From bac93bbf374d3c2718791d735a60709d0f61df01 Mon Sep 17 00:00:00 2001 From: Sagar Kharal Date: Fri, 4 Sep 2026 21:42:56 +0530 Subject: [PATCH 1/2] fix(import): report the collector's success label instead of inventing one The importer hardcoded success=true into every episode/v1 record while its episodes query read seven columns and never touched an outcome field. FORMAT.md defines success as the collector-labeled outcome and states the record is copied/merged from the source; the v3 format carries that label as a next.success feature with per-episode stats/next.success/min|max aggregates (lerobot/pusht declares it for all 206 episodes, uniformly False). The importer now reads stats/next.success/max as an eighth column when the source declares it (column discovery first: a corpus without the outcome feature is normal, not malformed), derives the episode outcome as MAX over the episode's frames, and stamps success together with a success_derivation naming the derivation so the methodology travels with the data. When the source declares no outcome feature the key is omitted entirely: FORMAT.md makes every episode/v1 key optional, and omission is the only encoding that cannot be read as a collector's judgment. The catalog promotion already renders an omitted key as SQL NULL; that behavior is pinned, not changed. CONVERTER_VERSION moves to v7 in the same commit: the label changes episode/v1 bytes that content_episode_id hashes. The fixture chain from #395 lands as tests through the real import, Episode reader, Catalog.append_episode, and SQL: max over a False,True transition reports true, an all-false source reports false, an unlabeled source omits the key and catalogs as NULL. Refs #395 --- .zcode/pr379_body.md | 15 ++ .zcode/receipt_timing.py | 35 +++ .zcode/success_catalog_fixture.py | 243 ++++++++++++++++++ .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 0 -> 286 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 0 -> 286 bytes .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 0 -> 4265 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 0 -> 4266 bytes .../catalog/format_version | 1 + .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 0 -> 243 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 0 -> 243 bytes .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 0 -> 292 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 0 -> 292 bytes .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 0 -> 342 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 0 -> 342 bytes .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 0 -> 203 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 0 -> 203 bytes .../meta/episodes/chunk-000/file-000.parquet | Bin 0 -> 2956 bytes .zcode/success_catalog_fixture/meta/info.json | 1 + .../meta/episodes/chunk-000/file-000.parquet | Bin 0 -> 2956 bytes ...servation_images_side-chunk000-file000.mp4 | Bin 0 -> 1042 bytes ...observation_images_up-chunk000-file000.mp4 | Bin 0 -> 1042 bytes .../landing/lerobot_episode_0001.mcap | Bin 0 -> 4626 bytes .../landing/lerobot_episode_0002.mcap | Bin 0 -> 4626 bytes .../prepared/prepared-manifest.json | 26 ++ .zcode/success_fixture.py | 214 +++++++++++++++ .../meta/episodes/chunk-000/file-000.parquet | Bin 0 -> 2956 bytes .zcode/success_fixture/meta/info.json | 1 + .../meta/episodes/chunk-000/file-000.parquet | Bin 0 -> 2956 bytes ...servation_images_side-chunk000-file000.mp4 | Bin 0 -> 1042 bytes ...observation_images_up-chunk000-file000.mp4 | Bin 0 -> 1042 bytes .../landing/lerobot_episode_0001.mcap | Bin 0 -> 4626 bytes .../landing/lerobot_episode_0002.mcap | Bin 0 -> 4626 bytes .../prepared/prepared-manifest.json | 26 ++ .zcode/success_issue_body.md | 66 +++++ src/hflow/importers/lerobot.py | 82 ++++-- tests/test_lerobot_converter.py | 216 ++++++++++++++++ 36 files changed, 902 insertions(+), 24 deletions(-) create mode 100644 .zcode/pr379_body.md create mode 100644 .zcode/receipt_timing.py create mode 100644 .zcode/success_catalog_fixture.py create mode 100644 .zcode/success_catalog_fixture/catalog/check_runs/ba19d7fb94cdba01-44ca08086eed.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/check_runs/e9e1a9ad1306d39d-9a5226c7b5ff.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/episodes/ba19d7fb94cdba01-44ca08086eed.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/episodes/e9e1a9ad1306d39d-9a5226c7b5ff.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/format_version create mode 100644 .zcode/success_catalog_fixture/catalog/intervals/ba19d7fb94cdba01-44ca08086eed.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/intervals/e9e1a9ad1306d39d-9a5226c7b5ff.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/measurements/ba19d7fb94cdba01-44ca08086eed.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/measurements/e9e1a9ad1306d39d-9a5226c7b5ff.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/observations/ba19d7fb94cdba01-44ca08086eed.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/observations/e9e1a9ad1306d39d-9a5226c7b5ff.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/tags/ba19d7fb94cdba01-44ca08086eed.parquet create mode 100644 .zcode/success_catalog_fixture/catalog/tags/e9e1a9ad1306d39d-9a5226c7b5ff.parquet create mode 100644 .zcode/success_catalog_fixture/meta/episodes/chunk-000/file-000.parquet create mode 100644 .zcode/success_catalog_fixture/meta/info.json create mode 100644 .zcode/success_catalog_fixture/prepared/_lerobot_cache/abc1234/meta/episodes/chunk-000/file-000.parquet create mode 100644 .zcode/success_catalog_fixture/prepared/_lerobot_cache/abc1234/videos/observation_images_side-chunk000-file000.mp4 create mode 100644 .zcode/success_catalog_fixture/prepared/_lerobot_cache/abc1234/videos/observation_images_up-chunk000-file000.mp4 create mode 100644 .zcode/success_catalog_fixture/prepared/landing/lerobot_episode_0001.mcap create mode 100644 .zcode/success_catalog_fixture/prepared/landing/lerobot_episode_0002.mcap create mode 100644 .zcode/success_catalog_fixture/prepared/prepared-manifest.json create mode 100644 .zcode/success_fixture.py create mode 100644 .zcode/success_fixture/meta/episodes/chunk-000/file-000.parquet create mode 100644 .zcode/success_fixture/meta/info.json create mode 100644 .zcode/success_fixture/prepared/_lerobot_cache/abc1234/meta/episodes/chunk-000/file-000.parquet create mode 100644 .zcode/success_fixture/prepared/_lerobot_cache/abc1234/videos/observation_images_side-chunk000-file000.mp4 create mode 100644 .zcode/success_fixture/prepared/_lerobot_cache/abc1234/videos/observation_images_up-chunk000-file000.mp4 create mode 100644 .zcode/success_fixture/prepared/landing/lerobot_episode_0001.mcap create mode 100644 .zcode/success_fixture/prepared/landing/lerobot_episode_0002.mcap create mode 100644 .zcode/success_fixture/prepared/prepared-manifest.json create mode 100644 .zcode/success_issue_body.md diff --git a/.zcode/pr379_body.md b/.zcode/pr379_body.md new file mode 100644 index 00000000..b7989248 --- /dev/null +++ b/.zcode/pr379_body.md @@ -0,0 +1,15 @@ +Closes #379. + +The manifest recorded how many episodes were converted but not which ones, so a truncated, missing, or swapped episode was undetectable from the delivery (#379's controlled result: one landing file cut to zero bytes, manifest unchanged). This adds the receipt. + +`prepared-manifest.json` moves to schema version 3: every delivered episode carries its published URI, its `content_episode_id` content address, and its byte size. The v2 top-level keys are unchanged, so existing readers keep working, and the entries carry everything a future verify command needs, keeping verify purely additive per the issue direction: the entries are done here, verify is deferred, not rejected. + +Per the constraints on the issue: the hash is taken inside `_convert_single_episode` while the canonical file is still on local disk, before `storage.publish`, so a bucket root never downloads its own upload to learn its content id; the recorded URI is the published object (a bucket prefix recipient has no local paths); and `content_episode_id` stays the single hashing implementation, reused rather than forked. The source cache is not hashed; the receipt covers what was delivered, not what it was made from. + +The truncation fixture from the issue lands as `test_manifest_content_id_detects_a_truncated_episode`: after truncation, both the size and the content id disagree with the manifest, which is the detection the receipt exists for. + +Cost: one linear sha256 read of each canonical file while it is already on local disk, 4.6 ms at fixture scale, about 0.2 s for a 100 MB episode. + +Gate: ruff, format, ty clean; 1471 passed / 6 skipped, the single failure (`utc-stats-test-date-collision`) reproduced on the base commit and unrelated, fix already in flight on its own branch. + +Refs #379, builds on #377's published URI list. diff --git a/.zcode/receipt_timing.py b/.zcode/receipt_timing.py new file mode 100644 index 00000000..e1df8636 --- /dev/null +++ b/.zcode/receipt_timing.py @@ -0,0 +1,35 @@ +"""Timing: the receipt cost — content_episode_id + stat per episode. + +Measures the hashing added by #379 on a real canonical episode file, +at fixture scale and extrapolated at delivery scale. + +Run: uv run --locked --all-extras python .zcode/receipt_timing.py +""" + +import statistics +import sys +import time +from pathlib import Path + +from hflow.catalog import content_episode_id + + +def main() -> None: + path = Path(sys.argv[1]) + size = __import__("pathlib").Path(path).stat().st_size + timings = [] + for _ in range(20): + started = time.perf_counter() + content_episode_id(path) + timings.append(time.perf_counter() - started) + median_us = statistics.median(timings) * 1_000_000 + print(f"episode file: {path} ({size} bytes)") + print(f"content_episode_id median of 20 runs: {median_us:.1f} us") + # sha256 throughput ~500 MB/s in CPython: a 100 MB episode costs ~0.2 s, + # one linear read of a file that is already on local disk pre-publish. + projected_100mb_s = 100 / 500 + print(f"projected at 100 MB episode (~500 MB/s sha256): ~{projected_100mb_s:.2f} s") + + +if __name__ == "__main__": + main() diff --git a/.zcode/success_catalog_fixture.py b/.zcode/success_catalog_fixture.py new file mode 100644 index 00000000..02c43184 --- /dev/null +++ b/.zcode/success_catalog_fixture.py @@ -0,0 +1,243 @@ +"""Success-stamp fixture: the #379 lesson applied — every stage real. + +Chain: real import (network/ffmpeg stubbed) -> real Episode.metadata read -> +real Catalog.append_episode -> real duckdb query. Shows a demo whose source +collector label is next.success=False being delivered as success=true and +returned by a buyer's success=true filter. + +Run: uv run --locked --all-extras python .zcode/success_catalog_fixture.py +""" + +import json +import shutil +from pathlib import Path + +import duckdb + +from hflow.catalog import Catalog +from hflow.episode import Episode +from hflow.importers.lerobot import import_lerobot_dataset +from hflow.transform import stamps_from_provenance + +ROOT = Path(".zcode/success_catalog_fixture") +EPISODES = 2 + + +def build_corpus(root: Path) -> dict: + info = { + "fps": 30, + "data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet", + "video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4", + "features": { + "action": {"dtype": "float32", "shape": [1]}, + "observation.state": {"dtype": "float32", "shape": [1]}, + "observation.images.up": {"dtype": "video", "shape": [480, 640, 3]}, + "observation.images.side": {"dtype": "video", "shape": [480, 640, 3]}, + "timestamp": {"dtype": "float32", "shape": [1]}, + "next.success": {"dtype": "bool", "shape": [1]}, + }, + "robot_type": "so101", + } + (root / "meta").mkdir(parents=True, exist_ok=True) + (root / "meta" / "info.json").write_text(json.dumps(info)) + + conn = duckdb.connect() + ep_cols = [ + "episode_index", + "length", + "data/chunk_index", + "data/file_index", + "dataset_from_index", + "dataset_to_index", + "videos/observation.images.up/chunk_index", + "videos/observation.images.up/file_index", + "videos/observation.images.up/from_timestamp", + "videos/observation.images.up/to_timestamp", + "videos/observation.images.side/chunk_index", + "videos/observation.images.side/file_index", + "videos/observation.images.side/from_timestamp", + "videos/observation.images.side/to_timestamp", + "tasks", + "stats/next.success/min", + "stats/next.success/max", + ] + rows = [ + [ + i, + 1, + "000", + "000", + i, + i + 1, + "000", + "000", + 0.0, + 0.0, + "000", + "000", + 0.0, + 0.0, + [f"task-{i}"], + [False], + [False], + ] + for i in range(EPISODES) + ] + ep_path = root / "meta" / "episodes" / "chunk-000" / "file-000.parquet" + ep_path.parent.mkdir(parents=True, exist_ok=True) + vals = ",".join( + "(" + + ",".join( + "[" + ",".join(f"'{x}'" for x in v) + "]" + if isinstance(v, list) + else f"'{v}'" + if isinstance(v, str) + else str(v) + for v in row + ) + + ")" + for row in rows + ) + conn.execute( + f"COPY (SELECT * FROM (VALUES {vals}) AS t({','.join(chr(34) + c + chr(34) for c in ep_cols)})) " + f"TO '{str(ep_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" + ) + + data_rows = [[i, i, 0, 0.0, [0.0], [0.5], False] for i in range(EPISODES)] + data_path = root / "data" / "chunk-000" / "file-000.parquet" + data_path.parent.mkdir(parents=True, exist_ok=True) + dvals = ",".join( + "(" + + ",".join( + str(v) + if not isinstance(v, (str, list)) + else ( + "'" + str(v) + "'" + if isinstance(v, str) + else "[" + ",".join(str(x) for x in v) + "]" + ) + for v in row + ) + + ")" + for row in data_rows + ) + conn.execute( + f"COPY (SELECT * FROM (VALUES {dvals}) AS t(index, episode_index, frame_index, " + 'timestamp, "observation.state", action, "next.success")) ' + f"TO '{str(data_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" + ) + conn.close() + return {"info": info} + + +def main() -> None: + shutil.rmtree(ROOT, ignore_errors=True) + corpus = build_corpus(ROOT) + output_dir = ROOT / "prepared" + + import hflow.importers.lerobot as prep + + stubs = { + "_hf_repo_info": lambda repo, revision: {"sha": "abc1234", "license": "apache-2.0"}, + "_fetch_info_json": lambda repo, rev, cache: corpus["info"], + "_hf_tree": lambda repo, rev, path: ( + [{"path": "meta/episodes/chunk-000/file-000.parquet", "type": "file"}] + if "episodes" in path + else [{"path": "meta/info.json", "type": "file"}] + ), + "_transcode_mp4_to_h264": lambda mp4_path, gop, fps: [ + b"\x00\x00\x00\x01\x09\x10\x00\x00\x00\x01\x67\x42\x00" + b"\x00\x00\x00\x01\x68\x88\x80\x00\x00\x00\x01\x65\x88" + ], + "_get_video_pts_times": lambda path: [0], + "ffmpeg_version": lambda: "fixture-ffmpeg", + } + originals = {name: getattr(prep, name) for name in stubs} + + def fake_download(url: str, dest: Path, **kw: object) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + if "meta/episodes" in url: + import shutil + + shutil.copy(ROOT / "meta" / "episodes" / "chunk-000" / "file-000.parquet", dest) + elif url.endswith("info.json"): + import shutil + + shutil.copy(ROOT / "meta" / "info.json", dest) + else: + import shutil + + shutil.copy(ROOT / "data" / "chunk-000" / "file-000.parquet", dest) + + stubs["_download_file"] = fake_download + try: + for name, fake in stubs.items(): + setattr(prep, name, fake) + import_lerobot_dataset( + dataset_repo="fake/repo", + output_dir=output_dir, + camera_keys=("observation.images.up", "observation.images.side"), + ) + finally: + for name, original in originals.items(): + setattr(prep, name, original) + + print("--- stage 1: what the source declares (collector's own label) ---") + conn = duckdb.connect() + src = conn.execute( + 'SELECT episode_index, "next.success" FROM read_parquet(' + f"'{ROOT / 'data' / 'chunk-000' / 'file-000.parquet'}') ORDER BY index" + ).fetchall() + for r in src: + print(f" data parquet frame (episode {r[0]}): next.success = {r[1]}") + agg = conn.execute( + 'SELECT episode_index, "stats/next.success/max" FROM read_parquet(' + f"'{ROOT / 'meta' / 'episodes' / 'chunk-000' / 'file-000.parquet'}') ORDER BY episode_index" + ).fetchall() + for r in agg: + print(f" episodes parquet aggregate (episode {r[0]}): stats/next.success/max = {r[1]}") + conn.close() + + print("--- stage 2: what the importer shipped (read back through the real Episode reader) ---") + landing = sorted((output_dir / "landing").glob("*.mcap")) + catalog_root = ROOT / "catalog" + catalog = Catalog(catalog_root) + for episode_path in landing: + with Episode(episode_path) as episode: + metadata = episode.metadata + stamps = stamps_from_provenance(metadata) + print(f" {episode_path.name}: episode/v1 success = {metadata.get('success')!r}") + catalog.append_episode( + canonical_path=episode_path, + stamps=stamps, + episode_metadata=metadata, + check_rows=[], + ) + + print("--- stage 3: the buyer's query (real catalog, real SQL) ---") + query_conn = duckdb.connect() + rows = query_conn.execute( + f"SELECT episode_id, task, success FROM read_parquet(" + f"'{catalog_root / 'episodes' / '*.parquet'}') WHERE success = 'true'" + ).fetchall() + print(f" SELECT ... WHERE success = 'true' -> {len(rows)} row(s):") + for r in rows: + print(f" episode_id={r[0]} task={r[1]!r} success={r[2]!r}") + total_rows = query_conn.execute( + f"SELECT COUNT(*) FROM read_parquet('{catalog_root / 'episodes' / '*.parquet'}')" + ).fetchall() + print(f" (catalog holds {total_rows[0][0]} episode(s) in total)") + query_conn.close() + print("--- verdict ---") + print( + " source label: next.success = False on every frame of every episode" + " (the collector's own outcome signal, declared in info.json)" + ) + print( + " delivered + cataloged: success = 'true' for every episode;" + " a buyer filtering successful demos receives all of them" + ) + + +if __name__ == "__main__": + main() diff --git a/.zcode/success_catalog_fixture/catalog/check_runs/ba19d7fb94cdba01-44ca08086eed.parquet b/.zcode/success_catalog_fixture/catalog/check_runs/ba19d7fb94cdba01-44ca08086eed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..080e6b600031d234dee729880796201852b1dd37 GIT binary patch literal 286 zcmZ{g!D<3Q42GlBB3nfix47V8FAFP(AhOVF#bXcB7Z}D#YzK8#GBZ9!AHhfPA)R6M z*pSorC;v~f{rq00?d(>;q>o-5a3`$#rISoYkoJo9B%!mmHImR+K!XPCjo54RLwsg! z+><9^Q5}SlCwiiOgD%8Y*CtX3o?XS-X(f-1->Y@fc^lOvCZcEQ!nm-F-XZ3iqx=%& zHzrjx5(?>x@z>vQI)_;AkoC%P6K>e7^G9HQSuUzYm6toOkC@|!LMd0PJ;q>o-5a3`$#rISoYkoJo9B%!mmHImR+K!XPCjo54RLwsg! z+><9^Q5}SlCwiiOgD%8Y*CtX3o?XS-X(f-1->Y@fc^lOvCZcEQ!nm-F-XZ3iqx=%& zHzrjx5(?>x@z>vQI)_;AkoC%P6K>e7^G9HQSuUzYm6toOkC@|!LMd0PJ6LlD1K_wE}G7WH-A_7prQkRS!_1suYA;s%YYwWX&eF z8QW<^Td6|80aY9-4&{J^5GPIu^;9nW0SFZg`cIQ>V4+`-V-Z=K6*~UAJWbwr6)+9rLE= zGpad^I+#(S^{_=utE=@Wv)i`UqK@gpz*enRTdwvE?Cce_EFF!-e5f2oEI`D2XNAyC zV|itb8f!EF?wXZFsT{LEzY*HcfPKsMD!$8eT9g)JpsXH(V9}6=-K|RCU`<+&L0;Gl zAvu6V(>82tK$%AwpbJtlhMsGL=u}{k`4j-hrBV!VxETV{0C4R(FNjAO>0vsC`6Bgg z2s#3nTP0)Dx3(%CW?c&Fiu;sdx3pd?H6iP@T4|}cRzg|V!ya8PB0UPN1rfTYZ3!m| z!>}vTv07!Y3Ij@=PT!&3>Px5)rA9cu*tMM&th7r#VIwo>;hdCrDNL25KS%X+zQ}Cf z(KKv$v(EH`3|N^xs@QF)vH=!Ol&fYRcd1)-O_F|;`-4?b%6Ehh1}~#&FSTA=xKnUU zhxSa1Rv)8sY)#YO=Uroy_Hng@rk`K5eP+-WfeM`cr`W+;RDPn0)S_xLEkbV! zx2igtVRT^D?<%-$)R$Kp$6m?!%oOVR&Ca@}k?Bs#)g-cbcBn4hf#sN!=3}WopoEzY zc9kL(Vw#zyP!pW5UEg(~f6|s;`}*n>lEj54^Q%aiCBCsm+O2R(QVHtyq^KMmg-%YQ zP;{927Hlx9OPRw=%ZubkYzEnIJ%&A-wE~bi8uJ{?F2Wsj!%W3QbD`5{G<;!YYS7ze z?V(Nh*r`OgpQ^m9J{0t|pk~J*1y#e<--+@gQTY z=*v{HfD(O$Bb&0W{*uxap$-S+lwT9pw>2I1*P3TR7s2D7oJ#Nb$C=i|@H8|n|x zSks)<3@>0|;yGbs@=d_aJkH3!rqvIpb>*(O59%{XjyMULphl$;Rf}F@fjS~YBjp4w ziX)?H2uRbI9|bmmAVtEEC2>wx=duw~0{#_X!?*$yp=QXMJW$7I6AhpdzXR~*vlMip zfS%RCvpVR`&krdL;-oH47J1vK$qn`QId~l*IG||wHPLN$S}ehSZ@1fj}%GMAQ-Q#bjXW9Y6}Sc*8@F({}nF5pA`QB3vgrn literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/catalog/episodes/e9e1a9ad1306d39d-9a5226c7b5ff.parquet b/.zcode/success_catalog_fixture/catalog/episodes/e9e1a9ad1306d39d-9a5226c7b5ff.parquet new file mode 100644 index 0000000000000000000000000000000000000000..942e8248f2614b918402960bb4ef1cfe6d30ff7a GIT binary patch literal 4266 zcmeHL&2Jk;6rWw})M@i!r=Dd~oW!!ym4Vp18z*iOSydZFJwS!3QV?pXtnu#HSz_-l z`{7o!l_~_BDh{ZJazH|e6DNcahj8H!;M8*uxxkq}paS0PdhPWlsmiGbDy{e9z2Cg| z=FOY8zx9=iS5=Jg8s1pJJlss+353q*JVJ=)vT&xGR3)24S8GaLuWjmpS;0ySEL;o0 zQvlv1>#M7E?ObcU-3H(k&c=Wf`2dJO@qot<`Izbp?yt{7D|f+)+%r8{lizcxq zLuK2cHgTvfcZsPRW=HN)$7)$#liG$01DlGXsMVfE>}N$S;4`t9PnE-nIf$60@c69g zw7s@kt5dB`8SvPwEaP&_{@hMzKLhqn%d7YglhMn$8y; z%eQ5jXx_9v`6vTcrq7603#zP7xTSK{=#dU}tFED^_j7+Z6_oOA?$iF8Xwpk<73c32 zY{RBq!=%*%RF17_`q#Xx?b05pmeAy@%a-qGw5dY{PXAMElRYX!@mbst(@RQJZHB4S z+uR+bEG$epv~Q^QMbgsL)!N#^V#aq2PR;MOw`5tLY^Piq>&q7g>e4;P_6(kj<@$se zrkZUl5f@^bsg+Q}&Nps?`skJ3cbDJ({`v&M{QT3obtF#fzP6{gn&F&q2@3WsFPdIYOF^h@Ujj%b8`bqg91_)CyPPbP=qzAhiXA{_BAr!5<4(;9rV=0V!u<_5c6? literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/catalog/format_version b/.zcode/success_catalog_fixture/catalog/format_version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/.zcode/success_catalog_fixture/catalog/format_version @@ -0,0 +1 @@ +1 diff --git a/.zcode/success_catalog_fixture/catalog/intervals/ba19d7fb94cdba01-44ca08086eed.parquet b/.zcode/success_catalog_fixture/catalog/intervals/ba19d7fb94cdba01-44ca08086eed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..df1deafcd39b60b5b2d8f7e88e7391cacd2b739b GIT binary patch literal 243 zcmWG=3^EjDlI$^Mkl;-zP0mh9iZ4#iNX<e%gjqpEh;F=%qxM3a{)CbXUFFy=BC1gcp*Y%sYS(^`FSu==Iqo;5St5VK2KR< zPHAd*kO9zhu$ z1`rTqkmS+OZ~^+n#Yq8btwNcho~fRxf<{tlW=@JiibZltVxqBeT8gIR92o`%T!=w} Kk%1uq7(M`g-bA$k literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/catalog/measurements/e9e1a9ad1306d39d-9a5226c7b5ff.parquet b/.zcode/success_catalog_fixture/catalog/measurements/e9e1a9ad1306d39d-9a5226c7b5ff.parquet new file mode 100644 index 0000000000000000000000000000000000000000..58f4b5e8810623f7ec08083534dec6826c05af70 GIT binary patch literal 292 zcmWG=3^EjDl3ZiTAie%gjqpEh;F=%qxM3a{)CbXUFFy=BC1gcp*Y%sYS(^`FSu==Iqo;5St5VK2KR< zPHAd*kO9zhu$ z1`rTqkmS+OZ~^+n#Yq8btwNcho~fRxf<{tlW=@JiibZltVxqBeT8gIR92o`%T!=w} Kk%1uq7(M`g-bA$k literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/catalog/observations/ba19d7fb94cdba01-44ca08086eed.parquet b/.zcode/success_catalog_fixture/catalog/observations/ba19d7fb94cdba01-44ca08086eed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..39ecdfff92817296c8084b8dc9bb46b19ec600bc GIT binary patch literal 342 zcmaKo!AiqG5QfLrQ%n`Xlm>dpp@9feC?()k=}9~kUtrnYnd*{cH|%bLzJrhCBlu7z zO>%XZ)5rY({4>X=w_P-ge{)zZu#p7{=Hv%eJX%5&rn6N*wQ>fDDNJENH@4C2r1U%4 z+A8fQ9&|!kFwHBP&|*L>S*MJi=(Q1!Y|DMeM&HYBRp?bk&hx5f+NGOe*}R~yUKjKO zEiW5lXc|${7;iwHKF=|rPdp%DObJo%du2%)i-e5(^zQZhcKrZ-Eg=rG?9g3(cuvNM jNxR$T+dRocqe@KhQ6lI2{R!{mpEbb$4KEh}$L^zlNyJf) literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/catalog/observations/e9e1a9ad1306d39d-9a5226c7b5ff.parquet b/.zcode/success_catalog_fixture/catalog/observations/e9e1a9ad1306d39d-9a5226c7b5ff.parquet new file mode 100644 index 0000000000000000000000000000000000000000..39ecdfff92817296c8084b8dc9bb46b19ec600bc GIT binary patch literal 342 zcmaKo!AiqG5QfLrQ%n`Xlm>dpp@9feC?()k=}9~kUtrnYnd*{cH|%bLzJrhCBlu7z zO>%XZ)5rY({4>X=w_P-ge{)zZu#p7{=Hv%eJX%5&rn6N*wQ>fDDNJENH@4C2r1U%4 z+A8fQ9&|!kFwHBP&|*L>S*MJi=(Q1!Y|DMeM&HYBRp?bk&hx5f+NGOe*}R~yUKjKO zEiW5lXc|${7;iwHKF=|rPdp%DObJo%du2%)i-e5(^zQZhcKrZ-Eg=rG?9g3(cuvNM jNxR$T+dRocqe@KhQ6lI2{R!{mpEbb$4KEh}$L^zlNyJf) literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/catalog/tags/ba19d7fb94cdba01-44ca08086eed.parquet b/.zcode/success_catalog_fixture/catalog/tags/ba19d7fb94cdba01-44ca08086eed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6fb89d94d058a6d353947417188244d34b3a3321 GIT binary patch literal 203 zcmWG=3^EjDlFTt>kl;-zP0mh9iZ4#iNX<*kO z9zhu$1`rTqkmS+OZ~~ S86(5MfQ%US1I-8kdIJCkS1+mn literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/catalog/tags/e9e1a9ad1306d39d-9a5226c7b5ff.parquet b/.zcode/success_catalog_fixture/catalog/tags/e9e1a9ad1306d39d-9a5226c7b5ff.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6fb89d94d058a6d353947417188244d34b3a3321 GIT binary patch literal 203 zcmWG=3^EjDlFTt>kl;-zP0mh9iZ4#iNX<*kO z9zhu$1`rTqkmS+OZ~~ S86(5MfQ%US1I-8kdIJCkS1+mn literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/meta/episodes/chunk-000/file-000.parquet b/.zcode/success_catalog_fixture/meta/episodes/chunk-000/file-000.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bdd4598590cf8fe52b060c7d103318637c0649e0 GIT binary patch literal 2956 zcmd5;?`ji45Z_)d*R-^1qm8@cHvaZcL7OUH?#fI5nS8Fr(+`GW#1_<4=OwTMeFZymPkOS87 z;I240v)kADu#c$sai24s-d=R?oq{0l~vYe4s*RMAt0y&%N{Se zZ0Hqvzu9D*mmJ&uS50%cNK;o+>C{y+N77VDQIeWSe5&ZG{!~B8C8~yK%?4gO(Mf)6 zl}y6c2q7pEsd^$+v1^Z1aV1qZgz>3@5+En~tK9pQsu6>xE(XAm^9h8>%98-!7XtY4 z&nQtD$wx(o0=|!hf-C8Wk26)z66GhNVCVB0MM6d^iQ*Fyx%g$Am$7I>MePz2`1e9! zKk$%Bfqr$f(j+IJRp10!9Oc(&B{}g8gp@xR7%xN#|1KT{-lu$th9tT93ivW3L$Wi( z^@+|Up+9JgpnX=BYjKj99{?Yf_>h~O>VeY0r`0G11oa1gn-uNwx-ANvhTh2i0vt`e z5cYy^8Hk?Pk(uJ-&FQAo22S7CqFaTD^`k?Ouo^#+g8RpXj-PZzkgm8@cHvaZcL7OUH?#fI5nS8Fr(+`GW#1_<4=OwTMeFZymPkOS87 z;I240v)kADu#c$sai24s-d=R?oq{0l~vYe4s*RMAt0y&%N{Se zZ0Hqvzu9D*mmJ&uS50%cNK;o+>C{y+N77VDQIeWSe5&ZG{!~B8C8~yK%?4gO(Mf)6 zl}y6c2q7pEsd^$+v1^Z1aV1qZgz>3@5+En~tK9pQsu6>xE(XAm^9h8>%98-!7XtY4 z&nQtD$wx(o0=|!hf-C8Wk26)z66GhNVCVB0MM6d^iQ*Fyx%g$Am$7I>MePz2`1e9! zKk$%Bfqr$f(j+IJRp10!9Oc(&B{}g8gp@xR7%xN#|1KT{-lu$th9tT93ivW3L$Wi( z^@+|Up+9JgpnX=BYjKj99{?Yf_>h~O>VeY0r`0G11oa1gn-uNwx-ANvhTh2i0vt`e z5cYy^8Hk?Pk(uJ-&FQAo22S7CqFaTD^`k?Ouo^#+g8RpXj-PZzkg_HU5-hY1m9MoB6)|W@b07%iA&qnx|%wYWR|-X@G2%0DzW21Je92K(hSG zFb}Y4aY@7RwbB5}CS{1ckmt9s6o$wf!?N^fi7lhgGor+Jb3i}zN8Y2~z8BsD4_%rC zN(w|z1wEfn3a2ygkPiaU_xb(G?0#qHhHG=XQ4n$wx#6I&M+w!6chaH`hv>O}Tkzr7 zje>r!gfS7vZ)tEeUF!xSqGYyMhIe_$d(kFA+Vv#%FDMF_%t4P&qLLVS9v1?bX5@{j z1$it01!JBnB}`)%?8@4y>#E(lR1FO5B`=r5sTjFhmIXs|dxqr_F7QtYB#?{85@K&n zLR{|!{C#_PZL=k+72hpToK1FPJ(H~^)#hFLndnmz$$dGqvb?dk$PH9uR@$bdoU^L+ zlC0|qz=2rxR4DeEWSX_ZK9XSD4cyU_HU5-hY1m9MoB6)|W@b07%iA&qnx|%wYWR|-X@G2%0DzW21Je92K(hSG zFb}Y4aY@7RwbB5}CS{1ckmt9s6o$wf!?N^fi7lhgGor+Jb3i}zN8Y2~z8BsD4_%rC zN(w|z1wEfn3a2ygkPiaU_xb(G?0#qHhHG=XQ4n$wx#6I&M+w!6chaH`hv>O}Tkzr7 zje>r!gfS7vZ)tEeUF!xSqGYyMhIe_$d(kFA+Vv#%FDMF_%t4P&qLLVS9v1?bX5@{j z1$it01!JBnB}`)%?8@4y>#E(lR1FO5B`=r5sTjFhmIXs|dxqr_F7QtYB#?{85@K&n zLR{|!{C#_PZL=k+72hpToK1FPJ(H~^)#hFLndnmz$$dGqvb?dk$PH9uR@$bdoU^L+ zlC0|qz=2rxR4DeEWSX_ZK9XSD4cyUY}j@U3_4*c&?uP9Y{jX&VWV0^YTq5CNq?lS2=P5<-*$rC96r^LiC~-)eX5 z1Wr*AQX?Vw0~Hbogj6Afka~xzJ(NoioVX$o8~`B>sF$kDynVa=NJ|yeTQXKWZ|CdH zn>RZ%&d$rrCs#^2RhlI|n1*Gu(PEog)$&-!rS)ptG~DW@X9v`iPY6kgOu?!}Jyv5u zwe4HJ4qy67Gq9O60xXaD6`(u-OR#1H^OdvZb4Lu$S8x*q2|g&Shjba1X5B-Md{aV1 zNP-mLyTP`s7VA=NnYCSy`aZ3%+jYv)z;itouv(|_J!q)6CH1hynAM_sMArkmP5r=V zyBa6u$3ube2^43KtC_V(CSOoTd}=bM?iZvZ(ww{+J?0OoNyl)QUq}Ewxf(t!y)UVg zCTnX$MwiB7lPl=piu7hhh*+#;IF`mdtIx-GT8k{<^iBlJsdpKDDxRDKT@3 zA4}ubz-#RcZM9xPg*DWzrTF3I)oYSE6NO733YQ2#I^s{?D^|0vIzD70UsN-3M0tot zE7QZL*W$zHC)9MqGupIj*LmPF`K+qcjld|#5XNkNOieYI*EWKJ0?(P~97+jqA0Q!8 zKxZ|ddR+q(OtagDMSabO9OzF0l#eqG!0e}pjhXCIxb2{#>ym_805u$LV zU8U`{zzAqxWt#-?zRKn@-^SEb9*xZ?BtN`6l%at3mAUN4Li!M3pLINw7U4baQYbVg z)w|{MK*t&i1*uCtsK8Y~KJ0@E!pCCFEOwt71A3#;cBvIfa*DkbbUa!NIV#A6n^ilW zeHfmajc0WoUrL__zTPl4sP0jhy=c_Ta%FxY`wz$OcTmD*Y-M;+illa9m=p1BvF8sOJ#IjI4YdW03~*ByvO!v z*T%4*ZQ*n;q*dW^)pqK13mZ#mPchAYBAQVSR0po-17L6`%_eOdQN&TYJb|*dOIx-> zBSsczb=p{{%r8=Nk@E8mUKR$=VL`D29S(zHF}I2hj|P}E-Ig&bh@waFt@eDMcn;p( zb=ygA;O+3YwQ#lY|^8t7sl z%&n(O2>3z`NN@u^ergQ_IBegzI4_?q5P?q%^7x-K4`b53>j(8YL7Tg*-#T*o79m^7 z85l@}#tRBbk?oz;UxE+6UQNCA-1gBCEG7sN!OM*VafuMKl>zo+A(x~sibbp^Ho79{ zqwe+JBG+6&4jOP zSy^02o&l=-V8>O7=}zR=Q`euFy7uI;zkd4c%WwW(oY*4E@fi{WXAs`>jwlbXYl1(H z?niKcM#Z?BAIyRKa}JCX;9#8FrwMSM=CXVL=KSA3ITTG4y}eDGdm9x;*Tp7LkyZHY zg+v?;61WiUA;Q?;@F#W%iFhT)Db9~$&BsyuVquvl4q0-aIMV#}3mB4#Lh+KPBIQuJk;`omEPtd>f55Fw{ literal 0 HcmV?d00001 diff --git a/.zcode/success_catalog_fixture/prepared/landing/lerobot_episode_0002.mcap b/.zcode/success_catalog_fixture/prepared/landing/lerobot_episode_0002.mcap new file mode 100644 index 0000000000000000000000000000000000000000..42fd05df85b63f767d7e5381292431b09d4263c7 GIT binary patch literal 4626 zcmeGgO>Y}j@U3_4*qb;BP9Y{jX&VWV0`|sEh=5X{$)N{C2_Z^>Qmpm*dA*8v-)cXc zz$q$1Y9s`Iph5zHkSc@_Qtwc;hjQtG6ITR+10ci!^-`6Yw{O=UX{myGOU8=l?R>nM zH*a=koSm1KPp-`6RcVHFVG5S5W}9u6Yo^CSmo{p)p}VzB&kCp~pAeD~nS@oFdaTZZ zn(dpu24C7qBe0k=1T2sFRiHcoOV~{h=BsBb=Z@%{ub>eG2|g&SMRXaKM#Dpnd{aV1 zNQxBUyUDi9HtW#xGP7Ne`aW%}TMf#xz;itouzJ}19yHY3l6u%=%xqIFrfY#^Q$Nsc zx6Da}u}C0#0>zo*YHlr-DHPQqpBl_*_(kc6G%K&hkA*{O+R+{67gIn_uSO3`?@Q{0 z!R+$Drc0yAmMiGsiu7hxh?uOcJ7$@AW{;2WxYXA+9Jc9n{dMa%CF#?Yd}?LsQfm4X zKUSIV240uX(02PJR9HjZdWIivUcDx%({Z@8fpCccWMlr+y<#=vs$&Ct6iR9?i6{@z zaCK_%^yTF6g>g08^mLontOgHUt}vo14L#6{GK6uYFsf#n%(L~NsK9eBK8G@gw-1mK zDWI{sPrZ(g30Agj-K2imhaBil0hCWNM?T1${h2vsOalNJJ`VxDQ|5&3uDp4kD-oh_ zrCp`%wLlMOPi30`@t(@>*|~R31&XQAlCXI+US+_LRAikA?Ihz&;B-gO=bu?ocQ+ z2Gu&1^FYTM3I(Y{J*dDnKt9+(1>s{UVU{{ijRL*dv|Vb(lAK~~1))bv5l01?$XG4( zti$lsYCfxJ_)_{b@U^DCK{b!M>_xq9RI2j}BmaO50%}s%@>v6&`x%}T_ye_#jKHM| z29~+h(*x$A>v1F#FENOIXc$n&xwaQlzOP+p4JgA-kVXMknX91d!cpN|4k(FxV_mjK zI~Ilo_ZCifLs}Cq*DR+&x3IBP_7u~RPee1ygKFRPd;kpYq|u_b9!H#87)M#lrESZh zF=GU14cc6&&M#79k@EBPUls<>VL>T`4u?T8nOnn#M+3~7PFo)qMA4(TsXgB(o`ZLH zO>|x!am>x>F6S=w79(sxJ@<6~k^dA9-5ybF;=*F@zWjYijN^?Nt0>_@aY^Q)SR{JF&=o-+ zcCY^yyXFcSckRjvnOs%UZ$KjyDrzSC?T^1-%@ihkgW1y*r*OiOa^vQb)Vi=uFU%;d z>f%EB3{d34NTz_Wr+LOor`suSTzxjJ{e2Xk6XGry(L3q3;r0o zAH)3_6_aj$FbD3>IWS3pgK=)3Ccu4~%kKS~^MC*3P|Bj{?QP=R+o(9aE{w!QR^hW3 z5^*$0;X<^B7-NIOpV$$kl9imSlw_5CU5LfXG5!HZ5`c3NBG{xQGKYg0fI?motL`=R zch|V{-qnC3HuyZ+ksC;n2^?wwWKtEYK?8A*jQeLk?&t5}KmqVYg!qPcnyvFV>>xaj z0|dbB2n|fR`J;k-4CfhuOCqAXromC1DiP$jh0h;}%St%PA2==od>0{-aE8CaaTcH! up)Mzz=i>$;@ktXM^@iXGm99=c=A0L-6y6&`?Aois* dict: + info = { + "fps": 30, + "data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet", + "video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4", + "features": { + "action": {"dtype": "float32", "shape": [1]}, + "observation.state": {"dtype": "float32", "shape": [1]}, + "observation.images.up": {"dtype": "video", "shape": [480, 640, 3]}, + "observation.images.side": {"dtype": "video", "shape": [480, 640, 3]}, + "timestamp": {"dtype": "float32", "shape": [1]}, + "next.success": {"dtype": "bool", "shape": [1]}, + }, + "robot_type": "so101", + } + (root / "meta").mkdir(parents=True, exist_ok=True) + (root / "meta" / "info.json").write_text(json.dumps(info)) + + conn = duckdb.connect() + ep_cols = [ + "episode_index", + "length", + "data/chunk_index", + "data/file_index", + "dataset_from_index", + "dataset_to_index", + "videos/observation.images.up/chunk_index", + "videos/observation.images.up/file_index", + "videos/observation.images.up/from_timestamp", + "videos/observation.images.up/to_timestamp", + "videos/observation.images.side/chunk_index", + "videos/observation.images.side/file_index", + "videos/observation.images.side/from_timestamp", + "videos/observation.images.side/to_timestamp", + "tasks", + "stats/next.success/min", + "stats/next.success/max", + ] + rows = [ + [ + i, + 1, + "000", + "000", + i, + i + 1, + "000", + "000", + 0.0, + 0.0, + "000", + "000", + 0.0, + 0.0, + [f"task-{i}"], + [False], + [False], + ] + for i in range(EPISODES) + ] + ep_path = root / "meta" / "episodes" / "chunk-000" / "file-000.parquet" + ep_path.parent.mkdir(parents=True, exist_ok=True) + vals = ",".join( + "(" + + ",".join( + "[" + ",".join(f"'{x}'" for x in v) + "]" + if isinstance(v, list) + else f"'{v}'" + if isinstance(v, str) + else str(v) + for v in row + ) + + ")" + for row in rows + ) + conn.execute( + f"COPY (SELECT * FROM (VALUES {vals}) AS t({','.join(chr(34) + c + chr(34) for c in ep_cols)})) " + f"TO '{str(ep_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" + ) + + data_rows = [[i, i, 0, 0.0, [0.0], [0.5], False] for i in range(EPISODES)] + data_path = root / "data" / "chunk-000" / "file-000.parquet" + data_path.parent.mkdir(parents=True, exist_ok=True) + dvals = ",".join( + "(" + + ",".join( + str(v) + if not isinstance(v, (str, list)) + else ( + "'" + str(v) + "'" + if isinstance(v, str) + else "[" + ",".join(str(x) for x in v) + "]" + ) + for v in row + ) + + ")" + for row in data_rows + ) + conn.execute( + f"COPY (SELECT * FROM (VALUES {dvals}) AS t(index, episode_index, frame_index, " + 'timestamp, "observation.state", action, "next.success")) ' + f"TO '{str(data_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" + ) + conn.close() + return {"info": info} + + +def main() -> None: + shutil.rmtree(ROOT, ignore_errors=True) + corpus = build_corpus(ROOT) + output_dir = ROOT / "prepared" + + stubs = { + "_hf_repo_info": lambda repo, revision: {"sha": "abc1234", "license": "apache-2.0"}, + "_fetch_info_json": lambda repo, rev, cache: corpus["info"], + "_hf_tree": lambda repo, rev, path: ( + [{"path": "meta/episodes/chunk-000/file-000.parquet", "type": "file"}] + if "episodes" in path + else [{"path": "meta/info.json", "type": "file"}] + ), + "_transcode_mp4_to_h264": lambda mp4_path, gop, fps: [ + b"\x00\x00\x00\x01\x09\x10\x00\x00\x00\x01\x67\x42\x00" + b"\x00\x00\x00\x01\x68\x88\x80\x00\x00\x00\x01\x65\x88" + ], + "_get_video_pts_times": lambda path: [0], + "ffmpeg_version": lambda: "fixture-ffmpeg", + } + originals = {name: getattr(prep, name) for name in stubs} + + def fake_download(url: str, dest: Path, **kw: object) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + if "meta/episodes" in url: + import shutil + + shutil.copy(ROOT / "meta" / "episodes" / "chunk-000" / "file-000.parquet", dest) + elif url.endswith("info.json"): + import shutil + + shutil.copy(ROOT / "meta" / "info.json", dest) + else: + import shutil + + shutil.copy(ROOT / "data" / "chunk-000" / "file-000.parquet", dest) + + stubs["_download_file"] = fake_download + try: + for name, fake in stubs.items(): + setattr(prep, name, fake) + prep.import_lerobot_dataset( + dataset_repo="fake/repo", + output_dir=output_dir, + camera_keys=("observation.images.up", "observation.images.side"), + ) + finally: + for name, original in originals.items(): + setattr(prep, name, original) + + print("--- what the source declares ---") + conn = duckdb.connect() + src = conn.execute( + 'SELECT episode_index, "next.success" FROM read_parquet(' + f"'{ROOT / 'data' / 'chunk-000' / 'file-000.parquet'}') ORDER BY index" + ).fetchall() + for r in src: + print(f" data parquet frame (episode {r[0]}): next.success = {r[1]}") + agg = conn.execute( + 'SELECT episode_index, "stats/next.success/max" FROM read_parquet(' + f"'{ROOT / 'meta' / 'episodes' / 'chunk-000' / 'file-000.parquet'}') ORDER BY episode_index" + ).fetchall() + for r in agg: + print(f" episodes parquet aggregate (episode {r[0]}): stats/next.success/max = {r[1]}") + conn.close() + + print("--- what hflow shipped ---") + for episode_path in sorted((output_dir / "landing").glob("*.mcap")): + with episode_path.open("rb") as stream: + for metadata in make_reader(stream).iter_metadata(): + if metadata.name == "episode/v1": + print(f" {episode_path.name}: episode/v1 = {json.dumps(metadata.metadata)}") + break + + print("--- consequence ---") + print(" source collector label: next.success = False (this demo never succeeded)") + print(' shipped episode/v1: "success": "true"') + print(" catalog promotion: success VARCHAR first-class column (catalog.py:59,68)") + print(" buyer filter success=true: matches this failed demo") + + +if __name__ == "__main__": + main() diff --git a/.zcode/success_fixture/meta/episodes/chunk-000/file-000.parquet b/.zcode/success_fixture/meta/episodes/chunk-000/file-000.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bdd4598590cf8fe52b060c7d103318637c0649e0 GIT binary patch literal 2956 zcmd5;?`ji45Z_)d*R-^1qm8@cHvaZcL7OUH?#fI5nS8Fr(+`GW#1_<4=OwTMeFZymPkOS87 z;I240v)kADu#c$sai24s-d=R?oq{0l~vYe4s*RMAt0y&%N{Se zZ0Hqvzu9D*mmJ&uS50%cNK;o+>C{y+N77VDQIeWSe5&ZG{!~B8C8~yK%?4gO(Mf)6 zl}y6c2q7pEsd^$+v1^Z1aV1qZgz>3@5+En~tK9pQsu6>xE(XAm^9h8>%98-!7XtY4 z&nQtD$wx(o0=|!hf-C8Wk26)z66GhNVCVB0MM6d^iQ*Fyx%g$Am$7I>MePz2`1e9! zKk$%Bfqr$f(j+IJRp10!9Oc(&B{}g8gp@xR7%xN#|1KT{-lu$th9tT93ivW3L$Wi( z^@+|Up+9JgpnX=BYjKj99{?Yf_>h~O>VeY0r`0G11oa1gn-uNwx-ANvhTh2i0vt`e z5cYy^8Hk?Pk(uJ-&FQAo22S7CqFaTD^`k?Ouo^#+g8RpXj-PZzkgm8@cHvaZcL7OUH?#fI5nS8Fr(+`GW#1_<4=OwTMeFZymPkOS87 z;I240v)kADu#c$sai24s-d=R?oq{0l~vYe4s*RMAt0y&%N{Se zZ0Hqvzu9D*mmJ&uS50%cNK;o+>C{y+N77VDQIeWSe5&ZG{!~B8C8~yK%?4gO(Mf)6 zl}y6c2q7pEsd^$+v1^Z1aV1qZgz>3@5+En~tK9pQsu6>xE(XAm^9h8>%98-!7XtY4 z&nQtD$wx(o0=|!hf-C8Wk26)z66GhNVCVB0MM6d^iQ*Fyx%g$Am$7I>MePz2`1e9! zKk$%Bfqr$f(j+IJRp10!9Oc(&B{}g8gp@xR7%xN#|1KT{-lu$th9tT93ivW3L$Wi( z^@+|Up+9JgpnX=BYjKj99{?Yf_>h~O>VeY0r`0G11oa1gn-uNwx-ANvhTh2i0vt`e z5cYy^8Hk?Pk(uJ-&FQAo22S7CqFaTD^`k?Ouo^#+g8RpXj-PZzkg_HU5-hY1m9MoB6)|W@b07%iA&qnx|%wYWR|-X@G2%0DzW21Je92K(hSG zFb}Y4aY@7RwbB5}CS{1ckmt9s6o$wf!?N^fi7lhgGor+Jb3i}zN8Y2~z8BsD4_%rC zN(w|z1wEfn3a2ygkPiaU_xb(G?0#qHhHG=XQ4n$wx#6I&M+w!6chaH`hv>O}Tkzr7 zje>r!gfS7vZ)tEeUF!xSqGYyMhIe_$d(kFA+Vv#%FDMF_%t4P&qLLVS9v1?bX5@{j z1$it01!JBnB}`)%?8@4y>#E(lR1FO5B`=r5sTjFhmIXs|dxqr_F7QtYB#?{85@K&n zLR{|!{C#_PZL=k+72hpToK1FPJ(H~^)#hFLndnmz$$dGqvb?dk$PH9uR@$bdoU^L+ zlC0|qz=2rxR4DeEWSX_ZK9XSD4cyU_HU5-hY1m9MoB6)|W@b07%iA&qnx|%wYWR|-X@G2%0DzW21Je92K(hSG zFb}Y4aY@7RwbB5}CS{1ckmt9s6o$wf!?N^fi7lhgGor+Jb3i}zN8Y2~z8BsD4_%rC zN(w|z1wEfn3a2ygkPiaU_xb(G?0#qHhHG=XQ4n$wx#6I&M+w!6chaH`hv>O}Tkzr7 zje>r!gfS7vZ)tEeUF!xSqGYyMhIe_$d(kFA+Vv#%FDMF_%t4P&qLLVS9v1?bX5@{j z1$it01!JBnB}`)%?8@4y>#E(lR1FO5B`=r5sTjFhmIXs|dxqr_F7QtYB#?{85@K&n zLR{|!{C#_PZL=k+72hpToK1FPJ(H~^)#hFLndnmz$$dGqvb?dk$PH9uR@$bdoU^L+ zlC0|qz=2rxR4DeEWSX_ZK9XSD4cyUY}j@U3_4*c&?uP9Y{jX&VWV0^YTq5CNq?lS2=P5<-*$rC96r^LiC~-)eX5 z1Wr*AQX?Vw0~Hbogj6Afka~xzJ(NoioVX$o8~`B>sF$kDynVa=NJ|yeTQXKWZ|CdH zn>RZ%&d$rrCs#^2RhlI|n1*Gu(PEog)$&-!rS)ptG~DW@X9v`iPY6kgOu?!}Jyv5u zwe4HJ4qy67Gq9O60xXaD6`(u-OR#1H^OdvZb4Lu$S8x*q2|g&Shjba1X5B-Md{aV1 zNP-mLyTP`s7VA=NnYCSy`aZ3%+jYv)z;itouv(|_J!q)6CH1hynAM_sMArkmP5r=V zyBa6u$3ube2^43KtC_V(CSOoTd}=bM?iZvZ(ww{+J?0OoNyl)QUq}Ewxf(t!y)UVg zCTnX$MwiB7lPl=piu7hhh*+#;IF`mdtIx-GT8k{<^iBlJsdpKDDxRDKT@3 zA4}ubz-#RcZM9xPg*DWzrTF3I)oYSE6NO733YQ2#I^s{?D^|0vIzD70UsN-3M0tot zE7QZL*W$zHC)9MqGupIj*LmPF`K+qcjld|#5XNkNOieYI*EWKJ0?(P~97+jqA0Q!8 zKxZ|ddR+q(OtagDMSabO9OzF0l#eqG!0e}pjhXCIxb2{#>ym_805u$LV zU8U`{zzAqxWt#-?zRKn@-^SEb9*xZ?BtN`6l%at3mAUN4Li!M3pLINw7U4baQYbVg z)w|{MK*t&i1*uCtsK8Y~KJ0@E!pCCFEOwt71A3#;cBvIfa*DkbbUa!NIV#A6n^ilW zeHfmajc0WoUrL__zTPl4sP0jhy=c_Ta%FxY`wz$OcTmD*Y-M;+illa9m=p1BvF8sOJ#IjI4YdW03~*ByvO!v z*T%4*ZQ*n;q*dW^)pqK13mZ#mPchAYBAQVSR0po-17L6`%_eOdQN&TYJb|*dOIx-> zBSsczb=p{{%r8=Nk@E8mUKR$=VL`D29S(zHF}I2hj|P}E-Ig&bh@waFt@eDMcn;p( zb=ygA;O+3YwQ#lY|^8t7sl z%&n(O2>3z`NN@u^ergQ_IBegzI4_?q5P?q%^7x-K4`b53>j(8YL7Tg*-#T*o79m^7 z85l@}#tRBbk?oz;UxE+6UQNCA-1gBCEG7sN!OM*VafuMKl>zo+A(x~sibbp^Ho79{ zqwe+JBG+6&4jOP zSy^02o&l=-V8>O7=}zR=Q`euFy7uI;zkd4c%WwW(oY*4E@fi{WXAs`>jwlbXYl1(H z?niKcM#Z?BAIyRKa}JCX;9#8FrwMSM=CXVL=KSA3ITTG4y}eDGdm9x;*Tp7LkyZHY zg+v?;61WiUA;Q?;@F#W%iFhT)Db9~$&BsyuVquvl4q0-aIMV#}3mB4#Lh+KPBIQuJk;`omEPtd>f55Fw{ literal 0 HcmV?d00001 diff --git a/.zcode/success_fixture/prepared/landing/lerobot_episode_0002.mcap b/.zcode/success_fixture/prepared/landing/lerobot_episode_0002.mcap new file mode 100644 index 0000000000000000000000000000000000000000..42fd05df85b63f767d7e5381292431b09d4263c7 GIT binary patch literal 4626 zcmeGgO>Y}j@U3_4*qb;BP9Y{jX&VWV0`|sEh=5X{$)N{C2_Z^>Qmpm*dA*8v-)cXc zz$q$1Y9s`Iph5zHkSc@_Qtwc;hjQtG6ITR+10ci!^-`6Yw{O=UX{myGOU8=l?R>nM zH*a=koSm1KPp-`6RcVHFVG5S5W}9u6Yo^CSmo{p)p}VzB&kCp~pAeD~nS@oFdaTZZ zn(dpu24C7qBe0k=1T2sFRiHcoOV~{h=BsBb=Z@%{ub>eG2|g&SMRXaKM#Dpnd{aV1 zNQxBUyUDi9HtW#xGP7Ne`aW%}TMf#xz;itouzJ}19yHY3l6u%=%xqIFrfY#^Q$Nsc zx6Da}u}C0#0>zo*YHlr-DHPQqpBl_*_(kc6G%K&hkA*{O+R+{67gIn_uSO3`?@Q{0 z!R+$Drc0yAmMiGsiu7hxh?uOcJ7$@AW{;2WxYXA+9Jc9n{dMa%CF#?Yd}?LsQfm4X zKUSIV240uX(02PJR9HjZdWIivUcDx%({Z@8fpCccWMlr+y<#=vs$&Ct6iR9?i6{@z zaCK_%^yTF6g>g08^mLontOgHUt}vo14L#6{GK6uYFsf#n%(L~NsK9eBK8G@gw-1mK zDWI{sPrZ(g30Agj-K2imhaBil0hCWNM?T1${h2vsOalNJJ`VxDQ|5&3uDp4kD-oh_ zrCp`%wLlMOPi30`@t(@>*|~R31&XQAlCXI+US+_LRAikA?Ihz&;B-gO=bu?ocQ+ z2Gu&1^FYTM3I(Y{J*dDnKt9+(1>s{UVU{{ijRL*dv|Vb(lAK~~1))bv5l01?$XG4( zti$lsYCfxJ_)_{b@U^DCK{b!M>_xq9RI2j}BmaO50%}s%@>v6&`x%}T_ye_#jKHM| z29~+h(*x$A>v1F#FENOIXc$n&xwaQlzOP+p4JgA-kVXMknX91d!cpN|4k(FxV_mjK zI~Ilo_ZCifLs}Cq*DR+&x3IBP_7u~RPee1ygKFRPd;kpYq|u_b9!H#87)M#lrESZh zF=GU14cc6&&M#79k@EBPUls<>VL>T`4u?T8nOnn#M+3~7PFo)qMA4(TsXgB(o`ZLH zO>|x!am>x>F6S=w79(sxJ@<6~k^dA9-5ybF;=*F@zWjYijN^?Nt0>_@aY^Q)SR{JF&=o-+ zcCY^yyXFcSckRjvnOs%UZ$KjyDrzSC?T^1-%@ihkgW1y*r*OiOa^vQb)Vi=uFU%;d z>f%EB3{d34NTz_Wr+LOor`suSTzxjJ{e2Xk6XGry(L3q3;r0o zAH)3_6_aj$FbD3>IWS3pgK=)3Ccu4~%kKS~^MC*3P|Bj{?QP=R+o(9aE{w!QR^hW3 z5^*$0;X<^B7-NIOpV$$kl9imSlw_5CU5LfXG5!HZ5`c3NBG{xQGKYg0fI?motL`=R zch|V{-qnC3HuyZ+ksC;n2^?wwWKtEYK?8A*jQeLk?&t5}KmqVYg!qPcnyvFV>>xaj z0|dbB2n|fR`J;k-4CfhuOCqAXromC1DiP$jh0h;}%St%PA2==od>0{-aE8CaaTcH! up)Mzz=i>$;@ktXM^@iXGm99=c=A0L-6y6&`?Aois* 2 row(s): + episode_id=ba19d7fb94cdba01 task='task-1' success='true' + episode_id=e9e1a9ad1306d39d task='task-0' success='true' + (catalog holds 2 episode(s) in total) +``` + +A demo whose collector label reads never-succeeded is delivered as `success: "true"` and returned by the exact filter a buyer would run. On `lerobot/pusht` at scale, the same disagreement holds for all 206 episodes: hflow's column says `true` where the source says `False`, 100 percent of the time. + +## Why now + +Third brick of the thread #376 and #379 opened: #376 removed a stamp asserting a measurement nobody took, #389 gave the delivery a checkable receipt. This is the last record in the family that speaks without evidence, and unlike the first two it speaks about content: which demos worked. That is the column a data buyer filters on. + +## What to build + +Open call, not assumed: + +1. **Read the label**: when the source carries an outcome feature (`next.success` or equivalent), derive the episode outcome and stamp it. Whether episode success is any-frame or last-frame is a methodology choice to name. +2. **Stamp unlabeled**: when the source carries no outcome feature, omit the key. FORMAT.md:119 already says all keys are optional; omitting is the only encoding that cannot be mistaken for a collector's judgment. +3. **Column semantics**: how the catalog column represents unlabeled episodes (NULL versus a value) is the catalog's decision to make once, deliberately. + +Combinations are plausible: read when present, omit when absent, document both. + +## Definition of done + +1. The importer never invents a collector label: the episode record is copied from the source or omits the key. +2. When the source declares an outcome feature, the delivered record reports it, with the derivation named. +3. The catalog `success` column carries real information or NULL, pinned by tests through the real import and catalog path. +4. The transform's copy path is untouched; non-LeRobot imports are unaffected. +5. The fixture above lands as a test. + +## Non-goals + +- Changing the transform's episode/v1 copy path +- Judging whether pusht's upstream conversion should have populated `next.success`; that is upstream's question +- Changing catalog dedupe or the promoted-keys mechanism + +## Validation + +```bash +uv run ruff check --fix +uv run ruff format +uv run ty check +uv run pytest -q tests/test_lerobot_converter.py tests/test_catalog.py +uv run pytest -q +``` diff --git a/src/hflow/importers/lerobot.py b/src/hflow/importers/lerobot.py index 7c6bfc8b..7245f2d5 100644 --- a/src/hflow/importers/lerobot.py +++ b/src/hflow/importers/lerobot.py @@ -54,7 +54,7 @@ # can prove a landing file belongs to this exact selection (#303). Those # fields change the canonical bytes that content_episode_id hashes, so v5 # and v6 outputs must not share a converter identity. -CONVERTER_VERSION = "lerobot-converter-v6" +CONVERTER_VERSION = "lerobot-converter-v7" # Canonical transform knobs that affect published bytes for this importer. IMPORT_GOP_SECONDS = 1.0 PRESENTATION_TIMESTAMP_EPSILON_S = 0.050 @@ -75,6 +75,9 @@ class _EpisodeRow(TypedDict): data_from: int data_to: int video_windows: NotRequired[dict[str, "_VideoWindow"]] + # MAX of the episode's collector-labeled next.success frames, present only + # when the source declares that outcome feature at all. + success_outcome: NotRequired[bool] class _VideoWindow(TypedDict): @@ -584,12 +587,25 @@ def _ensure_source_archive(dataset_source: DatasetSource, cache_dir: Path) -> _S ) connection = duckdb.connect() try: + # Column discovery first: the outcome aggregate is optional in v3 + # (not every corpus declares a collector-labeled next.success), and + # naming a missing column would fail the read. A corpus without one + # is normal, not malformed. + episodes_columns = [ + column_description[0] + for column_description in connection.execute( + f"SELECT * FROM {episode_metadata_relation} LIMIT 1" + ).description + ] + has_outcome_aggregate = "stats/next.success/max" in episodes_columns + episode_rows: list[_EpisodeRow] = [] parquet_episode_rows = connection.execute( f""" SELECT "episode_index", "tasks", "length", "data/chunk_index", "data/file_index", "dataset_from_index", "dataset_to_index" + {"," + chr(34) + "stats/next.success/max" + chr(34) if has_outcome_aggregate else ""} FROM {episode_metadata_relation} ORDER BY "episode_index" """ @@ -597,17 +613,25 @@ def _ensure_source_archive(dataset_source: DatasetSource, cache_dir: Path) -> _S for parquet_episode_row in parquet_episode_rows: tasks = parquet_episode_row[1] task = (str(tasks[0]) if tasks else "") if isinstance(tasks, list) else str(tasks or "") - episode_rows.append( - { - "episode_index": int(parquet_episode_row[0]), - "task": task, - "length": int(parquet_episode_row[2]), - "data_chunk": str(parquet_episode_row[3]).split("/")[-1], - "data_file": str(parquet_episode_row[4]).split("/")[-1], - "data_from": int(parquet_episode_row[5]), - "data_to": int(parquet_episode_row[6]), - } - ) + episode_row_value: _EpisodeRow = { + "episode_index": int(parquet_episode_row[0]), + "task": task, + "length": int(parquet_episode_row[2]), + "data_chunk": str(parquet_episode_row[3]).split("/")[-1], + "data_file": str(parquet_episode_row[4]).split("/")[-1], + "data_from": int(parquet_episode_row[5]), + "data_to": int(parquet_episode_row[6]), + } + if has_outcome_aggregate: + # Episode outcome = MAX over the episode's collector-labeled + # next.success frames: any success frame makes the episode a + # success. An empty aggregate carries no label either way. + outcome_frames = parquet_episode_row[7] + if outcome_frames is not None and len(outcome_frames) > 0: + episode_row_value["success_outcome"] = any( + bool(value) for value in outcome_frames + ) + episode_rows.append(episode_row_value) # Video window columns: videos//{chunk_index,file_index,from_timestamp,to_timestamp} flattened_column_names = [ @@ -1193,20 +1217,30 @@ def _feature_rows(feature_name: str) -> list | None: sequence=frame_index, ) + episode_record: dict[str, str] = { + "task": str(episode_row["task"] or ""), + "operator": "lerobot_converter", + "embodiment": str(source_archive["info"].get("robot_type") or "unknown"), + "source_dataset": dataset_source.repo_id, + "source_revision": dataset_source.revision, + "source_episode_index": str(episode_index), + "converter_version": CONVERTER_VERSION, + "camera_keys": _encode_camera_keys(camera_keys), + "gop_seconds": f"{IMPORT_GOP_SECONDS:g}", + } + # success is the collector's label, never ours: when the source + # declares the outcome feature, report MAX over the episode's + # frames and name the derivation so the methodology travels with + # the data; when it does not, the key is omitted rather than + # invented (FORMAT.md: every episode/v1 key is optional and the + # record is copied/merged from the source recording). + success_outcome = episode_row.get("success_outcome") + if success_outcome is not None: + episode_record["success"] = "true" if success_outcome else "false" + episode_record["success_derivation"] = "max(stats/next.success)" mcap_writer.add_metadata( name="episode/v1", - data={ - "task": str(episode_row["task"] or ""), - "operator": "lerobot_converter", - "success": "true", - "embodiment": str(source_archive["info"].get("robot_type") or "unknown"), - "source_dataset": dataset_source.repo_id, - "source_revision": dataset_source.revision, - "source_episode_index": str(episode_index), - "converter_version": CONVERTER_VERSION, - "camera_keys": _encode_camera_keys(camera_keys), - "gop_seconds": f"{IMPORT_GOP_SECONDS:g}", - }, + data=episode_record, ) mcap_writer.add_metadata( name="source-provenance/v1", diff --git a/tests/test_lerobot_converter.py b/tests/test_lerobot_converter.py index 99c64066..c890ece3 100755 --- a/tests/test_lerobot_converter.py +++ b/tests/test_lerobot_converter.py @@ -1866,3 +1866,219 @@ def should_not_convert(**_kwargs: object) -> prep._PublishedEpisode: manifest = json.loads((output_dir / "prepared-manifest.json").read_text()) assert manifest["episodes_converted"] == 0 assert len(manifest["episodes"]) == 2 + + +# --- success label: read the collector's outcome, never invent it (#395) ----- + + +def _build_success_label_corpus(root: Path, outcome_mode: str) -> dict: + """One two-frame episode. outcome_mode: 'transition', 'all-false', 'none'.""" + has_outcome = outcome_mode != "none" + info = { + "fps": 30, + "data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet", + "video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4", + "features": { + "action": {"dtype": "float32", "shape": [1]}, + "observation.state": {"dtype": "float32", "shape": [1]}, + "observation.images.up": {"dtype": "video", "shape": [480, 640, 3]}, + "timestamp": {"dtype": "float32", "shape": [1]}, + }, + "robot_type": "so101", + } + if has_outcome: + info["features"]["next.success"] = {"dtype": "bool", "shape": [1]} + (root / "meta").mkdir(parents=True, exist_ok=True) + (root / "meta" / "info.json").write_text(json.dumps(info)) + + import duckdb + + conn = duckdb.connect() + ep_cols = [ + "episode_index", + "length", + "data/chunk_index", + "data/file_index", + "dataset_from_index", + "dataset_to_index", + "videos/observation.images.up/chunk_index", + "videos/observation.images.up/file_index", + "videos/observation.images.up/from_timestamp", + "videos/observation.images.up/to_timestamp", + "tasks", + ] + row: list[object] = [0, 2, "000", "000", 0, 2, "000", "000", 0.0, 0.0, ["push the block"]] + if has_outcome: + stats_min, stats_max = ( + ([False], [True]) if outcome_mode == "transition" else ([False], [False]) + ) + ep_cols += ["stats/next.success/min", "stats/next.success/max"] + row += [stats_min, stats_max] + ep_path = root / "meta" / "episodes" / "chunk-000" / "file-000.parquet" + ep_path.parent.mkdir(parents=True, exist_ok=True) + vals = ( + "(" + + ",".join( + "[" + ",".join(str(bool(item)) for item in value) + "]" + if isinstance(value, list) and value and all(isinstance(item, bool) for item in value) + else "[" + ",".join(f"'{item}'" for item in value) + "]" + if isinstance(value, list) + else f"'{value}'" + if isinstance(value, str) + else str(value) + for value in row + ) + + ")" + ) + conn.execute( + f"COPY (SELECT * FROM (VALUES {vals}) AS t({','.join(chr(34) + c + chr(34) for c in ep_cols)})) " + f"TO '{str(ep_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" + ) + + frame_outcomes = [False, True] if outcome_mode == "transition" else [False, False] + data_cols = 'index, episode_index, frame_index, timestamp, "observation.state", action' + data_rows = [ + f"({index}, 0, {frame_index}, 0.0, [0.0], [0.5]" + for index, frame_index in enumerate(range(2)) + ] + if has_outcome: + data_cols += ', "next.success"' + data_rows = [ + data_row + f", {str(frame_outcomes[frame_index]).lower()})" + for frame_index, data_row in enumerate(data_rows) + ] + else: + data_rows = [data_row + ")" for data_row in data_rows] + data_path = root / "data" / "chunk-000" / "file-000.parquet" + data_path.parent.mkdir(parents=True, exist_ok=True) + conn.execute( + f"COPY (SELECT * FROM (VALUES {','.join(data_rows)}) AS t({data_cols})) " + f"TO '{str(data_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" + ) + conn.close() + return {"info": info} + + +def _import_success_label_corpus( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, outcome_mode: str +) -> Path: + root = tmp_path / "corpus" + corpus = _build_success_label_corpus(root, outcome_mode) + output_dir = tmp_path / "out" + + monkeypatch.setattr( + prep, "_hf_repo_info", lambda repo, revision: {"sha": "abc1234", "license": "apache-2.0"} + ) + monkeypatch.setattr(prep, "_fetch_info_json", lambda repo, rev, cache: corpus["info"]) + monkeypatch.setattr( + prep, + "_hf_tree", + lambda repo, rev, path: ( + [{"path": "meta/episodes/chunk-000/file-000.parquet", "type": "file"}] + if "episodes" in path + else [{"path": "meta/info.json", "type": "file"}] + ), + ) + + def fake_download(url: str, dest: Path, **_kwargs: object) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + if "meta/episodes" in url: + shutil.copy(root / "meta" / "episodes" / "chunk-000" / "file-000.parquet", dest) + elif url.endswith("info.json"): + shutil.copy(root / "meta" / "info.json", dest) + else: + shutil.copy(root / "data" / "chunk-000" / "file-000.parquet", dest) + + monkeypatch.setattr(prep, "_download_file", fake_download) + monkeypatch.setattr( + prep, + "_transcode_mp4_to_h264", + lambda mp4_path, gop, fps: ( + [ + b"\x00\x00\x00\x01\x09\x10\x00\x00\x00\x01\x67\x42\x00" + b"\x00\x00\x00\x01\x68\x88\x80\x00\x00\x00\x01\x65\x88" + ] + * 2 + ), + ) + monkeypatch.setattr(prep, "_get_video_pts_times", lambda path: [0, 0]) + monkeypatch.setattr(prep, "ffmpeg_version", lambda: "test-ffmpeg") + + prep.import_lerobot_dataset( + dataset_repo="fake/repo", + output_dir=output_dir, + camera_keys=("observation.images.up",), + ) + return output_dir + + +def test_success_label_reports_max_over_episode_frames( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """MAX over the collector's next.success frames: a False frame followed + by a True frame makes the episode a success, even though the LAST frame + is False. The derivation is stamped so the methodology travels.""" + from hflow.episode import Episode + + output_dir = _import_success_label_corpus(tmp_path, monkeypatch, "transition") + landing = sorted((output_dir / "landing").glob("*.mcap")) + with Episode(landing[0]) as episode: + record = episode.metadata_records["episode/v1"] + assert record["success"] == "true" + assert record["success_derivation"] == "max(stats/next.success)" + + +def test_success_label_reports_false_when_source_is_all_false( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An all-false source label ships as 'false', never as an invented + 'true': the collector's judgment, reported verbatim.""" + from hflow.episode import Episode + + output_dir = _import_success_label_corpus(tmp_path, monkeypatch, "all-false") + landing = sorted((output_dir / "landing").glob("*.mcap")) + with Episode(landing[0]) as episode: + record = episode.metadata_records["episode/v1"] + assert record["success"] == "false" + assert record["success_derivation"] == "max(stats/next.success)" + + +def test_success_label_omitted_when_source_has_no_outcome_feature( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A corpus without the outcome feature is normal, not malformed: the key + is omitted (never substituted), the import succeeds, and the catalog + promotion renders the omitted key as SQL NULL (catalog.py:886).""" + import duckdb + + from hflow.catalog import Catalog + from hflow.episode import Episode + from hflow.transform import stamps_from_provenance + + output_dir = _import_success_label_corpus(tmp_path, monkeypatch, "none") + landing = sorted((output_dir / "landing").glob("*.mcap")) + with Episode(landing[0]) as episode: + record = episode.metadata_records["episode/v1"] + assert "success" not in record + assert "success_derivation" not in record + + catalog_root = tmp_path / "catalog" + catalog = Catalog(catalog_root) + catalog.append_episode( + canonical_path=landing[0], + stamps=stamps_from_provenance(episode.metadata), + episode_metadata=dict(episode.metadata), + check_rows=[], + ) + + rows = duckdb.sql( + f"SELECT success FROM read_parquet('{catalog_root / 'episodes' / '*.parquet'}')" + ).fetchall() + assert len(rows) == 1 + assert rows[0][0] is None + + +def test_converter_version_bumped_with_the_label_support() -> None: + """The label changes episode/v1 bytes, which content_episode_id hashes: + the converter version moves with the change, not after it.""" + assert prep.CONVERTER_VERSION == "lerobot-converter-v7" From a6bc98bbf3a963c25732b22b16e96dc889f589a6 Mon Sep 17 00:00:00 2001 From: Kingston Date: Fri, 4 Sep 2026 11:16:48 -0700 Subject: [PATCH 2/2] fix(lerobot): drop the committed .zcode scratch tree and hold the empty aggregate The PR carried 34 files under .zcode/, including generated MCAPs, MP4s and catalog parquets. Removed, and .zcode/ now sits beside .claude/ and .agents/ in .gitignore so it cannot recur for anyone using that tool. An outcome column present but empty for an episode stamped success: "false", since any([]) is False. That is the same invention the hardcoded "true" was, one value over, and nothing caught it; it now has its own case. Also lifts the column name and derivation string into constants, replacing a chr(34) workaround for f-string quote nesting on 3.11. --- .gitignore | 1 + .zcode/pr379_body.md | 15 -- .zcode/receipt_timing.py | 35 --- .zcode/success_catalog_fixture.py | 243 ------------------ .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 286 -> 0 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 286 -> 0 bytes .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 4265 -> 0 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 4266 -> 0 bytes .../catalog/format_version | 1 - .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 243 -> 0 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 243 -> 0 bytes .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 292 -> 0 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 292 -> 0 bytes .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 342 -> 0 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 342 -> 0 bytes .../ba19d7fb94cdba01-44ca08086eed.parquet | Bin 203 -> 0 bytes .../e9e1a9ad1306d39d-9a5226c7b5ff.parquet | Bin 203 -> 0 bytes .../meta/episodes/chunk-000/file-000.parquet | Bin 2956 -> 0 bytes .zcode/success_catalog_fixture/meta/info.json | 1 - .../meta/episodes/chunk-000/file-000.parquet | Bin 2956 -> 0 bytes ...servation_images_side-chunk000-file000.mp4 | Bin 1042 -> 0 bytes ...observation_images_up-chunk000-file000.mp4 | Bin 1042 -> 0 bytes .../landing/lerobot_episode_0001.mcap | Bin 4626 -> 0 bytes .../landing/lerobot_episode_0002.mcap | Bin 4626 -> 0 bytes .../prepared/prepared-manifest.json | 26 -- .zcode/success_fixture.py | 214 --------------- .../meta/episodes/chunk-000/file-000.parquet | Bin 2956 -> 0 bytes .zcode/success_fixture/meta/info.json | 1 - .../meta/episodes/chunk-000/file-000.parquet | Bin 2956 -> 0 bytes ...servation_images_side-chunk000-file000.mp4 | Bin 1042 -> 0 bytes ...observation_images_up-chunk000-file000.mp4 | Bin 1042 -> 0 bytes .../landing/lerobot_episode_0001.mcap | Bin 4626 -> 0 bytes .../landing/lerobot_episode_0002.mcap | Bin 4626 -> 0 bytes .../prepared/prepared-manifest.json | 26 -- .zcode/success_issue_body.md | 66 ----- src/hflow/importers/lerobot.py | 16 +- tests/test_lerobot_converter.py | 36 ++- 37 files changed, 46 insertions(+), 635 deletions(-) delete mode 100644 .zcode/pr379_body.md delete mode 100644 .zcode/receipt_timing.py delete mode 100644 .zcode/success_catalog_fixture.py delete mode 100644 .zcode/success_catalog_fixture/catalog/check_runs/ba19d7fb94cdba01-44ca08086eed.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/check_runs/e9e1a9ad1306d39d-9a5226c7b5ff.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/episodes/ba19d7fb94cdba01-44ca08086eed.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/episodes/e9e1a9ad1306d39d-9a5226c7b5ff.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/format_version delete mode 100644 .zcode/success_catalog_fixture/catalog/intervals/ba19d7fb94cdba01-44ca08086eed.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/intervals/e9e1a9ad1306d39d-9a5226c7b5ff.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/measurements/ba19d7fb94cdba01-44ca08086eed.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/measurements/e9e1a9ad1306d39d-9a5226c7b5ff.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/observations/ba19d7fb94cdba01-44ca08086eed.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/observations/e9e1a9ad1306d39d-9a5226c7b5ff.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/tags/ba19d7fb94cdba01-44ca08086eed.parquet delete mode 100644 .zcode/success_catalog_fixture/catalog/tags/e9e1a9ad1306d39d-9a5226c7b5ff.parquet delete mode 100644 .zcode/success_catalog_fixture/meta/episodes/chunk-000/file-000.parquet delete mode 100644 .zcode/success_catalog_fixture/meta/info.json delete mode 100644 .zcode/success_catalog_fixture/prepared/_lerobot_cache/abc1234/meta/episodes/chunk-000/file-000.parquet delete mode 100644 .zcode/success_catalog_fixture/prepared/_lerobot_cache/abc1234/videos/observation_images_side-chunk000-file000.mp4 delete mode 100644 .zcode/success_catalog_fixture/prepared/_lerobot_cache/abc1234/videos/observation_images_up-chunk000-file000.mp4 delete mode 100644 .zcode/success_catalog_fixture/prepared/landing/lerobot_episode_0001.mcap delete mode 100644 .zcode/success_catalog_fixture/prepared/landing/lerobot_episode_0002.mcap delete mode 100644 .zcode/success_catalog_fixture/prepared/prepared-manifest.json delete mode 100644 .zcode/success_fixture.py delete mode 100644 .zcode/success_fixture/meta/episodes/chunk-000/file-000.parquet delete mode 100644 .zcode/success_fixture/meta/info.json delete mode 100644 .zcode/success_fixture/prepared/_lerobot_cache/abc1234/meta/episodes/chunk-000/file-000.parquet delete mode 100644 .zcode/success_fixture/prepared/_lerobot_cache/abc1234/videos/observation_images_side-chunk000-file000.mp4 delete mode 100644 .zcode/success_fixture/prepared/_lerobot_cache/abc1234/videos/observation_images_up-chunk000-file000.mp4 delete mode 100644 .zcode/success_fixture/prepared/landing/lerobot_episode_0001.mcap delete mode 100644 .zcode/success_fixture/prepared/landing/lerobot_episode_0002.mcap delete mode 100644 .zcode/success_fixture/prepared/prepared-manifest.json delete mode 100644 .zcode/success_issue_body.md diff --git a/.gitignore b/.gitignore index 15e29790..bcfb859a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ packages/hflow-server/src/hflow_server/static/ # Local maintainer tooling (agent skills, settings); not part of the public repo. .claude/ .agents/ +.zcode/ # Transient: `pnpm gen:api` dumps the schema here on its way to src/apiSchema.ts. .openapi.json diff --git a/.zcode/pr379_body.md b/.zcode/pr379_body.md deleted file mode 100644 index b7989248..00000000 --- a/.zcode/pr379_body.md +++ /dev/null @@ -1,15 +0,0 @@ -Closes #379. - -The manifest recorded how many episodes were converted but not which ones, so a truncated, missing, or swapped episode was undetectable from the delivery (#379's controlled result: one landing file cut to zero bytes, manifest unchanged). This adds the receipt. - -`prepared-manifest.json` moves to schema version 3: every delivered episode carries its published URI, its `content_episode_id` content address, and its byte size. The v2 top-level keys are unchanged, so existing readers keep working, and the entries carry everything a future verify command needs, keeping verify purely additive per the issue direction: the entries are done here, verify is deferred, not rejected. - -Per the constraints on the issue: the hash is taken inside `_convert_single_episode` while the canonical file is still on local disk, before `storage.publish`, so a bucket root never downloads its own upload to learn its content id; the recorded URI is the published object (a bucket prefix recipient has no local paths); and `content_episode_id` stays the single hashing implementation, reused rather than forked. The source cache is not hashed; the receipt covers what was delivered, not what it was made from. - -The truncation fixture from the issue lands as `test_manifest_content_id_detects_a_truncated_episode`: after truncation, both the size and the content id disagree with the manifest, which is the detection the receipt exists for. - -Cost: one linear sha256 read of each canonical file while it is already on local disk, 4.6 ms at fixture scale, about 0.2 s for a 100 MB episode. - -Gate: ruff, format, ty clean; 1471 passed / 6 skipped, the single failure (`utc-stats-test-date-collision`) reproduced on the base commit and unrelated, fix already in flight on its own branch. - -Refs #379, builds on #377's published URI list. diff --git a/.zcode/receipt_timing.py b/.zcode/receipt_timing.py deleted file mode 100644 index e1df8636..00000000 --- a/.zcode/receipt_timing.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Timing: the receipt cost — content_episode_id + stat per episode. - -Measures the hashing added by #379 on a real canonical episode file, -at fixture scale and extrapolated at delivery scale. - -Run: uv run --locked --all-extras python .zcode/receipt_timing.py -""" - -import statistics -import sys -import time -from pathlib import Path - -from hflow.catalog import content_episode_id - - -def main() -> None: - path = Path(sys.argv[1]) - size = __import__("pathlib").Path(path).stat().st_size - timings = [] - for _ in range(20): - started = time.perf_counter() - content_episode_id(path) - timings.append(time.perf_counter() - started) - median_us = statistics.median(timings) * 1_000_000 - print(f"episode file: {path} ({size} bytes)") - print(f"content_episode_id median of 20 runs: {median_us:.1f} us") - # sha256 throughput ~500 MB/s in CPython: a 100 MB episode costs ~0.2 s, - # one linear read of a file that is already on local disk pre-publish. - projected_100mb_s = 100 / 500 - print(f"projected at 100 MB episode (~500 MB/s sha256): ~{projected_100mb_s:.2f} s") - - -if __name__ == "__main__": - main() diff --git a/.zcode/success_catalog_fixture.py b/.zcode/success_catalog_fixture.py deleted file mode 100644 index 02c43184..00000000 --- a/.zcode/success_catalog_fixture.py +++ /dev/null @@ -1,243 +0,0 @@ -"""Success-stamp fixture: the #379 lesson applied — every stage real. - -Chain: real import (network/ffmpeg stubbed) -> real Episode.metadata read -> -real Catalog.append_episode -> real duckdb query. Shows a demo whose source -collector label is next.success=False being delivered as success=true and -returned by a buyer's success=true filter. - -Run: uv run --locked --all-extras python .zcode/success_catalog_fixture.py -""" - -import json -import shutil -from pathlib import Path - -import duckdb - -from hflow.catalog import Catalog -from hflow.episode import Episode -from hflow.importers.lerobot import import_lerobot_dataset -from hflow.transform import stamps_from_provenance - -ROOT = Path(".zcode/success_catalog_fixture") -EPISODES = 2 - - -def build_corpus(root: Path) -> dict: - info = { - "fps": 30, - "data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet", - "video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4", - "features": { - "action": {"dtype": "float32", "shape": [1]}, - "observation.state": {"dtype": "float32", "shape": [1]}, - "observation.images.up": {"dtype": "video", "shape": [480, 640, 3]}, - "observation.images.side": {"dtype": "video", "shape": [480, 640, 3]}, - "timestamp": {"dtype": "float32", "shape": [1]}, - "next.success": {"dtype": "bool", "shape": [1]}, - }, - "robot_type": "so101", - } - (root / "meta").mkdir(parents=True, exist_ok=True) - (root / "meta" / "info.json").write_text(json.dumps(info)) - - conn = duckdb.connect() - ep_cols = [ - "episode_index", - "length", - "data/chunk_index", - "data/file_index", - "dataset_from_index", - "dataset_to_index", - "videos/observation.images.up/chunk_index", - "videos/observation.images.up/file_index", - "videos/observation.images.up/from_timestamp", - "videos/observation.images.up/to_timestamp", - "videos/observation.images.side/chunk_index", - "videos/observation.images.side/file_index", - "videos/observation.images.side/from_timestamp", - "videos/observation.images.side/to_timestamp", - "tasks", - "stats/next.success/min", - "stats/next.success/max", - ] - rows = [ - [ - i, - 1, - "000", - "000", - i, - i + 1, - "000", - "000", - 0.0, - 0.0, - "000", - "000", - 0.0, - 0.0, - [f"task-{i}"], - [False], - [False], - ] - for i in range(EPISODES) - ] - ep_path = root / "meta" / "episodes" / "chunk-000" / "file-000.parquet" - ep_path.parent.mkdir(parents=True, exist_ok=True) - vals = ",".join( - "(" - + ",".join( - "[" + ",".join(f"'{x}'" for x in v) + "]" - if isinstance(v, list) - else f"'{v}'" - if isinstance(v, str) - else str(v) - for v in row - ) - + ")" - for row in rows - ) - conn.execute( - f"COPY (SELECT * FROM (VALUES {vals}) AS t({','.join(chr(34) + c + chr(34) for c in ep_cols)})) " - f"TO '{str(ep_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" - ) - - data_rows = [[i, i, 0, 0.0, [0.0], [0.5], False] for i in range(EPISODES)] - data_path = root / "data" / "chunk-000" / "file-000.parquet" - data_path.parent.mkdir(parents=True, exist_ok=True) - dvals = ",".join( - "(" - + ",".join( - str(v) - if not isinstance(v, (str, list)) - else ( - "'" + str(v) + "'" - if isinstance(v, str) - else "[" + ",".join(str(x) for x in v) + "]" - ) - for v in row - ) - + ")" - for row in data_rows - ) - conn.execute( - f"COPY (SELECT * FROM (VALUES {dvals}) AS t(index, episode_index, frame_index, " - 'timestamp, "observation.state", action, "next.success")) ' - f"TO '{str(data_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" - ) - conn.close() - return {"info": info} - - -def main() -> None: - shutil.rmtree(ROOT, ignore_errors=True) - corpus = build_corpus(ROOT) - output_dir = ROOT / "prepared" - - import hflow.importers.lerobot as prep - - stubs = { - "_hf_repo_info": lambda repo, revision: {"sha": "abc1234", "license": "apache-2.0"}, - "_fetch_info_json": lambda repo, rev, cache: corpus["info"], - "_hf_tree": lambda repo, rev, path: ( - [{"path": "meta/episodes/chunk-000/file-000.parquet", "type": "file"}] - if "episodes" in path - else [{"path": "meta/info.json", "type": "file"}] - ), - "_transcode_mp4_to_h264": lambda mp4_path, gop, fps: [ - b"\x00\x00\x00\x01\x09\x10\x00\x00\x00\x01\x67\x42\x00" - b"\x00\x00\x00\x01\x68\x88\x80\x00\x00\x00\x01\x65\x88" - ], - "_get_video_pts_times": lambda path: [0], - "ffmpeg_version": lambda: "fixture-ffmpeg", - } - originals = {name: getattr(prep, name) for name in stubs} - - def fake_download(url: str, dest: Path, **kw: object) -> None: - dest.parent.mkdir(parents=True, exist_ok=True) - if "meta/episodes" in url: - import shutil - - shutil.copy(ROOT / "meta" / "episodes" / "chunk-000" / "file-000.parquet", dest) - elif url.endswith("info.json"): - import shutil - - shutil.copy(ROOT / "meta" / "info.json", dest) - else: - import shutil - - shutil.copy(ROOT / "data" / "chunk-000" / "file-000.parquet", dest) - - stubs["_download_file"] = fake_download - try: - for name, fake in stubs.items(): - setattr(prep, name, fake) - import_lerobot_dataset( - dataset_repo="fake/repo", - output_dir=output_dir, - camera_keys=("observation.images.up", "observation.images.side"), - ) - finally: - for name, original in originals.items(): - setattr(prep, name, original) - - print("--- stage 1: what the source declares (collector's own label) ---") - conn = duckdb.connect() - src = conn.execute( - 'SELECT episode_index, "next.success" FROM read_parquet(' - f"'{ROOT / 'data' / 'chunk-000' / 'file-000.parquet'}') ORDER BY index" - ).fetchall() - for r in src: - print(f" data parquet frame (episode {r[0]}): next.success = {r[1]}") - agg = conn.execute( - 'SELECT episode_index, "stats/next.success/max" FROM read_parquet(' - f"'{ROOT / 'meta' / 'episodes' / 'chunk-000' / 'file-000.parquet'}') ORDER BY episode_index" - ).fetchall() - for r in agg: - print(f" episodes parquet aggregate (episode {r[0]}): stats/next.success/max = {r[1]}") - conn.close() - - print("--- stage 2: what the importer shipped (read back through the real Episode reader) ---") - landing = sorted((output_dir / "landing").glob("*.mcap")) - catalog_root = ROOT / "catalog" - catalog = Catalog(catalog_root) - for episode_path in landing: - with Episode(episode_path) as episode: - metadata = episode.metadata - stamps = stamps_from_provenance(metadata) - print(f" {episode_path.name}: episode/v1 success = {metadata.get('success')!r}") - catalog.append_episode( - canonical_path=episode_path, - stamps=stamps, - episode_metadata=metadata, - check_rows=[], - ) - - print("--- stage 3: the buyer's query (real catalog, real SQL) ---") - query_conn = duckdb.connect() - rows = query_conn.execute( - f"SELECT episode_id, task, success FROM read_parquet(" - f"'{catalog_root / 'episodes' / '*.parquet'}') WHERE success = 'true'" - ).fetchall() - print(f" SELECT ... WHERE success = 'true' -> {len(rows)} row(s):") - for r in rows: - print(f" episode_id={r[0]} task={r[1]!r} success={r[2]!r}") - total_rows = query_conn.execute( - f"SELECT COUNT(*) FROM read_parquet('{catalog_root / 'episodes' / '*.parquet'}')" - ).fetchall() - print(f" (catalog holds {total_rows[0][0]} episode(s) in total)") - query_conn.close() - print("--- verdict ---") - print( - " source label: next.success = False on every frame of every episode" - " (the collector's own outcome signal, declared in info.json)" - ) - print( - " delivered + cataloged: success = 'true' for every episode;" - " a buyer filtering successful demos receives all of them" - ) - - -if __name__ == "__main__": - main() diff --git a/.zcode/success_catalog_fixture/catalog/check_runs/ba19d7fb94cdba01-44ca08086eed.parquet b/.zcode/success_catalog_fixture/catalog/check_runs/ba19d7fb94cdba01-44ca08086eed.parquet deleted file mode 100644 index 080e6b600031d234dee729880796201852b1dd37..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 286 zcmZ{g!D<3Q42GlBB3nfix47V8FAFP(AhOVF#bXcB7Z}D#YzK8#GBZ9!AHhfPA)R6M z*pSorC;v~f{rq00?d(>;q>o-5a3`$#rISoYkoJo9B%!mmHImR+K!XPCjo54RLwsg! z+><9^Q5}SlCwiiOgD%8Y*CtX3o?XS-X(f-1->Y@fc^lOvCZcEQ!nm-F-XZ3iqx=%& zHzrjx5(?>x@z>vQI)_;AkoC%P6K>e7^G9HQSuUzYm6toOkC@|!LMd0PJ;q>o-5a3`$#rISoYkoJo9B%!mmHImR+K!XPCjo54RLwsg! z+><9^Q5}SlCwiiOgD%8Y*CtX3o?XS-X(f-1->Y@fc^lOvCZcEQ!nm-F-XZ3iqx=%& zHzrjx5(?>x@z>vQI)_;AkoC%P6K>e7^G9HQSuUzYm6toOkC@|!LMd0PJ6LlD1K_wE}G7WH-A_7prQkRS!_1suYA;s%YYwWX&eF z8QW<^Td6|80aY9-4&{J^5GPIu^;9nW0SFZg`cIQ>V4+`-V-Z=K6*~UAJWbwr6)+9rLE= zGpad^I+#(S^{_=utE=@Wv)i`UqK@gpz*enRTdwvE?Cce_EFF!-e5f2oEI`D2XNAyC zV|itb8f!EF?wXZFsT{LEzY*HcfPKsMD!$8eT9g)JpsXH(V9}6=-K|RCU`<+&L0;Gl zAvu6V(>82tK$%AwpbJtlhMsGL=u}{k`4j-hrBV!VxETV{0C4R(FNjAO>0vsC`6Bgg z2s#3nTP0)Dx3(%CW?c&Fiu;sdx3pd?H6iP@T4|}cRzg|V!ya8PB0UPN1rfTYZ3!m| z!>}vTv07!Y3Ij@=PT!&3>Px5)rA9cu*tMM&th7r#VIwo>;hdCrDNL25KS%X+zQ}Cf z(KKv$v(EH`3|N^xs@QF)vH=!Ol&fYRcd1)-O_F|;`-4?b%6Ehh1}~#&FSTA=xKnUU zhxSa1Rv)8sY)#YO=Uroy_Hng@rk`K5eP+-WfeM`cr`W+;RDPn0)S_xLEkbV! zx2igtVRT^D?<%-$)R$Kp$6m?!%oOVR&Ca@}k?Bs#)g-cbcBn4hf#sN!=3}WopoEzY zc9kL(Vw#zyP!pW5UEg(~f6|s;`}*n>lEj54^Q%aiCBCsm+O2R(QVHtyq^KMmg-%YQ zP;{927Hlx9OPRw=%ZubkYzEnIJ%&A-wE~bi8uJ{?F2Wsj!%W3QbD`5{G<;!YYS7ze z?V(Nh*r`OgpQ^m9J{0t|pk~J*1y#e<--+@gQTY z=*v{HfD(O$Bb&0W{*uxap$-S+lwT9pw>2I1*P3TR7s2D7oJ#Nb$C=i|@H8|n|x zSks)<3@>0|;yGbs@=d_aJkH3!rqvIpb>*(O59%{XjyMULphl$;Rf}F@fjS~YBjp4w ziX)?H2uRbI9|bmmAVtEEC2>wx=duw~0{#_X!?*$yp=QXMJW$7I6AhpdzXR~*vlMip zfS%RCvpVR`&krdL;-oH47J1vK$qn`QId~l*IG||wHPLN$S}ehSZ@1fj}%GMAQ-Q#bjXW9Y6}Sc*8@F({}nF5pA`QB3vgrn diff --git a/.zcode/success_catalog_fixture/catalog/episodes/e9e1a9ad1306d39d-9a5226c7b5ff.parquet b/.zcode/success_catalog_fixture/catalog/episodes/e9e1a9ad1306d39d-9a5226c7b5ff.parquet deleted file mode 100644 index 942e8248f2614b918402960bb4ef1cfe6d30ff7a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4266 zcmeHL&2Jk;6rWw})M@i!r=Dd~oW!!ym4Vp18z*iOSydZFJwS!3QV?pXtnu#HSz_-l z`{7o!l_~_BDh{ZJazH|e6DNcahj8H!;M8*uxxkq}paS0PdhPWlsmiGbDy{e9z2Cg| z=FOY8zx9=iS5=Jg8s1pJJlss+353q*JVJ=)vT&xGR3)24S8GaLuWjmpS;0ySEL;o0 zQvlv1>#M7E?ObcU-3H(k&c=Wf`2dJO@qot<`Izbp?yt{7D|f+)+%r8{lizcxq zLuK2cHgTvfcZsPRW=HN)$7)$#liG$01DlGXsMVfE>}N$S;4`t9PnE-nIf$60@c69g zw7s@kt5dB`8SvPwEaP&_{@hMzKLhqn%d7YglhMn$8y; z%eQ5jXx_9v`6vTcrq7603#zP7xTSK{=#dU}tFED^_j7+Z6_oOA?$iF8Xwpk<73c32 zY{RBq!=%*%RF17_`q#Xx?b05pmeAy@%a-qGw5dY{PXAMElRYX!@mbst(@RQJZHB4S z+uR+bEG$epv~Q^QMbgsL)!N#^V#aq2PR;MOw`5tLY^Piq>&q7g>e4;P_6(kj<@$se zrkZUl5f@^bsg+Q}&Nps?`skJ3cbDJ({`v&M{QT3obtF#fzP6{gn&F&q2@3WsFPdIYOF^h@Ujj%b8`bqg91_)CyPPbP=qzAhiXA{_BAr!5<4(;9rV=0V!u<_5c6? diff --git a/.zcode/success_catalog_fixture/catalog/format_version b/.zcode/success_catalog_fixture/catalog/format_version deleted file mode 100644 index d00491fd..00000000 --- a/.zcode/success_catalog_fixture/catalog/format_version +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/.zcode/success_catalog_fixture/catalog/intervals/ba19d7fb94cdba01-44ca08086eed.parquet b/.zcode/success_catalog_fixture/catalog/intervals/ba19d7fb94cdba01-44ca08086eed.parquet deleted file mode 100644 index df1deafcd39b60b5b2d8f7e88e7391cacd2b739b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 243 zcmWG=3^EjDlI$^Mkl;-zP0mh9iZ4#iNX<e%gjqpEh;F=%qxM3a{)CbXUFFy=BC1gcp*Y%sYS(^`FSu==Iqo;5St5VK2KR< zPHAd*kO9zhu$ z1`rTqkmS+OZ~^+n#Yq8btwNcho~fRxf<{tlW=@JiibZltVxqBeT8gIR92o`%T!=w} Kk%1uq7(M`g-bA$k diff --git a/.zcode/success_catalog_fixture/catalog/measurements/e9e1a9ad1306d39d-9a5226c7b5ff.parquet b/.zcode/success_catalog_fixture/catalog/measurements/e9e1a9ad1306d39d-9a5226c7b5ff.parquet deleted file mode 100644 index 58f4b5e8810623f7ec08083534dec6826c05af70..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 292 zcmWG=3^EjDl3ZiTAie%gjqpEh;F=%qxM3a{)CbXUFFy=BC1gcp*Y%sYS(^`FSu==Iqo;5St5VK2KR< zPHAd*kO9zhu$ z1`rTqkmS+OZ~^+n#Yq8btwNcho~fRxf<{tlW=@JiibZltVxqBeT8gIR92o`%T!=w} Kk%1uq7(M`g-bA$k diff --git a/.zcode/success_catalog_fixture/catalog/observations/ba19d7fb94cdba01-44ca08086eed.parquet b/.zcode/success_catalog_fixture/catalog/observations/ba19d7fb94cdba01-44ca08086eed.parquet deleted file mode 100644 index 39ecdfff92817296c8084b8dc9bb46b19ec600bc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 342 zcmaKo!AiqG5QfLrQ%n`Xlm>dpp@9feC?()k=}9~kUtrnYnd*{cH|%bLzJrhCBlu7z zO>%XZ)5rY({4>X=w_P-ge{)zZu#p7{=Hv%eJX%5&rn6N*wQ>fDDNJENH@4C2r1U%4 z+A8fQ9&|!kFwHBP&|*L>S*MJi=(Q1!Y|DMeM&HYBRp?bk&hx5f+NGOe*}R~yUKjKO zEiW5lXc|${7;iwHKF=|rPdp%DObJo%du2%)i-e5(^zQZhcKrZ-Eg=rG?9g3(cuvNM jNxR$T+dRocqe@KhQ6lI2{R!{mpEbb$4KEh}$L^zlNyJf) diff --git a/.zcode/success_catalog_fixture/catalog/observations/e9e1a9ad1306d39d-9a5226c7b5ff.parquet b/.zcode/success_catalog_fixture/catalog/observations/e9e1a9ad1306d39d-9a5226c7b5ff.parquet deleted file mode 100644 index 39ecdfff92817296c8084b8dc9bb46b19ec600bc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 342 zcmaKo!AiqG5QfLrQ%n`Xlm>dpp@9feC?()k=}9~kUtrnYnd*{cH|%bLzJrhCBlu7z zO>%XZ)5rY({4>X=w_P-ge{)zZu#p7{=Hv%eJX%5&rn6N*wQ>fDDNJENH@4C2r1U%4 z+A8fQ9&|!kFwHBP&|*L>S*MJi=(Q1!Y|DMeM&HYBRp?bk&hx5f+NGOe*}R~yUKjKO zEiW5lXc|${7;iwHKF=|rPdp%DObJo%du2%)i-e5(^zQZhcKrZ-Eg=rG?9g3(cuvNM jNxR$T+dRocqe@KhQ6lI2{R!{mpEbb$4KEh}$L^zlNyJf) diff --git a/.zcode/success_catalog_fixture/catalog/tags/ba19d7fb94cdba01-44ca08086eed.parquet b/.zcode/success_catalog_fixture/catalog/tags/ba19d7fb94cdba01-44ca08086eed.parquet deleted file mode 100644 index 6fb89d94d058a6d353947417188244d34b3a3321..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 203 zcmWG=3^EjDlFTt>kl;-zP0mh9iZ4#iNX<*kO z9zhu$1`rTqkmS+OZ~~ S86(5MfQ%US1I-8kdIJCkS1+mn diff --git a/.zcode/success_catalog_fixture/catalog/tags/e9e1a9ad1306d39d-9a5226c7b5ff.parquet b/.zcode/success_catalog_fixture/catalog/tags/e9e1a9ad1306d39d-9a5226c7b5ff.parquet deleted file mode 100644 index 6fb89d94d058a6d353947417188244d34b3a3321..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 203 zcmWG=3^EjDlFTt>kl;-zP0mh9iZ4#iNX<*kO z9zhu$1`rTqkmS+OZ~~ S86(5MfQ%US1I-8kdIJCkS1+mn diff --git a/.zcode/success_catalog_fixture/meta/episodes/chunk-000/file-000.parquet b/.zcode/success_catalog_fixture/meta/episodes/chunk-000/file-000.parquet deleted file mode 100644 index bdd4598590cf8fe52b060c7d103318637c0649e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2956 zcmd5;?`ji45Z_)d*R-^1qm8@cHvaZcL7OUH?#fI5nS8Fr(+`GW#1_<4=OwTMeFZymPkOS87 z;I240v)kADu#c$sai24s-d=R?oq{0l~vYe4s*RMAt0y&%N{Se zZ0Hqvzu9D*mmJ&uS50%cNK;o+>C{y+N77VDQIeWSe5&ZG{!~B8C8~yK%?4gO(Mf)6 zl}y6c2q7pEsd^$+v1^Z1aV1qZgz>3@5+En~tK9pQsu6>xE(XAm^9h8>%98-!7XtY4 z&nQtD$wx(o0=|!hf-C8Wk26)z66GhNVCVB0MM6d^iQ*Fyx%g$Am$7I>MePz2`1e9! zKk$%Bfqr$f(j+IJRp10!9Oc(&B{}g8gp@xR7%xN#|1KT{-lu$th9tT93ivW3L$Wi( z^@+|Up+9JgpnX=BYjKj99{?Yf_>h~O>VeY0r`0G11oa1gn-uNwx-ANvhTh2i0vt`e z5cYy^8Hk?Pk(uJ-&FQAo22S7CqFaTD^`k?Ouo^#+g8RpXj-PZzkgm8@cHvaZcL7OUH?#fI5nS8Fr(+`GW#1_<4=OwTMeFZymPkOS87 z;I240v)kADu#c$sai24s-d=R?oq{0l~vYe4s*RMAt0y&%N{Se zZ0Hqvzu9D*mmJ&uS50%cNK;o+>C{y+N77VDQIeWSe5&ZG{!~B8C8~yK%?4gO(Mf)6 zl}y6c2q7pEsd^$+v1^Z1aV1qZgz>3@5+En~tK9pQsu6>xE(XAm^9h8>%98-!7XtY4 z&nQtD$wx(o0=|!hf-C8Wk26)z66GhNVCVB0MM6d^iQ*Fyx%g$Am$7I>MePz2`1e9! zKk$%Bfqr$f(j+IJRp10!9Oc(&B{}g8gp@xR7%xN#|1KT{-lu$th9tT93ivW3L$Wi( z^@+|Up+9JgpnX=BYjKj99{?Yf_>h~O>VeY0r`0G11oa1gn-uNwx-ANvhTh2i0vt`e z5cYy^8Hk?Pk(uJ-&FQAo22S7CqFaTD^`k?Ouo^#+g8RpXj-PZzkg_HU5-hY1m9MoB6)|W@b07%iA&qnx|%wYWR|-X@G2%0DzW21Je92K(hSG zFb}Y4aY@7RwbB5}CS{1ckmt9s6o$wf!?N^fi7lhgGor+Jb3i}zN8Y2~z8BsD4_%rC zN(w|z1wEfn3a2ygkPiaU_xb(G?0#qHhHG=XQ4n$wx#6I&M+w!6chaH`hv>O}Tkzr7 zje>r!gfS7vZ)tEeUF!xSqGYyMhIe_$d(kFA+Vv#%FDMF_%t4P&qLLVS9v1?bX5@{j z1$it01!JBnB}`)%?8@4y>#E(lR1FO5B`=r5sTjFhmIXs|dxqr_F7QtYB#?{85@K&n zLR{|!{C#_PZL=k+72hpToK1FPJ(H~^)#hFLndnmz$$dGqvb?dk$PH9uR@$bdoU^L+ zlC0|qz=2rxR4DeEWSX_ZK9XSD4cyU_HU5-hY1m9MoB6)|W@b07%iA&qnx|%wYWR|-X@G2%0DzW21Je92K(hSG zFb}Y4aY@7RwbB5}CS{1ckmt9s6o$wf!?N^fi7lhgGor+Jb3i}zN8Y2~z8BsD4_%rC zN(w|z1wEfn3a2ygkPiaU_xb(G?0#qHhHG=XQ4n$wx#6I&M+w!6chaH`hv>O}Tkzr7 zje>r!gfS7vZ)tEeUF!xSqGYyMhIe_$d(kFA+Vv#%FDMF_%t4P&qLLVS9v1?bX5@{j z1$it01!JBnB}`)%?8@4y>#E(lR1FO5B`=r5sTjFhmIXs|dxqr_F7QtYB#?{85@K&n zLR{|!{C#_PZL=k+72hpToK1FPJ(H~^)#hFLndnmz$$dGqvb?dk$PH9uR@$bdoU^L+ zlC0|qz=2rxR4DeEWSX_ZK9XSD4cyUY}j@U3_4*c&?uP9Y{jX&VWV0^YTq5CNq?lS2=P5<-*$rC96r^LiC~-)eX5 z1Wr*AQX?Vw0~Hbogj6Afka~xzJ(NoioVX$o8~`B>sF$kDynVa=NJ|yeTQXKWZ|CdH zn>RZ%&d$rrCs#^2RhlI|n1*Gu(PEog)$&-!rS)ptG~DW@X9v`iPY6kgOu?!}Jyv5u zwe4HJ4qy67Gq9O60xXaD6`(u-OR#1H^OdvZb4Lu$S8x*q2|g&Shjba1X5B-Md{aV1 zNP-mLyTP`s7VA=NnYCSy`aZ3%+jYv)z;itouv(|_J!q)6CH1hynAM_sMArkmP5r=V zyBa6u$3ube2^43KtC_V(CSOoTd}=bM?iZvZ(ww{+J?0OoNyl)QUq}Ewxf(t!y)UVg zCTnX$MwiB7lPl=piu7hhh*+#;IF`mdtIx-GT8k{<^iBlJsdpKDDxRDKT@3 zA4}ubz-#RcZM9xPg*DWzrTF3I)oYSE6NO733YQ2#I^s{?D^|0vIzD70UsN-3M0tot zE7QZL*W$zHC)9MqGupIj*LmPF`K+qcjld|#5XNkNOieYI*EWKJ0?(P~97+jqA0Q!8 zKxZ|ddR+q(OtagDMSabO9OzF0l#eqG!0e}pjhXCIxb2{#>ym_805u$LV zU8U`{zzAqxWt#-?zRKn@-^SEb9*xZ?BtN`6l%at3mAUN4Li!M3pLINw7U4baQYbVg z)w|{MK*t&i1*uCtsK8Y~KJ0@E!pCCFEOwt71A3#;cBvIfa*DkbbUa!NIV#A6n^ilW zeHfmajc0WoUrL__zTPl4sP0jhy=c_Ta%FxY`wz$OcTmD*Y-M;+illa9m=p1BvF8sOJ#IjI4YdW03~*ByvO!v z*T%4*ZQ*n;q*dW^)pqK13mZ#mPchAYBAQVSR0po-17L6`%_eOdQN&TYJb|*dOIx-> zBSsczb=p{{%r8=Nk@E8mUKR$=VL`D29S(zHF}I2hj|P}E-Ig&bh@waFt@eDMcn;p( zb=ygA;O+3YwQ#lY|^8t7sl z%&n(O2>3z`NN@u^ergQ_IBegzI4_?q5P?q%^7x-K4`b53>j(8YL7Tg*-#T*o79m^7 z85l@}#tRBbk?oz;UxE+6UQNCA-1gBCEG7sN!OM*VafuMKl>zo+A(x~sibbp^Ho79{ zqwe+JBG+6&4jOP zSy^02o&l=-V8>O7=}zR=Q`euFy7uI;zkd4c%WwW(oY*4E@fi{WXAs`>jwlbXYl1(H z?niKcM#Z?BAIyRKa}JCX;9#8FrwMSM=CXVL=KSA3ITTG4y}eDGdm9x;*Tp7LkyZHY zg+v?;61WiUA;Q?;@F#W%iFhT)Db9~$&BsyuVquvl4q0-aIMV#}3mB4#Lh+KPBIQuJk;`omEPtd>f55Fw{ diff --git a/.zcode/success_catalog_fixture/prepared/landing/lerobot_episode_0002.mcap b/.zcode/success_catalog_fixture/prepared/landing/lerobot_episode_0002.mcap deleted file mode 100644 index 42fd05df85b63f767d7e5381292431b09d4263c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4626 zcmeGgO>Y}j@U3_4*qb;BP9Y{jX&VWV0`|sEh=5X{$)N{C2_Z^>Qmpm*dA*8v-)cXc zz$q$1Y9s`Iph5zHkSc@_Qtwc;hjQtG6ITR+10ci!^-`6Yw{O=UX{myGOU8=l?R>nM zH*a=koSm1KPp-`6RcVHFVG5S5W}9u6Yo^CSmo{p)p}VzB&kCp~pAeD~nS@oFdaTZZ zn(dpu24C7qBe0k=1T2sFRiHcoOV~{h=BsBb=Z@%{ub>eG2|g&SMRXaKM#Dpnd{aV1 zNQxBUyUDi9HtW#xGP7Ne`aW%}TMf#xz;itouzJ}19yHY3l6u%=%xqIFrfY#^Q$Nsc zx6Da}u}C0#0>zo*YHlr-DHPQqpBl_*_(kc6G%K&hkA*{O+R+{67gIn_uSO3`?@Q{0 z!R+$Drc0yAmMiGsiu7hxh?uOcJ7$@AW{;2WxYXA+9Jc9n{dMa%CF#?Yd}?LsQfm4X zKUSIV240uX(02PJR9HjZdWIivUcDx%({Z@8fpCccWMlr+y<#=vs$&Ct6iR9?i6{@z zaCK_%^yTF6g>g08^mLontOgHUt}vo14L#6{GK6uYFsf#n%(L~NsK9eBK8G@gw-1mK zDWI{sPrZ(g30Agj-K2imhaBil0hCWNM?T1${h2vsOalNJJ`VxDQ|5&3uDp4kD-oh_ zrCp`%wLlMOPi30`@t(@>*|~R31&XQAlCXI+US+_LRAikA?Ihz&;B-gO=bu?ocQ+ z2Gu&1^FYTM3I(Y{J*dDnKt9+(1>s{UVU{{ijRL*dv|Vb(lAK~~1))bv5l01?$XG4( zti$lsYCfxJ_)_{b@U^DCK{b!M>_xq9RI2j}BmaO50%}s%@>v6&`x%}T_ye_#jKHM| z29~+h(*x$A>v1F#FENOIXc$n&xwaQlzOP+p4JgA-kVXMknX91d!cpN|4k(FxV_mjK zI~Ilo_ZCifLs}Cq*DR+&x3IBP_7u~RPee1ygKFRPd;kpYq|u_b9!H#87)M#lrESZh zF=GU14cc6&&M#79k@EBPUls<>VL>T`4u?T8nOnn#M+3~7PFo)qMA4(TsXgB(o`ZLH zO>|x!am>x>F6S=w79(sxJ@<6~k^dA9-5ybF;=*F@zWjYijN^?Nt0>_@aY^Q)SR{JF&=o-+ zcCY^yyXFcSckRjvnOs%UZ$KjyDrzSC?T^1-%@ihkgW1y*r*OiOa^vQb)Vi=uFU%;d z>f%EB3{d34NTz_Wr+LOor`suSTzxjJ{e2Xk6XGry(L3q3;r0o zAH)3_6_aj$FbD3>IWS3pgK=)3Ccu4~%kKS~^MC*3P|Bj{?QP=R+o(9aE{w!QR^hW3 z5^*$0;X<^B7-NIOpV$$kl9imSlw_5CU5LfXG5!HZ5`c3NBG{xQGKYg0fI?motL`=R zch|V{-qnC3HuyZ+ksC;n2^?wwWKtEYK?8A*jQeLk?&t5}KmqVYg!qPcnyvFV>>xaj z0|dbB2n|fR`J;k-4CfhuOCqAXromC1DiP$jh0h;}%St%PA2==od>0{-aE8CaaTcH! up)Mzz=i>$;@ktXM^@iXGm99=c=A0L-6y6&`?Aois* dict: - info = { - "fps": 30, - "data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet", - "video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4", - "features": { - "action": {"dtype": "float32", "shape": [1]}, - "observation.state": {"dtype": "float32", "shape": [1]}, - "observation.images.up": {"dtype": "video", "shape": [480, 640, 3]}, - "observation.images.side": {"dtype": "video", "shape": [480, 640, 3]}, - "timestamp": {"dtype": "float32", "shape": [1]}, - "next.success": {"dtype": "bool", "shape": [1]}, - }, - "robot_type": "so101", - } - (root / "meta").mkdir(parents=True, exist_ok=True) - (root / "meta" / "info.json").write_text(json.dumps(info)) - - conn = duckdb.connect() - ep_cols = [ - "episode_index", - "length", - "data/chunk_index", - "data/file_index", - "dataset_from_index", - "dataset_to_index", - "videos/observation.images.up/chunk_index", - "videos/observation.images.up/file_index", - "videos/observation.images.up/from_timestamp", - "videos/observation.images.up/to_timestamp", - "videos/observation.images.side/chunk_index", - "videos/observation.images.side/file_index", - "videos/observation.images.side/from_timestamp", - "videos/observation.images.side/to_timestamp", - "tasks", - "stats/next.success/min", - "stats/next.success/max", - ] - rows = [ - [ - i, - 1, - "000", - "000", - i, - i + 1, - "000", - "000", - 0.0, - 0.0, - "000", - "000", - 0.0, - 0.0, - [f"task-{i}"], - [False], - [False], - ] - for i in range(EPISODES) - ] - ep_path = root / "meta" / "episodes" / "chunk-000" / "file-000.parquet" - ep_path.parent.mkdir(parents=True, exist_ok=True) - vals = ",".join( - "(" - + ",".join( - "[" + ",".join(f"'{x}'" for x in v) + "]" - if isinstance(v, list) - else f"'{v}'" - if isinstance(v, str) - else str(v) - for v in row - ) - + ")" - for row in rows - ) - conn.execute( - f"COPY (SELECT * FROM (VALUES {vals}) AS t({','.join(chr(34) + c + chr(34) for c in ep_cols)})) " - f"TO '{str(ep_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" - ) - - data_rows = [[i, i, 0, 0.0, [0.0], [0.5], False] for i in range(EPISODES)] - data_path = root / "data" / "chunk-000" / "file-000.parquet" - data_path.parent.mkdir(parents=True, exist_ok=True) - dvals = ",".join( - "(" - + ",".join( - str(v) - if not isinstance(v, (str, list)) - else ( - "'" + str(v) + "'" - if isinstance(v, str) - else "[" + ",".join(str(x) for x in v) + "]" - ) - for v in row - ) - + ")" - for row in data_rows - ) - conn.execute( - f"COPY (SELECT * FROM (VALUES {dvals}) AS t(index, episode_index, frame_index, " - 'timestamp, "observation.state", action, "next.success")) ' - f"TO '{str(data_path).replace(chr(39), chr(39) * 2)}' (FORMAT parquet)" - ) - conn.close() - return {"info": info} - - -def main() -> None: - shutil.rmtree(ROOT, ignore_errors=True) - corpus = build_corpus(ROOT) - output_dir = ROOT / "prepared" - - stubs = { - "_hf_repo_info": lambda repo, revision: {"sha": "abc1234", "license": "apache-2.0"}, - "_fetch_info_json": lambda repo, rev, cache: corpus["info"], - "_hf_tree": lambda repo, rev, path: ( - [{"path": "meta/episodes/chunk-000/file-000.parquet", "type": "file"}] - if "episodes" in path - else [{"path": "meta/info.json", "type": "file"}] - ), - "_transcode_mp4_to_h264": lambda mp4_path, gop, fps: [ - b"\x00\x00\x00\x01\x09\x10\x00\x00\x00\x01\x67\x42\x00" - b"\x00\x00\x00\x01\x68\x88\x80\x00\x00\x00\x01\x65\x88" - ], - "_get_video_pts_times": lambda path: [0], - "ffmpeg_version": lambda: "fixture-ffmpeg", - } - originals = {name: getattr(prep, name) for name in stubs} - - def fake_download(url: str, dest: Path, **kw: object) -> None: - dest.parent.mkdir(parents=True, exist_ok=True) - if "meta/episodes" in url: - import shutil - - shutil.copy(ROOT / "meta" / "episodes" / "chunk-000" / "file-000.parquet", dest) - elif url.endswith("info.json"): - import shutil - - shutil.copy(ROOT / "meta" / "info.json", dest) - else: - import shutil - - shutil.copy(ROOT / "data" / "chunk-000" / "file-000.parquet", dest) - - stubs["_download_file"] = fake_download - try: - for name, fake in stubs.items(): - setattr(prep, name, fake) - prep.import_lerobot_dataset( - dataset_repo="fake/repo", - output_dir=output_dir, - camera_keys=("observation.images.up", "observation.images.side"), - ) - finally: - for name, original in originals.items(): - setattr(prep, name, original) - - print("--- what the source declares ---") - conn = duckdb.connect() - src = conn.execute( - 'SELECT episode_index, "next.success" FROM read_parquet(' - f"'{ROOT / 'data' / 'chunk-000' / 'file-000.parquet'}') ORDER BY index" - ).fetchall() - for r in src: - print(f" data parquet frame (episode {r[0]}): next.success = {r[1]}") - agg = conn.execute( - 'SELECT episode_index, "stats/next.success/max" FROM read_parquet(' - f"'{ROOT / 'meta' / 'episodes' / 'chunk-000' / 'file-000.parquet'}') ORDER BY episode_index" - ).fetchall() - for r in agg: - print(f" episodes parquet aggregate (episode {r[0]}): stats/next.success/max = {r[1]}") - conn.close() - - print("--- what hflow shipped ---") - for episode_path in sorted((output_dir / "landing").glob("*.mcap")): - with episode_path.open("rb") as stream: - for metadata in make_reader(stream).iter_metadata(): - if metadata.name == "episode/v1": - print(f" {episode_path.name}: episode/v1 = {json.dumps(metadata.metadata)}") - break - - print("--- consequence ---") - print(" source collector label: next.success = False (this demo never succeeded)") - print(' shipped episode/v1: "success": "true"') - print(" catalog promotion: success VARCHAR first-class column (catalog.py:59,68)") - print(" buyer filter success=true: matches this failed demo") - - -if __name__ == "__main__": - main() diff --git a/.zcode/success_fixture/meta/episodes/chunk-000/file-000.parquet b/.zcode/success_fixture/meta/episodes/chunk-000/file-000.parquet deleted file mode 100644 index bdd4598590cf8fe52b060c7d103318637c0649e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2956 zcmd5;?`ji45Z_)d*R-^1qm8@cHvaZcL7OUH?#fI5nS8Fr(+`GW#1_<4=OwTMeFZymPkOS87 z;I240v)kADu#c$sai24s-d=R?oq{0l~vYe4s*RMAt0y&%N{Se zZ0Hqvzu9D*mmJ&uS50%cNK;o+>C{y+N77VDQIeWSe5&ZG{!~B8C8~yK%?4gO(Mf)6 zl}y6c2q7pEsd^$+v1^Z1aV1qZgz>3@5+En~tK9pQsu6>xE(XAm^9h8>%98-!7XtY4 z&nQtD$wx(o0=|!hf-C8Wk26)z66GhNVCVB0MM6d^iQ*Fyx%g$Am$7I>MePz2`1e9! zKk$%Bfqr$f(j+IJRp10!9Oc(&B{}g8gp@xR7%xN#|1KT{-lu$th9tT93ivW3L$Wi( z^@+|Up+9JgpnX=BYjKj99{?Yf_>h~O>VeY0r`0G11oa1gn-uNwx-ANvhTh2i0vt`e z5cYy^8Hk?Pk(uJ-&FQAo22S7CqFaTD^`k?Ouo^#+g8RpXj-PZzkgm8@cHvaZcL7OUH?#fI5nS8Fr(+`GW#1_<4=OwTMeFZymPkOS87 z;I240v)kADu#c$sai24s-d=R?oq{0l~vYe4s*RMAt0y&%N{Se zZ0Hqvzu9D*mmJ&uS50%cNK;o+>C{y+N77VDQIeWSe5&ZG{!~B8C8~yK%?4gO(Mf)6 zl}y6c2q7pEsd^$+v1^Z1aV1qZgz>3@5+En~tK9pQsu6>xE(XAm^9h8>%98-!7XtY4 z&nQtD$wx(o0=|!hf-C8Wk26)z66GhNVCVB0MM6d^iQ*Fyx%g$Am$7I>MePz2`1e9! zKk$%Bfqr$f(j+IJRp10!9Oc(&B{}g8gp@xR7%xN#|1KT{-lu$th9tT93ivW3L$Wi( z^@+|Up+9JgpnX=BYjKj99{?Yf_>h~O>VeY0r`0G11oa1gn-uNwx-ANvhTh2i0vt`e z5cYy^8Hk?Pk(uJ-&FQAo22S7CqFaTD^`k?Ouo^#+g8RpXj-PZzkg_HU5-hY1m9MoB6)|W@b07%iA&qnx|%wYWR|-X@G2%0DzW21Je92K(hSG zFb}Y4aY@7RwbB5}CS{1ckmt9s6o$wf!?N^fi7lhgGor+Jb3i}zN8Y2~z8BsD4_%rC zN(w|z1wEfn3a2ygkPiaU_xb(G?0#qHhHG=XQ4n$wx#6I&M+w!6chaH`hv>O}Tkzr7 zje>r!gfS7vZ)tEeUF!xSqGYyMhIe_$d(kFA+Vv#%FDMF_%t4P&qLLVS9v1?bX5@{j z1$it01!JBnB}`)%?8@4y>#E(lR1FO5B`=r5sTjFhmIXs|dxqr_F7QtYB#?{85@K&n zLR{|!{C#_PZL=k+72hpToK1FPJ(H~^)#hFLndnmz$$dGqvb?dk$PH9uR@$bdoU^L+ zlC0|qz=2rxR4DeEWSX_ZK9XSD4cyU_HU5-hY1m9MoB6)|W@b07%iA&qnx|%wYWR|-X@G2%0DzW21Je92K(hSG zFb}Y4aY@7RwbB5}CS{1ckmt9s6o$wf!?N^fi7lhgGor+Jb3i}zN8Y2~z8BsD4_%rC zN(w|z1wEfn3a2ygkPiaU_xb(G?0#qHhHG=XQ4n$wx#6I&M+w!6chaH`hv>O}Tkzr7 zje>r!gfS7vZ)tEeUF!xSqGYyMhIe_$d(kFA+Vv#%FDMF_%t4P&qLLVS9v1?bX5@{j z1$it01!JBnB}`)%?8@4y>#E(lR1FO5B`=r5sTjFhmIXs|dxqr_F7QtYB#?{85@K&n zLR{|!{C#_PZL=k+72hpToK1FPJ(H~^)#hFLndnmz$$dGqvb?dk$PH9uR@$bdoU^L+ zlC0|qz=2rxR4DeEWSX_ZK9XSD4cyUY}j@U3_4*c&?uP9Y{jX&VWV0^YTq5CNq?lS2=P5<-*$rC96r^LiC~-)eX5 z1Wr*AQX?Vw0~Hbogj6Afka~xzJ(NoioVX$o8~`B>sF$kDynVa=NJ|yeTQXKWZ|CdH zn>RZ%&d$rrCs#^2RhlI|n1*Gu(PEog)$&-!rS)ptG~DW@X9v`iPY6kgOu?!}Jyv5u zwe4HJ4qy67Gq9O60xXaD6`(u-OR#1H^OdvZb4Lu$S8x*q2|g&Shjba1X5B-Md{aV1 zNP-mLyTP`s7VA=NnYCSy`aZ3%+jYv)z;itouv(|_J!q)6CH1hynAM_sMArkmP5r=V zyBa6u$3ube2^43KtC_V(CSOoTd}=bM?iZvZ(ww{+J?0OoNyl)QUq}Ewxf(t!y)UVg zCTnX$MwiB7lPl=piu7hhh*+#;IF`mdtIx-GT8k{<^iBlJsdpKDDxRDKT@3 zA4}ubz-#RcZM9xPg*DWzrTF3I)oYSE6NO733YQ2#I^s{?D^|0vIzD70UsN-3M0tot zE7QZL*W$zHC)9MqGupIj*LmPF`K+qcjld|#5XNkNOieYI*EWKJ0?(P~97+jqA0Q!8 zKxZ|ddR+q(OtagDMSabO9OzF0l#eqG!0e}pjhXCIxb2{#>ym_805u$LV zU8U`{zzAqxWt#-?zRKn@-^SEb9*xZ?BtN`6l%at3mAUN4Li!M3pLINw7U4baQYbVg z)w|{MK*t&i1*uCtsK8Y~KJ0@E!pCCFEOwt71A3#;cBvIfa*DkbbUa!NIV#A6n^ilW zeHfmajc0WoUrL__zTPl4sP0jhy=c_Ta%FxY`wz$OcTmD*Y-M;+illa9m=p1BvF8sOJ#IjI4YdW03~*ByvO!v z*T%4*ZQ*n;q*dW^)pqK13mZ#mPchAYBAQVSR0po-17L6`%_eOdQN&TYJb|*dOIx-> zBSsczb=p{{%r8=Nk@E8mUKR$=VL`D29S(zHF}I2hj|P}E-Ig&bh@waFt@eDMcn;p( zb=ygA;O+3YwQ#lY|^8t7sl z%&n(O2>3z`NN@u^ergQ_IBegzI4_?q5P?q%^7x-K4`b53>j(8YL7Tg*-#T*o79m^7 z85l@}#tRBbk?oz;UxE+6UQNCA-1gBCEG7sN!OM*VafuMKl>zo+A(x~sibbp^Ho79{ zqwe+JBG+6&4jOP zSy^02o&l=-V8>O7=}zR=Q`euFy7uI;zkd4c%WwW(oY*4E@fi{WXAs`>jwlbXYl1(H z?niKcM#Z?BAIyRKa}JCX;9#8FrwMSM=CXVL=KSA3ITTG4y}eDGdm9x;*Tp7LkyZHY zg+v?;61WiUA;Q?;@F#W%iFhT)Db9~$&BsyuVquvl4q0-aIMV#}3mB4#Lh+KPBIQuJk;`omEPtd>f55Fw{ diff --git a/.zcode/success_fixture/prepared/landing/lerobot_episode_0002.mcap b/.zcode/success_fixture/prepared/landing/lerobot_episode_0002.mcap deleted file mode 100644 index 42fd05df85b63f767d7e5381292431b09d4263c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4626 zcmeGgO>Y}j@U3_4*qb;BP9Y{jX&VWV0`|sEh=5X{$)N{C2_Z^>Qmpm*dA*8v-)cXc zz$q$1Y9s`Iph5zHkSc@_Qtwc;hjQtG6ITR+10ci!^-`6Yw{O=UX{myGOU8=l?R>nM zH*a=koSm1KPp-`6RcVHFVG5S5W}9u6Yo^CSmo{p)p}VzB&kCp~pAeD~nS@oFdaTZZ zn(dpu24C7qBe0k=1T2sFRiHcoOV~{h=BsBb=Z@%{ub>eG2|g&SMRXaKM#Dpnd{aV1 zNQxBUyUDi9HtW#xGP7Ne`aW%}TMf#xz;itouzJ}19yHY3l6u%=%xqIFrfY#^Q$Nsc zx6Da}u}C0#0>zo*YHlr-DHPQqpBl_*_(kc6G%K&hkA*{O+R+{67gIn_uSO3`?@Q{0 z!R+$Drc0yAmMiGsiu7hxh?uOcJ7$@AW{;2WxYXA+9Jc9n{dMa%CF#?Yd}?LsQfm4X zKUSIV240uX(02PJR9HjZdWIivUcDx%({Z@8fpCccWMlr+y<#=vs$&Ct6iR9?i6{@z zaCK_%^yTF6g>g08^mLontOgHUt}vo14L#6{GK6uYFsf#n%(L~NsK9eBK8G@gw-1mK zDWI{sPrZ(g30Agj-K2imhaBil0hCWNM?T1${h2vsOalNJJ`VxDQ|5&3uDp4kD-oh_ zrCp`%wLlMOPi30`@t(@>*|~R31&XQAlCXI+US+_LRAikA?Ihz&;B-gO=bu?ocQ+ z2Gu&1^FYTM3I(Y{J*dDnKt9+(1>s{UVU{{ijRL*dv|Vb(lAK~~1))bv5l01?$XG4( zti$lsYCfxJ_)_{b@U^DCK{b!M>_xq9RI2j}BmaO50%}s%@>v6&`x%}T_ye_#jKHM| z29~+h(*x$A>v1F#FENOIXc$n&xwaQlzOP+p4JgA-kVXMknX91d!cpN|4k(FxV_mjK zI~Ilo_ZCifLs}Cq*DR+&x3IBP_7u~RPee1ygKFRPd;kpYq|u_b9!H#87)M#lrESZh zF=GU14cc6&&M#79k@EBPUls<>VL>T`4u?T8nOnn#M+3~7PFo)qMA4(TsXgB(o`ZLH zO>|x!am>x>F6S=w79(sxJ@<6~k^dA9-5ybF;=*F@zWjYijN^?Nt0>_@aY^Q)SR{JF&=o-+ zcCY^yyXFcSckRjvnOs%UZ$KjyDrzSC?T^1-%@ihkgW1y*r*OiOa^vQb)Vi=uFU%;d z>f%EB3{d34NTz_Wr+LOor`suSTzxjJ{e2Xk6XGry(L3q3;r0o zAH)3_6_aj$FbD3>IWS3pgK=)3Ccu4~%kKS~^MC*3P|Bj{?QP=R+o(9aE{w!QR^hW3 z5^*$0;X<^B7-NIOpV$$kl9imSlw_5CU5LfXG5!HZ5`c3NBG{xQGKYg0fI?motL`=R zch|V{-qnC3HuyZ+ksC;n2^?wwWKtEYK?8A*jQeLk?&t5}KmqVYg!qPcnyvFV>>xaj z0|dbB2n|fR`J;k-4CfhuOCqAXromC1DiP$jh0h;}%St%PA2==od>0{-aE8CaaTcH! up)Mzz=i>$;@ktXM^@iXGm99=c=A0L-6y6&`?Aois* 2 row(s): - episode_id=ba19d7fb94cdba01 task='task-1' success='true' - episode_id=e9e1a9ad1306d39d task='task-0' success='true' - (catalog holds 2 episode(s) in total) -``` - -A demo whose collector label reads never-succeeded is delivered as `success: "true"` and returned by the exact filter a buyer would run. On `lerobot/pusht` at scale, the same disagreement holds for all 206 episodes: hflow's column says `true` where the source says `False`, 100 percent of the time. - -## Why now - -Third brick of the thread #376 and #379 opened: #376 removed a stamp asserting a measurement nobody took, #389 gave the delivery a checkable receipt. This is the last record in the family that speaks without evidence, and unlike the first two it speaks about content: which demos worked. That is the column a data buyer filters on. - -## What to build - -Open call, not assumed: - -1. **Read the label**: when the source carries an outcome feature (`next.success` or equivalent), derive the episode outcome and stamp it. Whether episode success is any-frame or last-frame is a methodology choice to name. -2. **Stamp unlabeled**: when the source carries no outcome feature, omit the key. FORMAT.md:119 already says all keys are optional; omitting is the only encoding that cannot be mistaken for a collector's judgment. -3. **Column semantics**: how the catalog column represents unlabeled episodes (NULL versus a value) is the catalog's decision to make once, deliberately. - -Combinations are plausible: read when present, omit when absent, document both. - -## Definition of done - -1. The importer never invents a collector label: the episode record is copied from the source or omits the key. -2. When the source declares an outcome feature, the delivered record reports it, with the derivation named. -3. The catalog `success` column carries real information or NULL, pinned by tests through the real import and catalog path. -4. The transform's copy path is untouched; non-LeRobot imports are unaffected. -5. The fixture above lands as a test. - -## Non-goals - -- Changing the transform's episode/v1 copy path -- Judging whether pusht's upstream conversion should have populated `next.success`; that is upstream's question -- Changing catalog dedupe or the promoted-keys mechanism - -## Validation - -```bash -uv run ruff check --fix -uv run ruff format -uv run ty check -uv run pytest -q tests/test_lerobot_converter.py tests/test_catalog.py -uv run pytest -q -``` diff --git a/src/hflow/importers/lerobot.py b/src/hflow/importers/lerobot.py index 7245f2d5..e2c82e6f 100644 --- a/src/hflow/importers/lerobot.py +++ b/src/hflow/importers/lerobot.py @@ -57,6 +57,10 @@ CONVERTER_VERSION = "lerobot-converter-v7" # Canonical transform knobs that affect published bytes for this importer. IMPORT_GOP_SECONDS = 1.0 +# The v3 per-episode aggregate of the collector's frame-level next.success +# label. Optional: a corpus that declares no outcome feature has no such column. +_OUTCOME_AGGREGATE_COLUMN = "stats/next.success/max" +_SUCCESS_DERIVATION = f"max({_OUTCOME_AGGREGATE_COLUMN.removesuffix('/max')})" PRESENTATION_TIMESTAMP_EPSILON_S = 0.050 EPISODE_METADATA_TREE_PREFIX = PurePosixPath("meta/episodes") @@ -597,7 +601,13 @@ def _ensure_source_archive(dataset_source: DatasetSource, cache_dir: Path) -> _S f"SELECT * FROM {episode_metadata_relation} LIMIT 1" ).description ] - has_outcome_aggregate = "stats/next.success/max" in episodes_columns + # Built outside the f-string below: a quoted identifier cannot be + # nested in a same-quoted f-string on Python 3.11, which this repo + # still supports. + has_outcome_aggregate = _OUTCOME_AGGREGATE_COLUMN in episodes_columns + outcome_aggregate_selector = ( + f', "{_OUTCOME_AGGREGATE_COLUMN}"' if has_outcome_aggregate else "" + ) episode_rows: list[_EpisodeRow] = [] parquet_episode_rows = connection.execute( @@ -605,7 +615,7 @@ def _ensure_source_archive(dataset_source: DatasetSource, cache_dir: Path) -> _S SELECT "episode_index", "tasks", "length", "data/chunk_index", "data/file_index", "dataset_from_index", "dataset_to_index" - {"," + chr(34) + "stats/next.success/max" + chr(34) if has_outcome_aggregate else ""} + {outcome_aggregate_selector} FROM {episode_metadata_relation} ORDER BY "episode_index" """ @@ -1237,7 +1247,7 @@ def _feature_rows(feature_name: str) -> list | None: success_outcome = episode_row.get("success_outcome") if success_outcome is not None: episode_record["success"] = "true" if success_outcome else "false" - episode_record["success_derivation"] = "max(stats/next.success)" + episode_record["success_derivation"] = _SUCCESS_DERIVATION mcap_writer.add_metadata( name="episode/v1", data=episode_record, diff --git a/tests/test_lerobot_converter.py b/tests/test_lerobot_converter.py index c890ece3..5b145c6d 100755 --- a/tests/test_lerobot_converter.py +++ b/tests/test_lerobot_converter.py @@ -1872,7 +1872,10 @@ def should_not_convert(**_kwargs: object) -> prep._PublishedEpisode: def _build_success_label_corpus(root: Path, outcome_mode: str) -> dict: - """One two-frame episode. outcome_mode: 'transition', 'all-false', 'none'.""" + """One two-frame episode. + + outcome_mode: 'transition', 'all-false', 'empty-aggregate', or 'none'. + """ has_outcome = outcome_mode != "none" info = { "fps": 30, @@ -1909,9 +1912,14 @@ def _build_success_label_corpus(root: Path, outcome_mode: str) -> dict: ] row: list[object] = [0, 2, "000", "000", 0, 2, "000", "000", 0.0, 0.0, ["push the block"]] if has_outcome: - stats_min, stats_max = ( - ([False], [True]) if outcome_mode == "transition" else ([False], [False]) - ) + if outcome_mode == "transition": + stats_min, stats_max = [False], [True] + elif outcome_mode == "empty-aggregate": + # The column exists but carries no value for this episode, which + # is a declared feature with nothing recorded rather than a label. + stats_min, stats_max = [], [] + else: + stats_min, stats_max = [False], [False] ep_cols += ["stats/next.success/min", "stats/next.success/max"] row += [stats_min, stats_max] ep_path = root / "meta" / "episodes" / "chunk-000" / "file-000.parquet" @@ -2043,6 +2051,26 @@ def test_success_label_reports_false_when_source_is_all_false( assert record["success_derivation"] == "max(stats/next.success)" +def test_success_label_omitted_when_the_outcome_aggregate_is_empty( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A declared outcome feature with nothing recorded is not a label. + + Dropping the length check stamps ``success: "false"`` here, because + ``any([])`` is False. That is the same invention the hardcoded ``"true"`` + was, one value over, so the empty aggregate needs its own case rather than + riding on the no-feature one. + """ + from hflow.episode import Episode + + output_dir = _import_success_label_corpus(tmp_path, monkeypatch, "empty-aggregate") + landing = sorted((output_dir / "landing").glob("*.mcap")) + with Episode(landing[0]) as episode: + record = episode.metadata_records["episode/v1"] + assert "success" not in record + assert "success_derivation" not in record + + def test_success_label_omitted_when_source_has_no_outcome_feature( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: