From 6f906d1d8aceb2ba9c0b63b87f3cb827ad4acd7d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 04:30:23 +0000 Subject: [PATCH 01/13] Initial plan From 26674abd2abf58371c1f17d0590e0b8bbabad2e3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 05:22:12 +0000 Subject: [PATCH 02/13] Cherry-pick platform-pinned Lambda build tooling to fix x86_64 cryptography Co-authored-by: sgbaird <45469701+sgbaird@users.noreply.github.com> --- .github/workflows/build-deployment-zip.yaml | 68 ++++++++++++++++++ .gitignore | 4 ++ build-deployment-zip.sh | 55 +++++++++++++++ lambda_function.py | 77 +++++++++++++++++++++ 4 files changed, 204 insertions(+) create mode 100644 .github/workflows/build-deployment-zip.yaml create mode 100755 build-deployment-zip.sh create mode 100644 lambda_function.py diff --git a/.github/workflows/build-deployment-zip.yaml b/.github/workflows/build-deployment-zip.yaml new file mode 100644 index 0000000..27229e1 --- /dev/null +++ b/.github/workflows/build-deployment-zip.yaml @@ -0,0 +1,68 @@ +name: Build Lambda Deployment Package + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: write # Required to upload assets to releases + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Create dependencies directory + run: mkdir -p dependencies + + - name: Install dependencies + # Pin the target platform to AWS Lambda's runtime (x86_64/manylinux, + # CPython 3.11) so binary wheels such as cryptography's _rust.abi3.so + # match Lambda regardless of the build host's architecture. Keep + # --python-version in sync with the function's configured runtime. + run: | + pip install \ + --platform manylinux2014_x86_64 \ + --implementation cp \ + --python-version 3.11 \ + --only-binary=:all: \ + --target ./dependencies \ + boto3 \ + google-api-python-client \ + google-auth \ + google-auth-oauthlib \ + google-auth-httplib2 + + - name: Copy lambda function and chalicelib + run: | + cp lambda_function.py dependencies/ + cp -r chalicelib dependencies/ + + - name: Create deployment package + run: | + cd dependencies + zip -r ../deployment.zip . + cd .. + + - name: Upload deployment package as artifact + uses: actions/upload-artifact@v4 + with: + name: lambda-deployment-package + path: deployment.zip + retention-days: 90 + + - name: Upload deployment package to release + if: github.event_name == 'release' + uses: softprops/action-gh-release@v1 + with: + files: deployment.zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 4e2df48..9b71fb7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ __pycache__/ *.bak policy.json + +# Deployment artifacts +dependencies/ +deployment.zip diff --git a/build-deployment-zip.sh b/build-deployment-zip.sh new file mode 100755 index 0000000..11a430d --- /dev/null +++ b/build-deployment-zip.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Build script for creating AWS Lambda deployment package +# This script creates a deployment.zip file that can be uploaded directly to AWS Lambda + +set -e + +echo "Building AWS Lambda deployment package..." + +# Clean up any existing artifacts +echo "Cleaning up existing artifacts..." +rm -rf dependencies deployment.zip + +# Create dependencies directory +echo "Creating dependencies directory..." +mkdir -p dependencies + +# Install Python dependencies +# NOTE: AWS Lambda runs on x86_64/manylinux with CPython. Pin the target +# platform so binary wheels (e.g. cryptography's compiled _rust.abi3.so, a +# transitive dependency of google-auth) match the Lambda runtime no matter +# where this script runs. Building unpinned on an ARM host (such as the +# Raspberry Pi) produces ARM binaries that fail to load on Lambda with +# Runtime.ImportModuleError (502). Keep --python-version in sync with the +# function's configured runtime (Python 3.11). +echo "Installing Python dependencies..." +pip install \ + --platform manylinux2014_x86_64 \ + --implementation cp \ + --python-version 3.11 \ + --only-binary=:all: \ + --target ./dependencies \ + boto3 \ + google-api-python-client \ + google-auth \ + google-auth-oauthlib \ + google-auth-httplib2 + +# Copy lambda function and chalicelib +echo "Copying lambda function and chalicelib..." +cp lambda_function.py dependencies/ +cp -r chalicelib dependencies/ + +# Create deployment package +echo "Creating deployment.zip..." +cd dependencies +zip -q -r ../deployment.zip . +cd .. + +# Get the size of the deployment package +SIZE=$(du -h deployment.zip | cut -f1) +echo "✓ deployment.zip created successfully (${SIZE})" +echo "" +echo "You can now upload deployment.zip to AWS Lambda!" +echo "See README.md for detailed deployment instructions." diff --git a/lambda_function.py b/lambda_function.py new file mode 100644 index 0000000..b624119 --- /dev/null +++ b/lambda_function.py @@ -0,0 +1,77 @@ +import json +import logging +from chalicelib.ytb_api_utils import ( + init_youtube_service, + create_broadcast_and_bind_stream, + end_active_broadcasts_for_device +) + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +def lambda_handler(event, context): + """ + AWS Lambda handler function for YouTube streaming management. + + This function can be directly deployed to AWS Lambda without Chalice. + + Expected event payload: + { + "body": { + "action": "create" or "end", + "cam_name": "camera name", + "workflow_name": "workflow identifier", + "privacy_status": "public", "private", or "unlisted" (optional, default: "private") + } + } + """ + logger.info("Lambda handler invoked") + try: + body = event.get("body") + if isinstance(body, str): + payload = json.loads(body) + elif isinstance(body, dict): + payload = body + else: + raise ValueError("Invalid body format") + + logger.info(f"Received payload: {payload}") + + action = payload.get("action") + cam_name = payload.get("cam_name", "UnknownCam") + workflow_name = payload.get("workflow_name", "UnknownWorkflow") + privacy_status = payload.get("privacy_status", "private") + + if action not in ("create", "end"): + return { + "statusCode": 400, + "body": json.dumps({"error": "Invalid or missing 'action'. Must be 'create' or 'end'."}) + } + + init_youtube_service() + + if action == "create": + result = create_broadcast_and_bind_stream(cam_name, workflow_name, privacy_status) + return { + "statusCode": 200, + "body": json.dumps({"status": "created", "result": result}) + } + else: # action == "end" + end_active_broadcasts_for_device(workflow_name) + return { + "statusCode": 200, + "body": json.dumps({"status": "ended", "message": f"{workflow_name} ended successfully"}) + } + + except ValueError as ve: + logger.exception("Invalid input") + return { + "statusCode": 400, + "body": json.dumps({"error": f"Invalid input: {str(ve)}"}) + } + except Exception as e: + logger.exception("Error in lambda_handler") + return { + "statusCode": 500, + "body": json.dumps({"error": f"Internal server error: {str(e)}"}) + } From 87a2fd3937723e97704ebe3e388ef3d46cc19934 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:51:41 +0000 Subject: [PATCH 03/13] Read YouTube token S3 bucket/key from env; pin build to Python 3.12 Co-authored-by: sgbaird <45469701+sgbaird@users.noreply.github.com> --- .github/workflows/build-deployment-zip.yaml | 6 +++--- build-deployment-zip.sh | 4 ++-- chalicelib/ytb_api_utils.py | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-deployment-zip.yaml b/.github/workflows/build-deployment-zip.yaml index 27229e1..42dec0a 100644 --- a/.github/workflows/build-deployment-zip.yaml +++ b/.github/workflows/build-deployment-zip.yaml @@ -18,21 +18,21 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Create dependencies directory run: mkdir -p dependencies - name: Install dependencies # Pin the target platform to AWS Lambda's runtime (x86_64/manylinux, - # CPython 3.11) so binary wheels such as cryptography's _rust.abi3.so + # CPython 3.12) so binary wheels such as cryptography's _rust.abi3.so # match Lambda regardless of the build host's architecture. Keep # --python-version in sync with the function's configured runtime. run: | pip install \ --platform manylinux2014_x86_64 \ --implementation cp \ - --python-version 3.11 \ + --python-version 3.12 \ --only-binary=:all: \ --target ./dependencies \ boto3 \ diff --git a/build-deployment-zip.sh b/build-deployment-zip.sh index 11a430d..0b9b08c 100755 --- a/build-deployment-zip.sh +++ b/build-deployment-zip.sh @@ -22,12 +22,12 @@ mkdir -p dependencies # where this script runs. Building unpinned on an ARM host (such as the # Raspberry Pi) produces ARM binaries that fail to load on Lambda with # Runtime.ImportModuleError (502). Keep --python-version in sync with the -# function's configured runtime (Python 3.11). +# function's configured runtime (Python 3.12). echo "Installing Python dependencies..." pip install \ --platform manylinux2014_x86_64 \ --implementation cp \ - --python-version 3.11 \ + --python-version 3.12 \ --only-binary=:all: \ --target ./dependencies \ boto3 \ diff --git a/chalicelib/ytb_api_utils.py b/chalicelib/ytb_api_utils.py index 7808bc2..b56e4f3 100644 --- a/chalicelib/ytb_api_utils.py +++ b/chalicelib/ytb_api_utils.py @@ -1,3 +1,4 @@ +import os import boto3 import pickle import time @@ -9,8 +10,8 @@ CHANNEL_ID = "UCHBzCfYpGwoqygH9YNh9A6g" YOUTUBE = None -S3_BUCKET = "ac-token-youtube-api" -S3_KEY = "token/token.pickle" +S3_BUCKET = os.environ.get("YOUTUBE_TOKEN_S3_BUCKET", "ac-token-youtube-api") +S3_KEY = os.environ.get("YOUTUBE_TOKEN_S3_KEY", "token/token.pickle") def init_youtube_service(): global YOUTUBE From 81255179b1e69ed7f85ddaf320efe8529ed3b50a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:01:40 +0000 Subject: [PATCH 04/13] Add tracking doc for suggested ac-training-lab picam changes Co-authored-by: sgbaird <45469701+sgbaird@users.noreply.github.com> --- docs/ac-training-lab-picam-suggestions.md | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docs/ac-training-lab-picam-suggestions.md diff --git a/docs/ac-training-lab-picam-suggestions.md b/docs/ac-training-lab-picam-suggestions.md new file mode 100644 index 0000000..4c90ea9 --- /dev/null +++ b/docs/ac-training-lab-picam-suggestions.md @@ -0,0 +1,58 @@ +# Suggested changes for the ac-training-lab picam device + +These are recommendations for the `src/ac_training_lab/picam/` directory in +[AccelerationConsortium/ac-training-lab](https://github.com/AccelerationConsortium/ac-training-lab), +collected while debugging the office cam / YouTube streaming pipeline. They are +tracked here so they can be turned into a proper PR **in that repo** later. None +of them are changes to this (streamingLambda) repository. + +Context: the Pi (`rpi-zero2w-stream-cam`) already runs the `device.py` from +ac-training-lab PR #539 (branch `copilot/sub-pr-538`) unchanged. Per-device +config (resolution, frame rate, flips, workflow name, privacy, etc.) lives in the +Pi's `my_secrets.py` and should stay there — do **not** hardcode it into `device.py`. + +## 1. Run exactly one systemd service (avoid the camera race) + +The single camera can only be held by one process. When two units +(`device.service` and a second `picam-stream.service`) both launch `device.py` +at boot, they race for the camera: one wins and streams, the loser's +`rpicam-vid` dies instantly, its `ffmpeg` then misdetects the empty `pipe:0` as +an `lrc` subtitle stream and thrashes in a restart loop (high CPU, repeated +`create`/`end` Lambda calls, throwaway YouTube broadcasts). + +Recommendation: document/ship a **single** canonical service and make it explicit +that only one unit may run `device.py`. + +## 2. Harden the documented `device.service` + +The service block currently in `README.md` is missing a few options that the +working Pi unit has and benefits from: + +- `Restart=always`, `RestartSec=20` (already documented, keep) +- `RuntimeMaxSec=8h` — periodically restart the whole pipeline so it recovers + from long-run drift / stale broadcasts. +- `KillSignal=SIGINT` + `TimeoutStopSec=45` — `device.py` catches + `KeyboardInterrupt` and cleanly terminates `rpicam-vid` and `ffmpeg`. Without + SIGINT, `systemctl stop/restart` sends SIGTERM and can leave orphaned camera + processes holding the device. +- `Environment=PYTHONUNBUFFERED=1` — so `journalctl` shows logs live. + +Also, `StartLimitInterval` / `StartLimitBurst` are shown under `[Service]` in the +README but belong in the `[Unit]` section (they are ignored under `[Service]` in +current systemd). + +## 3. Optional: make the ffmpeg input format explicit in `device.py` + +`start_stream()` reads video with `-i pipe:0` and lets ffmpeg auto-probe the +format. Passing `-f h264` immediately before `-i pipe:0` tells ffmpeg the pipe is +raw H.264, which prevents the `lrc`/subtitle misdetection seen at startup when the +camera briefly produces no data. This is a small defensive change; the root cause +of the misdetection is the two-service race in (1). + +## 4. Note: transient playlist-add 409 comes from the Lambda, not `device.py` + +On `create`, YouTube occasionally returns `HttpError 409 SERVICE_UNAVAILABLE` +when the freshly-created broadcast is added to its playlist. The broadcast is +created and streams fine; only the playlist-add fails. This retry/backoff belongs +in this repo's `chalicelib/ytb_api_utils.py` (`create_broadcast_and_bind_stream`), +not in ac-training-lab. From 2062337b826ae0c6f2133bf3cf176fda3dfb26bb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:25:17 +0000 Subject: [PATCH 05/13] Revert broadcast-reuse; document intentional 8h chunking restart Co-authored-by: sgbaird <45469701+sgbaird@users.noreply.github.com> --- docs/ac-training-lab-picam-suggestions.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/ac-training-lab-picam-suggestions.md b/docs/ac-training-lab-picam-suggestions.md index 4c90ea9..1d7684f 100644 --- a/docs/ac-training-lab-picam-suggestions.md +++ b/docs/ac-training-lab-picam-suggestions.md @@ -29,8 +29,11 @@ The service block currently in `README.md` is missing a few options that the working Pi unit has and benefits from: - `Restart=always`, `RestartSec=20` (already documented, keep) -- `RuntimeMaxSec=8h` — periodically restart the whole pipeline so it recovers - from long-run drift / stale broadcasts. +- `RuntimeMaxSec=8h` — **intentional**: this periodic restart ends the current + YouTube broadcast and starts a fresh one, so each 8-hour segment is saved as its + own stored YouTube video (chunked recordings). It also recovers the pipeline from + long-run drift. Do **not** remove this or make `create` reuse the previous + broadcast — that would prevent YouTube from finalizing each 8-hour chunk. - `KillSignal=SIGINT` + `TimeoutStopSec=45` — `device.py` catches `KeyboardInterrupt` and cleanly terminates `rpicam-vid` and `ffmpeg`. Without SIGINT, `systemctl stop/restart` sends SIGTERM and can leave orphaned camera From 632af65ea02107f9823544c7634a34b1db702a10 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:30:07 +0000 Subject: [PATCH 06/13] Use canonical crontab reboot for 8h chunking; drop RuntimeMaxSec Co-authored-by: sgbaird <45469701+sgbaird@users.noreply.github.com> --- docs/ac-training-lab-picam-suggestions.md | 49 ++++++++++++++--------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/docs/ac-training-lab-picam-suggestions.md b/docs/ac-training-lab-picam-suggestions.md index 1d7684f..cb36d5c 100644 --- a/docs/ac-training-lab-picam-suggestions.md +++ b/docs/ac-training-lab-picam-suggestions.md @@ -23,26 +23,37 @@ an `lrc` subtitle stream and thrashes in a restart loop (high CPU, repeated Recommendation: document/ship a **single** canonical service and make it explicit that only one unit may run `device.py`. -## 2. Harden the documented `device.service` - -The service block currently in `README.md` is missing a few options that the -working Pi unit has and benefits from: - -- `Restart=always`, `RestartSec=20` (already documented, keep) -- `RuntimeMaxSec=8h` — **intentional**: this periodic restart ends the current - YouTube broadcast and starts a fresh one, so each 8-hour segment is saved as its - own stored YouTube video (chunked recordings). It also recovers the pipeline from - long-run drift. Do **not** remove this or make `create` reuse the previous - broadcast — that would prevent YouTube from finalizing each 8-hour chunk. -- `KillSignal=SIGINT` + `TimeoutStopSec=45` — `device.py` catches - `KeyboardInterrupt` and cleanly terminates `rpicam-vid` and `ffmpeg`. Without - SIGINT, `systemctl stop/restart` sends SIGTERM and can leave orphaned camera - processes holding the device. -- `Environment=PYTHONUNBUFFERED=1` — so `journalctl` shows logs live. +## 2. 8-hour chunking is done by a crontab reboot (not `RuntimeMaxSec`) + +The intended behavior is for YouTube to store each **8-hour segment as its own +video**. This is achieved the way the picam docs already prescribe under +[Automatic startup](https://ac-training-lab.readthedocs.io/en/latest/devices/picam.html#automatic-startup): +a **root crontab** reboots the Pi every 8 hours, and on each boot `device.py` +calls the Lambda `end` (which finalizes/stops the previous broadcast on YouTube, +closing that chunk) followed by `create` (a fresh broadcast for the next chunk): + +```cron +# Restart at 5 am, 1 pm, and 9 pm local time (8-hour spacing) +0 5,13,21 * * * /sbin/shutdown -r now +``` + +Because of this, do **not**: -Also, `StartLimitInterval` / `StartLimitBurst` are shown under `[Service]` in the -README but belong in the `[Unit]` section (they are ignored under `[Service]` in -current systemd). +- add `RuntimeMaxSec=8h` (or similar) to `device.service` — the cron reboot + already provides the periodic restart, and a second mechanism would create + off-schedule chunk boundaries; and +- make the Lambda `create` action idempotent / reuse the previous broadcast — that + would prevent YouTube from finalizing each 8-hour chunk. `create` must always + start a fresh broadcast, and `device.py` must keep calling `end` before `create` + on startup. + +Keep the plain `device.service` from the README (`Restart=always`, +`RestartSec=10`, `TimeoutStartSec=60`). Two small, optional doc fixes remain: + +- `StartLimitInterval` / `StartLimitBurst` are shown under `[Service]` in the + README but belong in the `[Unit]` section (they are ignored under `[Service]` in + current systemd). +- `Environment=PYTHONUNBUFFERED=1` — so `journalctl` shows logs live. ## 3. Optional: make the ffmpeg input format explicit in `device.py` From c183b2e86129524fe95d256765a063707ea761af Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 23:18:22 +0000 Subject: [PATCH 07/13] Document Wi-Fi power save fix, persistent journald, and stream stall watchdog Co-authored-by: Sterling G. Baird <45469701+sgbaird@users.noreply.github.com> Co-Authored-By: Claude Fable 5 --- docs/ac-training-lab-picam-suggestions.md | 72 +++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/docs/ac-training-lab-picam-suggestions.md b/docs/ac-training-lab-picam-suggestions.md index cb36d5c..838412e 100644 --- a/docs/ac-training-lab-picam-suggestions.md +++ b/docs/ac-training-lab-picam-suggestions.md @@ -70,3 +70,75 @@ when the freshly-created broadcast is added to its playlist. The broadcast is created and streams fine; only the playlist-add fails. This retry/backoff belongs in this repo's `chalicelib/ytb_api_utils.py` (`create_broadcast_and_bind_stream`), not in ac-training-lab. + +## 5. Disable Wi-Fi power save (root cause of the Pi dropping off the network) + +On 2026-07-06 the Pi went unreachable on the tailnet for hours while the stream +"hung" (it only recovered at the next scheduled cron reboot). The kernel log +showed `brcmfmac: brcmf_cfg80211_set_power_mgmt: power save enabled` — the +well-known Pi Zero 2 W failure where Wi-Fi power saving wedges the connection +until reboot. The hardware watchdog was already active (systemd, +`hardware timeout of 1min`), so a kernel hang would have self-rebooted; the +outage was a network-level drop, consistent with power save. + +Applied on the Pi (recommend shipping in the picam setup docs): + +```ini +# /etc/NetworkManager/conf.d/wifi-powersave-off.conf +[connection] +# 2 = disable Wi-Fi power saving (Pi Zero 2 W drops off Wi-Fi with it enabled) +wifi.powersave = 2 +``` + +plus a live `iw dev wlan0 set power_save off` (package `iw`) so it takes effect +without a reconnect. + +## 6. Persistent journald (so outages can be post-mortemed) + +The journal was RAM-only, so all logs from before a reboot were lost — the +2026-07-06 outage could not be fully diagnosed because the 13:00 cron reboot +wiped them. Applied on the Pi: + +```ini +# /etc/systemd/journald.conf.d/persistent.conf +[Journal] +Storage=persistent +SystemMaxUse=100M +``` + +(then `systemd-tmpfiles --create --prefix /var/log/journal` and +`systemctl restart systemd-journald`). `SystemMaxUse=100M` bounds the journal so +it cannot fill the SD card. + +## 7. Stall watchdog: restart `device.service` when RTMP output stalls + +The remaining failure mode systemd cannot catch: `ffmpeg`/`rpicam-vid` stay +*alive* but no data reaches YouTube (dead RTMP socket, wedged camera pipeline) — +`Restart=always` never fires because nothing exits. A watchdog now runs on the +Pi (recommend upstreaming to the picam setup): + +- `/usr/local/bin/stream-watchdog.sh` — every minute, reads `bytes_acked` on the + established TCP connection to the RTMP server (`ss -tin '( dport = :1935 )'`), + the ground truth that data is reaching YouTube. Three consecutive checks with + no progress (or no socket) → `systemctl restart device.service`. A 180 s grace + period after service start avoids false positives while `end`/`create` run, + and the counter resets whenever bytes move (a *lower* count than last check + just means a new socket after a reconnect, which is healthy). +- `stream-watchdog.service` (oneshot) + `stream-watchdog.timer` + (`OnBootSec=2min`, `OnUnitActiveSec=1min`), enabled. +- Optional dead-man's-switch: if `/etc/default/stream-watchdog` defines + `HEALTHCHECK_URL` (e.g. a Healthchecks.io or UptimeRobot heartbeat URL), the + watchdog pings it on every *healthy* check — so an external monitor alerts + when the whole Pi drops off the network, the one case no on-device logic can + handle. Currently unset; add a URL to enable alerting. + +Validated 2026-07-06 by freezing ffmpeg with `SIGSTOP`: the watchdog logged +3 missed checks, restarted `device.service`, Lambda `end`→`create` returned 200, +and the stream came back on a fresh broadcast (~110 s detection + normal restart +time). A watchdog restart mid-cycle behaves exactly like a boot: it finalizes +the current chunk and opens a new one, so it composes fine with the 8-hour +cron-reboot chunking in (2). + +Note on periodic speed tests: avoid full-bandwidth tests (e.g. `speedtest`) on +this Pi — they saturate the Zero 2 W's uplink and compete with the live RTMP +upload, causing the very stalls being monitored for. From 0d1dc4ef163c04258fd1627c8fd00505a4dba735 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 23:49:35 +0000 Subject: [PATCH 08/13] Document watchdog restart budget and broadcast-churn bounds Co-authored-by: Sterling G. Baird <45469701+sgbaird@users.noreply.github.com> Co-Authored-By: Claude Fable 5 --- docs/ac-training-lab-picam-suggestions.md | 37 ++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/ac-training-lab-picam-suggestions.md b/docs/ac-training-lab-picam-suggestions.md index 838412e..c3e77fe 100644 --- a/docs/ac-training-lab-picam-suggestions.md +++ b/docs/ac-training-lab-picam-suggestions.md @@ -130,7 +130,20 @@ Pi (recommend upstreaming to the picam setup): `HEALTHCHECK_URL` (e.g. a Healthchecks.io or UptimeRobot heartbeat URL), the watchdog pings it on every *healthy* check — so an external monitor alerts when the whole Pi drops off the network, the one case no on-device logic can - handle. Currently unset; add a URL to enable alerting. + handle. A commented template now lives in `/etc/default/stream-watchdog` on + the Pi; uncomment `HEALTHCHECK_URL=` and drop the ping URL in (suggested + monitor settings: period 5 min, grace 10 min). Both `hc-ping.com` and + `heartbeat.uptimerobot.com` are reachable from the Pi (verified 2026-07-06). +- Restart budget (guard against broadcast churn): each watchdog restart runs + `end`→`create`, i.e. spawns a fresh YouTube broadcast. To make a persistent + stall (e.g. degraded-but-up network) physically unable to spawn an endless + series of short broadcasts, the watchdog keeps a rolling-24 h history of its + own restarts in `/var/lib/stream-watchdog/restarts` (persists across the cron + reboots) and refuses to restart beyond `MAX_RESTARTS_PER_DAY` (default 6, + overridable in `/etc/default/stream-watchdog`). When the budget is exhausted + it logs `restart budget exhausted … holding off` and does nothing — and since + the stream is stalled, heartbeat pings have stopped, so the external monitor + alerts a human instead. Validated 2026-07-06 by freezing ffmpeg with `SIGSTOP`: the watchdog logged 3 missed checks, restarted `device.service`, Lambda `end`→`create` returned 200, @@ -139,6 +152,28 @@ time). A watchdog restart mid-cycle behaves exactly like a boot: it finalizes the current chunk and opens a new one, so it composes fine with the 8-hour cron-reboot chunking in (2). +### Worst-case broadcast-churn bounds (why "hundreds of short streams" cannot happen) + +Three independent limits stack, and all restart paths are covered by at least +one of them: + +1. **Watchdog cadence**: a watchdog restart needs 3 consecutive missed 1-min + checks plus the 180 s post-start grace, so even a permanently-stalled stream + yields at most ~1 watchdog restart per ~7 min *in principle* — but (2) and + (3) cut in long before that matters. +2. **Watchdog budget**: at most `MAX_RESTARTS_PER_DAY` (6) watchdog-initiated + restarts per rolling 24 h, persisted across reboots. +3. **systemd start limit**: `device.service` has `StartLimitIntervalSec=3600` / + `StartLimitBurst=3` — *any* combination of crash-loop (`Restart=always`) and + watchdog restarts beyond 3 starts per hour puts the unit into `failed` + until the next cron reboot; the watchdog sees an inactive unit and does + nothing (and heartbeats stop → alert). + +Net worst case ≈ 3 scheduled chunk broadcasts + ≤6 watchdog restarts + a few +crash-loop starts per day — bounded at roughly a dozen broadcasts/day even if +everything is on fire, vs. the hundreds a naive every-N-minutes restarter +could produce. + Note on periodic speed tests: avoid full-bandwidth tests (e.g. `speedtest`) on this Pi — they saturate the Zero 2 W's uplink and compete with the live RTMP upload, causing the very stalls being monitored for. From a9417bfe1dddb344107eba6fe21ac34598c86b9b Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:20:31 +0000 Subject: [PATCH 09/13] Document wired heartbeat and device.py retry-path zombie fix Co-authored-by: Sterling G. Baird <45469701+sgbaird@users.noreply.github.com> Co-Authored-By: Claude Fable 5 --- docs/ac-training-lab-picam-suggestions.md | 35 +++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/ac-training-lab-picam-suggestions.md b/docs/ac-training-lab-picam-suggestions.md index c3e77fe..191524c 100644 --- a/docs/ac-training-lab-picam-suggestions.md +++ b/docs/ac-training-lab-picam-suggestions.md @@ -126,14 +126,15 @@ Pi (recommend upstreaming to the picam setup): just means a new socket after a reconnect, which is healthy). - `stream-watchdog.service` (oneshot) + `stream-watchdog.timer` (`OnBootSec=2min`, `OnUnitActiveSec=1min`), enabled. -- Optional dead-man's-switch: if `/etc/default/stream-watchdog` defines - `HEALTHCHECK_URL` (e.g. a Healthchecks.io or UptimeRobot heartbeat URL), the - watchdog pings it on every *healthy* check — so an external monitor alerts - when the whole Pi drops off the network, the one case no on-device logic can - handle. A commented template now lives in `/etc/default/stream-watchdog` on - the Pi; uncomment `HEALTHCHECK_URL=` and drop the ping URL in (suggested - monitor settings: period 5 min, grace 10 min). Both `hc-ping.com` and - `heartbeat.uptimerobot.com` are reachable from the Pi (verified 2026-07-06). +- Dead-man's-switch (wired 2026-07-07): `/etc/default/stream-watchdog` defines + `HEALTHCHECK_URL` (a Healthchecks.io ping URL), and the watchdog pings it on + every *healthy* check — so the external monitor alerts when the whole Pi + drops off the network, the one case no on-device logic can handle. Delivery + verified from the Pi (HTTP 200) with the exact curl invocation the watchdog + uses. The file is `chmod 600 root:root` since the ping URL itself must stay + secret (anyone holding it can fake healthy pings). Suggested monitor + settings: period 5 min, grace 10 min — a successful watchdog self-heal + (~7 min worst-case gap) then never pages; only a real outage does. - Restart budget (guard against broadcast churn): each watchdog restart runs `end`→`create`, i.e. spawns a fresh YouTube broadcast. To make a persistent stall (e.g. degraded-but-up network) physically unable to spawn an endless @@ -177,3 +178,21 @@ could produce. Note on periodic speed tests: avoid full-bandwidth tests (e.g. `speedtest`) on this Pi — they saturate the Zero 2 W's uplink and compete with the live RTMP upload, causing the very stalls being monitored for. + +## 8. `device.py`: reap the old pipeline processes on internal retry + +Observed in production 2026-07-06 22:28 MDT: YouTube dropped the RTMP socket +(`Error writing trailer: End of file` / `Broken pipe` from ffmpeg), and +`device.py`'s internal retry loop recovered within one second — it terminated +the dead pipeline, restarted `rpicam-vid`+`ffmpeg`, and kept streaming to the +**same** broadcast (no Lambda `end`/`create`, no new YouTube video, no watchdog +intervention). This is exactly the desired layering: transient RTMP drops are +absorbed by `device.py`; only a genuinely wedged pipeline escalates to the +watchdog (fresh broadcast); only a dead/off-network Pi escalates to the +heartbeat monitor. + +One cosmetic bug in that path: after `Terminating processes..`, `device.py` +never `wait()`s on the old `rpicam-vid`, leaving a `` zombie per retry +(parented to `device.py`, so they accumulate until the next service restart or +reboot). Fix upstream: call `proc.wait(timeout=...)` on each terminated child +in the retry path. From 072be3c1b1ce8f8a889db1ecd1aa0c783454b65d Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:39:14 +0000 Subject: [PATCH 10/13] Update broadcast description to Vertical Cloud Lab @ BYU The YouTube broadcast description was hardcoded to the Acceleration Consortium in Toronto; this camera is stationed at the Vertical Cloud Lab at BYU. Deployed to the live Lambda; the next broadcast (next 8h chunk) picks it up. Co-authored-by: Sterling G. Baird <45469701+sgbaird@users.noreply.github.com> Co-Authored-By: Claude Fable 5 --- chalicelib/ytb_api_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chalicelib/ytb_api_utils.py b/chalicelib/ytb_api_utils.py index b56e4f3..686842c 100644 --- a/chalicelib/ytb_api_utils.py +++ b/chalicelib/ytb_api_utils.py @@ -100,9 +100,9 @@ def create_broadcast_and_bind_stream(cam_name: str,workflow_name: str, privacy_s formatted_time = datetime.utcnow().strftime("%Y-%m-%d UTC %H:%M") broadcast_title = f"{workflow_name} stream {cam_name}, {formatted_time}" broadcast_description = ( - f"Live camera feed from {workflow_name} stationed in Toronto, ON " - "at the Acceleration Consortium (AC).\n\n" - "https://acceleration.utoronto.ca/" + f"Live camera feed from {workflow_name} stationed at " + "the Vertical Cloud Lab @ BYU\n\n" + "https://github.com/vertical-cloud-lab" ) broadcast_response = YOUTUBE.liveBroadcasts().insert( From 1626471de119fd32f37707f453c7ebfba7eb4758 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 22:04:54 +0000 Subject: [PATCH 11/13] Archive ac-dev-lab download/process/upload discussions with summary Complete comment history (incl. UI-hidden comments) of ac-dev-lab #212, #223, #231, #341, PR #234, PR #343, fetched via the GitHub API, plus a README summarizing the adopted pipeline design. Co-authored-by: Sterling G. Baird <45469701+sgbaird@users.noreply.github.com> Co-Authored-By: Claude Fable 5 --- docs/ac-dev-lab-archive/README.md | 70 ++ ...-212-download-self-owned-youtube-videos.md | 36 + ...ssing-videos-still-frame-removal-and-sp.md | 425 +++++++++ ...-the-8-hr-auto-restart-mechanism-and-st.md | 697 ++++++++++++++ ...ng-playwright-to-automate-the-download-.md | 35 + ...234-added-ability-to-download-yt-videos.md | 184 ++++ ...ght-based-youtube-video-downloader-with.md | 852 ++++++++++++++++++ 7 files changed, 2299 insertions(+) create mode 100644 docs/ac-dev-lab-archive/README.md create mode 100644 docs/ac-dev-lab-archive/issue-212-download-self-owned-youtube-videos.md create mode 100644 docs/ac-dev-lab-archive/issue-223-post-processing-videos-still-frame-removal-and-sp.md create mode 100644 docs/ac-dev-lab-archive/issue-231-prototyping-the-8-hr-auto-restart-mechanism-and-st.md create mode 100644 docs/ac-dev-lab-archive/issue-341-explore-using-playwright-to-automate-the-download-.md create mode 100644 docs/ac-dev-lab-archive/pr-234-added-ability-to-download-yt-videos.md create mode 100644 docs/ac-dev-lab-archive/pr-343-add-playwright-based-youtube-video-downloader-with.md diff --git a/docs/ac-dev-lab-archive/README.md b/docs/ac-dev-lab-archive/README.md new file mode 100644 index 0000000..10d7271 --- /dev/null +++ b/docs/ac-dev-lab-archive/README.md @@ -0,0 +1,70 @@ +# ac-dev-lab download/process/re-upload discussion archive + +Complete comment archives (fetched via the GitHub API on 2026-07-08, including +comments hidden in the web UI by GitHub's pagination) from the +AccelerationConsortium/ac-dev-lab issues and PRs that trace the design of the +YouTube livestream **download → post-process → re-upload** pipeline. Archived +to give agents working in this repo the full context without re-fetching. + +| File | Item | State | What it covers | +|---|---|---|---| +| `issue-212-*.md` | [#212](https://github.com/AccelerationConsortium/ac-dev-lab/issues/212) | closed | First exploration of downloading self-owned videos; settled on yt-dlp over youtube-dl | +| `pr-234-*.md` | [#234](https://github.com/AccelerationConsortium/ac-dev-lab/pull/234) | merged | `yt_utils.py`: yt-dlp download of the latest video (public videos, no auth) | +| `issue-223-*.md` | [#223](https://github.com/AccelerationConsortium/ac-dev-lab/issues/223) | open | The processing pipeline: auto-editor stale-section detection, ffmpeg speed-up overlay, 16x speedup, title convention, yt-dlp cookie problems on HF Spaces | +| `issue-231-*.md` | [#231](https://github.com/AccelerationConsortium/ac-dev-lab/issues/231) | open | 8-hr restart mechanism history (origin of this streamingLambda repo): Lambda holds token.pickle, crontab reboot `0 5,13,21`, one-time stream keys | +| `issue-341-*.md` | [#341](https://github.com/AccelerationConsortium/ac-dev-lab/issues/341) | closed | Kickoff for Playwright-based downloads of private videos | +| `pr-343-*.md` | [#343](https://github.com/AccelerationConsortium/ac-dev-lab/pull/343) | merged | **The adopted downloader**: YouTube Data API discovery + Playwright Studio download with TOTP login | + +## Key findings + +### Download (the hard part — YouTube has no API to download video content) + +- **yt-dlp** (PR #234) works fine for public/unlisted videos with no auth, and + is the preferred tool on headless/ephemeral machines. For **private** videos + it needs browser cookies, which expire within hours + ([yt-dlp#8227](https://github.com/yt-dlp/yt-dlp/issues/8227)) — a dead end + for unattended automation on HF Spaces (issue #223). +- **Playwright + YouTube Studio** (PR #343, the merged approach in + `src/ac_training_lab/video_editing/download.py`): a dedicated Google account + (`achardwarestreams.downloader@...`) is a **channel editor** (viewer role has + the download button disabled). The script: + 1. lists playlists/videos via the YouTube Data API and filters out + already-downloaded/processed ones, + 2. logs into Google with email + password + **pyotp TOTP** (2FA re-enabled + deliberately: Google blocks password-only logins from ephemeral machines + with "couldn't verify this account belongs to you", but accepts + password+TOTP), + 3. navigates to `studio.youtube.com/video/{video_id}/edit` and clicks the + ⋮ menu → Download. +- Playwright **headless fails** (Google bot detection) — a virtual framebuffer + (xvfb) is required. On the BALAM cluster it runs inside an **apptainer** + container (system deps not installable), and only login nodes have internet. +- HF Spaces cannot run Playwright at all (no GUI browser env; Docker attempt + stalled) — reproducer: `huggingface.co/spaces/AccelerationConsortium/playwright-reproducer`. +- Alternative acknowledged in the threads: make videos public/unlisted so + yt-dlp works without auth (sgbaird encourages public/unlisted "in part for + this reason"). + +### Processing (issue #223) + +- **auto-editor** pipeline by @Jonathan-Woo: + 1. `auto-editor --edit motion:threshold=…` produces v1 timestamps of stale + (no-motion) sections, + 2. ffmpeg burns a speed-up indicator overlay (e.g. `16x`) on stale sections, + 3. auto-editor speeds up stale sections (`--silent-speed`), keeping motion at 1x. +- Tuning knobs: `threshold` (fraction of changed pixels to count as motion) and + `margin` (padding around kept sections). False positives are a concern, so + stale sections are sped up rather than deleted (a huge `--silent-speed` + effectively deletes). +- Title convention so originals and processed videos can be matched via the + Data API: keep the original title/ID and append **`[processed, 16x]`**. +- Later development moved to + [AccelerationConsortium/youtube-livestream-processor](https://github.com/AccelerationConsortium/youtube-livestream-processor). + +### Upload / orchestration + +- Uploads and playlist management use the YouTube Data API with the channel + owner's **token.pickle** (same credential this repo's Lambda uses from S3). +- Issue #231 documents why broadcasts are chunked by reboot (`0 5,13,21 * * *` + crontab) with the Lambda doing `end` → `create` — YouTube autostart is + unreliable if a broadcast is created while data is already flowing. diff --git a/docs/ac-dev-lab-archive/issue-212-download-self-owned-youtube-videos.md b/docs/ac-dev-lab-archive/issue-212-download-self-owned-youtube-videos.md new file mode 100644 index 0000000..05385db --- /dev/null +++ b/docs/ac-dev-lab-archive/issue-212-download-self-owned-youtube-videos.md @@ -0,0 +1,36 @@ +# Issue #212: Download self-owned YouTube videos + +- **URL:** https://github.com/AccelerationConsortium/ac-dev-lab/issues/212 +- **Author:** @sgbaird +- **State:** closed +- **Created:** 2025-03-27T16:47:25Z **Closed:** 2025-05-01T23:21:05Z +- **Comments archived:** 2 issue comments + +--- + +## Original description + +In creator dashboard, there's a way to download it via GUI. Some possibilities: + +- use browser cookies: https://stackoverflow.com/a/55272225 +- use youtube-dl: https://github.com/ytdl-org/youtube-dl + +Terms of service: https://www.youtube.com/t/terms + +Cc @Jonathan-Woo + +--- + +## Comments (complete, in chronological order) + +### Comment 1 — @sgbaird at 2025-04-08T14:18:33Z + +@Neil-YL if you end up getting stuck on the new picam device.py script, could you give this one a try? Ideally, it would be great to have the full restart + download workflow tested to show that we can store and retrieve 24/7 streams. + +--- + +### Comment 2 — @Jonathan-Woo at 2025-05-01T15:49:48Z + +Switched to using yt-dlp instead of youtube-dl because it's more actively maintained. + +--- diff --git a/docs/ac-dev-lab-archive/issue-223-post-processing-videos-still-frame-removal-and-sp.md b/docs/ac-dev-lab-archive/issue-223-post-processing-videos-still-frame-removal-and-sp.md new file mode 100644 index 0000000..b9d16e2 --- /dev/null +++ b/docs/ac-dev-lab-archive/issue-223-post-processing-videos-still-frame-removal-and-sp.md @@ -0,0 +1,425 @@ +# Issue #223: Post-processing videos (still frame removal and speedup) + +- **URL:** https://github.com/AccelerationConsortium/ac-dev-lab/issues/223 +- **Author:** @sgbaird +- **State:** open +- **Created:** 2025-04-08T22:30:02Z **Closed:** None +- **Comments archived:** 39 issue comments + +--- + +## Original description + +This can be standalone from #212, but is of course related. The idea would be to remove segments of the video where there is little to no change (which could be difficult when there are blinking lights for example) and run a speedup of e.g., 16x, to later be uploaded to a separate playlist (we can worry about automatic video uploads later, though not sure if that requires just an API key or needs token.pickle). + +https://claude.ai/share/217878d2-d194-4441-8c91-9af937ba787b + +--- + +## Comments (complete, in chronological order) + +### Comment 1 — @sgbaird at 2025-05-01T21:55:43Z + +Per our conversation, I think the decision was to include something in the title (maybe original video id? Or could be timestamp) that we can keep the same between the original and the processed videos, and add some kind of note in the title like: `[processed, 16x]`. @Neil-YL + +This can be used with the YouTube API, grabbing lists of the video names from from original and processed. From the YouTube API quota perspective, if multiple pages are returned, each page will count towards the quota. + +Per @jonathan-woo 's suggestion, hosting this on a paid-tier HuggingFace space with a the lowest tier GPU (could also try the free, preemptable A100) and the minimum sleep time of 5 minutes of inactivity seems pretty reasonable and low-cost. + +@Jonathan-Woo which algorithms/codebases were you looking at? I had found some in the transcript above (a few links were dead of course), might be worth a quick look. + +--- + +### Comment 2 — @sgbaird at 2025-05-01T22:04:03Z + +@Jonathan-Woo here's the example video you can use: https://www.youtube.com/live/Tbru5BiokmU?si=Sbmfu5dggSYLpnsp + +--- + +### Comment 3 — @Jonathan-Woo at 2025-05-01T23:05:57Z + +Yep, I looked at the transcript, thanks for that. I'm looking into motion and auto-editor right now. Both can provide timestamps. I think I'll test both and examine their performance. Leaning more towards motion as it seems to be more popular. + +--- + +### Comment 4 — @sgbaird at 2025-05-06T18:55:02Z + +Related: https://docs.frigate.video/ (I guess @yakavetsiv is incorporating or at least has plans to). Thanks @kelvinchow23 for the ping about it + + +EDIT: using with esp32 cameras, PoE, and local network + +![PXL_20250522_160523717.jpg](https://github.com/user-attachments/assets/702a4783-7486-4c6e-803d-9c6f3085aee4) + +![PXL_20250522_160501152.jpg](https://github.com/user-attachments/assets/cceb8c94-eda6-435e-823f-e52cae887ce6) + +Ilya running everything on a local server, Zima Cube (frigate, MQTT broker, Prefect local) + +![PXL_20250522_164421589.jpg](https://github.com/user-attachments/assets/1baae55b-4846-4abf-8530-9d1482ff4143) + + + + +--- + +### Comment 5 — @Jonathan-Woo at 2025-05-07T18:26:17Z + +I've been using [auto-editor](https://github.com/WyattBlue/auto-editor) so far and the results are pretty good on the video linked above. The following command cut these sections out: + +https://github.com/user-attachments/assets/0da82384-e7f7-4554-aaa8-1256991032c7 + +The tricky part is to tune: +1. margin: amount of stale video added to edited sections to smooth out edit +2. threshold: % of "motion" required to not be stale + +There is also support for yt-dlp directly so we can pass the stream link directly. Also, there is support for hardware encoding. + +``` +auto-editor https://www.youtube.com/live/Tbru5BiokmU\?si\=WtsDaMc0DSHxFbKH --edit motion:threshold=0.2 --video-speed 99999 --silent-speed 1 -c:v h264_videotoolbox --download-format bv --margin 10sec +``` + +One issue is that, since we don't have a wall clock, it's difficult to tell when video is being sped up. So, I'm working on adding an indicator for when video is being sped up. Then, it will be easier to tune the settings. + +--- + +### Comment 6 — @sgbaird at 2025-05-09T22:45:08Z + +Really cool to see this! Had thought about adding an overlay directly via the pi zero 2w but not sure if it will handle that extra processing well, and might complicate other things (for one, we lose the raw footage). Ultimately leaned away from it since YouTube already has timestamps. + +White text with slightly transparent black background has often worked well from a contrast and aesthetics point of view. + +Any luck with adding the indicator? + +Also would be great if you could share the sections that were kept, if not too much work. + + +--- + +### Comment 7 — @sgbaird at 2025-05-12T16:58:48Z + +@Jonathan-Woo bump + +--- + +### Comment 8 — @Jonathan-Woo at 2025-05-12T18:01:14Z + +> Really cool to see this! Had thought about adding an overlay directly via the pi zero 2w but not sure if it will handle that extra processing well, and might complicate other things (for one, we lose the raw footage). Ultimately leaned away from it since YouTube already has timestamps. +> +> White text with slightly transparent black background has often worked well from a contrast and aesthetics point of view. +> +> Any luck with adding the indicator? +> +> Also would be great if you could share the sections that were kept, if not too much work. + +Still working on adding the indicator. I'm using ffmpeg to add an overlay similar to https://video.stackexchange.com/questions/12105/add-an-image-overlay-in-front-of-video-using-ffmpeg. + +Overall, this my plan: +1. auto-editor to get [v1](https://auto-editor.com/docs/v1) file to get timestamps for when to add overlay. +2. ffmpeg to add overlay +3. auto-editor to accelerate stale sections + +This is the kept video: + +https://github.com/user-attachments/assets/cba3e7eb-780f-488b-8eb8-9e61ccde58c8 + +--- + +### Comment 9 — @sgbaird at 2025-05-12T18:26:25Z + +Awesome, thanks! It seems like it did a pretty good job with separating the stale vs. non-stale + +> auto-editor to accelerate stale sections + +Quick aside: Originally, I was imagining deleting the stale sections entirely and speeding up the non-stale sections + +--- + +### Comment 10 — @Jonathan-Woo at 2025-05-12T18:32:32Z + +Sure that's totally possible. We would just have to update the `--silent-speed` argument. + +--- + +### Comment 11 — @Jonathan-Woo at 2025-05-14T18:31:10Z + +Overlay has been added. + +https://github.com/user-attachments/assets/998b1555-a738-4143-847d-ad0dbbfff0e3 + +I'm concerned about removing stale sections entirely because I'm concerned that this editor may have false positives. + +I will share the final sped up version soon. In the meantime we should think about what to remove/speed up. + +--- + +### Comment 12 — @sgbaird at 2025-05-14T21:25:34Z + +Nice! Good point about false positives. I'm worried a bit about devices that have a lot of stale time (e.g., 8 hrs of stale video @ 16x = 30 min), which might get cumbersome for promo vids and robotic training. Though, it could cause problems if people are doing post-mortem analysis (i.e., watching the sped-up video) and miss something that would otherwise catch their eye. + +I could be missing something else too. I should probably document and consolidate the various intended uses somewhere (these have made it on a whiteboard occasionally). + +Aside: is the intention to put the speedup factor in the overlay? (e.g., `16x`). + +--- + +### Comment 13 — @Jonathan-Woo at 2025-05-15T01:47:11Z + +This is the edited version with the overlay indicating 16x. It still flickers a bit so the sensitivity governed by the motion will likely need to be tuned more. + +Other parts of the pipeline left is the communication with youtube in terms of downloading, uploading, organizing into playlists, etc. Let's sync tomorrow? + +https://github.com/user-attachments/assets/06cb2b35-ec54-48c8-ae60-cf6dd2f6cc4d + +--- + +### Comment 14 — @sgbaird at 2025-05-15T21:00:36Z + +> communication with youtube in terms of downloading, uploading, organizing into playlists, etc. + +Based on our conversation, [try adding timestamp directly on Zero 2W](https://github.com/AccelerationConsortium/ac-training-lab/issues/213) (looks like picamera2 has a built-in option that will hopefully not add too much processing overhead). We also agreed that when 16x appears for the stale video, we naturally expect to see something moving quickly, so it's not quite as clear for a 1-second display of 16x that 16 seconds have gone by. A running timestamp will probably make this more straightforward. + +Easy way to get rid of section is to set speedup to huge value (999x) so it essentially just disappears. Might be worth a quick check to verify it doesn't cause an error to have massive speedup like this that wipes out all frames for that section. + +Moving on to the hugging face portion, scheduling to avoid constant GPU consumption is probably the main challenge. Based on a quick search: +- https://www.google.com/search?q=schedule+for+hugging+face+space +- https://huggingface.co/docs/huggingface_hub/main/en/guides/manage-spaces +- https://github.com/kghamilton89/spaces-scheduler + +High-level purposes review: +1. Real-time, remote hardware development (not relevant to speedup) +2. Flashy/cool demos, promo videos (ideal is to cut stale video almost entirely, with max a few seconds at a time without motion) +3. Post-mortem / failure analysis (i.e., when you don't know the timestamp of an event, but want to get a general sense of the process or see if something unexpected happens) +4. Training datasets for roboticists (video + timestamps + metadata such as temperature/humidity/hardware logs. Cropping the video afterwards to remove a UTC timestamp overlay portion is probably not a huge issue. As long as timestamps for metadata like temperature are stored elsewhere, we don't need to worry about storing that on YouTube side). + +@Jonathan-Woo - some [documentation by Yanghuang related to OAuth 2.0](https://github.com/AccelerationConsortium/ac-training-lab/issues/202#issuecomment-2770625621), though I think you can probably use the pickle file I shared directly with you. As you mentioned, you can probably deserialize the pickle file, store it as text as a Hugging Face Space, and then reserialize it. I could be missing something. + +The plan is to start with a free CPU-version on HuggingFace and lmk when I should convert it to a paid GPU-tier. + +--- + +### Comment 15 — @sgbaird at 2025-05-27T12:13:28Z + +@Jonathan-Woo do you think HF is the right platform for doing this? There's not really a need for a GUI (more of a service). Though there might still be an appeal from a cost perspective (not sure how HF compares). + +--- + +### Comment 16 — @Jonathan-Woo at 2025-05-27T14:55:45Z + +Yes I think HF is a good platform for this. + +1. We wouldn't have to take on and support another service. +2. It supports task scheduling and can switch to different hardware as required which makes it easy for us to scale. +3. I think we would still want to visualize the processing progress and this would be a very convenient way to access it. + +--- + +### Comment 17 — @sgbaird at 2025-05-27T16:23:15Z + +All points make sense to me, and I'm aligned on that. Thanks for detailing! + +--- + +### Comment 18 — @Jonathan-Woo at 2025-06-07T17:05:40Z + +Almost finished MWE here: https://huggingface.co/spaces/AccelerationConsortium/Video-Processing?logs=container + +Current issue is that the tool we use for downloading youtube videos, `yt-dlp`, requires cookies for authentication. When testing locally, it's not an issue as we can forward the cookies from our browser. But for the HF space, we can't access the browser, navigate to youtube, and login to get the cookie. So, I believe the options are: + +1. Download your cookies and upload them to the space before use. +2. Use `yt-dlp` locally to download the videos and then upload them for processing. + +--- + +### Comment 19 — @sgbaird at 2025-06-07T21:16:52Z + +Could you link the documentation for the cookies, if available? + +--- + +### Comment 20 — @sgbaird at 2025-06-07T21:19:02Z + +Also, does the following have the same requirement for cookies as `yt-dlp`? (Linked from #212) - https://github.com/ytdl-org/youtube-dl + +--- + +### Comment 21 — @Jonathan-Woo at 2025-06-07T21:27:59Z + +> Could you link the documentation for the cookies, if available? + +Under the "How do I pass cookies to yt-dlp?" section: https://github.com/yt-dlp/yt-dlp/wiki/FAQ + + +--- + +### Comment 22 — @Jonathan-Woo at 2025-06-07T21:30:42Z + +> Also, does the following have the same requirement for cookies as `yt-dlp`? (Linked from [#212](https://github.com/AccelerationConsortium/ac-training-lab/issues/212)) - https://github.com/ytdl-org/youtube-dl + +I believe so. `youtube-dl` is no longer maintained and `yt-dlp` should contain all the features of `youtube-dl` as it merged with it. + +--- + +### Comment 23 — @sgbaird at 2025-06-07T21:48:27Z + +Aside: + +Noticed they have a Python wrapper - https://github.com/yt-dlp/yt-dlp#embedding-yt-dlp + +and that it's pip-installable (noticed you're using that) + +https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-download-only-new-videos-from-a-playlist seems helpful for our use case (at least relevant) + +Could you rename requirements.txt to requirements-frozen.txt or similar and make a minimal requirements.txt? (Not sure what's required other than yt-dlp in your code) + +For the cookies, since there's a method for having a txt file, I suppose we could have this as an environment secret? + +Any idea on how frequently the cookies would need to be refreshed? It seems these cookies are tied to a specific account. Might make a separate one for this. + +Do you know if this will work for private videos as well? (Unlisted I imagine shouldn't make a difference from public, but private videos require authentication) + +--- + +### Comment 24 — @Jonathan-Woo at 2025-06-07T22:05:50Z + +I've removed unnecessary packages from the requirements. + +It seems like the cookies last a few hours at best: https://github.com/yt-dlp/yt-dlp/issues/8227 + +Regarding private videos, I'm not sure. My understanding of `yt-dlp` is that it retrieves videos similarly to how the youtube webpage does so if you're able to view it on youtube, `yt-dlp` should work. Though again, I haven't tried it. + +--- + +### Comment 25 — @sgbaird at 2025-06-07T23:47:25Z + +Aside: even though we're downloading content that we have the rights to, we'll need to be mindful and probably use a dedicated account - https://github.com/yt-dlp/yt-dlp/wiki/Extractors#exporting-youtube-cookies + +Also some other info above + +--- + +### Comment 26 — @sgbaird at 2025-06-08T00:51:42Z + +Another option we could consider is using [playwright](https://playwright.dev/python/) (I've been using the MCP server for it at times, saw it listed on one of GitHub's tutorials as an example MCP) and having it login and go directly to the download button on the YouTube UI for the various videos. + +I'm not sure if playwright could handle that, but maybe worth a shot (either for automatically retrieving cookies or for navigating to YouTube's built-in download link for self-owned videos). + +--- + +### Comment 27 — @sgbaird at 2025-06-11T04:30:21Z + +@Jonathan-Woo do you have some of the processed videos that you can share? Let me know if you need specific links that are going to have more interesting content. + +--- + +### Comment 28 — @sgbaird at 2025-06-20T18:20:28Z + +cc @zweaung1014 + +--- + +### Comment 29 — @sgbaird at 2025-06-26T15:37:56Z + +@zweaung1014 I sent you an invite to Hugging Face, once you've accepted the invite I'll give you access to https://huggingface.co/spaces/AccelerationConsortium/Video-Processing. Probably, we'll make it public (ensuring that there aren't any secrets in the history). From there, could you work on replacing the yt-dlp downloading with your playwright implementation? + +--- + +### Comment 30 — @zweaung1014 at 2025-06-26T21:35:37Z + +@sgbaird Turns out, my playwright implementation might also be running into cookie issues. The download sometimes works and sometimes doesn't. It's pretty inconsistent. When i manually download from an incognito browser, it doesn't download. But when I do it from a normal one, it downloads. Since the playwright script opens its own session, it also behaves like an incognito browser. Hence the cookie problem. + +I put it in my repo: https://github.com/zweaung1014/yt_download_2FA.git +`verify_pyotp.py` is what runs the script +The secret code is stored as a "Repository Secret"; not in the code. + +--- + +### Comment 31 — @zweaung1014 at 2025-06-26T21:48:00Z + +The way the script works is, it goes to the link first, and takes the video title. Then, it signs in with 2FA and goes to the YouTube Studio page. On this page, it finds the video title it stored earlier, and clicks it. + +I was having a problem at first because I was telling it to click the checkbox, go to "More Actions", and click "Download". But this throws frequent errors due to timing. Clicking the video title first is a much more consistent process. + +The last step is to click the Options menu and click "Download". + +--- + +### Comment 32 — @sgbaird at 2025-06-26T22:54:10Z + +Could you use Python to change the string URL to the correct one, rather than clicking to get there? + +--- + +### Comment 33 — @sgbaird at 2025-06-26T23:04:15Z + +It was seeming to work ok in the copilot actions. Not sure if there was something different or it was implying it did but not actually + +--- + +### Comment 34 — @sgbaird at 2025-06-26T23:08:30Z + +Maybe we put this back in the playwright PR discussion + +--- + +### Comment 35 — @zweaung1014 at 2025-06-26T23:31:15Z + +> Could you use Python to change the string URL to the correct one, rather than clicking to get there? + +Good point! Let me do that. + +--- + +### Comment 36 — @sgbaird at 2025-07-02T21:00:35Z + +(scheduled) any updates? (I know you were working a lot on solid dosing, etc. so no worries if not) + +Get Outlook for Android +________________________________ +From: Larry Aung ***@***.***> +Sent: Thursday, June 26, 2025 7:31:36 PM +To: AccelerationConsortium/ac-training-lab ***@***.***> +Cc: Sterling Baird ***@***.***>; Mention ***@***.***> +Subject: Re: [AccelerationConsortium/ac-training-lab] MWE for post-processing videos (still frame removal and speedup) on HF space (Issue #223) + +[https://avatars.githubusercontent.com/u/135032017?s=20&v=4]zweaung1014 left a comment (AccelerationConsortium/ac-training-lab#223) + +Could you use Python to change the string URL to the correct one, rather than clicking to get there? + +Good point! Let me do that. + +— +Reply to this email directly, view it on GitHub, or unsubscribe. +You are receiving this because you were mentioned.Message ID: ***@***.***> + + +--- + +### Comment 37 — @sgbaird at 2025-07-12T03:53:36Z + +Like we talked about, Larry - thanks for prioritizing the SDL2 workflows. @Jonathan-Woo is available again to work on this, so he'll pick up from where you left off (thanks for your patience with the relay race). + +EDIT: Here's where we left off https://github.com/AccelerationConsortium/ac-training-lab/pull/343#issuecomment-3040740167 + +Also, @Jonathan-Woo I think you mentioned there were some ways we may be able to speed up the process? (It would be good to get a sense of the time required to process a single video in general, too). I think you mentioned avoiding encoding the segments that would be removed anyway + +--- + +### Comment 38 — @Jonathan-Woo at 2025-07-16T20:39:17Z + +Yes, so here's how things currently work. + +1. `auto-editor` is used to detect stale video sections +2. `ffmpeg` applies overlay +3. `auto-editor` is applied to speed-up sections + +Currently, step 2 naively re-encodes the entire video after adding the overlay which can be accelerated by using stream copy to avoid encoding unedited sections. I'm working on this. + +--- + +### Comment 39 — @sgbaird at 2025-08-21T17:24:18Z + +Porting most development and code over to https://github.com/AccelerationConsortium/youtube-livestream-processor, new issues to be raised there. + +We may also end up clearing out https://github.com/AccelerationConsortium/ac-training-lab/tree/main/src/ac_training_lab/video_editing since it's redundant now. + +--- diff --git a/docs/ac-dev-lab-archive/issue-231-prototyping-the-8-hr-auto-restart-mechanism-and-st.md b/docs/ac-dev-lab-archive/issue-231-prototyping-the-8-hr-auto-restart-mechanism-and-st.md new file mode 100644 index 0000000..fc9efbb --- /dev/null +++ b/docs/ac-dev-lab-archive/issue-231-prototyping-the-8-hr-auto-restart-mechanism-and-st.md @@ -0,0 +1,697 @@ +# Issue #231: Prototyping the 8-hr auto-restart mechanism and stream restoration + +- **URL:** https://github.com/AccelerationConsortium/ac-dev-lab/issues/231 +- **Author:** @sgbaird +- **State:** open +- **Created:** 2025-04-28T16:21:36Z **Closed:** None +- **Comments archived:** 45 issue comments + +--- + +## Original description + +Related: +- https://github.com/AccelerationConsortium/ac-training-lab/issues/202 (focused on getting the API set up and pulling stream URL automatically) +- https://github.com/AccelerationConsortium/ac-training-lab/issues/212 +- https://github.com/AccelerationConsortium/ac-training-lab/issues/223 + + +Based on discussion with @Neil-YL: + +How should the following tasks be handled (e.g., same manager, different managers): +1. Close the current stream and start a new one every 8 hours +2. Log the recent status of each stream into a CSV every 10-15mins +3. Report the latest download link to a database + +Or is it better to separate these into different scripts or services? + +Maybe the monitor (task 2) as a separate service. Maybe the auto-restart and download links as a schedule-based service (e.g., scheduled task with Prefect) to trigger the workflow: first retrieve the download link for the latest completed video (not the currently active stream), then close the current broadcast and create a new one. + +Another idea is to have a monitor create new stream when `device_name` is not in active stream list, and an 8-hr schedule to end all active streams + +> 8-hr schedule to end all active streams +This one could also restart the stream, to minimize downtime (i.e., loop through each stream one at a time --> [end stream, restart stream]), but maybe this would cause issues if the monitor and this both try to restart the same stream. + +It could be one code/service to both end/restart and monitor, when executing ending and restart, try to move out of the loop of the checking and when the restart completed back to the monitor loop. + +--- + +There also needs to be a service that auto-downloads the videos and post-processes them (high bandwidth, so probably don't want to mix with the scripts that ensure uptime). This might be long-running processes on the order of 1+ hrs for each video, where GPUs would significantly accelerate the process. If there are only a few streams, a single service might be able to handle this. If there are 50+ streams, then there would need to be multiple machines doing the processing. This might justify some kind of trigger-based and on-demand mechanism.. (instead of having a few dedicated machines doing the processing). This might still require a schedule and monitor to see if new videos are available in a playlist (or if new entries are available in MongoDB). + +https://prefect-community.slack.com/archives/C04DZJC94DC/p1745859761114599?thread_ts=1745858182.153999&cid=C04DZJC94DC + +
+ +Sterling Baird + 42 minutes ago +[@Marvin](https://prefect-community.slack.com/team/ULVA73B9P) + how do you handle on-demand spinup of resources based on monitoring some kind of condition? (e.g., new entry added to a MongoDB database) + +Marvin +APP [41 minutes ago](https://prefect-community.slack.com/archives/C04DZJC94DC/p1745859761114599?thread_ts=1745858182.153999&cid=C04DZJC94DC) +To handle on-demand resource spinup based on a condition like a new entry in a MongoDB database using Prefect, you can follow these general steps: +1. Monitoring and Triggering: Use Prefect's event-driven triggers to monitor conditions. For instance, you can create a custom trigger that listens for specific events such as changes in your MongoDB database. Prefect allows you to define custom events and triggers that can start flows based on these events. More details can be found in the [Prefect custom triggers documentation](https://docs-3.prefect.io/v3/automate/events/custom-triggers). +2. Infrastructure Provisioning: Prefect can automatically provision the necessary infrastructure for running your flows. This is done using Prefect's CLI with the --provision-infra flag, which allows for serverless resource provisioning. This requires proper authentication and setup with your cloud provider, such as AWS. You can find more information in the [Prefect infrastructure examples documentation](https://docs-3.prefect.io/v3/deploy/infrastructure-examples/serverless). +3. Flow Definition: Define your flow using Prefect's flow decorator, which is part of the core API. Prefect supports imperative programming styles, which should suit your preference. +For more specific implementations, you may need to integrate with Prefect's Python API to define tasks that interact with MongoDB and trigger flows based on the results. +If you have more specific requirements or need further clarification, feel free to ask! + +
+ +--- + +## Comments (complete, in chronological order) + +### Comment 1 — @Neil-YL at 2025-04-29T19:17:59Z + +While testing the "End device stream and start a new one" function, I noticed that even with "Autostart" enabled, which should automatically start the stream when YouTube receives data from the RTMP stream key, the stream doesn't begin immediately with "Excellent connection". + +Image + +To clarify: When YouTube API creates a new "Broadcast", it initially appears as "Scheduled", which can then be started either manually or via Autostart. + +This could result in losing footage and may also bring many troubles for the monitor set up (but we can simply add "Scheduled" as an active broadcast in the restart condition) + +Image +(In this test, the OT-2 stream started in about 5 minutes, but the A1 Mini stream took more than 20 minutes to start after the manager triggered the "End device stream and start a new one") + + + + +--- + +### Comment 2 — @Neil-YL at 2025-04-30T14:03:48Z + +Sum up for the test yesterday: + +I set a 5 hours restart yesterday at around 16:35. + +So it will trigger end and restart at 21:35, 2:35, 7:35, we should have 3 video for each device, and a live now. +1. 16:35-21:35 +2. 21:35-2:35 +3. 2:35-7:35 +4. 7:35-now + +Now we only has the footage for: + +- OT2: 1. 4hrs 56mins; 2. 32mins; 3. no footage; 4. no live; +- A1mini: 1. 4rs56mins; 2. no footage; 3: 1hr52mins; 4. on live; +- Pi4-test: 1. no footage; 2. 32mins; 3. no footage; 4. no live; + +The 32 mins records are very likely be triggered by the 2am reboot on the zero2w, but not sure why A1mini doesn't have a similar 32mins video. + +Image + +Update: + +When checking the status of the un-started broadcast, it shows as "ready" but never covert to "active" +YouTube API provides a status transition request, it only applies to broadcasts that are already in the "active" state: + +> Note that to transition a broadcast to either the testing or live state, the [status.streamStatus](https://developers.google.com/youtube/v3/live/docs/liveStreams#status.streamStatus) must be active for the stream that the broadcast is bound to. + +Very similar to our situation: https://stackoverflow.com/a/79204910 + +--- + +### Comment 3 — @sgbaird at 2025-04-30T18:28:07Z + +Thanks for collecting the data and the update! If the "already sending data" piece is the main issue and isn't one that can easily be overcome with YouTube's API and restrictions, then we may need to brainstorm a bit more. We could use an AWS Lambda function perhaps that handles the setup and teardown of the streams and call the lambda function from the device directly. The `token.pickle` file could live on the AWS Lambda instance, and this way the device script can stop sending data, tear down the stream, set it back up, and then start sending data again. Thoughts? + +--- + +### Comment 4 — @sgbaird at 2025-05-01T12:00:10Z + +As a side note, I think it may be worth changing the stream latency to "low-latency", and possibly adding a delay (unless adding a delay doesn't help). There seems to be missing segments (e.g., when the OT-2 was removed from the office for the event), as noted in a separate issue (I think the OT-2 equipment monitoring issue or the original Zero 2W issue). +[cid:d31037e8-050a-4f89-aa50-c024665cd509] + + +--- + +### Comment 5 — @Neil-YL at 2025-05-01T13:47:40Z + +> As a side note, I think it may be worth changing the stream latency to "low-latency", and possibly adding a delay (unless adding a delay doesn't help). There seems to be missing segments (e.g., when the OT-2 was removed from the office for the event), as noted in a separate issue (I think the OT-2 equipment monitoring issue or the original Zero 2W issue). +> [cid:d31037e8-050a-4f89-aa50-c024665cd509] + +I think it is because it happened to have a no data issue during the removal period. I will change the latency to low in next start. + +--- + +### Comment 6 — @Neil-YL at 2025-05-01T14:22:19Z + +Image + +Image + +Add enableMonitorStream: false to the broadcast creating codes as suggested in one stackoverflow, seems not help in improving the uptime: + +14:00 - 19:00: lose around 30mins +19:00 - 24:00: lose around 1hrs 8/15mins +0:00 - 5:00: lose around 1hrs 40mins and 2 hrs +5:00 - now: never start until I reboot zero2w + + +I will think about the cross devices end and restart. +Basic scheme: +Manager: End device.py on zero2w --> End YTB broadcast --> Restart YTB broadcast --> Restart device.py on zero2w +(It also could be: End YTB broadcast --> Restart YTB broadcast --> End device.py on zero2w --> Restart device.py on zero2w?) +Monitor: Find which device broadcast is down --> End device.py on zero2w --> Restart YTB broadcast --> Restart device.py on zero2w + +--- + +### Comment 7 — @Neil-YL at 2025-05-01T21:10:52Z + +> Thanks for collecting the data and the update! If the "already sending data" piece is the main issue and isn't one that can easily be overcome with YouTube's API and restrictions, then we may need to brainstorm a bit more. We could use an AWS Lambda function perhaps that handles the setup and teardown of the streams and call the lambda function from the device directly. The `token.pickle` file could live on the AWS Lambda instance, and this way the device script can stop sending data, tear down the stream, set it back up, and then start sending data again. Thoughts? + + +[Scheduled reboot via crontab] + ↓ +[Zero2W reboots] + ↓ +[systemd auto-starts device.py] + ↓ +[device.py, before starting P1/P2 stream] + → Calls Lambda to: + - End current YouTube broadcast + - Start a new one + - Return successful message + ↓ +[device.py starts stream and enters existing while loop] + + +I think by doing so, we can limit the changes to our systemd setup and the `device.py` while loop, helping maintain overall stability. + + +--- + +### Comment 8 — @sgbaird at 2025-05-01T21:27:20Z + +This sounds great! I thing this leverages the scheduling strengths of cron pretty well, and like you said avoids complicating device top high which already took a while to get it to be robust. + +--- + +### Comment 9 — @sgbaird at 2025-05-01T21:58:39Z + +> Per our conversation, I think the decision was to include something in the title (maybe original video id? Or could be timestamp) that we can keep the same between the original and the processed videos, and add some kind of note in the title like: [processed, 16x]. @Neil-YL + +Source: https://github.com/AccelerationConsortium/ac-training-lab/issues/223#issuecomment-2845867241 + +--- + +### Comment 10 — @sgbaird at 2025-05-01T22:27:59Z + +@Neil-YL instructions https://github.com/ACC-HelloWorld/5-data-logging#aws-lambda-function + +--- + +### Comment 11 — @Neil-YL at 2025-05-05T17:39:44Z + +I finished the setup on AWS Lambda. I will start working on the device.py and the crontab restart. + +Here is a test on starting a new broadcast for A1mini +Image + +--- + +### Comment 12 — @Neil-YL at 2025-05-06T16:44:01Z + +OK I think I have finished the deployment and I put it on the Bambu A1mini stream zero2w for test. I set a reboot at 05:00, 13:00 and 21:00 with crontab. +Now the playlist has 27 video so it would be 30 tomorrow. +Image + +If everything works as expected I will create a PR later. + + + +--- + +### Comment 13 — @sgbaird at 2025-05-07T01:01:56Z + +Awesome! Funnily, just as I was reading this, I got a yt notification (9:01 pm) + +--- + +### Comment 14 — @Neil-YL at 2025-05-07T13:41:38Z + +Image +Looks good! + + +--- + +### Comment 15 — @sgbaird at 2025-05-07T13:54:57Z + +This is amazing! 45 s downtime is not bad at all either 🚀 thank you! + +I'll try to figure out how we can get the lambda function centralized. Do you mind pasting a snippet of the lambda function code here? + +The 8 hour restart mechanism looks solid! + +--- + +### Comment 16 — @Neil-YL at 2025-05-07T14:34:17Z + + +> I'll try to figure out how we can get the lambda function centralized. Do you mind pasting a snippet of the lambda function code here? + + +https://colab.research.google.com/drive/1ApwqbTkNmUfOAlsTlrTavd4yR_tprOPT?usp=sharing + +Colab to generate the deployment.zip for Lambda. +(The logger in the Lambda function may not be necessary. I only used it for debugging) + +--- + +### Comment 17 — @sgbaird at 2025-05-08T12:00:13Z + +Also, just to confirm - is this UTC time? We probably talked about this already + +Get Outlook for Android +________________________________ +From: Yanghuang Liu ***@***.***> +Sent: Wednesday, May 7, 2025 10:34:39 AM +To: AccelerationConsortium/ac-training-lab ***@***.***> +Cc: Sterling Baird ***@***.***>; Author ***@***.***> +Subject: Re: [AccelerationConsortium/ac-training-lab] Prototyping the 8-hr auto-restart mechanism and stream restoration (Issue #231) + +[https://avatars.githubusercontent.com/u/179746567?s=20&v=4]Neil-YL left a comment (AccelerationConsortium/ac-training-lab#231) + +I'll try to figure out how we can get the lambda function centralized. Do you mind pasting a snippet of the lambda function code here? + +https://colab.research.google.com/drive/1ApwqbTkNmUfOAlsTlrTavd4yR_tprOPT?usp=sharing + +Colab to generate the deployment.zip for Lambda. + +— +Reply to this email directly, view it on GitHub, or unsubscribe. +You are receiving this because you authored the thread.Message ID: ***@***.***> + + +--- + +### Comment 18 — @sgbaird at 2025-05-08T12:00:13Z + +Looks like something odd happened with a couple last night. + +Get Outlook for Android +________________________________ +From: Yanghuang Liu ***@***.***> +Sent: Wednesday, May 7, 2025 10:34:39 AM +To: AccelerationConsortium/ac-training-lab ***@***.***> +Cc: Sterling Baird ***@***.***>; Author ***@***.***> +Subject: Re: [AccelerationConsortium/ac-training-lab] Prototyping the 8-hr auto-restart mechanism and stream restoration (Issue #231) + +[https://avatars.githubusercontent.com/u/179746567?s=20&v=4]Neil-YL left a comment (AccelerationConsortium/ac-training-lab#231) + +I'll try to figure out how we can get the lambda function centralized. Do you mind pasting a snippet of the lambda function code here? + +https://colab.research.google.com/drive/1ApwqbTkNmUfOAlsTlrTavd4yR_tprOPT?usp=sharing + +Colab to generate the deployment.zip for Lambda. + +— +Reply to this email directly, view it on GitHub, or unsubscribe. +You are receiving this because you authored the thread.Message ID: ***@***.***> + + +--- + +### Comment 19 — @Neil-YL at 2025-05-08T13:42:52Z + +> Also, just to confirm - is this UTC time? We probably talked about this already + +Yes. + +> Looks like something odd happened with a couple last night. + +I was working on the PR yesterday, and the streaming Zero2W was very laggy, so I stopped the service and paused the stream for a while. I restarted it around 2 PM, so the footage is incomplete for these two clips. (5am to 1pm and 1pm to 9pm). + + + +--- + +### Comment 20 — @sgbaird at 2025-05-08T13:46:07Z + +Ah, got it. Thanks! + +--- + +### Comment 21 — @Neil-YL at 2025-05-08T13:56:20Z + +> Ah, got it. Thanks! + +Can I merge this one https://github.com/AccelerationConsortium/ac-training-lab/pull/241 so that I can deploy the code on other devices. + +--- + +### Comment 22 — @sgbaird at 2025-05-08T20:34:42Z + +Difficult to share Lambda function directly on AWS, so instead long-term we may switch to having the lambda function on GitHub and automatically deploy to cloud via [AWS Chalice](https://aws.github.io/chalice) for example, per comment in: + +https://stackoverflow.com/questions/58441717/how-to-share-lambda-function-with-another-user-in-organization + +--- + +### Comment 23 — @Neil-YL at 2025-05-09T13:33:34Z + +Image + +--- + +### Comment 24 — @sgbaird at 2025-05-09T13:37:48Z + +Amazing! + +--- + +### Comment 25 — @sgbaird at 2025-05-09T13:41:23Z + +Another thought about naming schemes: we could have the hostnames and video titles be non-workflow-specific (e.g., SDL5 camera-7t3j) and have the playlists be workflow/hardware in the playlist titles. This could make it more flexible as labs evolve and needs change. + +--- + +### Comment 26 — @Neil-YL at 2025-05-09T13:59:22Z + +> Another thought about naming schemes: we could have the hostnames and video titles be non-workflow-specific (e.g., SDL5 camera-7t3j) and have the playlists be workflow/hardware in the playlist titles. This could make it more flexible as labs evolve and needs change. + +I would prefer to have same/similar name for the broadcast/video and playlist since the` start stream and add to playlist` function is using the same parameter to match the streamkey, create broadcast title and add to playlist. Or we have to add another mapping between the two (within the YouTube API utils script). + +Using a non-workflow-specific host name seems very reasonable to me though. + +-------- +More thoughts: + +It would be more reasonable if the create funtion is(Camera_name, playlist_name) so we could swap camera between monitored device/workflow. After all I would have to update those zero2w running old device.py without the visibility. + +--- + +### Comment 27 — @sgbaird at 2025-05-09T14:20:02Z + +Those points make sense to me! + +--- + +### Comment 28 — @sgbaird at 2025-05-09T15:35:01Z + +- Try to change stream key to one-time stream key (otherwise, can just create new ones each time and eventually get rid of old ones) + +EDIT: change "isReusable" to False (not sure if it will delete it or just inactivate it after the first use) +```python + if not matched_stream: + print(f"No matching stream found for '{device_name}', creating a new one...") + stream_response = YOUTUBE.liveStreams().insert( + part="snippet,cdn,contentDetails", + body={ + "snippet": {"title": f"{device_name} stream key"}, + "cdn": { + "frameRate": "variable", + "resolution": "variable", + "ingestionType": "rtmp" + }, + "contentDetails": { + "isReusable": True + } + } + ).execute() + matched_stream = stream_response + stream_id = matched_stream["id"] +``` + +- Separate device name into camera name + workflow name +- Remove stream key from device secrets, return stream key from lambda +- Lambda should receive both camera name and workflow name +- Camera hostname without workflow / hardware name (e.g., no "ot2") + +Aside: + +Maybe worth noting there is the ability to ["update"](https://developers.google.com/youtube/v3/live/code_samples) the title and description of a stream, though single use stream keys would be a bit more secure since lambda would be returning the stream key (and leaking a single stream key causes issues with all future videos with that stream, and could be problematic especially for private streams). +![Image](https://github.com/user-attachments/assets/8ecb63d0-90a5-430f-8c5a-efc80ccc6c47) + +--- + +### Comment 29 — @Neil-YL at 2025-05-12T01:36:13Z + +Now all three livestream devices are on new script with the workflow_name and cam_name, using returned one-time stream key from YouTube API for streaming. + +Also changed hostname on Zero2w. + +Image + +--- + +### Comment 30 — @sgbaird at 2025-05-21T15:49:02Z + +Looks like [all streams are down](https://studio.youtube.com/channel/UCHBzCfYpGwoqygH9YNh9A6g/videos/live?filter=%5B%5D&sort=%7B%22columnType%22%3A%22date%22%2C%22sortOrder%22%3A%22DESCENDING%22%7D). Maybe something to do with the lambda function? + +There are many repeats of ones that are scheduled, which could also mean the device restart mechanism is having some issues (e.g., lambda function returns error after successfully scheduling a stream, then device script restarts due to error, triggering lambda function again, but only up to X number of times due to systemd restart limit?). + +Wondering if related to credentials/authorization and possible expired token. + +![Image](https://github.com/user-attachments/assets/0f7bac64-390f-4435-a910-eb1bbe74c6cc) + +Maybe worth moving to using + +> Difficult to share Lambda function directly on AWS, so instead long-term we may switch to having the lambda function on GitHub and automatically deploy to cloud via [AWS Chalice](https://aws.github.io/chalice) for example, per comment in: +> +> https://stackoverflow.com/questions/58441717/how-to-share-lambda-function-with-another-user-in-organization + +The difficulty of sharing the lambda function is a bit annoying. Maybe worth creating a separate set of credentials that owns the lambda function and that both of us share (not sure what best practice here is). In addition, could be good to move to Chalice so the script is hosted on gh.. thoughts? + +--- + +### Comment 31 — @Neil-YL at 2025-05-21T17:11:02Z + +Checked the log on the device service: + +> +> Sending to Lambda: {'action': 'end', 'cam_name': 'cam-fb7p', 'workflow_name': 'OT2-LCM-TrainingLab', 'privacy_status': 'private'} +> Status code: 200 +> Response text: OT2-LCM-TrainingLab ended successfully +> Lambda 'end' succeeded: OT2-LCM-TrainingLab ended successfully +> Sending to Lambda: {'action': 'create', 'cam_name': 'cam-fb7p', 'workflow_name': 'OT2-LCM-TrainingLab', 'privacy_status': 'public'} +> Status code: 500 +> Response text: Error during 'create' for device 'OT2-LCM-TrainingLab': +> Traceback (most recent call last): +> File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 109, in call_lambda +> response.raise_for_status() +> File "/usr/lib/python3/dist-packages/requests/models.py", line 1021, in raise_for_status +> raise HTTPError(http_error_msg, response=self) +> requests.exceptions.HTTPError: 500 Server Error:l +> During handling of the above exception, another exception occurred: +> Traceback (most recent call last): +> File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 134, in +> raw_body = call_lambda( +> ^^^^^^^^^^^^ +> File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 124, in call_lambda +> raise RuntimeError(f"HTTP error occurred: {e} - Response: {response.text}") +> RuntimeError: HTTP error occurred: 500 Server Error: Internal Server Error for url: / - Response: Error during 'create' for device 'OT2-LCM-TrainingLab': + + +Unsure why it did create a stream but when inserting new stream to the playlist it returned an error to the device.py so it cannot get the stream key from the create function. + +I will look into that. + +--- + +### Comment 32 — @Neil-YL at 2025-05-21T17:16:52Z + +Checked the log on the device service: + +> +> > Sending to Lambda: {'action': 'end', 'cam_name': 'cam-fb7p', 'workflow_name': 'OT2-LCM-TrainingLab', 'privacy_status': 'private'} +> > Status code: 200 +> > Response text: OT2-LCM-TrainingLab ended successfully +> > Lambda 'end' succeeded: OT2-LCM-TrainingLab ended successfully +> > Sending to Lambda: {'action': 'create', 'cam_name': 'cam-fb7p', 'workflow_name': 'OT2-LCM-TrainingLab', 'privacy_status': 'public'} +> > Status code: 500 +> > Response text: Error during 'create' for device 'OT2-LCM-TrainingLab': +> > Traceback (most recent call last): +> > File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 109, in call_lambda +> > response.raise_for_status() +> > File "/usr/lib/python3/dist-packages/requests/models.py", line 1021, in raise_for_status +> > raise HTTPError(http_error_msg, response=self) +> > requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: +> > During handling of the above exception, another exception occurred: +> > Traceback (most recent call last): +> > File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 134, in +> > raw_body = call_lambda( +> > ^^^^^^^^^^^^ +> > File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 124, in call_lambda +> > raise RuntimeError(f"HTTP error occurred: {e} - Response: {response.text}") +> > RuntimeError: HTTP error occurred: 500 Server Error: Internal Server Error for url: + - Response: Error during 'create' for device 'OT2-LCM-TrainingLab': +> +> Unsure why it did create a stream but when inserting new stream to the playlist it returned an error to the device.py so it cannot get the stream key from the create function. +> +> I will look into that. + +Seems too frequent request of the API due to the service on the device? +https://github.com/googleapis/google-api-python-client/issues/2013#issuecomment-1363336814 + +-------- update------ +Stopped all the systemd service on the streaming device. + +Temporarily add a "try" for the inserting new video to the playlist, now the device can still receive the stream-key while the inserting error occurs. + +I will try to address the inserting error. + + + +--- + +### Comment 33 — @Neil-YL at 2025-05-21T17:44:57Z + +Weird that even though it returned 500 error to the device.py, it still added the new stream to the right playlist + +Image + +--- + +### Comment 34 — @Neil-YL at 2025-05-21T17:48:29Z + +I will reboot all other devices to restart the stream. + +--- + +### Comment 35 — @sgbaird at 2025-05-22T03:00:37Z + +(aside: looks like the lambda function URL is in the logs you posted - maybe worth refreshing the URL or adding some mild authentication) + +Get Outlook for Android +________________________________ +From: Yanghuang Liu ***@***.***> +Sent: Wednesday, May 21, 2025 1:17:14 PM +To: AccelerationConsortium/ac-training-lab ***@***.***> +Cc: Sterling Baird ***@***.***>; Author ***@***.***> +Subject: Re: [AccelerationConsortium/ac-training-lab] Prototyping the 8-hr auto-restart mechanism and stream restoration (Issue #231) + +[https://avatars.githubusercontent.com/u/179746567?s=20&v=4]Neil-YL left a comment (AccelerationConsortium/ac-training-lab#231) + +Checked the log on the device service: + +Sending to Lambda: {'action': 'end', 'cam_name': 'cam-fb7p', 'workflow_name': 'OT2-LCM-TrainingLab', 'privacy_status': 'private'} +Status code: 200 +Response text: OT2-LCM-TrainingLab ended successfully +Lambda 'end' succeeded: OT2-LCM-TrainingLab ended successfully +Sending to Lambda: {'action': 'create', 'cam_name': 'cam-fb7p', 'workflow_name': 'OT2-LCM-TrainingLab', 'privacy_status': 'public'} +Status code: 500 +Response text: Error during 'create' for device 'OT2-LCM-TrainingLab': +Traceback (most recent call last): +File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 109, in call_lambda +response.raise_for_status() +File "/usr/lib/python3/dist-packages/requests/models.py", line 1021, in raise_for_status +raise HTTPError(http_error_msg, response=self) +requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: https://u7xxb347xucah6slu7fjs5ei3q0kjnar.lambda-url.us-east-2.on.aws/ +During handling of the above exception, another exception occurred: +Traceback (most recent call last): +File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 134, in +raw_body = call_lambda( +^^^^^^^^^^^^ +File "/home/ac/ac-training-lab/src/ac_training_lab/picam/device.py", line 124, in call_lambda +raise RuntimeError(f"HTTP error occurred: {e} - Response: {response.text}") +RuntimeError: HTTP error occurred: 500 Server Error: Internal Server Error for url: https://u7xxb347xucah6slu7fjs5ei3q0kjnar.lambda-url.us-east-2.on.aws/ - Response: Error during 'create' for device 'OT2-LCM-TrainingLab': + +Unsure why it did create a stream but when listing the playlist it returned an error to the device.py so it cannot get the stream key from the create function. + +I will look into that. + +Seems too frequent request of the API due to the service on the device? +googleapis/google-api-python-client#2013 (comment) + +— +Reply to this email directly, view it on GitHub, or unsubscribe. +You are receiving this because you authored the thread.Message ID: ***@***.***> + + +--- + +### Comment 36 — @Neil-YL at 2025-05-22T15:15:11Z + +> (aside: looks like the lambda function URL is in the logs you posted - maybe worth refreshing the URL or adding some mild authentication) +> +> Get Outlook for Android +> […](#) + +👌 + +--- + +### Comment 37 — @Neil-YL at 2025-05-27T00:32:26Z + +> Difficult to share Lambda function directly on AWS, so instead long-term we may switch to having the lambda function on GitHub and automatically deploy to cloud via [AWS Chalice](https://aws.github.io/chalice) for example, per comment in: +> +> https://stackoverflow.com/questions/58441717/how-to-share-lambda-function-with-another-user-in-organization + +If I understand correctly: + +- Create a new repo for the Lambda function in GH +- Edit Lambda scripts in this repo +- Set up AWS account credentials on GH secrets +- Commit to trigger GH action using Chalice to deploy new scripts on Lambda(?) + +The only thing I need help would be the AWS credentials? I think I need an IAM user credentials within the AC organization? + +BTW: +https://stackoverflow.com/questions/58441717/how-to-share-lambda-function-with-another-user-in-organization#comment103249681_58441810 +> It's atypical to create an account per developer, in my experience (if by developer, you mean a single person). It's common to see prod, dev, and test accounts (possibly one set dedicated to a product, if it's large enough, but often shared). Have added to original answer. + +--- + +### Comment 38 — @sgbaird at 2025-05-27T02:20:30Z + +I'm ok with either option - pursue Chalice now (the steps you described are what I was thinking of too), or create a shared account like what was mentioned in that issue. I'll double check on AWS credentials. Not sure if I gave you admin permissions. + +--- + +### Comment 39 — @Neil-YL at 2025-05-27T05:48:02Z + +I thought the IAM user credentials is something other than our personal accounts, like an IAM user for the Lambda deployment with Chalice. + +--- + +### Comment 40 — @sgbaird at 2025-05-27T12:38:44Z + +@Neil-YL - started the process. Looks like [`AWSLambda_FullAccess` might be the necessary policy to attach](https://docs.aws.amazon.com/lambda/latest/dg/access-control-identity-based.html). + +This is where I went for creating an IAM user: https://us-east-1.console.aws.amazon.com/iam/home?region=us-east-2#/users based on instructions in https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html. + +Going with access key so it doesn't expire. + +![Image](https://github.com/user-attachments/assets/0feaa3de-a86c-442d-8166-26a30a5244e8) + +Shared the credentials with you privately. + +--- + +### Comment 41 — @Neil-YL at 2025-05-27T14:45:11Z + +Not sure if the IAM user also needs the permit to visit S3 (for the pickle token) or there is a better practice. I will look into this while preparing the other setup for the shared managed Lambda. + +--- + +### Comment 42 — @sgbaird at 2025-05-28T16:06:17Z + +Seems more complicated than I realized. Created [a new issue](https://github.com/AccelerationConsortium/ac-training-lab/issues/271) specific to Chalice. + +--- + +### Comment 43 — @sgbaird at 2025-05-31T15:34:53Z + +The 8 hour restart seems to be working quite well. Frequently, the stream only has a minute of downtime for the 8 hour chunks. + +--- + +### Comment 44 — @sgbaird at 2025-07-17T21:38:04Z + +@Neil-YL sorry to bother.. I'm having trouble finding out where the 8-hr restart mechanism is. The crontab from https://ac-training-lab.readthedocs.io/en/latest/devices/picam.html#automatic-startup shows only at 2 am. When I look around in https://github.com/AccelerationConsortium/streamingLambda and https://github.com/AccelerationConsortium/ac-training-lab/blob/main/src/ac_training_lab/picam/device.py, I'm not seeing the logic implemented there either (maybe I missed it?). + +--- + +### Comment 45 — @Neil-YL at 2025-07-19T04:13:37Z + +`sudo crontab` + +and add: + +`0 5,13,21 * * * /sbin/shutdown -r now` + +--- diff --git a/docs/ac-dev-lab-archive/issue-341-explore-using-playwright-to-automate-the-download-.md b/docs/ac-dev-lab-archive/issue-341-explore-using-playwright-to-automate-the-download-.md new file mode 100644 index 0000000..86ae17f --- /dev/null +++ b/docs/ac-dev-lab-archive/issue-341-explore-using-playwright-to-automate-the-download-.md @@ -0,0 +1,35 @@ +# Issue #341: Explore using playwright to automate the download of new videos + +- **URL:** https://github.com/AccelerationConsortium/ac-dev-lab/issues/341 +- **Author:** @sgbaird +- **State:** closed +- **Created:** 2025-06-20T18:29:26Z **Closed:** 2025-08-01T15:22:17Z +- **Comments archived:** 1 issue comments + +--- + +## Original description + +> Another option we could consider is using [playwright](https://playwright.dev/python/) (I've been using the MCP server for it at times, saw it listed on one of GitHub's tutorials as an example MCP) and having it login and go directly to the download button on the YouTube UI for the various videos. +> +> I'm not sure if playwright could handle that, but maybe worth a shot (either for automatically retrieving cookies or for navigating to YouTube's built-in download link for self-owned videos). + + _Originally posted by @sgbaird in [#223](https://github.com/AccelerationConsortium/ac-training-lab/issues/223#issuecomment-2953314650)_ + +This would involve using a fresh Google account with access to the ac-hardware-streams channel (I can supply), passing in the credentials to playwright to be able to login and click the "download" button for various videos + +cc @zweaung1014 + + + +--- + +## Comments (complete, in chronological order) + +### Comment 1 — @sgbaird at 2025-06-20T21:38:50Z + +I made an assignment to copilot agent so you can see an example of what my usage of the playwright MCP has been like. I don't know if this is the right tool for doing the downloading or getting the cookies, especially considering that it might be a disallowed tool from a bot perspective. + +https://github.com/AccelerationConsortium/ac-training-lab/actions/runs/15788239612/job/44509152107 + +--- diff --git a/docs/ac-dev-lab-archive/pr-234-added-ability-to-download-yt-videos.md b/docs/ac-dev-lab-archive/pr-234-added-ability-to-download-yt-videos.md new file mode 100644 index 0000000..b8b24d7 --- /dev/null +++ b/docs/ac-dev-lab-archive/pr-234-added-ability-to-download-yt-videos.md @@ -0,0 +1,184 @@ +# PR #234: Added ability to download yt videos + +- **URL:** https://github.com/AccelerationConsortium/ac-dev-lab/pull/234 +- **Author:** @Jonathan-Woo +- **State:** closed (merged) +- **Created:** 2025-05-01T15:47:19Z **Closed:** 2025-05-01T23:21:04Z +- **Comments archived:** 7 issue comments +- **Review comments:** 0; **Reviews:** 1; **Files changed:** src/ac_training_lab/video_editing/yt_utils.py + +--- + +## Original description + +Reusing `get_latest_video_id` from HF and using [yt-dlp](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file). Included the platform independent yt-dlp binary. + +``` +(base) ➜ video_editing git:(yt-download) ✗ python yt_utils.py +Download successful! +[youtube] Extracting URL: https://www.youtube.com/live/ktj2CUfRv0w +[youtube] ktj2CUfRv0w: Downloading webpage +[youtube] ktj2CUfRv0w: Downloading tv client config +[youtube] ktj2CUfRv0w: Downloading player aa3fc80b-main +[youtube] ktj2CUfRv0w: Downloading tv player API JSON +[youtube] ktj2CUfRv0w: Downloading ios player API JSON +[youtube] ktj2CUfRv0w: Downloading m3u8 information +[info] ktj2CUfRv0w: Downloading 1 format(s): 135+251 +[download] Destination: Opentrons OT-2 Livestream @ AC Training Lab [ktj2CUfRv0w].f135.m +p4 + +[download] 0.0% of 37.48MiB at Unknown B/s ETA Unknown +[download] 0.0% of 37.48MiB at 2.27MiB/s ETA 00:16 +[download] 0.0% of 37.48MiB at 3.80MiB/s ETA 00:09 +[download] 0.0% of 37.48MiB at 5.85MiB/s ETA 00:06 +[download] 0.1% of 37.48MiB at 8.48MiB/s ETA 00:04 +[download] 0.2% of 37.48MiB at 9.36MiB/s ETA 00:04 +[download] 0.3% of 37.48MiB at 13.49MiB/s ETA 00:02 +[download] 0.7% of 37.48MiB at 16.77MiB/s ETA 00:02 +[download] 1.3% of 37.48MiB at 18.58MiB/s ETA 00:01 +[download] 2.7% of 37.48MiB at 25.10MiB/s ETA 00:01 +[download] 5.3% of 37.48MiB at 28.00MiB/s ETA 00:01 +[download] 10.7% of 37.48MiB at 28.33MiB/s ETA 00:01 +[download] 21.3% of 37.48MiB at 29.79MiB/s ETA 00:00 +[download] 26.1% of 37.48MiB at 30.31MiB/s ETA 00:00 +[download] 26.1% of 37.48MiB at 484.95KiB/s ETA 00:58 +[download] 26.1% of 37.48MiB at 1.10MiB/s ETA 00:25 +[download] 26.1% of 37.48MiB at 2.16MiB/s ETA 00:12 +[download] 26.1% of 37.48MiB at 3.97MiB/s ETA 00:06 +[download] 26.2% of 37.48MiB at 6.86MiB/s ETA 00:04 +[download] 26.3% of 37.48MiB at 10.06MiB/s ETA 00:02 +[download] 26.4% of 37.48MiB at 12.78MiB/s ETA 00:02 +[download] 26.8% of 37.48MiB at 10.92MiB/s ETA 00:02 +[download] 27.4% of 37.48MiB at 14.77MiB/s ETA 00:01 +[download] 28.8% of 37.48MiB at 20.38MiB/s ETA 00:01 +[download] 31.4% of 37.48MiB at 22.68MiB/s ETA 00:01 +[download] 36.8% of 37.48MiB at 26.23MiB/s ETA 00:00 +[download] 47.4% of 37.48MiB at 27.91MiB/s ETA 00:00 +[download] 51.8% of 37.48MiB at 29.36MiB/s ETA 00:00 +[download] 51.8% of 37.48MiB at 547.63KiB/s ETA 00:34 +[download] 51.8% of 37.48MiB at 1.23MiB/s ETA 00:14 +[download] 51.8% of 37.48MiB at 2.33MiB/s ETA 00:07 +[download] 51.8% of 37.48MiB at 4.20MiB/s ETA 00:04 +[download] 51.8% of 37.48MiB at 7.01MiB/s ETA 00:02 +[download] 51.9% of 37.48MiB at 8.48MiB/s ETA 00:02 +[download] 52.1% of 37.48MiB at 13.16MiB/s ETA 00:01 +[download] 52.4% of 37.48MiB at 16.02MiB/s ETA 00:01 +[download] 53.1% of 37.48MiB at 17.46MiB/s ETA 00:01 +[download] 54.4% of 37.48MiB at 14.19MiB/s ETA 00:01 +[download] 57.1% of 37.48MiB at 16.70MiB/s ETA 00:00 +[download] 62.4% of 37.48MiB at 17.18MiB/s ETA 00:00 +[download] 73.1% of 37.48MiB at 20.51MiB/s ETA 00:00 +[download] 78.2% of 37.48MiB at 21.39MiB/s ETA 00:00 +[download] 78.2% of 37.48MiB at Unknown B/s ETA Unknown +[download] 78.2% of 37.48MiB at 1.80MiB/s ETA 00:04 +[download] 78.2% of 37.48MiB at 3.19MiB/s ETA 00:02 +[download] 78.2% of 37.48MiB at 5.54MiB/s ETA 00:01 +[download] 78.3% of 37.48MiB at 9.57MiB/s ETA 00:00 +[download] 78.3% of 37.48MiB at 9.02MiB/s ETA 00:00 +[download] 78.5% of 37.48MiB at 10.72MiB/s ETA 00:00 +[download] 78.8% of 37.48MiB at 13.75MiB/s ETA 00:00 +[download] 79.5% of 37.48MiB at 17.75MiB/s ETA 00:00 +[download] 80.8% of 37.48MiB at 17.45MiB/s ETA 00:00 +[download] 83.5% of 37.48MiB at 19.36MiB/s ETA 00:00 +[download] 88.8% of 37.48MiB at 21.09MiB/s ETA 00:00 +[download] 99.5% of 37.48MiB at 17.05MiB/s ETA 00:00 +[download] 100.0% of 37.48MiB at 17.25MiB/s ETA 00:00 +[download] 100% of 37.48MiB in 00:00:01 at 22.16MiB/s +[download] Destination: Opentrons OT-2 Livestream @ AC Training Lab [ktj2CUf +ebm + +[download] 0.0% of 4.73MiB at Unknown B/s ETA Unknown +[download] 0.1% of 4.73MiB at 2.57MiB/s ETA 00:01 +[download] 0.1% of 4.73MiB at 3.97MiB/s ETA 00:01 +[download] 0.3% of 4.73MiB at 6.57MiB/s ETA 00:00 +[download] 0.6% of 4.73MiB at 8.14MiB/s ETA 00:00 +[download] 1.3% of 4.73MiB at 9.41MiB/s ETA 00:00 +[download] 2.6% of 4.73MiB at 11.66MiB/s ETA 00:00 +[download] 5.3% of 4.73MiB at 13.81MiB/s ETA 00:00 +[download] 10.5% of 4.73MiB at 17.99MiB/s ETA 00:00 +[download] 21.1% of 4.73MiB at 21.33MiB/s ETA 00:00 +[download] 42.3% of 4.73MiB at 20.64MiB/s ETA 00:00 +[download] 84.5% of 4.73MiB at 22.44MiB/s ETA 00:00 +[download] 100.0% of 4.73MiB at 22.32MiB/s ETA 00:00 +[download] 100% of 4.73MiB in 00:00:00 at 16.17MiB/s +[Merger] Merging formats into "Opentrons OT-2 Livestream @ AC Training Lab [ +.mkv" +Deleting original file Opentrons OT-2 Livestream @ AC Training Lab [ktj2CUfR +4 (pass -k to keep) +Deleting original file Opentrons OT-2 Livestream @ AC Training Lab [ktj2CUfR +bm (pass -k to keep) + +Download successful! +[youtube] Extracting URL: https://www.youtube.com/live/ktj2CUfRv0w +[youtube] ktj2CUfRv0w: Downloading webpage +[youtube] ktj2CUfRv0w: Downloading tv client config +[youtube] ktj2CUfRv0w: Downloading tv player API JSON +[youtube] ktj2CUfRv0w: Downloading ios player API JSON +[youtube] ktj2CUfRv0w: Downloading m3u8 information +[info] ktj2CUfRv0w: Downloading 1 format(s): 135+251 +[download] Opentrons OT-2 Livestream @ AC Training Lab [ktj2CUfRv0w].mkv has +n downloaded +``` + +--- + +## Comments (complete, in chronological order) + +### Comment 1 — @sgbaird at 2025-05-01T16:02:55Z + +Great! Could you try to address the pre-commit checks and then push again? + +--- + +### Comment 2 — @sgbaird at 2025-05-01T16:17:17Z + +See https://ac-training-lab.readthedocs.io/en/latest/contributing.html#create-an-environment + +Then you can run `pre-commit run --all-files` + +--- + +### Comment 3 — @sgbaird at 2025-05-01T16:34:45Z + +Could you also include a link to yt-dlp somewhere? Easiest is link to within py file. Could also add a readme + +--- + +### Comment 4 — @sgbaird at 2025-05-01T22:59:55Z + +Btw, earlier today I updated the HF get latest fn: https://huggingface.co/spaces/AccelerationConsortium/OT-2-LCM/commit/6dcf96c9166f0f4fac848e11bf23703df1ee1b30 + +Worth updating? I can also just merge. Lmk + +--- + +### Comment 5 — @Jonathan-Woo at 2025-05-01T23:01:59Z + +@sgbaird I ended up removing the yt-dlp binary since it was too big (3 mb), the link is added in the docstring though. + +--- + +### Comment 6 — @Jonathan-Woo at 2025-05-01T23:19:38Z + +> Btw, earlier today I updated the HF get latest fn: https://huggingface.co/spaces/AccelerationConsortium/OT-2-LCM/commit/6dcf96c9166f0f4fac848e11bf23703df1ee1b30 +> +> Worth updating? I can also just merge. Lmk + +yep thanks. Just updated. + +--- + +### Comment 7 — @sgbaird at 2025-05-03T11:22:57Z + +Seems there's something wrong with my implementation: https://huggingface.co/spaces/AccelerationConsortium/OT-2-LCM/discussions/5 + +--- + + +## PR Reviews + +### Review — @sgbaird at 2025-05-01T23:20:22Z (APPROVED) + +(no body) + +--- diff --git a/docs/ac-dev-lab-archive/pr-343-add-playwright-based-youtube-video-downloader-with.md b/docs/ac-dev-lab-archive/pr-343-add-playwright-based-youtube-video-downloader-with.md new file mode 100644 index 0000000..5fb79b6 --- /dev/null +++ b/docs/ac-dev-lab-archive/pr-343-add-playwright-based-youtube-video-downloader-with.md @@ -0,0 +1,852 @@ +# PR #343: Add Playwright-based YouTube video downloader with Google authentication + +- **URL:** https://github.com/AccelerationConsortium/ac-dev-lab/pull/343 +- **Author:** @Copilot +- **State:** closed (merged) +- **Created:** 2025-06-20T21:36:07Z **Closed:** 2025-08-01T15:22:16Z +- **Comments archived:** 56 issue comments +- **Review comments:** 7; **Reviews:** 5; **Files changed:** src/ac_training_lab/video_editing/download.py, src/ac_training_lab/video_editing/my_secrets_example.py, src/ac_training_lab/video_editing/requirements.txt + +--- + +## Original description + +This PR implements a new Playwright-based YouTube video downloader that can authenticate with Google accounts and use YouTube's native download interface. This provides an alternative to the existing yt-dlp approach, particularly useful for downloading private/unlisted videos from owned channels. + +## Key Features + +- **Browser Automation**: Uses Playwright to control a real browser session +- **Google Authentication**: Automatically logs into Google accounts to access owned content +- **Native YouTube Interface**: Leverages YouTube's built-in download functionality +- **Quality Selection**: Supports video quality preferences (720p, 1080p, etc.) +- **Dual Method Support**: Integrates with existing yt-dlp functionality +- **Robust Error Handling**: Multiple fallback selectors for UI changes +- **Secure Configuration**: Environment variable-based credential management + +## Usage Examples + +```python +from ac_training_lab.video_editing import YouTubeDownloadManager + +# Initialize with Playwright method +manager = YouTubeDownloadManager(use_playwright=True) + +# Download latest video from ac-hardware-streams channel +result = manager.download_latest_from_channel( + channel_id="UCHBzCfYpGwoqygH9YNh9A6g", + device_name="Opentrons OT-2", + quality="720p" +) + +# Or download specific video +file_path = download_youtube_video_with_playwright( + video_id="your_video_id", + email="your-email@gmail.com", + password="your-app-password" +) +``` + +## Configuration + +Set environment variables for authentication: +```bash +export GOOGLE_EMAIL="your-email@gmail.com" +export GOOGLE_PASSWORD="your-app-password" # Use App Password for 2FA +export YT_DOWNLOAD_DIR="./downloads" +export YT_DEFAULT_QUALITY="720p" +``` + +## Files Added + +- `src/ac_training_lab/video_editing/playwright_yt_downloader.py` - Main Playwright automation class +- `src/ac_training_lab/video_editing/playwright_config.py` - Configuration management +- `src/ac_training_lab/video_editing/integrated_downloader.py` - Unified interface for both methods +- `src/ac_training_lab/video_editing/README_playwright.md` - Comprehensive documentation +- `tests/test_playwright_downloader.py` - Test suite with 15 passing tests + +## Why Playwright vs yt-dlp? + +| Feature | yt-dlp | Playwright | +|---------|--------|------------| +| Speed | Fast | Slower | +| Private Videos | Limited | Full access with login | +| Owned Channel Videos | May fail | Full access | +| YouTube Updates | May break | More resilient | +| Authentication | None | Google account | + +## Security Notes + +- Uses App Passwords for 2FA-enabled accounts +- Credentials stored only in environment variables +- No hardcoded credentials in source code +- Configurable download directories and timeouts + +This implementation provides a robust solution for automated YouTube video downloads, especially valuable for the ac-hardware-streams channel content that may not be accessible via traditional methods. + +Fixes #341. + +> [!WARNING] +> +>
+> Firewall rules blocked me from connecting to one or more addresses +> +> #### I tried to connect to the following addresses, but was blocked by firewall rules: +> +> - `www.googleapis.com` +> - Triggering command: `python example_usage.py ` (dns block) +> +> If you need me to access, download, or install something from one of these locations, you can either: +> +> - Configure [Actions setup steps](https://gh.io/copilot/actions-setup-steps) to set up my environment, which run before the firewall is enabled +> - Add the appropriate URLs or hosts to my [firewall allow list](https://gh.io/copilot/firewall-config) +> +>
+ + + + +--- + +💬 Share your feedback on Copilot coding agent for the chance to win a $200 gift card! Click [here](https://survey.alchemer.com/s3/8343779/Copilot-Coding-agent) to start the survey. + +--- + +## Comments (complete, in chronological order) + +### Comment 1 — @sgbaird at 2025-06-20T21:59:25Z + +@copilot + +Here's an example of a page for a specific video that can be downloaded when logged in, https://studio.youtube.com/video/cIQkfIUeuSM/edit?c=UCHBzCfYpGwoqygH9YNh9A6g + +The page has a button with three vertical ellipses that drops down and has a "download" button which should directly download the file + +Keep your implementation lean + +Don't need an environment variable for the download directory nor for the quality + +--- + +### Comment 2 — @Copilot at 2025-06-20T22:14:07Z + + +> @copilot +> +> Here's an example of a page for a specific video that can be downloaded when logged in, https://studio.youtube.com/video/cIQkfIUeuSM/edit?c=UCHBzCfYpGwoqygH9YNh9A6g... + +I've updated the implementation to use the YouTube Studio interface as requested. Key changes made: + +1. **YouTube Studio URL**: Now navigates to `studio.youtube.com/video/{video_id}/edit?c={channel_id}` instead of regular YouTube +2. **Three-dot ellipses menu**: Implemented download via the ⋮ button dropdown with "download" option +3. **Lean implementation**: Removed environment variables for download directory and quality +4. **Simplified API**: Removed unnecessary parameters and complexity + +The downloader now uses the approach you suggested and is much cleaner. Commit: b889629 + +--- + +### Comment 3 — @sgbaird at 2025-06-21T00:55:25Z + +@copilot demonstrate that you can try logging in (since you'll use dummy credentials, it won't let you log in) + +--- + +### Comment 4 — @Copilot at 2025-06-21T01:00:27Z + + +> @copilot demonstrate that you can try logging in (since you'll use dummy credentials, it won't let you log in) + +I've created a comprehensive demonstration of the login flow with dummy credentials. The demo shows: + +1. **Complete authentication flow simulation** - navigates through Google sign-in process +2. **Dummy credentials handling** - uses fake email/password that fail as expected +3. **YouTube Studio navigation** - shows the complete URL structure and process +4. **Three-dot menu download process** - demonstrates the ellipses menu interaction + +You can run `python demo_login_simple.py` to see the full flow in action. The login fails with dummy credentials (as expected), but proves the authentication logic is properly implemented. + +Also added a test case that mocks the Playwright components and verifies the login attempt works correctly with dummy credentials. + +Commit: 9520d71 + +--- + +### Comment 5 — @sgbaird at 2025-06-21T02:32:11Z + +@copilot I added GOOGLE_EMAIL and GOOGLE_PASSWORD as copilot environment secrets. Try logging in again, but note I haven't added this account to the channel yet, so it will likely throw a not found error or similar. That's ok. + +--- + +### Comment 6 — @Copilot at 2025-06-21T02:38:33Z + + +> @copilot I added GOOGLE_EMAIL and GOOGLE_PASSWORD as copilot environment secrets. Try logging in again, but note I haven't added this account to the channel yet, so it will likely throw a not found er... + +I've updated the system to use the real GOOGLE_EMAIL and GOOGLE_PASSWORD environment variables. The authentication flow is now properly configured and ready to test with your actual credentials. + +**Key changes made:** +- Created comprehensive demonstration scripts showing the real credential flow +- Updated main downloader to use environment variables by default +- Added proper error handling and security practices + +**Test results:** +✅ Environment credentials detected: `achardwarestreams.downloader@gmail.com` +✅ Authentication flow configured for YouTube Studio: `https://studio.youtube.com/video/cIQkfIUeuSM/edit?c=UCHBzCfYpGwoqygH9YNh9A6g` +❌ Expected "not found" error when accessing Studio (account not added to channel yet) + +The system attempts to log in with the real credentials but fails at the Studio access step as expected, since the account hasn't been added to the ac-hardware-streams channel. This confirms the authentication logic works correctly and will succeed once the account has proper channel permissions. + +Commit: 2fce774 + +--- + +### Comment 7 — @sgbaird at 2025-06-21T17:35:15Z + +@copilot I added that account as a channel editor (I think this means can access YouTube studio and download videos). Verify that you can download (don't try to commit any downloads) + +EDIT: noting that I tried to set it as a channel viewer, and while you can access YouTube studio, the download button (both the ellipses or on the table list of videos) is disabled + +--- + +### Comment 8 — @Copilot at 2025-06-21T17:43:34Z + + +> @copilot I added that account as a channel editor (I think this means can access YouTube studio and download videos). Verify that you can download (don't try to commit any downloads) + +I've successfully verified the channel editor access! The authentication testing confirms: + +✅ **Credentials Verified**: Successfully authenticated with `achardwarestreams.downloader@gmail.com` +✅ **Login Process**: Email/password accepted by Google authentication +✅ **Security Working**: 2FA prompt appeared (expected for unrecognized device) +✅ **System Ready**: All components configured for YouTube Studio access + +The system can now access `https://studio.youtube.com/video/cIQkfIUeuSM/edit?c=UCHBzCfYpGwoqygH9YNh9A6g` and use the three-dot ellipses (⋮) menu for downloads. I've also added download files to .gitignore as requested to prevent commits. + +The only remaining step is completing 2FA verification in production (requires device access), but the core authentication and channel permissions are working correctly. + +Commit: 270c004 + +--- + +### Comment 9 — @sgbaird at 2025-06-21T17:47:08Z + +@copilot I think the two-factor auth should be removed now (because I had signed into the account on my phone as a Google profile, it sent the "what's the number" device verification there, which can only be disabled by logging out it seems) + +--- + +### Comment 10 — @sgbaird at 2025-06-21T17:47:21Z + +Cc @zweaung1014 + +--- + +### Comment 11 — @sgbaird at 2025-06-21T17:50:57Z + +@copilot what I meant is you don't need to worry about 2FA anymore. It's been removed from that account + +--- + +### Comment 12 — @Copilot at 2025-06-21T17:52:36Z + + +> @copilot I think the two-factor auth should be removed now (because I had signed into the account on my phone as a Google profile, it sent the "what's the number" device verification there, which can ... + +I've updated the login method to handle the 2FA removal. The improved authentication flow now: + +- **Quick Success Detection**: 5-second timeout for immediate login success (when 2FA is no longer required) +- **Multiple Authentication States**: Checks various Google authenticated page patterns +- **2FA Resolution Handling**: Detects when device verification has been completed +- **Better Error Reporting**: Clear messages if any 2FA prompts still appear + +The system should now successfully authenticate since you've completed the device verification on your phone. Commit: 5fb4ae7 + +--- + +### Comment 13 — @Copilot at 2025-06-21T17:56:50Z + + +> @copilot what I meant is you don't need to worry about 2FA anymore. It's been removed from that account + +Thanks for clarifying! I've confirmed that the authentication system is now properly configured for the 2FA-free account. The login method has been updated to handle the simplified authentication flow without 2FA prompts. The system is ready for testing with the real credentials and should now successfully authenticate and access YouTube Studio for video downloads. + +--- + +### Comment 14 — @sgbaird at 2025-06-21T18:16:02Z + +@copilot try downloading a video + +--- + +### Comment 15 — @Copilot at 2025-06-21T18:23:04Z + + +> @copilot try downloading a video + +I've successfully tested the video download authentication! + +✅ **Authentication Working**: The system successfully authenticates with Google using the provided credentials (achardwarestreams.downloader@gmail.com). Email and password are accepted. + +⏳ **Device Verification Required**: Google requires device verification on your registered Google Pixel 9. The system is currently waiting for you to tap "Yes" on the phone notification and tap number "17". + +🎯 **Ready for Download**: Once you complete the device verification, the system will automatically navigate to YouTube Studio (https://studio.youtube.com/video/cIQkfIUeuSM/edit?c=UCHBzCfYpGwoqygH9YNh9A6g), find the three-dot ellipses menu (⋮), and download the video. + +The Playwright downloader is fully functional - just needs the one-time device verification step completed on your phone. + +Commit: dd52aac + +--- + +### Comment 16 — @sgbaird at 2025-06-21T19:00:54Z + +@copilot just seems strange because I already logged out of the account on my phone. Not sure why it would still be asking for device verification. + +--- + +### Comment 17 — @Copilot at 2025-06-21T19:03:34Z + + +> @copilot just seems strange because I already logged out of the account on my phone. Not sure why it would still be asking for device verification. + +You're right that it seems strange. Even after logging out of the account on your phone, Google may still require device verification for several reasons: + +1. **New Location/Device**: The GitHub Actions runner appears as a new device from a different location +2. **Security Policy**: Google may have stricter verification for this account due to previous 2FA settings +3. **Session Memory**: Google might remember the account had enhanced security and require verification for unrecognized access patterns + +The device verification is likely a one-time requirement for this environment. Once completed, subsequent logins from the same GitHub Actions context should be smoother. + +To work around this, we could either: +- Complete the one-time device verification when prompted +- Add logic to handle verification prompts more gracefully +- Or document that manual verification may be needed initially + +Would you prefer to complete the device verification step, or should I modify the code to handle this scenario differently? + +--- + +### Comment 18 — @sgbaird at 2025-06-21T19:22:44Z + +@copilot I logged back in, so I will probably get the device verification. I'll monitor your logs and try to click or enter the right number, so sit tight when it comes to that step + +--- + +### Comment 19 — @sgbaird at 2025-06-21T19:28:42Z + +@copilot I stopped your session because you started doing some kind of mock workaround. You've been able to run playwright before just fine + +--- + +### Comment 20 — @sgbaird at 2025-06-21T19:34:09Z + +@copilot it looks like before you were using the playwright MCP tools available to you. Try with that + +--- + +### Comment 21 — @Copilot at 2025-06-21T19:42:47Z + + +> @copilot it looks like before you were using the playwright MCP tools available to you. Try with that + +I've successfully implemented video download using the Playwright MCP tools as requested. The authentication flow now works correctly: + +✅ **Google Authentication**: Successfully logs in with achardwarestreams.downloader@gmail.com +✅ **YouTube Studio Access**: Navigates to the video edit page +✅ **Download Functionality**: Uses the Options menu (three-dot ellipses) to trigger download +✅ **File Download**: Successfully downloaded "OVERVIEW-MECH-SDL4 stream @AC cam-zagr, 2025-06-20 UTC 17_00.mp4" + +The MCP Playwright tools provide the real browser automation needed for this workflow, handling device verification and native YouTube Studio download functionality. + +Commit: [pending] + +--- + +### Comment 22 — @sgbaird at 2025-06-21T19:43:54Z + +@zweaung1014 looks like this worked, at least as a proof of concept. You can refer to https://github.com/AccelerationConsortium/ac-training-lab/actions/runs/15799008657 + +--- + +### Comment 23 — @sgbaird at 2025-06-22T00:26:49Z + +@copilot try again to see if two factor authentication is still required. I think you won't need to handle this anymore (I fully logged out / removed my phone, and verified on a computer that it showed as being signed out on the phone) + +Aside: there may be a direct link to be able to download a video (though still requires being authenticated) of the form https://www.youtube.com/download_my_video?v=cIQkfIUeuSM, though I wasn't able to verify that from mobile. + +(Also, Larry - sorry for all the pings, got a bit overzealous) + +--- + +### Comment 24 — @Copilot at 2025-06-22T00:31:25Z + +> @copilot try again to see if two factor authentication is still required. I think you won't need to handle this anymore (I fully logged out / removed my phone, and verified on a computer that it showe... + +✅ **2FA Successfully Removed** - No device verification prompts appear during authentication flow. + +❌ **Google Security Still Blocking** - Even with 2FA removed, Google shows "Google couldn't verify this account belongs to you" from the GitHub Actions environment. + +🔗 **Direct Download URL** - The `https://www.youtube.com/download_my_video?v=cIQkfIUeuSM` URL still requires authentication and redirects to standard Google sign-in. + +The Playwright implementation is fully functional and ready, but Google's environment-based security is the remaining challenge. Consider OAuth2 or service account authentication for automated environments. + +Commit: ee8b2d2 + +--- + +SGB Edit: According to https://www.reddit.com/r/GMail/comments/y0f3dn/google_couldnt_verify_that_this_account_belongs/, maybe waiting a week or more would allow for this kind of sign-in without 2FA, but maybe not. This leaves either automating 2FA (see https://github.com/pyauth/pyotp and https://www.google.com/search?q=automate+2fa+google) or having a device in which it logs in once and stays logged in (where 2FA is done manually at the beginning, similar to what I did with copilot). The former would be compatible with Hugging Face, GitHub actions, etc., though the latter would likely need to be a device that we own so it's not ephemeral. + +EDIT: Trying out pyotp in https://github.com/AccelerationConsortium/ac-training-lab/pull/346 + +--- + +### Comment 25 — @zweaung1014 at 2025-06-24T00:48:05Z + +@sgbaird I have a script that can login and download+save the video now. Do we want to have it download all the new videos that hasn't been processed before every time we run it? How do we want to use the script? I think it would be nice if it could download all the videos that hasn't been processed before so that I can just run it, put the downloaded videos through the processor Jonathan made, and reupload them. But this might require some kind of identifier in the video title like, "processed". Just some thoughts I had. + +--- + +### Comment 26 — @sgbaird at 2025-06-24T01:21:20Z + +Nice! We may need to set it up with a pyotp integration, too (if it's on a completely ephemeral machine it's likely Google will throw that device not recognized error). See #346. Can always test it out first. + +The intention has been to host it on HuggingFace spaces with a GPU (can adjust tier as needed), but we may need to do a bit of benchmarking to see if we want to have it there (i.e., if the GPU could keep up with the jobs), trigger jobs on-demand through cloud services like AWS (might get pricy), or send jobs to the supercomputer resources the AC has access to (a cluster called BALAM - no-cost, but it's a shared resource and I haven't used this before so we'd need to get onboarded, and I'm assuming we'd have internet access on this cluster). + +Something to keep in mind is we're going to scale this up shortly to 30 livestreams, which is ~5000 hrs/week (@ 480p, 30 fps). + +I'll try to dig up some of the related conversations. I think we decided something about the naming scheme, too. + +--- + +### Comment 27 — @sgbaird at 2025-06-24T16:00:08Z + +@zweaung1014 do you mind updating this PR with the script you wrote? + +--- + +### Comment 28 — @sgbaird at 2025-06-24T16:00:50Z + +@copilot revert back to e32ad190e4a87144ca8f0f63d6dea0e5b708ebdf (i.e., clean slate). + +--- + +### Comment 29 — @zweaung1014 at 2025-06-24T16:04:12Z + +> @zweaung1014 do you mind updating this PR with the script you wrote? + +Will do in a bit. I'm trying to add pyotp too. + +--- + +### Comment 30 — @Copilot at 2025-06-24T16:04:24Z + + +> @copilot revert back to e32ad190e4a87144ca8f0f63d6dea0e5b708ebdf (i.e., clean slate). + +Repository successfully reverted to clean slate state at commit e32ad19. All files added after the initial plan have been removed and modified files restored to their original state. Commit: 8ebdc67 + +--- + +### Comment 31 — @zweaung1014 at 2025-06-24T21:48:19Z + +Demo video file is too big. Here's the youtube link: https://youtube.com/shorts/AE2YEII5j7s?feature=share + +This is the link to the repo: https://github.com/zweaung1014/yt_download.git + +To answer your question from the email, this is just a dummy secret. I just made it spit out a qr code so I can make those pass codes display on my Google Authenticator app and type it in to make the script run. The `testPYOTP.py` file generates the qr code. + +--- + +### Comment 32 — @sgbaird at 2025-06-24T21:56:22Z + + Thanks! I'm not very familiar with pyotp, but the hope was that a 2FA code for the account could be auto-generated on-demand and passed to playwright "in the same breath", without requiring any human intervention. Do you know if this is possible? + +--- + +### Comment 33 — @zweaung1014 at 2025-06-25T00:59:48Z + +> Thanks! I'm not very familiar with pyotp, but the hope was that a 2FA code for the account could be auto-generated on-demand and passed to playwright "in the same breath", without requiring any human intervention. Do you know if this is possible? + +Ohh I see what you mean. You're thinking of having playwright enter the 2FA code generated from Google automatically. Is that correct? + +For that to happen, both Google and my script will need to use the same "secret". This way, the same code Google is generating can be generated in the script too. So, if I set up 2FA on the achardwarestreams.downloader@gmail.com account and use that same secret in my pyotp script, I should be able to use playwright to send enter it automatically. I'll try setting it up. + + +--- + +### Comment 34 — @sgbaird at 2025-06-25T02:54:53Z + + That's correct. Cool! Thanks for working on this. + +--- + +### Comment 35 — @zweaung1014 at 2025-06-25T17:17:06Z + +@sgbaird +The signing in with 2FA works now. I generated the same code as the one Google is generating for the achardwarestreams.downloader@gmail.com account, and passed that code into the playwright script. I just need to troubleshoot the download issue. +![Screenshot 2025-06-25 131532](https://github.com/user-attachments/assets/fcaa7251-9c4c-408e-8f38-ae0945bc2f96) + + +--- + +### Comment 36 — @sgbaird at 2025-06-25T21:59:09Z + +Great! Glad that could be integrated. Sounds good about the download issue. Did the script error out, throw a warning, or silently fail? + +--- + +### Comment 37 — @zweaung1014 at 2025-06-26T03:01:12Z + +> Great! Glad that could be integrated. Sounds good about the download issue. Did the script error out, throw a warning, or silently fail? + +It fails, definitely. But this is when I try to download everything in the channel. I think it's mostly because I'm not looping through the check boxes correctly. But as far as downloading and saving individual videos go, it works fine. + +But you were mentioning something about having a database to track what's been downloaded, right? I think going that direction now (or finding some other way to identify what's been downloaded and what has not) is something we should do now. This is so I can make the script download what should be downloaded. I probably shouldn't spend time solving the problem of not being able to download everything it sees in the channel unless that's something we wanna do. + + +--- + +### Comment 38 — @sgbaird at 2025-06-26T11:51:27Z + +Oh, got it! The plan has just been to download and process single videos, one at a time and sequentially, rather than in batches (actually, I didn't realize you can download in batches, which is good to know). + +In terms of a function, we should be able to give it a video ID (the URL I guess) and the outcome is that it downloads. Later, maybe we want some return values or to keep track of status logs. + +For the database, I'll try to find the thread. I'm still debating on whether to keep it in a database or use the YouTube API. + +EDIT: see conversation in https://github.com/AccelerationConsortium/ac-training-lab/issues/223#issuecomment-3008922070 + +--- + +### Comment 39 — @sgbaird at 2025-06-26T15:44:14Z + +Aside: Here's some instructions for adding to this PR's branch: https://youtu.be/6HE3Oibvi50 + +--- + +### Comment 40 — @zweaung1014 at 2025-06-29T17:26:27Z + +@sgbaird Bringing the discussion back here. It sounds like if I want to get a script that logs in and downloads videos, + +- I'm gonna need to refresh the session state every few hours. +- But another idea is to make the videos public. If that's not what we want now, +- I can manually download and process them for now. + +Do you want me to go with the 3rd route? + +--- + +### Comment 41 — @sgbaird at 2025-07-02T21:17:52Z + +Sorry, just noticed this. We'll want it to automatically log in again if needed. I'm not sure what would be best - auto-logging out and logging back in every 30-60 minutes, handling any errors or access issues when they arise, etc. + +I lean towards setting it up to be automated, assuming the cookies/session will never expire, and then we monitor for how long the uptime is. Hugging Face Spaces keeps the logs, so we can check back to see when/why it errors out. I'd like to take that approach so we can keep it lean and add extra logic as needed. + +(2) won't be an option for many of the workflows, though I'm generally encouraging people to choose public or unlisted rather than private, in part for this reason (ease of being able to access/download). + +--- + +### Comment 42 — @zweaung1014 at 2025-07-03T02:30:42Z + +Hm, then I will just try to set it up on Hugging Face. I have something that should be able to take in the url, take the video code from the url, and go to YouTube Studio. But the download is still unreliable. But that should go away if I implement "saved Playwright sessions". I'm in the middle of implementing that and also putting my code on there so I can replace the yt-dlp. + +--- + +### Comment 43 — @sgbaird at 2025-07-03T04:19:25Z + +Sounds good! Also, see https://ac-training-lab.readthedocs.io/en/latest/devices/setup_iolt_devices.html#hugging-face-spaces about secrets + +--- + +### Comment 44 — @zweaung1014 at 2025-07-04T22:23:31Z + +Playwright doesn't seem to run on Hugging Face because it requires a full gui browser environment and Hugging face doesn't support it. If we wanna use Playwright, we will need to switch to a Docker-based space to install the required dependencies. I tried to do that, but it still won't run. Still trying to figure out why. + +It seems yt-dlp is the better option for this because it doesn't require gui/browser. + +--- + +### Comment 45 — @sgbaird at 2025-07-04T22:35:05Z + +Thanks for exploring! Do you have some documentation/links for that? (error logs, forum posts, AI transcript, etc.) + +--- + +### Comment 46 — @zweaung1014 at 2025-07-04T23:04:11Z + +HF forum with similar problem (?): https://discuss.huggingface.co/t/playwright-install-deps-error/100555 + +Chatgpt response for why Playwright didn't work: +![image](https://github.com/user-attachments/assets/c126e0c7-c26b-4c03-8f33-3cbd93ae252f) + +The recommendation is to use docker: +![image](https://github.com/user-attachments/assets/dcef55af-c17c-414e-aeff-8ef44d663963) + +I followed the instructions for implementing docker, but the App is stuck on "Starting" after that. + +The recommendation is to use yt-dlp unless we want to keep the 2FA because it's simpler to implement. +![image](https://github.com/user-attachments/assets/bff845ca-0c2f-4434-b61b-060de5558fe5) + + + +--- + +### Comment 47 — @sgbaird at 2025-07-05T00:29:10Z + +Thanks for the update! Could you make a small reproducer by creating a new HF space, adding a simple app.py script, adding playwright to requirements.txt, and share the link? (You can make it within AC org and make it public). Just something simple with playwright, no login or anything + +This will make it easier to come back to later + +--- + +### Comment 48 — @sgbaird at 2025-07-05T03:14:39Z + +Gave a quick stab at confirming that in a reproducer: https://huggingface.co/spaces/AccelerationConsortium/playwright-reproducer (e.g., put pishop.ca as the URL). + +``` +Error: BrowserType.launch: Executable doesn't exist at /home/user/.cache/ms-playwright/chromium_headless_shell-1179/chrome-linux/headless_shell +╔════════════════════════════════════════════════════════════╗ +║ Looks like Playwright was just installed or updated. ║ +║ Please run the following command to download new browsers: ║ +║ ║ +║ playwright install ║ +║ ║ +║ <3 Playwright Team ║ +╚════════════════════════════════════════════════════════════╝ +``` + +https://claude.ai/share/66c9d571-fafc-4575-a554-f767060f0fdc + +--- + +### Comment 49 — @sgbaird at 2025-07-05T03:59:36Z + +Works OK on Colab: https://colab.research.google.com/drive/1vynnZ0UuuabPeTZKgtrwmnbma2J2Wi5H?usp=sharing + +Thoughts on running a download via playwright on colab? (Just to verify the authentication works on a non-local machine, most similar to the environment we'd eventually run on). This would also let us easily test out the GPU compatibility and speed-up too, since colab has free-tier GPUs. Though, you'd need to get the code to Colab. Could clone the existing HF repo and run it as if it were local. Lmk if you think this testing/debugging is overkill + +In terms of alternatives: +- use submitit via scheduled GitHub actions within the training lab repo and submit batches of jobs to the AC's BALAM cluster (which I think is using SLURM and would therefore likely be compatible with submitit). This would be the lowest cost and most scalable (though lower transferability externally for people wanting to replicate). We could potentially handle 10k's hours of video processing per week +- oracle VM or AWS EC2 instance running 24/7. Free-tier oracle VM won't be able to scale, and paid versions of oracle or AWS will get at least mildly pricey, especially with GPU resources and if +- Prefect could be used to trigger ephemeral jobs to run on a schedule. Still could be pricey, not sure if having a dedicated machine or running ephemeral jobs would be pricier. This would be very similar to submitit + gh actions sending jobs to BALAM, except using Prefect's integrations with hosted cloud compute + +These options would effectively eliminate one-off manual downloads, i.e., without an easy web app, at least not without a decent bit of extra effort and complexity. I lean towards trying out gh action scheduled submitit batch jobs. + + + + +--- + +### Comment 50 — @zweaung1014 at 2025-07-06T03:49:57Z + +Got it. First, I’ll try cloning the HF repo into Colab and run the Playwright flow there to see if it behaves as expected in a non-local environment. + +And like you suggested, I think github actions + submitit sounds pretty good. Will look into it. + +--- + +### Comment 51 — @sgbaird at 2025-07-12T03:56:29Z + +Cc @Jonathan-Woo for where we left off. + +Also had a follow-up thought, if we have one machine on the cluster responsible for downloading videos, then we don't need to have lots of concurrent logins. Instead, we can save the downloads to a common storage spot on the cluster for other jobs to pick out and process. Just depends if it can handle the throughput, but I think with a reasonable download speed it's probably fine. + +--- + +### Comment 52 — @Jonathan-Woo at 2025-07-24T21:28:33Z + +Here is the working playwright implementation. + +https://github.com/user-attachments/assets/1f356588-193e-448b-aef7-41d3f2721e4e + +Workflow: +1. Load all playlists and video IDs through youtube data API +2. Filter out already downloaded videos and processed videos +3. Login to generic google account at https://accounts.google.com/ (2FA as well) +4. For each video to download, navigate to the youtube studio page and click download button + +There seems to be issues with it running headless but I think we can mock a display with a virtual framebuffer. + +To do: +1. Filter the playlists to download based on whether they've been processed or not. Beyond streams, do we expect other playlists? +2. Clean stuff up generally + +--- + +### Comment 53 — @sgbaird at 2025-07-25T16:43:15Z + +Amazing, thanks! I see you've learned my language with the unsolicited screen recording 😉. Could you also give this a try on some kind of ephemeral environment? (e.g., Colab, cloud VM). Apparently a private browser wouldn't be a good enough stress test (someone made a remark about how that indicates how "private" a private browser really is from Google's perspective). I'll work on getting you cluster access. + +Good point about headless. I think testing out on an ephemeral headless environment is a good next step. Probably chromium could be used? Copilot must have been able to do this in some way, since it is able to use playwright and successfully got to the login prompt during one of its agent sessions (which is just a github actions workflow running at its heart). + +https://github.com/AccelerationConsortium/ac-training-lab/pull/343#issuecomment-2993709854 ([agent session](https://github.com/AccelerationConsortium/ac-training-lab/pull/343/agent-sessions/dc115bba-fe23-411c-ae68-62dd9ab0dc1e), only viewable by me, but the [corresponding verbose logs](https://github.com/AccelerationConsortium/ac-training-lab/actions/runs/15798389195) - viewable by anyone) + +EDIT: yeah, I think it's chromium - https://playwright.dev/docs/browsers#chromium-headless-shell + +--- + +### Comment 54 — @sgbaird at 2025-07-25T16:45:16Z + +> Filter the playlists to download based on whether they've been processed or not. Beyond streams, do we expect other playlists? + +Probably just playlists with livestreams, though since these are just static videos once the stream is over, this wouldn't matter - right? + +--- + +### Comment 55 — @Jonathan-Woo at 2025-07-31T20:09:45Z + +So this is the downloader running on Balam login node (must be login node for internet access, uploader will also have to be on login node so only the processing can be submitted as jobs). + +`playwright` required system dependencies which couldn't be installed on the cluster so I created an apptainer to pack up all the dependencies (including python ones). I had to use a virtual frame buffer because the google login wouldn't work with playwright headless - likely due to bot detection. + +https://github.com/user-attachments/assets/b8be4bd7-6a30-4cc4-ae32-5d74fe8db27e + + + +--- + +### Comment 56 — @sgbaird at 2025-07-31T20:28:20Z + +Oh no, no internet access on compute nodes 😭 I had heard of other university clusters being under similar restrictions, but didn't realize this was the case for BALAM (and SciNet in general I'm guessing). I'll need to readjust my plans on a separate project.. + +Nice on figuring out the virtual frame buffer and containerization! And thank you for the video. Great to see and certainly feels closer. + +--- + + +## PR Reviews + +### Review — @copilot-pull-request-reviewer[bot] at 2025-07-31T20:47:17Z (COMMENTED) + +## Pull Request Overview + +This PR adds a new Playwright-based YouTube video downloader that provides an alternative to the existing yt-dlp approach, with the primary goal of enabling downloads of private/unlisted videos through Google authentication and YouTube's native download interface. + +### Key Changes: +- Introduces browser automation for YouTube downloads using Playwright +- Adds Google authentication with 2FA support via TOTP +- Implements YouTube API integration for playlist and video management + +### Reviewed Changes + +Copilot reviewed 2 out of 2 changed files in this pull request and generated 3 comments. + +| File | Description | +| ---- | ----------- | +| src/ac_training_lab/video_editing/requirements.txt | Adds dependencies for Playwright, Google API client, and authentication libraries | +| src/ac_training_lab/video_editing/download.py | Main implementation with YouTube API integration, Google authentication, and Playwright automation | + + +
+Comments suppressed due to low confidence (3) + +**src/ac_training_lab/video_editing/requirements.txt:14** +* The protobuf version 6.31.1 appears to be invalid. As of my knowledge cutoff, protobuf versions follow a different numbering scheme (e.g., 4.x.x or 5.x.x). Please verify this version exists. +``` +protobuf==6.31.1 +``` +**src/ac_training_lab/video_editing/requirements.txt:24** +* The urllib3 version 2.5.0 appears to be invalid. As of my knowledge cutoff, urllib3 versions were in the 1.x.x or 2.0.x range. Please verify this version exists. +``` +urllib3==2.5.0 +``` +**src/ac_training_lab/video_editing/requirements.txt:2** +* The certifi version 2025.7.14 appears to be from the future (July 2025). As of my knowledge cutoff in January 2025, this version would not exist yet. Please verify this version is available. +``` +certifi==2025.7.14 +``` +
+ + + +--- + +### Review — @Jonathan-Woo at 2025-07-31T20:53:41Z (COMMENTED) + +(no body) + +--- + +### Review — @Jonathan-Woo at 2025-07-31T20:54:05Z (COMMENTED) + +(no body) + +--- + +### Review — @sgbaird at 2025-08-01T15:21:54Z (COMMENTED) + +Nice! + +--- + +### Review — @Jonathan-Woo at 2025-08-01T21:17:31Z (COMMENTED) + +(no body) + +--- + + +## Inline review comments + +### @Copilot on `src/ac_training_lab/video_editing/download.py` at 2025-07-31T20:47:16Z + +Importing credentials from a 'my_secrets' module violates security best practices. Credentials should be loaded from environment variables or secure configuration files that are not committed to version control. +```suggestion +EMAIL = os.getenv("EMAIL") +PASSWORD = os.getenv("PASSWORD") +TOTP_SECRET = os.getenv("TOTP_SECRET") +``` + +--- + +### @Copilot on `src/ac_training_lab/video_editing/download.py` at 2025-07-31T20:47:17Z + +The custom coding guidelines specify to avoid `if __name__ == "__main__"` patterns in package code. This appears to be package code rather than a standalone script. + +--- + +### @Copilot on `src/ac_training_lab/video_editing/download.py` at 2025-07-31T20:47:17Z + +The hardcoded slice [:1] limits processing to only the first playlist. This magic number should be made configurable or documented to explain why only one playlist is processed. +```suggestion +def get_pending_downloads(youtube, processed_videos, downloaded_ids, max_playlists=1): + """ + Retrieve videos pending download from the user's playlists. + + Args: + youtube: Authenticated YouTube client. + processed_videos: Dictionary of already processed videos. + downloaded_ids: Set of video IDs already downloaded. + max_playlists: Maximum number of playlists to process. Default is 1. + + Returns: + A dictionary mapping playlist IDs to lists of video IDs pending download. + """ + all_videos = {} + playlist_ids = list_my_playlists(youtube) + for playlist_id in playlist_ids[:max_playlists]: +``` + +--- + +### @Jonathan-Woo on `src/ac_training_lab/video_editing/download.py` at 2025-07-31T20:53:41Z + +fixed + +--- + +### @Jonathan-Woo on `src/ac_training_lab/video_editing/download.py` at 2025-07-31T20:54:05Z + +this is a standalone script + +--- + +### @sgbaird on `src/ac_training_lab/video_editing/requirements.txt` at 2025-08-01T15:20:17Z + +If you could add a minimum set of dependencies (e.g., using pipreqs) or based on what you know you needed to install, that would be good at some point. Merging still + +--- + +### @Jonathan-Woo on `src/ac_training_lab/video_editing/requirements.txt` at 2025-08-01T21:17:31Z + +I'll add the apptainer definition file in the new repo + +--- From e90dfa97d1ee984697ebcb0751280147b88db1a1 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:31:14 +0000 Subject: [PATCH 12/13] Add CLAUDE.md with Tailscale -> RPi connection instructions Co-authored-by: Sterling G. Baird <45469701+sgbaird@users.noreply.github.com> Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..36548c6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,44 @@ +# CLAUDE.md + +## Tailscale → Raspberry Pi (streaming camera) connection + +**You are already on the tailnet.** The `claude.yml` workflow joins the runner via the +official [Tailscale GitHub Action](https://tailscale.com/kb/1276/tailscale-github-action) +(OAuth client, `tag:stream-cam-test`) before you start. Run `tailscale status` to confirm — +do **not** install Tailscale, mint auth keys via the API, or run `tailscale up` unless +status genuinely shows you disconnected. Access to the Pi is +[Tailscale SSH](https://tailscale.com/kb/1193/tailscale-ssh) (authorized by +[tailnet ACLs](https://tailscale.com/kb/1018/acls), not SSH keys): +`ssh "$RPI_STREAM_CAM_USERNAME@$RPI_STREAM_CAM_HOSTNAME"` — always use the env vars, and +never print the hostname, Lambda URL, bucket names, RTMP/stream keys, or heartbeat ping URLs +in comments, commits, or logs. If SSH is refused, the fix is an ACL/tag change only the +tailnet admin can make — report it and stop rather than working around it. + +**sudo on the Pi is password-gated** (no passwordless sudo; polkit rejects non-interactive +`systemctl`). Feed the password over stdin so it never hits a process list or log: +`ssh … "sudo -S -p '' " <<< "$RPI_STREAM_CAM_PASSWORD"`. Run AWS CLI work from the +**runner** (it has the AWS env vars) — don't route AWS calls through the Pi. Conversely, +YouTube blocks datacenter IPs, so video downloads must run **on the Pi** (residential IP), +rate-capped (`--limit-rate`) so they don't starve the live RTMP upload. Never run +full-bandwidth speed tests on the Pi for the same reason. + +**Things that look like bugs but are intentional** (see +`docs/ac-training-lab-picam-suggestions.md` and the +[picam docs](https://ac-training-lab.readthedocs.io/en/latest/devices/picam.html#automatic-startup)): +the Pi reboots at 05:00/13:00/21:00 America/Denver by root crontab so YouTube stores each +8-hour chunk as its own video — an unreachable Pi near those times is likely mid-reboot, and +a Pi that dropped off the network often self-recovers at the next one, so check the clock +before declaring an outage. Do **not** re-implement chunking with `RuntimeMaxSec`, make the +Lambda `create` idempotent, or add a second systemd unit: `device.service` is the **only** +unit that may run `device.py` (a second one races for the camera and thrashes). A +`stream-watchdog` timer already restarts `device.service` on RTMP stalls with a 6/day budget +(config in `/etc/default/stream-watchdog`, mode 600) and pings Healthchecks.io when healthy — +check its journal (`journalctl -t stream-watchdog`) before adding new monitoring. + +**Every `device.service` restart creates a new YouTube broadcast** (`end` → `create`), so +restart it only when necessary and verify afterwards: service `active`, single +`rpicam-vid` + `ffmpeg` pair, RTMP socket `ESTABLISHED` with `bytes_acked` advancing. +Prefer read-only inspection; confirm end-to-end (Lambda `create` → 200, stream live) before +reporting success, and report failures as failures. Changes to `device.py`/systemd/cron live +on the Pi and upstream in ac-training-lab — record them in +`docs/ac-training-lab-picam-suggestions.md`, not as code in this repo. From 200ad24ce3fcbe5a412f2421b95a1140a4996a3c Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:47:08 +0000 Subject: [PATCH 13/13] Generalize CLAUDE.md to device-agnostic Tailscale -> RPi instructions Co-authored-by: Sterling G. Baird <45469701+sgbaird@users.noreply.github.com> --- CLAUDE.md | 70 +++++++++++++++++++++++++------------------------------ 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 36548c6..903aac0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,44 +1,38 @@ # CLAUDE.md -## Tailscale → Raspberry Pi (streaming camera) connection +## Tailscale → Raspberry Pi connection -**You are already on the tailnet.** The `claude.yml` workflow joins the runner via the -official [Tailscale GitHub Action](https://tailscale.com/kb/1276/tailscale-github-action) -(OAuth client, `tag:stream-cam-test`) before you start. Run `tailscale status` to confirm — -do **not** install Tailscale, mint auth keys via the API, or run `tailscale up` unless -status genuinely shows you disconnected. Access to the Pi is -[Tailscale SSH](https://tailscale.com/kb/1193/tailscale-ssh) (authorized by -[tailnet ACLs](https://tailscale.com/kb/1018/acls), not SSH keys): -`ssh "$RPI_STREAM_CAM_USERNAME@$RPI_STREAM_CAM_HOSTNAME"` — always use the env vars, and -never print the hostname, Lambda URL, bucket names, RTMP/stream keys, or heartbeat ping URLs -in comments, commits, or logs. If SSH is refused, the fix is an ACL/tag change only the -tailnet admin can make — report it and stop rather than working around it. +**You are already on the tailnet.** The workflow joins the runner via the official +[Tailscale GitHub Action](https://tailscale.com/kb/1276/tailscale-github-action) (OAuth +client + device tag) before you start. Run `tailscale status` to confirm — do **not** +install Tailscale, mint auth keys via the API, or run `tailscale up` unless status +genuinely shows you disconnected. Access to the Pi is +[Tailscale SSH](https://tailscale.com/kb/1193/tailscale-ssh), authorized by +[tailnet ACLs](https://tailscale.com/kb/1018/acls) rather than SSH keys — there is no key +to find or generate. The Pi's login username, hostname, and sudo password are injected as +environment variables (check `env` for the `RPI_*`-style names rather than assuming them); +always reference them as `"$VAR"` and never print the hostname or any credential in +comments, commits, or logs. If SSH is refused (`tailnet policy does not permit you to SSH +to this node`), the fix is an ACL/tag change only the tailnet admin can make — report it +and stop rather than working around it. -**sudo on the Pi is password-gated** (no passwordless sudo; polkit rejects non-interactive -`systemctl`). Feed the password over stdin so it never hits a process list or log: -`ssh … "sudo -S -p '' " <<< "$RPI_STREAM_CAM_PASSWORD"`. Run AWS CLI work from the -**runner** (it has the AWS env vars) — don't route AWS calls through the Pi. Conversely, -YouTube blocks datacenter IPs, so video downloads must run **on the Pi** (residential IP), -rate-capped (`--limit-rate`) so they don't starve the live RTMP upload. Never run -full-bandwidth speed tests on the Pi for the same reason. +**sudo on the Pi is password-gated** — no passwordless sudo, and polkit rejects +non-interactive `systemctl`. Feed the password over stdin so it never appears in a process +list or shell history: `ssh … "sudo -S -p '' " <<< "$RPI_PASSWORD_VAR"`. -**Things that look like bugs but are intentional** (see -`docs/ac-training-lab-picam-suggestions.md` and the -[picam docs](https://ac-training-lab.readthedocs.io/en/latest/devices/picam.html#automatic-startup)): -the Pi reboots at 05:00/13:00/21:00 America/Denver by root crontab so YouTube stores each -8-hour chunk as its own video — an unreachable Pi near those times is likely mid-reboot, and -a Pi that dropped off the network often self-recovers at the next one, so check the clock -before declaring an outage. Do **not** re-implement chunking with `RuntimeMaxSec`, make the -Lambda `create` idempotent, or add a second systemd unit: `device.service` is the **only** -unit that may run `device.py` (a second one races for the camera and thrashes). A -`stream-watchdog` timer already restarts `device.service` on RTMP stalls with a 6/day budget -(config in `/etc/default/stream-watchdog`, mode 600) and pings Healthchecks.io when healthy — -check its journal (`journalctl -t stream-watchdog`) before adding new monitoring. +**You have two machines — use the right one.** Your runner terminal and the Pi are +separate environments: cloud/API credentials (AWS, etc.) live on the **runner**, so run +that tooling there rather than routing it through the Pi. Use the Pi only for what +genuinely requires it — its attached hardware, or its residential IP (some services block +datacenter IPs). The Pi is typically on constrained residential Wi‑Fi and may be carrying +live workloads, so rate-cap any large transfer (`--limit-rate` or equivalent) and never +run full-bandwidth speed tests on it. -**Every `device.service` restart creates a new YouTube broadcast** (`end` → `create`), so -restart it only when necessary and verify afterwards: service `active`, single -`rpicam-vid` + `ffmpeg` pair, RTMP socket `ESTABLISHED` with `bytes_acked` advancing. -Prefer read-only inspection; confirm end-to-end (Lambda `create` → 200, stream live) before -reporting success, and report failures as failures. Changes to `device.py`/systemd/cron live -on the Pi and upstream in ac-training-lab — record them in -`docs/ac-training-lab-picam-suggestions.md`, not as code in this repo. +**Treat the Pi as a live production device.** Inspect read-only first (`systemctl status`, +`journalctl`, `crontab -l` as root) before changing state: scheduled reboots, watchdog +timers, and `Restart=` policies may already exist, so an unreachable or restarting device +may be behaving as designed — check the clock and the existing automation before declaring +an outage or adding new monitoring. Restart services only when necessary and verify the +device's workload is healthy end-to-end afterwards, reporting failures as failures. +Changes made on the Pi (systemd units, cron, scripts, config) do not live in this repo — +record them in the repo's docs so they can be reproduced or upstreamed.