Dk updates

app.py

@@ -72,81 +72,64 @@ def get_latest_frame() -> bytes:
 
 
 
-def action_to_finger_mapping(action: str) -> str:
-    """
-    Map action names to receiver finger/command codes.
-
-    Map actions to relay target names expected by firmware.
-
-    Supported relay targets:
-    - wrist_left, wrist_right, thumb, index, middle, ring, pinky
-    - x = reset/all off
-    """
-    mapping = {
-        "clench_hand": "x",      # reset/end sequence
-        "close_index": "index",
-        "close_middle": "middle",
-        "close_pinky": "pinky",
-        "close_thumb": "thumb",
-        "close_ring": "ring",
-        "wrist_left": "wrist_left",
-        "wrist_right": "wrist_right",
-        # These are not relay-compatible and will be skipped:
-        # "biceps_flex": requires different command type
-        # "lean_left": GVS command, not relay
-        # "lean_right": GVS command, not relay
-    }
-    return mapping.get(action, "x")
+# Maps each LLM action to how it drives the hardware:
+#   relay_target: relay electrode to select before stimulating
+#   ems_channel:  stimulator channel to fire
+# Only channel 2 is wired to the relay board, so all three actions route
+# through it. The firmware enables one electrode at a time, which makes the
+# actions mutually exclusive (one action per step).
+ACTION_SPEC = {
+    "grab":        {"relay_target": "grab",        "ems_channel": 2},
+    "arm_left":  {"relay_target": "arm_left",  "ems_channel": 2},
+    "arm_right": {"relay_target": "arm_right", "ems_channel": 2},
+}
 
 
 
 
 def transform_actions_to_receiver_format(claude_response: dict) -> dict:
     """
-    Transform Claude's response format to receiver.py's timestamped format.
-
-    Handles both formats:
+    Transform the LLM's response format to receiver.py's timestamped format.
+
+    Handles both numeric ("1", "2") and "sequence_1" style step keys.
+
     INPUT (numeric keys):
     {
-      "1": [["close_middle", 1.0], ["clench_hand", 0.5]],
-      "2": [["close_pinky", 2.0]]
+      "1": [["grab", 1.0]],
+      "2": [["arm_left", 1.5]]
     }
-
-    INPUT (sequence keys):
-    {
-      'sequence_1': [['close_middle', 1.0], ['clench_hand', 0.5]],
-      'sequence_2': [['close_pinky', 2.0]]
-    }
-
+
     OUTPUT (for receiver.py):
     {
-      "0": [
-        {"type": "RELAY", "finger": "m"},
-        {"type": "EMS", "channel": 1, "amplitude": 60, "duration": 1.0, "frequency": 100}
+      "0.0": [
+        {"type": "RELAY", "target": "grab"},
+        {"type": "EMS", "channel": 2, "amplitude": 60, "duration": 1.0, ...}
       ],
-      "1.0": [
-        {"type": "RELAY", "finger": "x"},
-        {"type": "EMS", "channel": 1, "amplitude": 60, "duration": 0.5, "frequency": 100}
+      "2.0": [
+        {"type": "RELAY", "target": "arm_left"},
+        {"type": "EMS", "channel": 2, "amplitude": 60, "duration": 1.5, ...}
       ],
-      "1.5": [
-        {"type": "RELAY", "finger": "p"},
-        {"type": "EMS", "channel": 1, "amplitude": 60, "duration": 2.0, "frequency": 100}
+      "4.5": [
+        {"type": "RELAY", "target": "x"}
       ]
     }
-
+
+    Per-action behavior comes from ACTION_SPEC: each action selects its relay
+    electrode, then fires EMS on channel 2. Only one relay is active at a time,
+    so actions are mutually exclusive.
+
     Timing logic:
-    - Each action starts at cumulative_time (sum of all previous durations)
-    - Duration in the action is how long the EMS stimulation lasts
-    - All supported actions select one relay target then stimulate EMS channel 1
-    - Unsupported actions (biceps, lean) are logged but not sent
+    - Each action starts at cumulative_time (sum of all previous durations).
+    - Duration in the action is how long the EMS stimulation lasts.
+    - Unknown actions are logged and skipped.
     """
     receiver_format = {}
     current_time = 0.0
-    
+
     # Determine key format and sort accordingly
     numeric_keys = [k for k in claude_response.keys() if k.isdigit()]
     sequence_keys = [k for k in claude_response.keys() if k.startswith('sequence_')]
-    
+
     if numeric_keys:
         # Sort by numeric value
         sorted_keys = sorted(numeric_keys, key=lambda x: int(x))
@@ -156,50 +139,48 @@ def transform_actions_to_receiver_format(claude_response: dict) -> dict:
     else:
         # Unknown format, use as-is
         sorted_keys = list(claude_response.keys())
-    
+
     for key in sorted_keys:
         actions = claude_response[key]
-        
+
         for action_name, duration in actions:
-            # Map action to finger code
-            finger_code = action_to_finger_mapping(action_name)
-
-            # Create timestamped entry key
-            time_key = str(current_time)
-
-            # Skip unsupported actions (biceps, lean, etc.)
-            if action_name in ["biceps_flex", "lean_left", "lean_right"]:
+            spec = ACTION_SPEC.get(action_name)
+
+            # Skip unknown actions, but still advance time so later steps stay ordered.
+            if spec is None:
                 print(f"[!] Skipping unsupported action: {action_name}")
                 current_time += float(duration) + 1.0
                 continue
 
+            time_key = str(current_time)
             if time_key not in receiver_format:
                 receiver_format[time_key] = []
 
-            # All supported actions: RELAY select first, then EMS on channel 1
-            receiver_format[time_key].append({
-                "type": "RELAY",
-                "finger": finger_code
-            })
+            # Route the relevant arm relay first (grab drives the channel directly).
+            if spec["relay_target"] is not None:
+                receiver_format[time_key].append({
+                    "type": "RELAY",
+                    "target": spec["relay_target"]
+                })
             receiver_format[time_key].append({
                 "type": "EMS",
-                "channel": 1,
+                "channel": spec["ems_channel"],
                 "amplitude": EMS_AMPLITUDE,
                 "duration": float(duration),
                 "frequency": EMS_FREQUENCY,
                 "pulse_width": EMS_PULSE_WIDTH
             })
-            
+
             # Move to next action time (current duration + 1 second buffer for relay to open)
             current_time += float(duration) + 1.0
-    
-    # Always append "x" command (disable all fingers) at the end
+
+    # Always append "x" command (reset relay) at the end
     final_time = str(current_time)
     receiver_format[final_time] = [{
         "type": "RELAY",
-        "finger": "x"
+        "target": "x"
     }]
-    
+
     return receiver_format
 
 
@@ -342,6 +323,7 @@ def main():
     except KeyboardInterrupt:
         print("\n[*] Interrupted")
     finally:
+        listener.stop()
         cap.release()
         cv2.destroyAllWindows()
         print("[*] Exiting...")

firmware/human_operator_ems/human_operator_ems.ino

@@ -1,19 +1,27 @@
-// Electrode select lines (user-provided config)
-const int electrode1 = 2;  // wrist_left
-const int electrode2 = 4;  // wrist_right
-const int electrode3 = 3;  // thumb
-const int electrode4 = 5;  // index
-const int electrode5 = 6;  // middle
-const int electrode6 = 7;  // ring
-const int electrode7 = 8;  // pinky
+// Electrode relay router.
+const int electrode1 = 2;  // arm_left
+const int electrode2 = 4;  // arm_right
+const int electrode3 = 3;  // grab (grip)
 
 // Most relay boards are active-low (LOW = ON, HIGH = OFF).
 // If your board is active-high, set this to false.
 bool relayActiveLow = false;
 
-const int relayPins[] = {electrode1, electrode2, electrode3, electrode4, electrode5, electrode6, electrode7};
+const int relayPins[] = {electrode1, electrode2, electrode3};
 const int relayPinCount = sizeof(relayPins) / sizeof(relayPins[0]);
 
+// Auto-off safety. If the stimulator runs continuously, the relay hold time IS
+// the stimulation time. A selected electrode is dropped when its hold expires,
+// so it can never get stuck on if commands stop arriving.
+// holdUntil == 0 means nothing is scheduled.
+unsigned long holdUntil = 0;
+const unsigned long DEFAULT_HOLD_MS = 2000;  // fallback if a select passes 0
+const unsigned long MAX_HOLD_MS = 5000;      // hard cap so nothing sticks on too long
+
+// Per-action hold times. The firmware owns these; the host just sends the name.
+const unsigned long GRAB_HOLD_MS = 2000;     // grip holds long enough to grab
+const unsigned long ARM_HOLD_MS = 300;     // a arm turn is a quick flick
+
 int relayOnLevel() {
   return relayActiveLow ? LOW : HIGH;
 }
@@ -32,9 +40,12 @@ void allOff() {
   }
 }
 
-void selectOnly(int pin) {
+void select(int pin, unsigned long holdMs) {
+  if (holdMs == 0) holdMs = DEFAULT_HOLD_MS;
+  if (holdMs > MAX_HOLD_MS) holdMs = MAX_HOLD_MS;
   allOff();
   setRelayState(pin, true);
+  holdUntil = millis() + holdMs;
 }
 
 void runTestSweep() {
@@ -58,26 +69,30 @@ void setup() {
   // Initialize serial communication so we can chat with it
   Serial.begin(115200);
 
-  // Initialize all electrode select pins as outputs
+  // Initialize the electrode select pins as outputs
   pinMode(electrode1, OUTPUT);
   pinMode(electrode2, OUTPUT);
   pinMode(electrode3, OUTPUT);
-  pinMode(electrode4, OUTPUT);
-  pinMode(electrode5, OUTPUT);
-  pinMode(electrode6, OUTPUT);
-  pinMode(electrode7, OUTPUT);
 
   allOff();
 
   // Print the instructions to the Serial Monitor
-  Serial.println("Electrode Router Ready.");
+  Serial.println("Electrode Relay Router Ready.");
   Serial.println("Commands (name or alias):");
-  Serial.println("wrist_left/w, wrist_right/q, thumb/t, index/i, middle/m, ring/r, pinky/p");
+  Serial.println("arm_left/w, arm_right/q, grab/g");
   Serial.println("x = all off, test = sweep channels");
   Serial.println("mode_low = active-low relays, mode_high = active-high relays");
 }
 
 void loop() {
+  // Safety watchdog: drop the active electrode once its hold time expires.
+  // (long) cast handles millis() rollover correctly.
+  if (holdUntil != 0 && (long)(millis() - holdUntil) >= 0) {
+    allOff();
+    holdUntil = 0;
+    Serial.println("Auto-off (hold expired)");
+  }
+
   if (Serial.available() > 0) {
     String command = Serial.readStringUntil('\n');
     command.trim();
@@ -86,43 +101,34 @@ void loop() {
       return;
     }
 
-    if (isCommand(command, "wrist_left", 'w')) {
-      selectOnly(electrode1);
-      Serial.println("Selected: wrist_left");
-    } else if (isCommand(command, "wrist_right", 'q')) {
-      selectOnly(electrode2);
-      Serial.println("Selected: wrist_right");
-    } else if (isCommand(command, "thumb", 't')) {
-      selectOnly(electrode3);
-      Serial.println("Selected: thumb");
-    } else if (isCommand(command, "index", 'i')) {
-      selectOnly(electrode4);
-      Serial.println("Selected: index");
-    } else if (isCommand(command, "middle", 'm')) {
-      selectOnly(electrode5);
-      Serial.println("Selected: middle");
-    } else if (isCommand(command, "ring", 'r')) {
-      selectOnly(electrode6);
-      Serial.println("Selected: ring");
-    } else if (isCommand(command, "pinky", 'p')) {
-      selectOnly(electrode7);
-      Serial.println("Selected: pinky");
+    if (isCommand(command, "arm_left", 'l')) {
+      select(electrode1, ARM_HOLD_MS);
+      Serial.println("Selected: arm_left");
+    } else if (isCommand(command, "arm_right", 'q')) {
+      select(electrode2, ARM_HOLD_MS);
+      Serial.println("Selected: arm_right");
+    } else if (isCommand(command, "grab", 'g')) {
+      select(electrode3, GRAB_HOLD_MS);
+      Serial.println("Selected: grab");
     } else if (command.equalsIgnoreCase("x")) {
       allOff();
+      holdUntil = 0;
       Serial.println("All electrodes OFF");
     } else if (command.equalsIgnoreCase("test")) {
       runTestSweep();
     } else if (command.equalsIgnoreCase("mode_low")) {
       relayActiveLow = true;
       allOff();
+      holdUntil = 0;
       Serial.println("Relay polarity set to ACTIVE-LOW");
     } else if (command.equalsIgnoreCase("mode_high")) {
       relayActiveLow = false;
       allOff();
+      holdUntil = 0;
       Serial.println("Relay polarity set to ACTIVE-HIGH");
     } else {
       Serial.print("Unknown command: ");
       Serial.println(command);
     }
   }
-}
+}

requirements.txt

@@ -1,15 +1,22 @@
 # llm
 anthropic
+openai
 numpy
 opencv-python
 python-dotenv
 pynput
 
 # stt
 realtimestt
+pygame
 
 # server
 flask
+requests
+
+# hardware / serial 
+pyserial
 
 # manual control app
-PyQt5
+PyQt5
+matplotlib

run_hardware.sh

@@ -2,10 +2,12 @@
 set -euo pipefail
 
 ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
-PYTHON_BIN="$ROOT_DIR/.venv/bin/python"
+# Defaults to the repo venv; override with PYTHON_BIN to use conda/system Python,
+# e.g. PYTHON_BIN="$(which python)" ./run_hardware.sh
+PYTHON_BIN="${PYTHON_BIN:-$ROOT_DIR/.venv/bin/python}"
 
 # Defaults can be overridden when invoking the script.
-RELAY_PORT="${RELAY_PORT:-/dev/cu.usbserial-210}"
+RELAY_PORT="${RELAY_PORT:-/dev/cu.usbmodem34B7DA631B182}"
 RECEIVER_HOST="${RECEIVER_HOST:-127.0.0.1}"
 RECEIVER_PORT="${RECEIVER_PORT:-5001}"
 RECEIVER_URL="http://${RECEIVER_HOST}:${RECEIVER_PORT}/execute"