feat: animated Clawd avatar for camera bubble with face tracking

Cargo.toml

@@ -65,6 +65,7 @@ cidre = { git = "https://github.com/CapSoftware/cidre", rev = "bf84b67079a8", fe
     "io_surface",
     "mtl",
     "vt",
+    "vn",
 ], default-features = false }
 
 windows = "0.60.0"

apps/desktop/src-tauri/src/general_settings.rs

@@ -124,6 +124,10 @@ pub struct GeneralSettingsStore {
     #[serde(default)]
     pub auto_zoom_on_clicks: bool,
     #[serde(default)]
+    pub avatar_mode: bool,
+    #[serde(default)]
+    pub avatar_background: cap_project::AvatarBackground,
+    #[serde(default)]
     pub post_deletion_behaviour: PostDeletionBehaviour,
     #[serde(default = "default_excluded_windows")]
     pub excluded_windows: Vec<WindowExclusion>,
@@ -203,6 +207,8 @@ impl Default for GeneralSettingsStore {
             recording_countdown: Some(3),
             enable_native_camera_preview: default_enable_native_camera_preview(),
             auto_zoom_on_clicks: false,
+            avatar_mode: false,
+            avatar_background: cap_project::AvatarBackground::Dark,
             post_deletion_behaviour: PostDeletionBehaviour::DoNothing,
             excluded_windows: default_excluded_windows(),
             delete_instant_recordings_after_upload: false,

apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx

@@ -26,6 +26,7 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) {
 			enableNotifications: true,
 			enableNativeCameraPreview: false,
 			autoZoomOnClicks: false,
+			avatarMode: false,
 			custom_cursor_capture2: true,
 		},
 	);
@@ -96,6 +97,15 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) {
 						/>
 					</div>
 				</div>
+				<h3 class="text-sm text-gray-12 w-fit">Camera Features</h3>
+				<div class="px-3 rounded-xl border divide-y divide-gray-3 border-gray-3 bg-gray-2">
+					<ToggleSettingItem
+						label="Avatar mode (Clawd)"
+						description="Replace your camera feed with an animated Clawd character that mimics your facial expressions. Perfect for creating engaging tutorials."
+						value={!!settings.avatarMode}
+						onChange={(value) => handleChange("avatarMode", value)}
+					/>
+				</div>
 			</div>
 		</div>
 	);

crates/face-tracking/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "cap-face-tracking"
+version = "0.1.0"
+edition = "2024"
+
+[lints]
+workspace = true
+
+[dependencies]
+tracing.workspace = true
+
+[target.'cfg(target_os = "macos")'.dependencies]
+cidre = { workspace = true }

crates/face-tracking/src/lib.rs

@@ -0,0 +1,30 @@
+#[derive(Clone, Debug, Default)]
+pub struct FacePose {
+    pub head_pitch: f32,
+    pub head_yaw: f32,
+    pub head_roll: f32,
+    pub mouth_open: f32,
+    pub left_eye_open: f32,
+    pub right_eye_open: f32,
+    pub confidence: f32,
+}
+
+#[cfg(target_os = "macos")]
+mod macos;
+
+#[cfg(target_os = "macos")]
+pub use macos::FaceTracker;
+
+#[cfg(not(target_os = "macos"))]
+pub struct FaceTracker;
+
+#[cfg(not(target_os = "macos"))]
+impl FaceTracker {
+    pub fn new() -> Self {
+        Self
+    }
+
+    pub fn track(&mut self, _rgba_data: &[u8], _width: u32, _height: u32) -> FacePose {
+        FacePose::default()
+    }
+}

crates/face-tracking/src/macos.rs

@@ -0,0 +1,243 @@
+use std::ffi::c_void;
+
+use cidre::{arc, cv, ns, objc, vn};
+
+use crate::FacePose;
+
+pub struct FaceTracker {
+    landmarks_request: arc::R<vn::Request>,
+}
+
+impl FaceTracker {
+    pub fn new() -> Self {
+        let landmarks_request = create_landmarks_request();
+        Self { landmarks_request }
+    }
+
+    pub fn track(&mut self, rgba_data: &[u8], width: u32, height: u32) -> FacePose {
+        match self.track_inner(rgba_data, width, height) {
+            Some(pose) => pose,
+            None => FacePose::default(),
+        }
+    }
+
+    fn track_inner(&mut self, rgba_data: &[u8], width: u32, height: u32) -> Option<FacePose> {
+        let w = width as usize;
+        let h = height as usize;
+        let src_row_bytes = w * 4;
+        let expected_len = src_row_bytes * h;
+        if rgba_data.len() < expected_len {
+            tracing::warn!(
+                "RGBA data too small: {} < {}",
+                rgba_data.len(),
+                expected_len
+            );
+            return None;
+        }
+
+        let mut pixel_buf = cv::PixelBuf::new(w, h, cv::PixelFormat::_32_BGRA, None).ok()?;
+
+        unsafe {
+            pixel_buf
+                .lock_base_addr(cv::pixel_buffer::LockFlags::DEFAULT)
+                .result()
+                .ok()?;
+        }
+
+        let dst_base = unsafe { CVPixelBufferGetBaseAddress(&pixel_buf) };
+        let dst_row_bytes = unsafe { CVPixelBufferGetBytesPerRow(&pixel_buf) };
+
+        if dst_base.is_null() {
+            unsafe {
+                pixel_buf.unlock_lock_base_addr(cv::pixel_buffer::LockFlags::DEFAULT);
+            }
+            return None;
+        }
+
+        unsafe {
+            rgba_to_bgra_copy(
+                rgba_data,
+                dst_base as *mut u8,
+                w,
+                h,
+                src_row_bytes,
+                dst_row_bytes,
+            );
+        }
+
+        unsafe {
+            pixel_buf.unlock_lock_base_addr(cv::pixel_buffer::LockFlags::DEFAULT);
+        }
+
+        let handler = vn::ImageRequestHandler::with_cv_pixel_buf(&pixel_buf, None)?;
+
+        let mut rect_request = vn::DetectFaceRectanglesRequest::new();
+        rect_request.set_revision(vn::DetectFaceRectanglesRequest::REVISION_3);
+
+        let requests =
+            ns::Array::<vn::Request>::from_slice(&[&rect_request, &self.landmarks_request]);
+        if handler.perform(&requests).is_err() {
+            return None;
+        }
+
+        let landmarks_results: Option<arc::R<ns::Array<vn::FaceObservation>>> = unsafe {
+            let raw: *const vn::Request = &*self.landmarks_request;
+            let face_req: &vn::DetectFaceRectanglesRequest =
+                &*(raw as *const vn::DetectFaceRectanglesRequest);
+            face_req.results()
+        };
+
+        let face_obs = landmarks_results
+            .as_ref()
+            .filter(|r| !r.is_empty())
+            .and_then(|r| r.get(0).ok());
+
+        let face_obs = match face_obs {
+            Some(obs) => obs,
+            None => {
+                let results = rect_request.results()?;
+                if results.is_empty() {
+                    return None;
+                }
+                results.get(0).ok()?
+            }
+        };
+
+        let head_roll = face_obs.roll().map(|n| n.as_f32()).unwrap_or(0.0);
+        let head_yaw = face_obs.yaw().map(|n| n.as_f32()).unwrap_or(0.0);
+        let head_pitch = face_obs.pitch().map(|n| n.as_f32()).unwrap_or(0.0);
+        let confidence = face_obs.confidence();
+
+        let (mouth_open, left_eye_open, right_eye_open) = extract_landmark_features(&face_obs);
+
+        Some(FacePose {
+            head_pitch,
+            head_yaw,
+            head_roll,
+            mouth_open,
+            left_eye_open,
+            right_eye_open,
+            confidence,
+        })
+    }
+}
+
+fn create_landmarks_request() -> arc::R<vn::Request> {
+    unsafe {
+        let cls = objc::objc_getClass(b"VNDetectFaceLandmarksRequest\0".as_ptr());
+        match cls {
+            Some(cls) => {
+                let cls: &objc::Class<vn::Request> = std::mem::transmute(cls);
+                cls.new()
+            }
+            None => {
+                tracing::warn!(
+                    "VNDetectFaceLandmarksRequest not found, falling back to rectangles"
+                );
+                let req = vn::DetectFaceRectanglesRequest::new();
+                std::mem::transmute(req)
+            }
+        }
+    }
+}
+
+fn extract_landmark_features(face: &vn::FaceObservation) -> (f32, f32, f32) {
+    let landmarks = match face.landmarks() {
+        Some(l) => l,
+        None => return (0.0, 1.0, 1.0),
+    };
+
+    let mouth_open = compute_mouth_openness(&landmarks);
+    let left_eye_open = compute_eye_openness(landmarks.left_eye());
+    let right_eye_open = compute_eye_openness(landmarks.right_eye());
+
+    (mouth_open, left_eye_open, right_eye_open)
+}
+
+fn compute_mouth_openness(landmarks: &vn::FaceLandmarks2d) -> f32 {
+    let inner_lips = match landmarks.inner_lips() {
+        Some(region) => region,
+        None => return 0.0,
+    };
+
+    let points = inner_lips.normalized_points();
+    if points.len() < 6 {
+        return 0.0;
+    }
+
+    let top = points[2];
+    let bottom = points[points.len() - 2];
+    let vertical = (top.y - bottom.y).abs() as f32;
+
+    let left = points[0];
+    let right = points[points.len() / 2];
+    let horizontal = (right.x - left.x).abs() as f32;
+
+    if horizontal < 1e-6 {
+        return 0.0;
+    }
+
+    let ratio = vertical / horizontal;
+    (ratio * 3.0).clamp(0.0, 1.0)
+}
+
+fn compute_eye_openness(eye_region: Option<arc::R<vn::FaceLandmarkRegion2d>>) -> f32 {
+    let region = match eye_region {
+        Some(r) => r,
+        None => return 1.0,
+    };
+
+    let points = region.normalized_points();
+    if points.len() < 6 {
+        return 1.0;
+    }
+
+    let half = points.len() / 2;
+    let top = points[half / 2];
+    let bottom = points[half + half / 2];
+    let vertical = (top.y - bottom.y).abs() as f32;
+
+    let left = points[0];
+    let right = points[half];
+    let horizontal = (right.x - left.x).abs() as f32;
+
+    if horizontal < 1e-6 {
+        return 1.0;
+    }
+
+    let ratio = vertical / horizontal;
+    (ratio * 4.0).clamp(0.0, 1.0)
+}
+
+unsafe fn rgba_to_bgra_copy(
+    src: &[u8],
+    dst: *mut u8,
+    width: usize,
+    height: usize,
+    src_row_bytes: usize,
+    dst_row_bytes: usize,
+) {
+    for y in 0..height {
+        let src_row = &src[y * src_row_bytes..];
+        let dst_row = unsafe { dst.add(y * dst_row_bytes) };
+        for x in 0..width {
+            let si = x * 4;
+            let di = x * 4;
+            let r = src_row[si];
+            let g = src_row[si + 1];
+            let b = src_row[si + 2];
+            let a = src_row[si + 3];
+            unsafe {
+                *dst_row.add(di) = b;
+                *dst_row.add(di + 1) = g;
+                *dst_row.add(di + 2) = r;
+                *dst_row.add(di + 3) = a;
+            }
+        }
+    }
+}
+
+unsafe extern "C-unwind" {
+    fn CVPixelBufferGetBaseAddress(pixel_buffer: &cv::PixelBuf) -> *mut c_void;
+    fn CVPixelBufferGetBytesPerRow(pixel_buffer: &cv::PixelBuf) -> usize;
+}

crates/project/src/configuration.rs

@@ -1182,6 +1182,15 @@ impl ProjectConfiguration {
     }
 }
 
+#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+pub enum AvatarBackground {
+    #[default]
+    Dark,
+    Light,
+    Gradient,
+}
+
 pub const SLOW_SMOOTHING_SAMPLES: usize = 24;
 pub const REGULAR_SMOOTHING_SAMPLES: usize = 16;
 pub const FAST_SMOOTHING_SAMPLES: usize = 10;

crates/rendering/Cargo.toml

@@ -9,6 +9,7 @@ workspace = true
 [dependencies]
 anyhow.workspace = true
 bytemuck = { version = "1.7", features = ["derive"] }
+cap-face-tracking = { path = "../face-tracking" }
 cap-flags = { path = "../flags" }
 cap-project = { path = "../project" }
 cap-video-decode = { path = "../video-decode" }