feat(web-watcher): set audible on browser events from media sessions

mobile/src/main/java/net/activitywatch/android/watcher/BrowserAudibleDetector.kt

@@ -0,0 +1,61 @@
+package net.activitywatch.android.watcher
+
+import android.content.ComponentName
+import android.content.Context
+import android.media.session.MediaSessionManager
+import android.media.session.PlaybackState
+import android.os.SystemClock
+import android.util.Log
+
+// Reports whether a browser is currently playing audio, for the `audible` field on
+// aw-watcher-android-web events. A browser is considered audible when it owns a media
+// session in STATE_PLAYING (Chrome/Firefox publish one for page audio/video). That needs
+// the MediaWatcher notification-listener access; without it the answer is always false.
+//
+// AudioManager.isMusicActive() is deliberately not used as a fallback: it's device-wide,
+// so a music app in the background would mark silent browser sessions audible, and
+// aw-webui treats audible browser events as not-AFK evidence.
+internal class BrowserAudibleDetector(context: Context) {
+    private val TAG = "BrowserAudibleDetector"
+    private val appContext = context.applicationContext
+    private val listenerComponent = ComponentName(appContext, MediaWatcher::class.java)
+    private val sessionManager =
+        appContext.getSystemService(Context.MEDIA_SESSION_SERVICE) as? MediaSessionManager
+
+    private var cachedBrowser: String? = null
+    private var cachedAt = 0L
+    private var cachedResult = false
+
+    // onAccessibilityEvent runs on the service's main thread and fires many times a second
+    // while scrolling; getActiveSessions is a binder IPC that also builds a MediaController
+    // per session, and blocking that thread is what produced the WebWatcher ANRs in
+    // aw-android#261. So the answer is cached briefly instead of being recomputed per event.
+    fun isAudible(browserPackage: String): Boolean {
+        val nowMs = SystemClock.elapsedRealtime()
+        if (browserPackage == cachedBrowser && nowMs - cachedAt < CACHE_MS) return cachedResult
+
+        cachedResult = browserHasPlayingSession(browserPackage)
+        cachedBrowser = browserPackage
+        cachedAt = nowMs
+        return cachedResult
+    }
+
+    private fun browserHasPlayingSession(browserPackage: String): Boolean {
+        val manager = sessionManager ?: return false
+        if (!MediaWatcher.isNotificationAccessGranted(appContext)) return false
+        return try {
+            manager.getActiveSessions(listenerComponent).any { controller ->
+                controller.packageName == browserPackage &&
+                    controller.playbackState?.state == PlaybackState.STATE_PLAYING
+            }
+        } catch (e: SecurityException) {
+            // Access can be revoked between the settings check and the call.
+            Log.w(TAG, "Media session access denied: ${e.message}")
+            false
+        }
+    }
+
+    companion object {
+        private const val CACHE_MS = 1000L
+    }
+}

mobile/src/main/java/net/activitywatch/android/watcher/BrowserSessionTracker.kt

@@ -7,6 +7,7 @@ internal data class CompletedBrowserSession(
     val url: String,
     val browser: String,
     val title: String,
+    val audible: Boolean,
     val start: Instant,
     val duration: Duration
 )
@@ -20,33 +21,37 @@ internal class BrowserSessionTracker(
     private var lastUrl: String? = null
     private var lastBrowser: String? = null
     private var lastWindowTitle: String? = null
+    private var lastAudible: Boolean = false
 
     // Returns the just-completed session (previous url/browser/title) when the url or
     // browser changes, so the caller can log it. We wait for the url to change before
     // logging so we have a chance to receive the page title, which often only arrives
     // after the page loads and/or the user interacts with it.
-    fun handleUrl(newUrl: String?, newBrowser: String?): CompletedBrowserSession? {
-        if (newUrl == lastUrl && newBrowser == lastBrowser) return null
-
-        val completed = lastUrl?.let { url ->
-            lastBrowser?.let { browser ->
-                val start = lastUrlTimestamp!!
-                CompletedBrowserSession(
-                    url = url,
-                    browser = browser,
-                    title = lastWindowTitle ?: "",
-                    start = start,
-                    // Clock can step backward (NTP sync, manual change) between `start` and now;
-                    // don't report a negative duration in that case.
-                    duration = Duration.between(start, now()).coerceAtLeast(Duration.ZERO)
-                )
-            }
-        }
+    fun handleUrl(newUrl: String?, newBrowser: String?, audible: Boolean = false): CompletedBrowserSession? {
+        // Same page: nothing to log for the url, but playback may have started/stopped.
+        if (newUrl == lastUrl && newBrowser == lastBrowser) return handleAudible(audible)
+
+        val completed = completeCurrentSession()
 
         lastUrlTimestamp = now()
         lastUrl = newUrl
         lastBrowser = newBrowser
         lastWindowTitle = null
+        lastAudible = audible
+        return completed
+    }
+
+    // Splits the current session when its audible state flips, so the logged events carry
+    // the right `audible` value for each stretch of time (like the desktop web watcher,
+    // where a data change starts a new event). The url, browser and title carry over into
+    // the new session because it's still the same page.
+    fun handleAudible(audible: Boolean): CompletedBrowserSession? {
+        if (lastUrl == null || audible == lastAudible) return null
+
+        val completed = completeCurrentSession()
+
+        lastUrlTimestamp = now()
+        lastAudible = audible
         return completed
     }
 
@@ -56,4 +61,20 @@ internal class BrowserSessionTracker(
         lastWindowTitle = newWindowTitle
         return true
     }
+
+    private fun completeCurrentSession(): CompletedBrowserSession? {
+        val url = lastUrl ?: return null
+        val browser = lastBrowser ?: return null
+        val start = lastUrlTimestamp!!
+        return CompletedBrowserSession(
+            url = url,
+            browser = browser,
+            title = lastWindowTitle ?: "",
+            audible = lastAudible,
+            start = start,
+            // Clock can step backward (NTP sync, manual change) between `start` and now;
+            // don't report a negative duration in that case.
+            duration = Duration.between(start, now()).coerceAtLeast(Duration.ZERO)
+        )
+    }
 }

mobile/src/main/java/net/activitywatch/android/watcher/WebWatcher.kt

@@ -45,6 +45,7 @@ class WebWatcher : AccessibilityService() {
     @Volatile private var ri : RustInterface? = null
     private var lastWindowId: Int? = null
     private val sessionTracker = BrowserSessionTracker()
+    private lateinit var audibleDetector: BrowserAudibleDetector
 
     // Applies stripProtocol uniformly to whatever extractor matched, so the logged url is
     // formatted identically no matter which browser/view-variant produced it.
@@ -69,6 +70,7 @@ class WebWatcher : AccessibilityService() {
     override fun onCreate() {
         super.onCreate()
         Log.i(TAG, "Creating WebWatcher")
+        audibleDetector = BrowserAudibleDetector(this)
         // createBucketHelper() blocks on the datastore worker. Doing that on the
         // accessibility service's main thread produced "Executing service
         // WebWatcher" ANRs whenever the worker was busy (aw-android#261), so
@@ -113,11 +115,15 @@ class WebWatcher : AccessibilityService() {
                 try {
                     val browser = packageName!!
                     val newUrl = extractUrl(browser, event)
+                    val audible = audibleDetector.isAudible(browser)
 
                     if (newUrl == null) {
                         maybeDumpTree(browser)
+                        // Still on the previous url; only the audible state may have moved.
+                        handleAudible(audible)
                     } else {
-                        handleUrl(newUrl, newBrowser = browser)
+                        // Also covers the same-url case: the tracker splits on an audible change.
+                        handleUrl(newUrl, newBrowser = browser, audible = audible)
                     }
                     findWebView(source)?.let { webView ->
                         handleWindowTitle(webView.text.toString())
@@ -178,9 +184,16 @@ class WebWatcher : AccessibilityService() {
         }
     }
 
-    private fun handleUrl(newUrl : String?, newBrowser: String?) {
-        newUrl?.let { Log.i(TAG, "Url: $it, browser: $newBrowser") }
-        sessionTracker.handleUrl(newUrl, newBrowser)?.let { logBrowserEvent(it) }
+    private fun handleUrl(newUrl : String?, newBrowser: String?, audible: Boolean = false) {
+        newUrl?.let { Log.i(TAG, "Url: $it, browser: $newBrowser, audible: $audible") }
+        sessionTracker.handleUrl(newUrl, newBrowser, audible)?.let { logBrowserEvent(it) }
+    }
+
+    private fun handleAudible(audible: Boolean) {
+        sessionTracker.handleAudible(audible)?.let {
+            Log.i(TAG, "Audible changed to $audible; splitting session")
+            logBrowserEvent(it)
+        }
     }
 
     private fun handleWindowTitle(newWindowTitle: String) {
@@ -194,7 +207,7 @@ class WebWatcher : AccessibilityService() {
             .put("url", session.url)
             .put("browser", session.browser)
             .put("title", session.title)
-            .put("audible", false) // TODO
+            .put("audible", session.audible)
             .put("incognito", false) // TODO
 
         Log.i(TAG, "Registered event: $data")

mobile/src/test/java/net/activitywatch/android/watcher/BrowserSessionTrackerTest.kt

@@ -1,7 +1,9 @@
 package net.activitywatch.android.watcher
 
 import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
 import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
 import org.junit.Test
 import org.threeten.bp.Instant
 
@@ -127,4 +129,89 @@ class BrowserSessionTrackerTest {
         checkNotNull(completed)
         assertEquals(0L, completed.duration.seconds)
     }
+    @Test
+    fun `audible defaults to false and is attached to the completed session`() {
+        val clock = FakeClock(Instant.ofEpochSecond(1000))
+        val tracker = BrowserSessionTracker(clock::now)
+
+        tracker.handleUrl("example.com", "chrome")
+        val first = tracker.handleUrl("example.org", "chrome", audible = true)
+        val second = tracker.handleUrl("example.net", "chrome")
+
+        checkNotNull(first)
+        assertFalse(first.audible)
+        checkNotNull(second)
+        assertTrue(second.audible)
+    }
+
+    @Test
+    fun `audible change splits the session and keeps url browser and title`() {
+        val clock = FakeClock(Instant.ofEpochSecond(1000))
+        val tracker = BrowserSessionTracker(clock::now)
+
+        tracker.handleUrl("example.com", "chrome", audible = false)
+        tracker.handleWindowTitle("Example Domain")
+        clock.advanceSeconds(10)
+        val silent = tracker.handleAudible(true)
+        clock.advanceSeconds(20)
+        val playing = tracker.handleUrl("example.org", "chrome")
+
+        checkNotNull(silent)
+        assertEquals("example.com", silent.url)
+        assertEquals("chrome", silent.browser)
+        assertEquals("Example Domain", silent.title)
+        assertFalse(silent.audible)
+        assertEquals(Instant.ofEpochSecond(1000), silent.start)
+        assertEquals(10L, silent.duration.seconds)
+
+        checkNotNull(playing)
+        assertEquals("example.com", playing.url)
+        assertEquals("Example Domain", playing.title)
+        assertTrue(playing.audible)
+        assertEquals(Instant.ofEpochSecond(1010), playing.start)
+        assertEquals(20L, playing.duration.seconds)
+    }
+
+    @Test
+    fun `same url with a changed audible state splits the session via handleUrl`() {
+        val clock = FakeClock(Instant.ofEpochSecond(1000))
+        val tracker = BrowserSessionTracker(clock::now)
+
+        tracker.handleUrl("example.com", "chrome", audible = false)
+        clock.advanceSeconds(10)
+        val silent = tracker.handleUrl("example.com", "chrome", audible = true)
+        clock.advanceSeconds(20)
+        val playing = tracker.handleUrl("example.org", "chrome")
+
+        checkNotNull(silent)
+        assertFalse(silent.audible)
+        assertEquals(10L, silent.duration.seconds)
+        checkNotNull(playing)
+        assertEquals("example.com", playing.url)
+        assertTrue(playing.audible)
+        assertEquals(20L, playing.duration.seconds)
+    }
+
+    @Test
+    fun `unchanged audible state does not split the session`() {
+        val clock = FakeClock(Instant.ofEpochSecond(1000))
+        val tracker = BrowserSessionTracker(clock::now)
+
+        tracker.handleUrl("example.com", "chrome", audible = true)
+        clock.advanceSeconds(5)
+
+        assertNull(tracker.handleAudible(true))
+    }
+
+    @Test
+    fun `audible without an active session is ignored`() {
+        val tracker = BrowserSessionTracker()
+
+        assertNull(tracker.handleAudible(true))
+
+        // Ending a session (window changed away) also leaves nothing to split.
+        tracker.handleUrl("example.com", "chrome")
+        tracker.handleUrl(null, null)
+        assertNull(tracker.handleAudible(true))
+    }
 }