diff --git a/README.md b/README.md index ae168c40..b4aac9f2 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,22 @@ Higher values → leaner bandwidth, less resilience. --- +## Object Detection (Optional) +PixelPilot features an optional, real-time object detection overlay using Google's MediaPipe Object Detection library. + +### Features +* **Live Overlay**: Draws bounding boxes, class names, and confidence scores directly over the standard video stream. +* **Hardware Acceleration**: Toggle between CPU and GPU delegates to optimize performance for your device. +* **Model Choices**: Supports switching between `EfficientDet-Lite0` (lightweight/faster) and `EfficientDet-Lite2` (larger/more accurate) models. +* **Automated Setup**: The necessary TFLite model files are downloaded automatically from Google's CDN during the Gradle build phase. + +### How to Use +1. In the application settings menu, click on the **Object Detection** submenu. +2. Select **Enable** to toggle the detection overlay on/off. +3. Choose your preferred hardware **Delegate** (CPU/GPU) and **Model** (Lite0/Lite2) based on your device's processing capabilities. + +--- + ## List of potential improvements: * adaptive link [x] * 40 MHz bandwidth [?] - works but buggy diff --git a/app/build.gradle b/app/build.gradle index 3bb89146..3bfee20a 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -1,5 +1,6 @@ plugins { alias(libs.plugins.androidApplication) + alias(libs.plugins.gradleDownload) } android { @@ -40,10 +41,29 @@ android { buildFeatures { viewBinding = true } + androidResources { + noCompress 'tflite' + } buildToolsVersion '35.0.0' ndkVersion '26.1.10909125' } +project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets' + +task downloadModelFile0(type: de.undercouch.gradle.tasks.download.Download) { + src 'https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/float32/1/efficientdet_lite0.tflite' + dest project.ext.ASSET_DIR + '/efficientdet-lite0.tflite' + overwrite false +} + +task downloadModelFile1(type: de.undercouch.gradle.tasks.download.Download) { + src 'https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite2/float32/1/efficientdet_lite2.tflite' + dest project.ext.ASSET_DIR + '/efficientdet-lite2.tflite' + overwrite false +} + +preBuild.dependsOn downloadModelFile0, downloadModelFile1 + dependencies { implementation(project(":app:mavlink")) implementation(project(":app:videonative")) @@ -52,4 +72,5 @@ dependencies { implementation(libs.appcompat) implementation(libs.material) implementation(libs.philjay.mpandroidchart) + implementation(libs.mediapipe.tasks.vision) } diff --git a/app/src/main/assets/efficientdet-lite0.tflite b/app/src/main/assets/efficientdet-lite0.tflite new file mode 100644 index 00000000..74fa351d Binary files /dev/null and b/app/src/main/assets/efficientdet-lite0.tflite differ diff --git a/app/src/main/assets/efficientdet-lite2.tflite b/app/src/main/assets/efficientdet-lite2.tflite new file mode 100644 index 00000000..fffedaa4 Binary files /dev/null and b/app/src/main/assets/efficientdet-lite2.tflite differ diff --git a/app/src/main/java/com/openipc/pixelpilot/ObjectDetectorHelper.java b/app/src/main/java/com/openipc/pixelpilot/ObjectDetectorHelper.java new file mode 100644 index 00000000..6dff4181 --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/ObjectDetectorHelper.java @@ -0,0 +1,106 @@ +package com.openipc.pixelpilot; + +import android.content.Context; +import android.graphics.Bitmap; +import android.os.SystemClock; +import android.util.Log; + +import com.google.mediapipe.framework.image.BitmapImageBuilder; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.core.BaseOptions; +import com.google.mediapipe.tasks.core.Delegate; +import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetector; +import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetectorResult; + +import java.util.Collections; +import java.util.List; + +public class ObjectDetectorHelper { + private static final String TAG = "ObjectDetectorHelper"; + + public static final int DELEGATE_CPU = 0; + public static final int DELEGATE_GPU = 1; + + public static final int MODEL_EFFICIENTDETV0 = 0; + public static final int MODEL_EFFICIENTDETV2 = 1; + + private final Context context; + private final float threshold; + private final int maxResults; + private final int currentDelegate; + private final int currentModel; + + private ObjectDetector objectDetector; + + public ObjectDetectorHelper(Context context, float threshold, int maxResults, int delegate, int model) { + this.context = context; + this.threshold = threshold; + this.maxResults = maxResults; + this.currentDelegate = delegate; + this.currentModel = model; + setupObjectDetector(); + } + + private void setupObjectDetector() { + BaseOptions.Builder baseOptionsBuilder = BaseOptions.builder(); + if (currentDelegate == DELEGATE_GPU) { + baseOptionsBuilder.setDelegate(Delegate.GPU); + } else { + baseOptionsBuilder.setDelegate(Delegate.CPU); + } + + String modelName = "efficientdet-lite0.tflite"; + if (currentModel == MODEL_EFFICIENTDETV2) { + modelName = "efficientdet-lite2.tflite"; + } + baseOptionsBuilder.setModelAssetPath(modelName); + + try { + ObjectDetector.ObjectDetectorOptions options = ObjectDetector.ObjectDetectorOptions.builder() + .setBaseOptions(baseOptionsBuilder.build()) + .setScoreThreshold(threshold) + .setMaxResults(maxResults) + .setRunningMode(RunningMode.IMAGE) + .build(); + objectDetector = ObjectDetector.createFromOptions(context, options); + } catch (Exception e) { + Log.e(TAG, "Failed to load model: " + e.getMessage(), e); + } + } + + public ResultBundle detectImage(Bitmap image) { + if (objectDetector == null) return null; + long startTime = SystemClock.uptimeMillis(); + MPImage mpImage = new BitmapImageBuilder(image).build(); + try { + ObjectDetectorResult detectionResult = objectDetector.detect(mpImage); + long inferenceTime = SystemClock.uptimeMillis() - startTime; + return new ResultBundle(Collections.singletonList(detectionResult), inferenceTime, image.getHeight(), image.getWidth()); + } catch (Exception e) { + Log.e(TAG, "Detection failed: " + e.getMessage(), e); + return null; + } + } + + public void clear() { + if (objectDetector != null) { + objectDetector.close(); + objectDetector = null; + } + } + + public static class ResultBundle { + public final List results; + public final long inferenceTime; + public final int inputImageHeight; + public final int inputImageWidth; + + public ResultBundle(List results, long inferenceTime, int inputImageHeight, int inputImageWidth) { + this.results = results; + this.inferenceTime = inferenceTime; + this.inputImageHeight = inputImageHeight; + this.inputImageWidth = inputImageWidth; + } + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/OverlayView.java b/app/src/main/java/com/openipc/pixelpilot/OverlayView.java new file mode 100644 index 00000000..0ced971b --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/OverlayView.java @@ -0,0 +1,103 @@ +package com.openipc.pixelpilot; + +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.Rect; +import android.graphics.RectF; +import android.util.AttributeSet; +import android.view.View; + +import com.google.mediapipe.tasks.components.containers.Detection; +import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetectorResult; + +public class OverlayView extends View { + private ObjectDetectorResult results; + private final Paint boxPaint = new Paint(); + private final Paint textBackgroundPaint = new Paint(); + private final Paint textPaint = new Paint(); + private float scaleFactor = 1.0f; + private final Rect bounds = new Rect(); + private int outputWidth = 0; + private int outputHeight = 0; + + private static final int BOUNDING_RECT_TEXT_PADDING = 8; + + public OverlayView(Context context, AttributeSet attrs) { + super(context, attrs); + initPaints(); + } + + private void initPaints() { + textBackgroundPaint.setColor(Color.BLACK); + textBackgroundPaint.setStyle(Paint.Style.FILL); + textBackgroundPaint.setTextSize(40f); + + textPaint.setColor(Color.WHITE); + textPaint.setStyle(Paint.Style.FILL); + textPaint.setTextSize(40f); + + boxPaint.setColor(Color.GREEN); + boxPaint.setStrokeWidth(6f); + boxPaint.setStyle(Paint.Style.STROKE); + } + + public void clear() { + results = null; + invalidate(); + } + + @Override + protected void onDraw(Canvas canvas) { + super.onDraw(canvas); + if (results == null || results.detections() == null) return; + + for (Detection detection : results.detections()) { + RectF box = detection.boundingBox(); + + // Adjust box coordinates to view scaling + float left = box.left * scaleFactor; + float top = box.top * scaleFactor; + float right = box.right * scaleFactor; + float bottom = box.bottom * scaleFactor; + + // Draw bounding box + canvas.drawRect(left, top, right, bottom, boxPaint); + + // Category text + if (!detection.categories().isEmpty()) { + String categoryName = detection.categories().get(0).categoryName(); + float score = detection.categories().get(0).score(); + String text = String.format("%s %.2f", categoryName, score); + + // Draw background rect for text + textBackgroundPaint.getTextBounds(text, 0, text.length(), bounds); + float textWidth = bounds.width(); + float textHeight = bounds.height(); + + canvas.drawRect( + left, + top, + left + textWidth + BOUNDING_RECT_TEXT_PADDING, + top + textHeight + BOUNDING_RECT_TEXT_PADDING, + textBackgroundPaint + ); + + // Draw text + canvas.drawText(text, left, top + textHeight, textPaint); + } + } + } + + public void setResults(ObjectDetectorResult detectionResults, int inputHeight, int inputWidth) { + this.results = detectionResults; + this.outputWidth = inputWidth; + this.outputHeight = inputHeight; + + // Calculate scaling factor to scale bounding boxes to match actual display size of the View + scaleFactor = Math.min(getWidth() * 1.0f / inputWidth, getHeight() * 1.0f / inputHeight); + + invalidate(); + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index e3c8da26..af5a50b9 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -80,6 +80,13 @@ import java.util.Locale; import java.util.Timer; import java.util.TimerTask; +import android.graphics.Bitmap; +import android.os.SystemClock; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import android.view.TextureView; +import android.graphics.SurfaceTexture; +import android.view.Surface; // Most basic implementation of an activity that uses VideoNative to stream a video // Into an Android Surface View @@ -112,6 +119,12 @@ public void run() { private ConstraintSet constraintSet; private WfbNgLink wfbLink; + private ObjectDetectorHelper objectDetectorHelper; + private ExecutorService objectDetectionExecutor; + private volatile boolean isObjectDetectionEnabled = false; + private volatile boolean isDetecting = false; + private final Object detectorLock = new Object(); + private static final String PREF_DRONE_USERNAME = "drone_username"; private static final String PREF_DRONE_PASSWORD = "drone_password"; @@ -344,7 +357,7 @@ private void setupVRVideoPlayers() { private void setupStandardVideoPlayer() { binding.surfaceViewRight.setVisibility(View.GONE); binding.surfaceViewLeft.setVisibility(View.GONE); - binding.mainVideo.getHolder().addCallback(videoPlayer.configure1(0)); + binding.mainVideo.setSurfaceTextureListener(videoPlayer.configureTextureView(0)); } // ---------------------------------------------------------------------------- @@ -574,6 +587,9 @@ private void showSettingsMenu(View anchor) { // Help submenu setupHelpSubMenu(popup); + // Object Detection submenu + setupObjectDetectionSubMenu(popup); + popup.show(); } @@ -1407,6 +1423,8 @@ public void unregisterReceivers() { protected void onPause() { super.onPause(); + stopObjectDetectionLoop(); + unregisterReceivers(); videoPlayer.stop(); @@ -1446,6 +1464,10 @@ protected void onResume() { updateUdpForwardingState(); videoPlayer.startAudio(); + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + boolean odEnabled = prefs.getBoolean("od_enabled", false); + setObjectDetectionEnabled(odEnabled); + osdManager.restoreOSDConfig(); startVpnService(); @@ -1737,4 +1759,172 @@ public void onReceivedHttpAuthRequest( } }); } + + private void setupObjectDetectionSubMenu(PopupMenu popup) { + SubMenu odMenu = popup.getMenu().addSubMenu("Object Detection"); + + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + boolean odEnabled = prefs.getBoolean("od_enabled", false); + + MenuItem enableItem = odMenu.add("Enable"); + enableItem.setCheckable(true); + enableItem.setChecked(odEnabled); + enableItem.setOnMenuItemClickListener(item -> { + boolean newState = !item.isChecked(); + item.setChecked(newState); + setObjectDetectionEnabled(newState); + return true; + }); + + SubMenu delegateMenu = odMenu.addSubMenu("Delegate"); + int savedDelegate = prefs.getInt("od_delegate", ObjectDetectorHelper.DELEGATE_CPU); + + MenuItem cpuItem = delegateMenu.add("CPU"); + cpuItem.setCheckable(true); + cpuItem.setChecked(savedDelegate == ObjectDetectorHelper.DELEGATE_CPU); + cpuItem.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_delegate", ObjectDetectorHelper.DELEGATE_CPU).apply(); + restartObjectDetector(); + return true; + }); + + MenuItem gpuItem = delegateMenu.add("GPU"); + gpuItem.setCheckable(true); + gpuItem.setChecked(savedDelegate == ObjectDetectorHelper.DELEGATE_GPU); + gpuItem.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_delegate", ObjectDetectorHelper.DELEGATE_GPU).apply(); + restartObjectDetector(); + return true; + }); + + SubMenu modelMenu = odMenu.addSubMenu("Model"); + int savedModel = prefs.getInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV0); + + MenuItem v0Item = modelMenu.add("EfficientDet-Lite0"); + v0Item.setCheckable(true); + v0Item.setChecked(savedModel == ObjectDetectorHelper.MODEL_EFFICIENTDETV0); + v0Item.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV0).apply(); + restartObjectDetector(); + return true; + }); + + MenuItem v2Item = modelMenu.add("EfficientDet-Lite2"); + v2Item.setCheckable(true); + v2Item.setChecked(savedModel == ObjectDetectorHelper.MODEL_EFFICIENTDETV2); + v2Item.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV2).apply(); + restartObjectDetector(); + return true; + }); + } + + private void setObjectDetectionEnabled(boolean enabled) { + isObjectDetectionEnabled = enabled; + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + prefs.edit().putBoolean("od_enabled", enabled).apply(); + + if (enabled) { + binding.detectionOverlay.setVisibility(View.VISIBLE); + startObjectDetectionLoop(); + } else { + binding.detectionOverlay.setVisibility(View.GONE); + binding.detectionOverlay.clear(); + stopObjectDetectionLoop(); + } + } + + private void restartObjectDetector() { + if (isObjectDetectionEnabled) { + stopObjectDetectionLoop(); + startObjectDetectionLoop(); + } + } + + private void startObjectDetectionLoop() { + if (isVRMode) return; // Standard mode only + if (objectDetectionExecutor == null) { + objectDetectionExecutor = Executors.newSingleThreadExecutor(); + } + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + int delegate = prefs.getInt("od_delegate", ObjectDetectorHelper.DELEGATE_CPU); + int model = prefs.getInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV0); + + objectDetectionExecutor.execute(() -> { + synchronized (detectorLock) { + if (objectDetectorHelper != null) { + objectDetectorHelper.clear(); + } + objectDetectorHelper = new ObjectDetectorHelper( + VideoActivity.this, + 0.5f, + 3, + delegate, + model + ); + } + + isDetecting = true; + while (isDetecting && isObjectDetectionEnabled) { + Bitmap bitmap = null; + try { + long start = SystemClock.uptimeMillis(); + if (binding.mainVideo != null && binding.mainVideo.isAvailable()) { + bitmap = binding.mainVideo.getBitmap(); + } + + if (bitmap != null) { + ObjectDetectorHelper.ResultBundle result = null; + synchronized (detectorLock) { + if (objectDetectorHelper != null) { + result = objectDetectorHelper.detectImage(bitmap); + } + } + + if (result != null && !result.results.isEmpty() && isObjectDetectionEnabled) { + final ObjectDetectorHelper.ResultBundle finalResult = result; + runOnUiThread(() -> { + if (isObjectDetectionEnabled) { + binding.detectionOverlay.setResults(finalResult.results.get(0), finalResult.inputImageHeight, finalResult.inputImageWidth); + } + }); + } else { + runOnUiThread(() -> { + if (isObjectDetectionEnabled) { + binding.detectionOverlay.clear(); + } + }); + } + } + + long sleepTime = 100 - (SystemClock.uptimeMillis() - start); // ~10 FPS + if (sleepTime > 0) { + Thread.sleep(sleepTime); + } + } catch (InterruptedException e) { + break; + } catch (Exception e) { + Log.e(TAG, "Error in detection loop", e); + } finally { + if (bitmap != null) { + bitmap.recycle(); + } + } + } + }); + } + + private void stopObjectDetectionLoop() { + isDetecting = false; + if (objectDetectionExecutor != null) { + objectDetectionExecutor.shutdownNow(); + objectDetectionExecutor = null; + } + synchronized (detectorLock) { + if (objectDetectorHelper != null) { + objectDetectorHelper.clear(); + objectDetectorHelper = null; + } + } + } } diff --git a/app/src/main/res/layout/activity_video.xml b/app/src/main/res/layout/activity_video.xml index f888475f..cabe29e6 100644 --- a/app/src/main/res/layout/activity_video.xml +++ b/app/src/main/res/layout/activity_video.xml @@ -5,7 +5,7 @@ android:layout_width="match_parent" android:layout_height="match_parent"> - + +