Extract SurfaceCapture from ScreenEncoder

Extract an interface SurfaceCapture from ScreenEncoder, representing a
video source which can be rendered to a Surface for encoding.

Split ScreenEncoder into:
 - ScreenCapture, implementing SurfaceCapture to capture the device
   screen,
 - SurfaceEncoder, to encode any SurfaceCapture.

This separation prepares the introduction of another SurfaceCapture
implementation to capture the camera instead of the device screen.

PR #4213 <https://github.com/Genymobile/scrcpy/pull/4213>

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>

server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java

@@ -0,0 +1,83 @@
+package com.genymobile.scrcpy;
+
+import com.genymobile.scrcpy.wrappers.SurfaceControl;
+
+import android.graphics.Rect;
+import android.os.Build;
+import android.os.IBinder;
+import android.view.Surface;
+
+public class ScreenCapture extends SurfaceCapture implements Device.RotationListener, Device.FoldListener {
+
+    private final Device device;
+    private IBinder display;
+
+    public ScreenCapture(Device device) {
+        this.device = device;
+    }
+
+    @Override
+    public void init() {
+        display = createDisplay();
+        device.setRotationListener(this);
+        device.setFoldListener(this);
+    }
+
+    @Override
+    public void start(Surface surface) {
+        ScreenInfo screenInfo = device.getScreenInfo();
+        Rect contentRect = screenInfo.getContentRect();
+
+        // does not include the locked video orientation
+        Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
+        int videoRotation = screenInfo.getVideoRotation();
+        int layerStack = device.getLayerStack();
+        setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
+    }
+
+    @Override
+    public void release() {
+        device.setRotationListener(null);
+        device.setFoldListener(null);
+        SurfaceControl.destroyDisplay(display);
+    }
+
+    @Override
+    public Size getSize() {
+        return device.getScreenInfo().getVideoSize();
+    }
+
+    @Override
+    public void setMaxSize(int maxSize) {
+        device.setMaxSize(maxSize);
+    }
+
+    @Override
+    public void onFoldChanged(int displayId, boolean folded) {
+        requestReset();
+    }
+
+    @Override
+    public void onRotationChanged(int rotation) {
+        requestReset();
+    }
+
+    private static IBinder createDisplay() {
+        // Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
+        // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
+        boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S".equals(
+                Build.VERSION.CODENAME));
+        return SurfaceControl.createDisplay("scrcpy", secure);
+    }
+
+    private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
+        SurfaceControl.openTransaction();
+        try {
+            SurfaceControl.setDisplaySurface(display, surface);
+            SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
+            SurfaceControl.setDisplayLayerStack(display, layerStack);
+        } finally {
+            SurfaceControl.closeTransaction();
+        }
+    }
+}

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -132,7 +132,8 @@ public final class Server {
             if (video) {
                 Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecMeta(),
                         options.getSendFrameMeta());
-                ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
+                ScreenCapture screenCapture = new ScreenCapture(device);
+                SurfaceEncoder screenEncoder = new SurfaceEncoder(screenCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
                         options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());
                 asyncProcessors.add(screenEncoder);
             }

server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java

@@ -0,0 +1,61 @@
+package com.genymobile.scrcpy;
+
+import android.view.Surface;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A video source which can be rendered on a Surface for encoding.
+ */
+public abstract class SurfaceCapture {
+
+    private final AtomicBoolean resetCapture = new AtomicBoolean();
+
+    /**
+     * Request the encoding session to be restarted, for example if the capture implementation detects that the video source size has changed (on
+     * device rotation for example).
+     */
+    protected void requestReset() {
+        resetCapture.set(true);
+    }
+
+    /**
+     * Consume the reset request (intended to be called by the encoder).
+     *
+     * @return {@code true} if a reset request was pending, {@code false} otherwise.
+     */
+    public boolean consumeReset() {
+        return resetCapture.getAndSet(false);
+    }
+
+    /**
+     * Called once before the capture starts.
+     */
+    public abstract void init();
+
+    /**
+     * Called after the capture ends (if and only if {@link #init()} has been called).
+     */
+    public abstract void release();
+
+    /**
+     * Start the capture to the target surface.
+     *
+     * @param surface the surface which will be encoded
+     */
+    public abstract void start(Surface surface);
+
+    /**
+     * Return the video size
+     *
+     * @return the video size
+     */
+    public abstract Size getSize();
+
+    /**
+     * Set the maximum capture size (set by the encoder if it does not support the current size).
+     *
+     * @param maxSize Maximum size
+     */
+    public abstract void setMaxSize(int maxSize);
+}

server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java

@@ -1,13 +1,8 @@
 package com.genymobile.scrcpy;
 
-import com.genymobile.scrcpy.wrappers.SurfaceControl;
-
-import android.graphics.Rect;
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
-import android.os.Build;
-import android.os.IBinder;
 import android.os.Looper;
 import android.os.SystemClock;
 import android.view.Surface;
@@ -17,7 +12,7 @@ import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-public class ScreenEncoder implements Device.RotationListener, Device.FoldListener, AsyncProcessor {
+public class SurfaceEncoder implements AsyncProcessor {
 
     private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
     private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms
@@ -27,9 +22,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
     private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800};
     private static final int MAX_CONSECUTIVE_ERRORS = 3;
 
-    private final AtomicBoolean resetCapture = new AtomicBoolean();
+    private final SurfaceCapture capture;
-
-    private final Device device;
     private final Streamer streamer;
     private final String encoderName;
     private final List<CodecOption> codecOptions;
@@ -43,9 +36,9 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
     private Thread thread;
     private final AtomicBoolean stopped = new AtomicBoolean();
 
-    public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
+    public SurfaceEncoder(SurfaceCapture capture, Streamer streamer, int videoBitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
             boolean downsizeOnError) {
-        this.device = device;
+        this.capture = capture;
         this.streamer = streamer;
         this.videoBitRate = videoBitRate;
         this.maxFps = maxFps;
@@ -54,51 +47,29 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
         this.downsizeOnError = downsizeOnError;
     }
 
-    @Override
-    public void onFoldChanged(int displayId, boolean folded) {
-        resetCapture.set(true);
-    }
-
-    @Override
-    public void onRotationChanged(int rotation) {
-        resetCapture.set(true);
-    }
-
-    private boolean consumeResetCapture() {
-        return resetCapture.getAndSet(false);
-    }
-
     private void streamScreen() throws IOException, ConfigurationException {
         Codec codec = streamer.getCodec();
         MediaCodec mediaCodec = createMediaCodec(codec, encoderName);
         MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions);
-        IBinder display = createDisplay();
-        device.setRotationListener(this);
-        device.setFoldListener(this);
 
-        streamer.writeVideoHeader(device.getScreenInfo().getVideoSize());
+        capture.init();
 
-        boolean alive;
         try {
-            do {
+            streamer.writeVideoHeader(capture.getSize());
-                ScreenInfo screenInfo = device.getScreenInfo();
+
-                Rect contentRect = screenInfo.getContentRect();
+            boolean alive;
 
-                // include the locked video orientation
+            do {
-                Rect videoRect = screenInfo.getVideoSize().toRect();
+                Size size = capture.getSize();
-                format.setInteger(MediaFormat.KEY_WIDTH, videoRect.width());
+                format.setInteger(MediaFormat.KEY_WIDTH, size.getWidth());
-                format.setInteger(MediaFormat.KEY_HEIGHT, videoRect.height());
+                format.setInteger(MediaFormat.KEY_HEIGHT, size.getHeight());
 
                 Surface surface = null;
                 try {
                     mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
                     surface = mediaCodec.createInputSurface();
 
-                    // does not include the locked video orientation
+                    capture.start(surface);
-                    Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
-                    int videoRotation = screenInfo.getVideoRotation();
-                    int layerStack = device.getLayerStack();
-                    setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
 
                     mediaCodec.start();
 
@@ -107,7 +78,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
                     mediaCodec.stop();
                 } catch (IllegalStateException | IllegalArgumentException e) {
                     Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage());
-                    if (!prepareRetry(device, screenInfo)) {
+                    if (!prepareRetry(size)) {
                         throw e;
                     }
                     Ln.i("Retrying...");
@@ -121,13 +92,11 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
             } while (alive);
         } finally {
             mediaCodec.release();
-            device.setRotationListener(null);
+            capture.release();
-            device.setFoldListener(null);
-            SurfaceControl.destroyDisplay(display);
         }
     }
 
-    private boolean prepareRetry(Device device, ScreenInfo screenInfo) {
+    private boolean prepareRetry(Size currentSize) {
         if (firstFrameSent) {
             ++consecutiveErrors;
             if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
@@ -147,7 +116,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
 
         // Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising)
 
-        int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
+        int newMaxSize = chooseMaxSizeFallback(currentSize);
         if (newMaxSize == 0) {
             // Must definitively fail
             return false;
@@ -155,7 +124,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
 
         // Retry with a smaller device size
         Ln.i("Retrying with -m" + newMaxSize + "...");
-        device.setMaxSize(newMaxSize);
+        capture.setMaxSize(newMaxSize);
         return true;
     }
 
@@ -176,14 +145,14 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
         boolean alive = true;
         MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
 
-        while (!consumeResetCapture() && !eof) {
+        while (!capture.consumeReset() && !eof) {
             if (stopped.get()) {
                 alive = false;
                 break;
             }
             int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
             try {
-                if (consumeResetCapture()) {
+                if (capture.consumeReset()) {
                     // must restart encoding with new size
                     break;
                 }
@@ -264,25 +233,6 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
         return format;
     }
 
-    private static IBinder createDisplay() {
-        // Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
-        // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
-        boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S"
-                .equals(Build.VERSION.CODENAME));
-        return SurfaceControl.createDisplay("scrcpy", secure);
-    }
-
-    private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
-        SurfaceControl.openTransaction();
-        try {
-            SurfaceControl.setDisplaySurface(display, surface);
-            SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
-            SurfaceControl.setDisplayLayerStack(display, layerStack);
-        } finally {
-            SurfaceControl.closeTransaction();
-        }
-    }
-
     @Override
     public void start(TerminationListener listener) {
         thread = new Thread(() -> {