From a2fb1b40f690a1e4556a2ac058d7c069513b7402 Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Sun, 16 Jul 2023 17:07:19 +0800 Subject: [PATCH] Extract SurfaceCapture from ScreenEncoder Extract an interface SurfaceCapture from ScreenEncoder, representing a video source which can be rendered to a Surface for encoding. Split ScreenEncoder into: - ScreenCapture, implementing SurfaceCapture to capture the device screen, - SurfaceEncoder, to encode any SurfaceCapture. This separation prepares the introduction of another SurfaceCapture implementation to capture the camera instead of the device screen. PR #4213 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- .../com/genymobile/scrcpy/ScreenCapture.java | 83 +++++++++++++++++ .../java/com/genymobile/scrcpy/Server.java | 3 +- .../com/genymobile/scrcpy/SurfaceCapture.java | 61 +++++++++++++ ...ScreenEncoder.java => SurfaceEncoder.java} | 90 +++++-------------- 4 files changed, 166 insertions(+), 71 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java create mode 100644 server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java rename server/src/main/java/com/genymobile/scrcpy/{ScreenEncoder.java => SurfaceEncoder.java} (74%) diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java new file mode 100644 index 00000000..f9ac66b8 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java @@ -0,0 +1,83 @@ +package com.genymobile.scrcpy; + +import com.genymobile.scrcpy.wrappers.SurfaceControl; + +import android.graphics.Rect; +import android.os.Build; +import android.os.IBinder; +import android.view.Surface; + +public class ScreenCapture extends SurfaceCapture implements Device.RotationListener, Device.FoldListener { + + private final Device device; + private IBinder display; + + public ScreenCapture(Device device) { + this.device = device; + } + + @Override + public void init() { + display = createDisplay(); + device.setRotationListener(this); + device.setFoldListener(this); + } + + @Override + public void start(Surface surface) { + ScreenInfo screenInfo = device.getScreenInfo(); + Rect contentRect = screenInfo.getContentRect(); + + // does not include the locked video orientation + Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect(); + int videoRotation = screenInfo.getVideoRotation(); + int layerStack = device.getLayerStack(); + setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); + } + + @Override + public void release() { + device.setRotationListener(null); + device.setFoldListener(null); + SurfaceControl.destroyDisplay(display); + } + + @Override + public Size getSize() { + return device.getScreenInfo().getVideoSize(); + } + + @Override + public void setMaxSize(int maxSize) { + device.setMaxSize(maxSize); + } + + @Override + public void onFoldChanged(int displayId, boolean folded) { + requestReset(); + } + + @Override + public void onRotationChanged(int rotation) { + requestReset(); + } + + private static IBinder createDisplay() { + // Since Android 12 (preview), secure displays could not be created with shell permissions anymore. + // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S". + boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S".equals( + Build.VERSION.CODENAME)); + return SurfaceControl.createDisplay("scrcpy", secure); + } + + private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) { + SurfaceControl.openTransaction(); + try { + SurfaceControl.setDisplaySurface(display, surface); + SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect); + SurfaceControl.setDisplayLayerStack(display, layerStack); + } finally { + SurfaceControl.closeTransaction(); + } + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index dc85c965..0f73a19b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -132,7 +132,8 @@ public final class Server { if (video) { Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecMeta(), options.getSendFrameMeta()); - ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), + ScreenCapture screenCapture = new ScreenCapture(device); + SurfaceEncoder screenEncoder = new SurfaceEncoder(screenCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError()); asyncProcessors.add(screenEncoder); } diff --git a/server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java b/server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java new file mode 100644 index 00000000..45a0fd2f --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java @@ -0,0 +1,61 @@ +package com.genymobile.scrcpy; + +import android.view.Surface; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A video source which can be rendered on a Surface for encoding. + */ +public abstract class SurfaceCapture { + + private final AtomicBoolean resetCapture = new AtomicBoolean(); + + /** + * Request the encoding session to be restarted, for example if the capture implementation detects that the video source size has changed (on + * device rotation for example). + */ + protected void requestReset() { + resetCapture.set(true); + } + + /** + * Consume the reset request (intended to be called by the encoder). + * + * @return {@code true} if a reset request was pending, {@code false} otherwise. + */ + public boolean consumeReset() { + return resetCapture.getAndSet(false); + } + + /** + * Called once before the capture starts. + */ + public abstract void init(); + + /** + * Called after the capture ends (if and only if {@link #init()} has been called). + */ + public abstract void release(); + + /** + * Start the capture to the target surface. + * + * @param surface the surface which will be encoded + */ + public abstract void start(Surface surface); + + /** + * Return the video size + * + * @return the video size + */ + public abstract Size getSize(); + + /** + * Set the maximum capture size (set by the encoder if it does not support the current size). + * + * @param maxSize Maximum size + */ + public abstract void setMaxSize(int maxSize); +} diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java similarity index 74% rename from server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java rename to server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java index 5a9db10d..4af31e89 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java @@ -1,13 +1,8 @@ package com.genymobile.scrcpy; -import com.genymobile.scrcpy.wrappers.SurfaceControl; - -import android.graphics.Rect; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaFormat; -import android.os.Build; -import android.os.IBinder; import android.os.Looper; import android.os.SystemClock; import android.view.Surface; @@ -17,7 +12,7 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -public class ScreenEncoder implements Device.RotationListener, Device.FoldListener, AsyncProcessor { +public class SurfaceEncoder implements AsyncProcessor { private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms @@ -27,9 +22,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800}; private static final int MAX_CONSECUTIVE_ERRORS = 3; - private final AtomicBoolean resetCapture = new AtomicBoolean(); - - private final Device device; + private final SurfaceCapture capture; private final Streamer streamer; private final String encoderName; private final List codecOptions; @@ -43,9 +36,9 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen private Thread thread; private final AtomicBoolean stopped = new AtomicBoolean(); - public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int maxFps, List codecOptions, String encoderName, + public SurfaceEncoder(SurfaceCapture capture, Streamer streamer, int videoBitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { - this.device = device; + this.capture = capture; this.streamer = streamer; this.videoBitRate = videoBitRate; this.maxFps = maxFps; @@ -54,51 +47,29 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen this.downsizeOnError = downsizeOnError; } - @Override - public void onFoldChanged(int displayId, boolean folded) { - resetCapture.set(true); - } - - @Override - public void onRotationChanged(int rotation) { - resetCapture.set(true); - } - - private boolean consumeResetCapture() { - return resetCapture.getAndSet(false); - } - private void streamScreen() throws IOException, ConfigurationException { Codec codec = streamer.getCodec(); MediaCodec mediaCodec = createMediaCodec(codec, encoderName); MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions); - IBinder display = createDisplay(); - device.setRotationListener(this); - device.setFoldListener(this); - streamer.writeVideoHeader(device.getScreenInfo().getVideoSize()); + capture.init(); - boolean alive; try { - do { - ScreenInfo screenInfo = device.getScreenInfo(); - Rect contentRect = screenInfo.getContentRect(); + streamer.writeVideoHeader(capture.getSize()); + + boolean alive; - // include the locked video orientation - Rect videoRect = screenInfo.getVideoSize().toRect(); - format.setInteger(MediaFormat.KEY_WIDTH, videoRect.width()); - format.setInteger(MediaFormat.KEY_HEIGHT, videoRect.height()); + do { + Size size = capture.getSize(); + format.setInteger(MediaFormat.KEY_WIDTH, size.getWidth()); + format.setInteger(MediaFormat.KEY_HEIGHT, size.getHeight()); Surface surface = null; try { mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); surface = mediaCodec.createInputSurface(); - // does not include the locked video orientation - Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect(); - int videoRotation = screenInfo.getVideoRotation(); - int layerStack = device.getLayerStack(); - setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); + capture.start(surface); mediaCodec.start(); @@ -107,7 +78,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen mediaCodec.stop(); } catch (IllegalStateException | IllegalArgumentException e) { Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage()); - if (!prepareRetry(device, screenInfo)) { + if (!prepareRetry(size)) { throw e; } Ln.i("Retrying..."); @@ -121,13 +92,11 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen } while (alive); } finally { mediaCodec.release(); - device.setRotationListener(null); - device.setFoldListener(null); - SurfaceControl.destroyDisplay(display); + capture.release(); } } - private boolean prepareRetry(Device device, ScreenInfo screenInfo) { + private boolean prepareRetry(Size currentSize) { if (firstFrameSent) { ++consecutiveErrors; if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { @@ -147,7 +116,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen // Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising) - int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize()); + int newMaxSize = chooseMaxSizeFallback(currentSize); if (newMaxSize == 0) { // Must definitively fail return false; @@ -155,7 +124,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen // Retry with a smaller device size Ln.i("Retrying with -m" + newMaxSize + "..."); - device.setMaxSize(newMaxSize); + capture.setMaxSize(newMaxSize); return true; } @@ -176,14 +145,14 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen boolean alive = true; MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); - while (!consumeResetCapture() && !eof) { + while (!capture.consumeReset() && !eof) { if (stopped.get()) { alive = false; break; } int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1); try { - if (consumeResetCapture()) { + if (capture.consumeReset()) { // must restart encoding with new size break; } @@ -264,25 +233,6 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen return format; } - private static IBinder createDisplay() { - // Since Android 12 (preview), secure displays could not be created with shell permissions anymore. - // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S". - boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S" - .equals(Build.VERSION.CODENAME)); - return SurfaceControl.createDisplay("scrcpy", secure); - } - - private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) { - SurfaceControl.openTransaction(); - try { - SurfaceControl.setDisplaySurface(display, surface); - SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect); - SurfaceControl.setDisplayLayerStack(display, layerStack); - } finally { - SurfaceControl.closeTransaction(); - } - } - @Override public void start(TerminationListener listener) { thread = new Thread(() -> {