From 87972e2022686b1176bfaf0c678e703856c2b027 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Feb 2023 21:08:43 +0100 Subject: [PATCH] Extract video streaming to a separate class ScreenEncoder handled both capture/encoding and sending over the network. Move the streaming part to a separate VideoStreamer. --- .../com/genymobile/scrcpy/ScreenEncoder.java | 49 +++++------------- .../java/com/genymobile/scrcpy/Server.java | 7 +-- .../com/genymobile/scrcpy/VideoStreamer.java | 51 +++++++++++++++++++ 3 files changed, 67 insertions(+), 40 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 95f89ed5..1d66c0d1 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -12,7 +12,6 @@ import android.os.IBinder; import android.os.SystemClock; import android.view.Surface; -import java.io.FileDescriptor; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -22,6 +21,10 @@ import java.util.concurrent.atomic.AtomicBoolean; public class ScreenEncoder implements Device.RotationListener { + public interface Callbacks { + void onPacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException; + } + private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms private static final String KEY_MAX_FPS_TO_ENCODER = "max-fps-to-encoder"; @@ -30,25 +33,18 @@ public class ScreenEncoder implements Device.RotationListener { private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800}; private static final int MAX_CONSECUTIVE_ERRORS = 3; - private static final long PACKET_FLAG_CONFIG = 1L << 63; - private static final long PACKET_FLAG_KEY_FRAME = 1L << 62; - private final AtomicBoolean rotationChanged = new AtomicBoolean(); - private final ByteBuffer headerBuffer = ByteBuffer.allocate(12); private final String encoderName; private final List codecOptions; private final int bitRate; private final int maxFps; - private final boolean sendFrameMeta; private final boolean downsizeOnError; private boolean firstFrameSent; private int consecutiveErrors; - public ScreenEncoder(boolean sendFrameMeta, int bitRate, int maxFps, List codecOptions, String encoderName, - boolean downsizeOnError) { - this.sendFrameMeta = sendFrameMeta; + public ScreenEncoder(int bitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { this.bitRate = bitRate; this.maxFps = maxFps; this.codecOptions = codecOptions; @@ -65,7 +61,7 @@ public class ScreenEncoder implements Device.RotationListener { return rotationChanged.getAndSet(false); } - public void streamScreen(Device device, FileDescriptor fd) throws IOException { + public void streamScreen(Device device, Callbacks callbacks) throws IOException { MediaCodec codec = createCodec(encoderName); MediaFormat format = createFormat(bitRate, maxFps, codecOptions); IBinder display = createDisplay(); @@ -94,7 +90,7 @@ public class ScreenEncoder implements Device.RotationListener { codec.start(); - alive = encode(codec, fd); + alive = encode(codec, callbacks); // do not call stop() on exception, it would trigger an IllegalStateException codec.stop(); } catch (IllegalStateException | IllegalArgumentException e) { @@ -163,7 +159,7 @@ public class ScreenEncoder implements Device.RotationListener { return 0; } - private boolean encode(MediaCodec codec, FileDescriptor fd) throws IOException { + private boolean encode(MediaCodec codec, Callbacks callbacks) throws IOException { boolean eof = false; MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); @@ -179,16 +175,14 @@ public class ScreenEncoder implements Device.RotationListener { if (outputBufferId >= 0) { ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId); - if (sendFrameMeta) { - writeFrameMeta(fd, bufferInfo, codecBuffer.remaining()); - } - - IO.writeFully(fd, codecBuffer); - if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { + boolean isConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; + if (!isConfig) { // If this is not a config packet, then it contains a frame firstFrameSent = true; consecutiveErrors = 0; } + + callbacks.onPacket(codecBuffer, bufferInfo); } } finally { if (outputBufferId >= 0) { @@ -200,25 +194,6 @@ public class ScreenEncoder implements Device.RotationListener { return !eof; } - private void writeFrameMeta(FileDescriptor fd, MediaCodec.BufferInfo bufferInfo, int packetSize) throws IOException { - headerBuffer.clear(); - - long pts; - if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { - pts = PACKET_FLAG_CONFIG; // non-media data packet - } else { - pts = bufferInfo.presentationTimeUs; - if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) { - pts |= PACKET_FLAG_KEY_FRAME; - } - } - - headerBuffer.putLong(pts); - headerBuffer.putInt(packetSize); - headerBuffer.flip(); - IO.writeFully(fd, headerBuffer); - } - private static MediaCodecInfo[] listEncoders() { List result = new ArrayList<>(); MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS); diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index a4f17262..46612069 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -89,8 +89,8 @@ public final class Server { Size videoSize = device.getScreenInfo().getVideoSize(); connection.sendDeviceMeta(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight()); } - ScreenEncoder screenEncoder = new ScreenEncoder(options.getSendFrameMeta(), options.getBitRate(), options.getMaxFps(), codecOptions, - options.getEncoderName(), options.getDownsizeOnError()); + ScreenEncoder screenEncoder = new ScreenEncoder(options.getBitRate(), options.getMaxFps(), codecOptions, options.getEncoderName(), + options.getDownsizeOnError()); Controller controller = null; if (control) { @@ -103,7 +103,8 @@ public final class Server { try { // synchronous - screenEncoder.streamScreen(device, connection.getVideoFd()); + VideoStreamer videoStreamer = new VideoStreamer(connection.getVideoFd(), options.getSendFrameMeta()); + screenEncoder.streamScreen(device, videoStreamer); } catch (IOException e) { // this is expected on close Ln.d("Screen streaming stopped"); diff --git a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java b/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java new file mode 100644 index 00000000..77e51499 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java @@ -0,0 +1,51 @@ +package com.genymobile.scrcpy; + +import android.media.MediaCodec; + +import java.io.FileDescriptor; +import java.io.IOException; +import java.nio.ByteBuffer; + +public final class VideoStreamer implements ScreenEncoder.Callbacks { + + private static final long PACKET_FLAG_CONFIG = 1L << 63; + private static final long PACKET_FLAG_KEY_FRAME = 1L << 62; + + private final FileDescriptor fd; + private final boolean sendFrameMeta; + + private final ByteBuffer headerBuffer = ByteBuffer.allocate(12); + + public VideoStreamer(FileDescriptor fd, boolean sendFrameMeta) { + this.fd = fd; + this.sendFrameMeta = sendFrameMeta; + } + + @Override + public void onPacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException { + if (sendFrameMeta) { + writeFrameMeta(fd, bufferInfo, codecBuffer.remaining()); + } + + IO.writeFully(fd, codecBuffer); + } + + private void writeFrameMeta(FileDescriptor fd, MediaCodec.BufferInfo bufferInfo, int packetSize) throws IOException { + headerBuffer.clear(); + + long ptsAndFlags; + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + ptsAndFlags = PACKET_FLAG_CONFIG; // non-media data packet + } else { + ptsAndFlags = bufferInfo.presentationTimeUs; + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) { + ptsAndFlags |= PACKET_FLAG_KEY_FRAME; + } + } + + headerBuffer.putLong(ptsAndFlags); + headerBuffer.putInt(packetSize); + headerBuffer.flip(); + IO.writeFully(fd, headerBuffer); + } +}