You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
scrcpy/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

287 lines
11 KiB
Java

package com.genymobile.scrcpy;
import com.genymobile.scrcpy.wrappers.SurfaceControl;
import android.graphics.Rect;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaFormat;
import android.os.Build;
import android.os.IBinder;
import android.os.SystemClock;
import android.view.Surface;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class ScreenEncoder implements Device.RotationListener {
public interface Callbacks {
void onPacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException;
}
private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms
private static final String KEY_MAX_FPS_TO_ENCODER = "max-fps-to-encoder";
// Keep the values in descending order
private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800};
private static final int MAX_CONSECUTIVE_ERRORS = 3;
private final AtomicBoolean rotationChanged = new AtomicBoolean();
private final String videoMimeType;
private final String encoderName;
private final List<CodecOption> codecOptions;
private final int bitRate;
private final int maxFps;
private final boolean downsizeOnError;
private boolean firstFrameSent;
private int consecutiveErrors;
public ScreenEncoder(String videoMimeType, int bitRate, int maxFps, List<CodecOption> codecOptions, String encoderName, boolean downsizeOnError) {
this.videoMimeType = videoMimeType;
this.bitRate = bitRate;
this.maxFps = maxFps;
this.codecOptions = codecOptions;
this.encoderName = encoderName;
this.downsizeOnError = downsizeOnError;
}
@Override
public void onRotationChanged(int rotation) {
rotationChanged.set(true);
}
public boolean consumeRotationChange() {
return rotationChanged.getAndSet(false);
}
public void streamScreen(Device device, Callbacks callbacks) throws IOException {
MediaCodec codec = createCodec(videoMimeType, encoderName);
MediaFormat format = createFormat(videoMimeType, bitRate, maxFps, codecOptions);
IBinder display = createDisplay();
device.setRotationListener(this);
boolean alive;
try {
do {
ScreenInfo screenInfo = device.getScreenInfo();
Rect contentRect = screenInfo.getContentRect();
// include the locked video orientation
Rect videoRect = screenInfo.getVideoSize().toRect();
format.setInteger(MediaFormat.KEY_WIDTH, videoRect.width());
format.setInteger(MediaFormat.KEY_HEIGHT, videoRect.height());
Surface surface = null;
try {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
surface = codec.createInputSurface();
// does not include the locked video orientation
Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
int videoRotation = screenInfo.getVideoRotation();
int layerStack = device.getLayerStack();
setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
codec.start();
alive = encode(codec, callbacks);
// do not call stop() on exception, it would trigger an IllegalStateException
codec.stop();
} catch (IllegalStateException | IllegalArgumentException e) {
Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage());
if (!prepareRetry(device, screenInfo)) {
throw e;
}
Ln.i("Retrying...");
alive = true;
} finally {
codec.reset();
if (surface != null) {
surface.release();
}
}
} while (alive);
} finally {
codec.release();
device.setRotationListener(null);
SurfaceControl.destroyDisplay(display);
}
}
private boolean prepareRetry(Device device, ScreenInfo screenInfo) {
if (firstFrameSent) {
++consecutiveErrors;
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
// Definitively fail
return false;
}
// Wait a bit to increase the probability that retrying will fix the problem
SystemClock.sleep(50);
return true;
}
if (!downsizeOnError) {
// Must fail immediately
return false;
}
// Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising)
int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
Ln.i("newMaxSize = " + newMaxSize);
if (newMaxSize == 0) {
// Must definitively fail
return false;
}
// Retry with a smaller device size
Ln.i("Retrying with -m" + newMaxSize + "...");
device.setMaxSize(newMaxSize);
return true;
}
private static int chooseMaxSizeFallback(Size failedSize) {
int currentMaxSize = Math.max(failedSize.getWidth(), failedSize.getHeight());
for (int value : MAX_SIZE_FALLBACK) {
if (value < currentMaxSize) {
// We found a smaller value to reduce the video size
return value;
}
}
// No fallback, fail definitively
return 0;
}
private boolean encode(MediaCodec codec, Callbacks callbacks) throws IOException {
boolean eof = false;
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
while (!consumeRotationChange() && !eof) {
int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
try {
if (consumeRotationChange()) {
// must restart encoding with new size
break;
}
eof = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
if (outputBufferId >= 0) {
ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId);
boolean isConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0;
if (!isConfig) {
// If this is not a config packet, then it contains a frame
firstFrameSent = true;
consecutiveErrors = 0;
}
callbacks.onPacket(codecBuffer, bufferInfo);
}
} finally {
if (outputBufferId >= 0) {
codec.releaseOutputBuffer(outputBufferId, false);
}
}
}
return !eof;
}
private static MediaCodecInfo[] listEncoders(String videoMimeType) {
List<MediaCodecInfo> result = new ArrayList<>();
MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
for (MediaCodecInfo codecInfo : list.getCodecInfos()) {
if (codecInfo.isEncoder() && Arrays.asList(codecInfo.getSupportedTypes()).contains(videoMimeType)) {
result.add(codecInfo);
}
}
return result.toArray(new MediaCodecInfo[result.size()]);
}
private static MediaCodec createCodec(String videoMimeType, String encoderName) throws IOException {
if (encoderName != null) {
Ln.d("Creating encoder by name: '" + encoderName + "'");
try {
return MediaCodec.createByCodecName(encoderName);
} catch (IllegalArgumentException e) {
MediaCodecInfo[] encoders = listEncoders(videoMimeType);
throw new InvalidEncoderException(encoderName, encoders);
}
}
MediaCodec codec = MediaCodec.createEncoderByType(videoMimeType);
Ln.d("Using encoder: '" + codec.getName() + "'");
return codec;
}
private static void setCodecOption(MediaFormat format, CodecOption codecOption) {
String key = codecOption.getKey();
Object value = codecOption.getValue();
if (value instanceof Integer) {
format.setInteger(key, (Integer) value);
} else if (value instanceof Long) {
format.setLong(key, (Long) value);
} else if (value instanceof Float) {
format.setFloat(key, (Float) value);
} else if (value instanceof String) {
format.setString(key, (String) value);
}
Ln.d("Codec option set: " + key + " (" + value.getClass().getSimpleName() + ") = " + value);
}
private static MediaFormat createFormat(String videoMimeType, int bitRate, int maxFps, List<CodecOption> codecOptions) {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, videoMimeType);
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
// must be present to configure the encoder, but does not impact the actual frame rate, which is variable
format.setInteger(MediaFormat.KEY_FRAME_RATE, 60);
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL);
// display the very first frame, and recover from bad quality when no new frames
format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, REPEAT_FRAME_DELAY_US); // µs
if (maxFps > 0) {
// The key existed privately before Android 10:
// <https://android.googlesource.com/platform/frameworks/base/+/625f0aad9f7a259b6881006ad8710adce57d1384%5E%21/>
// <https://github.com/Genymobile/scrcpy/issues/488#issuecomment-567321437>
format.setFloat(KEY_MAX_FPS_TO_ENCODER, maxFps);
}
if (codecOptions != null) {
for (CodecOption option : codecOptions) {
setCodecOption(format, option);
}
}
return format;
}
private static IBinder createDisplay() {
// Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
// On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S"
.equals(Build.VERSION.CODENAME));
return SurfaceControl.createDisplay("scrcpy", secure);
}
private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
SurfaceControl.openTransaction();
try {
SurfaceControl.setDisplaySurface(display, surface);
SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
SurfaceControl.setDisplayLayerStack(display, layerStack);
} finally {
SurfaceControl.closeTransaction();
}
}
}