From 04f19f42a7866d9900c168753f3f40424edde596 Mon Sep 17 00:00:00 2001 From: Andrew Gunnerson Date: Fri, 27 Oct 2023 20:20:19 -0400 Subject: [PATCH] Add support for high frame rate camera capture Add --camera-high-speed to enable high frame rate camera capture. If the option is enabled, then --camera-fps is mandatory. Co-authored-by: Romain VImont Signed-off-by: Andrew Gunnerson Signed-off-by: Romain Vimont --- app/src/cli.c | 17 ++++++++++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 1 + app/src/server.c | 3 ++ app/src/server.h | 1 + .../com/genymobile/scrcpy/CameraCapture.java | 31 ++++++++++++++----- .../java/com/genymobile/scrcpy/LogUtils.java | 28 ++++++++++++++--- .../java/com/genymobile/scrcpy/Options.java | 8 +++++ .../java/com/genymobile/scrcpy/Server.java | 2 +- 10 files changed, 80 insertions(+), 13 deletions(-) diff --git a/app/src/cli.c b/app/src/cli.c index 2c697dfa..2e803144 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -89,6 +89,7 @@ enum { OPT_CAMERA_FACING, OPT_CAMERA_AR, OPT_CAMERA_FPS, + OPT_CAMERA_HIGH_SPEED, }; struct sc_option { @@ -229,6 +230,13 @@ static const struct sc_option options[] = { .text = "Select the device camera by its facing direction.\n" "Possible values are \"front\", \"back\" and \"external\".", }, + { + .longopt_id = OPT_CAMERA_HIGH_SPEED, + .longopt = "camera-high-speed", + .text = "Enable high-speed camera capture mode.\n" + "This mode is restricted to specific resolutions and frame " + "rates, listed by --list-camera-sizes.", + }, { .longopt_id = OPT_CAMERA_SIZE, .longopt = "camera-size", @@ -2180,6 +2188,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } break; + case OPT_CAMERA_HIGH_SPEED: + opts->camera_high_speed = true; + break; default: // getopt prints the error message on stderr return false; @@ -2302,6 +2313,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } } + if (opts->camera_high_speed && !opts->camera_fps) { + LOGE("--camera-high-speed requires an explicit --camera-fps value"); + return false; + } + if (opts->control) { LOGI("Camera video source: control disabled"); opts->control = false; @@ -2310,6 +2326,7 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], || opts->camera_ar || opts->camera_facing != SC_CAMERA_FACING_ANY || opts->camera_fps + || opts->camera_high_speed || opts->camera_size) { LOGE("Camera options are only available with --video-source=camera"); return false; diff --git a/app/src/options.c b/app/src/options.c index 8601678b..6c72d767 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -86,5 +86,6 @@ const struct scrcpy_options scrcpy_options_default = { .audio = true, .require_audio = false, .kill_adb_on_close = false, + .camera_high_speed = false, .list = 0, }; diff --git a/app/src/options.h b/app/src/options.h index a712f443..18b437d8 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -199,6 +199,7 @@ struct scrcpy_options { bool audio; bool require_audio; bool kill_adb_on_close; + bool camera_high_speed; #define SC_OPTION_LIST_ENCODERS 0x1 #define SC_OPTION_LIST_DISPLAYS 0x2 #define SC_OPTION_LIST_CAMERAS 0x4 diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 64067cf6..1d0e90c1 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -386,6 +386,7 @@ scrcpy(struct scrcpy_options *options) { .cleanup = options->cleanup, .power_on = options->power_on, .kill_adb_on_close = options->kill_adb_on_close, + .camera_high_speed = options->camera_high_speed, .list = options->list, }; diff --git a/app/src/server.c b/app/src/server.c index 8a91952a..2b3439da 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -311,6 +311,9 @@ execute_server(struct sc_server *server, if (params->camera_fps) { ADD_PARAM("camera_fps=%" PRIu16, params->camera_fps); } + if (params->camera_high_speed) { + ADD_PARAM("camera_high_speed=true"); + } if (params->show_touches) { ADD_PARAM("show_touches=true"); } diff --git a/app/src/server.h b/app/src/server.h index ed1f307e..062af0a9 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -63,6 +63,7 @@ struct sc_server_params { bool cleanup; bool power_on; bool kill_adb_on_close; + bool camera_high_speed; uint8_t list; }; diff --git a/server/src/main/java/com/genymobile/scrcpy/CameraCapture.java b/server/src/main/java/com/genymobile/scrcpy/CameraCapture.java index 983b76f8..d1d9b8ab 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CameraCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/CameraCapture.java @@ -8,6 +8,7 @@ import android.graphics.Rect; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession; import android.hardware.camera2.CameraDevice; import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureFailure; @@ -40,6 +41,7 @@ public class CameraCapture extends SurfaceCapture { private int maxSize; private final CameraAspectRatio aspectRatio; private final int fps; + private final boolean highSpeed; private String cameraId; private Size size; @@ -51,13 +53,15 @@ public class CameraCapture extends SurfaceCapture { private final AtomicBoolean disconnected = new AtomicBoolean(); - public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, int fps) { + public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, int fps, + boolean highSpeed) { this.explicitCameraId = explicitCameraId; this.cameraFacing = cameraFacing; this.explicitSize = explicitSize; this.maxSize = maxSize; this.aspectRatio = aspectRatio; this.fps = fps; + this.highSpeed = highSpeed; } @Override @@ -73,7 +77,7 @@ public class CameraCapture extends SurfaceCapture { throw new IOException("No matching camera found"); } - size = selectSize(cameraId, explicitSize, maxSize, aspectRatio); + size = selectSize(cameraId, explicitSize, maxSize, aspectRatio, highSpeed); if (size == null) { throw new IOException("Could not select camera size"); } @@ -112,7 +116,8 @@ public class CameraCapture extends SurfaceCapture { } @TargetApi(Build.VERSION_CODES.N) - private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio) throws CameraAccessException { + private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, boolean highSpeed) + throws CameraAccessException { if (explicitSize != null) { return explicitSize; } @@ -121,7 +126,7 @@ public class CameraCapture extends SurfaceCapture { CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraId); StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); - android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class); + android.util.Size[] sizes = highSpeed ? configs.getHighSpeedVideoSizes() : configs.getOutputSizes(MediaCodec.class); Stream stream = Arrays.stream(sizes); if (maxSize > 0) { stream = stream.filter(it -> it.getWidth() <= maxSize && it.getHeight() <= maxSize); @@ -221,7 +226,7 @@ public class CameraCapture extends SurfaceCapture { this.maxSize = maxSize; try { - size = selectSize(cameraId, null, maxSize, aspectRatio); + size = selectSize(cameraId, null, maxSize, aspectRatio, highSpeed); return true; } catch (CameraAccessException e) { Ln.w("Could not select camera size", e); @@ -282,7 +287,9 @@ public class CameraCapture extends SurfaceCapture { CompletableFuture future = new CompletableFuture<>(); OutputConfiguration outputConfig = new OutputConfiguration(surface); List outputs = Arrays.asList(outputConfig); - SessionConfiguration sessionConfig = new SessionConfiguration(SessionConfiguration.SESSION_REGULAR, outputs, cameraExecutor, + + int sessionType = highSpeed ? SessionConfiguration.SESSION_HIGH_SPEED : SessionConfiguration.SESSION_REGULAR; + SessionConfiguration sessionConfig = new SessionConfiguration(sessionType, outputs, cameraExecutor, new CameraCaptureSession.StateCallback() { @Override public void onConfigured(CameraCaptureSession session) { @@ -317,7 +324,7 @@ public class CameraCapture extends SurfaceCapture { @TargetApi(Build.VERSION_CODES.S) private void setRepeatingRequest(CameraCaptureSession session, CaptureRequest request) throws CameraAccessException, InterruptedException { - session.setRepeatingRequest(request, new CameraCaptureSession.CaptureCallback() { + CameraCaptureSession.CaptureCallback callback = new CameraCaptureSession.CaptureCallback() { @Override public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber) { // Called for each frame captured, do nothing @@ -327,7 +334,15 @@ public class CameraCapture extends SurfaceCapture { public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request, CaptureFailure failure) { Ln.w("Camera capture failed: frame " + failure.getFrameNumber()); } - }, cameraHandler); + }; + + if (highSpeed) { + CameraConstrainedHighSpeedCaptureSession highSpeedSession = (CameraConstrainedHighSpeedCaptureSession) session; + List requests = highSpeedSession.createHighSpeedRequestList(request); + highSpeedSession.setRepeatingBurst(requests, callback, cameraHandler); + } else { + session.setRepeatingRequest(request, callback, cameraHandler); + } } @Override diff --git a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java index 329f2570..2b6a5589 100644 --- a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java +++ b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java @@ -11,7 +11,10 @@ import android.hardware.camera2.params.StreamConfigurationMap; import android.media.MediaCodec; import android.util.Range; +import java.util.Arrays; import java.util.List; +import java.util.Set; +import java.util.SortedSet; import java.util.TreeSet; public final class LogUtils { @@ -103,18 +106,27 @@ public final class LogUtils { // Capture frame rates for low-FPS mode are the same for every resolution Range[] lowFpsRanges = characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES); - TreeSet uniqueLowFps = new TreeSet<>(); - for (Range range : lowFpsRanges) { - uniqueLowFps.add(range.getUpper()); - } + SortedSet uniqueLowFps = getUniqueSet(lowFpsRanges); builder.append("fps=").append(uniqueLowFps).append(')'); if (includeSizes) { StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class); for (android.util.Size size : sizes) { builder.append("\n - ").append(size.getWidth()).append('x').append(size.getHeight()); } + + android.util.Size[] highSpeedSizes = configs.getHighSpeedVideoSizes(); + if (highSpeedSizes.length > 0) { + builder.append("\n High speed capture:"); + for (android.util.Size size : highSpeedSizes) { + Range[] highFpsRanges = configs.getHighSpeedVideoFpsRanges(); + SortedSet uniqueHighFps = getUniqueSet(highFpsRanges); + builder.append("\n - ").append(size.getWidth()).append("x").append(size.getHeight()); + builder.append(" (fps=").append(uniqueHighFps).append(')'); + } + } } } } @@ -123,4 +135,12 @@ public final class LogUtils { } return builder.toString(); } + + private static SortedSet getUniqueSet(Range[] ranges) { + SortedSet set = new TreeSet<>(); + for (Range range : ranges) { + set.add(range.getUpper()); + } + return set; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 843fe9f1..9b1d8d8d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -29,6 +29,7 @@ public class Options { private CameraFacing cameraFacing; private CameraAspectRatio cameraAspectRatio; private int cameraFps; + private boolean cameraHighSpeed; private boolean showTouches; private boolean stayAwake; private List videoCodecOptions; @@ -141,6 +142,10 @@ public class Options { return cameraFps; } + public boolean getCameraHighSpeed() { + return cameraHighSpeed; + } + public boolean getShowTouches() { return showTouches; } @@ -392,6 +397,9 @@ public class Options { case "camera_fps": options.cameraFps = Integer.parseInt(value); break; + case "camera_high_speed": + options.cameraHighSpeed = Boolean.parseBoolean(value); + break; case "send_device_meta": options.sendDeviceMeta = Boolean.parseBoolean(value); break; diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index dffd95ad..44f0e7cb 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -144,7 +144,7 @@ public final class Server { surfaceCapture = new ScreenCapture(device); } else { surfaceCapture = new CameraCapture(options.getCameraId(), options.getCameraFacing(), options.getCameraSize(), - options.getMaxSize(), options.getCameraAspectRatio(), options.getCameraFps()); + options.getMaxSize(), options.getCameraAspectRatio(), options.getCameraFps(), options.getCameraHighSpeed()); } SurfaceEncoder surfaceEncoder = new SurfaceEncoder(surfaceCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());