From 6af4bd601f028d4e3d96eb90326a2330f03bc960 Mon Sep 17 00:00:00 2001 From: Andrew Gunnerson Date: Fri, 27 Oct 2023 20:20:19 -0400 Subject: [PATCH] Add support for high frame rate camera capture Add --camera-high-speed to enable high frame rate camera capture. If the option is enabled, then --camera-fps is mandatory. PR #4213 Co-authored-by: Romain Vimont Signed-off-by: Andrew Gunnerson Signed-off-by: Romain Vimont --- app/data/bash-completion/scrcpy | 1 + app/data/zsh-completion/_scrcpy | 1 + app/scrcpy.1 | 6 ++++ app/src/cli.c | 17 ++++++++++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 1 + app/src/server.c | 3 ++ app/src/server.h | 1 + .../com/genymobile/scrcpy/CameraCapture.java | 31 ++++++++++++++----- .../java/com/genymobile/scrcpy/LogUtils.java | 26 +++++++++++++--- .../java/com/genymobile/scrcpy/Options.java | 8 +++++ .../java/com/genymobile/scrcpy/Server.java | 2 +- 13 files changed, 86 insertions(+), 13 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 9743e44a..eaed88b7 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -14,6 +14,7 @@ _scrcpy() { --camera-id= --camera-facing= --camera-fps= + --camera-high-speed --camera-size= --crop= -d --select-usb diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 1ad96ad5..4b1e5868 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -18,6 +18,7 @@ arguments=( '--audio-output-buffer=[Configure the size of the SDL audio output buffer (in milliseconds)]' {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' '--camera-ar=[Select the camera size by its aspect ratio]' + '--camera-high-speed=[Enable high-speed camera capture mode]' '--camera-id=[Specify the camera id to mirror]' '--camera-facing=[Select the device camera by its facing direction]:facing:(front back external)' '--camera-fps=[Specify the camera capture frame rate]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index e3b1b6f0..2e5cbc60 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -81,6 +81,12 @@ Select the camera size by its aspect ratio (+/- 10%). Possible values are "sensor" (use the camera sensor aspect ratio), ":" (e.g. "4:3") and "" (e.g. "1.6"). +.TP +.B \-\-camera\-high\-speed +Enable high-speed camera capture mode. + +This mode is restricted to specific resolutions and frame rates, listed by --list-camera-sizes. + .TP .BI "\-\-camera\-id " id Specify the device camera id to mirror. diff --git a/app/src/cli.c b/app/src/cli.c index b82d332d..56b5cfb2 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -89,6 +89,7 @@ enum { OPT_CAMERA_FACING, OPT_CAMERA_AR, OPT_CAMERA_FPS, + OPT_CAMERA_HIGH_SPEED, }; struct sc_option { @@ -229,6 +230,13 @@ static const struct sc_option options[] = { .text = "Select the device camera by its facing direction.\n" "Possible values are \"front\", \"back\" and \"external\".", }, + { + .longopt_id = OPT_CAMERA_HIGH_SPEED, + .longopt = "camera-high-speed", + .text = "Enable high-speed camera capture mode.\n" + "This mode is restricted to specific resolutions and frame " + "rates, listed by --list-camera-sizes.", + }, { .longopt_id = OPT_CAMERA_SIZE, .longopt = "camera-size", @@ -2180,6 +2188,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } break; + case OPT_CAMERA_HIGH_SPEED: + opts->camera_high_speed = true; + break; default: // getopt prints the error message on stderr return false; @@ -2296,6 +2307,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } } + if (opts->camera_high_speed && !opts->camera_fps) { + LOGE("--camera-high-speed requires an explicit --camera-fps value"); + return false; + } + if (opts->control) { LOGI("Camera video source: control disabled"); opts->control = false; @@ -2304,6 +2320,7 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], || opts->camera_ar || opts->camera_facing != SC_CAMERA_FACING_ANY || opts->camera_fps + || opts->camera_high_speed || opts->camera_size) { LOGE("Camera options are only available with --video-source=camera"); return false; diff --git a/app/src/options.c b/app/src/options.c index 8601678b..6c72d767 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -86,5 +86,6 @@ const struct scrcpy_options scrcpy_options_default = { .audio = true, .require_audio = false, .kill_adb_on_close = false, + .camera_high_speed = false, .list = 0, }; diff --git a/app/src/options.h b/app/src/options.h index a712f443..18b437d8 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -199,6 +199,7 @@ struct scrcpy_options { bool audio; bool require_audio; bool kill_adb_on_close; + bool camera_high_speed; #define SC_OPTION_LIST_ENCODERS 0x1 #define SC_OPTION_LIST_DISPLAYS 0x2 #define SC_OPTION_LIST_CAMERAS 0x4 diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 64067cf6..1d0e90c1 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -386,6 +386,7 @@ scrcpy(struct scrcpy_options *options) { .cleanup = options->cleanup, .power_on = options->power_on, .kill_adb_on_close = options->kill_adb_on_close, + .camera_high_speed = options->camera_high_speed, .list = options->list, }; diff --git a/app/src/server.c b/app/src/server.c index 8a91952a..2b3439da 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -311,6 +311,9 @@ execute_server(struct sc_server *server, if (params->camera_fps) { ADD_PARAM("camera_fps=%" PRIu16, params->camera_fps); } + if (params->camera_high_speed) { + ADD_PARAM("camera_high_speed=true"); + } if (params->show_touches) { ADD_PARAM("show_touches=true"); } diff --git a/app/src/server.h b/app/src/server.h index ed1f307e..062af0a9 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -63,6 +63,7 @@ struct sc_server_params { bool cleanup; bool power_on; bool kill_adb_on_close; + bool camera_high_speed; uint8_t list; }; diff --git a/server/src/main/java/com/genymobile/scrcpy/CameraCapture.java b/server/src/main/java/com/genymobile/scrcpy/CameraCapture.java index 9edc600c..b9da3658 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CameraCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/CameraCapture.java @@ -8,6 +8,7 @@ import android.graphics.Rect; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession; import android.hardware.camera2.CameraDevice; import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureFailure; @@ -40,6 +41,7 @@ public class CameraCapture extends SurfaceCapture { private int maxSize; private final CameraAspectRatio aspectRatio; private final int fps; + private final boolean highSpeed; private String cameraId; private Size size; @@ -51,13 +53,15 @@ public class CameraCapture extends SurfaceCapture { private final AtomicBoolean disconnected = new AtomicBoolean(); - public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, int fps) { + public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, int fps, + boolean highSpeed) { this.explicitCameraId = explicitCameraId; this.cameraFacing = cameraFacing; this.explicitSize = explicitSize; this.maxSize = maxSize; this.aspectRatio = aspectRatio; this.fps = fps; + this.highSpeed = highSpeed; } @Override @@ -73,7 +77,7 @@ public class CameraCapture extends SurfaceCapture { throw new IOException("No matching camera found"); } - size = selectSize(cameraId, explicitSize, maxSize, aspectRatio); + size = selectSize(cameraId, explicitSize, maxSize, aspectRatio, highSpeed); if (size == null) { throw new IOException("Could not select camera size"); } @@ -112,7 +116,8 @@ public class CameraCapture extends SurfaceCapture { } @TargetApi(Build.VERSION_CODES.N) - private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio) throws CameraAccessException { + private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, boolean highSpeed) + throws CameraAccessException { if (explicitSize != null) { return explicitSize; } @@ -121,7 +126,7 @@ public class CameraCapture extends SurfaceCapture { CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraId); StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); - android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class); + android.util.Size[] sizes = highSpeed ? configs.getHighSpeedVideoSizes() : configs.getOutputSizes(MediaCodec.class); Stream stream = Arrays.stream(sizes); if (maxSize > 0) { stream = stream.filter(it -> it.getWidth() <= maxSize && it.getHeight() <= maxSize); @@ -221,7 +226,7 @@ public class CameraCapture extends SurfaceCapture { this.maxSize = maxSize; try { - size = selectSize(cameraId, null, maxSize, aspectRatio); + size = selectSize(cameraId, null, maxSize, aspectRatio, highSpeed); return size != null; } catch (CameraAccessException e) { Ln.w("Could not select camera size", e); @@ -282,7 +287,9 @@ public class CameraCapture extends SurfaceCapture { CompletableFuture future = new CompletableFuture<>(); OutputConfiguration outputConfig = new OutputConfiguration(surface); List outputs = Arrays.asList(outputConfig); - SessionConfiguration sessionConfig = new SessionConfiguration(SessionConfiguration.SESSION_REGULAR, outputs, cameraExecutor, + + int sessionType = highSpeed ? SessionConfiguration.SESSION_HIGH_SPEED : SessionConfiguration.SESSION_REGULAR; + SessionConfiguration sessionConfig = new SessionConfiguration(sessionType, outputs, cameraExecutor, new CameraCaptureSession.StateCallback() { @Override public void onConfigured(CameraCaptureSession session) { @@ -317,7 +324,7 @@ public class CameraCapture extends SurfaceCapture { @TargetApi(Build.VERSION_CODES.S) private void setRepeatingRequest(CameraCaptureSession session, CaptureRequest request) throws CameraAccessException, InterruptedException { - session.setRepeatingRequest(request, new CameraCaptureSession.CaptureCallback() { + CameraCaptureSession.CaptureCallback callback = new CameraCaptureSession.CaptureCallback() { @Override public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber) { // Called for each frame captured, do nothing @@ -327,7 +334,15 @@ public class CameraCapture extends SurfaceCapture { public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request, CaptureFailure failure) { Ln.w("Camera capture failed: frame " + failure.getFrameNumber()); } - }, cameraHandler); + }; + + if (highSpeed) { + CameraConstrainedHighSpeedCaptureSession highSpeedSession = (CameraConstrainedHighSpeedCaptureSession) session; + List requests = highSpeedSession.createHighSpeedRequestList(request); + highSpeedSession.setRepeatingBurst(requests, callback, cameraHandler); + } else { + session.setRepeatingRequest(request, callback, cameraHandler); + } } @Override diff --git a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java index 329f2570..8dc54629 100644 --- a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java +++ b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java @@ -12,6 +12,7 @@ import android.media.MediaCodec; import android.util.Range; import java.util.List; +import java.util.SortedSet; import java.util.TreeSet; public final class LogUtils { @@ -103,18 +104,27 @@ public final class LogUtils { // Capture frame rates for low-FPS mode are the same for every resolution Range[] lowFpsRanges = characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES); - TreeSet uniqueLowFps = new TreeSet<>(); - for (Range range : lowFpsRanges) { - uniqueLowFps.add(range.getUpper()); - } + SortedSet uniqueLowFps = getUniqueSet(lowFpsRanges); builder.append("fps=").append(uniqueLowFps).append(')'); if (includeSizes) { StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class); for (android.util.Size size : sizes) { builder.append("\n - ").append(size.getWidth()).append('x').append(size.getHeight()); } + + android.util.Size[] highSpeedSizes = configs.getHighSpeedVideoSizes(); + if (highSpeedSizes.length > 0) { + builder.append("\n High speed capture (--camera-high-speed):"); + for (android.util.Size size : highSpeedSizes) { + Range[] highFpsRanges = configs.getHighSpeedVideoFpsRanges(); + SortedSet uniqueHighFps = getUniqueSet(highFpsRanges); + builder.append("\n - ").append(size.getWidth()).append("x").append(size.getHeight()); + builder.append(" (fps=").append(uniqueHighFps).append(')'); + } + } } } } @@ -123,4 +133,12 @@ public final class LogUtils { } return builder.toString(); } + + private static SortedSet getUniqueSet(Range[] ranges) { + SortedSet set = new TreeSet<>(); + for (Range range : ranges) { + set.add(range.getUpper()); + } + return set; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 843fe9f1..9b1d8d8d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -29,6 +29,7 @@ public class Options { private CameraFacing cameraFacing; private CameraAspectRatio cameraAspectRatio; private int cameraFps; + private boolean cameraHighSpeed; private boolean showTouches; private boolean stayAwake; private List videoCodecOptions; @@ -141,6 +142,10 @@ public class Options { return cameraFps; } + public boolean getCameraHighSpeed() { + return cameraHighSpeed; + } + public boolean getShowTouches() { return showTouches; } @@ -392,6 +397,9 @@ public class Options { case "camera_fps": options.cameraFps = Integer.parseInt(value); break; + case "camera_high_speed": + options.cameraHighSpeed = Boolean.parseBoolean(value); + break; case "send_device_meta": options.sendDeviceMeta = Boolean.parseBoolean(value); break; diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index ca72d584..a86b8130 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -144,7 +144,7 @@ public final class Server { surfaceCapture = new ScreenCapture(device); } else { surfaceCapture = new CameraCapture(options.getCameraId(), options.getCameraFacing(), options.getCameraSize(), - options.getMaxSize(), options.getCameraAspectRatio(), options.getCameraFps()); + options.getMaxSize(), options.getCameraAspectRatio(), options.getCameraFps(), options.getCameraHighSpeed()); } SurfaceEncoder surfaceEncoder = new SurfaceEncoder(surfaceCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());