Add support for high frame rate camera capture

Add --camera-high-speed to enable high frame rate camera capture. If
the option is enabled, then --camera-fps is mandatory.

PR #4213 <https://github.com/Genymobile/scrcpy/pull/4213>

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Andrew Gunnerson <accounts+github@chiller3.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>

app/data/bash-completion/scrcpy

@@ -14,6 +14,7 @@ _scrcpy() {
         --camera-id=
         --camera-facing=
         --camera-fps=
+        --camera-high-speed
         --camera-size=
         --crop=
         -d --select-usb

app/data/zsh-completion/_scrcpy

@@ -18,6 +18,7 @@ arguments=(
     '--audio-output-buffer=[Configure the size of the SDL audio output buffer (in milliseconds)]'
     {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]'
     '--camera-ar=[Select the camera size by its aspect ratio]'
+    '--camera-high-speed=[Enable high-speed camera capture mode]'
     '--camera-id=[Specify the camera id to mirror]'
     '--camera-facing=[Select the device camera by its facing direction]:facing:(front back external)'
     '--camera-fps=[Specify the camera capture frame rate]'

app/scrcpy.1

@@ -81,6 +81,12 @@ Select the camera size by its aspect ratio (+/- 10%).
 
 Possible values are "sensor" (use the camera sensor aspect ratio), "<num>:<den>" (e.g. "4:3") and "<value>" (e.g. "1.6").
 
+.TP
+.B \-\-camera\-high\-speed
+Enable high-speed camera capture mode.
+
+This mode is restricted to specific resolutions and frame rates, listed by --list-camera-sizes.
+
 .TP
 .BI "\-\-camera\-id " id
 Specify the device camera id to mirror.

app/src/cli.c

@@ -89,6 +89,7 @@ enum {
     OPT_CAMERA_FACING,
     OPT_CAMERA_AR,
     OPT_CAMERA_FPS,
+    OPT_CAMERA_HIGH_SPEED,
 };
 
 struct sc_option {
@@ -229,6 +230,13 @@ static const struct sc_option options[] = {
         .text = "Select the device camera by its facing direction.\n"
                 "Possible values are \"front\", \"back\" and \"external\".",
     },
+    {
+        .longopt_id = OPT_CAMERA_HIGH_SPEED,
+        .longopt = "camera-high-speed",
+        .text = "Enable high-speed camera capture mode.\n"
+                "This mode is restricted to specific resolutions and frame "
+                "rates, listed by --list-camera-sizes.",
+    },
     {
         .longopt_id = OPT_CAMERA_SIZE,
         .longopt = "camera-size",
@@ -2180,6 +2188,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                     return false;
                 }
                 break;
+            case OPT_CAMERA_HIGH_SPEED:
+                opts->camera_high_speed = true;
+                break;
             default:
                 // getopt prints the error message on stderr
                 return false;
@@ -2296,6 +2307,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
             }
         }
 
+        if (opts->camera_high_speed && !opts->camera_fps) {
+            LOGE("--camera-high-speed requires an explicit --camera-fps value");
+            return false;
+        }
+
         if (opts->control) {
             LOGI("Camera video source: control disabled");
             opts->control = false;
@@ -2304,6 +2320,7 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
             || opts->camera_ar
             || opts->camera_facing != SC_CAMERA_FACING_ANY
             || opts->camera_fps
+            || opts->camera_high_speed
             || opts->camera_size) {
         LOGE("Camera options are only available with --video-source=camera");
         return false;

app/src/options.c

@@ -86,5 +86,6 @@ const struct scrcpy_options scrcpy_options_default = {
     .audio = true,
     .require_audio = false,
     .kill_adb_on_close = false,
+    .camera_high_speed = false,
     .list = 0,
 };

app/src/options.h

@@ -199,6 +199,7 @@ struct scrcpy_options {
     bool audio;
     bool require_audio;
     bool kill_adb_on_close;
+    bool camera_high_speed;
 #define SC_OPTION_LIST_ENCODERS 0x1
 #define SC_OPTION_LIST_DISPLAYS 0x2
 #define SC_OPTION_LIST_CAMERAS 0x4

app/src/scrcpy.c

@@ -386,6 +386,7 @@ scrcpy(struct scrcpy_options *options) {
         .cleanup = options->cleanup,
         .power_on = options->power_on,
         .kill_adb_on_close = options->kill_adb_on_close,
+        .camera_high_speed = options->camera_high_speed,
         .list = options->list,
     };
 

app/src/server.c

@@ -311,6 +311,9 @@ execute_server(struct sc_server *server,
     if (params->camera_fps) {
         ADD_PARAM("camera_fps=%" PRIu16, params->camera_fps);
     }
+    if (params->camera_high_speed) {
+        ADD_PARAM("camera_high_speed=true");
+    }
     if (params->show_touches) {
         ADD_PARAM("show_touches=true");
     }

app/src/server.h

@@ -63,6 +63,7 @@ struct sc_server_params {
     bool cleanup;
     bool power_on;
     bool kill_adb_on_close;
+    bool camera_high_speed;
     uint8_t list;
 };
 

server/src/main/java/com/genymobile/scrcpy/CameraCapture.java

@@ -8,6 +8,7 @@ import android.graphics.Rect;
 import android.hardware.camera2.CameraAccessException;
 import android.hardware.camera2.CameraCaptureSession;
 import android.hardware.camera2.CameraCharacteristics;
+import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession;
 import android.hardware.camera2.CameraDevice;
 import android.hardware.camera2.CameraManager;
 import android.hardware.camera2.CaptureFailure;
@@ -40,6 +41,7 @@ public class CameraCapture extends SurfaceCapture {
     private int maxSize;
     private final CameraAspectRatio aspectRatio;
     private final int fps;
+    private final boolean highSpeed;
 
     private String cameraId;
     private Size size;
@@ -51,13 +53,15 @@ public class CameraCapture extends SurfaceCapture {
 
     private final AtomicBoolean disconnected = new AtomicBoolean();
 
-    public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, int fps) {
+    public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, int fps,
+            boolean highSpeed) {
         this.explicitCameraId = explicitCameraId;
         this.cameraFacing = cameraFacing;
         this.explicitSize = explicitSize;
         this.maxSize = maxSize;
         this.aspectRatio = aspectRatio;
         this.fps = fps;
+        this.highSpeed = highSpeed;
     }
 
     @Override
@@ -73,7 +77,7 @@ public class CameraCapture extends SurfaceCapture {
                 throw new IOException("No matching camera found");
             }
 
-            size = selectSize(cameraId, explicitSize, maxSize, aspectRatio);
+            size = selectSize(cameraId, explicitSize, maxSize, aspectRatio, highSpeed);
             if (size == null) {
                 throw new IOException("Could not select camera size");
             }
@@ -112,7 +116,8 @@ public class CameraCapture extends SurfaceCapture {
     }
 
     @TargetApi(Build.VERSION_CODES.N)
-    private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio) throws CameraAccessException {
+    private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, boolean highSpeed)
+            throws CameraAccessException {
         if (explicitSize != null) {
             return explicitSize;
         }
@@ -121,7 +126,7 @@ public class CameraCapture extends SurfaceCapture {
         CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraId);
 
         StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
-        android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class);
+        android.util.Size[] sizes = highSpeed ? configs.getHighSpeedVideoSizes() : configs.getOutputSizes(MediaCodec.class);
         Stream<android.util.Size> stream = Arrays.stream(sizes);
         if (maxSize > 0) {
             stream = stream.filter(it -> it.getWidth() <= maxSize && it.getHeight() <= maxSize);
@@ -221,7 +226,7 @@ public class CameraCapture extends SurfaceCapture {
 
         this.maxSize = maxSize;
         try {
-            size = selectSize(cameraId, null, maxSize, aspectRatio);
+            size = selectSize(cameraId, null, maxSize, aspectRatio, highSpeed);
             return size != null;
         } catch (CameraAccessException e) {
             Ln.w("Could not select camera size", e);
@@ -282,7 +287,9 @@ public class CameraCapture extends SurfaceCapture {
         CompletableFuture<CameraCaptureSession> future = new CompletableFuture<>();
         OutputConfiguration outputConfig = new OutputConfiguration(surface);
         List<OutputConfiguration> outputs = Arrays.asList(outputConfig);
-        SessionConfiguration sessionConfig = new SessionConfiguration(SessionConfiguration.SESSION_REGULAR, outputs, cameraExecutor,
+
+        int sessionType = highSpeed ? SessionConfiguration.SESSION_HIGH_SPEED : SessionConfiguration.SESSION_REGULAR;
+        SessionConfiguration sessionConfig = new SessionConfiguration(sessionType, outputs, cameraExecutor,
                 new CameraCaptureSession.StateCallback() {
                     @Override
                     public void onConfigured(CameraCaptureSession session) {
@@ -317,7 +324,7 @@ public class CameraCapture extends SurfaceCapture {
 
     @TargetApi(Build.VERSION_CODES.S)
     private void setRepeatingRequest(CameraCaptureSession session, CaptureRequest request) throws CameraAccessException, InterruptedException {
-        session.setRepeatingRequest(request, new CameraCaptureSession.CaptureCallback() {
+        CameraCaptureSession.CaptureCallback callback = new CameraCaptureSession.CaptureCallback() {
             @Override
             public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber) {
                 // Called for each frame captured, do nothing
@@ -327,7 +334,15 @@ public class CameraCapture extends SurfaceCapture {
             public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request, CaptureFailure failure) {
                 Ln.w("Camera capture failed: frame " + failure.getFrameNumber());
             }
-        }, cameraHandler);
+        };
+
+        if (highSpeed) {
+            CameraConstrainedHighSpeedCaptureSession highSpeedSession = (CameraConstrainedHighSpeedCaptureSession) session;
+            List<CaptureRequest> requests = highSpeedSession.createHighSpeedRequestList(request);
+            highSpeedSession.setRepeatingBurst(requests, callback, cameraHandler);
+        } else {
+            session.setRepeatingRequest(request, callback, cameraHandler);
+        }
     }
 
     @Override

server/src/main/java/com/genymobile/scrcpy/LogUtils.java

@@ -12,6 +12,7 @@ import android.media.MediaCodec;
 import android.util.Range;
 
 import java.util.List;
+import java.util.SortedSet;
 import java.util.TreeSet;
 
 public final class LogUtils {
@@ -103,18 +104,27 @@ public final class LogUtils {
 
                     // Capture frame rates for low-FPS mode are the same for every resolution
                     Range<Integer>[] lowFpsRanges = characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES);
-                    TreeSet<Integer> uniqueLowFps = new TreeSet<>();
+                    SortedSet<Integer> uniqueLowFps = getUniqueSet(lowFpsRanges);
-                    for (Range<Integer> range : lowFpsRanges) {
-                        uniqueLowFps.add(range.getUpper());
-                    }
                     builder.append("fps=").append(uniqueLowFps).append(')');
 
                     if (includeSizes) {
                         StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
+
                         android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class);
                         for (android.util.Size size : sizes) {
                             builder.append("\n        - ").append(size.getWidth()).append('x').append(size.getHeight());
                         }
+
+                        android.util.Size[] highSpeedSizes = configs.getHighSpeedVideoSizes();
+                        if (highSpeedSizes.length > 0) {
+                            builder.append("\n      High speed capture (--camera-high-speed):");
+                            for (android.util.Size size : highSpeedSizes) {
+                                Range<Integer>[] highFpsRanges = configs.getHighSpeedVideoFpsRanges();
+                                SortedSet<Integer> uniqueHighFps = getUniqueSet(highFpsRanges);
+                                builder.append("\n        - ").append(size.getWidth()).append("x").append(size.getHeight());
+                                builder.append(" (fps=").append(uniqueHighFps).append(')');
+                            }
+                        }
                     }
                 }
             }
@@ -123,4 +133,12 @@ public final class LogUtils {
         }
         return builder.toString();
     }
+
+    private static SortedSet<Integer> getUniqueSet(Range<Integer>[] ranges) {
+        SortedSet<Integer> set = new TreeSet<>();
+        for (Range<Integer> range : ranges) {
+            set.add(range.getUpper());
+        }
+        return set;
+    }
 }

server/src/main/java/com/genymobile/scrcpy/Options.java

@@ -29,6 +29,7 @@ public class Options {
     private CameraFacing cameraFacing;
     private CameraAspectRatio cameraAspectRatio;
     private int cameraFps;
+    private boolean cameraHighSpeed;
     private boolean showTouches;
     private boolean stayAwake;
     private List<CodecOption> videoCodecOptions;
@@ -141,6 +142,10 @@ public class Options {
         return cameraFps;
     }
 
+    public boolean getCameraHighSpeed() {
+        return cameraHighSpeed;
+    }
+
     public boolean getShowTouches() {
         return showTouches;
     }
@@ -392,6 +397,9 @@ public class Options {
                 case "camera_fps":
                     options.cameraFps = Integer.parseInt(value);
                     break;
+                case "camera_high_speed":
+                    options.cameraHighSpeed = Boolean.parseBoolean(value);
+                    break;
                 case "send_device_meta":
                     options.sendDeviceMeta = Boolean.parseBoolean(value);
                     break;

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -144,7 +144,7 @@ public final class Server {
                     surfaceCapture = new ScreenCapture(device);
                 } else {
                     surfaceCapture = new CameraCapture(options.getCameraId(), options.getCameraFacing(), options.getCameraSize(),
-                            options.getMaxSize(), options.getCameraAspectRatio(), options.getCameraFps());
+                            options.getMaxSize(), options.getCameraAspectRatio(), options.getCameraFps(), options.getCameraHighSpeed());
                 }
                 SurfaceEncoder surfaceEncoder = new SurfaceEncoder(surfaceCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
                         options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());