From 3001f8a2d581a921920d18c088d42e0cd747bf41 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 3 Dec 2023 18:00:05 +0100 Subject: [PATCH 01/79] Adapt AudioRecord workaround to Android 14 Android 14 added a new int parameter "halInputFlags" to an internal method: Fixes #4492 --- .../com/genymobile/scrcpy/Workarounds.java | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Workarounds.java b/server/src/main/java/com/genymobile/scrcpy/Workarounds.java index 8781a783..448e7099 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Workarounds.java +++ b/server/src/main/java/com/genymobile/scrcpy/Workarounds.java @@ -285,16 +285,28 @@ public final class Workarounds { Method getParcelMethod = attributionSourceState.getClass().getDeclaredMethod("getParcel"); Parcel attributionSourceParcel = (Parcel) getParcelMethod.invoke(attributionSourceState); - // private native int native_setup(Object audiorecordThis, - // Object /*AudioAttributes*/ attributes, - // int[] sampleRate, int channelMask, int channelIndexMask, int audioFormat, - // int buffSizeInBytes, int[] sessionId, @NonNull Parcel attributionSource, - // long nativeRecordInJavaObj, int maxSharedAudioHistoryMs); - Method nativeSetupMethod = AudioRecord.class.getDeclaredMethod("native_setup", Object.class, Object.class, int[].class, int.class, - int.class, int.class, int.class, int[].class, Parcel.class, long.class, int.class); - nativeSetupMethod.setAccessible(true); - initResult = (int) nativeSetupMethod.invoke(audioRecord, new WeakReference(audioRecord), attributes, sampleRateArray, - channelMask, channelIndexMask, audioRecord.getAudioFormat(), bufferSizeInBytes, session, attributionSourceParcel, 0L, 0); + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { + // private native int native_setup(Object audiorecordThis, + // Object /*AudioAttributes*/ attributes, + // int[] sampleRate, int channelMask, int channelIndexMask, int audioFormat, + // int buffSizeInBytes, int[] sessionId, @NonNull Parcel attributionSource, + // long nativeRecordInJavaObj, int maxSharedAudioHistoryMs); + Method nativeSetupMethod = AudioRecord.class.getDeclaredMethod("native_setup", Object.class, Object.class, int[].class, + int.class, int.class, int.class, int.class, int[].class, Parcel.class, long.class, int.class); + nativeSetupMethod.setAccessible(true); + initResult = (int) nativeSetupMethod.invoke(audioRecord, new WeakReference(audioRecord), attributes, + sampleRateArray, channelMask, channelIndexMask, audioRecord.getAudioFormat(), bufferSizeInBytes, session, + attributionSourceParcel, 0L, 0); + } else { + // Android 14 added a new int parameter "halInputFlags" + // + Method nativeSetupMethod = AudioRecord.class.getDeclaredMethod("native_setup", Object.class, Object.class, int[].class, + int.class, int.class, int.class, int.class, int[].class, Parcel.class, long.class, int.class, int.class); + nativeSetupMethod.setAccessible(true); + initResult = (int) nativeSetupMethod.invoke(audioRecord, new WeakReference(audioRecord), attributes, + sampleRateArray, channelMask, channelIndexMask, audioRecord.getAudioFormat(), bufferSizeInBytes, session, + attributionSourceParcel, 0L, 0, 0); + } } } From 5ce8672ebc56b7286e1078a39abc64903e5664d0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 3 Dec 2023 21:22:30 +0100 Subject: [PATCH 02/79] Add clipboard workaround for IQOO device Fixes #4492 --- .../scrcpy/wrappers/ClipboardManager.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java index 783a3407..0866d42d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java @@ -41,8 +41,13 @@ public final class ClipboardManager { getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, String.class, int.class, int.class); getMethodVersion = 2; } catch (NoSuchMethodException e3) { - getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, int.class, String.class); - getMethodVersion = 3; + try { + getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, int.class, String.class); + getMethodVersion = 3; + } catch (NoSuchMethodException e4) { + getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, String.class, int.class, int.class, boolean.class); + getMethodVersion = 4; + } } } } @@ -87,8 +92,11 @@ public final class ClipboardManager { return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID); case 2: return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID, 0); - default: + case 3: return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, FakeContext.ROOT_UID, null); + default: + // The last boolean parameter is "userOperate" + return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID, 0, true); } } From 1beec99f8283713b1fbf0b3704eb4dceecc9a590 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 3 Dec 2023 10:48:53 +0100 Subject: [PATCH 03/79] Explicitly exit cleanup process This avoids an internal crash reported in `adb logcat`. Refs #4456 --- server/src/main/java/com/genymobile/scrcpy/CleanUp.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/com/genymobile/scrcpy/CleanUp.java b/server/src/main/java/com/genymobile/scrcpy/CleanUp.java index b3a1aac1..c84e25bb 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CleanUp.java +++ b/server/src/main/java/com/genymobile/scrcpy/CleanUp.java @@ -187,5 +187,7 @@ public final class CleanUp { Device.setScreenPowerMode(Device.POWER_MODE_NORMAL); } } + + System.exit(0); } } From c9a4d2b38f318a4887e3db48a7d1bcf0affc0250 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 1 Dec 2023 19:14:22 +0100 Subject: [PATCH 04/79] Use up-to-date values on display fold change When a display is folded or unfolded, the maxSize may have been updated since the option was passed, and deviceSize must be updated. Refs #4469 --- server/src/main/java/com/genymobile/scrcpy/Device.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java index b51ad8d3..2324ce90 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/Device.java @@ -45,11 +45,11 @@ public final class Device { void onClipboardTextChanged(String text); } - private final Size deviceSize; private final Rect crop; private int maxSize; private final int lockVideoOrientation; + private Size deviceSize; private ScreenInfo screenInfo; private RotationListener rotationListener; private FoldListener foldListener; @@ -116,8 +116,8 @@ public final class Device { return; } - screenInfo = ScreenInfo.computeScreenInfo(displayInfo.getRotation(), displayInfo.getSize(), options.getCrop(), - options.getMaxSize(), options.getLockVideoOrientation()); + deviceSize = displayInfo.getSize(); + screenInfo = ScreenInfo.computeScreenInfo(displayInfo.getRotation(), deviceSize, crop, maxSize, lockVideoOrientation); // notify if (foldListener != null) { foldListener.onFoldChanged(displayId, folded); From 5a6b8310cae1e0741b4375ca760d9c8dd49822c4 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 13 Dec 2023 12:50:06 +0100 Subject: [PATCH 05/79] Add note about official website --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8fabd556..0a3d03cb 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +**This GitHub repo () is the only official +source for the project. Do not download releases from random websites, even if +their name contains `scrcpy`.** + # scrcpy (v2.3.1) scrcpy From cbce42336dce0bb4cd2c47ed7dfa56d40fc3aca1 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 13 Dec 2023 13:41:34 +0100 Subject: [PATCH 06/79] Fix manpage syntax The '-' character must be escaped. Fixes #4528 --- app/scrcpy.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 0c34b4e2..8ca4a773 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -124,7 +124,7 @@ Use USB device (if there is exactly one, like adb -d). Also see \fB\-e\fR (\fB\-\-select\-tcpip\fR). .TP -.BI "\-\-disable-screensaver" +.BI "\-\-disable\-screensaver" Disable screensaver while scrcpy is running. .TP From af69689ec1d52b442b0a4e1461ca71853c660cc6 Mon Sep 17 00:00:00 2001 From: Harsh Shandilya Date: Thu, 14 Dec 2023 21:21:57 +0530 Subject: [PATCH 07/79] Fix bash completion syntax PR #4532 Signed-off-by: Romain Vimont --- app/data/bash-completion/scrcpy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 0c854310..a0490157 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -115,8 +115,7 @@ _scrcpy() { COMPREPLY=($(compgen -W 'front back external' -- "$cur")) return ;; - --orientation - --display-orientation) + --orientation|--display-orientation) COMPREPLY=($(compgen -> '0 90 180 270 flip0 flip90 flip180 flip270' -- "$cur")) return ;; From 604dfd7c6b07f583521c92f8391d3fdb6b1576e8 Mon Sep 17 00:00:00 2001 From: Harsh Shandilya Date: Thu, 14 Dec 2023 21:25:32 +0530 Subject: [PATCH 08/79] Fix incorrect compgen usage PR #4532 Signed-off-by: Romain Vimont --- app/data/bash-completion/scrcpy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index a0490157..78aa539d 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -116,11 +116,11 @@ _scrcpy() { return ;; --orientation|--display-orientation) - COMPREPLY=($(compgen -> '0 90 180 270 flip0 flip90 flip180 flip270' -- "$cur")) + COMPREPLY=($(compgen -W '0 90 180 270 flip0 flip90 flip180 flip270' -- "$cur")) return ;; --record-orientation) - COMPREPLY=($(compgen -> '0 90 180 270' -- "$cur")) + COMPREPLY=($(compgen -W '0 90 180 270' -- "$cur")) return ;; --lock-video-orientation) From d2ed4510a76e0d273f9088bce6e85334b922c9cb Mon Sep 17 00:00:00 2001 From: Till Rathmann Date: Wed, 13 Dec 2023 17:04:02 +0100 Subject: [PATCH 09/79] Simulate tilt multitouch event by pressing Shift PR #4529 Signed-off-by: Romain Vimont --- app/scrcpy.1 | 6 +++++- app/src/cli.c | 6 +++++- app/src/input_manager.c | 40 ++++++++++++++++++++++++++++++++-------- app/src/input_manager.h | 2 ++ doc/control.md | 8 ++++++-- doc/shortcuts.md | 3 ++- 6 files changed, 52 insertions(+), 13 deletions(-) diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 8ca4a773..beaa99ab 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -642,7 +642,11 @@ Enable/disable FPS counter (print frames/second in logs) .TP .B Ctrl+click-and-move -Pinch-to-zoom from the center of the screen +Pinch-to-zoom and rotate from the center of the screen + +.TP +.B Shift+click-and-move +Tilt (slide vertically with two fingers) .TP .B Drag & drop APK file diff --git a/app/src/cli.c b/app/src/cli.c index fd4525f5..c580c959 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -947,7 +947,11 @@ static const struct sc_shortcut shortcuts[] = { }, { .shortcuts = { "Ctrl+click-and-move" }, - .text = "Pinch-to-zoom from the center of the screen", + .text = "Pinch-to-zoom and rotate from the center of the screen", + }, + { + .shortcuts = { "Shift+click-and-move" }, + .text = "Tilt (slide vertically with two fingers)", }, { .shortcuts = { "Drag & drop APK file" }, diff --git a/app/src/input_manager.c b/app/src/input_manager.c index 9a487836..76cfbd92 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -76,6 +76,8 @@ sc_input_manager_init(struct sc_input_manager *im, im->sdl_shortcut_mods.count = shortcut_mods->count; im->vfinger_down = false; + im->vfinger_invert_x = false; + im->vfinger_invert_y = false; im->last_keycode = SDLK_UNKNOWN; im->last_mod = 0; @@ -347,9 +349,14 @@ simulate_virtual_finger(struct sc_input_manager *im, } static struct sc_point -inverse_point(struct sc_point point, struct sc_size size) { - point.x = size.width - point.x; - point.y = size.height - point.y; +inverse_point(struct sc_point point, struct sc_size size, + bool invert_x, bool invert_y) { + if (invert_x) { + point.x = size.width - point.x; + } + if (invert_y) { + point.y = size.height - point.y; + } return point; } @@ -605,7 +612,9 @@ sc_input_manager_process_mouse_motion(struct sc_input_manager *im, struct sc_point mouse = sc_screen_convert_window_to_frame_coords(im->screen, event->x, event->y); - struct sc_point vfinger = inverse_point(mouse, im->screen->frame_size); + struct sc_point vfinger = inverse_point(mouse, im->screen->frame_size, + im->vfinger_invert_x, + im->vfinger_invert_y); simulate_virtual_finger(im, AMOTION_EVENT_ACTION_MOVE, vfinger); } } @@ -726,7 +735,7 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im, return; } - // Pinch-to-zoom simulation. + // Pinch-to-zoom, rotate and tilt simulation. // // If Ctrl is hold when the left-click button is pressed, then // pinch-to-zoom mode is enabled: on every mouse event until the left-click @@ -735,14 +744,29 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im, // // In other words, the center of the rotation/scaling is the center of the // screen. -#define CTRL_PRESSED (SDL_GetModState() & (KMOD_LCTRL | KMOD_RCTRL)) + // + // To simulate a tilt gesture (a vertical slide with two fingers), Shift + // can be used instead of Ctrl. The "virtual finger" has a position + // inverted with respect to the vertical axis of symmetry in the middle of + // the screen. + const SDL_Keymod keymod = SDL_GetModState(); + const bool ctrl_pressed = keymod & KMOD_CTRL; + const bool shift_pressed = keymod & KMOD_SHIFT; if (event->button == SDL_BUTTON_LEFT && - ((down && !im->vfinger_down && CTRL_PRESSED) || + ((down && !im->vfinger_down && + ((ctrl_pressed && !shift_pressed) || + (!ctrl_pressed && shift_pressed))) || (!down && im->vfinger_down))) { struct sc_point mouse = sc_screen_convert_window_to_frame_coords(im->screen, event->x, event->y); - struct sc_point vfinger = inverse_point(mouse, im->screen->frame_size); + if (down) { + im->vfinger_invert_x = ctrl_pressed || shift_pressed; + im->vfinger_invert_y = ctrl_pressed; + } + struct sc_point vfinger = inverse_point(mouse, im->screen->frame_size, + im->vfinger_invert_x, + im->vfinger_invert_y); enum android_motionevent_action action = down ? AMOTION_EVENT_ACTION_DOWN : AMOTION_EVENT_ACTION_UP; diff --git a/app/src/input_manager.h b/app/src/input_manager.h index b5a762eb..2ce11b03 100644 --- a/app/src/input_manager.h +++ b/app/src/input_manager.h @@ -32,6 +32,8 @@ struct sc_input_manager { } sdl_shortcut_mods; bool vfinger_down; + bool vfinger_invert_x; + bool vfinger_invert_y; // Tracks the number of identical consecutive shortcut key down events. // Not to be confused with event->repeat, which counts the number of diff --git a/doc/control.md b/doc/control.md index 0b060775..595e910e 100644 --- a/doc/control.md +++ b/doc/control.md @@ -85,7 +85,7 @@ way as MOD+Shift+v). To disable automatic clipboard synchronization, use `--no-clipboard-autosync`. -## Pinch-to-zoom +## Pinch-to-zoom, rotate and tilt simulation To simulate "pinch-to-zoom": Ctrl+_click-and-move_. @@ -93,8 +93,12 @@ More precisely, hold down Ctrl while pressing the left-click button. Until the left-click button is released, all mouse movements scale and rotate the content (if supported by the app) relative to the center of the screen. +To simulate a tilt gesture: Shift+_click-and-move-up-or-down_. + Technically, _scrcpy_ generates additional touch events from a "virtual finger" -at a location inverted through the center of the screen. +at a location inverted through the center of the screen. When pressing +Ctrl the x and y coordinates are inverted. Using Shift +only inverts x. ## Key repeat diff --git a/doc/shortcuts.md b/doc/shortcuts.md index c0fc2842..21bccbd9 100644 --- a/doc/shortcuts.md +++ b/doc/shortcuts.md @@ -49,7 +49,8 @@ _[Super] is typically the Windows or Cmd key._ | Synchronize clipboards and paste⁵ | MOD+v | Inject computer clipboard text | MOD+Shift+v | Enable/disable FPS counter (on stdout) | MOD+i - | Pinch-to-zoom | Ctrl+_click-and-move_ + | Pinch-to-zoom/rotate | Ctrl+_click-and-move_ + | Tilt (slide vertically with 2 fingers) | Shift+_click-and-move_ | Drag & drop APK file | Install APK from computer | Drag & drop non-APK file | [Push file to device](control.md#push-file-to-device) From 4cd61b5a9001043f1054502f0c29465707260cb1 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 16 Dec 2023 20:12:58 +0100 Subject: [PATCH 10/79] Fix checkstyle violation Reported by checkstyle: > [ant:checkstyle] [INFO] > scrcpy/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java:48: > Line is longer than 150 characters (found 167). [LineLength] --- .../java/com/genymobile/scrcpy/wrappers/ClipboardManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java index 0866d42d..15f0ee74 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java @@ -45,7 +45,8 @@ public final class ClipboardManager { getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, int.class, String.class); getMethodVersion = 3; } catch (NoSuchMethodException e4) { - getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, String.class, int.class, int.class, boolean.class); + getPrimaryClipMethod = manager.getClass() + .getMethod("getPrimaryClip", String.class, String.class, int.class, int.class, boolean.class); getMethodVersion = 4; } } From ec41896c853325d670e361f641f4490ed71b641f Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 16 Dec 2023 20:06:45 +0100 Subject: [PATCH 11/79] Fix integer overflow for audio packet duration The result is assigned to a long (64-bit signed integer), but the intermediate multiplication was stored in an int (32-bit signed integer). This value is only used as a fallback when no timestamp could be retrieved, that's why it did not cause too much harm so far. Fixes #4536 --- server/src/main/java/com/genymobile/scrcpy/AudioCapture.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java index e3de50e6..76e2f63b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java @@ -159,7 +159,7 @@ public final class AudioCapture { pts = nextPts; } - long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE); + long durationUs = r * 1000000L / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE); nextPts = pts + durationUs; if (previousPts != 0 && pts < previousPts + ONE_SAMPLE_US) { From 6a58891e13bc3d5b531aa93718591ec495c4fb6f Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 16 Dec 2023 20:09:08 +0100 Subject: [PATCH 12/79] Use current time as initial timestamp on error If the initial timestamp could not be retrieved, use the current time as returned by System.nanoTime(). In practice, it is the same time base as AudioRecord timestamps. Fixes #4536 --- server/src/main/java/com/genymobile/scrcpy/AudioCapture.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java index 76e2f63b..45634c70 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java @@ -153,7 +153,8 @@ public final class AudioCapture { previousRecorderTimestamp = timestamp.nanoTime; } else { if (nextPts == 0) { - Ln.w("Could not get any audio timestamp"); + Ln.w("Could not get initial audio timestamp"); + nextPts = System.nanoTime() / 1000; } // compute from previous timestamp and packet size pts = nextPts; From cd4056d0f333aee26efa8ba083e08c59b6d63a70 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 2 Jan 2024 10:22:06 +0100 Subject: [PATCH 13/79] Fix include formatting --- app/src/audio_player.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/src/audio_player.h b/app/src/audio_player.h index a03e9e35..30378246 100644 --- a/app/src/audio_player.h +++ b/app/src/audio_player.h @@ -4,16 +4,16 @@ #include "common.h" #include -#include "trait/frame_sink.h" -#include -#include -#include -#include - #include #include #include +#include "trait/frame_sink.h" +#include "util/audiobuf.h" +#include "util/average.h" +#include "util/thread.h" +#include "util/tick.h" + struct sc_audio_player { struct sc_frame_sink frame_sink; From d067a11478e5a6312325bdd051c4ffd1362945c5 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 7 Jan 2024 21:06:09 +0100 Subject: [PATCH 14/79] Do not power on if no video Power on the device on start only if video capture is enabled. Note that it only impacts display mirroring, since control is completely disabled if video source is camera. Refs 110b3a16f6d02124a4567d2ab79fcb74d78f949f --- app/src/cli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/src/cli.c b/app/src/cli.c index c580c959..f7d7e390 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -2398,6 +2398,8 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], if (!opts->video) { opts->video_playback = false; + // Do not power on the device on start if video capture is disabled + opts->power_on = false; } if (!opts->audio) { From 2ad93d1fc0094ba8263a05f0b162f2607aa68ea9 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 15 Jan 2024 22:01:19 +0100 Subject: [PATCH 15/79] Fix scrcpy_otg() return value on error The function now returns an enum scrcpy_exit_code, not a bool. --- app/src/usb/scrcpy_otg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/usb/scrcpy_otg.c b/app/src/usb/scrcpy_otg.c index 6a7fd79b..dfb0b9e9 100644 --- a/app/src/usb/scrcpy_otg.c +++ b/app/src/usb/scrcpy_otg.c @@ -62,7 +62,7 @@ scrcpy_otg(struct scrcpy_options *options) { // Minimal SDL initialization if (SDL_Init(SDL_INIT_EVENTS)) { LOGE("Could not initialize SDL: %s", SDL_GetError()); - return false; + return SCRCPY_EXIT_FAILURE; } atexit(SDL_Quit); From 5187f7254e4b556e3731fdb77fda68ae2e9fddce Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 16 Jan 2024 18:59:36 +0100 Subject: [PATCH 16/79] Add another clipboard workaround for IQOO device Fixes #4589 Refs 5ce8672ebc56b7286e1078a39abc64903e5664d0 Refs #4492 --- .../scrcpy/wrappers/ClipboardManager.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java index 15f0ee74..daea6db3 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java @@ -45,9 +45,16 @@ public final class ClipboardManager { getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, int.class, String.class); getMethodVersion = 3; } catch (NoSuchMethodException e4) { - getPrimaryClipMethod = manager.getClass() - .getMethod("getPrimaryClip", String.class, String.class, int.class, int.class, boolean.class); - getMethodVersion = 4; + try { + getPrimaryClipMethod = manager.getClass() + .getMethod("getPrimaryClip", String.class, String.class, int.class, int.class, boolean.class); + getMethodVersion = 4; + } catch (NoSuchMethodException e5) { + getPrimaryClipMethod = manager.getClass() + .getMethod("getPrimaryClip", String.class, String.class, String.class, String.class, int.class, int.class, + boolean.class); + getMethodVersion = 5; + } } } } @@ -95,9 +102,11 @@ public final class ClipboardManager { return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID, 0); case 3: return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, FakeContext.ROOT_UID, null); - default: + case 4: // The last boolean parameter is "userOperate" return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID, 0, true); + default: + return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, null, null, FakeContext.ROOT_UID, 0, true); } } From 7c53a29d72cb0725e960c1b92732193251984558 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 26 Jan 2024 13:13:14 +0100 Subject: [PATCH 17/79] Remove useless run script This script was outdated and redundant with ./run. --- meson.build | 2 -- scripts/run-scrcpy.sh | 2 -- 2 files changed, 4 deletions(-) delete mode 100755 scripts/run-scrcpy.sh diff --git a/meson.build b/meson.build index 11b974e0..4ae91f69 100644 --- a/meson.build +++ b/meson.build @@ -16,5 +16,3 @@ endif if get_option('compile_server') subdir('server') endif - -run_target('run', command: ['scripts/run-scrcpy.sh']) diff --git a/scripts/run-scrcpy.sh b/scripts/run-scrcpy.sh deleted file mode 100755 index e93b639f..00000000 --- a/scripts/run-scrcpy.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -SCRCPY_SERVER_PATH="$MESON_BUILD_ROOT/server/scrcpy-server" "$MESON_BUILD_ROOT/app/scrcpy" From 3333e67452e52a1e0cd1c68181067f9eccdeb582 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 1 Feb 2024 09:18:14 +0100 Subject: [PATCH 18/79] Fix memory leak on error Fixes #4636 --- app/src/adb/adb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/adb/adb.c b/app/src/adb/adb.c index 54375451..15c9c85a 100644 --- a/app/src/adb/adb.c +++ b/app/src/adb/adb.c @@ -458,6 +458,7 @@ sc_adb_list_devices(struct sc_intr *intr, unsigned flags, // in the buffer in a single pass LOGW("Result of \"adb devices -l\" does not fit in 64Kb. " "Please report an issue."); + free(buf); return false; } From d25cbc55f2238e2f9e621c83f27d675a7de913aa Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 9 Feb 2024 18:32:48 +0100 Subject: [PATCH 19/79] Remove unused field --- .../java/com/genymobile/scrcpy/wrappers/ContentProvider.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java index 8171988e..89c1d0e2 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java @@ -42,8 +42,6 @@ public final class ContentProvider implements Closeable { private Method callMethod; private int callMethodVersion; - private Object attributionSource; - ContentProvider(ActivityManager manager, Object provider, String name, IBinder token) { this.manager = manager; this.provider = provider; From 05b5deacadf25f706a62b981a1558c75b37dea93 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 9 Feb 2024 18:36:27 +0100 Subject: [PATCH 20/79] Move service managers creation Create the service managers from each manager wrapper class rather than from their getter in ServiceManager. The way a wrapper retrieve the underlying service is an implementation detail, and it must be consistent with the way it accesses it, so it is better to write the creation in the wrapper. --- .../scrcpy/wrappers/ActivityManager.java | 15 ++++- .../scrcpy/wrappers/ClipboardManager.java | 13 ++++- .../scrcpy/wrappers/DisplayManager.java | 17 +++++- .../scrcpy/wrappers/InputManager.java | 24 +++++++- .../scrcpy/wrappers/PowerManager.java | 7 ++- .../scrcpy/wrappers/ServiceManager.java | 58 +++---------------- .../scrcpy/wrappers/StatusBarManager.java | 7 ++- .../scrcpy/wrappers/WindowManager.java | 7 ++- 8 files changed, 92 insertions(+), 56 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java index 75115618..fd0a7798 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java @@ -26,7 +26,20 @@ public final class ActivityManager { private Method startActivityAsUserWithFeatureMethod; private Method forceStopPackageMethod; - public ActivityManager(IInterface manager) { + static ActivityManager create() { + try { + // On old Android versions, the ActivityManager is not exposed via AIDL, + // so use ActivityManagerNative.getDefault() + Class cls = Class.forName("android.app.ActivityManagerNative"); + Method getDefaultMethod = cls.getDeclaredMethod("getDefault"); + IInterface am = (IInterface) getDefaultMethod.invoke(null); + return new ActivityManager(am); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + private ActivityManager(IInterface manager) { this.manager = manager; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java index daea6db3..2a09b200 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java @@ -20,7 +20,18 @@ public final class ClipboardManager { private int setMethodVersion; private int addListenerMethodVersion; - public ClipboardManager(IInterface manager) { + static ClipboardManager create() { + IInterface clipboard = ServiceManager.getService("clipboard", "android.content.IClipboard"); + if (clipboard == null) { + // Some devices have no clipboard manager + // + // + return null; + } + return new ClipboardManager(clipboard); + } + + private ClipboardManager(IInterface manager) { this.manager = manager; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java index 17b9ae4d..80785a9f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java @@ -5,16 +5,31 @@ import com.genymobile.scrcpy.DisplayInfo; import com.genymobile.scrcpy.Ln; import com.genymobile.scrcpy.Size; +import android.annotation.SuppressLint; import android.view.Display; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.regex.Matcher; import java.util.regex.Pattern; +@SuppressLint("PrivateApi,DiscouragedPrivateApi") public final class DisplayManager { private final Object manager; // instance of hidden class android.hardware.display.DisplayManagerGlobal - public DisplayManager(Object manager) { + static DisplayManager create() { + try { + Class clazz = Class.forName("android.hardware.display.DisplayManagerGlobal"); + Method getInstanceMethod = clazz.getDeclaredMethod("getInstance"); + Object dmg = getInstanceMethod.invoke(null); + return new DisplayManager(dmg); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new AssertionError(e); + } + } + + private DisplayManager(Object manager) { this.manager = manager; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java index ef0a4f50..c7c72dc9 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java @@ -2,12 +2,14 @@ package com.genymobile.scrcpy.wrappers; import com.genymobile.scrcpy.Ln; +import android.annotation.SuppressLint; import android.view.InputEvent; import android.view.MotionEvent; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +@SuppressLint("PrivateApi,DiscouragedPrivateApi") public final class InputManager { public static final int INJECT_INPUT_EVENT_MODE_ASYNC = 0; @@ -20,7 +22,27 @@ public final class InputManager { private static Method setDisplayIdMethod; private static Method setActionButtonMethod; - public InputManager(Object manager) { + static InputManager create() { + try { + Class inputManagerClass = getInputManagerClass(); + Method getInstanceMethod = inputManagerClass.getDeclaredMethod("getInstance"); + Object im = getInstanceMethod.invoke(null); + return new InputManager(im); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new AssertionError(e); + } + } + + private static Class getInputManagerClass() { + try { + // Parts of the InputManager class have been moved to a new InputManagerGlobal class in Android 14 preview + return Class.forName("android.hardware.input.InputManagerGlobal"); + } catch (ClassNotFoundException e) { + return android.hardware.input.InputManager.class; + } + } + + private InputManager(Object manager) { this.manager = manager; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java index 93722687..942a5880 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java @@ -13,7 +13,12 @@ public final class PowerManager { private final IInterface manager; private Method isScreenOnMethod; - public PowerManager(IInterface manager) { + static PowerManager create() { + IInterface manager = ServiceManager.getService("power", "android.os.IPowerManager"); + return new PowerManager(manager); + } + + private PowerManager(IInterface manager) { this.manager = manager; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java index 85602c19..a8a56dab 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java @@ -9,7 +9,6 @@ import android.os.IBinder; import android.os.IInterface; import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @SuppressLint("PrivateApi,DiscouragedPrivateApi") @@ -38,7 +37,7 @@ public final class ServiceManager { /* not instantiable */ } - private static IInterface getService(String service, String type) { + static IInterface getService(String service, String type) { try { IBinder binder = (IBinder) GET_SERVICE_METHOD.invoke(null, service); Method asInterfaceMethod = Class.forName(type + "$Stub").getMethod("asInterface", IBinder.class); @@ -50,90 +49,51 @@ public final class ServiceManager { public static WindowManager getWindowManager() { if (windowManager == null) { - windowManager = new WindowManager(getService("window", "android.view.IWindowManager")); + windowManager = WindowManager.create(); } return windowManager; } public static DisplayManager getDisplayManager() { if (displayManager == null) { - try { - Class clazz = Class.forName("android.hardware.display.DisplayManagerGlobal"); - Method getInstanceMethod = clazz.getDeclaredMethod("getInstance"); - Object dmg = getInstanceMethod.invoke(null); - displayManager = new DisplayManager(dmg); - } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { - throw new AssertionError(e); - } + displayManager = DisplayManager.create(); } return displayManager; } - public static Class getInputManagerClass() { - try { - // Parts of the InputManager class have been moved to a new InputManagerGlobal class in Android 14 preview - return Class.forName("android.hardware.input.InputManagerGlobal"); - } catch (ClassNotFoundException e) { - return android.hardware.input.InputManager.class; - } - } - public static InputManager getInputManager() { if (inputManager == null) { - try { - Class inputManagerClass = getInputManagerClass(); - Method getInstanceMethod = inputManagerClass.getDeclaredMethod("getInstance"); - Object im = getInstanceMethod.invoke(null); - inputManager = new InputManager(im); - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { - throw new AssertionError(e); - } + inputManager = InputManager.create(); } return inputManager; } public static PowerManager getPowerManager() { if (powerManager == null) { - powerManager = new PowerManager(getService("power", "android.os.IPowerManager")); + powerManager = PowerManager.create(); } return powerManager; } public static StatusBarManager getStatusBarManager() { if (statusBarManager == null) { - statusBarManager = new StatusBarManager(getService("statusbar", "com.android.internal.statusbar.IStatusBarService")); + statusBarManager = StatusBarManager.create(); } return statusBarManager; } public static ClipboardManager getClipboardManager() { if (clipboardManager == null) { - IInterface clipboard = getService("clipboard", "android.content.IClipboard"); - if (clipboard == null) { - // Some devices have no clipboard manager - // - // - return null; - } - clipboardManager = new ClipboardManager(clipboard); + // May be null, some devices have no clipboard manager + clipboardManager = ClipboardManager.create(); } return clipboardManager; } public static ActivityManager getActivityManager() { if (activityManager == null) { - try { - // On old Android versions, the ActivityManager is not exposed via AIDL, - // so use ActivityManagerNative.getDefault() - Class cls = Class.forName("android.app.ActivityManagerNative"); - Method getDefaultMethod = cls.getDeclaredMethod("getDefault"); - IInterface am = (IInterface) getDefaultMethod.invoke(null); - activityManager = new ActivityManager(am); - } catch (Exception e) { - throw new AssertionError(e); - } + activityManager = ActivityManager.create(); } - return activityManager; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java index 9126d5ed..e65cef5c 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java @@ -16,7 +16,12 @@ public final class StatusBarManager { private boolean expandSettingsPanelMethodNewVersion = true; private Method collapsePanelsMethod; - public StatusBarManager(IInterface manager) { + static StatusBarManager create() { + IInterface manager = ServiceManager.getService("statusbar", "com.android.internal.statusbar.IStatusBarService"); + return new StatusBarManager(manager); + } + + private StatusBarManager(IInterface manager) { this.manager = manager; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java index a746be5c..99b9148f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java @@ -17,7 +17,12 @@ public final class WindowManager { private Method isRotationFrozenMethod; private Method thawRotationMethod; - public WindowManager(IInterface manager) { + static WindowManager create() { + IInterface manager = ServiceManager.getService("window", "android.view.IWindowManager"); + return new WindowManager(manager); + } + + private WindowManager(IInterface manager) { this.manager = manager; } From f7b4a18b4398c013b2bed9630b7bb35eb50ad97e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 9 Feb 2024 18:39:57 +0100 Subject: [PATCH 21/79] Catch generic ReflectiveOperationException This exception is a super-type of: - ClassNotFoundException - IllegalAccessException - InstantiationException - InvocationTargetException - NoSuchFieldException - NoSuchMethodException Use it to simplify. --- .../scrcpy/wrappers/ActivityManager.java | 7 +++---- .../scrcpy/wrappers/ClipboardManager.java | 15 ++++++--------- .../scrcpy/wrappers/ContentProvider.java | 6 ++---- .../scrcpy/wrappers/DisplayControl.java | 5 ++--- .../scrcpy/wrappers/DisplayManager.java | 9 ++++----- .../genymobile/scrcpy/wrappers/InputManager.java | 9 ++++----- .../genymobile/scrcpy/wrappers/PowerManager.java | 3 +-- .../scrcpy/wrappers/StatusBarManager.java | 7 +++---- .../scrcpy/wrappers/SurfaceControl.java | 9 ++++----- .../genymobile/scrcpy/wrappers/WindowManager.java | 9 ++++----- 10 files changed, 33 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java index fd0a7798..367ea2e7 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java @@ -13,7 +13,6 @@ import android.os.IBinder; import android.os.IInterface; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @SuppressLint("PrivateApi,DiscouragedPrivateApi") @@ -34,7 +33,7 @@ public final class ActivityManager { Method getDefaultMethod = cls.getDeclaredMethod("getDefault"); IInterface am = (IInterface) getDefaultMethod.invoke(null); return new ActivityManager(am); - } catch (Exception e) { + } catch (ReflectiveOperationException e) { throw new AssertionError(e); } } @@ -89,7 +88,7 @@ public final class ActivityManager { return null; } return new ContentProvider(this, provider, name, token); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException | NoSuchFieldException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return null; } @@ -99,7 +98,7 @@ public final class ActivityManager { try { Method method = getRemoveContentProviderExternalMethod(); method.invoke(manager, name, token); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java index 2a09b200..2c8d9907 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java @@ -8,7 +8,6 @@ import android.content.IOnPrimaryClipChangedListener; import android.os.Build; import android.os.IInterface; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; public final class ClipboardManager { @@ -98,8 +97,7 @@ public final class ClipboardManager { return setPrimaryClipMethod; } - private static ClipData getPrimaryClip(Method method, int methodVersion, IInterface manager) - throws InvocationTargetException, IllegalAccessException { + private static ClipData getPrimaryClip(Method method, int methodVersion, IInterface manager) throws ReflectiveOperationException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME); } @@ -121,8 +119,7 @@ public final class ClipboardManager { } } - private static void setPrimaryClip(Method method, int methodVersion, IInterface manager, ClipData clipData) - throws InvocationTargetException, IllegalAccessException { + private static void setPrimaryClip(Method method, int methodVersion, IInterface manager, ClipData clipData) throws ReflectiveOperationException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { method.invoke(manager, clipData, FakeContext.PACKAGE_NAME); return; @@ -149,7 +146,7 @@ public final class ClipboardManager { return null; } return clipData.getItemAt(0).getText(); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return null; } @@ -161,14 +158,14 @@ public final class ClipboardManager { ClipData clipData = ClipData.newPlainText(null, text); setPrimaryClip(method, setMethodVersion, manager, clipData); return true; - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return false; } } private static void addPrimaryClipChangedListener(Method method, int methodVersion, IInterface manager, IOnPrimaryClipChangedListener listener) - throws InvocationTargetException, IllegalAccessException { + throws ReflectiveOperationException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { method.invoke(manager, listener, FakeContext.PACKAGE_NAME); return; @@ -220,7 +217,7 @@ public final class ClipboardManager { Method method = getAddPrimaryClipChangedListener(); addPrimaryClipChangedListener(method, addListenerMethodVersion, manager, listener); return true; - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return false; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java index 89c1d0e2..a03f824e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java @@ -11,7 +11,6 @@ import android.os.Bundle; import android.os.IBinder; import java.io.Closeable; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; public final class ContentProvider implements Closeable { @@ -75,8 +74,7 @@ public final class ContentProvider implements Closeable { return callMethod; } - private Bundle call(String callMethod, String arg, Bundle extras) - throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + private Bundle call(String callMethod, String arg, Bundle extras) throws ReflectiveOperationException { try { Method method = getCallMethod(); Object[] args; @@ -97,7 +95,7 @@ public final class ContentProvider implements Closeable { } } return (Bundle) method.invoke(provider, args); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); throw e; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayControl.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayControl.java index 4e19beb9..ba3e9ee0 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayControl.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayControl.java @@ -7,7 +7,6 @@ import android.annotation.TargetApi; import android.os.Build; import android.os.IBinder; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @SuppressLint({"PrivateApi", "SoonBlockedPrivateApi", "BlockedPrivateApi"}) @@ -55,7 +54,7 @@ public final class DisplayControl { try { Method method = getGetPhysicalDisplayTokenMethod(); return (IBinder) method.invoke(null, physicalDisplayId); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return null; } @@ -72,7 +71,7 @@ public final class DisplayControl { try { Method method = getGetPhysicalDisplayIdsMethod(); return (long[]) method.invoke(null); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return null; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java index 80785a9f..33a061ba 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java @@ -9,7 +9,6 @@ import android.annotation.SuppressLint; import android.view.Display; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -24,7 +23,7 @@ public final class DisplayManager { Method getInstanceMethod = clazz.getDeclaredMethod("getInstance"); Object dmg = getInstanceMethod.invoke(null); return new DisplayManager(dmg); - } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + } catch (ReflectiveOperationException e) { throw new AssertionError(e); } } @@ -75,7 +74,7 @@ public final class DisplayManager { try { Field filed = Display.class.getDeclaredField(flagString); flags |= filed.getInt(null); - } catch (NoSuchFieldException | IllegalAccessException e) { + } catch (ReflectiveOperationException e) { // Silently ignore, some flags reported by "dumpsys display" are @TestApi } } @@ -97,7 +96,7 @@ public final class DisplayManager { int layerStack = cls.getDeclaredField("layerStack").getInt(displayInfo); int flags = cls.getDeclaredField("flags").getInt(displayInfo); return new DisplayInfo(displayId, new Size(width, height), rotation, layerStack, flags); - } catch (Exception e) { + } catch (ReflectiveOperationException e) { throw new AssertionError(e); } } @@ -105,7 +104,7 @@ public final class DisplayManager { public int[] getDisplayIds() { try { return (int[]) manager.getClass().getMethod("getDisplayIds").invoke(manager); - } catch (Exception e) { + } catch (ReflectiveOperationException e) { throw new AssertionError(e); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java index c7c72dc9..16ecb09f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java @@ -6,7 +6,6 @@ import android.annotation.SuppressLint; import android.view.InputEvent; import android.view.MotionEvent; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @SuppressLint("PrivateApi,DiscouragedPrivateApi") @@ -28,7 +27,7 @@ public final class InputManager { Method getInstanceMethod = inputManagerClass.getDeclaredMethod("getInstance"); Object im = getInstanceMethod.invoke(null); return new InputManager(im); - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + } catch (ReflectiveOperationException e) { throw new AssertionError(e); } } @@ -57,7 +56,7 @@ public final class InputManager { try { Method method = getInjectInputEventMethod(); return (boolean) method.invoke(manager, inputEvent, mode); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return false; } @@ -75,7 +74,7 @@ public final class InputManager { Method method = getSetDisplayIdMethod(); method.invoke(inputEvent, displayId); return true; - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Cannot associate a display id to the input event", e); return false; } @@ -93,7 +92,7 @@ public final class InputManager { Method method = getSetActionButtonMethod(); method.invoke(motionEvent, actionButton); return true; - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Cannot set action button on MotionEvent", e); return false; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java index 942a5880..36d5f1ac 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java @@ -6,7 +6,6 @@ import android.annotation.SuppressLint; import android.os.Build; import android.os.IInterface; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; public final class PowerManager { @@ -35,7 +34,7 @@ public final class PowerManager { try { Method method = getIsScreenOnMethod(); return (boolean) method.invoke(manager); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return false; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java index e65cef5c..af217da2 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java @@ -4,7 +4,6 @@ import com.genymobile.scrcpy.Ln; import android.os.IInterface; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; public final class StatusBarManager { @@ -67,7 +66,7 @@ public final class StatusBarManager { } else { method.invoke(manager); } - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); } } @@ -82,7 +81,7 @@ public final class StatusBarManager { // old version method.invoke(manager); } - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); } } @@ -91,7 +90,7 @@ public final class StatusBarManager { try { Method method = getCollapsePanelsMethod(); method.invoke(manager); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java index 98259e7f..4a3d0bfe 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java @@ -8,7 +8,6 @@ import android.os.Build; import android.os.IBinder; import android.view.Surface; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @SuppressLint("PrivateApi") @@ -109,7 +108,7 @@ public final class SurfaceControl { // call getInternalDisplayToken() return (IBinder) method.invoke(null); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return null; } @@ -126,7 +125,7 @@ public final class SurfaceControl { try { Method method = getGetPhysicalDisplayTokenMethod(); return (IBinder) method.invoke(null, physicalDisplayId); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return null; } @@ -152,7 +151,7 @@ public final class SurfaceControl { try { Method method = getGetPhysicalDisplayIdsMethod(); return (long[]) method.invoke(null); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return null; } @@ -170,7 +169,7 @@ public final class SurfaceControl { Method method = getSetDisplayPowerModeMethod(); method.invoke(null, displayToken, mode); return true; - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return false; } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java index 99b9148f..b19dace9 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java @@ -7,7 +7,6 @@ import android.os.IInterface; import android.view.IDisplayFoldListener; import android.view.IRotationWatcher; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; public final class WindowManager { @@ -66,7 +65,7 @@ public final class WindowManager { try { Method method = getGetRotationMethod(); return (int) method.invoke(manager); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return 0; } @@ -76,7 +75,7 @@ public final class WindowManager { try { Method method = getFreezeRotationMethod(); method.invoke(manager, rotation); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); } } @@ -85,7 +84,7 @@ public final class WindowManager { try { Method method = getIsRotationFrozenMethod(); return (boolean) method.invoke(manager); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return false; } @@ -95,7 +94,7 @@ public final class WindowManager { try { Method method = getThawRotationMethod(); method.invoke(manager); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { + } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); } } From be3f949aa5c4dcad276ac0f2b36671a3309ce12f Mon Sep 17 00:00:00 2001 From: wuderek Date: Fri, 9 Feb 2024 16:02:48 +0800 Subject: [PATCH 22/79] Adapt to display API changes The method SurfaceControl.createDisplay() has been removed in AOSP. Use DisplayManager to create a VirtualDisplay object instead. Fixes #4646 Fixes #4656 PR #4657 Signed-off-by: Romain Vimont --- .../java/com/genymobile/scrcpy/Device.java | 4 +++ .../com/genymobile/scrcpy/ScreenCapture.java | 29 +++++++++++++++++-- .../scrcpy/wrappers/DisplayManager.java | 16 ++++++++++ .../scrcpy/wrappers/SurfaceControl.java | 8 ++--- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java index 2324ce90..33b09a57 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/Device.java @@ -164,6 +164,10 @@ public final class Device { } } + public int getDisplayId() { + return displayId; + } + public synchronized void setMaxSize(int newMaxSize) { maxSize = newMaxSize; screenInfo = ScreenInfo.computeScreenInfo(screenInfo.getReverseVideoRotation(), deviceSize, crop, newMaxSize, lockVideoOrientation); diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java index e048354a..95214188 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java @@ -1,8 +1,10 @@ package com.genymobile.scrcpy; +import com.genymobile.scrcpy.wrappers.ServiceManager; import com.genymobile.scrcpy.wrappers.SurfaceControl; import android.graphics.Rect; +import android.hardware.display.VirtualDisplay; import android.os.Build; import android.os.IBinder; import android.view.Surface; @@ -11,6 +13,7 @@ public class ScreenCapture extends SurfaceCapture implements Device.RotationList private final Device device; private IBinder display; + private VirtualDisplay virtualDisplay; public ScreenCapture(Device device) { this.device = device; @@ -34,9 +37,29 @@ public class ScreenCapture extends SurfaceCapture implements Device.RotationList if (display != null) { SurfaceControl.destroyDisplay(display); + display = null; + } + if (virtualDisplay != null) { + virtualDisplay.release(); + virtualDisplay = null; + } + + try { + display = createDisplay(); + setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); + Ln.d("Display: using SurfaceControl API"); + } catch (Exception surfaceControlException) { + Rect videoRect = screenInfo.getVideoSize().toRect(); + try { + virtualDisplay = ServiceManager.getDisplayManager() + .createVirtualDisplay("scrcpy", videoRect.width(), videoRect.height(), device.getDisplayId(), surface); + Ln.d("Display: using DisplayManager API"); + } catch (Exception displayManagerException) { + Ln.e("Could not create display using SurfaceControl", surfaceControlException); + Ln.e("Could not create display using DisplayManager", displayManagerException); + throw new AssertionError("Could not create display"); + } } - display = createDisplay(); - setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); } @Override @@ -69,7 +92,7 @@ public class ScreenCapture extends SurfaceCapture implements Device.RotationList requestReset(); } - private static IBinder createDisplay() { + private static IBinder createDisplay() throws Exception { // Since Android 12 (preview), secure displays could not be created with shell permissions anymore. // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S". boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S".equals( diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java index 33a061ba..2ff82d04 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java @@ -6,7 +6,9 @@ import com.genymobile.scrcpy.Ln; import com.genymobile.scrcpy.Size; import android.annotation.SuppressLint; +import android.hardware.display.VirtualDisplay; import android.view.Display; +import android.view.Surface; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -16,6 +18,7 @@ import java.util.regex.Pattern; @SuppressLint("PrivateApi,DiscouragedPrivateApi") public final class DisplayManager { private final Object manager; // instance of hidden class android.hardware.display.DisplayManagerGlobal + private Method createVirtualDisplayMethod; static DisplayManager create() { try { @@ -108,4 +111,17 @@ public final class DisplayManager { throw new AssertionError(e); } } + + private Method getCreateVirtualDisplayMethod() throws NoSuchMethodException { + if (createVirtualDisplayMethod == null) { + createVirtualDisplayMethod = android.hardware.display.DisplayManager.class + .getMethod("createVirtualDisplay", String.class, int.class, int.class, int.class, Surface.class); + } + return createVirtualDisplayMethod; + } + + public VirtualDisplay createVirtualDisplay(String name, int width, int height, int displayIdToMirror, Surface surface) throws Exception { + Method method = getCreateVirtualDisplayMethod(); + return (VirtualDisplay) method.invoke(null, name, width, height, displayIdToMirror, surface); + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java index 4a3d0bfe..f0e351a2 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java @@ -77,12 +77,8 @@ public final class SurfaceControl { } } - public static IBinder createDisplay(String name, boolean secure) { - try { - return (IBinder) CLASS.getMethod("createDisplay", String.class, boolean.class).invoke(null, name, secure); - } catch (Exception e) { - throw new AssertionError(e); - } + public static IBinder createDisplay(String name, boolean secure) throws Exception { + return (IBinder) CLASS.getMethod("createDisplay", String.class, boolean.class).invoke(null, name, secure); } private static Method getGetBuiltInDisplayMethod() throws NoSuchMethodException { From 9efa162949c2a3e3e42564862ff390700270394d Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 3 Feb 2024 18:52:10 +0100 Subject: [PATCH 23/79] Configure clean up actions dynamically Some actions may be performed when scrcpy exits, currently: - disable "show touches" - restore "stay on while plugged in" - power off screen - restore "power mode" (to disable "turn screen off") They are performed from a separate process so that they can be executed even when scrcpy-server is killed (e.g. if the device is unplugged). The clean up actions to perform were configured when scrcpy started. Given that there is no method to read the current "power mode" in Android, and that "turn screen off" can be applied at any time using an scrcpy shortcut, there was no way to determine if "power mode" had to be restored on exit. Therefore, it was always restored to "normal", even when not necessary. However, setting the "power mode" is quite fragile on some devices, and may cause some issues, so it is preferable to call it only when necessary (when "turn screen off" has actually been called). For that purpose, make the scrcpy-server main process and the clean up process communicate the actions to perform over a pipe (stdin/stdout), so that they can be changed dynamically. In particular, when the power mode is changed at runtime, notify the clean up process. Refs 1beec99f8283713b1fbf0b3704eb4dceecc9a590 Refs #4456 Refs #4624 PR #4649 --- .../java/com/genymobile/scrcpy/CleanUp.java | 218 +++++++----------- .../com/genymobile/scrcpy/Controller.java | 8 +- .../java/com/genymobile/scrcpy/Server.java | 83 ++++--- 3 files changed, 140 insertions(+), 169 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/CleanUp.java b/server/src/main/java/com/genymobile/scrcpy/CleanUp.java index c84e25bb..f9b1efd6 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CleanUp.java +++ b/server/src/main/java/com/genymobile/scrcpy/CleanUp.java @@ -1,11 +1,8 @@ package com.genymobile.scrcpy; -import android.os.Parcel; -import android.os.Parcelable; -import android.util.Base64; - import java.io.File; import java.io.IOException; +import java.io.OutputStream; /** * Handle the cleanup of scrcpy, even if the main process is killed. @@ -14,127 +11,59 @@ import java.io.IOException; */ public final class CleanUp { - // A simple struct to be passed from the main process to the cleanup process - public static class Config implements Parcelable { - - public static final Creator CREATOR = new Creator() { - @Override - public Config createFromParcel(Parcel in) { - return new Config(in); - } - - @Override - public Config[] newArray(int size) { - return new Config[size]; - } - }; - - private static final int FLAG_DISABLE_SHOW_TOUCHES = 1; - private static final int FLAG_RESTORE_NORMAL_POWER_MODE = 2; - private static final int FLAG_POWER_OFF_SCREEN = 4; - - private int displayId; - - // Restore the value (between 0 and 7), -1 to not restore - // - private int restoreStayOn = -1; - - private boolean disableShowTouches; - private boolean restoreNormalPowerMode; - private boolean powerOffScreen; - - public Config() { - // Default constructor, the fields are initialized by CleanUp.configure() - } - - protected Config(Parcel in) { - displayId = in.readInt(); - restoreStayOn = in.readInt(); - byte options = in.readByte(); - disableShowTouches = (options & FLAG_DISABLE_SHOW_TOUCHES) != 0; - restoreNormalPowerMode = (options & FLAG_RESTORE_NORMAL_POWER_MODE) != 0; - powerOffScreen = (options & FLAG_POWER_OFF_SCREEN) != 0; - } - - @Override - public void writeToParcel(Parcel dest, int flags) { - dest.writeInt(displayId); - dest.writeInt(restoreStayOn); - byte options = 0; - if (disableShowTouches) { - options |= FLAG_DISABLE_SHOW_TOUCHES; - } - if (restoreNormalPowerMode) { - options |= FLAG_RESTORE_NORMAL_POWER_MODE; - } - if (powerOffScreen) { - options |= FLAG_POWER_OFF_SCREEN; - } - dest.writeByte(options); - } + private static final int MSG_TYPE_MASK = 0b11; + private static final int MSG_TYPE_RESTORE_STAY_ON = 0; + private static final int MSG_TYPE_DISABLE_SHOW_TOUCHES = 1; + private static final int MSG_TYPE_RESTORE_NORMAL_POWER_MODE = 2; + private static final int MSG_TYPE_POWER_OFF_SCREEN = 3; - private boolean hasWork() { - return disableShowTouches || restoreStayOn != -1 || restoreNormalPowerMode || powerOffScreen; - } + private static final int MSG_PARAM_SHIFT = 2; - @Override - public int describeContents() { - return 0; - } + private final OutputStream out; - byte[] serialize() { - Parcel parcel = Parcel.obtain(); - writeToParcel(parcel, 0); - byte[] bytes = parcel.marshall(); - parcel.recycle(); - return bytes; - } + public CleanUp(OutputStream out) { + this.out = out; + } - static Config deserialize(byte[] bytes) { - Parcel parcel = Parcel.obtain(); - parcel.unmarshall(bytes, 0, bytes.length); - parcel.setDataPosition(0); - return CREATOR.createFromParcel(parcel); - } + public static CleanUp configure(int displayId) throws IOException { + String[] cmd = {"app_process", "/", CleanUp.class.getName(), String.valueOf(displayId)}; - static Config fromBase64(String base64) { - byte[] bytes = Base64.decode(base64, Base64.NO_WRAP); - return deserialize(bytes); - } + ProcessBuilder builder = new ProcessBuilder(cmd); + builder.environment().put("CLASSPATH", Server.SERVER_PATH); + Process process = builder.start(); + return new CleanUp(process.getOutputStream()); + } - String toBase64() { - byte[] bytes = serialize(); - return Base64.encodeToString(bytes, Base64.NO_WRAP); + private boolean sendMessage(int type, int param) { + assert (type & ~MSG_TYPE_MASK) == 0; + int msg = type | param << MSG_PARAM_SHIFT; + try { + out.write(msg); + out.flush(); + return true; + } catch (IOException e) { + Ln.w("Could not configure cleanup (type=" + type + ", param=" + param + ")", e); + return false; } } - private CleanUp() { - // not instantiable + public boolean setRestoreStayOn(int restoreValue) { + // Restore the value (between 0 and 7), -1 to not restore + // + assert restoreValue >= -1 && restoreValue <= 7; + return sendMessage(MSG_TYPE_RESTORE_STAY_ON, restoreValue & 0b1111); } - public static void configure(int displayId, int restoreStayOn, boolean disableShowTouches, boolean restoreNormalPowerMode, boolean powerOffScreen) - throws IOException { - Config config = new Config(); - config.displayId = displayId; - config.disableShowTouches = disableShowTouches; - config.restoreStayOn = restoreStayOn; - config.restoreNormalPowerMode = restoreNormalPowerMode; - config.powerOffScreen = powerOffScreen; - - if (config.hasWork()) { - startProcess(config); - } else { - // There is no additional clean up to do when scrcpy dies - unlinkSelf(); - } + public boolean setDisableShowTouches(boolean disableOnExit) { + return sendMessage(MSG_TYPE_DISABLE_SHOW_TOUCHES, disableOnExit ? 1 : 0); } - private static void startProcess(Config config) throws IOException { - String[] cmd = {"app_process", "/", CleanUp.class.getName(), config.toBase64()}; + public boolean setRestoreNormalPowerMode(boolean restoreOnExit) { + return sendMessage(MSG_TYPE_RESTORE_NORMAL_POWER_MODE, restoreOnExit ? 1 : 0); + } - ProcessBuilder builder = new ProcessBuilder(cmd); - builder.environment().put("CLASSPATH", Server.SERVER_PATH); - builder.start(); + public boolean setPowerOffScreen(boolean powerOffScreenOnExit) { + return sendMessage(MSG_TYPE_POWER_OFF_SCREEN, powerOffScreenOnExit ? 1 : 0); } public static void unlinkSelf() { @@ -148,41 +77,66 @@ public final class CleanUp { public static void main(String... args) { unlinkSelf(); + int displayId = Integer.parseInt(args[0]); + + int restoreStayOn = -1; + boolean disableShowTouches = false; + boolean restoreNormalPowerMode = false; + boolean powerOffScreen = false; + try { // Wait for the server to die - System.in.read(); + int msg; + while ((msg = System.in.read()) != -1) { + int type = msg & MSG_TYPE_MASK; + int param = msg >> MSG_PARAM_SHIFT; + switch (type) { + case MSG_TYPE_RESTORE_STAY_ON: + restoreStayOn = param > 7 ? -1 : param; + break; + case MSG_TYPE_DISABLE_SHOW_TOUCHES: + disableShowTouches = param != 0; + break; + case MSG_TYPE_RESTORE_NORMAL_POWER_MODE: + restoreNormalPowerMode = param != 0; + break; + case MSG_TYPE_POWER_OFF_SCREEN: + powerOffScreen = param != 0; + break; + default: + Ln.w("Unexpected msg type: " + type); + break; + } + } } catch (IOException e) { // Expected when the server is dead } Ln.i("Cleaning up"); - Config config = Config.fromBase64(args[0]); - - if (config.disableShowTouches || config.restoreStayOn != -1) { - if (config.disableShowTouches) { - Ln.i("Disabling \"show touches\""); - try { - Settings.putValue(Settings.TABLE_SYSTEM, "show_touches", "0"); - } catch (SettingsException e) { - Ln.e("Could not restore \"show_touches\"", e); - } + if (disableShowTouches) { + Ln.i("Disabling \"show touches\""); + try { + Settings.putValue(Settings.TABLE_SYSTEM, "show_touches", "0"); + } catch (SettingsException e) { + Ln.e("Could not restore \"show_touches\"", e); } - if (config.restoreStayOn != -1) { - Ln.i("Restoring \"stay awake\""); - try { - Settings.putValue(Settings.TABLE_GLOBAL, "stay_on_while_plugged_in", String.valueOf(config.restoreStayOn)); - } catch (SettingsException e) { - Ln.e("Could not restore \"stay_on_while_plugged_in\"", e); - } + } + + if (restoreStayOn != -1) { + Ln.i("Restoring \"stay awake\""); + try { + Settings.putValue(Settings.TABLE_GLOBAL, "stay_on_while_plugged_in", String.valueOf(restoreStayOn)); + } catch (SettingsException e) { + Ln.e("Could not restore \"stay_on_while_plugged_in\"", e); } } if (Device.isScreenOn()) { - if (config.powerOffScreen) { + if (powerOffScreen) { Ln.i("Power off screen"); - Device.powerOffScreen(config.displayId); - } else if (config.restoreNormalPowerMode) { + Device.powerOffScreen(displayId); + } else if (restoreNormalPowerMode) { Ln.i("Restoring normal power mode"); Device.setScreenPowerMode(Device.POWER_MODE_NORMAL); } diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 3b0e9031..c0763012 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -28,6 +28,7 @@ public class Controller implements AsyncProcessor { private final Device device; private final DesktopConnection connection; + private final CleanUp cleanUp; private final DeviceMessageSender sender; private final boolean clipboardAutosync; private final boolean powerOn; @@ -41,9 +42,10 @@ public class Controller implements AsyncProcessor { private boolean keepPowerModeOff; - public Controller(Device device, DesktopConnection connection, boolean clipboardAutosync, boolean powerOn) { + public Controller(Device device, DesktopConnection connection, CleanUp cleanUp, boolean clipboardAutosync, boolean powerOn) { this.device = device; this.connection = connection; + this.cleanUp = cleanUp; this.clipboardAutosync = clipboardAutosync; this.powerOn = powerOn; initPointers(); @@ -170,6 +172,10 @@ public class Controller implements AsyncProcessor { if (setPowerModeOk) { keepPowerModeOff = mode == Device.POWER_MODE_OFF; Ln.i("Device screen turned " + (mode == Device.POWER_MODE_OFF ? "off" : "on")); + if (cleanUp != null) { + boolean mustRestoreOnExit = mode != Device.POWER_MODE_NORMAL; + cleanUp.setRestoreNormalPowerMode(mustRestoreOnExit); + } } } break; diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index e4a95140..bcafa133 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -51,46 +51,47 @@ public final class Server { // not instantiable } - private static void initAndCleanUp(Options options) { - boolean mustDisableShowTouchesOnCleanUp = false; - int restoreStayOn = -1; - boolean restoreNormalPowerMode = options.getControl(); // only restore power mode if control is enabled - if (options.getShowTouches() || options.getStayAwake()) { - if (options.getShowTouches()) { - try { - String oldValue = Settings.getAndPutValue(Settings.TABLE_SYSTEM, "show_touches", "1"); - // If "show touches" was disabled, it must be disabled back on clean up - mustDisableShowTouchesOnCleanUp = !"1".equals(oldValue); - } catch (SettingsException e) { - Ln.e("Could not change \"show_touches\"", e); + private static void initAndCleanUp(Options options, CleanUp cleanUp) { + // This method is called from its own thread, so it may only configure cleanup actions which are NOT dynamic (i.e. they are configured once + // and for all, they cannot be changed from another thread) + + if (options.getShowTouches()) { + try { + String oldValue = Settings.getAndPutValue(Settings.TABLE_SYSTEM, "show_touches", "1"); + // If "show touches" was disabled, it must be disabled back on clean up + if (!"1".equals(oldValue)) { + if (!cleanUp.setDisableShowTouches(true)) { + Ln.e("Could not disable show touch on exit"); + } } + } catch (SettingsException e) { + Ln.e("Could not change \"show_touches\"", e); } + } - if (options.getStayAwake()) { - int stayOn = BatteryManager.BATTERY_PLUGGED_AC | BatteryManager.BATTERY_PLUGGED_USB | BatteryManager.BATTERY_PLUGGED_WIRELESS; + if (options.getStayAwake()) { + int stayOn = BatteryManager.BATTERY_PLUGGED_AC | BatteryManager.BATTERY_PLUGGED_USB | BatteryManager.BATTERY_PLUGGED_WIRELESS; + try { + String oldValue = Settings.getAndPutValue(Settings.TABLE_GLOBAL, "stay_on_while_plugged_in", String.valueOf(stayOn)); try { - String oldValue = Settings.getAndPutValue(Settings.TABLE_GLOBAL, "stay_on_while_plugged_in", String.valueOf(stayOn)); - try { - restoreStayOn = Integer.parseInt(oldValue); - if (restoreStayOn == stayOn) { - // No need to restore - restoreStayOn = -1; + int restoreStayOn = Integer.parseInt(oldValue); + if (restoreStayOn != stayOn) { + // Restore only if the current value is different + if (!cleanUp.setRestoreStayOn(restoreStayOn)) { + Ln.e("Could not restore stay on on exit"); } - } catch (NumberFormatException e) { - restoreStayOn = 0; } - } catch (SettingsException e) { - Ln.e("Could not change \"stay_on_while_plugged_in\"", e); + } catch (NumberFormatException e) { + // ignore } + } catch (SettingsException e) { + Ln.e("Could not change \"stay_on_while_plugged_in\"", e); } } - if (options.getCleanup()) { - try { - CleanUp.configure(options.getDisplayId(), restoreStayOn, mustDisableShowTouchesOnCleanUp, restoreNormalPowerMode, - options.getPowerOffScreenOnClose()); - } catch (IOException e) { - Ln.e("Could not configure cleanup", e); + if (options.getPowerOffScreenOnClose()) { + if (!cleanUp.setPowerOffScreen(true)) { + Ln.e("Could not power off screen on exit"); } } } @@ -101,7 +102,13 @@ public final class Server { throw new ConfigurationException("Camera mirroring is not supported"); } - Thread initThread = startInitThread(options); + CleanUp cleanUp = null; + Thread initThread = null; + + if (options.getCleanup()) { + cleanUp = CleanUp.configure(options.getDisplayId()); + initThread = startInitThread(options, cleanUp); + } int scid = options.getScid(); boolean tunnelForward = options.isTunnelForward(); @@ -124,7 +131,7 @@ public final class Server { } if (control) { - Controller controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn()); + Controller controller = new Controller(device, connection, cleanUp, options.getClipboardAutosync(), options.getPowerOn()); device.setClipboardListener(text -> controller.getSender().pushClipboardText(text)); asyncProcessors.add(controller); } @@ -167,7 +174,9 @@ public final class Server { completion.await(); } finally { - initThread.interrupt(); + if (initThread != null) { + initThread.interrupt(); + } for (AsyncProcessor asyncProcessor : asyncProcessors) { asyncProcessor.stop(); } @@ -175,7 +184,9 @@ public final class Server { connection.shutdown(); try { - initThread.join(); + if (initThread != null) { + initThread.join(); + } for (AsyncProcessor asyncProcessor : asyncProcessors) { asyncProcessor.join(); } @@ -187,8 +198,8 @@ public final class Server { } } - private static Thread startInitThread(final Options options) { - Thread thread = new Thread(() -> initAndCleanUp(options), "init-cleanup"); + private static Thread startInitThread(final Options options, final CleanUp cleanUp) { + Thread thread = new Thread(() -> initAndCleanUp(options, cleanUp), "init-cleanup"); thread.start(); return thread; } From d47ecef1b561322872437368edbfff69dc5ad0bd Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 1 Feb 2024 17:27:14 +0100 Subject: [PATCH 24/79] Limit buffering time value This avoids unreasonable values which could lead to integer overflow. PR #4572 --- app/src/cli.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/src/cli.c b/app/src/cli.c index f7d7e390..b2b02ecd 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -1385,7 +1385,11 @@ parse_max_fps(const char *s, uint16_t *max_fps) { static bool parse_buffering_time(const char *s, sc_tick *tick) { long value; - bool ok = parse_integer_arg(s, &value, false, 0, 0x7FFFFFFF, + // In practice, buffering time should not exceed a few seconds. + // Limit it to some arbitrary value (1 hour) to prevent 32-bit overflow + // when multiplied by the audio sample size and the number of samples per + // millisecond. + bool ok = parse_integer_arg(s, &value, false, 0, 60 * 60 * 1000, "buffering time"); if (!ok) { return false; From cfa4f7e2f2ac867dd6d6278a48cc470e82d42f37 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 1 Jan 2024 19:22:55 +0100 Subject: [PATCH 25/79] Replace locks by atomics in audio player The audio output thread only reads samples from the buffer, and most of the time, the audio receiver thread only writes samples to the buffer. In these cases, using atomics avoids lock contention. There are still corner cases where the audio receiver thread needs to "read" samples (and drop them), so lock only in these cases. PR #4572 --- app/meson.build | 9 +- app/src/audio_player.c | 202 ++++++++++++++++++-------------------- app/src/audio_player.h | 22 ++--- app/src/util/audiobuf.c | 112 +++++++++++++++++++++ app/src/util/audiobuf.h | 79 +++++---------- app/src/util/bytebuf.c | 104 -------------------- app/src/util/bytebuf.h | 114 --------------------- app/tests/test_audiobuf.c | 128 ++++++++++++++++++++++++ app/tests/test_bytebuf.c | 126 ------------------------ 9 files changed, 376 insertions(+), 520 deletions(-) create mode 100644 app/src/util/audiobuf.c delete mode 100644 app/src/util/bytebuf.c delete mode 100644 app/src/util/bytebuf.h create mode 100644 app/tests/test_audiobuf.c delete mode 100644 app/tests/test_bytebuf.c diff --git a/app/meson.build b/app/meson.build index 88e2df9a..caf5ee5c 100644 --- a/app/meson.build +++ b/app/meson.build @@ -34,8 +34,8 @@ src = [ 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/util/acksync.c', + 'src/util/audiobuf.c', 'src/util/average.c', - 'src/util/bytebuf.c', 'src/util/file.c', 'src/util/intmap.c', 'src/util/intr.c', @@ -212,9 +212,10 @@ if get_option('buildtype') == 'debug' ['test_binary', [ 'tests/test_binary.c', ]], - ['test_bytebuf', [ - 'tests/test_bytebuf.c', - 'src/util/bytebuf.c', + ['test_audiobuf', [ + 'tests/test_audiobuf.c', + 'src/util/audiobuf.c', + 'src/util/memory.c', ]], ['test_cli', [ 'tests/test_cli.c', diff --git a/app/src/audio_player.c b/app/src/audio_player.c index 8f0ad7fb..728d3f2a 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -66,8 +66,7 @@ static void SDLCALL sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) { struct sc_audio_player *ap = userdata; - // This callback is called with the lock used by SDL_AudioDeviceLock(), so - // the audiobuf is protected + // This callback is called with the lock used by SDL_LockAudioDevice() assert(len_int > 0); size_t len = len_int; @@ -77,8 +76,9 @@ sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) { LOGD("[Audio] SDL callback requests %" PRIu32 " samples", count); #endif - uint32_t buffered_samples = sc_audiobuf_can_read(&ap->buf); - if (!ap->played) { + bool played = atomic_load_explicit(&ap->played, memory_order_relaxed); + if (!played) { + uint32_t buffered_samples = sc_audiobuf_can_read(&ap->buf); // Part of the buffering is handled by inserting initial silence. The // remaining (margin) last samples will be handled by compensation. uint32_t margin = 30 * ap->sample_rate / 1000; // 30ms @@ -93,10 +93,7 @@ sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) { } } - uint32_t read = MIN(buffered_samples, count); - if (read) { - sc_audiobuf_read(&ap->buf, stream, read); - } + uint32_t read = sc_audiobuf_read(&ap->buf, stream, count); if (read < count) { uint32_t silence = count - read; @@ -109,13 +106,16 @@ sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) { silence); memset(stream + TO_BYTES(read), 0, TO_BYTES(silence)); - if (ap->received) { + bool received = atomic_load_explicit(&ap->received, + memory_order_relaxed); + if (received) { // Inserting additional samples immediately increases buffering - ap->underflow += silence; + atomic_fetch_add_explicit(&ap->underflow, silence, + memory_order_relaxed); } } - ap->played = true; + atomic_store_explicit(&ap->played, true, memory_order_relaxed); } static uint8_t * @@ -162,123 +162,119 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, // swr_convert() returns the number of samples which would have been // written if the buffer was big enough. - uint32_t samples_written = MIN(ret, dst_nb_samples); + uint32_t samples = MIN(ret, dst_nb_samples); #ifndef SC_AUDIO_PLAYER_NDEBUG - LOGD("[Audio] %" PRIu32 " samples written to buffer", samples_written); + LOGD("[Audio] %" PRIu32 " samples written to buffer", samples); #endif - // Since this function is the only writer, the current available space is - // at least the previous available space. In practice, it should almost - // always be possible to write without lock. - bool lockless_write = samples_written <= ap->previous_can_write; - if (lockless_write) { - sc_audiobuf_prepare_write(&ap->buf, swr_buf, samples_written); + uint32_t cap = sc_audiobuf_capacity(&ap->buf); + if (samples > cap) { + // Very very unlikely: a single resampled frame should never + // exceed the audio buffer size (or something is very wrong). + // Ignore the first bytes in swr_buf to avoid memory corruption anyway. + swr_buf += TO_BYTES(samples - cap); + samples = cap; } - SDL_LockAudioDevice(ap->device); - - uint32_t buffered_samples = sc_audiobuf_can_read(&ap->buf); - - if (lockless_write) { - sc_audiobuf_commit_write(&ap->buf, samples_written); - } else { - uint32_t can_write = sc_audiobuf_can_write(&ap->buf); - if (samples_written > can_write) { - // Entering this branch is very unlikely, the audio buffer is - // allocated with a size sufficient to store 1 second more than the - // target buffering. If this happens, though, we have to skip old - // samples. - uint32_t cap = sc_audiobuf_capacity(&ap->buf); - if (samples_written > cap) { - // Very very unlikely: a single resampled frame should never - // exceed the audio buffer size (or something is very wrong). - // Ignore the first bytes in swr_buf - swr_buf += TO_BYTES(samples_written - cap); - // This change in samples_written will impact the - // instant_compensation below - samples_written = cap; - } - - assert(samples_written >= can_write); - if (samples_written > can_write) { - uint32_t skip_samples = samples_written - can_write; - assert(buffered_samples >= skip_samples); - sc_audiobuf_skip(&ap->buf, skip_samples); - buffered_samples -= skip_samples; - if (ap->played) { - // Dropping input samples instantly decreases buffering - ap->avg_buffering.avg -= skip_samples; - } - } - - // It should remain exactly the expected size to write the new - // samples. - assert(sc_audiobuf_can_write(&ap->buf) == samples_written); + uint32_t skipped_samples = 0; + + uint32_t written = sc_audiobuf_write(&ap->buf, swr_buf, samples); + if (written < samples) { + uint32_t remaining = samples - written; + + // All samples that could be written without locking have been written, + // now we need to lock to drop/consume old samples + SDL_LockAudioDevice(ap->device); + + // Retry with the lock + written += sc_audiobuf_write(&ap->buf, + swr_buf + TO_BYTES(written), + remaining); + if (written < samples) { + remaining = samples - written; + // Still insufficient, drop old samples to make space + skipped_samples = sc_audiobuf_read(&ap->buf, NULL, remaining); + assert(skipped_samples == remaining); + + // Now there is enough space + uint32_t w = sc_audiobuf_write(&ap->buf, + swr_buf + TO_BYTES(written), + remaining); + assert(w == remaining); + (void) w; } - sc_audiobuf_write(&ap->buf, swr_buf, samples_written); + SDL_UnlockAudioDevice(ap->device); } - buffered_samples += samples_written; - assert(buffered_samples == sc_audiobuf_can_read(&ap->buf)); - - // Read with lock held, to be used after unlocking - bool played = ap->played; - uint32_t underflow = ap->underflow; - + uint32_t underflow = 0; + uint32_t max_buffered_samples; + bool played = atomic_load_explicit(&ap->played, memory_order_relaxed); if (played) { - uint32_t max_buffered_samples = ap->target_buffering - + 12 * ap->output_buffer - + ap->target_buffering / 10; - if (buffered_samples > max_buffered_samples) { - uint32_t skip_samples = buffered_samples - max_buffered_samples; - sc_audiobuf_skip(&ap->buf, skip_samples); - LOGD("[Audio] Buffering threshold exceeded, skipping %" PRIu32 - " samples", skip_samples); - } + underflow = atomic_exchange_explicit(&ap->underflow, 0, + memory_order_relaxed); - // reset (the current value was copied to a local variable) - ap->underflow = 0; + max_buffered_samples = ap->target_buffering + + 12 * ap->output_buffer + + ap->target_buffering / 10; } else { // SDL playback not started yet, do not accumulate more than // max_initial_buffering samples, this would cause unnecessary delay // (and glitches to compensate) on start. - uint32_t max_initial_buffering = ap->target_buffering - + 2 * ap->output_buffer; - if (buffered_samples > max_initial_buffering) { - uint32_t skip_samples = buffered_samples - max_initial_buffering; - sc_audiobuf_skip(&ap->buf, skip_samples); + max_buffered_samples = ap->target_buffering + 2 * ap->output_buffer; + } + + uint32_t can_read = sc_audiobuf_can_read(&ap->buf); + if (can_read > max_buffered_samples) { + uint32_t skip_samples = 0; + + SDL_LockAudioDevice(ap->device); + can_read = sc_audiobuf_can_read(&ap->buf); + if (can_read > max_buffered_samples) { + skip_samples = can_read - max_buffered_samples; + uint32_t r = sc_audiobuf_read(&ap->buf, NULL, skip_samples); + assert(r == skip_samples); + (void) r; + skipped_samples += skip_samples; + } + SDL_UnlockAudioDevice(ap->device); + + if (skip_samples) { + if (played) { + LOGD("[Audio] Buffering threshold exceeded, skipping %" PRIu32 + " samples", skip_samples); #ifndef SC_AUDIO_PLAYER_NDEBUG - LOGD("[Audio] Playback not started, skipping %" PRIu32 " samples", - skip_samples); + } else { + LOGD("[Audio] Playback not started, skipping %" PRIu32 + " samples", skip_samples); #endif + } } } - ap->previous_can_write = sc_audiobuf_can_write(&ap->buf); - ap->received = true; - - SDL_UnlockAudioDevice(ap->device); + atomic_store_explicit(&ap->received, true, memory_order_relaxed); if (played) { // Number of samples added (or removed, if negative) for compensation - int32_t instant_compensation = - (int32_t) samples_written - frame->nb_samples; + int32_t instant_compensation = (int32_t) written - frame->nb_samples; + // Inserting silence instantly increases buffering int32_t inserted_silence = (int32_t) underflow; + // Dropping input samples instantly decreases buffering + int32_t dropped = (int32_t) skipped_samples; // The compensation must apply instantly, it must not be smoothed - ap->avg_buffering.avg += instant_compensation + inserted_silence; - + ap->avg_buffering.avg += + instant_compensation + inserted_silence - dropped; // However, the buffering level must be smoothed - sc_average_push(&ap->avg_buffering, buffered_samples); + sc_average_push(&ap->avg_buffering, can_read); #ifndef SC_AUDIO_PLAYER_NDEBUG - LOGD("[Audio] buffered_samples=%" PRIu32 " avg_buffering=%f", - buffered_samples, sc_average_get(&ap->avg_buffering)); + LOGD("[Audio] can_read=%" PRIu32 " avg_buffering=%f", + can_read, sc_average_get(&ap->avg_buffering)); #endif - ap->samples_since_resync += samples_written; + ap->samples_since_resync += written; if (ap->samples_since_resync >= ap->sample_rate) { // Recompute compensation every second ap->samples_since_resync = 0; @@ -288,7 +284,7 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, if (abs(diff) < (int) ap->sample_rate / 1000) { // Do not compensate for less than 1ms, the error is just noise diff = 0; - } else if (diff < 0 && buffered_samples < ap->target_buffering) { + } else if (diff < 0 && can_read < ap->target_buffering) { // Do not accelerate if the instant buffering level is below // the average, this would increase underflow diff = 0; @@ -300,8 +296,7 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, int abs_max_diff = distance / 50; diff = CLAMP(diff, -abs_max_diff, abs_max_diff); LOGV("[Audio] Buffering: target=%" PRIu32 " avg=%f cur=%" PRIu32 - " compensation=%d", ap->target_buffering, avg, - buffered_samples, diff); + " compensation=%d", ap->target_buffering, avg, can_read, diff); if (diff != ap->compensation) { int ret = swr_set_compensation(swr_ctx, diff, distance); @@ -397,7 +392,7 @@ sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, // producer and the consumer. It's too big on purpose, to guarantee that // the producer and the consumer will be able to access it in parallel // without locking. - size_t audiobuf_samples = ap->target_buffering + ap->sample_rate; + uint32_t audiobuf_samples = ap->target_buffering + ap->sample_rate; size_t sample_size = ap->nb_channels * ap->out_bytes_per_sample; bool ok = sc_audiobuf_init(&ap->buf, sample_size, audiobuf_samples); @@ -413,16 +408,15 @@ sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, } ap->swr_buf_alloc_size = initial_swr_buf_size; - ap->previous_can_write = sc_audiobuf_can_write(&ap->buf); - // Samples are produced and consumed by blocks, so the buffering must be // smoothed to get a relatively stable value. sc_average_init(&ap->avg_buffering, 32); ap->samples_since_resync = 0; ap->received = false; - ap->played = false; - ap->underflow = 0; + atomic_init(&ap->played, false); + atomic_init(&ap->received, false); + atomic_init(&ap->underflow, 0); ap->compensation = 0; // The thread calling open() is the thread calling push(), which fills the diff --git a/app/src/audio_player.h b/app/src/audio_player.h index 30378246..0c677363 100644 --- a/app/src/audio_player.h +++ b/app/src/audio_player.h @@ -3,6 +3,7 @@ #include "common.h" +#include #include #include #include @@ -32,13 +33,9 @@ struct sc_audio_player { uint16_t output_buffer; // Audio buffer to communicate between the receiver and the SDL audio - // callback (protected by SDL_AudioDeviceLock()) + // callback struct sc_audiobuf buf; - // The previous empty space in the buffer (only used by the receiver - // thread) - uint32_t previous_can_write; - // Resampler (only used from the receiver thread) struct SwrContext *swr_ctx; @@ -47,7 +44,7 @@ struct sc_audio_player { // The number of channels is the same for input and output unsigned nb_channels; // The number of bytes per sample for a single channel - unsigned out_bytes_per_sample; + size_t out_bytes_per_sample; // Target buffer for resampling (only used by the receiver thread) uint8_t *swr_buf; @@ -61,19 +58,16 @@ struct sc_audio_player { uint32_t samples_since_resync; // Number of silence samples inserted since the last received packet - // (protected by SDL_AudioDeviceLock()) - uint32_t underflow; + atomic_uint_least32_t underflow; // Current applied compensation value (only used by the receiver thread) int compensation; - // Set to true the first time a sample is received (protected by - // SDL_AudioDeviceLock()) - bool received; + // Set to true the first time a sample is received + atomic_bool received; - // Set to true the first time the SDL callback is called (protected by - // SDL_AudioDeviceLock()) - bool played; + // Set to true the first time the SDL callback is called + atomic_bool played; const struct sc_audio_player_callbacks *cbs; void *cbs_userdata; diff --git a/app/src/util/audiobuf.c b/app/src/util/audiobuf.c new file mode 100644 index 00000000..3597f7ee --- /dev/null +++ b/app/src/util/audiobuf.c @@ -0,0 +1,112 @@ +#include "audiobuf.h" + +#include +#include +#include +#include + +bool +sc_audiobuf_init(struct sc_audiobuf *buf, size_t sample_size, + uint32_t capacity) { + assert(sample_size); + assert(capacity); + + // The actual capacity is (alloc_size - 1) so that head == tail is + // non-ambiguous + buf->alloc_size = capacity + 1; + buf->data = sc_allocarray(buf->alloc_size, sample_size); + if (!buf->data) { + LOG_OOM(); + return false; + } + + buf->sample_size = sample_size; + atomic_init(&buf->head, 0); + atomic_init(&buf->tail, 0); + + return true; +} + +void +sc_audiobuf_destroy(struct sc_audiobuf *buf) { + free(buf->data); +} + +uint32_t +sc_audiobuf_read(struct sc_audiobuf *buf, void *to_, uint32_t samples_count) { + assert(samples_count); + + uint8_t *to = to_; + + // Only the reader thread can write tail without synchronization, so + // memory_order_relaxed is sufficient + uint32_t tail = atomic_load_explicit(&buf->tail, memory_order_relaxed); + + // The head cursor is updated after the data is written to the array + uint32_t head = atomic_load_explicit(&buf->head, memory_order_acquire); + + uint32_t can_read = (buf->alloc_size + head - tail) % buf->alloc_size; + if (samples_count > can_read) { + samples_count = can_read; + } + + if (to) { + uint32_t right_count = buf->alloc_size - tail; + if (right_count > samples_count) { + right_count = samples_count; + } + memcpy(to, + buf->data + (tail * buf->sample_size), + right_count * buf->sample_size); + + if (samples_count > right_count) { + uint32_t left_count = samples_count - right_count; + memcpy(to + (right_count * buf->sample_size), + buf->data, + left_count * buf->sample_size); + } + } + + uint32_t new_tail = (tail + samples_count) % buf->alloc_size; + atomic_store_explicit(&buf->tail, new_tail, memory_order_release); + + return samples_count; +} + +uint32_t +sc_audiobuf_write(struct sc_audiobuf *buf, const void *from_, + uint32_t samples_count) { + const uint8_t *from = from_; + + // Only the writer thread can write head, so memory_order_relaxed is + // sufficient + uint32_t head = atomic_load_explicit(&buf->head, memory_order_relaxed); + + // The tail cursor is updated after the data is consumed by the reader + uint32_t tail = atomic_load_explicit(&buf->tail, memory_order_acquire); + + uint32_t can_write = (buf->alloc_size + tail - head - 1) % buf->alloc_size; + if (samples_count > can_write) { + samples_count = can_write; + } + + uint32_t right_count = buf->alloc_size - head; + if (right_count > samples_count) { + right_count = samples_count; + } + memcpy(buf->data + (head * buf->sample_size), + from, + right_count * buf->sample_size); + + if (samples_count > right_count) { + uint32_t left_count = samples_count - right_count; + memcpy(buf->data, + from + (right_count * buf->sample_size), + left_count * buf->sample_size); + } + + uint32_t new_head = (head + samples_count) % buf->alloc_size; + atomic_store_explicit(&buf->head, new_head, memory_order_release); + + return samples_count; +} diff --git a/app/src/util/audiobuf.h b/app/src/util/audiobuf.h index 8616d539..5e7dd4a0 100644 --- a/app/src/util/audiobuf.h +++ b/app/src/util/audiobuf.h @@ -3,19 +3,25 @@ #include "common.h" +#include +#include #include #include -#include "util/bytebuf.h" - /** * Wrapper around bytebuf to read and write samples * * Each sample takes sample_size bytes. */ struct sc_audiobuf { - struct sc_bytebuf buf; + uint8_t *data; + uint32_t alloc_size; // in samples size_t sample_size; + + atomic_uint_least32_t head; // writer cursor, in samples + atomic_uint_least32_t tail; // reader cursor, in samples + // empty: tail == head + // full: ((tail + 1) % alloc_size) == head }; static inline uint32_t @@ -29,66 +35,31 @@ sc_audiobuf_to_bytes(struct sc_audiobuf *buf, uint32_t samples) { return samples * buf->sample_size; } -static inline bool +bool sc_audiobuf_init(struct sc_audiobuf *buf, size_t sample_size, - uint32_t capacity) { - buf->sample_size = sample_size; - return sc_bytebuf_init(&buf->buf, capacity * sample_size + 1); -} - -static inline void -sc_audiobuf_read(struct sc_audiobuf *buf, uint8_t *to, uint32_t samples) { - size_t bytes = sc_audiobuf_to_bytes(buf, samples); - sc_bytebuf_read(&buf->buf, to, bytes); -} + uint32_t capacity); -static inline void -sc_audiobuf_skip(struct sc_audiobuf *buf, uint32_t samples) { - size_t bytes = sc_audiobuf_to_bytes(buf, samples); - sc_bytebuf_skip(&buf->buf, bytes); -} +void +sc_audiobuf_destroy(struct sc_audiobuf *buf); -static inline void -sc_audiobuf_write(struct sc_audiobuf *buf, const uint8_t *from, - uint32_t samples) { - size_t bytes = sc_audiobuf_to_bytes(buf, samples); - sc_bytebuf_write(&buf->buf, from, bytes); -} +uint32_t +sc_audiobuf_read(struct sc_audiobuf *buf, void *to, uint32_t samples_count); -static inline void -sc_audiobuf_prepare_write(struct sc_audiobuf *buf, const uint8_t *from, - uint32_t samples) { - size_t bytes = sc_audiobuf_to_bytes(buf, samples); - sc_bytebuf_prepare_write(&buf->buf, from, bytes); -} - -static inline void -sc_audiobuf_commit_write(struct sc_audiobuf *buf, uint32_t samples) { - size_t bytes = sc_audiobuf_to_bytes(buf, samples); - sc_bytebuf_commit_write(&buf->buf, bytes); -} - -static inline uint32_t -sc_audiobuf_can_read(struct sc_audiobuf *buf) { - size_t bytes = sc_bytebuf_can_read(&buf->buf); - return sc_audiobuf_to_samples(buf, bytes); -} - -static inline uint32_t -sc_audiobuf_can_write(struct sc_audiobuf *buf) { - size_t bytes = sc_bytebuf_can_write(&buf->buf); - return sc_audiobuf_to_samples(buf, bytes); -} +uint32_t +sc_audiobuf_write(struct sc_audiobuf *buf, const void *from, + uint32_t samples_count); static inline uint32_t sc_audiobuf_capacity(struct sc_audiobuf *buf) { - size_t bytes = sc_bytebuf_capacity(&buf->buf); - return sc_audiobuf_to_samples(buf, bytes); + assert(buf->alloc_size); + return buf->alloc_size - 1; } -static inline void -sc_audiobuf_destroy(struct sc_audiobuf *buf) { - sc_bytebuf_destroy(&buf->buf); +static inline uint32_t +sc_audiobuf_can_read(struct sc_audiobuf *buf) { + uint32_t head = atomic_load_explicit(&buf->head, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&buf->tail, memory_order_acquire); + return (buf->alloc_size + head - tail) % buf->alloc_size; } #endif diff --git a/app/src/util/bytebuf.c b/app/src/util/bytebuf.c deleted file mode 100644 index 93544d72..00000000 --- a/app/src/util/bytebuf.c +++ /dev/null @@ -1,104 +0,0 @@ -#include "bytebuf.h" - -#include -#include -#include - -#include "util/log.h" - -bool -sc_bytebuf_init(struct sc_bytebuf *buf, size_t alloc_size) { - assert(alloc_size); - buf->data = malloc(alloc_size); - if (!buf->data) { - LOG_OOM(); - return false; - } - - buf->alloc_size = alloc_size; - buf->head = 0; - buf->tail = 0; - - return true; -} - -void -sc_bytebuf_destroy(struct sc_bytebuf *buf) { - free(buf->data); -} - -void -sc_bytebuf_read(struct sc_bytebuf *buf, uint8_t *to, size_t len) { - assert(len); - assert(len <= sc_bytebuf_can_read(buf)); - assert(buf->tail != buf->head); // the buffer could not be empty - - size_t right_limit = buf->tail < buf->head ? buf->head : buf->alloc_size; - size_t right_len = right_limit - buf->tail; - if (len < right_len) { - right_len = len; - } - memcpy(to, buf->data + buf->tail, right_len); - - if (len > right_len) { - memcpy(to + right_len, buf->data, len - right_len); - } - - buf->tail = (buf->tail + len) % buf->alloc_size; -} - -void -sc_bytebuf_skip(struct sc_bytebuf *buf, size_t len) { - assert(len); - assert(len <= sc_bytebuf_can_read(buf)); - assert(buf->tail != buf->head); // the buffer could not be empty - - buf->tail = (buf->tail + len) % buf->alloc_size; -} - -static inline void -sc_bytebuf_write_step0(struct sc_bytebuf *buf, const uint8_t *from, - size_t len) { - size_t right_len = buf->alloc_size - buf->head; - if (len < right_len) { - right_len = len; - } - memcpy(buf->data + buf->head, from, right_len); - - if (len > right_len) { - memcpy(buf->data, from + right_len, len - right_len); - } -} - -static inline void -sc_bytebuf_write_step1(struct sc_bytebuf *buf, size_t len) { - buf->head = (buf->head + len) % buf->alloc_size; -} - -void -sc_bytebuf_write(struct sc_bytebuf *buf, const uint8_t *from, size_t len) { - assert(len); - assert(len <= sc_bytebuf_can_write(buf)); - - sc_bytebuf_write_step0(buf, from, len); - sc_bytebuf_write_step1(buf, len); -} - -void -sc_bytebuf_prepare_write(struct sc_bytebuf *buf, const uint8_t *from, - size_t len) { - // *This function MUST NOT access buf->tail (even in assert()).* - // The purpose of this function is to allow a reader and a writer to access - // different parts of the buffer in parallel simultaneously. It is intended - // to be called without lock (only sc_bytebuf_commit_write() is intended to - // be called with lock held). - - assert(len < buf->alloc_size - 1); - sc_bytebuf_write_step0(buf, from, len); -} - -void -sc_bytebuf_commit_write(struct sc_bytebuf *buf, size_t len) { - assert(len <= sc_bytebuf_can_write(buf)); - sc_bytebuf_write_step1(buf, len); -} diff --git a/app/src/util/bytebuf.h b/app/src/util/bytebuf.h deleted file mode 100644 index 1448f752..00000000 --- a/app/src/util/bytebuf.h +++ /dev/null @@ -1,114 +0,0 @@ -#ifndef SC_BYTEBUF_H -#define SC_BYTEBUF_H - -#include "common.h" - -#include -#include - -struct sc_bytebuf { - uint8_t *data; - // The actual capacity is (allocated - 1) so that head == tail is - // non-ambiguous - size_t alloc_size; - size_t head; // writter cursor - size_t tail; // reader cursor - // empty: tail == head - // full: ((tail + 1) % alloc_size) == head -}; - -bool -sc_bytebuf_init(struct sc_bytebuf *buf, size_t alloc_size); - -/** - * Copy from the bytebuf to a user-provided array - * - * The caller must check that len <= sc_bytebuf_read_available() (it is an - * error to attempt to read more bytes than available). - * - * This function is guaranteed not to write to buf->head. - */ -void -sc_bytebuf_read(struct sc_bytebuf *buf, uint8_t *to, size_t len); - -/** - * Drop len bytes from the buffer - * - * The caller must check that len <= sc_bytebuf_read_available() (it is an - * error to attempt to skip more bytes than available). - * - * This function is guaranteed not to write to buf->head. - * - * It is equivalent to call sc_bytebuf_read() to some array and discard the - * array (but this function is more efficient since there is no copy). - */ -void -sc_bytebuf_skip(struct sc_bytebuf *buf, size_t len); - -/** - * Copy the user-provided array to the bytebuf - * - * The caller must check that len <= sc_bytebuf_write_available() (it is an - * error to write more bytes than the remaining available space). - * - * This function is guaranteed not to write to buf->tail. - */ -void -sc_bytebuf_write(struct sc_bytebuf *buf, const uint8_t *from, size_t len); - -/** - * Copy the user-provided array to the bytebuf, but do not advance the cursor - * - * The caller must check that len <= sc_bytebuf_write_available() (it is an - * error to write more bytes than the remaining available space). - * - * After this function is called, the write must be committed with - * sc_bytebuf_commit_write(). - * - * The purpose of this mechanism is to acquire a lock only to commit the write, - * but not to perform the actual copy. - * - * This function is guaranteed not to access buf->tail. - */ -void -sc_bytebuf_prepare_write(struct sc_bytebuf *buf, const uint8_t *from, - size_t len); - -/** - * Commit a prepared write - */ -void -sc_bytebuf_commit_write(struct sc_bytebuf *buf, size_t len); - -/** - * Return the number of bytes which can be read - * - * It is an error to read more bytes than available. - */ -static inline size_t -sc_bytebuf_can_read(struct sc_bytebuf *buf) { - return (buf->alloc_size + buf->head - buf->tail) % buf->alloc_size; -} - -/** - * Return the number of bytes which can be written - * - * It is an error to write more bytes than available. - */ -static inline size_t -sc_bytebuf_can_write(struct sc_bytebuf *buf) { - return (buf->alloc_size + buf->tail - buf->head - 1) % buf->alloc_size; -} - -/** - * Return the actual capacity of the buffer (can_read() + can_write()) - */ -static inline size_t -sc_bytebuf_capacity(struct sc_bytebuf *buf) { - return buf->alloc_size - 1; -} - -void -sc_bytebuf_destroy(struct sc_bytebuf *buf); - -#endif diff --git a/app/tests/test_audiobuf.c b/app/tests/test_audiobuf.c new file mode 100644 index 00000000..94d0f07a --- /dev/null +++ b/app/tests/test_audiobuf.c @@ -0,0 +1,128 @@ +#include "common.h" + +#include +#include + +#include "util/audiobuf.h" + +static void test_audiobuf_simple(void) { + struct sc_audiobuf buf; + uint32_t data[20]; + + bool ok = sc_audiobuf_init(&buf, 4, 20); + assert(ok); + + uint32_t samples[] = {1, 2, 3, 4, 5}; + uint32_t w = sc_audiobuf_write(&buf, samples, 5); + assert(w == 5); + + uint32_t r = sc_audiobuf_read(&buf, data, 4); + assert(r == 4); + assert(!memcmp(data, samples, 16)); + + uint32_t samples2[] = {6, 7, 8}; + w = sc_audiobuf_write(&buf, samples2, 3); + assert(w == 3); + + uint32_t single = 9; + w = sc_audiobuf_write(&buf, &single, 1); + assert(w == 1); + + r = sc_audiobuf_read(&buf, &data[4], 8); + assert(r == 5); + + uint32_t expected[] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + assert(!memcmp(data, expected, 36)); + + sc_audiobuf_destroy(&buf); +} + +static void test_audiobuf_boundaries(void) { + struct sc_audiobuf buf; + uint32_t data[20]; + + bool ok = sc_audiobuf_init(&buf, 4, 20); + assert(ok); + + uint32_t samples[] = {1, 2, 3, 4, 5, 6}; + uint32_t w = sc_audiobuf_write(&buf, samples, 6); + assert(w == 6); + + w = sc_audiobuf_write(&buf, samples, 6); + assert(w == 6); + + w = sc_audiobuf_write(&buf, samples, 6); + assert(w == 6); + + uint32_t r = sc_audiobuf_read(&buf, data, 9); + assert(r == 9); + + uint32_t expected[] = {1, 2, 3, 4, 5, 6, 1, 2, 3}; + assert(!memcmp(data, expected, 36)); + + uint32_t samples2[] = {7, 8, 9, 10, 11}; + w = sc_audiobuf_write(&buf, samples2, 5); + assert(w == 5); + + uint32_t single = 12; + w = sc_audiobuf_write(&buf, &single, 1); + assert(w == 1); + + w = sc_audiobuf_read(&buf, NULL, 3); + assert(w == 3); + + assert(sc_audiobuf_can_read(&buf) == 12); + + r = sc_audiobuf_read(&buf, data, 12); + assert(r == 12); + + uint32_t expected2[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; + assert(!memcmp(data, expected2, 48)); + + sc_audiobuf_destroy(&buf); +} + +static void test_audiobuf_partial_read_write(void) { + struct sc_audiobuf buf; + uint32_t data[15]; + + bool ok = sc_audiobuf_init(&buf, 4, 10); + assert(ok); + + uint32_t samples[] = {1, 2, 3, 4, 5, 6}; + uint32_t w = sc_audiobuf_write(&buf, samples, 6); + assert(w == 6); + + w = sc_audiobuf_write(&buf, samples, 6); + assert(w == 4); + + w = sc_audiobuf_write(&buf, samples, 6); + assert(w == 0); + + uint32_t r = sc_audiobuf_read(&buf, data, 3); + assert(r == 3); + + uint32_t expected[] = {1, 2, 3}; + assert(!memcmp(data, expected, 12)); + + w = sc_audiobuf_write(&buf, samples, 6); + assert(w == 3); + + r = sc_audiobuf_read(&buf, data, 15); + assert(r == 10); + uint32_t expected2[] = {4, 5, 6, 1, 2, 3, 4, 1, 2, 3}; + assert(!memcmp(data, expected2, 12)); + + sc_audiobuf_destroy(&buf); +} + +int main(int argc, char *argv[]) { + (void) argc; + (void) argv; + + test_audiobuf_simple(); + test_audiobuf_boundaries(); + test_audiobuf_partial_read_write(); + + return 0; +} diff --git a/app/tests/test_bytebuf.c b/app/tests/test_bytebuf.c deleted file mode 100644 index 8e9d7c57..00000000 --- a/app/tests/test_bytebuf.c +++ /dev/null @@ -1,126 +0,0 @@ -#include "common.h" - -#include -#include - -#include "util/bytebuf.h" - -static void test_bytebuf_simple(void) { - struct sc_bytebuf buf; - uint8_t data[20]; - - bool ok = sc_bytebuf_init(&buf, 20); - assert(ok); - - sc_bytebuf_write(&buf, (uint8_t *) "hello", sizeof("hello") - 1); - assert(sc_bytebuf_can_read(&buf) == 5); - - sc_bytebuf_read(&buf, data, 4); - assert(!strncmp((char *) data, "hell", 4)); - - sc_bytebuf_write(&buf, (uint8_t *) " world", sizeof(" world") - 1); - assert(sc_bytebuf_can_read(&buf) == 7); - - sc_bytebuf_write(&buf, (uint8_t *) "!", 1); - assert(sc_bytebuf_can_read(&buf) == 8); - - sc_bytebuf_read(&buf, &data[4], 8); - assert(sc_bytebuf_can_read(&buf) == 0); - - data[12] = '\0'; - assert(!strcmp((char *) data, "hello world!")); - assert(sc_bytebuf_can_read(&buf) == 0); - - sc_bytebuf_destroy(&buf); -} - -static void test_bytebuf_boundaries(void) { - struct sc_bytebuf buf; - uint8_t data[20]; - - bool ok = sc_bytebuf_init(&buf, 20); - assert(ok); - - sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); - assert(sc_bytebuf_can_read(&buf) == 6); - - sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); - assert(sc_bytebuf_can_read(&buf) == 12); - - sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); - assert(sc_bytebuf_can_read(&buf) == 18); - - sc_bytebuf_read(&buf, data, 9); - assert(!strncmp((char *) data, "hello hel", 9)); - assert(sc_bytebuf_can_read(&buf) == 9); - - sc_bytebuf_write(&buf, (uint8_t *) "world", sizeof("world") - 1); - assert(sc_bytebuf_can_read(&buf) == 14); - - sc_bytebuf_write(&buf, (uint8_t *) "!", 1); - assert(sc_bytebuf_can_read(&buf) == 15); - - sc_bytebuf_skip(&buf, 3); - assert(sc_bytebuf_can_read(&buf) == 12); - - sc_bytebuf_read(&buf, data, 12); - data[12] = '\0'; - assert(!strcmp((char *) data, "hello world!")); - assert(sc_bytebuf_can_read(&buf) == 0); - - sc_bytebuf_destroy(&buf); -} - -static void test_bytebuf_two_steps_write(void) { - struct sc_bytebuf buf; - uint8_t data[20]; - - bool ok = sc_bytebuf_init(&buf, 20); - assert(ok); - - sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); - assert(sc_bytebuf_can_read(&buf) == 6); - - sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); - assert(sc_bytebuf_can_read(&buf) == 12); - - sc_bytebuf_prepare_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); - assert(sc_bytebuf_can_read(&buf) == 12); // write not committed yet - - sc_bytebuf_read(&buf, data, 9); - assert(!strncmp((char *) data, "hello hel", 3)); - assert(sc_bytebuf_can_read(&buf) == 3); - - sc_bytebuf_commit_write(&buf, sizeof("hello ") - 1); - assert(sc_bytebuf_can_read(&buf) == 9); - - sc_bytebuf_prepare_write(&buf, (uint8_t *) "world", sizeof("world") - 1); - assert(sc_bytebuf_can_read(&buf) == 9); // write not committed yet - - sc_bytebuf_commit_write(&buf, sizeof("world") - 1); - assert(sc_bytebuf_can_read(&buf) == 14); - - sc_bytebuf_write(&buf, (uint8_t *) "!", 1); - assert(sc_bytebuf_can_read(&buf) == 15); - - sc_bytebuf_skip(&buf, 3); - assert(sc_bytebuf_can_read(&buf) == 12); - - sc_bytebuf_read(&buf, data, 12); - data[12] = '\0'; - assert(!strcmp((char *) data, "hello world!")); - assert(sc_bytebuf_can_read(&buf) == 0); - - sc_bytebuf_destroy(&buf); -} - -int main(int argc, char *argv[]) { - (void) argc; - (void) argv; - - test_bytebuf_simple(); - test_bytebuf_boundaries(); - test_bytebuf_two_steps_write(); - - return 0; -} From 44abed5c68d657c664457eb562318168876d2208 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 7 Jan 2024 18:23:24 +0100 Subject: [PATCH 26/79] Improve audio compensation thresholds Use different thresholds for enabling and disabling compensation. Concretely, enable compensation if the difference between the average and the target buffering levels exceeds 4 ms (instead of 1 ms). This avoids unnecessary compensation due to small noise in buffering level estimation. But keep a smaller threshold (1 ms) for disabling compensation, so that the buffering level is restored closer to the target value. This avoids to keep the actual level close to the compensation threshold. PR #4572 --- app/src/audio_player.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/src/audio_player.c b/app/src/audio_player.c index 728d3f2a..c70964b9 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -281,8 +281,15 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, float avg = sc_average_get(&ap->avg_buffering); int diff = ap->target_buffering - avg; - if (abs(diff) < (int) ap->sample_rate / 1000) { - // Do not compensate for less than 1ms, the error is just noise + + // Enable compensation when the difference exceeds +/- 4ms. + // Disable compensation when the difference is lower than +/- 1ms. + int threshold = ap->compensation != 0 + ? ap->sample_rate / 1000 /* 1ms */ + : ap->sample_rate * 4 / 1000; /* 4ms */ + + if (abs(diff) < threshold) { + // Do not compensate for small values, the error is just noise diff = 0; } else if (diff < 0 && can_read < ap->target_buffering) { // Do not accelerate if the instant buffering level is below From edac4b8a9a4a261a82400828b17d7b7ae0b33cd0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 7 Jan 2024 18:27:56 +0100 Subject: [PATCH 27/79] Increase buffering level smoothness The buffering level does not change continuously: it increases abruptly when a packet is received, and decreases abruptly when an audio block is consumed. To estimate the buffering level, a rolling average is used. To make the buffering more stable, increase the smoothness of this rolling average. This decreases the risk of enabling audio compensation due to an estimation error. PR #4572 --- app/src/audio_player.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/audio_player.c b/app/src/audio_player.c index c70964b9..4552b0f7 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -417,7 +417,7 @@ sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, // Samples are produced and consumed by blocks, so the buffering must be // smoothed to get a relatively stable value. - sc_average_init(&ap->avg_buffering, 32); + sc_average_init(&ap->avg_buffering, 128); ap->samples_since_resync = 0; ap->received = false; From dfa3f97a87c0b42289920777731b4da95369d7ed Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 24 Jan 2024 14:46:16 +0100 Subject: [PATCH 28/79] Fix audio player comment PR #4572 --- app/src/audio_player.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/audio_player.c b/app/src/audio_player.c index 4552b0f7..4d101b01 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -293,7 +293,7 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, diff = 0; } else if (diff < 0 && can_read < ap->target_buffering) { // Do not accelerate if the instant buffering level is below - // the average, this would increase underflow + // the target, this would increase underflow diff = 0; } // Compensate the diff over 4 seconds (but will be recomputed after From 4502126e3b2ab2d5a82f636e32524929b3b0d07e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 2 Feb 2024 15:02:09 +0100 Subject: [PATCH 29/79] Use early return to avoid additional indentation PR #4572 --- app/src/audio_player.c | 107 +++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/app/src/audio_player.c b/app/src/audio_player.c index 4d101b01..e978cd9f 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -253,66 +253,67 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, } atomic_store_explicit(&ap->received, true, memory_order_relaxed); + if (!played) { + // Nothing more to do + return true; + } - if (played) { - // Number of samples added (or removed, if negative) for compensation - int32_t instant_compensation = (int32_t) written - frame->nb_samples; - // Inserting silence instantly increases buffering - int32_t inserted_silence = (int32_t) underflow; - // Dropping input samples instantly decreases buffering - int32_t dropped = (int32_t) skipped_samples; + // Number of samples added (or removed, if negative) for compensation + int32_t instant_compensation = (int32_t) written - frame->nb_samples; + // Inserting silence instantly increases buffering + int32_t inserted_silence = (int32_t) underflow; + // Dropping input samples instantly decreases buffering + int32_t dropped = (int32_t) skipped_samples; - // The compensation must apply instantly, it must not be smoothed - ap->avg_buffering.avg += - instant_compensation + inserted_silence - dropped; + // The compensation must apply instantly, it must not be smoothed + ap->avg_buffering.avg += instant_compensation + inserted_silence - dropped; - // However, the buffering level must be smoothed - sc_average_push(&ap->avg_buffering, can_read); + // However, the buffering level must be smoothed + sc_average_push(&ap->avg_buffering, can_read); #ifndef SC_AUDIO_PLAYER_NDEBUG - LOGD("[Audio] can_read=%" PRIu32 " avg_buffering=%f", - can_read, sc_average_get(&ap->avg_buffering)); + LOGD("[Audio] can_read=%" PRIu32 " avg_buffering=%f", + can_read, sc_average_get(&ap->avg_buffering)); #endif - ap->samples_since_resync += written; - if (ap->samples_since_resync >= ap->sample_rate) { - // Recompute compensation every second - ap->samples_since_resync = 0; - - float avg = sc_average_get(&ap->avg_buffering); - int diff = ap->target_buffering - avg; - - // Enable compensation when the difference exceeds +/- 4ms. - // Disable compensation when the difference is lower than +/- 1ms. - int threshold = ap->compensation != 0 - ? ap->sample_rate / 1000 /* 1ms */ - : ap->sample_rate * 4 / 1000; /* 4ms */ - - if (abs(diff) < threshold) { - // Do not compensate for small values, the error is just noise - diff = 0; - } else if (diff < 0 && can_read < ap->target_buffering) { - // Do not accelerate if the instant buffering level is below - // the target, this would increase underflow - diff = 0; - } - // Compensate the diff over 4 seconds (but will be recomputed after - // 1 second) - int distance = 4 * ap->sample_rate; - // Limit compensation rate to 2% - int abs_max_diff = distance / 50; - diff = CLAMP(diff, -abs_max_diff, abs_max_diff); - LOGV("[Audio] Buffering: target=%" PRIu32 " avg=%f cur=%" PRIu32 - " compensation=%d", ap->target_buffering, avg, can_read, diff); - - if (diff != ap->compensation) { - int ret = swr_set_compensation(swr_ctx, diff, distance); - if (ret < 0) { - LOGW("Resampling compensation failed: %d", ret); - // not fatal - } else { - ap->compensation = diff; - } + ap->samples_since_resync += written; + if (ap->samples_since_resync >= ap->sample_rate) { + // Recompute compensation every second + ap->samples_since_resync = 0; + + float avg = sc_average_get(&ap->avg_buffering); + int diff = ap->target_buffering - avg; + + // Enable compensation when the difference exceeds +/- 4ms. + // Disable compensation when the difference is lower than +/- 1ms. + int threshold = ap->compensation != 0 + ? ap->sample_rate / 1000 /* 1ms */ + : ap->sample_rate * 4 / 1000; /* 4ms */ + + if (abs(diff) < threshold) { + // Do not compensate for small values, the error is just noise + diff = 0; + } else if (diff < 0 && can_read < ap->target_buffering) { + // Do not accelerate if the instant buffering level is below the + // target, this would increase underflow + diff = 0; + } + // Compensate the diff over 4 seconds (but will be recomputed after 1 + // second) + int distance = 4 * ap->sample_rate; + // Limit compensation rate to 2% + int abs_max_diff = distance / 50; + diff = CLAMP(diff, -abs_max_diff, abs_max_diff); + LOGV("[Audio] Buffering: target=%" PRIu32 " avg=%f cur=%" PRIu32 + " compensation=%d", ap->target_buffering, avg, can_read, diff); + + if (diff != ap->compensation) { + int ret = swr_set_compensation(swr_ctx, diff, distance); + if (ret < 0) { + LOGW("Resampling compensation failed: %d", ret); + // not fatal + } else { + ap->compensation = diff; } } } From c12fdf900fb82e8ce958881e24a9f4f6185c0db7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 7 Feb 2024 17:50:14 +0100 Subject: [PATCH 30/79] Minimize buffer underflow on starting If playback starts too early, insert silence until the buffer is filled up to at least target_buffering before playing. PR #4572 --- app/src/audio_player.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/src/audio_player.c b/app/src/audio_player.c index e978cd9f..ea44e8d9 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -79,10 +79,9 @@ sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) { bool played = atomic_load_explicit(&ap->played, memory_order_relaxed); if (!played) { uint32_t buffered_samples = sc_audiobuf_can_read(&ap->buf); - // Part of the buffering is handled by inserting initial silence. The - // remaining (margin) last samples will be handled by compensation. - uint32_t margin = 30 * ap->sample_rate / 1000; // 30ms - if (buffered_samples + margin < ap->target_buffering) { + // Wait until the buffer is filled up to at least target_buffering + // before playing + if (buffered_samples < ap->target_buffering) { LOGV("[Audio] Inserting initial buffering silence: %" PRIu32 " samples", count); // Delay playback starting to reach the target buffering. Fill the From a7cf4daf3bcb573add846c2f4c98b94944c05cb9 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 8 Feb 2024 12:31:03 +0100 Subject: [PATCH 31/79] Avoid negative average buffering The assumption that underflow and overbuffering are caused by jitter (and that the delay between the producer and consumer will be caught up) does not always hold. For example, if the consumer does not consume at the expected rate (the SDL callback is not called often enough, which is an audio output issue), many samples will be dropped due to overbuffering, decreasing the average buffering indefinitely. Prevent the average buffering to become negative to limit the consequences of an unexpected behavior. PR #4572 --- app/src/audio_player.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/src/audio_player.c b/app/src/audio_player.c index ea44e8d9..bd799c51 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -266,6 +266,16 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, // The compensation must apply instantly, it must not be smoothed ap->avg_buffering.avg += instant_compensation + inserted_silence - dropped; + if (ap->avg_buffering.avg < 0) { + // Since dropping samples instantly reduces buffering, the difference + // is applied immediately to the average value, assuming that the delay + // between the producer and the consumer will be caught up. + // + // However, when this assumption is not valid, the average buffering + // may decrease indefinitely. Prevent it to become negative to limit + // the consequences. + ap->avg_buffering.avg = 0; + } // However, the buffering level must be smoothed sc_average_push(&ap->avg_buffering, can_read); From 25f1e703b7637c3eb1382e435113688520a38d36 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 22 Feb 2024 18:52:57 +0100 Subject: [PATCH 32/79] Extract ControlChannel class This prevents many components from depending on the whole DesktopConnection. --- .../com/genymobile/scrcpy/ControlChannel.java | 33 +++++++++++++++++++ .../com/genymobile/scrcpy/Controller.java | 10 +++--- .../genymobile/scrcpy/DesktopConnection.java | 32 ++++-------------- .../scrcpy/DeviceMessageSender.java | 10 +++--- .../java/com/genymobile/scrcpy/Server.java | 3 +- 5 files changed, 51 insertions(+), 37 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/ControlChannel.java diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlChannel.java b/server/src/main/java/com/genymobile/scrcpy/ControlChannel.java new file mode 100644 index 00000000..4677cfda --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/ControlChannel.java @@ -0,0 +1,33 @@ +package com.genymobile.scrcpy; + +import android.net.LocalSocket; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public final class ControlChannel { + private final InputStream inputStream; + private final OutputStream outputStream; + + private final ControlMessageReader reader = new ControlMessageReader(); + private final DeviceMessageWriter writer = new DeviceMessageWriter(); + + public ControlChannel(LocalSocket controlSocket) throws IOException { + this.inputStream = controlSocket.getInputStream(); + this.outputStream = controlSocket.getOutputStream(); + } + + public ControlMessage recv() throws IOException { + ControlMessage msg = reader.next(); + while (msg == null) { + reader.readFrom(inputStream); + msg = reader.next(); + } + return msg; + } + + public void send(DeviceMessage msg) throws IOException { + writer.writeTo(msg, outputStream); + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index c0763012..257f732b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -27,7 +27,7 @@ public class Controller implements AsyncProcessor { private Thread thread; private final Device device; - private final DesktopConnection connection; + private final ControlChannel controlChannel; private final CleanUp cleanUp; private final DeviceMessageSender sender; private final boolean clipboardAutosync; @@ -42,14 +42,14 @@ public class Controller implements AsyncProcessor { private boolean keepPowerModeOff; - public Controller(Device device, DesktopConnection connection, CleanUp cleanUp, boolean clipboardAutosync, boolean powerOn) { + public Controller(Device device, ControlChannel controlChannel, CleanUp cleanUp, boolean clipboardAutosync, boolean powerOn) { this.device = device; - this.connection = connection; + this.controlChannel = controlChannel; this.cleanUp = cleanUp; this.clipboardAutosync = clipboardAutosync; this.powerOn = powerOn; initPointers(); - sender = new DeviceMessageSender(connection); + sender = new DeviceMessageSender(controlChannel); } private void initPointers() { @@ -123,7 +123,7 @@ public class Controller implements AsyncProcessor { } private void handleEvent() throws IOException { - ControlMessage msg = connection.receiveControlMessage(); + ControlMessage msg = controlChannel.recv(); switch (msg.getType()) { case ControlMessage.TYPE_INJECT_KEYCODE: if (device.supportsInputEvents()) { diff --git a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java index 8bc743f8..d693ad61 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java +++ b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java @@ -7,8 +7,6 @@ import android.net.LocalSocketAddress; import java.io.Closeable; import java.io.FileDescriptor; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.charset.StandardCharsets; public final class DesktopConnection implements Closeable { @@ -24,25 +22,16 @@ public final class DesktopConnection implements Closeable { private final FileDescriptor audioFd; private final LocalSocket controlSocket; - private final InputStream controlInputStream; - private final OutputStream controlOutputStream; - - private final ControlMessageReader reader = new ControlMessageReader(); - private final DeviceMessageWriter writer = new DeviceMessageWriter(); + private final ControlChannel controlChannel; private DesktopConnection(LocalSocket videoSocket, LocalSocket audioSocket, LocalSocket controlSocket) throws IOException { this.videoSocket = videoSocket; - this.controlSocket = controlSocket; this.audioSocket = audioSocket; - if (controlSocket != null) { - controlInputStream = controlSocket.getInputStream(); - controlOutputStream = controlSocket.getOutputStream(); - } else { - controlInputStream = null; - controlOutputStream = null; - } + this.controlSocket = controlSocket; + videoFd = videoSocket != null ? videoSocket.getFileDescriptor() : null; audioFd = audioSocket != null ? audioSocket.getFileDescriptor() : null; + controlChannel = controlSocket != null ? new ControlChannel(controlSocket) : null; } private static LocalSocket connect(String abstractName) throws IOException { @@ -179,16 +168,7 @@ public final class DesktopConnection implements Closeable { return audioFd; } - public ControlMessage receiveControlMessage() throws IOException { - ControlMessage msg = reader.next(); - while (msg == null) { - reader.readFrom(controlInputStream); - msg = reader.next(); - } - return msg; - } - - public void sendDeviceMessage(DeviceMessage msg) throws IOException { - writer.writeTo(msg, controlOutputStream); + public ControlChannel getControlChannel() { + return controlChannel; } } diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java index 94e842ee..efb7b975 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java @@ -4,7 +4,7 @@ import java.io.IOException; public final class DeviceMessageSender { - private final DesktopConnection connection; + private final ControlChannel controlChannel; private Thread thread; @@ -12,8 +12,8 @@ public final class DeviceMessageSender { private long ack; - public DeviceMessageSender(DesktopConnection connection) { - this.connection = connection; + public DeviceMessageSender(ControlChannel controlChannel) { + this.controlChannel = controlChannel; } public synchronized void pushClipboardText(String text) { @@ -43,11 +43,11 @@ public final class DeviceMessageSender { if (sequence != DeviceMessage.SEQUENCE_INVALID) { DeviceMessage event = DeviceMessage.createAckClipboard(sequence); - connection.sendDeviceMessage(event); + controlChannel.send(event); } if (text != null) { DeviceMessage event = DeviceMessage.createClipboard(text); - connection.sendDeviceMessage(event); + controlChannel.send(event); } } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index bcafa133..3936648d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -131,7 +131,8 @@ public final class Server { } if (control) { - Controller controller = new Controller(device, connection, cleanUp, options.getClipboardAutosync(), options.getPowerOn()); + ControlChannel controlChannel = connection.getControlChannel(); + Controller controller = new Controller(device, controlChannel, cleanUp, options.getClipboardAutosync(), options.getPowerOn()); device.setClipboardListener(text -> controller.getSender().pushClipboardText(text)); asyncProcessors.add(controller); } From 9e22f3bf1cca9a957173193250eaeb084ab0c245 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 23 Feb 2024 19:59:54 +0100 Subject: [PATCH 33/79] Replace unsigned char by uint8_t for buffers For consistency. --- app/src/control_msg.c | 4 +- app/src/control_msg.h | 2 +- app/src/controller.c | 2 +- app/src/device_msg.c | 3 +- app/src/device_msg.h | 3 +- app/src/receiver.c | 5 ++- app/src/server.c | 2 +- app/src/usb/aoa_hid.c | 4 +- app/src/usb/aoa_hid.h | 2 +- app/tests/test_control_msg_serialize.c | 57 +++++++++++++------------ app/tests/test_device_msg_deserialize.c | 10 ++--- 11 files changed, 47 insertions(+), 47 deletions(-) diff --git a/app/src/control_msg.c b/app/src/control_msg.c index d4d6c62a..e173dac7 100644 --- a/app/src/control_msg.c +++ b/app/src/control_msg.c @@ -87,7 +87,7 @@ write_position(uint8_t *buf, const struct sc_position *position) { // write length (4 bytes) + string (non null-terminated) static size_t -write_string(const char *utf8, size_t max_len, unsigned char *buf) { +write_string(const char *utf8, size_t max_len, uint8_t *buf) { size_t len = sc_str_utf8_truncation_index(utf8, max_len); sc_write32be(buf, len); memcpy(&buf[4], utf8, len); @@ -95,7 +95,7 @@ write_string(const char *utf8, size_t max_len, unsigned char *buf) { } size_t -sc_control_msg_serialize(const struct sc_control_msg *msg, unsigned char *buf) { +sc_control_msg_serialize(const struct sc_control_msg *msg, uint8_t *buf) { buf[0] = msg->type; switch (msg->type) { case SC_CONTROL_MSG_TYPE_INJECT_KEYCODE: diff --git a/app/src/control_msg.h b/app/src/control_msg.h index b90a00b3..04eeb83b 100644 --- a/app/src/control_msg.h +++ b/app/src/control_msg.h @@ -98,7 +98,7 @@ struct sc_control_msg { // buf size must be at least CONTROL_MSG_MAX_SIZE // return the number of bytes written size_t -sc_control_msg_serialize(const struct sc_control_msg *msg, unsigned char *buf); +sc_control_msg_serialize(const struct sc_control_msg *msg, uint8_t *buf); void sc_control_msg_log(const struct sc_control_msg *msg); diff --git a/app/src/controller.c b/app/src/controller.c index 0139e42c..250321fe 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -84,7 +84,7 @@ sc_controller_push_msg(struct sc_controller *controller, static bool process_msg(struct sc_controller *controller, const struct sc_control_msg *msg) { - static unsigned char serialized_msg[SC_CONTROL_MSG_MAX_SIZE]; + static uint8_t serialized_msg[SC_CONTROL_MSG_MAX_SIZE]; size_t length = sc_control_msg_serialize(msg, serialized_msg); if (!length) { return false; diff --git a/app/src/device_msg.c b/app/src/device_msg.c index 265c7505..9925cf97 100644 --- a/app/src/device_msg.c +++ b/app/src/device_msg.c @@ -8,8 +8,7 @@ #include "util/log.h" ssize_t -device_msg_deserialize(const unsigned char *buf, size_t len, - struct device_msg *msg) { +device_msg_deserialize(const uint8_t *buf, size_t len, struct device_msg *msg) { if (len < 5) { // at least type + empty string length return 0; // not available diff --git a/app/src/device_msg.h b/app/src/device_msg.h index e8d9fed4..3b68a61a 100644 --- a/app/src/device_msg.h +++ b/app/src/device_msg.h @@ -30,8 +30,7 @@ struct device_msg { // return the number of bytes consumed (0 for no msg available, -1 on error) ssize_t -device_msg_deserialize(const unsigned char *buf, size_t len, - struct device_msg *msg); +device_msg_deserialize(const uint8_t *buf, size_t len, struct device_msg *msg); void device_msg_destroy(struct device_msg *msg); diff --git a/app/src/receiver.c b/app/src/receiver.c index e715a8e6..c08cd6cf 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -1,6 +1,7 @@ #include "receiver.h" #include +#include #include #include "device_msg.h" @@ -51,7 +52,7 @@ process_msg(struct sc_receiver *receiver, struct device_msg *msg) { } static ssize_t -process_msgs(struct sc_receiver *receiver, const unsigned char *buf, size_t len) { +process_msgs(struct sc_receiver *receiver, const uint8_t *buf, size_t len) { size_t head = 0; for (;;) { struct device_msg msg; @@ -78,7 +79,7 @@ static int run_receiver(void *data) { struct sc_receiver *receiver = data; - static unsigned char buf[DEVICE_MSG_MAX_SIZE]; + static uint8_t buf[DEVICE_MSG_MAX_SIZE]; size_t head = 0; for (;;) { diff --git a/app/src/server.c b/app/src/server.c index d4726c2a..4d55e994 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -498,7 +498,7 @@ sc_server_init(struct sc_server *server, const struct sc_server_params *params, static bool device_read_info(struct sc_intr *intr, sc_socket device_socket, struct sc_server_info *info) { - unsigned char buf[SC_DEVICE_NAME_FIELD_LENGTH]; + uint8_t buf[SC_DEVICE_NAME_FIELD_LENGTH]; ssize_t r = net_recv_all_intr(intr, device_socket, buf, sizeof(buf)); if (r < SC_DEVICE_NAME_FIELD_LENGTH) { LOGE("Could not retrieve device information"); diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index fb64e57c..9bad5296 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -113,7 +113,7 @@ sc_aoa_register_hid(struct sc_aoa *aoa, uint16_t accessory_id, static bool sc_aoa_set_hid_report_desc(struct sc_aoa *aoa, uint16_t accessory_id, - const unsigned char *report_desc, + const uint8_t *report_desc, uint16_t report_desc_size) { uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; uint8_t request = ACCESSORY_SET_HID_REPORT_DESC; @@ -150,7 +150,7 @@ sc_aoa_set_hid_report_desc(struct sc_aoa *aoa, uint16_t accessory_id, bool sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id, - const unsigned char *report_desc, uint16_t report_desc_size) { + const uint8_t *report_desc, uint16_t report_desc_size) { bool ok = sc_aoa_register_hid(aoa, accessory_id, report_desc_size); if (!ok) { return false; diff --git a/app/src/usb/aoa_hid.h b/app/src/usb/aoa_hid.h index 8803c1d9..fb5e1d28 100644 --- a/app/src/usb/aoa_hid.h +++ b/app/src/usb/aoa_hid.h @@ -57,7 +57,7 @@ sc_aoa_join(struct sc_aoa *aoa); bool sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id, - const unsigned char *report_desc, uint16_t report_desc_size); + const uint8_t *report_desc, uint16_t report_desc_size); bool sc_aoa_unregister_hid(struct sc_aoa *aoa, uint16_t accessory_id); diff --git a/app/tests/test_control_msg_serialize.c b/app/tests/test_control_msg_serialize.c index b2eef49c..80d33fc3 100644 --- a/app/tests/test_control_msg_serialize.c +++ b/app/tests/test_control_msg_serialize.c @@ -1,6 +1,7 @@ #include "common.h" #include +#include #include #include "control_msg.h" @@ -16,11 +17,11 @@ static void test_serialize_inject_keycode(void) { }, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 14); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_INJECT_KEYCODE, 0x01, // AKEY_EVENT_ACTION_UP 0x00, 0x00, 0x00, 0x42, // AKEYCODE_ENTER @@ -38,11 +39,11 @@ static void test_serialize_inject_text(void) { }, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 18); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_INJECT_TEXT, 0x00, 0x00, 0x00, 0x0d, // text length 'h', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // text @@ -58,11 +59,11 @@ static void test_serialize_inject_text_long(void) { text[SC_CONTROL_MSG_INJECT_TEXT_MAX_LENGTH] = '\0'; msg.inject_text.text = text; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 5 + SC_CONTROL_MSG_INJECT_TEXT_MAX_LENGTH); - unsigned char expected[5 + SC_CONTROL_MSG_INJECT_TEXT_MAX_LENGTH]; + uint8_t expected[5 + SC_CONTROL_MSG_INJECT_TEXT_MAX_LENGTH]; expected[0] = SC_CONTROL_MSG_TYPE_INJECT_TEXT; expected[1] = 0x00; expected[2] = 0x00; @@ -95,11 +96,11 @@ static void test_serialize_inject_touch_event(void) { }, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 32); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT, 0x00, // AKEY_EVENT_ACTION_DOWN 0x12, 0x34, 0x56, 0x78, 0x87, 0x65, 0x43, 0x21, // pointer id @@ -132,11 +133,11 @@ static void test_serialize_inject_scroll_event(void) { }, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 21); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x04, 0x02, // 260 1026 0x04, 0x38, 0x07, 0x80, // 1080 1920 @@ -155,11 +156,11 @@ static void test_serialize_back_or_screen_on(void) { }, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 2); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON, 0x01, // AKEY_EVENT_ACTION_UP }; @@ -171,11 +172,11 @@ static void test_serialize_expand_notification_panel(void) { .type = SC_CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 1); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL, }; assert(!memcmp(buf, expected, sizeof(expected))); @@ -186,11 +187,11 @@ static void test_serialize_expand_settings_panel(void) { .type = SC_CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 1); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL, }; assert(!memcmp(buf, expected, sizeof(expected))); @@ -201,11 +202,11 @@ static void test_serialize_collapse_panels(void) { .type = SC_CONTROL_MSG_TYPE_COLLAPSE_PANELS, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 1); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_COLLAPSE_PANELS, }; assert(!memcmp(buf, expected, sizeof(expected))); @@ -219,11 +220,11 @@ static void test_serialize_get_clipboard(void) { }, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 2); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_GET_CLIPBOARD, SC_COPY_KEY_COPY, }; @@ -240,11 +241,11 @@ static void test_serialize_set_clipboard(void) { }, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 27); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_SET_CLIPBOARD, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // sequence 1, // paste @@ -269,11 +270,11 @@ static void test_serialize_set_clipboard_long(void) { text[SC_CONTROL_MSG_CLIPBOARD_TEXT_MAX_LENGTH] = '\0'; msg.set_clipboard.text = text; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == SC_CONTROL_MSG_MAX_SIZE); - unsigned char expected[SC_CONTROL_MSG_MAX_SIZE] = { + uint8_t expected[SC_CONTROL_MSG_MAX_SIZE] = { SC_CONTROL_MSG_TYPE_SET_CLIPBOARD, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // sequence 1, // paste @@ -296,11 +297,11 @@ static void test_serialize_set_screen_power_mode(void) { }, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 2); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE, 0x02, // SC_SCREEN_POWER_MODE_NORMAL }; @@ -312,11 +313,11 @@ static void test_serialize_rotate_device(void) { .type = SC_CONTROL_MSG_TYPE_ROTATE_DEVICE, }; - unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; size_t size = sc_control_msg_serialize(&msg, buf); assert(size == 1); - const unsigned char expected[] = { + const uint8_t expected[] = { SC_CONTROL_MSG_TYPE_ROTATE_DEVICE, }; assert(!memcmp(buf, expected, sizeof(expected))); diff --git a/app/tests/test_device_msg_deserialize.c b/app/tests/test_device_msg_deserialize.c index 835096c0..a1a3f695 100644 --- a/app/tests/test_device_msg_deserialize.c +++ b/app/tests/test_device_msg_deserialize.c @@ -1,14 +1,14 @@ #include "common.h" #include +#include +#include #include #include "device_msg.h" -#include - static void test_deserialize_clipboard(void) { - const unsigned char input[] = { + const uint8_t input[] = { DEVICE_MSG_TYPE_CLIPBOARD, 0x00, 0x00, 0x00, 0x03, // text length 0x41, 0x42, 0x43, // "ABC" @@ -26,7 +26,7 @@ static void test_deserialize_clipboard(void) { } static void test_deserialize_clipboard_big(void) { - unsigned char input[DEVICE_MSG_MAX_SIZE]; + uint8_t input[DEVICE_MSG_MAX_SIZE]; input[0] = DEVICE_MSG_TYPE_CLIPBOARD; input[1] = (DEVICE_MSG_TEXT_MAX_LENGTH & 0xff000000u) >> 24; input[2] = (DEVICE_MSG_TEXT_MAX_LENGTH & 0x00ff0000u) >> 16; @@ -48,7 +48,7 @@ static void test_deserialize_clipboard_big(void) { } static void test_deserialize_ack_set_clipboard(void) { - const unsigned char input[] = { + const uint8_t input[] = { DEVICE_MSG_TYPE_ACK_CLIPBOARD, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // sequence }; From 9858eff85625abcbf392ae57220df1b1b03f793b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 23 Feb 2024 20:01:30 +0100 Subject: [PATCH 34/79] Fix device message deserialization checks If any message is incomplete, the deserialization method must return immediately. --- app/src/device_msg.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/app/src/device_msg.c b/app/src/device_msg.c index 9925cf97..f9f22a85 100644 --- a/app/src/device_msg.c +++ b/app/src/device_msg.c @@ -9,17 +9,20 @@ ssize_t device_msg_deserialize(const uint8_t *buf, size_t len, struct device_msg *msg) { - if (len < 5) { - // at least type + empty string length - return 0; // not available + if (!len) { + return 0; // no message } msg->type = buf[0]; switch (msg->type) { case DEVICE_MSG_TYPE_CLIPBOARD: { + if (len < 5) { + // at least type + empty string length + return 0; // no complete message + } size_t clipboard_len = sc_read32be(&buf[1]); if (clipboard_len > len - 5) { - return 0; // not available + return 0; // no complete message } char *text = malloc(clipboard_len + 1); if (!text) { @@ -35,6 +38,9 @@ device_msg_deserialize(const uint8_t *buf, size_t len, struct device_msg *msg) { return 5 + clipboard_len; } case DEVICE_MSG_TYPE_ACK_CLIPBOARD: { + if (len < 9) { + return 0; // no complete message + } uint64_t sequence = sc_read64be(&buf[1]); msg->ack_clipboard.sequence = sequence; return 9; From 78a7e4f293f59499fbb4be850a29e891171fcf4f Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 23 Feb 2024 20:05:12 +0100 Subject: [PATCH 35/79] Use sc_ prefix for device sender --- app/src/device_msg.c | 5 +++-- app/src/device_msg.h | 11 ++++++----- app/src/receiver.c | 8 ++++---- app/tests/test_device_msg_deserialize.c | 16 ++++++++-------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/app/src/device_msg.c b/app/src/device_msg.c index f9f22a85..0cadc49c 100644 --- a/app/src/device_msg.c +++ b/app/src/device_msg.c @@ -8,7 +8,8 @@ #include "util/log.h" ssize_t -device_msg_deserialize(const uint8_t *buf, size_t len, struct device_msg *msg) { +sc_device_msg_deserialize(const uint8_t *buf, size_t len, + struct sc_device_msg *msg) { if (!len) { return 0; // no message } @@ -52,7 +53,7 @@ device_msg_deserialize(const uint8_t *buf, size_t len, struct device_msg *msg) { } void -device_msg_destroy(struct device_msg *msg) { +sc_device_msg_destroy(struct sc_device_msg *msg) { if (msg->type == DEVICE_MSG_TYPE_CLIPBOARD) { free(msg->clipboard.text); } diff --git a/app/src/device_msg.h b/app/src/device_msg.h index 3b68a61a..3f541cf5 100644 --- a/app/src/device_msg.h +++ b/app/src/device_msg.h @@ -11,13 +11,13 @@ // type: 1 byte; length: 4 bytes #define DEVICE_MSG_TEXT_MAX_LENGTH (DEVICE_MSG_MAX_SIZE - 5) -enum device_msg_type { +enum sc_device_msg_type { DEVICE_MSG_TYPE_CLIPBOARD, DEVICE_MSG_TYPE_ACK_CLIPBOARD, }; -struct device_msg { - enum device_msg_type type; +struct sc_device_msg { + enum sc_device_msg_type type; union { struct { char *text; // owned, to be freed by free() @@ -30,9 +30,10 @@ struct device_msg { // return the number of bytes consumed (0 for no msg available, -1 on error) ssize_t -device_msg_deserialize(const uint8_t *buf, size_t len, struct device_msg *msg); +sc_device_msg_deserialize(const uint8_t *buf, size_t len, + struct sc_device_msg *msg); void -device_msg_destroy(struct device_msg *msg); +sc_device_msg_destroy(struct sc_device_msg *msg); #endif diff --git a/app/src/receiver.c b/app/src/receiver.c index c08cd6cf..408e1db7 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -27,7 +27,7 @@ sc_receiver_destroy(struct sc_receiver *receiver) { } static void -process_msg(struct sc_receiver *receiver, struct device_msg *msg) { +process_msg(struct sc_receiver *receiver, struct sc_device_msg *msg) { switch (msg->type) { case DEVICE_MSG_TYPE_CLIPBOARD: { char *current = SDL_GetClipboardText(); @@ -55,8 +55,8 @@ static ssize_t process_msgs(struct sc_receiver *receiver, const uint8_t *buf, size_t len) { size_t head = 0; for (;;) { - struct device_msg msg; - ssize_t r = device_msg_deserialize(&buf[head], len - head, &msg); + struct sc_device_msg msg; + ssize_t r = sc_device_msg_deserialize(&buf[head], len - head, &msg); if (r == -1) { return -1; } @@ -65,7 +65,7 @@ process_msgs(struct sc_receiver *receiver, const uint8_t *buf, size_t len) { } process_msg(receiver, &msg); - device_msg_destroy(&msg); + sc_device_msg_destroy(&msg); head += r; assert(head <= len); diff --git a/app/tests/test_device_msg_deserialize.c b/app/tests/test_device_msg_deserialize.c index a1a3f695..bfbcefd6 100644 --- a/app/tests/test_device_msg_deserialize.c +++ b/app/tests/test_device_msg_deserialize.c @@ -14,15 +14,15 @@ static void test_deserialize_clipboard(void) { 0x41, 0x42, 0x43, // "ABC" }; - struct device_msg msg; - ssize_t r = device_msg_deserialize(input, sizeof(input), &msg); + struct sc_device_msg msg; + ssize_t r = sc_device_msg_deserialize(input, sizeof(input), &msg); assert(r == 8); assert(msg.type == DEVICE_MSG_TYPE_CLIPBOARD); assert(msg.clipboard.text); assert(!strcmp("ABC", msg.clipboard.text)); - device_msg_destroy(&msg); + sc_device_msg_destroy(&msg); } static void test_deserialize_clipboard_big(void) { @@ -35,8 +35,8 @@ static void test_deserialize_clipboard_big(void) { memset(input + 5, 'a', DEVICE_MSG_TEXT_MAX_LENGTH); - struct device_msg msg; - ssize_t r = device_msg_deserialize(input, sizeof(input), &msg); + struct sc_device_msg msg; + ssize_t r = sc_device_msg_deserialize(input, sizeof(input), &msg); assert(r == DEVICE_MSG_MAX_SIZE); assert(msg.type == DEVICE_MSG_TYPE_CLIPBOARD); @@ -44,7 +44,7 @@ static void test_deserialize_clipboard_big(void) { assert(strlen(msg.clipboard.text) == DEVICE_MSG_TEXT_MAX_LENGTH); assert(msg.clipboard.text[0] == 'a'); - device_msg_destroy(&msg); + sc_device_msg_destroy(&msg); } static void test_deserialize_ack_set_clipboard(void) { @@ -53,8 +53,8 @@ static void test_deserialize_ack_set_clipboard(void) { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // sequence }; - struct device_msg msg; - ssize_t r = device_msg_deserialize(input, sizeof(input), &msg); + struct sc_device_msg msg; + ssize_t r = sc_device_msg_deserialize(input, sizeof(input), &msg); assert(r == 9); assert(msg.type == DEVICE_MSG_TYPE_ACK_CLIPBOARD); From 746eaea55683e8e97ba7763bc0fa567227004c5d Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 26 Feb 2024 08:33:11 +0100 Subject: [PATCH 36/79] Add missing clipboard workaround for IQOO device The first part of the workaround fixed getPrimaryClip(). This part fixes setPrimaryClip(). Fixes #4703 Refs 5ce8672ebc56b7286e1078a39abc64903e5664d0 Refs #4492 --- .../scrcpy/wrappers/ClipboardManager.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java index 2c8d9907..ed5c8d75 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java @@ -87,9 +87,15 @@ public final class ClipboardManager { setPrimaryClipMethod = manager.getClass().getMethod("setPrimaryClip", ClipData.class, String.class, String.class, int.class); setMethodVersion = 1; } catch (NoSuchMethodException e2) { - setPrimaryClipMethod = manager.getClass() - .getMethod("setPrimaryClip", ClipData.class, String.class, String.class, int.class, int.class); - setMethodVersion = 2; + try { + setPrimaryClipMethod = manager.getClass() + .getMethod("setPrimaryClip", ClipData.class, String.class, String.class, int.class, int.class); + setMethodVersion = 2; + } catch (NoSuchMethodException e3) { + setPrimaryClipMethod = manager.getClass() + .getMethod("setPrimaryClip", ClipData.class, String.class, String.class, int.class, int.class, boolean.class); + setMethodVersion = 3; + } } } } @@ -132,9 +138,12 @@ public final class ClipboardManager { case 1: method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID); break; - default: + case 2: method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID, 0); break; + default: + // The last boolean parameter is "userOperate" + method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID, 0, true); } } From d894e270a7719b92e38b4f5e0294b9d55e90a6df Mon Sep 17 00:00:00 2001 From: eiyooooo Date: Sat, 24 Feb 2024 01:10:35 +0800 Subject: [PATCH 37/79] Add rotation support for non-default display Use new methods introduced by this commit: PR #4698 Signed-off-by: Romain Vimont --- .../com/genymobile/scrcpy/Controller.java | 2 +- .../java/com/genymobile/scrcpy/Device.java | 19 +++-- .../scrcpy/wrappers/WindowManager.java | 76 ++++++++++++++++--- 3 files changed, 82 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 257f732b..73d6ad57 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -180,7 +180,7 @@ public class Controller implements AsyncProcessor { } break; case ControlMessage.TYPE_ROTATE_DEVICE: - Device.rotateDevice(); + device.rotateDevice(); break; default: // do nothing diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java index 33b09a57..8d0ee231 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/Device.java @@ -359,21 +359,30 @@ public final class Device { /** * Disable auto-rotation (if enabled), set the screen rotation and re-enable auto-rotation (if it was enabled). */ - public static void rotateDevice() { + public void rotateDevice() { WindowManager wm = ServiceManager.getWindowManager(); - boolean accelerometerRotation = !wm.isRotationFrozen(); + boolean accelerometerRotation = !wm.isRotationFrozen(displayId); - int currentRotation = wm.getRotation(); + int currentRotation = getCurrentRotation(displayId); int newRotation = (currentRotation & 1) ^ 1; // 0->1, 1->0, 2->1, 3->0 String newRotationString = newRotation == 0 ? "portrait" : "landscape"; Ln.i("Device rotation requested: " + newRotationString); - wm.freezeRotation(newRotation); + wm.freezeRotation(displayId, newRotation); // restore auto-rotate if necessary if (accelerometerRotation) { - wm.thawRotation(); + wm.thawRotation(displayId); } } + + private static int getCurrentRotation(int displayId) { + if (displayId == 0) { + return ServiceManager.getWindowManager().getRotation(); + } + + DisplayInfo displayInfo = ServiceManager.getDisplayManager().getDisplayInfo(displayId); + return displayInfo.getRotation(); + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java index b19dace9..d9654b1b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java @@ -13,8 +13,11 @@ public final class WindowManager { private final IInterface manager; private Method getRotationMethod; private Method freezeRotationMethod; + private Method freezeDisplayRotationMethod; private Method isRotationFrozenMethod; + private Method isDisplayRotationFrozenMethod; private Method thawRotationMethod; + private Method thawDisplayRotationMethod; static WindowManager create() { IInterface manager = ServiceManager.getService("window", "android.view.IWindowManager"); @@ -47,6 +50,15 @@ public final class WindowManager { return freezeRotationMethod; } + // New method added by this commit: + // + private Method getFreezeDisplayRotationMethod() throws NoSuchMethodException { + if (freezeDisplayRotationMethod == null) { + freezeDisplayRotationMethod = manager.getClass().getMethod("freezeDisplayRotation", int.class, int.class); + } + return freezeDisplayRotationMethod; + } + private Method getIsRotationFrozenMethod() throws NoSuchMethodException { if (isRotationFrozenMethod == null) { isRotationFrozenMethod = manager.getClass().getMethod("isRotationFrozen"); @@ -54,6 +66,15 @@ public final class WindowManager { return isRotationFrozenMethod; } + // New method added by this commit: + // + private Method getIsDisplayRotationFrozenMethod() throws NoSuchMethodException { + if (isDisplayRotationFrozenMethod == null) { + isDisplayRotationFrozenMethod = manager.getClass().getMethod("isDisplayRotationFrozen", int.class); + } + return isDisplayRotationFrozenMethod; + } + private Method getThawRotationMethod() throws NoSuchMethodException { if (thawRotationMethod == null) { thawRotationMethod = manager.getClass().getMethod("thawRotation"); @@ -61,6 +82,15 @@ public final class WindowManager { return thawRotationMethod; } + // New method added by this commit: + // + private Method getThawDisplayRotationMethod() throws NoSuchMethodException { + if (thawDisplayRotationMethod == null) { + thawDisplayRotationMethod = manager.getClass().getMethod("thawDisplayRotation", int.class); + } + return thawDisplayRotationMethod; + } + public int getRotation() { try { Method method = getGetRotationMethod(); @@ -71,29 +101,57 @@ public final class WindowManager { } } - public void freezeRotation(int rotation) { + public void freezeRotation(int displayId, int rotation) { try { - Method method = getFreezeRotationMethod(); - method.invoke(manager, rotation); + try { + Method method = getFreezeDisplayRotationMethod(); + method.invoke(manager, displayId, rotation); + } catch (ReflectiveOperationException e) { + if (displayId == 0) { + Method method = getFreezeRotationMethod(); + method.invoke(manager, rotation); + } else { + Ln.e("Could not invoke method", e); + } + } } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); } } - public boolean isRotationFrozen() { + public boolean isRotationFrozen(int displayId) { try { - Method method = getIsRotationFrozenMethod(); - return (boolean) method.invoke(manager); + try { + Method method = getIsDisplayRotationFrozenMethod(); + return (boolean) method.invoke(manager, displayId); + } catch (ReflectiveOperationException e) { + if (displayId == 0) { + Method method = getIsRotationFrozenMethod(); + return (boolean) method.invoke(manager); + } else { + Ln.e("Could not invoke method", e); + return false; + } + } } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); return false; } } - public void thawRotation() { + public void thawRotation(int displayId) { try { - Method method = getThawRotationMethod(); - method.invoke(manager); + try { + Method method = getThawDisplayRotationMethod(); + method.invoke(manager, displayId); + } catch (ReflectiveOperationException e) { + if (displayId == 0) { + Method method = getThawRotationMethod(); + method.invoke(manager); + } else { + Ln.e("Could not invoke method", e); + } + } } catch (ReflectiveOperationException e) { Ln.e("Could not invoke method", e); } From 295102a6d91e5a71fbcc7fa3f17a2cea9c9eb9e6 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 27 Feb 2024 09:59:23 +0100 Subject: [PATCH 38/79] Check device messages assumptions at runtime Do not assume the server behaves correctly (scrcpy should not require the device to be trusted). --- app/src/receiver.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/src/receiver.c b/app/src/receiver.c index 408e1db7..6be705e3 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -43,9 +43,19 @@ process_msg(struct sc_receiver *receiver, struct sc_device_msg *msg) { break; } case DEVICE_MSG_TYPE_ACK_CLIPBOARD: - assert(receiver->acksync); LOGD("Ack device clipboard sequence=%" PRIu64_, msg->ack_clipboard.sequence); + + // This is a programming error to receive this message if there is + // no ACK synchronization mechanism + assert(receiver->acksync); + + // Also check at runtime (do not trust the server) + if (!receiver->acksync) { + LOGE("Received unexpected ack"); + return; + } + sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence); break; } From f6459dd742f356fade275e2178aa9ecee05c23cc Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 29 Feb 2024 10:26:49 +0100 Subject: [PATCH 39/79] Fix FAQ link Refs ad05a018003a66b0a5f8afefb0d2f16a392d3077 --- FAQ.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FAQ.md b/FAQ.md index a6eaeefa..6d02361b 100644 --- a/FAQ.md +++ b/FAQ.md @@ -222,7 +222,7 @@ java.lang.IllegalStateException at android.media.MediaCodec.native_dequeueOutputBuffer(Native Method) ``` -then try with another [encoder](doc/video.md#codec). +then try with another [encoder](doc/video.md#encoder). ## Translations From ffa238b9d35bb9c882537d32724bbadbe4da7ef6 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 29 Feb 2024 23:55:44 +0100 Subject: [PATCH 40/79] Remove duplicate lines in libusb script --- app/prebuilt-deps/prepare-libusb.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/prebuilt-deps/prepare-libusb.sh b/app/prebuilt-deps/prepare-libusb.sh index 228a5bfa..b31c45eb 100755 --- a/app/prebuilt-deps/prepare-libusb.sh +++ b/app/prebuilt-deps/prepare-libusb.sh @@ -26,8 +26,6 @@ cd "$DEP_DIR" 7z x "../$FILENAME" \ "libusb-$VERSION-binaries/libusb-MinGW-Win32/" \ - "libusb-$VERSION-binaries/libusb-MinGW-Win32/" \ - "libusb-$VERSION-binaries/libusb-MinGW-x64/" \ "libusb-$VERSION-binaries/libusb-MinGW-x64/" mv "libusb-$VERSION-binaries/libusb-MinGW-Win32" . From a97641757237fc9342f2d1b4b9573a99761ffc21 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 1 Mar 2024 00:11:48 +0100 Subject: [PATCH 41/79] Fix typo in error message --- app/src/display.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/display.c b/app/src/display.c index 906b5d65..ba15cd14 100644 --- a/app/src/display.c +++ b/app/src/display.c @@ -59,7 +59,7 @@ sc_display_init(struct sc_display *display, SDL_Window *window, bool mipmaps) { LOGI("Trilinear filtering disabled"); } } else if (mipmaps) { - LOGD("Trilinear filtering disabled (not an OpenGL renderer"); + LOGD("Trilinear filtering disabled (not an OpenGL renderer)"); } display->pending.flags = 0; From c0a1aee8cea2ce6a5dbe39117d0505786ea0db7b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 20 Jan 2024 18:19:39 +0100 Subject: [PATCH 42/79] Always pass input manager instance Some functions in input_manager.c only have access to a sub-object (for example the controller). For consistency, always pass the whole input manager instance. This will allow to add assertions when keyboard and mouse could be disabled separately. PR #4473 --- app/src/input_manager.c | 193 ++++++++++++++++++++++------------------ 1 file changed, 107 insertions(+), 86 deletions(-) diff --git a/app/src/input_manager.c b/app/src/input_manager.c index 76cfbd92..8e7a6402 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -87,8 +87,10 @@ sc_input_manager_init(struct sc_input_manager *im, } static void -send_keycode(struct sc_controller *controller, enum android_keycode keycode, +send_keycode(struct sc_input_manager *im, enum android_keycode keycode, enum sc_action action, const char *name) { + assert(im->controller); + // send DOWN event struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_INJECT_KEYCODE; @@ -99,100 +101,109 @@ send_keycode(struct sc_controller *controller, enum android_keycode keycode, msg.inject_keycode.metastate = 0; msg.inject_keycode.repeat = 0; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { LOGW("Could not request 'inject %s'", name); } } static inline void -action_home(struct sc_controller *controller, enum sc_action action) { - send_keycode(controller, AKEYCODE_HOME, action, "HOME"); +action_home(struct sc_input_manager *im, enum sc_action action) { + send_keycode(im, AKEYCODE_HOME, action, "HOME"); } static inline void -action_back(struct sc_controller *controller, enum sc_action action) { - send_keycode(controller, AKEYCODE_BACK, action, "BACK"); +action_back(struct sc_input_manager *im, enum sc_action action) { + send_keycode(im, AKEYCODE_BACK, action, "BACK"); } static inline void -action_app_switch(struct sc_controller *controller, enum sc_action action) { - send_keycode(controller, AKEYCODE_APP_SWITCH, action, "APP_SWITCH"); +action_app_switch(struct sc_input_manager *im, enum sc_action action) { + send_keycode(im, AKEYCODE_APP_SWITCH, action, "APP_SWITCH"); } static inline void -action_power(struct sc_controller *controller, enum sc_action action) { - send_keycode(controller, AKEYCODE_POWER, action, "POWER"); +action_power(struct sc_input_manager *im, enum sc_action action) { + send_keycode(im, AKEYCODE_POWER, action, "POWER"); } static inline void -action_volume_up(struct sc_controller *controller, enum sc_action action) { - send_keycode(controller, AKEYCODE_VOLUME_UP, action, "VOLUME_UP"); +action_volume_up(struct sc_input_manager *im, enum sc_action action) { + send_keycode(im, AKEYCODE_VOLUME_UP, action, "VOLUME_UP"); } static inline void -action_volume_down(struct sc_controller *controller, enum sc_action action) { - send_keycode(controller, AKEYCODE_VOLUME_DOWN, action, "VOLUME_DOWN"); +action_volume_down(struct sc_input_manager *im, enum sc_action action) { + send_keycode(im, AKEYCODE_VOLUME_DOWN, action, "VOLUME_DOWN"); } static inline void -action_menu(struct sc_controller *controller, enum sc_action action) { - send_keycode(controller, AKEYCODE_MENU, action, "MENU"); +action_menu(struct sc_input_manager *im, enum sc_action action) { + send_keycode(im, AKEYCODE_MENU, action, "MENU"); } // turn the screen on if it was off, press BACK otherwise // If the screen is off, it is turned on only on ACTION_DOWN static void -press_back_or_turn_screen_on(struct sc_controller *controller, +press_back_or_turn_screen_on(struct sc_input_manager *im, enum sc_action action) { + assert(im->controller); + struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON; msg.back_or_screen_on.action = action == SC_ACTION_DOWN ? AKEY_EVENT_ACTION_DOWN : AKEY_EVENT_ACTION_UP; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { LOGW("Could not request 'press back or turn screen on'"); } } static void -expand_notification_panel(struct sc_controller *controller) { +expand_notification_panel(struct sc_input_manager *im) { + assert(im->controller); + struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { LOGW("Could not request 'expand notification panel'"); } } static void -expand_settings_panel(struct sc_controller *controller) { +expand_settings_panel(struct sc_input_manager *im) { + assert(im->controller); + struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { LOGW("Could not request 'expand settings panel'"); } } static void -collapse_panels(struct sc_controller *controller) { +collapse_panels(struct sc_input_manager *im) { + assert(im->controller); + struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_COLLAPSE_PANELS; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { LOGW("Could not request 'collapse notification panel'"); } } static bool -get_device_clipboard(struct sc_controller *controller, - enum sc_copy_key copy_key) { +get_device_clipboard(struct sc_input_manager *im, enum sc_copy_key copy_key) { + assert(im->controller); + struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_GET_CLIPBOARD; msg.get_clipboard.copy_key = copy_key; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { LOGW("Could not request 'get device clipboard'"); return false; } @@ -201,8 +212,10 @@ get_device_clipboard(struct sc_controller *controller, } static bool -set_device_clipboard(struct sc_controller *controller, bool paste, +set_device_clipboard(struct sc_input_manager *im, bool paste, uint64_t sequence) { + assert(im->controller); + char *text = SDL_GetClipboardText(); if (!text) { LOGW("Could not get clipboard text: %s", SDL_GetError()); @@ -222,7 +235,7 @@ set_device_clipboard(struct sc_controller *controller, bool paste, msg.set_clipboard.text = text_dup; msg.set_clipboard.paste = paste; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { free(text_dup); LOGW("Could not request 'set device clipboard'"); return false; @@ -232,19 +245,23 @@ set_device_clipboard(struct sc_controller *controller, bool paste, } static void -set_screen_power_mode(struct sc_controller *controller, +set_screen_power_mode(struct sc_input_manager *im, enum sc_screen_power_mode mode) { + assert(im->controller); + struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE; msg.set_screen_power_mode.mode = mode; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { LOGW("Could not request 'set screen power mode'"); } } static void -switch_fps_counter_state(struct sc_fps_counter *fps_counter) { +switch_fps_counter_state(struct sc_input_manager *im) { + struct sc_fps_counter *fps_counter = &im->screen->fps_counter; + // the started state can only be written from the current thread, so there // is no ToCToU issue if (sc_fps_counter_is_started(fps_counter)) { @@ -256,7 +273,9 @@ switch_fps_counter_state(struct sc_fps_counter *fps_counter) { } static void -clipboard_paste(struct sc_controller *controller) { +clipboard_paste(struct sc_input_manager *im) { + assert(im->controller); + char *text = SDL_GetClipboardText(); if (!text) { LOGW("Could not get clipboard text: %s", SDL_GetError()); @@ -278,25 +297,28 @@ clipboard_paste(struct sc_controller *controller) { struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_INJECT_TEXT; msg.inject_text.text = text_dup; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { free(text_dup); LOGW("Could not request 'paste clipboard'"); } } static void -rotate_device(struct sc_controller *controller) { +rotate_device(struct sc_input_manager *im) { + assert(im->controller); + struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_ROTATE_DEVICE; - if (!sc_controller_push_msg(controller, &msg)) { + if (!sc_controller_push_msg(im->controller, &msg)) { LOGW("Could not request device rotation"); } } static void -apply_orientation_transform(struct sc_screen *screen, +apply_orientation_transform(struct sc_input_manager *im, enum sc_orientation transform) { + struct sc_screen *screen = im->screen; enum sc_orientation new_orientation = sc_orientation_apply(screen->orientation, transform); sc_screen_set_orientation(screen, new_orientation); @@ -364,7 +386,7 @@ static void sc_input_manager_process_key(struct sc_input_manager *im, const SDL_KeyboardEvent *event) { // controller is NULL if --no-control is requested - struct sc_controller *controller = im->controller; + bool control = im->controller; SDL_Keycode keycode = event->keysym.sym; uint16_t mod = event->keysym.mod; @@ -390,68 +412,68 @@ sc_input_manager_process_key(struct sc_input_manager *im, enum sc_action action = down ? SC_ACTION_DOWN : SC_ACTION_UP; switch (keycode) { case SDLK_h: - if (controller && !shift && !repeat) { - action_home(controller, action); + if (control && !shift && !repeat) { + action_home(im, action); } return; case SDLK_b: // fall-through case SDLK_BACKSPACE: - if (controller && !shift && !repeat) { - action_back(controller, action); + if (control && !shift && !repeat) { + action_back(im, action); } return; case SDLK_s: - if (controller && !shift && !repeat) { - action_app_switch(controller, action); + if (control && !shift && !repeat) { + action_app_switch(im, action); } return; case SDLK_m: - if (controller && !shift && !repeat) { - action_menu(controller, action); + if (control && !shift && !repeat) { + action_menu(im, action); } return; case SDLK_p: - if (controller && !shift && !repeat) { - action_power(controller, action); + if (control && !shift && !repeat) { + action_power(im, action); } return; case SDLK_o: - if (controller && !repeat && down) { + if (control && !repeat && down) { enum sc_screen_power_mode mode = shift ? SC_SCREEN_POWER_MODE_NORMAL : SC_SCREEN_POWER_MODE_OFF; - set_screen_power_mode(controller, mode); + set_screen_power_mode(im, mode); } return; case SDLK_DOWN: if (shift) { if (!repeat & down) { - apply_orientation_transform(im->screen, + apply_orientation_transform(im, SC_ORIENTATION_FLIP_180); } - } else if (controller) { + } else if (control) { // forward repeated events - action_volume_down(controller, action); + action_volume_down(im, action); } return; case SDLK_UP: if (shift) { if (!repeat & down) { - apply_orientation_transform(im->screen, + apply_orientation_transform(im, SC_ORIENTATION_FLIP_180); } - } else if (controller) { + } else if (control) { // forward repeated events - action_volume_up(controller, action); + action_volume_up(im, action); } return; case SDLK_LEFT: if (!repeat && down) { if (shift) { - apply_orientation_transform(im->screen, + apply_orientation_transform(im, SC_ORIENTATION_FLIP_0); } else { - apply_orientation_transform(im->screen, + apply_orientation_transform(im, SC_ORIENTATION_270); } } @@ -459,34 +481,33 @@ sc_input_manager_process_key(struct sc_input_manager *im, case SDLK_RIGHT: if (!repeat && down) { if (shift) { - apply_orientation_transform(im->screen, + apply_orientation_transform(im, SC_ORIENTATION_FLIP_0); } else { - apply_orientation_transform(im->screen, + apply_orientation_transform(im, SC_ORIENTATION_90); } } return; case SDLK_c: - if (controller && !shift && !repeat && down) { - get_device_clipboard(controller, SC_COPY_KEY_COPY); + if (control && !shift && !repeat && down) { + get_device_clipboard(im, SC_COPY_KEY_COPY); } return; case SDLK_x: - if (controller && !shift && !repeat && down) { - get_device_clipboard(controller, SC_COPY_KEY_CUT); + if (control && !shift && !repeat && down) { + get_device_clipboard(im, SC_COPY_KEY_CUT); } return; case SDLK_v: - if (controller && !repeat && down) { + if (control && !repeat && down) { if (shift || im->legacy_paste) { // inject the text as input events - clipboard_paste(controller); + clipboard_paste(im); } else { // store the text in the device clipboard and paste, // without requesting an acknowledgment - set_device_clipboard(controller, true, - SC_SEQUENCE_INVALID); + set_device_clipboard(im, true, SC_SEQUENCE_INVALID); } } return; @@ -507,23 +528,23 @@ sc_input_manager_process_key(struct sc_input_manager *im, return; case SDLK_i: if (!shift && !repeat && down) { - switch_fps_counter_state(&im->screen->fps_counter); + switch_fps_counter_state(im); } return; case SDLK_n: - if (controller && !repeat && down) { + if (control && !repeat && down) { if (shift) { - collapse_panels(controller); + collapse_panels(im); } else if (im->key_repeat == 0) { - expand_notification_panel(controller); + expand_notification_panel(im); } else { - expand_settings_panel(controller); + expand_settings_panel(im); } } return; case SDLK_r: - if (controller && !shift && !repeat && down) { - rotate_device(controller); + if (control && !shift && !repeat && down) { + rotate_device(im); } return; } @@ -531,7 +552,7 @@ sc_input_manager_process_key(struct sc_input_manager *im, return; } - if (!controller) { + if (!control) { return; } @@ -540,7 +561,7 @@ sc_input_manager_process_key(struct sc_input_manager *im, if (im->clipboard_autosync && is_ctrl_v) { if (im->legacy_paste) { // inject the text as input events - clipboard_paste(controller); + clipboard_paste(im); return; } @@ -550,7 +571,7 @@ sc_input_manager_process_key(struct sc_input_manager *im, // Synchronize the computer clipboard to the device clipboard before // sending Ctrl+v, to allow seamless copy-paste. - bool ok = set_device_clipboard(controller, false, sequence); + bool ok = set_device_clipboard(im, false, sequence); if (!ok) { LOGW("Clipboard could not be synchronized, Ctrl+v not injected"); return; @@ -652,7 +673,7 @@ sc_input_manager_process_touch(struct sc_input_manager *im, static void sc_input_manager_process_mouse_button(struct sc_input_manager *im, const SDL_MouseButtonEvent *event) { - struct sc_controller *controller = im->controller; + bool control = im->controller; if (event->which == SDL_TOUCH_MOUSEID) { // simulated from touch events, so it's a duplicate @@ -661,27 +682,27 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im, bool down = event->type == SDL_MOUSEBUTTONDOWN; if (!im->forward_all_clicks) { - if (controller) { + if (control) { enum sc_action action = down ? SC_ACTION_DOWN : SC_ACTION_UP; if (event->button == SDL_BUTTON_X1) { - action_app_switch(controller, action); + action_app_switch(im, action); return; } if (event->button == SDL_BUTTON_X2 && down) { if (event->clicks < 2) { - expand_notification_panel(controller); + expand_notification_panel(im); } else { - expand_settings_panel(controller); + expand_settings_panel(im); } return; } if (event->button == SDL_BUTTON_RIGHT) { - press_back_or_turn_screen_on(controller, action); + press_back_or_turn_screen_on(im, action); return; } if (event->button == SDL_BUTTON_MIDDLE) { - action_home(controller, action); + action_home(im, action); return; } } @@ -704,7 +725,7 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im, // otherwise, send the click event to the device } - if (!controller) { + if (!control) { return; } From 35add3daee0907f4ca4e706c60d1c00a27702a79 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 20 Jan 2024 18:34:33 +0100 Subject: [PATCH 43/79] Accept disabled keyboard or mouse The input manager assumed that if a controller was present, then both a key processor and a mouse processor were present. Remove this assumption, to support disabling keyboard and mouse separately. This prepares the introduction of new command line options --keyboard and --mouse. PR #4473 --- app/src/input_manager.c | 55 ++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/app/src/input_manager.c b/app/src/input_manager.c index 8e7a6402..7186186f 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -52,8 +52,11 @@ is_shortcut_mod(struct sc_input_manager *im, uint16_t sdl_mod) { void sc_input_manager_init(struct sc_input_manager *im, const struct sc_input_manager_params *params) { - assert(!params->controller || (params->kp && params->kp->ops)); - assert(!params->controller || (params->mp && params->mp->ops)); + // A key/mouse processor may not be present if there is no controller + assert((!params->kp && !params->mp) || params->controller); + // A processor must have ops initialized + assert(!params->kp || params->kp->ops); + assert(!params->mp || params->mp->ops); im->controller = params->controller; im->fp = params->fp; @@ -89,7 +92,7 @@ sc_input_manager_init(struct sc_input_manager *im, static void send_keycode(struct sc_input_manager *im, enum android_keycode keycode, enum sc_action action, const char *name) { - assert(im->controller); + assert(im->controller && im->kp); // send DOWN event struct sc_control_msg msg; @@ -146,7 +149,7 @@ action_menu(struct sc_input_manager *im, enum sc_action action) { static void press_back_or_turn_screen_on(struct sc_input_manager *im, enum sc_action action) { - assert(im->controller); + assert(im->controller && im->kp); struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON; @@ -197,7 +200,7 @@ collapse_panels(struct sc_input_manager *im) { static bool get_device_clipboard(struct sc_input_manager *im, enum sc_copy_key copy_key) { - assert(im->controller); + assert(im->controller && im->kp); struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_GET_CLIPBOARD; @@ -214,7 +217,7 @@ get_device_clipboard(struct sc_input_manager *im, enum sc_copy_key copy_key) { static bool set_device_clipboard(struct sc_input_manager *im, bool paste, uint64_t sequence) { - assert(im->controller); + assert(im->controller && im->kp); char *text = SDL_GetClipboardText(); if (!text) { @@ -274,7 +277,7 @@ switch_fps_counter_state(struct sc_input_manager *im) { static void clipboard_paste(struct sc_input_manager *im) { - assert(im->controller); + assert(im->controller && im->kp); char *text = SDL_GetClipboardText(); if (!text) { @@ -412,28 +415,28 @@ sc_input_manager_process_key(struct sc_input_manager *im, enum sc_action action = down ? SC_ACTION_DOWN : SC_ACTION_UP; switch (keycode) { case SDLK_h: - if (control && !shift && !repeat) { + if (im->kp && !shift && !repeat) { action_home(im, action); } return; case SDLK_b: // fall-through case SDLK_BACKSPACE: - if (control && !shift && !repeat) { + if (im->kp && !shift && !repeat) { action_back(im, action); } return; case SDLK_s: - if (control && !shift && !repeat) { + if (im->kp && !shift && !repeat) { action_app_switch(im, action); } return; case SDLK_m: - if (control && !shift && !repeat) { + if (im->kp && !shift && !repeat) { action_menu(im, action); } return; case SDLK_p: - if (control && !shift && !repeat) { + if (im->kp && !shift && !repeat) { action_power(im, action); } return; @@ -451,7 +454,7 @@ sc_input_manager_process_key(struct sc_input_manager *im, apply_orientation_transform(im, SC_ORIENTATION_FLIP_180); } - } else if (control) { + } else if (im->kp) { // forward repeated events action_volume_down(im, action); } @@ -462,7 +465,7 @@ sc_input_manager_process_key(struct sc_input_manager *im, apply_orientation_transform(im, SC_ORIENTATION_FLIP_180); } - } else if (control) { + } else if (im->kp) { // forward repeated events action_volume_up(im, action); } @@ -490,17 +493,17 @@ sc_input_manager_process_key(struct sc_input_manager *im, } return; case SDLK_c: - if (control && !shift && !repeat && down) { + if (im->kp && !shift && !repeat && down) { get_device_clipboard(im, SC_COPY_KEY_COPY); } return; case SDLK_x: - if (control && !shift && !repeat && down) { + if (im->kp && !shift && !repeat && down) { get_device_clipboard(im, SC_COPY_KEY_CUT); } return; case SDLK_v: - if (control && !repeat && down) { + if (im->kp && !repeat && down) { if (shift || im->legacy_paste) { // inject the text as input events clipboard_paste(im); @@ -552,7 +555,7 @@ sc_input_manager_process_key(struct sc_input_manager *im, return; } - if (!control) { + if (!im->kp) { return; } @@ -685,7 +688,7 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im, if (control) { enum sc_action action = down ? SC_ACTION_DOWN : SC_ACTION_UP; - if (event->button == SDL_BUTTON_X1) { + if (im->kp && event->button == SDL_BUTTON_X1) { action_app_switch(im, action); return; } @@ -697,11 +700,11 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im, } return; } - if (event->button == SDL_BUTTON_RIGHT) { + if (im->kp && event->button == SDL_BUTTON_RIGHT) { press_back_or_turn_screen_on(im, action); return; } - if (event->button == SDL_BUTTON_MIDDLE) { + if (im->kp && event->button == SDL_BUTTON_MIDDLE) { action_home(im, action); return; } @@ -725,7 +728,7 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im, // otherwise, send the click event to the device } - if (!control) { + if (!im->mp) { return; } @@ -865,7 +868,7 @@ sc_input_manager_handle_event(struct sc_input_manager *im, bool control = im->controller; switch (event->type) { case SDL_TEXTINPUT: - if (!control) { + if (!im->kp) { break; } sc_input_manager_process_text_input(im, &event->text); @@ -877,13 +880,13 @@ sc_input_manager_handle_event(struct sc_input_manager *im, sc_input_manager_process_key(im, &event->key); break; case SDL_MOUSEMOTION: - if (!control) { + if (!im->mp) { break; } sc_input_manager_process_mouse_motion(im, &event->motion); break; case SDL_MOUSEWHEEL: - if (!control) { + if (!im->mp) { break; } sc_input_manager_process_mouse_wheel(im, &event->wheel); @@ -897,7 +900,7 @@ sc_input_manager_handle_event(struct sc_input_manager *im, case SDL_FINGERMOTION: case SDL_FINGERDOWN: case SDL_FINGERUP: - if (!control) { + if (!im->mp) { break; } sc_input_manager_process_touch(im, &event->tfinger); From ea98d49baed749eea0f3c9a02c9f019e37c85af6 Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:17:35 +0800 Subject: [PATCH 44/79] Introduce --keyboard and --mouse Until now, there was two modes for keyboard and mouse: - event injection using the Android system API (default) - HID/AOA over USB For this reason, the options were exposed as simple flags: - -K or --hid-keyboard to enable physical keyboard simulation (AOA) - -M or --hid-mouse to enable physical mouse simulation (AOA) Replace them by explicit --keyboard and --mouse options, with 3 possible values: - disabled - sdk (default) - aoa This will allow to add a new mode (uhid). PR #4473 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- app/data/bash-completion/scrcpy | 12 ++- app/data/zsh-completion/_scrcpy | 4 +- app/scrcpy.1 | 47 +++++---- app/src/cli.c | 175 ++++++++++++++++++++++++++------ app/src/options.c | 4 +- app/src/options.h | 12 ++- app/src/scrcpy.c | 36 ++++--- app/src/usb/scrcpy_otg.c | 15 ++- 8 files changed, 219 insertions(+), 86 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 78aa539d..b2009c56 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -27,8 +27,8 @@ _scrcpy() { --force-adb-forward --forward-all-clicks -h --help + --keyboard= --kill-adb-on-close - -K --hid-keyboard --legacy-paste --list-camera-sizes --list-cameras @@ -37,8 +37,8 @@ _scrcpy() { --lock-video-orientation --lock-video-orientation= -m --max-size= - -M --hid-mouse --max-fps= + --mouse= -n --no-control -N --no-playback --no-audio @@ -115,6 +115,14 @@ _scrcpy() { COMPREPLY=($(compgen -W 'front back external' -- "$cur")) return ;; + --keyboard) + COMPREPLY=($(compgen -W 'disabled sdk aoa' -- "$cur")) + return + ;; + --mouse) + COMPREPLY=($(compgen -W 'disabled sdk aoa' -- "$cur")) + return + ;; --orientation|--display-orientation) COMPREPLY=($(compgen -W '0 90 180 270 flip0 flip90 flip180 flip270' -- "$cur")) return diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 3c7ca217..a4611632 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -34,8 +34,8 @@ arguments=( '--force-adb-forward[Do not attempt to use \"adb reverse\" to connect to the device]' '--forward-all-clicks[Forward clicks to device]' {-h,--help}'[Print the help]' + '--keyboard[Set the keyboard input mode]:mode:(disabled sdk aoa)' '--kill-adb-on-close[Kill adb when scrcpy terminates]' - {-K,--hid-keyboard}'[Simulate a physical keyboard by using HID over AOAv2]' '--legacy-paste[Inject computer clipboard text as a sequence of key events on Ctrl+v]' '--list-camera-sizes[List the valid camera capture sizes]' '--list-cameras[List cameras available on the device]' @@ -43,8 +43,8 @@ arguments=( '--list-encoders[List video and audio encoders available on the device]' '--lock-video-orientation=[Lock video orientation]:orientation:(unlocked initial 0 90 180 270)' {-m,--max-size=}'[Limit both the width and height of the video to value]' - {-M,--hid-mouse}'[Simulate a physical mouse by using HID over AOAv2]' '--max-fps=[Limit the frame rate of screen capture]' + '--mouse[Set the mouse input mode]:mode:(disabled sdk aoa)' {-n,--no-control}'[Disable device control \(mirror the device in read only\)]' {-N,--no-playback}'[Disable video and audio playback]' '--no-audio[Disable audio forwarding]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index beaa99ab..ed2e620e 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -172,24 +172,26 @@ By default, right-click triggers BACK (or POWER on) and middle-click triggers HO Print this help. .TP -.B \-\-kill\-adb\-on\-close -Kill adb when scrcpy terminates. +.BI "\-\-keyboard " mode +Select how to send keyboard inputs to the device. -.TP -.B \-K, \-\-hid\-keyboard -Simulate a physical keyboard by using HID over AOAv2. +Possible values are "disabled", "sdk" and "aoa": -This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method. - -It may only work over USB. + - "disabled" does not send keyboard inputs to the device. + - "sdk" uses the Android system API to deliver keyboard events to applications. + - "aoa" simulates a physical keyboard using the AOAv2 protocol. It may only work over USB. -The keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly: +For "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly: adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS -However, the option is only available when the HID keyboard is enabled (or a physical keyboard is connected). +This option is only available when the HID keyboard is enabled (or a physical keyboard is connected). -Also see \fB\-\-hid\-mouse\fR. +Also see \fB\-\-mouse\fR. + +.TP +.B \-\-kill\-adb\-on\-close +Kill adb when scrcpy terminates. .TP .B \-\-legacy\-paste @@ -230,20 +232,25 @@ Limit both the width and height of the video to \fIvalue\fR. The other dimension Default is 0 (unlimited). .TP -.B \-M, \-\-hid\-mouse -Simulate a physical mouse by using HID over AOAv2. +.BI "\-\-max\-fps " value +Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions). + +.TP +.BI "\-\-mouse " mode +Select how to send mouse inputs to the device. -In this mode, the computer mouse is captured to control the device directly (relative mouse mode). +Possible values are "disabled", "sdk" and "aoa": -LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer. + - "disabled" does not send mouse inputs to the device. + - "sdk" uses the Android system API to deliver mouse events to applications. + - "aoa" simulates a physical mouse using the AOAv2 protocol. It may only work over USB. -It may only work over USB. +In "aoa" mode, the computer mouse is captured to control the device directly (relative mouse mode). -Also see \fB\-\-hid\-keyboard\fR. +LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer. + +Also see \fB\-\-keyboard\fR. -.TP -.BI "\-\-max\-fps " value -Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions). .TP .B \-n, \-\-no\-control diff --git a/app/src/cli.c b/app/src/cli.c index b2b02ecd..364590a4 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -93,6 +93,8 @@ enum { OPT_DISPLAY_ORIENTATION, OPT_RECORD_ORIENTATION, OPT_ORIENTATION, + OPT_KEYBOARD, + OPT_MOUSE, }; struct sc_option { @@ -358,27 +360,35 @@ static const struct sc_option options[] = { .longopt = "help", .text = "Print this help.", }, + { + .longopt_id = OPT_KEYBOARD, + .longopt = "keyboard", + .argdesc = "mode", + .text = "Select how to send keyboard inputs to the device.\n" + "Possible values are \"disabled\", \"sdk\" and \"aoa\".\n" + "\"disabled\" does not send keyboard inputs to the device.\n" + "\"sdk\" uses the Android system API to deliver keyboard " + "events to applications.\n" + "\"aoa\" simulates a physical keyboard using the AOAv2 " + "protocol. It may only work over USB.\n" + "For \"aoa\", the keyboard layout must be configured (once and " + "for all) on the device, via Settings -> System -> Languages " + "and input -> Physical keyboard. This settings page can be " + "started directly: `adb shell am start -a " + "android.settings.HARD_KEYBOARD_SETTINGS`.\n" + "This option is only available when the HID keyboard is " + "enabled (or a physical keyboard is connected).\n" + "Also see --mouse.", + }, { .longopt_id = OPT_KILL_ADB_ON_CLOSE, .longopt = "kill-adb-on-close", .text = "Kill adb when scrcpy terminates.", }, { + // deprecated .shortopt = 'K', .longopt = "hid-keyboard", - .text = "Simulate a physical keyboard by using HID over AOAv2.\n" - "It provides a better experience for IME users, and allows to " - "generate non-ASCII characters, contrary to the default " - "injection method.\n" - "It may only work over USB.\n" - "The keyboard layout must be configured (once and for all) on " - "the device, via Settings -> System -> Languages and input -> " - "Physical keyboard. This settings page can be started " - "directly: `adb shell am start -a " - "android.settings.HARD_KEYBOARD_SETTINGS`.\n" - "However, the option is only available when the HID keyboard " - "is enabled (or a physical keyboard is connected).\n" - "Also see --hid-mouse.", }, { .longopt_id = OPT_LEGACY_PASTE, @@ -432,15 +442,9 @@ static const struct sc_option options[] = { "Default is 0 (unlimited).", }, { + // deprecated .shortopt = 'M', .longopt = "hid-mouse", - .text = "Simulate a physical mouse by using HID over AOAv2.\n" - "In this mode, the computer mouse is captured to control the " - "device directly (relative mouse mode).\n" - "LAlt, LSuper or RSuper toggle the capture mode, to give " - "control of the mouse back to the computer.\n" - "It may only work over USB.\n" - "Also see --hid-keyboard.", }, { .longopt_id = OPT_MAX_FPS, @@ -449,6 +453,23 @@ static const struct sc_option options[] = { .text = "Limit the frame rate of screen capture (officially supported " "since Android 10, but may work on earlier versions).", }, + { + .longopt_id = OPT_MOUSE, + .longopt = "mouse", + .argdesc = "mode", + .text = "Select how to send mouse inputs to the device.\n" + "Possible values are \"disabled\", \"sdk\" and \"aoa\".\n" + "\"disabled\" does not send mouse inputs to the device.\n" + "\"sdk\" uses the Android system API to deliver mouse events" + "to applications.\n" + "\"aoa\" simulates a physical mouse using the AOAv2 protocol. " + "It may only work over USB.\n" + "In \"aoa\" mode, the computer mouse is captured to control " + "the device directly (relative mouse mode).\n" + "LAlt, LSuper or RSuper toggle the capture mode, to give " + "control of the mouse back to the computer.\n" + "Also see --keyboard.", + }, { .shortopt = 'n', .longopt = "no-control", @@ -543,10 +564,10 @@ static const struct sc_option options[] = { "mirroring is disabled.\n" "LAlt, LSuper or RSuper toggle the mouse capture mode, to give " "control of the mouse back to the computer.\n" - "If any of --hid-keyboard or --hid-mouse is set, only enable " - "keyboard or mouse respectively, otherwise enable both.\n" + "Keyboard and mouse may be disabled separately using" + "--keyboard=disabled and --mouse=disabled.\n" "It may only work over USB.\n" - "See --hid-keyboard and --hid-mouse.", + "See --keyboard and --mouse.", }, { .shortopt = 'p', @@ -1906,6 +1927,58 @@ parse_camera_fps(const char *s, uint16_t *camera_fps) { return true; } +static bool +parse_keyboard(const char *optarg, enum sc_keyboard_input_mode *mode) { + if (!strcmp(optarg, "disabled")) { + *mode = SC_KEYBOARD_INPUT_MODE_DISABLED; + return true; + } + + if (!strcmp(optarg, "sdk")) { + *mode = SC_KEYBOARD_INPUT_MODE_SDK; + return true; + } + + if (!strcmp(optarg, "aoa")) { +#ifdef HAVE_USB + *mode = SC_KEYBOARD_INPUT_MODE_AOA; + return true; +#else + LOGE("--keyboard=aoa is disabled."); + return false; +#endif + } + + LOGE("Unsupported keyboard: %s (expected disabled, sdk or aoa)", optarg); + return false; +} + +static bool +parse_mouse(const char *optarg, enum sc_mouse_input_mode *mode) { + if (!strcmp(optarg, "disabled")) { + *mode = SC_MOUSE_INPUT_MODE_DISABLED; + return true; + } + + if (!strcmp(optarg, "sdk")) { + *mode = SC_MOUSE_INPUT_MODE_SDK; + return true; + } + + if (!strcmp(optarg, "aoa")) { +#ifdef HAVE_USB + *mode = SC_MOUSE_INPUT_MODE_AOA; + return true; +#else + LOGE("--mouse=aoa is disabled."); + return false; +#endif + } + + LOGE("Unsupported mouse: %s (expected disabled, sdk or aoa)", optarg); + return false; +} + static bool parse_time_limit(const char *s, sc_tick *tick) { long value; @@ -1995,12 +2068,19 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], break; case 'K': #ifdef HAVE_USB - opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID; + LOGW("-K/--hid-keyboard is deprecated, use --keyboard=aoa " + "instead."); + opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_AOA; break; #else LOGE("HID over AOA (-K/--hid-keyboard) is disabled."); return false; #endif + case OPT_KEYBOARD: + if (!parse_keyboard(optarg, &opts->keyboard_input_mode)) { + return false; + } + break; case OPT_MAX_FPS: if (!parse_max_fps(optarg, &opts->max_fps)) { return false; @@ -2013,12 +2093,18 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], break; case 'M': #ifdef HAVE_USB - opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_HID; + LOGW("-M/--hid-mouse is deprecated, use --mouse=aoa instead."); + opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_AOA; break; #else LOGE("HID over AOA (-M/--hid-mouse) is disabled."); return false; #endif + case OPT_MOUSE: + if (!parse_mouse(optarg, &opts->mouse_input_mode)) { + return false; + } + break; case OPT_LOCK_VIDEO_ORIENTATION: if (!parse_lock_video_orientation(optarg, &opts->lock_video_orientation)) { @@ -2465,6 +2551,37 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } #endif + if (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AUTO) { + opts->keyboard_input_mode = otg ? SC_KEYBOARD_INPUT_MODE_AOA + : SC_KEYBOARD_INPUT_MODE_SDK; + } + if (opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_AUTO) { + opts->mouse_input_mode = otg ? SC_MOUSE_INPUT_MODE_AOA + : SC_MOUSE_INPUT_MODE_SDK; + } + + if (otg) { + enum sc_keyboard_input_mode kmode = opts->keyboard_input_mode; + if (kmode != SC_KEYBOARD_INPUT_MODE_AOA + && kmode != SC_KEYBOARD_INPUT_MODE_DISABLED) { + LOGE("In OTG mode, --keyboard only supports aoa or disabled."); + return false; + } + + enum sc_mouse_input_mode mmode = opts->mouse_input_mode; + if (mmode != SC_MOUSE_INPUT_MODE_AOA + && mmode != SC_MOUSE_INPUT_MODE_DISABLED) { + LOGE("In OTG mode, --mouse only supports aoa or disabled."); + return false; + } + + if (kmode == SC_KEYBOARD_INPUT_MODE_DISABLED + && mmode == SC_MOUSE_INPUT_MODE_DISABLED) { + LOGE("Could not disable both keyboard and mouse in OTG mode."); + return false; + } + } + if ((opts->tunnel_host || opts->tunnel_port) && !opts->force_adb_forward) { LOGI("Tunnel host/port is set, " "--force-adb-forward automatically enabled."); @@ -2625,12 +2742,12 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } # ifdef _WIN32 - if (!otg && (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID - || opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID)) { + if (!otg && (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA + || opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA)) { LOGE("On Windows, it is not possible to open a USB device already open " "by another process (like adb)."); - LOGE("Therefore, -K/--hid-keyboard and -M/--hid-mouse may only work in " - "OTG mode (--otg)."); + LOGE("Therefore, --keyboard=aoa and --mouse=aoa may only work in OTG" + "mode (--otg)."); return false; } # endif diff --git a/app/src/options.c b/app/src/options.c index a13df585..7a885aa5 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -21,8 +21,8 @@ const struct scrcpy_options scrcpy_options_default = { .video_source = SC_VIDEO_SOURCE_DISPLAY, .audio_source = SC_AUDIO_SOURCE_AUTO, .record_format = SC_RECORD_FORMAT_AUTO, - .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, - .mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT, + .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_AUTO, + .mouse_input_mode = SC_MOUSE_INPUT_MODE_AUTO, .camera_facing = SC_CAMERA_FACING_ANY, .port_range = { .first = DEFAULT_LOCAL_PORT_RANGE_FIRST, diff --git a/app/src/options.h b/app/src/options.h index 11e64fa1..1fb31c1a 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -140,13 +140,17 @@ enum sc_lock_video_orientation { }; enum sc_keyboard_input_mode { - SC_KEYBOARD_INPUT_MODE_INJECT, - SC_KEYBOARD_INPUT_MODE_HID, + SC_KEYBOARD_INPUT_MODE_AUTO, + SC_KEYBOARD_INPUT_MODE_DISABLED, + SC_KEYBOARD_INPUT_MODE_SDK, + SC_KEYBOARD_INPUT_MODE_AOA, }; enum sc_mouse_input_mode { - SC_MOUSE_INPUT_MODE_INJECT, - SC_MOUSE_INPUT_MODE_HID, + SC_MOUSE_INPUT_MODE_AUTO, + SC_MOUSE_INPUT_MODE_DISABLED, + SC_MOUSE_INPUT_MODE_SDK, + SC_MOUSE_INPUT_MODE_AOA, }; enum sc_key_inject_mode { diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index cf2e7e47..24177f15 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -543,11 +543,11 @@ scrcpy(struct scrcpy_options *options) { if (options->control) { #ifdef HAVE_USB - bool use_hid_keyboard = - options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID; - bool use_hid_mouse = - options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID; - if (use_hid_keyboard || use_hid_mouse) { + bool use_aoa_keyboard = + options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA; + bool use_aoa_mouse = + options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA; + if (use_aoa_keyboard || use_aoa_mouse) { bool ok = sc_acksync_init(&s->acksync); if (!ok) { goto end; @@ -590,7 +590,7 @@ scrcpy(struct scrcpy_options *options) { goto aoa_hid_end; } - if (use_hid_keyboard) { + if (use_aoa_keyboard) { if (sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) { hid_keyboard_initialized = true; kp = &s->keyboard_hid.key_processor; @@ -599,7 +599,7 @@ scrcpy(struct scrcpy_options *options) { } } - if (use_hid_mouse) { + if (use_aoa_mouse) { if (sc_hid_mouse_init(&s->mouse_hid, &s->aoa)) { hid_mouse_initialized = true; mp = &s->mouse_hid.mouse_processor; @@ -634,25 +634,23 @@ aoa_hid_end: } } - if (use_hid_keyboard && !hid_keyboard_initialized) { - LOGE("Fallback to default keyboard injection method " - "(-K/--hid-keyboard ignored)"); - options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT; + if (use_aoa_keyboard && !hid_keyboard_initialized) { + LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)"); + options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_SDK; } - if (use_hid_mouse && !hid_mouse_initialized) { - LOGE("Fallback to default mouse injection method " - "(-M/--hid-mouse ignored)"); - options->mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT; + if (use_aoa_mouse && !hid_mouse_initialized) { + LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)"); + options->mouse_input_mode = SC_MOUSE_INPUT_MODE_SDK; } } #else - assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID); - assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID); + assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_AOA); + assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_AOA); #endif // keyboard_input_mode may have been reset if HID mode failed - if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) { + if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_SDK) { sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, options->key_inject_mode, options->forward_key_repeat); @@ -660,7 +658,7 @@ aoa_hid_end: } // mouse_input_mode may have been reset if HID mode failed - if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_INJECT) { + if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_SDK) { sc_mouse_inject_init(&s->mouse_inject, &s->controller); mp = &s->mouse_inject.mouse_processor; } diff --git a/app/src/usb/scrcpy_otg.c b/app/src/usb/scrcpy_otg.c index dfb0b9e9..5955e909 100644 --- a/app/src/usb/scrcpy_otg.c +++ b/app/src/usb/scrcpy_otg.c @@ -117,16 +117,15 @@ scrcpy_otg(struct scrcpy_options *options) { } aoa_initialized = true; + assert(options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA + || options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_DISABLED); + assert(options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA + || options->mouse_input_mode == SC_MOUSE_INPUT_MODE_DISABLED); + bool enable_keyboard = - options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID; + options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA; bool enable_mouse = - options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID; - - // If neither --hid-keyboard or --hid-mouse is passed, enable both - if (!enable_keyboard && !enable_mouse) { - enable_keyboard = true; - enable_mouse = true; - } + options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA; if (enable_keyboard) { ok = sc_hid_keyboard_init(&s->keyboard, &s->aoa); From 48adae1728c6870a88a596ea092f98c76c7586b7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 25 Jan 2024 19:39:35 +0100 Subject: [PATCH 45/79] Fix HID mouse documentation The size of a mouse HID event is 4 bytes. PR #4473 --- app/src/usb/hid_mouse.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/src/usb/hid_mouse.c b/app/src/usb/hid_mouse.c index bab89940..06e2a224 100644 --- a/app/src/usb/hid_mouse.c +++ b/app/src/usb/hid_mouse.c @@ -10,7 +10,8 @@ #define HID_MOUSE_ACCESSORY_ID 2 -// 1 byte for buttons + padding, 1 byte for X position, 1 byte for Y position +// 1 byte for buttons + padding, 1 byte for X position, 1 byte for Y position, +// 1 byte for wheel motion #define HID_MOUSE_EVENT_SIZE 4 /** @@ -90,11 +91,12 @@ static const unsigned char mouse_report_desc[] = { }; /** - * A mouse HID event is 3 bytes long: + * A mouse HID event is 4 bytes long: * * - byte 0: buttons state * - byte 1: relative x motion (signed byte from -127 to 127) * - byte 2: relative y motion (signed byte from -127 to 127) + * - byte 3: wheel motion (-1, 0 or 1) * * 7 6 5 4 3 2 1 0 * +---------------+ @@ -112,7 +114,7 @@ static const unsigned char mouse_report_desc[] = { * +---------------+ * byte 2: |. . . . . . . .| relative y motion * +---------------+ - * byte 3: |. . . . . . . .| wheel motion (-1, 0 or 1) + * byte 3: |. . . . . . . .| wheel motion * +---------------+ * * As an example, here is the report for a motion of (x=5, y=-4) with left From 29ce03e3370d6d79c042e02abaa6666c4930d0ee Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 25 Jan 2024 19:45:21 +0100 Subject: [PATCH 46/79] Rename "buffer" to "data" The variable name is intended to match the parameter name of libusb_control_transfer(). PR #4473 --- app/src/usb/aoa_hid.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index 9bad5296..eb47f415 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -97,10 +97,10 @@ sc_aoa_register_hid(struct sc_aoa *aoa, uint16_t accessory_id, // index (arg1): total length of the HID report descriptor uint16_t value = accessory_id; uint16_t index = report_desc_size; - unsigned char *buffer = NULL; + unsigned char *data = NULL; uint16_t length = 0; int result = libusb_control_transfer(aoa->usb->handle, request_type, - request, value, index, buffer, length, + request, value, index, data, length, DEFAULT_TIMEOUT); if (result < 0) { LOGE("REGISTER_HID: libusb error: %s", libusb_strerror(result)); @@ -130,14 +130,14 @@ sc_aoa_set_hid_report_desc(struct sc_aoa *aoa, uint16_t accessory_id, * See */ // value (arg0): accessory assigned ID for the HID device - // index (arg1): offset of data (buffer) in descriptor + // index (arg1): offset of data in descriptor uint16_t value = accessory_id; uint16_t index = 0; // libusb_control_transfer expects a pointer to non-const - unsigned char *buffer = (unsigned char *) report_desc; + unsigned char *data = (unsigned char *) report_desc; uint16_t length = report_desc_size; int result = libusb_control_transfer(aoa->usb->handle, request_type, - request, value, index, buffer, length, + request, value, index, data, length, DEFAULT_TIMEOUT); if (result < 0) { LOGE("SET_HID_REPORT_DESC: libusb error: %s", libusb_strerror(result)); @@ -177,10 +177,10 @@ sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { // index (arg1): 0 (unused) uint16_t value = event->accessory_id; uint16_t index = 0; - unsigned char *buffer = event->buffer; + unsigned char *data = event->buffer; uint16_t length = event->size; int result = libusb_control_transfer(aoa->usb->handle, request_type, - request, value, index, buffer, length, + request, value, index, data, length, DEFAULT_TIMEOUT); if (result < 0) { LOGE("SEND_HID_EVENT: libusb error: %s", libusb_strerror(result)); @@ -200,10 +200,10 @@ sc_aoa_unregister_hid(struct sc_aoa *aoa, const uint16_t accessory_id) { // index (arg1): 0 uint16_t value = accessory_id; uint16_t index = 0; - unsigned char *buffer = NULL; + unsigned char *data = NULL; uint16_t length = 0; int result = libusb_control_transfer(aoa->usb->handle, request_type, - request, value, index, buffer, length, + request, value, index, data, length, DEFAULT_TIMEOUT); if (result < 0) { LOGE("UNREGISTER_HID: libusb error: %s", libusb_strerror(result)); From ae303b8d07bf84a96d97c7cead9e0a405a5e1482 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 25 Jan 2024 20:01:34 +0100 Subject: [PATCH 47/79] Rename hid event "buffer" to "data" This fields contains the HID event data (there is no "bufferization"). PR #4473 --- app/src/usb/aoa_hid.c | 12 ++++++------ app/src/usb/aoa_hid.h | 4 ++-- app/src/usb/hid_keyboard.c | 24 ++++++++++++------------ app/src/usb/hid_mouse.c | 36 ++++++++++++++++++------------------ 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index eb47f415..1f2f7c79 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -27,7 +27,7 @@ sc_hid_event_log(const struct sc_hid_event *event) { return; } for (unsigned i = 0; i < event->size; ++i) { - snprintf(buffer + i * 3, 4, " %02x", event->buffer[i]); + snprintf(buffer + i * 3, 4, " %02x", event->data[i]); } LOGV("HID Event: [%d]%s", event->accessory_id, buffer); free(buffer); @@ -35,16 +35,16 @@ sc_hid_event_log(const struct sc_hid_event *event) { void sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, - unsigned char *buffer, uint16_t buffer_size) { + unsigned char *data, uint16_t size) { hid_event->accessory_id = accessory_id; - hid_event->buffer = buffer; - hid_event->size = buffer_size; + hid_event->data = data; + hid_event->size = size; hid_event->ack_to_wait = SC_SEQUENCE_INVALID; } void sc_hid_event_destroy(struct sc_hid_event *hid_event) { - free(hid_event->buffer); + free(hid_event->data); } bool @@ -177,7 +177,7 @@ sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { // index (arg1): 0 (unused) uint16_t value = event->accessory_id; uint16_t index = 0; - unsigned char *data = event->buffer; + unsigned char *data = event->data; uint16_t length = event->size; int result = libusb_control_transfer(aoa->usb->handle, request_type, request, value, index, data, length, diff --git a/app/src/usb/aoa_hid.h b/app/src/usb/aoa_hid.h index fb5e1d28..a726938a 100644 --- a/app/src/usb/aoa_hid.h +++ b/app/src/usb/aoa_hid.h @@ -14,7 +14,7 @@ struct sc_hid_event { uint16_t accessory_id; - unsigned char *buffer; + unsigned char *data; uint16_t size; uint64_t ack_to_wait; }; @@ -22,7 +22,7 @@ struct sc_hid_event { // Takes ownership of buffer void sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, - unsigned char *buffer, uint16_t buffer_size); + unsigned char *data, uint16_t size); void sc_hid_event_destroy(struct sc_hid_event *hid_event); diff --git a/app/src/usb/hid_keyboard.c b/app/src/usb/hid_keyboard.c index e717006a..8bd0866a 100644 --- a/app/src/usb/hid_keyboard.c +++ b/app/src/usb/hid_keyboard.c @@ -233,17 +233,17 @@ sdl_keymod_to_hid_modifiers(uint16_t mod) { static bool sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) { - unsigned char *buffer = malloc(HID_KEYBOARD_EVENT_SIZE); - if (!buffer) { + unsigned char *data = malloc(HID_KEYBOARD_EVENT_SIZE); + if (!data) { LOG_OOM(); return false; } - buffer[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE; - buffer[1] = HID_RESERVED; - memset(&buffer[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS); + data[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE; + data[1] = HID_RESERVED; + memset(&data[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS); - sc_hid_event_init(hid_event, HID_KEYBOARD_ACCESSORY_ID, buffer, + sc_hid_event_init(hid_event, HID_KEYBOARD_ACCESSORY_ID, data, HID_KEYBOARD_EVENT_SIZE); return true; } @@ -282,9 +282,9 @@ convert_hid_keyboard_event(struct sc_hid_keyboard *kb, kb->keys[scancode] ? "true" : "false"); } - hid_event->buffer[HID_KEYBOARD_INDEX_MODIFIER] = modifiers; + hid_event->data[HID_KEYBOARD_INDEX_MODIFIER] = modifiers; - unsigned char *keys_buffer = &hid_event->buffer[HID_KEYBOARD_INDEX_KEYS]; + unsigned char *keys_data = &hid_event->data[HID_KEYBOARD_INDEX_KEYS]; // Re-calculate pressed keys every time int keys_pressed_count = 0; for (int i = 0; i < SC_HID_KEYBOARD_KEYS; ++i) { @@ -296,11 +296,11 @@ convert_hid_keyboard_event(struct sc_hid_keyboard *kb, // - Modifiers // - Reserved // - ErrorRollOver * HID_MAX_KEYS - memset(keys_buffer, HID_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS); + memset(keys_data, HID_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS); goto end; } - keys_buffer[keys_pressed_count] = i; + keys_data[keys_pressed_count] = i; ++keys_pressed_count; } } @@ -331,11 +331,11 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t mods_state) { unsigned i = 0; if (capslock) { - hid_event.buffer[HID_KEYBOARD_INDEX_KEYS + i] = SC_SCANCODE_CAPSLOCK; + hid_event.data[HID_KEYBOARD_INDEX_KEYS + i] = SC_SCANCODE_CAPSLOCK; ++i; } if (numlock) { - hid_event.buffer[HID_KEYBOARD_INDEX_KEYS + i] = SC_SCANCODE_NUMLOCK; + hid_event.data[HID_KEYBOARD_INDEX_KEYS + i] = SC_SCANCODE_NUMLOCK; ++i; } diff --git a/app/src/usb/hid_mouse.c b/app/src/usb/hid_mouse.c index 06e2a224..45ae6441 100644 --- a/app/src/usb/hid_mouse.c +++ b/app/src/usb/hid_mouse.c @@ -133,13 +133,13 @@ static const unsigned char mouse_report_desc[] = { static bool sc_hid_mouse_event_init(struct sc_hid_event *hid_event) { - unsigned char *buffer = calloc(1, HID_MOUSE_EVENT_SIZE); - if (!buffer) { + unsigned char *data = calloc(1, HID_MOUSE_EVENT_SIZE); + if (!data) { LOG_OOM(); return false; } - sc_hid_event_init(hid_event, HID_MOUSE_ACCESSORY_ID, buffer, + sc_hid_event_init(hid_event, HID_MOUSE_ACCESSORY_ID, data, HID_MOUSE_EVENT_SIZE); return true; } @@ -175,11 +175,11 @@ sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, return; } - unsigned char *buffer = hid_event.buffer; - buffer[0] = buttons_state_to_hid_buttons(event->buttons_state); - buffer[1] = CLAMP(event->xrel, -127, 127); - buffer[2] = CLAMP(event->yrel, -127, 127); - buffer[3] = 0; // wheel coordinates only used for scrolling + unsigned char *data = hid_event.data; + data[0] = buttons_state_to_hid_buttons(event->buttons_state); + data[1] = CLAMP(event->xrel, -127, 127); + data[2] = CLAMP(event->yrel, -127, 127); + data[3] = 0; // wheel coordinates only used for scrolling if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { sc_hid_event_destroy(&hid_event); @@ -197,11 +197,11 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, return; } - unsigned char *buffer = hid_event.buffer; - buffer[0] = buttons_state_to_hid_buttons(event->buttons_state); - buffer[1] = 0; // no x motion - buffer[2] = 0; // no y motion - buffer[3] = 0; // wheel coordinates only used for scrolling + unsigned char *data = hid_event.data; + data[0] = buttons_state_to_hid_buttons(event->buttons_state); + data[1] = 0; // no x motion + data[2] = 0; // no y motion + data[3] = 0; // wheel coordinates only used for scrolling if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { sc_hid_event_destroy(&hid_event); @@ -219,13 +219,13 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, return; } - unsigned char *buffer = hid_event.buffer; - buffer[0] = 0; // buttons state irrelevant (and unknown) - buffer[1] = 0; // no x motion - buffer[2] = 0; // no y motion + unsigned char *data = hid_event.data; + data[0] = 0; // buttons state irrelevant (and unknown) + data[1] = 0; // no x motion + data[2] = 0; // no y motion // In practice, vscroll is always -1, 0 or 1, but in theory other values // are possible - buffer[3] = CLAMP(event->vscroll, -127, 127); + data[3] = CLAMP(event->vscroll, -127, 127); // Horizontal scrolling ignored if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { From 2d32557fdea3e83dd8ec2a73e122338cbe5d417b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 25 Jan 2024 20:17:37 +0100 Subject: [PATCH 48/79] Embed HID event data In the implementation, an HID event is at most 8 bytes. Embed the data in the HID event structure to avoid allocations and simplify the code. PR #4473 --- app/src/usb/aoa_hid.c | 26 ++------------------------ app/src/usb/aoa_hid.h | 14 ++++---------- app/src/usb/hid_keyboard.c | 28 ++++++++-------------------- app/src/usb/hid_mouse.c | 31 +++++++++---------------------- 4 files changed, 23 insertions(+), 76 deletions(-) diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index 1f2f7c79..5db7ab94 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -33,20 +33,6 @@ sc_hid_event_log(const struct sc_hid_event *event) { free(buffer); } -void -sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, - unsigned char *data, uint16_t size) { - hid_event->accessory_id = accessory_id; - hid_event->data = data; - hid_event->size = size; - hid_event->ack_to_wait = SC_SEQUENCE_INVALID; -} - -void -sc_hid_event_destroy(struct sc_hid_event *hid_event) { - free(hid_event->data); -} - bool sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb, struct sc_acksync *acksync) { @@ -76,12 +62,7 @@ sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb, void sc_aoa_destroy(struct sc_aoa *aoa) { - // Destroy remaining events - while (!sc_vecdeque_is_empty(&aoa->queue)) { - struct sc_hid_event *event = sc_vecdeque_popref(&aoa->queue); - assert(event); - sc_hid_event_destroy(event); - } + sc_vecdeque_destroy(&aoa->queue); sc_cond_destroy(&aoa->event_cond); sc_mutex_destroy(&aoa->mutex); @@ -177,7 +158,7 @@ sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { // index (arg1): 0 (unused) uint16_t value = event->accessory_id; uint16_t index = 0; - unsigned char *data = event->data; + unsigned char *data = (uint8_t *) event->data; // discard const uint16_t length = event->size; int result = libusb_control_transfer(aoa->usb->handle, request_type, request, value, index, data, length, @@ -271,17 +252,14 @@ run_aoa_thread(void *data) { if (result == SC_ACKSYNC_WAIT_TIMEOUT) { LOGW("Ack not received after 500ms, discarding HID event"); - sc_hid_event_destroy(&event); continue; } else if (result == SC_ACKSYNC_WAIT_INTR) { // stopped - sc_hid_event_destroy(&event); break; } } bool ok = sc_aoa_send_hid_event(aoa, &event); - sc_hid_event_destroy(&event); if (!ok) { LOGW("Could not send HID event to USB device"); } diff --git a/app/src/usb/aoa_hid.h b/app/src/usb/aoa_hid.h index a726938a..2cbd1a23 100644 --- a/app/src/usb/aoa_hid.h +++ b/app/src/usb/aoa_hid.h @@ -12,21 +12,15 @@ #include "util/tick.h" #include "util/vecdeque.h" +#define SC_HID_MAX_SIZE 8 + struct sc_hid_event { uint16_t accessory_id; - unsigned char *data; - uint16_t size; + uint8_t data[SC_HID_MAX_SIZE]; + uint8_t size; uint64_t ack_to_wait; }; -// Takes ownership of buffer -void -sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, - unsigned char *data, uint16_t size); - -void -sc_hid_event_destroy(struct sc_hid_event *hid_event); - struct sc_hid_event_queue SC_VECDEQUE(struct sc_hid_event); struct sc_aoa { diff --git a/app/src/usb/hid_keyboard.c b/app/src/usb/hid_keyboard.c index 8bd0866a..dcf56313 100644 --- a/app/src/usb/hid_keyboard.c +++ b/app/src/usb/hid_keyboard.c @@ -231,21 +231,17 @@ sdl_keymod_to_hid_modifiers(uint16_t mod) { return modifiers; } -static bool +static void sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) { - unsigned char *data = malloc(HID_KEYBOARD_EVENT_SIZE); - if (!data) { - LOG_OOM(); - return false; - } + hid_event->accessory_id = HID_KEYBOARD_ACCESSORY_ID; + hid_event->size = HID_KEYBOARD_EVENT_SIZE; + hid_event->ack_to_wait = SC_SEQUENCE_INVALID; + + uint8_t *data = hid_event->data; data[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE; data[1] = HID_RESERVED; memset(&data[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS); - - sc_hid_event_init(hid_event, HID_KEYBOARD_ACCESSORY_ID, data, - HID_KEYBOARD_EVENT_SIZE); - return true; } static inline bool @@ -268,10 +264,7 @@ convert_hid_keyboard_event(struct sc_hid_keyboard *kb, return false; } - if (!sc_hid_keyboard_event_init(hid_event)) { - LOGW("Could not initialize HID keyboard event"); - return false; - } + sc_hid_keyboard_event_init(hid_event); unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->mods_state); @@ -324,10 +317,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t mods_state) { } struct sc_hid_event hid_event; - if (!sc_hid_keyboard_event_init(&hid_event)) { - LOGW("Could not initialize HID keyboard event"); - return false; - } + sc_hid_keyboard_event_init(&hid_event); unsigned i = 0; if (capslock) { @@ -340,7 +330,6 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t mods_state) { } if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { - sc_hid_event_destroy(&hid_event); LOGW("Could not request HID event (mod lock state)"); return false; } @@ -382,7 +371,6 @@ sc_key_processor_process_key(struct sc_key_processor *kp, } if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { - sc_hid_event_destroy(&hid_event); LOGW("Could not request HID event (key)"); } } diff --git a/app/src/usb/hid_mouse.c b/app/src/usb/hid_mouse.c index 45ae6441..a47534c1 100644 --- a/app/src/usb/hid_mouse.c +++ b/app/src/usb/hid_mouse.c @@ -131,17 +131,13 @@ static const unsigned char mouse_report_desc[] = { * +---------------+ */ -static bool +static void sc_hid_mouse_event_init(struct sc_hid_event *hid_event) { - unsigned char *data = calloc(1, HID_MOUSE_EVENT_SIZE); - if (!data) { - LOG_OOM(); - return false; - } - - sc_hid_event_init(hid_event, HID_MOUSE_ACCESSORY_ID, data, - HID_MOUSE_EVENT_SIZE); - return true; + hid_event->accessory_id = HID_MOUSE_ACCESSORY_ID; + hid_event->size = HID_MOUSE_EVENT_SIZE; + hid_event->ack_to_wait = SC_SEQUENCE_INVALID; + // Leave hid_event->data uninitialized, it will be fully initialized by + // callers } static unsigned char @@ -171,9 +167,7 @@ sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, struct sc_hid_mouse *mouse = DOWNCAST(mp); struct sc_hid_event hid_event; - if (!sc_hid_mouse_event_init(&hid_event)) { - return; - } + sc_hid_mouse_event_init(&hid_event); unsigned char *data = hid_event.data; data[0] = buttons_state_to_hid_buttons(event->buttons_state); @@ -182,7 +176,6 @@ sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, data[3] = 0; // wheel coordinates only used for scrolling if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { - sc_hid_event_destroy(&hid_event); LOGW("Could not request HID event (mouse motion)"); } } @@ -193,9 +186,7 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, struct sc_hid_mouse *mouse = DOWNCAST(mp); struct sc_hid_event hid_event; - if (!sc_hid_mouse_event_init(&hid_event)) { - return; - } + sc_hid_mouse_event_init(&hid_event); unsigned char *data = hid_event.data; data[0] = buttons_state_to_hid_buttons(event->buttons_state); @@ -204,7 +195,6 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, data[3] = 0; // wheel coordinates only used for scrolling if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { - sc_hid_event_destroy(&hid_event); LOGW("Could not request HID event (mouse click)"); } } @@ -215,9 +205,7 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, struct sc_hid_mouse *mouse = DOWNCAST(mp); struct sc_hid_event hid_event; - if (!sc_hid_mouse_event_init(&hid_event)) { - return; - } + sc_hid_mouse_event_init(&hid_event); unsigned char *data = hid_event.data; data[0] = 0; // buttons state irrelevant (and unknown) @@ -229,7 +217,6 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, // Horizontal scrolling ignored if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { - sc_hid_event_destroy(&hid_event); LOGW("Could not request HID event (mouse scroll)"); } } From f2d62031561b4d1660c2c5bb3910e7a723d317f7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 25 Jan 2024 20:32:37 +0100 Subject: [PATCH 49/79] Extract HID events struct An event contained several fields: - the accessory id - the HID event data - a field ack_to_wait specific to the AOA implementation. Extract the HID event part to prepare the factorization of HID event creation. PR #4473 --- app/src/hid/hid_event.h | 15 +++++++++++++++ app/src/usb/aoa_hid.c | 34 ++++++++++++++++++++++------------ app/src/usb/aoa_hid.h | 22 ++++++++++++++++------ app/src/usb/hid_keyboard.c | 21 ++++++++++----------- app/src/usb/hid_mouse.c | 11 ++++++----- 5 files changed, 69 insertions(+), 34 deletions(-) create mode 100644 app/src/hid/hid_event.h diff --git a/app/src/hid/hid_event.h b/app/src/hid/hid_event.h new file mode 100644 index 00000000..e17f8569 --- /dev/null +++ b/app/src/hid/hid_event.h @@ -0,0 +1,15 @@ +#ifndef SC_HID_EVENT_H +#define SC_HID_EVENT_H + +#include "common.h" + +#include + +#define SC_HID_MAX_SIZE 8 + +struct sc_hid_event { + uint8_t data[SC_HID_MAX_SIZE]; + uint8_t size; +}; + +#endif diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index 5db7ab94..d6b418a0 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -14,10 +14,10 @@ #define DEFAULT_TIMEOUT 1000 -#define SC_HID_EVENT_QUEUE_MAX 64 +#define SC_AOA_EVENT_QUEUE_MAX 64 static void -sc_hid_event_log(const struct sc_hid_event *event) { +sc_hid_event_log(uint16_t accessory_id, const struct sc_hid_event *event) { // HID Event: [00] FF FF FF FF... assert(event->size); unsigned buffer_size = event->size * 3 + 1; @@ -29,7 +29,7 @@ sc_hid_event_log(const struct sc_hid_event *event) { for (unsigned i = 0; i < event->size; ++i) { snprintf(buffer + i * 3, 4, " %02x", event->data[i]); } - LOGV("HID Event: [%d]%s", event->accessory_id, buffer); + LOGV("HID Event: [%d]%s", accessory_id, buffer); free(buffer); } @@ -38,7 +38,7 @@ sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb, struct sc_acksync *acksync) { sc_vecdeque_init(&aoa->queue); - if (!sc_vecdeque_reserve(&aoa->queue, SC_HID_EVENT_QUEUE_MAX)) { + if (!sc_vecdeque_reserve(&aoa->queue, SC_AOA_EVENT_QUEUE_MAX)) { return false; } @@ -150,13 +150,14 @@ sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id, } static bool -sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { +sc_aoa_send_hid_event(struct sc_aoa *aoa, uint16_t accessory_id, + const struct sc_hid_event *event) { uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; uint8_t request = ACCESSORY_SEND_HID_EVENT; // // value (arg0): accessory assigned ID for the HID device // index (arg1): 0 (unused) - uint16_t value = event->accessory_id; + uint16_t value = accessory_id; uint16_t index = 0; unsigned char *data = (uint8_t *) event->data; // discard const uint16_t length = event->size; @@ -173,7 +174,7 @@ sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { } bool -sc_aoa_unregister_hid(struct sc_aoa *aoa, const uint16_t accessory_id) { +sc_aoa_unregister_hid(struct sc_aoa *aoa, uint16_t accessory_id) { uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; uint8_t request = ACCESSORY_UNREGISTER_HID; // @@ -196,16 +197,25 @@ sc_aoa_unregister_hid(struct sc_aoa *aoa, const uint16_t accessory_id) { } bool -sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { +sc_aoa_push_hid_event_with_ack_to_wait(struct sc_aoa *aoa, + uint16_t accessory_id, + const struct sc_hid_event *event, + uint64_t ack_to_wait) { if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) { - sc_hid_event_log(event); + sc_hid_event_log(accessory_id, event); } sc_mutex_lock(&aoa->mutex); bool full = sc_vecdeque_is_full(&aoa->queue); if (!full) { bool was_empty = sc_vecdeque_is_empty(&aoa->queue); - sc_vecdeque_push_noresize(&aoa->queue, *event); + + struct sc_aoa_event *aoa_event = + sc_vecdeque_push_hole_noresize(&aoa->queue); + aoa_event->hid = *event; + aoa_event->accessory_id = accessory_id; + aoa_event->ack_to_wait = ack_to_wait; + if (was_empty) { sc_cond_signal(&aoa->event_cond); } @@ -233,7 +243,7 @@ run_aoa_thread(void *data) { } assert(!sc_vecdeque_is_empty(&aoa->queue)); - struct sc_hid_event event = sc_vecdeque_pop(&aoa->queue); + struct sc_aoa_event event = sc_vecdeque_pop(&aoa->queue); uint64_t ack_to_wait = event.ack_to_wait; sc_mutex_unlock(&aoa->mutex); @@ -259,7 +269,7 @@ run_aoa_thread(void *data) { } } - bool ok = sc_aoa_send_hid_event(aoa, &event); + bool ok = sc_aoa_send_hid_event(aoa, event.accessory_id, &event.hid); if (!ok) { LOGW("Could not send HID event to USB device"); } diff --git a/app/src/usb/aoa_hid.h b/app/src/usb/aoa_hid.h index 2cbd1a23..33a1f136 100644 --- a/app/src/usb/aoa_hid.h +++ b/app/src/usb/aoa_hid.h @@ -6,6 +6,7 @@ #include +#include "hid/hid_event.h" #include "usb.h" #include "util/acksync.h" #include "util/thread.h" @@ -14,14 +15,13 @@ #define SC_HID_MAX_SIZE 8 -struct sc_hid_event { +struct sc_aoa_event { + struct sc_hid_event hid; uint16_t accessory_id; - uint8_t data[SC_HID_MAX_SIZE]; - uint8_t size; uint64_t ack_to_wait; }; -struct sc_hid_event_queue SC_VECDEQUE(struct sc_hid_event); +struct sc_aoa_event_queue SC_VECDEQUE(struct sc_aoa_event); struct sc_aoa { struct sc_usb *usb; @@ -29,7 +29,7 @@ struct sc_aoa { sc_mutex mutex; sc_cond event_cond; bool stopped; - struct sc_hid_event_queue queue; + struct sc_aoa_event_queue queue; struct sc_acksync *acksync; }; @@ -57,6 +57,16 @@ bool sc_aoa_unregister_hid(struct sc_aoa *aoa, uint16_t accessory_id); bool -sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event); +sc_aoa_push_hid_event_with_ack_to_wait(struct sc_aoa *aoa, + uint16_t accessory_id, + const struct sc_hid_event *event, + uint64_t ack_to_wait); + +static inline bool +sc_aoa_push_hid_event(struct sc_aoa *aoa, uint16_t accessory_id, + const struct sc_hid_event *event) { + return sc_aoa_push_hid_event_with_ack_to_wait(aoa, accessory_id, event, + SC_SEQUENCE_INVALID); +} #endif diff --git a/app/src/usb/hid_keyboard.c b/app/src/usb/hid_keyboard.c index dcf56313..9b87a27a 100644 --- a/app/src/usb/hid_keyboard.c +++ b/app/src/usb/hid_keyboard.c @@ -233,9 +233,7 @@ sdl_keymod_to_hid_modifiers(uint16_t mod) { static void sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) { - hid_event->accessory_id = HID_KEYBOARD_ACCESSORY_ID; hid_event->size = HID_KEYBOARD_EVENT_SIZE; - hid_event->ack_to_wait = SC_SEQUENCE_INVALID; uint8_t *data = hid_event->data; @@ -329,7 +327,8 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t mods_state) { ++i; } - if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { + if (!sc_aoa_push_hid_event(kb->aoa, HID_KEYBOARD_ACCESSORY_ID, + &hid_event)) { LOGW("Could not request HID event (mod lock state)"); return false; } @@ -362,15 +361,15 @@ sc_key_processor_process_key(struct sc_key_processor *kp, } } - if (ack_to_wait) { - // Ctrl+v is pressed, so clipboard synchronization has been - // requested. Wait until clipboard synchronization is acknowledged - // by the server, otherwise it could paste the old clipboard - // content. - hid_event.ack_to_wait = ack_to_wait; - } + // If ack_to_wait is != SC_SEQUENCE_INVALID, then Ctrl+v is pressed, so + // clipboard synchronization has been requested. Wait until clipboard + // synchronization is acknowledged by the server, otherwise it could + // paste the old clipboard content. - if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { + if (!sc_aoa_push_hid_event_with_ack_to_wait(kb->aoa, + HID_KEYBOARD_ACCESSORY_ID, + &hid_event, + ack_to_wait)) { LOGW("Could not request HID event (key)"); } } diff --git a/app/src/usb/hid_mouse.c b/app/src/usb/hid_mouse.c index a47534c1..de961265 100644 --- a/app/src/usb/hid_mouse.c +++ b/app/src/usb/hid_mouse.c @@ -133,9 +133,7 @@ static const unsigned char mouse_report_desc[] = { static void sc_hid_mouse_event_init(struct sc_hid_event *hid_event) { - hid_event->accessory_id = HID_MOUSE_ACCESSORY_ID; hid_event->size = HID_MOUSE_EVENT_SIZE; - hid_event->ack_to_wait = SC_SEQUENCE_INVALID; // Leave hid_event->data uninitialized, it will be fully initialized by // callers } @@ -175,7 +173,8 @@ sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, data[2] = CLAMP(event->yrel, -127, 127); data[3] = 0; // wheel coordinates only used for scrolling - if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { + if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, + &hid_event)) { LOGW("Could not request HID event (mouse motion)"); } } @@ -194,7 +193,8 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, data[2] = 0; // no y motion data[3] = 0; // wheel coordinates only used for scrolling - if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { + if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, + &hid_event)) { LOGW("Could not request HID event (mouse click)"); } } @@ -216,7 +216,8 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, data[3] = CLAMP(event->vscroll, -127, 127); // Horizontal scrolling ignored - if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { + if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, + &hid_event)) { LOGW("Could not request HID event (mouse scroll)"); } } From 91485e2863603732348d86a4ed06cafa20d1225f Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 25 Jan 2024 22:57:02 +0100 Subject: [PATCH 50/79] Extract keyboard HID handling Split the keyboard implementation using AOA and the code handling HID events, so that HID events can be reused for another protocol (UHID). PR #4473 --- app/meson.build | 3 +- app/src/{usb => hid}/hid_keyboard.c | 225 +++++++++------------------- app/src/{usb => hid}/hid_keyboard.h | 24 +-- app/src/scrcpy.c | 32 ++-- app/src/usb/keyboard_aoa.c | 109 ++++++++++++++ app/src/usb/keyboard_aoa.h | 27 ++++ app/src/usb/scrcpy_otg.c | 8 +- app/src/usb/screen_otg.h | 6 +- 8 files changed, 245 insertions(+), 189 deletions(-) rename app/src/{usb => hid}/hid_keyboard.c (56%) rename app/src/{usb => hid}/hid_keyboard.h (68%) create mode 100644 app/src/usb/keyboard_aoa.c create mode 100644 app/src/usb/keyboard_aoa.h diff --git a/app/meson.build b/app/meson.build index caf5ee5c..6d572b7b 100644 --- a/app/meson.build +++ b/app/meson.build @@ -31,6 +31,7 @@ src = [ 'src/screen.c', 'src/server.c', 'src/version.c', + 'src/hid/hid_keyboard.c', 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/util/acksync.c', @@ -88,7 +89,7 @@ usb_support = get_option('usb') if usb_support src += [ 'src/usb/aoa_hid.c', - 'src/usb/hid_keyboard.c', + 'src/usb/keyboard_aoa.c', 'src/usb/hid_mouse.c', 'src/usb/scrcpy_otg.c', 'src/usb/screen_otg.c', diff --git a/app/src/usb/hid_keyboard.c b/app/src/hid/hid_keyboard.c similarity index 56% rename from app/src/usb/hid_keyboard.c rename to app/src/hid/hid_keyboard.c index 9b87a27a..f3001df4 100644 --- a/app/src/usb/hid_keyboard.c +++ b/app/src/hid/hid_keyboard.c @@ -1,40 +1,34 @@ #include "hid_keyboard.h" -#include +#include -#include "input_events.h" #include "util/log.h" -/** Downcast key processor to hid_keyboard */ -#define DOWNCAST(KP) container_of(KP, struct sc_hid_keyboard, key_processor) +#define SC_HID_MOD_NONE 0x00 +#define SC_HID_MOD_LEFT_CONTROL (1 << 0) +#define SC_HID_MOD_LEFT_SHIFT (1 << 1) +#define SC_HID_MOD_LEFT_ALT (1 << 2) +#define SC_HID_MOD_LEFT_GUI (1 << 3) +#define SC_HID_MOD_RIGHT_CONTROL (1 << 4) +#define SC_HID_MOD_RIGHT_SHIFT (1 << 5) +#define SC_HID_MOD_RIGHT_ALT (1 << 6) +#define SC_HID_MOD_RIGHT_GUI (1 << 7) -#define HID_KEYBOARD_ACCESSORY_ID 1 - -#define HID_MODIFIER_NONE 0x00 -#define HID_MODIFIER_LEFT_CONTROL (1 << 0) -#define HID_MODIFIER_LEFT_SHIFT (1 << 1) -#define HID_MODIFIER_LEFT_ALT (1 << 2) -#define HID_MODIFIER_LEFT_GUI (1 << 3) -#define HID_MODIFIER_RIGHT_CONTROL (1 << 4) -#define HID_MODIFIER_RIGHT_SHIFT (1 << 5) -#define HID_MODIFIER_RIGHT_ALT (1 << 6) -#define HID_MODIFIER_RIGHT_GUI (1 << 7) - -#define HID_KEYBOARD_INDEX_MODIFIER 0 -#define HID_KEYBOARD_INDEX_KEYS 2 +#define SC_HID_KEYBOARD_INDEX_MODS 0 +#define SC_HID_KEYBOARD_INDEX_KEYS 2 // USB HID protocol says 6 keys in an event is the requirement for BIOS // keyboard support, though OS could support more keys via modifying the report // desc. 6 should be enough for scrcpy. -#define HID_KEYBOARD_MAX_KEYS 6 -#define HID_KEYBOARD_EVENT_SIZE \ - (HID_KEYBOARD_INDEX_KEYS + HID_KEYBOARD_MAX_KEYS) +#define SC_HID_KEYBOARD_MAX_KEYS 6 +#define SC_HID_KEYBOARD_EVENT_SIZE \ + (SC_HID_KEYBOARD_INDEX_KEYS + SC_HID_KEYBOARD_MAX_KEYS) -#define HID_RESERVED 0x00 -#define HID_ERROR_ROLL_OVER 0x01 +#define SC_HID_RESERVED 0x00 +#define SC_HID_ERROR_ROLL_OVER 0x01 /** - * For HID over AOAv2, only report descriptor is needed. + * For HID, only report descriptor is needed. * * The specification is available here: * @@ -53,7 +47,7 @@ * * (change vid:pid' to your device's vendor ID and product ID). */ -static const unsigned char keyboard_report_desc[] = { +const uint8_t SC_HID_KEYBOARD_REPORT_DESC[] = { // Usage Page (Generic Desktop) 0x05, 0x01, // Usage (Keyboard) @@ -119,7 +113,7 @@ static const unsigned char keyboard_report_desc[] = { // Report Size (8) 0x75, 0x08, // Report Count (6) - 0x95, HID_KEYBOARD_MAX_KEYS, + 0x95, SC_HID_KEYBOARD_MAX_KEYS, // Input (Data, Array): Keys 0x81, 0x00, @@ -127,6 +121,9 @@ static const unsigned char keyboard_report_desc[] = { 0xC0 }; +const size_t SC_HID_KEYBOARD_REPORT_DESC_LEN = + sizeof(SC_HID_KEYBOARD_REPORT_DESC); + /** * A keyboard HID event is 8 bytes long: * @@ -201,45 +198,50 @@ static const unsigned char keyboard_report_desc[] = { * +---------------+ */ -static unsigned char -sdl_keymod_to_hid_modifiers(uint16_t mod) { - unsigned char modifiers = HID_MODIFIER_NONE; +static void +sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) { + hid_event->size = SC_HID_KEYBOARD_EVENT_SIZE; + + uint8_t *data = hid_event->data; + + data[SC_HID_KEYBOARD_INDEX_MODS] = SC_HID_MOD_NONE; + data[1] = SC_HID_RESERVED; + memset(&data[SC_HID_KEYBOARD_INDEX_KEYS], 0, SC_HID_KEYBOARD_MAX_KEYS); +} + +static uint16_t +sc_hid_mod_from_sdl_keymod(uint16_t mod) { + uint16_t mods = SC_HID_MOD_NONE; if (mod & SC_MOD_LCTRL) { - modifiers |= HID_MODIFIER_LEFT_CONTROL; + mods |= SC_HID_MOD_LEFT_CONTROL; } if (mod & SC_MOD_LSHIFT) { - modifiers |= HID_MODIFIER_LEFT_SHIFT; + mods |= SC_HID_MOD_LEFT_SHIFT; } if (mod & SC_MOD_LALT) { - modifiers |= HID_MODIFIER_LEFT_ALT; + mods |= SC_HID_MOD_LEFT_ALT; } if (mod & SC_MOD_LGUI) { - modifiers |= HID_MODIFIER_LEFT_GUI; + mods |= SC_HID_MOD_LEFT_GUI; } if (mod & SC_MOD_RCTRL) { - modifiers |= HID_MODIFIER_RIGHT_CONTROL; + mods |= SC_HID_MOD_RIGHT_CONTROL; } if (mod & SC_MOD_RSHIFT) { - modifiers |= HID_MODIFIER_RIGHT_SHIFT; + mods |= SC_HID_MOD_RIGHT_SHIFT; } if (mod & SC_MOD_RALT) { - modifiers |= HID_MODIFIER_RIGHT_ALT; + mods |= SC_HID_MOD_RIGHT_ALT; } if (mod & SC_MOD_RGUI) { - modifiers |= HID_MODIFIER_RIGHT_GUI; + mods |= SC_HID_MOD_RIGHT_GUI; } - return modifiers; + return mods; } -static void -sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) { - hid_event->size = HID_KEYBOARD_EVENT_SIZE; - - uint8_t *data = hid_event->data; - - data[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE; - data[1] = HID_RESERVED; - memset(&data[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS); +void +sc_hid_keyboard_init(struct sc_hid_keyboard *hid) { + memset(hid->keys, false, SC_HID_KEYBOARD_KEYS); } static inline bool @@ -247,10 +249,10 @@ scancode_is_modifier(enum sc_scancode scancode) { return scancode >= SC_SCANCODE_LCTRL && scancode <= SC_SCANCODE_RGUI; } -static bool -convert_hid_keyboard_event(struct sc_hid_keyboard *kb, - struct sc_hid_event *hid_event, - const struct sc_key_event *event) { +bool +sc_hid_keyboard_event_from_key(struct sc_hid_keyboard *hid, + struct sc_hid_event *hid_event, + const struct sc_key_event *event) { enum sc_scancode scancode = event->scancode; assert(scancode >= 0); @@ -264,30 +266,31 @@ convert_hid_keyboard_event(struct sc_hid_keyboard *kb, sc_hid_keyboard_event_init(hid_event); - unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->mods_state); + uint16_t mods = sc_hid_mod_from_sdl_keymod(event->mods_state); if (scancode < SC_HID_KEYBOARD_KEYS) { // Pressed is true and released is false - kb->keys[scancode] = (event->action == SC_ACTION_DOWN); + hid->keys[scancode] = (event->action == SC_ACTION_DOWN); LOGV("keys[%02x] = %s", scancode, - kb->keys[scancode] ? "true" : "false"); + hid->keys[scancode] ? "true" : "false"); } - hid_event->data[HID_KEYBOARD_INDEX_MODIFIER] = modifiers; + hid_event->data[SC_HID_KEYBOARD_INDEX_MODS] = mods; - unsigned char *keys_data = &hid_event->data[HID_KEYBOARD_INDEX_KEYS]; + uint8_t *keys_data = &hid_event->data[SC_HID_KEYBOARD_INDEX_KEYS]; // Re-calculate pressed keys every time int keys_pressed_count = 0; for (int i = 0; i < SC_HID_KEYBOARD_KEYS; ++i) { - if (kb->keys[i]) { + if (hid->keys[i]) { // USB HID protocol says that if keys exceeds report count, a // phantom state should be reported - if (keys_pressed_count >= HID_KEYBOARD_MAX_KEYS) { + if (keys_pressed_count >= SC_HID_KEYBOARD_MAX_KEYS) { // Phantom state: // - Modifiers // - Reserved // - ErrorRollOver * HID_MAX_KEYS - memset(keys_data, HID_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS); + memset(keys_data, SC_HID_ERROR_ROLL_OVER, + SC_HID_KEYBOARD_MAX_KEYS); goto end; } @@ -299,120 +302,32 @@ convert_hid_keyboard_event(struct sc_hid_keyboard *kb, end: LOGV("hid keyboard: key %-4s scancode=%02x (%u) mod=%02x", event->action == SC_ACTION_DOWN ? "down" : "up", event->scancode, - event->scancode, modifiers); + event->scancode, mods); return true; } - -static bool -push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t mods_state) { +bool +sc_hid_keyboard_event_from_mods(struct sc_hid_event *event, + uint16_t mods_state) { bool capslock = mods_state & SC_MOD_CAPS; bool numlock = mods_state & SC_MOD_NUM; if (!capslock && !numlock) { // Nothing to do - return true; + return false; } - struct sc_hid_event hid_event; - sc_hid_keyboard_event_init(&hid_event); + sc_hid_keyboard_event_init(event); unsigned i = 0; if (capslock) { - hid_event.data[HID_KEYBOARD_INDEX_KEYS + i] = SC_SCANCODE_CAPSLOCK; + event->data[SC_HID_KEYBOARD_INDEX_KEYS + i] = SC_SCANCODE_CAPSLOCK; ++i; } if (numlock) { - hid_event.data[HID_KEYBOARD_INDEX_KEYS + i] = SC_SCANCODE_NUMLOCK; + event->data[SC_HID_KEYBOARD_INDEX_KEYS + i] = SC_SCANCODE_NUMLOCK; ++i; } - if (!sc_aoa_push_hid_event(kb->aoa, HID_KEYBOARD_ACCESSORY_ID, - &hid_event)) { - LOGW("Could not request HID event (mod lock state)"); - return false; - } - - LOGD("HID keyboard state synchronized"); - return true; } - -static void -sc_key_processor_process_key(struct sc_key_processor *kp, - const struct sc_key_event *event, - uint64_t ack_to_wait) { - if (event->repeat) { - // In USB HID protocol, key repeat is handled by the host (Android), so - // just ignore key repeat here. - return; - } - - struct sc_hid_keyboard *kb = DOWNCAST(kp); - - struct sc_hid_event hid_event; - // Not all keys are supported, just ignore unsupported keys - if (convert_hid_keyboard_event(kb, &hid_event, event)) { - if (!kb->mod_lock_synchronized) { - // Inject CAPSLOCK and/or NUMLOCK if necessary to synchronize - // keyboard state - if (push_mod_lock_state(kb, event->mods_state)) { - kb->mod_lock_synchronized = true; - } - } - - // If ack_to_wait is != SC_SEQUENCE_INVALID, then Ctrl+v is pressed, so - // clipboard synchronization has been requested. Wait until clipboard - // synchronization is acknowledged by the server, otherwise it could - // paste the old clipboard content. - - if (!sc_aoa_push_hid_event_with_ack_to_wait(kb->aoa, - HID_KEYBOARD_ACCESSORY_ID, - &hid_event, - ack_to_wait)) { - LOGW("Could not request HID event (key)"); - } - } -} - -bool -sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) { - kb->aoa = aoa; - - bool ok = sc_aoa_setup_hid(aoa, HID_KEYBOARD_ACCESSORY_ID, - keyboard_report_desc, - ARRAY_LEN(keyboard_report_desc)); - if (!ok) { - LOGW("Register HID keyboard failed"); - return false; - } - - // Reset all states - memset(kb->keys, false, SC_HID_KEYBOARD_KEYS); - - kb->mod_lock_synchronized = false; - - static const struct sc_key_processor_ops ops = { - .process_key = sc_key_processor_process_key, - // Never forward text input via HID (all the keys are injected - // separately) - .process_text = NULL, - }; - - // Clipboard synchronization is requested over the control socket, while HID - // events are sent over AOA, so it must wait for clipboard synchronization - // to be acknowledged by the device before injecting Ctrl+v. - kb->key_processor.async_paste = true; - kb->key_processor.ops = &ops; - - return true; -} - -void -sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb) { - // Unregister HID keyboard so the soft keyboard shows again on Android - bool ok = sc_aoa_unregister_hid(kb->aoa, HID_KEYBOARD_ACCESSORY_ID); - if (!ok) { - LOGW("Could not unregister HID keyboard"); - } -} diff --git a/app/src/usb/hid_keyboard.h b/app/src/hid/hid_keyboard.h similarity index 68% rename from app/src/usb/hid_keyboard.h rename to app/src/hid/hid_keyboard.h index 7173a898..ddd2cc91 100644 --- a/app/src/usb/hid_keyboard.h +++ b/app/src/hid/hid_keyboard.h @@ -5,8 +5,8 @@ #include -#include "aoa_hid.h" -#include "trait/key_processor.h" +#include "hid/hid_event.h" +#include "input_events.h" // See "SDL2/SDL_scancode.h". // Maybe SDL_Keycode is used by most people, but SDL_Scancode is taken from USB @@ -14,6 +14,9 @@ // 0x65 is Application, typically AT-101 Keyboard ends here. #define SC_HID_KEYBOARD_KEYS 0x66 +extern const uint8_t SC_HID_KEYBOARD_REPORT_DESC[]; +extern const size_t SC_HID_KEYBOARD_REPORT_DESC_LEN; + /** * HID keyboard events are sequence-based, every time keyboard state changes * it sends an array of currently pressed keys, the host is responsible for @@ -27,18 +30,19 @@ * phantom state. */ struct sc_hid_keyboard { - struct sc_key_processor key_processor; // key processor trait - - struct sc_aoa *aoa; bool keys[SC_HID_KEYBOARD_KEYS]; - - bool mod_lock_synchronized; }; +void +sc_hid_keyboard_init(struct sc_hid_keyboard *hid); + bool -sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa); +sc_hid_keyboard_event_from_key(struct sc_hid_keyboard *hid, + struct sc_hid_event *hid_event, + const struct sc_key_event *event); -void -sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb); +bool +sc_hid_keyboard_event_from_mods(struct sc_hid_event *event, + uint16_t mods_state); #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 24177f15..1d5e67dc 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -27,7 +27,7 @@ #include "server.h" #ifdef HAVE_USB # include "usb/aoa_hid.h" -# include "usb/hid_keyboard.h" +# include "usb/keyboard_aoa.h" # include "usb/hid_mouse.h" # include "usb/usb.h" #endif @@ -65,7 +65,7 @@ struct scrcpy { union { struct sc_keyboard_inject keyboard_inject; #ifdef HAVE_USB - struct sc_hid_keyboard keyboard_hid; + struct sc_keyboard_aoa keyboard_aoa; #endif }; union { @@ -330,7 +330,7 @@ scrcpy(struct scrcpy_options *options) { bool audio_demuxer_started = false; #ifdef HAVE_USB bool aoa_hid_initialized = false; - bool hid_keyboard_initialized = false; + bool keyboard_aoa_initialized = false; bool hid_mouse_initialized = false; #endif bool controller_initialized = false; @@ -543,11 +543,11 @@ scrcpy(struct scrcpy_options *options) { if (options->control) { #ifdef HAVE_USB - bool use_aoa_keyboard = + bool use_keyboard_aoa = options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA; bool use_aoa_mouse = options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA; - if (use_aoa_keyboard || use_aoa_mouse) { + if (use_keyboard_aoa || use_aoa_mouse) { bool ok = sc_acksync_init(&s->acksync); if (!ok) { goto end; @@ -590,10 +590,10 @@ scrcpy(struct scrcpy_options *options) { goto aoa_hid_end; } - if (use_aoa_keyboard) { - if (sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) { - hid_keyboard_initialized = true; - kp = &s->keyboard_hid.key_processor; + if (use_keyboard_aoa) { + if (sc_keyboard_aoa_init(&s->keyboard_aoa, &s->aoa)) { + keyboard_aoa_initialized = true; + kp = &s->keyboard_aoa.key_processor; } else { LOGE("Could not initialize HID keyboard"); } @@ -608,7 +608,7 @@ scrcpy(struct scrcpy_options *options) { } } - bool need_aoa = hid_keyboard_initialized || hid_mouse_initialized; + bool need_aoa = keyboard_aoa_initialized || hid_mouse_initialized; if (!need_aoa || !sc_aoa_start(&s->aoa)) { sc_acksync_destroy(&s->acksync); @@ -624,9 +624,9 @@ scrcpy(struct scrcpy_options *options) { aoa_hid_end: if (!aoa_hid_initialized) { - if (hid_keyboard_initialized) { - sc_hid_keyboard_destroy(&s->keyboard_hid); - hid_keyboard_initialized = false; + if (keyboard_aoa_initialized) { + sc_keyboard_aoa_destroy(&s->keyboard_aoa); + keyboard_aoa_initialized = false; } if (hid_mouse_initialized) { sc_hid_mouse_destroy(&s->mouse_hid); @@ -634,7 +634,7 @@ aoa_hid_end: } } - if (use_aoa_keyboard && !hid_keyboard_initialized) { + if (use_keyboard_aoa && !keyboard_aoa_initialized) { LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)"); options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_SDK; } @@ -813,8 +813,8 @@ end: // end-of-stream #ifdef HAVE_USB if (aoa_hid_initialized) { - if (hid_keyboard_initialized) { - sc_hid_keyboard_destroy(&s->keyboard_hid); + if (keyboard_aoa_initialized) { + sc_keyboard_aoa_destroy(&s->keyboard_aoa); } if (hid_mouse_initialized) { sc_hid_mouse_destroy(&s->mouse_hid); diff --git a/app/src/usb/keyboard_aoa.c b/app/src/usb/keyboard_aoa.c new file mode 100644 index 00000000..b69d6cd8 --- /dev/null +++ b/app/src/usb/keyboard_aoa.c @@ -0,0 +1,109 @@ +#include "keyboard_aoa.h" + +#include + +#include "input_events.h" +#include "util/log.h" + +/** Downcast key processor to keyboard_aoa */ +#define DOWNCAST(KP) container_of(KP, struct sc_keyboard_aoa, key_processor) + +#define HID_KEYBOARD_ACCESSORY_ID 1 + +static bool +push_mod_lock_state(struct sc_keyboard_aoa *kb, uint16_t mods_state) { + struct sc_hid_event hid_event; + if (!sc_hid_keyboard_event_from_mods(&hid_event, mods_state)) { + // Nothing to do + return true; + } + + if (!sc_aoa_push_hid_event(kb->aoa, HID_KEYBOARD_ACCESSORY_ID, + &hid_event)) { + LOGW("Could not request HID event (mod lock state)"); + return false; + } + + LOGD("HID keyboard state synchronized"); + + return true; +} + +static void +sc_key_processor_process_key(struct sc_key_processor *kp, + const struct sc_key_event *event, + uint64_t ack_to_wait) { + if (event->repeat) { + // In USB HID protocol, key repeat is handled by the host (Android), so + // just ignore key repeat here. + return; + } + + struct sc_keyboard_aoa *kb = DOWNCAST(kp); + + struct sc_hid_event hid_event; + + // Not all keys are supported, just ignore unsupported keys + if (sc_hid_keyboard_event_from_key(&kb->hid, &hid_event, event)) { + if (!kb->mod_lock_synchronized) { + // Inject CAPSLOCK and/or NUMLOCK if necessary to synchronize + // keyboard state + if (push_mod_lock_state(kb, event->mods_state)) { + kb->mod_lock_synchronized = true; + } + } + + // If ack_to_wait is != SC_SEQUENCE_INVALID, then Ctrl+v is pressed, so + // clipboard synchronization has been requested. Wait until clipboard + // synchronization is acknowledged by the server, otherwise it could + // paste the old clipboard content. + + if (!sc_aoa_push_hid_event_with_ack_to_wait(kb->aoa, + HID_KEYBOARD_ACCESSORY_ID, + &hid_event, + ack_to_wait)) { + LOGW("Could not request HID event (key)"); + } + } +} + +bool +sc_keyboard_aoa_init(struct sc_keyboard_aoa *kb, struct sc_aoa *aoa) { + kb->aoa = aoa; + + bool ok = sc_aoa_setup_hid(aoa, HID_KEYBOARD_ACCESSORY_ID, + SC_HID_KEYBOARD_REPORT_DESC, + SC_HID_KEYBOARD_REPORT_DESC_LEN); + if (!ok) { + LOGW("Register HID keyboard failed"); + return false; + } + + sc_hid_keyboard_init(&kb->hid); + + kb->mod_lock_synchronized = false; + + static const struct sc_key_processor_ops ops = { + .process_key = sc_key_processor_process_key, + // Never forward text input via HID (all the keys are injected + // separately) + .process_text = NULL, + }; + + // Clipboard synchronization is requested over the control socket, while HID + // events are sent over AOA, so it must wait for clipboard synchronization + // to be acknowledged by the device before injecting Ctrl+v. + kb->key_processor.async_paste = true; + kb->key_processor.ops = &ops; + + return true; +} + +void +sc_keyboard_aoa_destroy(struct sc_keyboard_aoa *kb) { + // Unregister HID keyboard so the soft keyboard shows again on Android + bool ok = sc_aoa_unregister_hid(kb->aoa, HID_KEYBOARD_ACCESSORY_ID); + if (!ok) { + LOGW("Could not unregister HID keyboard"); + } +} diff --git a/app/src/usb/keyboard_aoa.h b/app/src/usb/keyboard_aoa.h new file mode 100644 index 00000000..565b9177 --- /dev/null +++ b/app/src/usb/keyboard_aoa.h @@ -0,0 +1,27 @@ +#ifndef SC_KEYBOARD_AOA_H +#define SC_KEYBOARD_AOA_H + +#include "common.h" + +#include + +#include "aoa_hid.h" +#include "hid/hid_keyboard.h" +#include "trait/key_processor.h" + +struct sc_keyboard_aoa { + struct sc_key_processor key_processor; // key processor trait + + struct sc_hid_keyboard hid; + struct sc_aoa *aoa; + + bool mod_lock_synchronized; +}; + +bool +sc_keyboard_aoa_init(struct sc_keyboard_aoa *kb, struct sc_aoa *aoa); + +void +sc_keyboard_aoa_destroy(struct sc_keyboard_aoa *kb); + +#endif diff --git a/app/src/usb/scrcpy_otg.c b/app/src/usb/scrcpy_otg.c index 5955e909..9064ad10 100644 --- a/app/src/usb/scrcpy_otg.c +++ b/app/src/usb/scrcpy_otg.c @@ -10,7 +10,7 @@ struct scrcpy_otg { struct sc_usb usb; struct sc_aoa aoa; - struct sc_hid_keyboard keyboard; + struct sc_keyboard_aoa keyboard; struct sc_hid_mouse mouse; struct sc_screen_otg screen_otg; @@ -73,7 +73,7 @@ scrcpy_otg(struct scrcpy_options *options) { enum scrcpy_exit_code ret = SCRCPY_EXIT_FAILURE; - struct sc_hid_keyboard *keyboard = NULL; + struct sc_keyboard_aoa *keyboard = NULL; struct sc_hid_mouse *mouse = NULL; bool usb_device_initialized = false; bool usb_connected = false; @@ -128,7 +128,7 @@ scrcpy_otg(struct scrcpy_options *options) { options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA; if (enable_keyboard) { - ok = sc_hid_keyboard_init(&s->keyboard, &s->aoa); + ok = sc_keyboard_aoa_init(&s->keyboard, &s->aoa); if (!ok) { goto end; } @@ -188,7 +188,7 @@ end: sc_hid_mouse_destroy(&s->mouse); } if (keyboard) { - sc_hid_keyboard_destroy(&s->keyboard); + sc_keyboard_aoa_destroy(&s->keyboard); } if (aoa_initialized) { diff --git a/app/src/usb/screen_otg.h b/app/src/usb/screen_otg.h index a0acf40b..cfc3bfa2 100644 --- a/app/src/usb/screen_otg.h +++ b/app/src/usb/screen_otg.h @@ -6,11 +6,11 @@ #include #include -#include "hid_keyboard.h" +#include "keyboard_aoa.h" #include "hid_mouse.h" struct sc_screen_otg { - struct sc_hid_keyboard *keyboard; + struct sc_keyboard_aoa *keyboard; struct sc_hid_mouse *mouse; SDL_Window *window; @@ -22,7 +22,7 @@ struct sc_screen_otg { }; struct sc_screen_otg_params { - struct sc_hid_keyboard *keyboard; + struct sc_keyboard_aoa *keyboard; struct sc_hid_mouse *mouse; const char *window_title; From d95276467b93eb4b045fd15000eacaecf7c1874c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 25 Jan 2024 23:04:09 +0100 Subject: [PATCH 51/79] Extract mouse HID handling Split the mouse implementation using AOA and the code handling HID events, so that HID events can be reused for another protocol (UHID). PR #4473 --- app/meson.build | 3 +- app/src/{usb => hid}/hid_mouse.c | 113 +++++------------------ app/src/hid/hid_mouse.h | 26 ++++++ app/src/scrcpy.c | 32 +++---- app/src/usb/mouse_aoa.c | 89 ++++++++++++++++++ app/src/usb/{hid_mouse.h => mouse_aoa.h} | 10 +- app/src/usb/scrcpy_otg.c | 8 +- app/src/usb/screen_otg.h | 6 +- 8 files changed, 169 insertions(+), 118 deletions(-) rename app/src/{usb => hid}/hid_mouse.c (59%) create mode 100644 app/src/hid/hid_mouse.h create mode 100644 app/src/usb/mouse_aoa.c rename app/src/usb/{hid_mouse.h => mouse_aoa.h} (55%) diff --git a/app/meson.build b/app/meson.build index 6d572b7b..f78afa15 100644 --- a/app/meson.build +++ b/app/meson.build @@ -32,6 +32,7 @@ src = [ 'src/server.c', 'src/version.c', 'src/hid/hid_keyboard.c', + 'src/hid/hid_mouse.c', 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/util/acksync.c', @@ -90,7 +91,7 @@ if usb_support src += [ 'src/usb/aoa_hid.c', 'src/usb/keyboard_aoa.c', - 'src/usb/hid_mouse.c', + 'src/usb/mouse_aoa.c', 'src/usb/scrcpy_otg.c', 'src/usb/screen_otg.c', 'src/usb/usb.c', diff --git a/app/src/usb/hid_mouse.c b/app/src/hid/hid_mouse.c similarity index 59% rename from app/src/usb/hid_mouse.c rename to app/src/hid/hid_mouse.c index de961265..9d814448 100644 --- a/app/src/usb/hid_mouse.c +++ b/app/src/hid/hid_mouse.c @@ -1,15 +1,5 @@ #include "hid_mouse.h" -#include - -#include "input_events.h" -#include "util/log.h" - -/** Downcast mouse processor to hid_mouse */ -#define DOWNCAST(MP) container_of(MP, struct sc_hid_mouse, mouse_processor) - -#define HID_MOUSE_ACCESSORY_ID 2 - // 1 byte for buttons + padding, 1 byte for X position, 1 byte for Y position, // 1 byte for wheel motion #define HID_MOUSE_EVENT_SIZE 4 @@ -24,7 +14,7 @@ * * §4 Generic Desktop Page (0x01) (p26) */ -static const unsigned char mouse_report_desc[] = { +const uint8_t SC_HID_MOUSE_REPORT_DESC[] = { // Usage Page (Generic Desktop) 0x05, 0x01, // Usage (Mouse) @@ -90,6 +80,9 @@ static const unsigned char mouse_report_desc[] = { 0xC0, }; +const size_t SC_HID_MOUSE_REPORT_DESC_LEN = + sizeof(SC_HID_MOUSE_REPORT_DESC); + /** * A mouse HID event is 4 bytes long: * @@ -138,9 +131,9 @@ sc_hid_mouse_event_init(struct sc_hid_event *hid_event) { // callers } -static unsigned char -buttons_state_to_hid_buttons(uint8_t buttons_state) { - unsigned char c = 0; +static uint8_t +sc_hid_buttons_from_buttons_state(uint8_t buttons_state) { + uint8_t c = 0; if (buttons_state & SC_MOUSE_BUTTON_LEFT) { c |= 1 << 0; } @@ -159,55 +152,36 @@ buttons_state_to_hid_buttons(uint8_t buttons_state) { return c; } -static void -sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, - const struct sc_mouse_motion_event *event) { - struct sc_hid_mouse *mouse = DOWNCAST(mp); - - struct sc_hid_event hid_event; - sc_hid_mouse_event_init(&hid_event); +void +sc_hid_mouse_event_from_motion(struct sc_hid_event *hid_event, + const struct sc_mouse_motion_event *event) { + sc_hid_mouse_event_init(hid_event); - unsigned char *data = hid_event.data; - data[0] = buttons_state_to_hid_buttons(event->buttons_state); + uint8_t *data = hid_event->data; + data[0] = sc_hid_buttons_from_buttons_state(event->buttons_state); data[1] = CLAMP(event->xrel, -127, 127); data[2] = CLAMP(event->yrel, -127, 127); data[3] = 0; // wheel coordinates only used for scrolling - - if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, - &hid_event)) { - LOGW("Could not request HID event (mouse motion)"); - } } -static void -sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, - const struct sc_mouse_click_event *event) { - struct sc_hid_mouse *mouse = DOWNCAST(mp); - - struct sc_hid_event hid_event; - sc_hid_mouse_event_init(&hid_event); +void +sc_hid_mouse_event_from_click(struct sc_hid_event *hid_event, + const struct sc_mouse_click_event *event) { + sc_hid_mouse_event_init(hid_event); - unsigned char *data = hid_event.data; - data[0] = buttons_state_to_hid_buttons(event->buttons_state); + uint8_t *data = hid_event->data; + data[0] = sc_hid_buttons_from_buttons_state(event->buttons_state); data[1] = 0; // no x motion data[2] = 0; // no y motion data[3] = 0; // wheel coordinates only used for scrolling - - if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, - &hid_event)) { - LOGW("Could not request HID event (mouse click)"); - } } -static void -sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, - const struct sc_mouse_scroll_event *event) { - struct sc_hid_mouse *mouse = DOWNCAST(mp); - - struct sc_hid_event hid_event; - sc_hid_mouse_event_init(&hid_event); +void +sc_hid_mouse_event_from_scroll(struct sc_hid_event *hid_event, + const struct sc_mouse_scroll_event *event) { + sc_hid_mouse_event_init(hid_event); - unsigned char *data = hid_event.data; + uint8_t *data = hid_event->data; data[0] = 0; // buttons state irrelevant (and unknown) data[1] = 0; // no x motion data[2] = 0; // no y motion @@ -215,43 +189,4 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, // are possible data[3] = CLAMP(event->vscroll, -127, 127); // Horizontal scrolling ignored - - if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, - &hid_event)) { - LOGW("Could not request HID event (mouse scroll)"); - } -} - -bool -sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa) { - mouse->aoa = aoa; - - bool ok = sc_aoa_setup_hid(aoa, HID_MOUSE_ACCESSORY_ID, mouse_report_desc, - ARRAY_LEN(mouse_report_desc)); - if (!ok) { - LOGW("Register HID mouse failed"); - return false; - } - - static const struct sc_mouse_processor_ops ops = { - .process_mouse_motion = sc_mouse_processor_process_mouse_motion, - .process_mouse_click = sc_mouse_processor_process_mouse_click, - .process_mouse_scroll = sc_mouse_processor_process_mouse_scroll, - // Touch events not supported (coordinates are not relative) - .process_touch = NULL, - }; - - mouse->mouse_processor.ops = &ops; - - mouse->mouse_processor.relative_mode = true; - - return true; -} - -void -sc_hid_mouse_destroy(struct sc_hid_mouse *mouse) { - bool ok = sc_aoa_unregister_hid(mouse->aoa, HID_MOUSE_ACCESSORY_ID); - if (!ok) { - LOGW("Could not unregister HID mouse"); - } } diff --git a/app/src/hid/hid_mouse.h b/app/src/hid/hid_mouse.h new file mode 100644 index 00000000..e514d7d9 --- /dev/null +++ b/app/src/hid/hid_mouse.h @@ -0,0 +1,26 @@ +#ifndef SC_HID_MOUSE_H +#define SC_HID_MOUSE_H + +#endif + +#include "common.h" + +#include + +#include "hid/hid_event.h" +#include "input_events.h" + +extern const uint8_t SC_HID_MOUSE_REPORT_DESC[]; +extern const size_t SC_HID_MOUSE_REPORT_DESC_LEN; + +void +sc_hid_mouse_event_from_motion(struct sc_hid_event *hid_event, + const struct sc_mouse_motion_event *event); + +void +sc_hid_mouse_event_from_click(struct sc_hid_event *hid_event, + const struct sc_mouse_click_event *event); + +void +sc_hid_mouse_event_from_scroll(struct sc_hid_event *hid_event, + const struct sc_mouse_scroll_event *event); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 1d5e67dc..bd448052 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -28,7 +28,7 @@ #ifdef HAVE_USB # include "usb/aoa_hid.h" # include "usb/keyboard_aoa.h" -# include "usb/hid_mouse.h" +# include "usb/mouse_aoa.h" # include "usb/usb.h" #endif #include "util/acksync.h" @@ -71,7 +71,7 @@ struct scrcpy { union { struct sc_mouse_inject mouse_inject; #ifdef HAVE_USB - struct sc_hid_mouse mouse_hid; + struct sc_mouse_aoa mouse_aoa; #endif }; struct sc_timeout timeout; @@ -331,7 +331,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_USB bool aoa_hid_initialized = false; bool keyboard_aoa_initialized = false; - bool hid_mouse_initialized = false; + bool mouse_aoa_initialized = false; #endif bool controller_initialized = false; bool controller_started = false; @@ -545,9 +545,9 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_USB bool use_keyboard_aoa = options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA; - bool use_aoa_mouse = + bool use_mouse_aoa = options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA; - if (use_keyboard_aoa || use_aoa_mouse) { + if (use_keyboard_aoa || use_mouse_aoa) { bool ok = sc_acksync_init(&s->acksync); if (!ok) { goto end; @@ -599,16 +599,16 @@ scrcpy(struct scrcpy_options *options) { } } - if (use_aoa_mouse) { - if (sc_hid_mouse_init(&s->mouse_hid, &s->aoa)) { - hid_mouse_initialized = true; - mp = &s->mouse_hid.mouse_processor; + if (use_mouse_aoa) { + if (sc_mouse_aoa_init(&s->mouse_aoa, &s->aoa)) { + mouse_aoa_initialized = true; + mp = &s->mouse_aoa.mouse_processor; } else { LOGE("Could not initialized HID mouse"); } } - bool need_aoa = keyboard_aoa_initialized || hid_mouse_initialized; + bool need_aoa = keyboard_aoa_initialized || mouse_aoa_initialized; if (!need_aoa || !sc_aoa_start(&s->aoa)) { sc_acksync_destroy(&s->acksync); @@ -628,9 +628,9 @@ aoa_hid_end: sc_keyboard_aoa_destroy(&s->keyboard_aoa); keyboard_aoa_initialized = false; } - if (hid_mouse_initialized) { - sc_hid_mouse_destroy(&s->mouse_hid); - hid_mouse_initialized = false; + if (mouse_aoa_initialized) { + sc_mouse_aoa_destroy(&s->mouse_aoa); + mouse_aoa_initialized = false; } } @@ -639,7 +639,7 @@ aoa_hid_end: options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_SDK; } - if (use_aoa_mouse && !hid_mouse_initialized) { + if (use_mouse_aoa && !mouse_aoa_initialized) { LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)"); options->mouse_input_mode = SC_MOUSE_INPUT_MODE_SDK; } @@ -816,8 +816,8 @@ end: if (keyboard_aoa_initialized) { sc_keyboard_aoa_destroy(&s->keyboard_aoa); } - if (hid_mouse_initialized) { - sc_hid_mouse_destroy(&s->mouse_hid); + if (mouse_aoa_initialized) { + sc_mouse_aoa_destroy(&s->mouse_aoa); } sc_aoa_stop(&s->aoa); sc_usb_stop(&s->usb); diff --git a/app/src/usb/mouse_aoa.c b/app/src/usb/mouse_aoa.c new file mode 100644 index 00000000..93b32328 --- /dev/null +++ b/app/src/usb/mouse_aoa.c @@ -0,0 +1,89 @@ +#include "mouse_aoa.h" + +#include + +#include "hid/hid_mouse.h" +#include "input_events.h" +#include "util/log.h" + +/** Downcast mouse processor to mouse_aoa */ +#define DOWNCAST(MP) container_of(MP, struct sc_mouse_aoa, mouse_processor) + +#define HID_MOUSE_ACCESSORY_ID 2 + +static void +sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, + const struct sc_mouse_motion_event *event) { + struct sc_mouse_aoa *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + sc_hid_mouse_event_from_motion(&hid_event, event); + + if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, + &hid_event)) { + LOGW("Could not request HID event (mouse motion)"); + } +} + +static void +sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, + const struct sc_mouse_click_event *event) { + struct sc_mouse_aoa *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + sc_hid_mouse_event_from_click(&hid_event, event); + + if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, + &hid_event)) { + LOGW("Could not request HID event (mouse click)"); + } +} + +static void +sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, + const struct sc_mouse_scroll_event *event) { + struct sc_mouse_aoa *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + sc_hid_mouse_event_from_scroll(&hid_event, event); + + if (!sc_aoa_push_hid_event(mouse->aoa, HID_MOUSE_ACCESSORY_ID, + &hid_event)) { + LOGW("Could not request HID event (mouse scroll)"); + } +} + +bool +sc_mouse_aoa_init(struct sc_mouse_aoa *mouse, struct sc_aoa *aoa) { + mouse->aoa = aoa; + + bool ok = sc_aoa_setup_hid(aoa, HID_MOUSE_ACCESSORY_ID, + SC_HID_MOUSE_REPORT_DESC, + SC_HID_MOUSE_REPORT_DESC_LEN); + if (!ok) { + LOGW("Register HID mouse failed"); + return false; + } + + static const struct sc_mouse_processor_ops ops = { + .process_mouse_motion = sc_mouse_processor_process_mouse_motion, + .process_mouse_click = sc_mouse_processor_process_mouse_click, + .process_mouse_scroll = sc_mouse_processor_process_mouse_scroll, + // Touch events not supported (coordinates are not relative) + .process_touch = NULL, + }; + + mouse->mouse_processor.ops = &ops; + + mouse->mouse_processor.relative_mode = true; + + return true; +} + +void +sc_mouse_aoa_destroy(struct sc_mouse_aoa *mouse) { + bool ok = sc_aoa_unregister_hid(mouse->aoa, HID_MOUSE_ACCESSORY_ID); + if (!ok) { + LOGW("Could not unregister HID mouse"); + } +} diff --git a/app/src/usb/hid_mouse.h b/app/src/usb/mouse_aoa.h similarity index 55% rename from app/src/usb/hid_mouse.h rename to app/src/usb/mouse_aoa.h index b89f7795..afaed761 100644 --- a/app/src/usb/hid_mouse.h +++ b/app/src/usb/mouse_aoa.h @@ -1,5 +1,5 @@ -#ifndef SC_HID_MOUSE_H -#define SC_HID_MOUSE_H +#ifndef SC_MOUSE_AOA_H +#define SC_MOUSE_AOA_H #include "common.h" @@ -8,16 +8,16 @@ #include "aoa_hid.h" #include "trait/mouse_processor.h" -struct sc_hid_mouse { +struct sc_mouse_aoa { struct sc_mouse_processor mouse_processor; // mouse processor trait struct sc_aoa *aoa; }; bool -sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa); +sc_mouse_aoa_init(struct sc_mouse_aoa *mouse, struct sc_aoa *aoa); void -sc_hid_mouse_destroy(struct sc_hid_mouse *mouse); +sc_mouse_aoa_destroy(struct sc_mouse_aoa *mouse); #endif diff --git a/app/src/usb/scrcpy_otg.c b/app/src/usb/scrcpy_otg.c index 9064ad10..c1d38da3 100644 --- a/app/src/usb/scrcpy_otg.c +++ b/app/src/usb/scrcpy_otg.c @@ -11,7 +11,7 @@ struct scrcpy_otg { struct sc_usb usb; struct sc_aoa aoa; struct sc_keyboard_aoa keyboard; - struct sc_hid_mouse mouse; + struct sc_mouse_aoa mouse; struct sc_screen_otg screen_otg; }; @@ -74,7 +74,7 @@ scrcpy_otg(struct scrcpy_options *options) { enum scrcpy_exit_code ret = SCRCPY_EXIT_FAILURE; struct sc_keyboard_aoa *keyboard = NULL; - struct sc_hid_mouse *mouse = NULL; + struct sc_mouse_aoa *mouse = NULL; bool usb_device_initialized = false; bool usb_connected = false; bool aoa_started = false; @@ -136,7 +136,7 @@ scrcpy_otg(struct scrcpy_options *options) { } if (enable_mouse) { - ok = sc_hid_mouse_init(&s->mouse, &s->aoa); + ok = sc_mouse_aoa_init(&s->mouse, &s->aoa); if (!ok) { goto end; } @@ -185,7 +185,7 @@ end: sc_usb_stop(&s->usb); if (mouse) { - sc_hid_mouse_destroy(&s->mouse); + sc_mouse_aoa_destroy(&s->mouse); } if (keyboard) { sc_keyboard_aoa_destroy(&s->keyboard); diff --git a/app/src/usb/screen_otg.h b/app/src/usb/screen_otg.h index cfc3bfa2..c4e03b87 100644 --- a/app/src/usb/screen_otg.h +++ b/app/src/usb/screen_otg.h @@ -7,11 +7,11 @@ #include #include "keyboard_aoa.h" -#include "hid_mouse.h" +#include "mouse_aoa.h" struct sc_screen_otg { struct sc_keyboard_aoa *keyboard; - struct sc_hid_mouse *mouse; + struct sc_mouse_aoa *mouse; SDL_Window *window; SDL_Renderer *renderer; @@ -23,7 +23,7 @@ struct sc_screen_otg { struct sc_screen_otg_params { struct sc_keyboard_aoa *keyboard; - struct sc_hid_mouse *mouse; + struct sc_mouse_aoa *mouse; const char *window_title; bool always_on_top; From 2e7f6a6fc4b3bf8347b7bba85335cb005d0fc63e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 25 Jan 2024 23:08:20 +0100 Subject: [PATCH 52/79] Rename default keyboard implementation to "sdk" Rename {keyboard,mouse}_inject to {keyboard,mouse}_sdk. All implementations "inject" key events and mouse events, what differs is the mechanism. For these implementations, the Android SDK API is used to inject events. Note that the input mode enum variants were already renamed (SC_KEYBOARD_INPUT_MODE_SDK and SC_MOUSE_INPUT_MODE_SDK). PR #4473 --- app/meson.build | 4 +- app/src/{keyboard_inject.c => keyboard_sdk.c} | 46 +++++++++---------- app/src/{keyboard_inject.h => keyboard_sdk.h} | 14 +++--- app/src/{mouse_inject.c => mouse_sdk.c} | 31 ++++++------- app/src/{mouse_inject.h => mouse_sdk.h} | 9 ++-- app/src/scrcpy.c | 20 ++++---- 6 files changed, 61 insertions(+), 63 deletions(-) rename app/src/{keyboard_inject.c => keyboard_sdk.c} (91%) rename app/src/{keyboard_inject.h => keyboard_sdk.h} (61%) rename app/src/{mouse_inject.c => mouse_sdk.c} (84%) rename app/src/{mouse_inject.h => mouse_sdk.h} (58%) diff --git a/app/meson.build b/app/meson.build index f78afa15..3ec9781a 100644 --- a/app/meson.build +++ b/app/meson.build @@ -20,8 +20,8 @@ src = [ 'src/fps_counter.c', 'src/frame_buffer.c', 'src/input_manager.c', - 'src/keyboard_inject.c', - 'src/mouse_inject.c', + 'src/keyboard_sdk.c', + 'src/mouse_sdk.c', 'src/opengl.c', 'src/options.c', 'src/packet_merger.c', diff --git a/app/src/keyboard_inject.c b/app/src/keyboard_sdk.c similarity index 91% rename from app/src/keyboard_inject.c rename to app/src/keyboard_sdk.c index fe297310..726f65a9 100644 --- a/app/src/keyboard_inject.c +++ b/app/src/keyboard_sdk.c @@ -1,4 +1,4 @@ -#include "keyboard_inject.h" +#include "keyboard_sdk.h" #include @@ -9,8 +9,8 @@ #include "util/intmap.h" #include "util/log.h" -/** Downcast key processor to sc_keyboard_inject */ -#define DOWNCAST(KP) container_of(KP, struct sc_keyboard_inject, key_processor) +/** Downcast key processor to sc_keyboard_sdk */ +#define DOWNCAST(KP) container_of(KP, struct sc_keyboard_sdk, key_processor) static enum android_keyevent_action convert_keycode_action(enum sc_action action) { @@ -271,20 +271,20 @@ sc_key_processor_process_key(struct sc_key_processor *kp, // is set before injecting Ctrl+v. (void) ack_to_wait; - struct sc_keyboard_inject *ki = DOWNCAST(kp); + struct sc_keyboard_sdk *kb = DOWNCAST(kp); if (event->repeat) { - if (!ki->forward_key_repeat) { + if (!kb->forward_key_repeat) { return; } - ++ki->repeat; + ++kb->repeat; } else { - ki->repeat = 0; + kb->repeat = 0; } struct sc_control_msg msg; - if (convert_input_key(event, &msg, ki->key_inject_mode, ki->repeat)) { - if (!sc_controller_push_msg(ki->controller, &msg)) { + if (convert_input_key(event, &msg, kb->key_inject_mode, kb->repeat)) { + if (!sc_controller_push_msg(kb->controller, &msg)) { LOGW("Could not request 'inject keycode'"); } } @@ -293,14 +293,14 @@ sc_key_processor_process_key(struct sc_key_processor *kp, static void sc_key_processor_process_text(struct sc_key_processor *kp, const struct sc_text_event *event) { - struct sc_keyboard_inject *ki = DOWNCAST(kp); + struct sc_keyboard_sdk *kb = DOWNCAST(kp); - if (ki->key_inject_mode == SC_KEY_INJECT_MODE_RAW) { + if (kb->key_inject_mode == SC_KEY_INJECT_MODE_RAW) { // Never inject text events return; } - if (ki->key_inject_mode == SC_KEY_INJECT_MODE_MIXED) { + if (kb->key_inject_mode == SC_KEY_INJECT_MODE_MIXED) { char c = event->text[0]; if (isalpha(c) || c == ' ') { assert(event->text[1] == '\0'); @@ -316,22 +316,22 @@ sc_key_processor_process_text(struct sc_key_processor *kp, LOGW("Could not strdup input text"); return; } - if (!sc_controller_push_msg(ki->controller, &msg)) { + if (!sc_controller_push_msg(kb->controller, &msg)) { free(msg.inject_text.text); LOGW("Could not request 'inject text'"); } } void -sc_keyboard_inject_init(struct sc_keyboard_inject *ki, - struct sc_controller *controller, - enum sc_key_inject_mode key_inject_mode, - bool forward_key_repeat) { - ki->controller = controller; - ki->key_inject_mode = key_inject_mode; - ki->forward_key_repeat = forward_key_repeat; +sc_keyboard_sdk_init(struct sc_keyboard_sdk *kb, + struct sc_controller *controller, + enum sc_key_inject_mode key_inject_mode, + bool forward_key_repeat) { + kb->controller = controller; + kb->key_inject_mode = key_inject_mode; + kb->forward_key_repeat = forward_key_repeat; - ki->repeat = 0; + kb->repeat = 0; static const struct sc_key_processor_ops ops = { .process_key = sc_key_processor_process_key, @@ -339,6 +339,6 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki, }; // Key injection and clipboard synchronization are serialized - ki->key_processor.async_paste = false; - ki->key_processor.ops = &ops; + kb->key_processor.async_paste = false; + kb->key_processor.ops = &ops; } diff --git a/app/src/keyboard_inject.h b/app/src/keyboard_sdk.h similarity index 61% rename from app/src/keyboard_inject.h rename to app/src/keyboard_sdk.h index b7781c1f..700ba90b 100644 --- a/app/src/keyboard_inject.h +++ b/app/src/keyboard_sdk.h @@ -1,5 +1,5 @@ -#ifndef SC_KEYBOARD_INJECT_H -#define SC_KEYBOARD_INJECT_H +#ifndef SC_KEYBOARD_SDK_H +#define SC_KEYBOARD_SDK_H #include "common.h" @@ -9,7 +9,7 @@ #include "options.h" #include "trait/key_processor.h" -struct sc_keyboard_inject { +struct sc_keyboard_sdk { struct sc_key_processor key_processor; // key processor trait struct sc_controller *controller; @@ -23,9 +23,9 @@ struct sc_keyboard_inject { }; void -sc_keyboard_inject_init(struct sc_keyboard_inject *ki, - struct sc_controller *controller, - enum sc_key_inject_mode key_inject_mode, - bool forward_key_repeat); +sc_keyboard_sdk_init(struct sc_keyboard_sdk *kb, + struct sc_controller *controller, + enum sc_key_inject_mode key_inject_mode, + bool forward_key_repeat); #endif diff --git a/app/src/mouse_inject.c b/app/src/mouse_sdk.c similarity index 84% rename from app/src/mouse_inject.c rename to app/src/mouse_sdk.c index 71b7a64d..620fb52c 100644 --- a/app/src/mouse_inject.c +++ b/app/src/mouse_sdk.c @@ -1,4 +1,4 @@ -#include "mouse_inject.h" +#include "mouse_sdk.h" #include @@ -9,8 +9,8 @@ #include "util/intmap.h" #include "util/log.h" -/** Downcast mouse processor to sc_mouse_inject */ -#define DOWNCAST(MP) container_of(MP, struct sc_mouse_inject, mouse_processor) +/** Downcast mouse processor to sc_mouse_sdk */ +#define DOWNCAST(MP) container_of(MP, struct sc_mouse_sdk, mouse_processor) static enum android_motionevent_buttons convert_mouse_buttons(uint32_t state) { @@ -63,7 +63,7 @@ sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, return; } - struct sc_mouse_inject *mi = DOWNCAST(mp); + struct sc_mouse_sdk *m = DOWNCAST(mp); struct sc_control_msg msg = { .type = SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT, @@ -76,7 +76,7 @@ sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, }, }; - if (!sc_controller_push_msg(mi->controller, &msg)) { + if (!sc_controller_push_msg(m->controller, &msg)) { LOGW("Could not request 'inject mouse motion event'"); } } @@ -84,7 +84,7 @@ sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, static void sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, const struct sc_mouse_click_event *event) { - struct sc_mouse_inject *mi = DOWNCAST(mp); + struct sc_mouse_sdk *m = DOWNCAST(mp); struct sc_control_msg msg = { .type = SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT, @@ -98,7 +98,7 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, }, }; - if (!sc_controller_push_msg(mi->controller, &msg)) { + if (!sc_controller_push_msg(m->controller, &msg)) { LOGW("Could not request 'inject mouse click event'"); } } @@ -106,7 +106,7 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, static void sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, const struct sc_mouse_scroll_event *event) { - struct sc_mouse_inject *mi = DOWNCAST(mp); + struct sc_mouse_sdk *m = DOWNCAST(mp); struct sc_control_msg msg = { .type = SC_CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT, @@ -118,7 +118,7 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, }, }; - if (!sc_controller_push_msg(mi->controller, &msg)) { + if (!sc_controller_push_msg(m->controller, &msg)) { LOGW("Could not request 'inject mouse scroll event'"); } } @@ -126,7 +126,7 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, static void sc_mouse_processor_process_touch(struct sc_mouse_processor *mp, const struct sc_touch_event *event) { - struct sc_mouse_inject *mi = DOWNCAST(mp); + struct sc_mouse_sdk *m = DOWNCAST(mp); struct sc_control_msg msg = { .type = SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT, @@ -139,15 +139,14 @@ sc_mouse_processor_process_touch(struct sc_mouse_processor *mp, }, }; - if (!sc_controller_push_msg(mi->controller, &msg)) { + if (!sc_controller_push_msg(m->controller, &msg)) { LOGW("Could not request 'inject touch event'"); } } void -sc_mouse_inject_init(struct sc_mouse_inject *mi, - struct sc_controller *controller) { - mi->controller = controller; +sc_mouse_sdk_init(struct sc_mouse_sdk *m, struct sc_controller *controller) { + m->controller = controller; static const struct sc_mouse_processor_ops ops = { .process_mouse_motion = sc_mouse_processor_process_mouse_motion, @@ -156,7 +155,7 @@ sc_mouse_inject_init(struct sc_mouse_inject *mi, .process_touch = sc_mouse_processor_process_touch, }; - mi->mouse_processor.ops = &ops; + m->mouse_processor.ops = &ops; - mi->mouse_processor.relative_mode = false; + m->mouse_processor.relative_mode = false; } diff --git a/app/src/mouse_inject.h b/app/src/mouse_sdk.h similarity index 58% rename from app/src/mouse_inject.h rename to app/src/mouse_sdk.h index 59a6a5d8..444a6ad5 100644 --- a/app/src/mouse_inject.h +++ b/app/src/mouse_sdk.h @@ -1,5 +1,5 @@ -#ifndef SC_MOUSE_INJECT_H -#define SC_MOUSE_INJECT_H +#ifndef SC_MOUSE_SDK_H +#define SC_MOUSE_SDK_H #include "common.h" @@ -9,14 +9,13 @@ #include "screen.h" #include "trait/mouse_processor.h" -struct sc_mouse_inject { +struct sc_mouse_sdk { struct sc_mouse_processor mouse_processor; // mouse processor trait struct sc_controller *controller; }; void -sc_mouse_inject_init(struct sc_mouse_inject *mi, - struct sc_controller *controller); +sc_mouse_sdk_init(struct sc_mouse_sdk *m, struct sc_controller *controller); #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index bd448052..876b400a 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -20,8 +20,8 @@ #include "demuxer.h" #include "events.h" #include "file_pusher.h" -#include "keyboard_inject.h" -#include "mouse_inject.h" +#include "keyboard_sdk.h" +#include "mouse_sdk.h" #include "recorder.h" #include "screen.h" #include "server.h" @@ -63,13 +63,13 @@ struct scrcpy { struct sc_acksync acksync; #endif union { - struct sc_keyboard_inject keyboard_inject; + struct sc_keyboard_sdk keyboard_sdk; #ifdef HAVE_USB struct sc_keyboard_aoa keyboard_aoa; #endif }; union { - struct sc_mouse_inject mouse_inject; + struct sc_mouse_sdk mouse_sdk; #ifdef HAVE_USB struct sc_mouse_aoa mouse_aoa; #endif @@ -651,16 +651,16 @@ aoa_hid_end: // keyboard_input_mode may have been reset if HID mode failed if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_SDK) { - sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, - options->key_inject_mode, - options->forward_key_repeat); - kp = &s->keyboard_inject.key_processor; + sc_keyboard_sdk_init(&s->keyboard_sdk, &s->controller, + options->key_inject_mode, + options->forward_key_repeat); + kp = &s->keyboard_sdk.key_processor; } // mouse_input_mode may have been reset if HID mode failed if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_SDK) { - sc_mouse_inject_init(&s->mouse_inject, &s->controller); - mp = &s->mouse_inject.mouse_processor; + sc_mouse_sdk_init(&s->mouse_sdk, &s->controller); + mp = &s->mouse_sdk.mouse_processor; } if (!sc_controller_init(&s->controller, s->server.control_socket, From 107f7a83abe60ff1a25db42aa864ed13f5a4c0c1 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 25 Feb 2024 14:54:17 +0100 Subject: [PATCH 53/79] Extract binary to hex string conversion PR #4473 --- app/src/usb/aoa_hid.c | 14 +++++--------- app/src/util/str.c | 20 ++++++++++++++++++++ app/src/util/str.h | 6 ++++++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index d6b418a0..50bc33fe 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -5,6 +5,7 @@ #include "aoa_hid.h" #include "util/log.h" +#include "util/str.h" // See . #define ACCESSORY_REGISTER_HID 54 @@ -20,17 +21,12 @@ static void sc_hid_event_log(uint16_t accessory_id, const struct sc_hid_event *event) { // HID Event: [00] FF FF FF FF... assert(event->size); - unsigned buffer_size = event->size * 3 + 1; - char *buffer = malloc(buffer_size); - if (!buffer) { - LOG_OOM(); + char *hex = sc_str_to_hex_string(event->data, event->size); + if (!hex) { return; } - for (unsigned i = 0; i < event->size; ++i) { - snprintf(buffer + i * 3, 4, " %02x", event->data[i]); - } - LOGV("HID Event: [%d]%s", accessory_id, buffer); - free(buffer); + LOGV("HID Event: [%d] %s", accessory_id, hex); + free(hex); } bool diff --git a/app/src/util/str.c b/app/src/util/str.c index d78aa9d7..755369d8 100644 --- a/app/src/util/str.c +++ b/app/src/util/str.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -333,3 +334,22 @@ sc_str_remove_trailing_cr(char *s, size_t len) { } return len; } + +char * +sc_str_to_hex_string(const uint8_t *data, size_t size) { + size_t buffer_size = size * 3 + 1; + char *buffer = malloc(buffer_size); + if (!buffer) { + LOG_OOM(); + return NULL; + } + + for (size_t i = 0; i < size; ++i) { + snprintf(buffer + i * 3, 4, "%02X ", data[i]); + } + + // Remove the final space + buffer[size * 3] = '\0'; + + return buffer; +} diff --git a/app/src/util/str.h b/app/src/util/str.h index 4f7eeeda..20da26f0 100644 --- a/app/src/util/str.h +++ b/app/src/util/str.h @@ -138,4 +138,10 @@ sc_str_index_of_column(const char *s, unsigned col, const char *seps); size_t sc_str_remove_trailing_cr(char *s, size_t len); +/** + * Convert binary data to hexadecimal string + */ +char * +sc_str_to_hex_string(const uint8_t *data, size_t len); + #endif From 4d2c2514fc466fc1dd8cfb972c2697a843e28d70 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 24 Feb 2024 22:33:48 +0100 Subject: [PATCH 54/79] Initialize controller in two steps There is a dependency cycle in the initialization order: - keyboard depends on controller - controller depends on acksync - acksync depends on keyboard initialization To break this cycle, bind the async instance to the controller in a second step. PR #4473 --- app/src/controller.c | 11 ++++++++--- app/src/controller.h | 7 +++++-- app/src/receiver.c | 5 ++--- app/src/receiver.h | 3 +-- app/src/scrcpy.c | 5 +++-- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/app/src/controller.c b/app/src/controller.c index 250321fe..5a5bfde9 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -7,8 +7,7 @@ #define SC_CONTROL_MSG_QUEUE_MAX 64 bool -sc_controller_init(struct sc_controller *controller, sc_socket control_socket, - struct sc_acksync *acksync) { +sc_controller_init(struct sc_controller *controller, sc_socket control_socket) { sc_vecdeque_init(&controller->queue); bool ok = sc_vecdeque_reserve(&controller->queue, SC_CONTROL_MSG_QUEUE_MAX); @@ -16,7 +15,7 @@ sc_controller_init(struct sc_controller *controller, sc_socket control_socket, return false; } - ok = sc_receiver_init(&controller->receiver, control_socket, acksync); + ok = sc_receiver_init(&controller->receiver, control_socket); if (!ok) { sc_vecdeque_destroy(&controller->queue); return false; @@ -43,6 +42,12 @@ sc_controller_init(struct sc_controller *controller, sc_socket control_socket, return true; } +void +sc_controller_set_acksync(struct sc_controller *controller, + struct sc_acksync *acksync) { + controller->receiver.acksync = acksync; +} + void sc_controller_destroy(struct sc_controller *controller) { sc_cond_destroy(&controller->msg_cond); diff --git a/app/src/controller.h b/app/src/controller.h index a044b2bf..767e1731 100644 --- a/app/src/controller.h +++ b/app/src/controller.h @@ -25,8 +25,11 @@ struct sc_controller { }; bool -sc_controller_init(struct sc_controller *controller, sc_socket control_socket, - struct sc_acksync *acksync); +sc_controller_init(struct sc_controller *controller, sc_socket control_socket); + +void +sc_controller_set_acksync(struct sc_controller *controller, + struct sc_acksync *acksync); void sc_controller_destroy(struct sc_controller *controller); diff --git a/app/src/receiver.c b/app/src/receiver.c index 6be705e3..97299b3f 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -8,15 +8,14 @@ #include "util/log.h" bool -sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket, - struct sc_acksync *acksync) { +sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) { bool ok = sc_mutex_init(&receiver->mutex); if (!ok) { return false; } receiver->control_socket = control_socket; - receiver->acksync = acksync; + receiver->acksync = NULL; return true; } diff --git a/app/src/receiver.h b/app/src/receiver.h index eb959fb8..43f89615 100644 --- a/app/src/receiver.h +++ b/app/src/receiver.h @@ -20,8 +20,7 @@ struct sc_receiver { }; bool -sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket, - struct sc_acksync *acksync); +sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket); void sc_receiver_destroy(struct sc_receiver *receiver); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 876b400a..7ecda6d0 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -663,12 +663,13 @@ aoa_hid_end: mp = &s->mouse_sdk.mouse_processor; } - if (!sc_controller_init(&s->controller, s->server.control_socket, - acksync)) { + if (!sc_controller_init(&s->controller, s->server.control_socket)) { goto end; } controller_initialized = true; + sc_controller_set_acksync(&s->controller, acksync); + if (!sc_controller_start(&s->controller)) { goto end; } From 604e59ac7bd907567348627712185f1fa88e4659 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 24 Feb 2024 22:33:58 +0100 Subject: [PATCH 55/79] Initialize controller before keyboards The UHID keyboard initializer will need the controller. PR #4473 --- app/src/scrcpy.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 7ecda6d0..a407dff1 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -542,6 +542,13 @@ scrcpy(struct scrcpy_options *options) { struct sc_mouse_processor *mp = NULL; if (options->control) { + if (!sc_controller_init(&s->controller, s->server.control_socket)) { + goto end; + } + controller_initialized = true; + + controller = &s->controller; + #ifdef HAVE_USB bool use_keyboard_aoa = options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA; @@ -663,18 +670,12 @@ aoa_hid_end: mp = &s->mouse_sdk.mouse_processor; } - if (!sc_controller_init(&s->controller, s->server.control_socket)) { - goto end; - } - controller_initialized = true; - sc_controller_set_acksync(&s->controller, acksync); if (!sc_controller_start(&s->controller)) { goto end; } controller_started = true; - controller = &s->controller; } // There is a controller if and only if control is enabled From 4d5b67cc8018753a02a786cd4ca27e38cad50d65 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 26 Feb 2024 10:20:01 +0100 Subject: [PATCH 56/79] Log controller handling errors On close, the controller is expected to throw an IOException because the socket is closed, so the exception was ignored. However, message handling actions may also throw IOException, and they must not be silently ignored. PR #4473 --- .../com/genymobile/scrcpy/Controller.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 73d6ad57..a3508c96 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -81,8 +81,9 @@ public class Controller implements AsyncProcessor { SystemClock.sleep(500); } - while (!Thread.currentThread().isInterrupted()) { - handleEvent(); + boolean alive = true; + while (!Thread.currentThread().isInterrupted() && alive) { + alive = handleEvent(); } } @@ -92,7 +93,7 @@ public class Controller implements AsyncProcessor { try { control(); } catch (IOException e) { - // this is expected on close + Ln.e("Controller error", e); } finally { Ln.d("Controller stopped"); listener.onTerminated(true); @@ -122,8 +123,15 @@ public class Controller implements AsyncProcessor { return sender; } - private void handleEvent() throws IOException { - ControlMessage msg = controlChannel.recv(); + private boolean handleEvent() throws IOException { + ControlMessage msg; + try { + msg = controlChannel.recv(); + } catch (IOException e) { + // this is expected on close + return false; + } + switch (msg.getType()) { case ControlMessage.TYPE_INJECT_KEYCODE: if (device.supportsInputEvents()) { @@ -185,6 +193,8 @@ public class Controller implements AsyncProcessor { default: // do nothing } + + return true; } private boolean injectKeycode(int action, int keycode, int repeat, int metaState) { From 840680f546be59d1581ae4014a6546e137447cbc Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:17:35 +0800 Subject: [PATCH 57/79] Add UHID keyboard support Use the following command: scrcpy --keyboard=uhid PR #4473 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- app/data/bash-completion/scrcpy | 2 +- app/data/zsh-completion/_scrcpy | 2 +- app/meson.build | 1 + app/scrcpy.1 | 7 +- app/src/cli.c | 25 +++- app/src/control_msg.c | 28 ++++ app/src/control_msg.h | 13 ++ app/src/options.h | 1 + app/src/scrcpy.c | 13 +- app/src/uhid/keyboard_uhid.c | 72 +++++++++ app/src/uhid/keyboard_uhid.h | 23 +++ app/tests/test_control_msg_serialize.c | 49 +++++++ .../com/genymobile/scrcpy/ControlMessage.java | 28 ++++ .../scrcpy/ControlMessageReader.java | 61 +++++++- .../com/genymobile/scrcpy/Controller.java | 10 ++ .../com/genymobile/scrcpy/UhidManager.java | 138 ++++++++++++++++++ .../scrcpy/ControlMessageReaderTest.java | 44 ++++++ 17 files changed, 497 insertions(+), 20 deletions(-) create mode 100644 app/src/uhid/keyboard_uhid.c create mode 100644 app/src/uhid/keyboard_uhid.h create mode 100644 server/src/main/java/com/genymobile/scrcpy/UhidManager.java diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index b2009c56..904ccdeb 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -116,7 +116,7 @@ _scrcpy() { return ;; --keyboard) - COMPREPLY=($(compgen -W 'disabled sdk aoa' -- "$cur")) + COMPREPLY=($(compgen -W 'disabled sdk uhid aoa' -- "$cur")) return ;; --mouse) diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index a4611632..f81d2b22 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -34,7 +34,7 @@ arguments=( '--force-adb-forward[Do not attempt to use \"adb reverse\" to connect to the device]' '--forward-all-clicks[Forward clicks to device]' {-h,--help}'[Print the help]' - '--keyboard[Set the keyboard input mode]:mode:(disabled sdk aoa)' + '--keyboard[Set the keyboard input mode]:mode:(disabled sdk uhid aoa)' '--kill-adb-on-close[Kill adb when scrcpy terminates]' '--legacy-paste[Inject computer clipboard text as a sequence of key events on Ctrl+v]' '--list-camera-sizes[List the valid camera capture sizes]' diff --git a/app/meson.build b/app/meson.build index 3ec9781a..9a2d2838 100644 --- a/app/meson.build +++ b/app/meson.build @@ -35,6 +35,7 @@ src = [ 'src/hid/hid_mouse.c', 'src/trait/frame_source.c', 'src/trait/packet_source.c', + 'src/uhid/keyboard_uhid.c', 'src/util/acksync.c', 'src/util/audiobuf.c', 'src/util/average.c', diff --git a/app/scrcpy.1 b/app/scrcpy.1 index ed2e620e..1dfcab2b 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -175,13 +175,14 @@ Print this help. .BI "\-\-keyboard " mode Select how to send keyboard inputs to the device. -Possible values are "disabled", "sdk" and "aoa": +Possible values are "disabled", "sdk", "uhid" and "aoa": - "disabled" does not send keyboard inputs to the device. - "sdk" uses the Android system API to deliver keyboard events to applications. - - "aoa" simulates a physical keyboard using the AOAv2 protocol. It may only work over USB. + - "uhid" simulates a physical HID keyboard using the Linux HID kernel module on the device. + - "aoa" simulates a physical HID keyboard using the AOAv2 protocol. It may only work over USB. -For "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly: +For "uhid" and "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly: adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS diff --git a/app/src/cli.c b/app/src/cli.c index 364590a4..59cd5699 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -365,19 +365,22 @@ static const struct sc_option options[] = { .longopt = "keyboard", .argdesc = "mode", .text = "Select how to send keyboard inputs to the device.\n" - "Possible values are \"disabled\", \"sdk\" and \"aoa\".\n" + "Possible values are \"disabled\", \"sdk\", \"uhid\" and " + "\"aoa\".\n" "\"disabled\" does not send keyboard inputs to the device.\n" "\"sdk\" uses the Android system API to deliver keyboard " "events to applications.\n" + "\"uhid\" simulates a physical HID keyboard using the Linux " + "UHID kernel module on the device.\n" "\"aoa\" simulates a physical keyboard using the AOAv2 " "protocol. It may only work over USB.\n" - "For \"aoa\", the keyboard layout must be configured (once and " - "for all) on the device, via Settings -> System -> Languages " - "and input -> Physical keyboard. This settings page can be " - "started directly: `adb shell am start -a " + "For \"uhid\" and \"aoa\", the keyboard layout must be " + "configured (once and for all) on the device, via Settings -> " + "System -> Languages and input -> Physical keyboard. This " + "settings page can be started directly: `adb shell am start -a " "android.settings.HARD_KEYBOARD_SETTINGS`.\n" - "This option is only available when the HID keyboard is " - "enabled (or a physical keyboard is connected).\n" + "This option is only available when a HID keyboard is enabled " + "(or a physical keyboard is connected).\n" "Also see --mouse.", }, { @@ -1939,6 +1942,11 @@ parse_keyboard(const char *optarg, enum sc_keyboard_input_mode *mode) { return true; } + if (!strcmp(optarg, "uhid")) { + *mode = SC_KEYBOARD_INPUT_MODE_UHID; + return true; + } + if (!strcmp(optarg, "aoa")) { #ifdef HAVE_USB *mode = SC_KEYBOARD_INPUT_MODE_AOA; @@ -1949,7 +1957,8 @@ parse_keyboard(const char *optarg, enum sc_keyboard_input_mode *mode) { #endif } - LOGE("Unsupported keyboard: %s (expected disabled, sdk or aoa)", optarg); + LOGE("Unsupported keyboard: %s (expected disabled, sdk, uhid and aoa)", + optarg); return false; } diff --git a/app/src/control_msg.c b/app/src/control_msg.c index e173dac7..88575b4e 100644 --- a/app/src/control_msg.c +++ b/app/src/control_msg.c @@ -146,6 +146,17 @@ sc_control_msg_serialize(const struct sc_control_msg *msg, uint8_t *buf) { case SC_CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE: buf[1] = msg->set_screen_power_mode.mode; return 2; + case SC_CONTROL_MSG_TYPE_UHID_CREATE: + sc_write16be(&buf[1], msg->uhid_create.id); + sc_write16be(&buf[3], msg->uhid_create.report_desc_size); + memcpy(&buf[5], msg->uhid_create.report_desc, + msg->uhid_create.report_desc_size); + return 5 + msg->uhid_create.report_desc_size; + case SC_CONTROL_MSG_TYPE_UHID_INPUT: + sc_write16be(&buf[1], msg->uhid_input.id); + sc_write16be(&buf[3], msg->uhid_input.size); + memcpy(&buf[5], msg->uhid_input.data, msg->uhid_input.size); + return 5 + msg->uhid_input.size; case SC_CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL: case SC_CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL: case SC_CONTROL_MSG_TYPE_COLLAPSE_PANELS: @@ -242,6 +253,23 @@ sc_control_msg_log(const struct sc_control_msg *msg) { case SC_CONTROL_MSG_TYPE_ROTATE_DEVICE: LOG_CMSG("rotate device"); break; + case SC_CONTROL_MSG_TYPE_UHID_CREATE: + LOG_CMSG("UHID create [%" PRIu16 "] report_desc_size=%" PRIu16, + msg->uhid_create.id, msg->uhid_create.report_desc_size); + break; + case SC_CONTROL_MSG_TYPE_UHID_INPUT: { + char *hex = sc_str_to_hex_string(msg->uhid_input.data, + msg->uhid_input.size); + if (hex) { + LOG_CMSG("UHID input [%" PRIu16 "] %s", + msg->uhid_input.id, hex); + free(hex); + } else { + LOG_CMSG("UHID input [%" PRIu16 "] size=%" PRIu16, + msg->uhid_input.id, msg->uhid_input.size); + } + break; + } default: LOG_CMSG("unknown type: %u", (unsigned) msg->type); break; diff --git a/app/src/control_msg.h b/app/src/control_msg.h index 04eeb83b..550168c2 100644 --- a/app/src/control_msg.h +++ b/app/src/control_msg.h @@ -10,6 +10,7 @@ #include "android/input.h" #include "android/keycodes.h" #include "coords.h" +#include "hid/hid_event.h" #define SC_CONTROL_MSG_MAX_SIZE (1 << 18) // 256k @@ -37,6 +38,8 @@ enum sc_control_msg_type { SC_CONTROL_MSG_TYPE_SET_CLIPBOARD, SC_CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE, SC_CONTROL_MSG_TYPE_ROTATE_DEVICE, + SC_CONTROL_MSG_TYPE_UHID_CREATE, + SC_CONTROL_MSG_TYPE_UHID_INPUT, }; enum sc_screen_power_mode { @@ -92,6 +95,16 @@ struct sc_control_msg { struct { enum sc_screen_power_mode mode; } set_screen_power_mode; + struct { + uint16_t id; + uint16_t report_desc_size; + const uint8_t *report_desc; // pointer to static data + } uhid_create; + struct { + uint16_t id; + uint16_t size; + uint8_t data[SC_HID_MAX_SIZE]; + } uhid_input; }; }; diff --git a/app/src/options.h b/app/src/options.h index 1fb31c1a..6d62fac0 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -143,6 +143,7 @@ enum sc_keyboard_input_mode { SC_KEYBOARD_INPUT_MODE_AUTO, SC_KEYBOARD_INPUT_MODE_DISABLED, SC_KEYBOARD_INPUT_MODE_SDK, + SC_KEYBOARD_INPUT_MODE_UHID, SC_KEYBOARD_INPUT_MODE_AOA, }; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index a407dff1..d01d3619 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -25,6 +25,7 @@ #include "recorder.h" #include "screen.h" #include "server.h" +#include "uhid/keyboard_uhid.h" #ifdef HAVE_USB # include "usb/aoa_hid.h" # include "usb/keyboard_aoa.h" @@ -64,6 +65,7 @@ struct scrcpy { #endif union { struct sc_keyboard_sdk keyboard_sdk; + struct sc_keyboard_uhid keyboard_uhid; #ifdef HAVE_USB struct sc_keyboard_aoa keyboard_aoa; #endif @@ -656,15 +658,22 @@ aoa_hid_end: assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_AOA); #endif - // keyboard_input_mode may have been reset if HID mode failed + // keyboard_input_mode may have been reset if AOA mode failed if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_SDK) { sc_keyboard_sdk_init(&s->keyboard_sdk, &s->controller, options->key_inject_mode, options->forward_key_repeat); kp = &s->keyboard_sdk.key_processor; + } else if (options->keyboard_input_mode + == SC_KEYBOARD_INPUT_MODE_UHID) { + bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller); + if (!ok) { + goto end; + } + kp = &s->keyboard_uhid.key_processor; } - // mouse_input_mode may have been reset if HID mode failed + // mouse_input_mode may have been reset if AOA mode failed if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_SDK) { sc_mouse_sdk_init(&s->mouse_sdk, &s->controller); mp = &s->mouse_sdk.mouse_processor; diff --git a/app/src/uhid/keyboard_uhid.c b/app/src/uhid/keyboard_uhid.c new file mode 100644 index 00000000..d974d578 --- /dev/null +++ b/app/src/uhid/keyboard_uhid.c @@ -0,0 +1,72 @@ +#include "keyboard_uhid.h" + +#include "util/log.h" + +/** Downcast key processor to keyboard_uhid */ +#define DOWNCAST(KP) container_of(KP, struct sc_keyboard_uhid, key_processor) + +#define UHID_KEYBOARD_ID 1 + +static void +sc_key_processor_process_key(struct sc_key_processor *kp, + const struct sc_key_event *event, + uint64_t ack_to_wait) { + (void) ack_to_wait; + + if (event->repeat) { + // In USB HID protocol, key repeat is handled by the host (Android), so + // just ignore key repeat here. + return; + } + + struct sc_keyboard_uhid *kb = DOWNCAST(kp); + + struct sc_hid_event hid_event; + + // Not all keys are supported, just ignore unsupported keys + if (sc_hid_keyboard_event_from_key(&kb->hid, &hid_event, event)) { + struct sc_control_msg msg; + msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT; + msg.uhid_input.id = UHID_KEYBOARD_ID; + + assert(hid_event.size <= SC_HID_MAX_SIZE); + memcpy(msg.uhid_input.data, hid_event.data, hid_event.size); + msg.uhid_input.size = hid_event.size; + + if (!sc_controller_push_msg(kb->controller, &msg)) { + LOGE("Could not send UHID_INPUT message (key)"); + } + } +} + +bool +sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb, + struct sc_controller *controller) { + sc_hid_keyboard_init(&kb->hid); + + kb->controller = controller; + + static const struct sc_key_processor_ops ops = { + .process_key = sc_key_processor_process_key, + // Never forward text input via HID (all the keys are injected + // separately) + .process_text = NULL, + }; + + // Clipboard synchronization is requested over the same control socket, so + // there is no need for a specific synchronization mechanism + kb->key_processor.async_paste = false; + kb->key_processor.ops = &ops; + + struct sc_control_msg msg; + msg.type = SC_CONTROL_MSG_TYPE_UHID_CREATE; + msg.uhid_create.id = UHID_KEYBOARD_ID; + msg.uhid_create.report_desc = SC_HID_KEYBOARD_REPORT_DESC; + msg.uhid_create.report_desc_size = SC_HID_KEYBOARD_REPORT_DESC_LEN; + if (!sc_controller_push_msg(controller, &msg)) { + LOGE("Could not send UHID_CREATE message (keyboard)"); + return false; + } + + return true; +} diff --git a/app/src/uhid/keyboard_uhid.h b/app/src/uhid/keyboard_uhid.h new file mode 100644 index 00000000..854ba008 --- /dev/null +++ b/app/src/uhid/keyboard_uhid.h @@ -0,0 +1,23 @@ +#ifndef SC_KEYBOARD_UHID_H +#define SC_KEYBOARD_UHID_H + +#include "common.h" + +#include + +#include "controller.h" +#include "hid/hid_keyboard.h" +#include "trait/key_processor.h" + +struct sc_keyboard_uhid { + struct sc_key_processor key_processor; // key processor trait + + struct sc_hid_keyboard hid; + struct sc_controller *controller; +}; + +bool +sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb, + struct sc_controller *controller); + +#endif diff --git a/app/tests/test_control_msg_serialize.c b/app/tests/test_control_msg_serialize.c index 80d33fc3..0ab61153 100644 --- a/app/tests/test_control_msg_serialize.c +++ b/app/tests/test_control_msg_serialize.c @@ -323,6 +323,53 @@ static void test_serialize_rotate_device(void) { assert(!memcmp(buf, expected, sizeof(expected))); } +static void test_serialize_uhid_create(void) { + const uint8_t report_desc[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + struct sc_control_msg msg = { + .type = SC_CONTROL_MSG_TYPE_UHID_CREATE, + .uhid_create = { + .id = 42, + .report_desc_size = sizeof(report_desc), + .report_desc = report_desc, + }, + }; + + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; + size_t size = sc_control_msg_serialize(&msg, buf); + assert(size == 16); + + const uint8_t expected[] = { + SC_CONTROL_MSG_TYPE_UHID_CREATE, + 0, 42, // id + 0, 11, // size + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + }; + assert(!memcmp(buf, expected, sizeof(expected))); +} + +static void test_serialize_uhid_input(void) { + struct sc_control_msg msg = { + .type = SC_CONTROL_MSG_TYPE_UHID_INPUT, + .uhid_input = { + .id = 42, + .size = 5, + .data = {1, 2, 3, 4, 5}, + }, + }; + + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; + size_t size = sc_control_msg_serialize(&msg, buf); + assert(size == 10); + + const uint8_t expected[] = { + SC_CONTROL_MSG_TYPE_UHID_INPUT, + 0, 42, // id + 0, 5, // size + 1, 2, 3, 4, 5, + }; + assert(!memcmp(buf, expected, sizeof(expected))); +} + int main(int argc, char *argv[]) { (void) argc; (void) argv; @@ -341,5 +388,7 @@ int main(int argc, char *argv[]) { test_serialize_set_clipboard_long(); test_serialize_set_screen_power_mode(); test_serialize_rotate_device(); + test_serialize_uhid_create(); + test_serialize_uhid_input(); return 0; } diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java index e1800374..74bf5610 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java @@ -17,6 +17,8 @@ public final class ControlMessage { public static final int TYPE_SET_CLIPBOARD = 9; public static final int TYPE_SET_SCREEN_POWER_MODE = 10; public static final int TYPE_ROTATE_DEVICE = 11; + public static final int TYPE_UHID_CREATE = 12; + public static final int TYPE_UHID_INPUT = 13; public static final long SEQUENCE_INVALID = 0; @@ -40,6 +42,8 @@ public final class ControlMessage { private boolean paste; private int repeat; private long sequence; + private int id; + private byte[] data; private ControlMessage() { } @@ -123,6 +127,22 @@ public final class ControlMessage { return msg; } + public static ControlMessage createUhidCreate(int id, byte[] reportDesc) { + ControlMessage msg = new ControlMessage(); + msg.type = TYPE_UHID_CREATE; + msg.id = id; + msg.data = reportDesc; + return msg; + } + + public static ControlMessage createUhidInput(int id, byte[] data) { + ControlMessage msg = new ControlMessage(); + msg.type = TYPE_UHID_INPUT; + msg.id = id; + msg.data = data; + return msg; + } + public int getType() { return type; } @@ -186,4 +206,12 @@ public final class ControlMessage { public long getSequence() { return sequence; } + + public int getId() { + return id; + } + + public byte[] getData() { + return data; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java index d95c36d8..24aa73c0 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java @@ -15,6 +15,8 @@ public class ControlMessageReader { static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1; static final int GET_CLIPBOARD_LENGTH = 1; static final int SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH = 9; + static final int UHID_CREATE_FIXED_PAYLOAD_LENGTH = 4; + static final int UHID_INPUT_FIXED_PAYLOAD_LENGTH = 4; private static final int MESSAGE_MAX_SIZE = 1 << 18; // 256k @@ -86,6 +88,12 @@ public class ControlMessageReader { case ControlMessage.TYPE_ROTATE_DEVICE: msg = ControlMessage.createEmpty(type); break; + case ControlMessage.TYPE_UHID_CREATE: + msg = parseUhidCreate(); + break; + case ControlMessage.TYPE_UHID_INPUT: + msg = parseUhidInput(); + break; default: Ln.w("Unknown event type: " + type); msg = null; @@ -110,12 +118,21 @@ public class ControlMessageReader { return ControlMessage.createInjectKeycode(action, keycode, repeat, metaState); } - private String parseString() { - if (buffer.remaining() < 4) { - return null; + private int parseBufferLength(int sizeBytes) { + assert sizeBytes > 0 && sizeBytes <= 4; + if (buffer.remaining() < sizeBytes) { + return -1; + } + int value = 0; + for (int i = 0; i < sizeBytes; ++i) { + value = (value << 8) | (buffer.get() & 0xFF); } - int len = buffer.getInt(); - if (buffer.remaining() < len) { + return value; + } + + private String parseString() { + int len = parseBufferLength(4); + if (len == -1 || buffer.remaining() < len) { return null; } int position = buffer.position(); @@ -124,6 +141,16 @@ public class ControlMessageReader { return new String(rawBuffer, position, len, StandardCharsets.UTF_8); } + private byte[] parseByteArray(int sizeBytes) { + int len = parseBufferLength(sizeBytes); + if (len == -1 || buffer.remaining() < len) { + return null; + } + byte[] data = new byte[len]; + buffer.get(data); + return data; + } + private ControlMessage parseInjectText() { String text = parseString(); if (text == null) { @@ -193,6 +220,30 @@ public class ControlMessageReader { return ControlMessage.createSetScreenPowerMode(mode); } + private ControlMessage parseUhidCreate() { + if (buffer.remaining() < UHID_CREATE_FIXED_PAYLOAD_LENGTH) { + return null; + } + int id = buffer.getShort(); + byte[] data = parseByteArray(2); + if (data == null) { + return null; + } + return ControlMessage.createUhidCreate(id, data); + } + + private ControlMessage parseUhidInput() { + if (buffer.remaining() < UHID_INPUT_FIXED_PAYLOAD_LENGTH) { + return null; + } + int id = buffer.getShort(); + byte[] data = parseByteArray(2); + if (data == null) { + return null; + } + return ControlMessage.createUhidInput(id, data); + } + private static Position readPosition(ByteBuffer buffer) { int x = buffer.getInt(); int y = buffer.getInt(); diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index a3508c96..d757d577 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -26,6 +26,8 @@ public class Controller implements AsyncProcessor { private Thread thread; + private final UhidManager uhidManager; + private final Device device; private final ControlChannel controlChannel; private final CleanUp cleanUp; @@ -50,6 +52,7 @@ public class Controller implements AsyncProcessor { this.powerOn = powerOn; initPointers(); sender = new DeviceMessageSender(controlChannel); + uhidManager = new UhidManager(); } private void initPointers() { @@ -96,6 +99,7 @@ public class Controller implements AsyncProcessor { Ln.e("Controller error", e); } finally { Ln.d("Controller stopped"); + uhidManager.closeAll(); listener.onTerminated(true); } }, "control-recv"); @@ -190,6 +194,12 @@ public class Controller implements AsyncProcessor { case ControlMessage.TYPE_ROTATE_DEVICE: device.rotateDevice(); break; + case ControlMessage.TYPE_UHID_CREATE: + uhidManager.open(msg.getId(), msg.getData()); + break; + case ControlMessage.TYPE_UHID_INPUT: + uhidManager.writeInput(msg.getId(), msg.getData()); + break; default: // do nothing } diff --git a/server/src/main/java/com/genymobile/scrcpy/UhidManager.java b/server/src/main/java/com/genymobile/scrcpy/UhidManager.java new file mode 100644 index 00000000..96458bf0 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/UhidManager.java @@ -0,0 +1,138 @@ +package com.genymobile.scrcpy; + +import android.system.ErrnoException; +import android.system.Os; +import android.system.OsConstants; +import android.util.ArrayMap; + +import java.io.FileDescriptor; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +public final class UhidManager { + + // Linux: include/uapi/linux/uhid.h + private static final int UHID_CREATE2 = 11; + private static final int UHID_INPUT2 = 12; + + // Linux: include/uapi/linux/input.h + private static final short BUS_VIRTUAL = 0x06; + + private final ArrayMap fds = new ArrayMap<>(); + + public void open(int id, byte[] reportDesc) throws IOException { + try { + FileDescriptor fd = Os.open("/dev/uhid", OsConstants.O_RDWR, 0); + try { + FileDescriptor old = fds.put(id, fd); + if (old != null) { + Ln.w("Duplicate UHID id: " + id); + close(old); + } + + byte[] req = buildUhidCreate2Req(reportDesc); + Os.write(fd, req, 0, req.length); + } catch (Exception e) { + close(fd); + throw e; + } + } catch (ErrnoException e) { + throw new IOException(e); + } + } + + public void writeInput(int id, byte[] data) throws IOException { + FileDescriptor fd = fds.get(id); + if (fd == null) { + Ln.w("Unknown UHID id: " + id); + return; + } + + try { + byte[] req = buildUhidInput2Req(data); + Os.write(fd, req, 0, req.length); + } catch (ErrnoException e) { + throw new IOException(e); + } + } + + private static byte[] buildUhidCreate2Req(byte[] reportDesc) { + /* + * struct uhid_event { + * uint32_t type; + * union { + * // ... + * struct uhid_create2_req { + * uint8_t name[128]; + * uint8_t phys[64]; + * uint8_t uniq[64]; + * uint16_t rd_size; + * uint16_t bus; + * uint32_t vendor; + * uint32_t product; + * uint32_t version; + * uint32_t country; + * uint8_t rd_data[HID_MAX_DESCRIPTOR_SIZE]; + * }; + * }; + * } __attribute__((__packed__)); + */ + + byte[] empty = new byte[256]; + ByteBuffer buf = ByteBuffer.allocate(280 + reportDesc.length).order(ByteOrder.nativeOrder()); + buf.putInt(UHID_CREATE2); + buf.put("scrcpy".getBytes(StandardCharsets.US_ASCII)); + buf.put(empty, 0, 256 - "scrcpy".length()); + buf.putShort((short) reportDesc.length); + buf.putShort(BUS_VIRTUAL); + buf.putInt(0); // vendor id + buf.putInt(0); // product id + buf.putInt(0); // version + buf.putInt(0); // country; + buf.put(reportDesc); + return buf.array(); + } + + private static byte[] buildUhidInput2Req(byte[] data) { + /* + * struct uhid_event { + * uint32_t type; + * union { + * // ... + * struct uhid_input2_req { + * uint16_t size; + * uint8_t data[UHID_DATA_MAX]; + * }; + * }; + * } __attribute__((__packed__)); + */ + + ByteBuffer buf = ByteBuffer.allocate(6 + data.length).order(ByteOrder.nativeOrder()); + buf.putInt(UHID_INPUT2); + buf.putShort((short) data.length); + buf.put(data); + return buf.array(); + } + + public void close(int id) { + FileDescriptor fd = fds.get(id); + assert fd != null; + close(fd); + } + + public void closeAll() { + for (FileDescriptor fd : fds.values()) { + close(fd); + } + } + + private static void close(FileDescriptor fd) { + try { + Os.close(fd); + } catch (ErrnoException e) { + Ln.e("Failed to close uhid: " + e.getMessage()); + } + } +} diff --git a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java index 47097c78..7cc67c3e 100644 --- a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java +++ b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java @@ -322,6 +322,50 @@ public class ControlMessageReaderTest { Assert.assertEquals(ControlMessage.TYPE_ROTATE_DEVICE, event.getType()); } + @Test + public void testParseUhidCreate() throws IOException { + ControlMessageReader reader = new ControlMessageReader(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + dos.writeByte(ControlMessage.TYPE_UHID_CREATE); + dos.writeShort(42); // id + byte[] data = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + dos.writeShort(data.length); // size + dos.write(data); + + byte[] packet = bos.toByteArray(); + + reader.readFrom(new ByteArrayInputStream(packet)); + ControlMessage event = reader.next(); + + Assert.assertEquals(ControlMessage.TYPE_UHID_CREATE, event.getType()); + Assert.assertEquals(42, event.getId()); + Assert.assertArrayEquals(data, event.getData()); + } + + @Test + public void testParseUhidInput() throws IOException { + ControlMessageReader reader = new ControlMessageReader(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + dos.writeByte(ControlMessage.TYPE_UHID_INPUT); + dos.writeShort(42); // id + byte[] data = {1, 2, 3, 4, 5}; + dos.writeShort(data.length); // size + dos.write(data); + + byte[] packet = bos.toByteArray(); + + reader.readFrom(new ByteArrayInputStream(packet)); + ControlMessage event = reader.next(); + + Assert.assertEquals(ControlMessage.TYPE_UHID_INPUT, event.getType()); + Assert.assertEquals(42, event.getId()); + Assert.assertArrayEquals(data, event.getData()); + } + @Test public void testMultiEvents() throws IOException { ControlMessageReader reader = new ControlMessageReader(); From 021c5d371ad0d2c932b981151f45f794d8843ebe Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 24 Feb 2024 22:27:53 +0100 Subject: [PATCH 58/79] Refactor DeviceMessageSender Refactor DeviceMessage as a queue of message. This will allow to add other message types. PR #4473 --- .../com/genymobile/scrcpy/Controller.java | 6 ++- .../com/genymobile/scrcpy/DeviceMessage.java | 2 - .../scrcpy/DeviceMessageSender.java | 42 ++++--------------- .../java/com/genymobile/scrcpy/Server.java | 5 ++- 4 files changed, 17 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index d757d577..b925dd80 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -413,7 +413,8 @@ public class Controller implements AsyncProcessor { if (!clipboardAutosync) { String clipboardText = Device.getClipboardText(); if (clipboardText != null) { - sender.pushClipboardText(clipboardText); + DeviceMessage msg = DeviceMessage.createClipboard(clipboardText); + sender.send(msg); } } } @@ -431,7 +432,8 @@ public class Controller implements AsyncProcessor { if (sequence != ControlMessage.SEQUENCE_INVALID) { // Acknowledgement requested - sender.pushAckClipboard(sequence); + DeviceMessage msg = DeviceMessage.createAckClipboard(sequence); + sender.send(msg); } return ok; diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java index 5b7c4de5..2e333e3f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java @@ -5,8 +5,6 @@ public final class DeviceMessage { public static final int TYPE_CLIPBOARD = 0; public static final int TYPE_ACK_CLIPBOARD = 1; - public static final long SEQUENCE_INVALID = ControlMessage.SEQUENCE_INVALID; - private int type; private String text; private long sequence; diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java index efb7b975..af14bb4e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java @@ -1,54 +1,30 @@ package com.genymobile.scrcpy; import java.io.IOException; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; public final class DeviceMessageSender { private final ControlChannel controlChannel; private Thread thread; - - private String clipboardText; - - private long ack; + private final BlockingQueue queue = new ArrayBlockingQueue<>(16); public DeviceMessageSender(ControlChannel controlChannel) { this.controlChannel = controlChannel; } - public synchronized void pushClipboardText(String text) { - clipboardText = text; - notify(); - } - - public synchronized void pushAckClipboard(long sequence) { - ack = sequence; - notify(); + public void send(DeviceMessage msg) { + if (!queue.offer(msg)) { + Ln.w("Device message dropped: " + msg.getType()); + } } private void loop() throws IOException, InterruptedException { while (!Thread.currentThread().isInterrupted()) { - String text; - long sequence; - synchronized (this) { - while (ack == DeviceMessage.SEQUENCE_INVALID && clipboardText == null) { - wait(); - } - text = clipboardText; - clipboardText = null; - - sequence = ack; - ack = DeviceMessage.SEQUENCE_INVALID; - } - - if (sequence != DeviceMessage.SEQUENCE_INVALID) { - DeviceMessage event = DeviceMessage.createAckClipboard(sequence); - controlChannel.send(event); - } - if (text != null) { - DeviceMessage event = DeviceMessage.createClipboard(text); - controlChannel.send(event); - } + DeviceMessage msg = queue.take(); + controlChannel.send(msg); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 3936648d..587a46df 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -133,7 +133,10 @@ public final class Server { if (control) { ControlChannel controlChannel = connection.getControlChannel(); Controller controller = new Controller(device, controlChannel, cleanUp, options.getClipboardAutosync(), options.getPowerOn()); - device.setClipboardListener(text -> controller.getSender().pushClipboardText(text)); + device.setClipboardListener(text -> { + DeviceMessage msg = DeviceMessage.createClipboard(text); + controller.getSender().send(msg); + }); asyncProcessors.add(controller); } From 87da68ee0d74831a2b44230c573a3b315c8fd7d3 Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Fri, 12 Jan 2024 23:32:30 +0800 Subject: [PATCH 59/79] Handle UHID output Use UHID output reports to synchronize CapsLock and VerrNum states. PR #4473 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- app/meson.build | 1 + app/src/controller.c | 6 +- app/src/controller.h | 5 +- app/src/device_msg.c | 37 +++++- app/src/device_msg.h | 6 + app/src/receiver.c | 38 ++++++ app/src/receiver.h | 2 + app/src/scrcpy.c | 9 +- app/src/uhid/keyboard_uhid.c | 109 ++++++++++++++++-- app/src/uhid/keyboard_uhid.h | 6 +- app/src/uhid/uhid_output.c | 25 ++++ app/src/uhid/uhid_output.h | 45 ++++++++ app/tests/test_device_msg_deserialize.c | 23 ++++ .../com/genymobile/scrcpy/Controller.java | 2 +- .../com/genymobile/scrcpy/DeviceMessage.java | 19 +++ .../scrcpy/DeviceMessageWriter.java | 7 ++ .../com/genymobile/scrcpy/UhidManager.java | 80 +++++++++++++ .../scrcpy/DeviceMessageWriterTest.java | 23 ++++ 18 files changed, 423 insertions(+), 20 deletions(-) create mode 100644 app/src/uhid/uhid_output.c create mode 100644 app/src/uhid/uhid_output.h diff --git a/app/meson.build b/app/meson.build index 9a2d2838..3695e0f9 100644 --- a/app/meson.build +++ b/app/meson.build @@ -36,6 +36,7 @@ src = [ 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/uhid/keyboard_uhid.c', + 'src/uhid/uhid_output.c', 'src/util/acksync.c', 'src/util/audiobuf.c', 'src/util/average.c', diff --git a/app/src/controller.c b/app/src/controller.c index 5a5bfde9..499cfd3c 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -43,9 +43,11 @@ sc_controller_init(struct sc_controller *controller, sc_socket control_socket) { } void -sc_controller_set_acksync(struct sc_controller *controller, - struct sc_acksync *acksync) { +sc_controller_configure(struct sc_controller *controller, + struct sc_acksync *acksync, + struct sc_uhid_devices *uhid_devices) { controller->receiver.acksync = acksync; + controller->receiver.uhid_devices = uhid_devices; } void diff --git a/app/src/controller.h b/app/src/controller.h index 767e1731..1e44427e 100644 --- a/app/src/controller.h +++ b/app/src/controller.h @@ -28,8 +28,9 @@ bool sc_controller_init(struct sc_controller *controller, sc_socket control_socket); void -sc_controller_set_acksync(struct sc_controller *controller, - struct sc_acksync *acksync); +sc_controller_configure(struct sc_controller *controller, + struct sc_acksync *acksync, + struct sc_uhid_devices *uhid_devices); void sc_controller_destroy(struct sc_controller *controller); diff --git a/app/src/device_msg.c b/app/src/device_msg.c index 0cadc49c..7621c040 100644 --- a/app/src/device_msg.c +++ b/app/src/device_msg.c @@ -46,6 +46,31 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len, msg->ack_clipboard.sequence = sequence; return 9; } + case DEVICE_MSG_TYPE_UHID_OUTPUT: { + if (len < 5) { + // at least id + size + return 0; // not available + } + uint16_t id = sc_read16be(&buf[1]); + size_t size = sc_read16be(&buf[3]); + if (size < len - 5) { + return 0; // not available + } + uint8_t *data = malloc(size); + if (!data) { + LOG_OOM(); + return -1; + } + if (size) { + memcpy(data, &buf[5], size); + } + + msg->uhid_output.id = id; + msg->uhid_output.size = size; + msg->uhid_output.data = data; + + return 5 + size; + } default: LOGW("Unknown device message type: %d", (int) msg->type); return -1; // error, we cannot recover @@ -54,7 +79,15 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len, void sc_device_msg_destroy(struct sc_device_msg *msg) { - if (msg->type == DEVICE_MSG_TYPE_CLIPBOARD) { - free(msg->clipboard.text); + switch (msg->type) { + case DEVICE_MSG_TYPE_CLIPBOARD: + free(msg->clipboard.text); + break; + case DEVICE_MSG_TYPE_UHID_OUTPUT: + free(msg->uhid_output.data); + break; + default: + // nothing to do + break; } } diff --git a/app/src/device_msg.h b/app/src/device_msg.h index 3f541cf5..86b2ccb7 100644 --- a/app/src/device_msg.h +++ b/app/src/device_msg.h @@ -14,6 +14,7 @@ enum sc_device_msg_type { DEVICE_MSG_TYPE_CLIPBOARD, DEVICE_MSG_TYPE_ACK_CLIPBOARD, + DEVICE_MSG_TYPE_UHID_OUTPUT, }; struct sc_device_msg { @@ -25,6 +26,11 @@ struct sc_device_msg { struct { uint64_t sequence; } ack_clipboard; + struct { + uint16_t id; + uint16_t size; + uint8_t *data; // owned, to be freed by free() + } uhid_output; }; }; diff --git a/app/src/receiver.c b/app/src/receiver.c index 97299b3f..f4ebd3f8 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -1,11 +1,13 @@ #include "receiver.h" #include +#include #include #include #include "device_msg.h" #include "util/log.h" +#include "util/str.h" bool sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) { @@ -16,6 +18,7 @@ sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) { receiver->control_socket = control_socket; receiver->acksync = NULL; + receiver->uhid_devices = NULL; return true; } @@ -57,6 +60,41 @@ process_msg(struct sc_receiver *receiver, struct sc_device_msg *msg) { sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence); break; + case DEVICE_MSG_TYPE_UHID_OUTPUT: + if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) { + char *hex = sc_str_to_hex_string(msg->uhid_output.data, + msg->uhid_output.size); + if (hex) { + LOGV("UHID output [%" PRIu16 "] %s", + msg->uhid_output.id, hex); + free(hex); + } else { + LOGV("UHID output [%" PRIu16 "] size=%" PRIu16, + msg->uhid_output.id, msg->uhid_output.size); + } + } + + // This is a programming error to receive this message if there is + // no uhid_devices instance + assert(receiver->uhid_devices); + + // Also check at runtime (do not trust the server) + if (!receiver->uhid_devices) { + LOGE("Received unexpected HID output message"); + return; + } + + struct sc_uhid_receiver *uhid_receiver = + sc_uhid_devices_get_receiver(receiver->uhid_devices, + msg->uhid_output.id); + if (uhid_receiver) { + uhid_receiver->ops->process_output(uhid_receiver, + msg->uhid_output.data, + msg->uhid_output.size); + } else { + LOGW("No UHID receiver for id %" PRIu16, msg->uhid_output.id); + } + break; } } diff --git a/app/src/receiver.h b/app/src/receiver.h index 43f89615..ba84c0ab 100644 --- a/app/src/receiver.h +++ b/app/src/receiver.h @@ -5,6 +5,7 @@ #include +#include "uhid/uhid_output.h" #include "util/acksync.h" #include "util/net.h" #include "util/thread.h" @@ -17,6 +18,7 @@ struct sc_receiver { sc_mutex mutex; struct sc_acksync *acksync; + struct sc_uhid_devices *uhid_devices; }; bool diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index d01d3619..7b1a6d5c 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -62,6 +62,7 @@ struct scrcpy { struct sc_aoa aoa; // sequence/ack helper to synchronize clipboard and Ctrl+v via HID struct sc_acksync acksync; + struct sc_uhid_devices uhid_devices; #endif union { struct sc_keyboard_sdk keyboard_sdk; @@ -342,6 +343,7 @@ scrcpy(struct scrcpy_options *options) { bool timeout_started = false; struct sc_acksync *acksync = NULL; + struct sc_uhid_devices *uhid_devices = NULL; uint32_t scid = scrcpy_generate_scid(); @@ -666,10 +668,13 @@ aoa_hid_end: kp = &s->keyboard_sdk.key_processor; } else if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_UHID) { - bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller); + sc_uhid_devices_init(&s->uhid_devices); + bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller, + &s->uhid_devices); if (!ok) { goto end; } + uhid_devices = &s->uhid_devices; kp = &s->keyboard_uhid.key_processor; } @@ -679,7 +684,7 @@ aoa_hid_end: mp = &s->mouse_sdk.mouse_processor; } - sc_controller_set_acksync(&s->controller, acksync); + sc_controller_configure(&s->controller, acksync, uhid_devices); if (!sc_controller_start(&s->controller)) { goto end; diff --git a/app/src/uhid/keyboard_uhid.c b/app/src/uhid/keyboard_uhid.c index d974d578..f537bc29 100644 --- a/app/src/uhid/keyboard_uhid.c +++ b/app/src/uhid/keyboard_uhid.c @@ -5,8 +5,52 @@ /** Downcast key processor to keyboard_uhid */ #define DOWNCAST(KP) container_of(KP, struct sc_keyboard_uhid, key_processor) +/** Downcast uhid_receiver to keyboard_uhid */ +#define DOWNCAST_RECEIVER(UR) \ + container_of(UR, struct sc_keyboard_uhid, uhid_receiver) + #define UHID_KEYBOARD_ID 1 +static void +sc_keyboard_uhid_send_input(struct sc_keyboard_uhid *kb, + const struct sc_hid_event *event) { + struct sc_control_msg msg; + msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT; + msg.uhid_input.id = UHID_KEYBOARD_ID; + + assert(event->size <= SC_HID_MAX_SIZE); + memcpy(msg.uhid_input.data, event->data, event->size); + msg.uhid_input.size = event->size; + + if (!sc_controller_push_msg(kb->controller, &msg)) { + LOGE("Could not send UHID_INPUT message (key)"); + } +} + +static void +sc_keyboard_uhid_synchronize_mod(struct sc_keyboard_uhid *kb) { + SDL_Keymod sdl_mod = SDL_GetModState(); + uint16_t mod = sc_mods_state_from_sdl(sdl_mod) & (SC_MOD_CAPS | SC_MOD_NUM); + + uint16_t device_mod = + atomic_load_explicit(&kb->device_mod, memory_order_relaxed); + uint16_t diff = mod ^ device_mod; + + if (diff) { + // Inherently racy (the HID output reports arrive asynchronously in + // response to key presses), but will re-synchronize on next key press + // or HID output anyway + atomic_store_explicit(&kb->device_mod, mod, memory_order_relaxed); + + struct sc_hid_event hid_event; + sc_hid_keyboard_event_from_mods(&hid_event, diff); + + LOGV("HID keyboard state synchronized"); + + sc_keyboard_uhid_send_input(kb, &hid_event); + } +} + static void sc_key_processor_process_key(struct sc_key_processor *kp, const struct sc_key_event *event, @@ -25,26 +69,63 @@ sc_key_processor_process_key(struct sc_key_processor *kp, // Not all keys are supported, just ignore unsupported keys if (sc_hid_keyboard_event_from_key(&kb->hid, &hid_event, event)) { - struct sc_control_msg msg; - msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT; - msg.uhid_input.id = UHID_KEYBOARD_ID; + if (event->scancode == SC_SCANCODE_CAPSLOCK) { + atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_CAPS, + memory_order_relaxed); + } else if (event->scancode == SC_SCANCODE_NUMLOCK) { + atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_NUM, + memory_order_relaxed); + } else { + // Synchronize modifiers (only if the scancode itself does not + // change the modifiers) + sc_keyboard_uhid_synchronize_mod(kb); + } + sc_keyboard_uhid_send_input(kb, &hid_event); + } +} - assert(hid_event.size <= SC_HID_MAX_SIZE); - memcpy(msg.uhid_input.data, hid_event.data, hid_event.size); - msg.uhid_input.size = hid_event.size; +static unsigned +sc_keyboard_uhid_to_sc_mod(uint8_t hid_led) { + // + // (chapter 11: LED page) + unsigned mod = 0; + if (hid_led & 0x01) { + mod |= SC_MOD_NUM; + } + if (hid_led & 0x02) { + mod |= SC_MOD_CAPS; + } + return mod; +} - if (!sc_controller_push_msg(kb->controller, &msg)) { - LOGE("Could not send UHID_INPUT message (key)"); - } +static void +sc_uhid_receiver_process_output(struct sc_uhid_receiver *receiver, + const uint8_t *data, size_t len) { + // Called from the thread receiving device messages + + assert(len); + + // Also check at runtime (do not trust the server) + if (!len) { + LOGE("Unexpected empty HID output message"); + return; } + + struct sc_keyboard_uhid *kb = DOWNCAST_RECEIVER(receiver); + + uint8_t hid_led = data[0]; + uint16_t device_mod = sc_keyboard_uhid_to_sc_mod(hid_led); + atomic_store_explicit(&kb->device_mod, device_mod, memory_order_relaxed); } bool sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb, - struct sc_controller *controller) { + struct sc_controller *controller, + struct sc_uhid_devices *uhid_devices) { sc_hid_keyboard_init(&kb->hid); kb->controller = controller; + atomic_init(&kb->device_mod, 0); static const struct sc_key_processor_ops ops = { .process_key = sc_key_processor_process_key, @@ -58,6 +139,14 @@ sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb, kb->key_processor.async_paste = false; kb->key_processor.ops = &ops; + static const struct sc_uhid_receiver_ops uhid_receiver_ops = { + .process_output = sc_uhid_receiver_process_output, + }; + + kb->uhid_receiver.id = UHID_KEYBOARD_ID; + kb->uhid_receiver.ops = &uhid_receiver_ops; + sc_uhid_devices_add_receiver(uhid_devices, &kb->uhid_receiver); + struct sc_control_msg msg; msg.type = SC_CONTROL_MSG_TYPE_UHID_CREATE; msg.uhid_create.id = UHID_KEYBOARD_ID; diff --git a/app/src/uhid/keyboard_uhid.h b/app/src/uhid/keyboard_uhid.h index 854ba008..5e1be70c 100644 --- a/app/src/uhid/keyboard_uhid.h +++ b/app/src/uhid/keyboard_uhid.h @@ -7,17 +7,21 @@ #include "controller.h" #include "hid/hid_keyboard.h" +#include "uhid/uhid_output.h" #include "trait/key_processor.h" struct sc_keyboard_uhid { struct sc_key_processor key_processor; // key processor trait + struct sc_uhid_receiver uhid_receiver; struct sc_hid_keyboard hid; struct sc_controller *controller; + atomic_uint_least16_t device_mod; }; bool sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb, - struct sc_controller *controller); + struct sc_controller *controller, + struct sc_uhid_devices *uhid_devices); #endif diff --git a/app/src/uhid/uhid_output.c b/app/src/uhid/uhid_output.c new file mode 100644 index 00000000..3b095faf --- /dev/null +++ b/app/src/uhid/uhid_output.c @@ -0,0 +1,25 @@ +#include "uhid_output.h" + +#include + +void +sc_uhid_devices_init(struct sc_uhid_devices *devices) { + devices->count = 0; +} + +void +sc_uhid_devices_add_receiver(struct sc_uhid_devices *devices, + struct sc_uhid_receiver *receiver) { + assert(devices->count < SC_UHID_MAX_RECEIVERS); + devices->receivers[devices->count++] = receiver; +} + +struct sc_uhid_receiver * +sc_uhid_devices_get_receiver(struct sc_uhid_devices *devices, uint16_t id) { + for (size_t i = 0; i < devices->count; ++i) { + if (devices->receivers[i]->id == id) { + return devices->receivers[i]; + } + } + return NULL; +} diff --git a/app/src/uhid/uhid_output.h b/app/src/uhid/uhid_output.h new file mode 100644 index 00000000..e13eed87 --- /dev/null +++ b/app/src/uhid/uhid_output.h @@ -0,0 +1,45 @@ +#ifndef SC_UHID_OUTPUT_H +#define SC_UHID_OUTPUT_H + +#include "common.h" + +#include +#include + +/** + * The communication with UHID devices is bidirectional. + * + * This component manages the registration of receivers to handle UHID output + * messages (sent from the device to the computer). + */ + +struct sc_uhid_receiver { + uint16_t id; + + const struct sc_uhid_receiver_ops *ops; +}; + +struct sc_uhid_receiver_ops { + void + (*process_output)(struct sc_uhid_receiver *receiver, + const uint8_t *data, size_t len); +}; + +#define SC_UHID_MAX_RECEIVERS 1 + +struct sc_uhid_devices { + struct sc_uhid_receiver *receivers[SC_UHID_MAX_RECEIVERS]; + unsigned count; +}; + +void +sc_uhid_devices_init(struct sc_uhid_devices *devices); + +void +sc_uhid_devices_add_receiver(struct sc_uhid_devices *devices, + struct sc_uhid_receiver *receiver); + +struct sc_uhid_receiver * +sc_uhid_devices_get_receiver(struct sc_uhid_devices *devices, uint16_t id); + +#endif diff --git a/app/tests/test_device_msg_deserialize.c b/app/tests/test_device_msg_deserialize.c index bfbcefd6..a64a3eb7 100644 --- a/app/tests/test_device_msg_deserialize.c +++ b/app/tests/test_device_msg_deserialize.c @@ -61,6 +61,28 @@ static void test_deserialize_ack_set_clipboard(void) { assert(msg.ack_clipboard.sequence == UINT64_C(0x0102030405060708)); } +static void test_deserialize_uhid_output(void) { + const uint8_t input[] = { + DEVICE_MSG_TYPE_UHID_OUTPUT, + 0, 42, // id + 0, 5, // size + 0x01, 0x02, 0x03, 0x04, 0x05, // data + }; + + struct sc_device_msg msg; + ssize_t r = sc_device_msg_deserialize(input, sizeof(input), &msg); + assert(r == 10); + + assert(msg.type == DEVICE_MSG_TYPE_UHID_OUTPUT); + assert(msg.uhid_output.id == 42); + assert(msg.uhid_output.size == 5); + + uint8_t expected[] = {1, 2, 3, 4, 5}; + assert(!memcmp(msg.uhid_output.data, expected, sizeof(expected))); + + sc_device_msg_destroy(&msg); +} + int main(int argc, char *argv[]) { (void) argc; (void) argv; @@ -68,5 +90,6 @@ int main(int argc, char *argv[]) { test_deserialize_clipboard(); test_deserialize_clipboard_big(); test_deserialize_ack_set_clipboard(); + test_deserialize_uhid_output(); return 0; } diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index b925dd80..5ba0c577 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -52,7 +52,7 @@ public class Controller implements AsyncProcessor { this.powerOn = powerOn; initPointers(); sender = new DeviceMessageSender(controlChannel); - uhidManager = new UhidManager(); + uhidManager = new UhidManager(sender); } private void initPointers() { diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java index 2e333e3f..a8987eb6 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java @@ -4,10 +4,13 @@ public final class DeviceMessage { public static final int TYPE_CLIPBOARD = 0; public static final int TYPE_ACK_CLIPBOARD = 1; + public static final int TYPE_UHID_OUTPUT = 2; private int type; private String text; private long sequence; + private int id; + private byte[] data; private DeviceMessage() { } @@ -26,6 +29,14 @@ public final class DeviceMessage { return event; } + public static DeviceMessage createUhidOutput(int id, byte[] data) { + DeviceMessage event = new DeviceMessage(); + event.type = TYPE_UHID_OUTPUT; + event.id = id; + event.data = data; + return event; + } + public int getType() { return type; } @@ -37,4 +48,12 @@ public final class DeviceMessage { public long getSequence() { return sequence; } + + public int getId() { + return id; + } + + public byte[] getData() { + return data; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java index bcd8d206..f5d57c98 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java @@ -29,6 +29,13 @@ public class DeviceMessageWriter { buffer.putLong(msg.getSequence()); output.write(rawBuffer, 0, buffer.position()); break; + case DeviceMessage.TYPE_UHID_OUTPUT: + buffer.putShort((short) msg.getId()); + byte[] data = msg.getData(); + buffer.putShort((short) data.length); + buffer.put(data); + output.write(rawBuffer, 0, buffer.position()); + break; default: Ln.w("Unknown device message: " + msg.getType()); break; diff --git a/server/src/main/java/com/genymobile/scrcpy/UhidManager.java b/server/src/main/java/com/genymobile/scrcpy/UhidManager.java index 96458bf0..a39288a5 100644 --- a/server/src/main/java/com/genymobile/scrcpy/UhidManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/UhidManager.java @@ -1,5 +1,8 @@ package com.genymobile.scrcpy; +import android.os.Build; +import android.os.HandlerThread; +import android.os.MessageQueue; import android.system.ErrnoException; import android.system.Os; import android.system.OsConstants; @@ -7,6 +10,7 @@ import android.util.ArrayMap; import java.io.FileDescriptor; import java.io.IOException; +import java.io.InterruptedIOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; @@ -14,13 +18,31 @@ import java.nio.charset.StandardCharsets; public final class UhidManager { // Linux: include/uapi/linux/uhid.h + private static final int UHID_OUTPUT = 6; private static final int UHID_CREATE2 = 11; private static final int UHID_INPUT2 = 12; // Linux: include/uapi/linux/input.h private static final short BUS_VIRTUAL = 0x06; + private static final int SIZE_OF_UHID_EVENT = 4380; // sizeof(struct uhid_event) + private final ArrayMap fds = new ArrayMap<>(); + private final ByteBuffer buffer = ByteBuffer.allocate(SIZE_OF_UHID_EVENT).order(ByteOrder.nativeOrder()); + + private final DeviceMessageSender sender; + private final HandlerThread thread = new HandlerThread("UHidManager"); + private final MessageQueue queue; + + public UhidManager(DeviceMessageSender sender) { + this.sender = sender; + thread.start(); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + queue = thread.getLooper().getQueue(); + } else { + queue = null; + } + } public void open(int id, byte[] reportDesc) throws IOException { try { @@ -34,6 +56,8 @@ public final class UhidManager { byte[] req = buildUhidCreate2Req(reportDesc); Os.write(fd, req, 0, req.length); + + registerUhidListener(id, fd); } catch (Exception e) { close(fd); throw e; @@ -43,6 +67,62 @@ public final class UhidManager { } } + private void registerUhidListener(int id, FileDescriptor fd) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + queue.addOnFileDescriptorEventListener(fd, MessageQueue.OnFileDescriptorEventListener.EVENT_INPUT, (fd2, events) -> { + try { + buffer.clear(); + int r = Os.read(fd2, buffer); + buffer.flip(); + if (r > 0) { + int type = buffer.getInt(); + if (type == UHID_OUTPUT) { + byte[] data = extractHidOutputData(buffer); + if (data != null) { + DeviceMessage msg = DeviceMessage.createUhidOutput(id, data); + sender.send(msg); + } + } + } + } catch (ErrnoException | InterruptedIOException e) { + Ln.e("Failed to read UHID output", e); + return 0; + } + return events; + }); + } + } + + private static byte[] extractHidOutputData(ByteBuffer buffer) { + /* + * #define UHID_DATA_MAX 4096 + * struct uhid_event { + * uint32_t type; + * union { + * // ... + * struct uhid_output_req { + * __u8 data[UHID_DATA_MAX]; + * __u16 size; + * __u8 rtype; + * }; + * }; + * } __attribute__((__packed__)); + */ + + if (buffer.remaining() < 4099) { + Ln.w("Incomplete HID output"); + return null; + } + int size = buffer.getShort(buffer.position() + 4096) & 0xFFFF; + if (size > 4096) { + Ln.w("Incorrect HID output size: " + size); + return null; + } + byte[] data = new byte[size]; + buffer.get(data); + return data; + } + public void writeInput(int id, byte[] data) throws IOException { FileDescriptor fd = fds.get(id); if (fd == null) { diff --git a/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java b/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java index 7b917d33..d7f926ba 100644 --- a/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java +++ b/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java @@ -52,4 +52,27 @@ public class DeviceMessageWriterTest { Assert.assertArrayEquals(expected, actual); } + + @Test + public void testSerializeUhidOutput() throws IOException { + DeviceMessageWriter writer = new DeviceMessageWriter(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + dos.writeByte(DeviceMessage.TYPE_UHID_OUTPUT); + dos.writeShort(42); // id + byte[] data = {1, 2, 3, 4, 5}; + dos.writeShort(data.length); + dos.write(data); + + byte[] expected = bos.toByteArray(); + + DeviceMessage msg = DeviceMessage.createUhidOutput(42, data); + bos = new ByteArrayOutputStream(); + writer.writeTo(msg, bos); + + byte[] actual = bos.toByteArray(); + + Assert.assertArrayEquals(expected, actual); + } } From f557188dc835e0a1b108d56b30641510901ecf13 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 24 Feb 2024 22:38:32 +0100 Subject: [PATCH 60/79] Create UhidManager only on first use There is no need to create a UhidManager instance (with its thread) if no UHID is used. PR #4473 --- .../java/com/genymobile/scrcpy/Controller.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 5ba0c577..fd320d3f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -26,7 +26,7 @@ public class Controller implements AsyncProcessor { private Thread thread; - private final UhidManager uhidManager; + private UhidManager uhidManager; private final Device device; private final ControlChannel controlChannel; @@ -52,7 +52,13 @@ public class Controller implements AsyncProcessor { this.powerOn = powerOn; initPointers(); sender = new DeviceMessageSender(controlChannel); - uhidManager = new UhidManager(sender); + } + + private UhidManager getUhidManager() { + if (uhidManager == null) { + uhidManager = new UhidManager(sender); + } + return uhidManager; } private void initPointers() { @@ -99,7 +105,9 @@ public class Controller implements AsyncProcessor { Ln.e("Controller error", e); } finally { Ln.d("Controller stopped"); - uhidManager.closeAll(); + if (uhidManager != null) { + uhidManager.closeAll(); + } listener.onTerminated(true); } }, "control-recv"); @@ -195,10 +203,10 @@ public class Controller implements AsyncProcessor { device.rotateDevice(); break; case ControlMessage.TYPE_UHID_CREATE: - uhidManager.open(msg.getId(), msg.getData()); + getUhidManager().open(msg.getId(), msg.getData()); break; case ControlMessage.TYPE_UHID_INPUT: - uhidManager.writeInput(msg.getId(), msg.getData()); + getUhidManager().writeInput(msg.getId(), msg.getData()); break; default: // do nothing From 54dede36307edc69553d7de620f6b4318e48c678 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 26 Feb 2024 20:22:34 +0100 Subject: [PATCH 61/79] Fix startActivity() for supporting API < 30 Call the older startActivityAsUser() instead of startActivityAsUserWithFeature() so that it also works on older Android versions. Fixes #4704 PR #4473 --- .../com/genymobile/scrcpy/AudioCapture.java | 2 +- .../scrcpy/wrappers/ActivityManager.java | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java index 45634c70..3934ad49 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java @@ -79,7 +79,7 @@ public final class AudioCapture { intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); intent.addCategory(Intent.CATEGORY_LAUNCHER); intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity")); - ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent); + ServiceManager.getActivityManager().startActivity(intent); } private static void stopWorkaroundAndroid11() { diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java index 367ea2e7..d4bee165 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java @@ -22,7 +22,7 @@ public final class ActivityManager { private Method getContentProviderExternalMethod; private boolean getContentProviderExternalMethodNewVersion = true; private Method removeContentProviderExternalMethod; - private Method startActivityAsUserWithFeatureMethod; + private Method startActivityAsUserMethod; private Method forceStopPackageMethod; static ActivityManager create() { @@ -107,26 +107,25 @@ public final class ActivityManager { return getContentProviderExternal("settings", new Binder()); } - private Method getStartActivityAsUserWithFeatureMethod() throws NoSuchMethodException, ClassNotFoundException { - if (startActivityAsUserWithFeatureMethod == null) { + private Method getStartActivityAsUserMethod() throws NoSuchMethodException, ClassNotFoundException { + if (startActivityAsUserMethod == null) { Class iApplicationThreadClass = Class.forName("android.app.IApplicationThread"); Class profilerInfo = Class.forName("android.app.ProfilerInfo"); - startActivityAsUserWithFeatureMethod = manager.getClass() - .getMethod("startActivityAsUserWithFeature", iApplicationThreadClass, String.class, String.class, Intent.class, String.class, - IBinder.class, String.class, int.class, int.class, profilerInfo, Bundle.class, int.class); + startActivityAsUserMethod = manager.getClass() + .getMethod("startActivityAsUser", iApplicationThreadClass, String.class, Intent.class, String.class, IBinder.class, String.class, + int.class, int.class, profilerInfo, Bundle.class, int.class); } - return startActivityAsUserWithFeatureMethod; + return startActivityAsUserMethod; } @SuppressWarnings("ConstantConditions") - public int startActivityAsUserWithFeature(Intent intent) { + public int startActivity(Intent intent) { try { - Method method = getStartActivityAsUserWithFeatureMethod(); + Method method = getStartActivityAsUserMethod(); return (int) method.invoke( /* this */ manager, /* caller */ null, /* callingPackage */ FakeContext.PACKAGE_NAME, - /* callingFeatureId */ null, /* intent */ intent, /* resolvedType */ null, /* resultTo */ null, From 151a6225d44f795bcc6066e8af9ccc65fefbaded Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 24 Feb 2024 22:30:30 +0100 Subject: [PATCH 62/79] Add shortcut to open keyboard settings The keyboard settings can be opened by: adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS Add a shortcut (MOD+k) for convenience if the current keyboard is HID. PR #4473 --- app/scrcpy.1 | 6 +++++- app/src/cli.c | 9 +++++++-- app/src/control_msg.c | 4 ++++ app/src/control_msg.h | 1 + app/src/input_manager.c | 19 +++++++++++++++++++ app/src/keyboard_sdk.c | 1 + app/src/trait/key_processor.h | 7 +++++++ app/src/uhid/keyboard_uhid.c | 1 + app/src/usb/keyboard_aoa.c | 1 + app/tests/test_control_msg_serialize.c | 16 ++++++++++++++++ doc/shortcuts.md | 1 + .../com/genymobile/scrcpy/ControlMessage.java | 1 + .../scrcpy/ControlMessageReader.java | 1 + .../com/genymobile/scrcpy/Controller.java | 10 ++++++++++ .../scrcpy/ControlMessageReaderTest.java | 16 ++++++++++++++++ 15 files changed, 91 insertions(+), 3 deletions(-) diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 1dfcab2b..7e856664 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -182,7 +182,7 @@ Possible values are "disabled", "sdk", "uhid" and "aoa": - "uhid" simulates a physical HID keyboard using the Linux HID kernel module on the device. - "aoa" simulates a physical HID keyboard using the AOAv2 protocol. It may only work over USB. -For "uhid" and "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly: +For "uhid" and "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly using the shortcut MOD+k (except in OTG mode), or by executing: adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS @@ -644,6 +644,10 @@ Copy computer clipboard to device, then paste (inject PASTE keycode, Android >= .B MOD+Shift+v Inject computer clipboard text as a sequence of key events +.TP +.B MOD+k +Open keyboard settings on the device (for HID keyboard only) + .TP .B MOD+i Enable/disable FPS counter (print frames/second in logs) diff --git a/app/src/cli.c b/app/src/cli.c index 59cd5699..c1c68e92 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -377,8 +377,9 @@ static const struct sc_option options[] = { "For \"uhid\" and \"aoa\", the keyboard layout must be " "configured (once and for all) on the device, via Settings -> " "System -> Languages and input -> Physical keyboard. This " - "settings page can be started directly: `adb shell am start -a " - "android.settings.HARD_KEYBOARD_SETTINGS`.\n" + "settings page can be started directly using the shortcut " + "MOD+k (except in OTG mode) or by executing: `adb shell am " + "start -a android.settings.HARD_KEYBOARD_SETTINGS`.\n" "This option is only available when a HID keyboard is enabled " "(or a physical keyboard is connected).\n" "Also see --mouse.", @@ -965,6 +966,10 @@ static const struct sc_shortcut shortcuts[] = { .shortcuts = { "MOD+Shift+v" }, .text = "Inject computer clipboard text as a sequence of key events", }, + { + .shortcuts = { "MOD+k" }, + .text = "Open keyboard settings on the device (for HID keyboard only)", + }, { .shortcuts = { "MOD+i" }, .text = "Enable/disable FPS counter (print frames/second in logs)", diff --git a/app/src/control_msg.c b/app/src/control_msg.c index 88575b4e..b3da5fe5 100644 --- a/app/src/control_msg.c +++ b/app/src/control_msg.c @@ -161,6 +161,7 @@ sc_control_msg_serialize(const struct sc_control_msg *msg, uint8_t *buf) { case SC_CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL: case SC_CONTROL_MSG_TYPE_COLLAPSE_PANELS: case SC_CONTROL_MSG_TYPE_ROTATE_DEVICE: + case SC_CONTROL_MSG_TYPE_OPEN_HARD_KEYBOARD_SETTINGS: // no additional data return 1; default: @@ -270,6 +271,9 @@ sc_control_msg_log(const struct sc_control_msg *msg) { } break; } + case SC_CONTROL_MSG_TYPE_OPEN_HARD_KEYBOARD_SETTINGS: + LOG_CMSG("open hard keyboard settings"); + break; default: LOG_CMSG("unknown type: %u", (unsigned) msg->type); break; diff --git a/app/src/control_msg.h b/app/src/control_msg.h index 550168c2..cd1340ef 100644 --- a/app/src/control_msg.h +++ b/app/src/control_msg.h @@ -40,6 +40,7 @@ enum sc_control_msg_type { SC_CONTROL_MSG_TYPE_ROTATE_DEVICE, SC_CONTROL_MSG_TYPE_UHID_CREATE, SC_CONTROL_MSG_TYPE_UHID_INPUT, + SC_CONTROL_MSG_TYPE_OPEN_HARD_KEYBOARD_SETTINGS, }; enum sc_screen_power_mode { diff --git a/app/src/input_manager.c b/app/src/input_manager.c index 7186186f..f26c4164 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -318,6 +318,18 @@ rotate_device(struct sc_input_manager *im) { } } +static void +open_hard_keyboard_settings(struct sc_input_manager *im) { + assert(im->controller); + + struct sc_control_msg msg; + msg.type = SC_CONTROL_MSG_TYPE_OPEN_HARD_KEYBOARD_SETTINGS; + + if (!sc_controller_push_msg(im->controller, &msg)) { + LOGW("Could not request opening hard keyboard settings"); + } +} + static void apply_orientation_transform(struct sc_input_manager *im, enum sc_orientation transform) { @@ -550,6 +562,13 @@ sc_input_manager_process_key(struct sc_input_manager *im, rotate_device(im); } return; + case SDLK_k: + if (control && !shift && !repeat && down + && im->kp && im->kp->hid) { + // Only if the current keyboard is hid + open_hard_keyboard_settings(im); + } + return; } return; diff --git a/app/src/keyboard_sdk.c b/app/src/keyboard_sdk.c index 726f65a9..00b7f92a 100644 --- a/app/src/keyboard_sdk.c +++ b/app/src/keyboard_sdk.c @@ -340,5 +340,6 @@ sc_keyboard_sdk_init(struct sc_keyboard_sdk *kb, // Key injection and clipboard synchronization are serialized kb->key_processor.async_paste = false; + kb->key_processor.hid = false; kb->key_processor.ops = &ops; } diff --git a/app/src/trait/key_processor.h b/app/src/trait/key_processor.h index 8c51b11d..96374413 100644 --- a/app/src/trait/key_processor.h +++ b/app/src/trait/key_processor.h @@ -23,6 +23,13 @@ struct sc_key_processor { */ bool async_paste; + /** + * Set by the implementation to indicate that the keyboard is HID. In + * practice, it is used to react on a shortcut to open the hard keyboard + * settings only if the keyboard is HID. + */ + bool hid; + const struct sc_key_processor_ops *ops; }; diff --git a/app/src/uhid/keyboard_uhid.c b/app/src/uhid/keyboard_uhid.c index f537bc29..515a3fd9 100644 --- a/app/src/uhid/keyboard_uhid.c +++ b/app/src/uhid/keyboard_uhid.c @@ -137,6 +137,7 @@ sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb, // Clipboard synchronization is requested over the same control socket, so // there is no need for a specific synchronization mechanism kb->key_processor.async_paste = false; + kb->key_processor.hid = true; kb->key_processor.ops = &ops; static const struct sc_uhid_receiver_ops uhid_receiver_ops = { diff --git a/app/src/usb/keyboard_aoa.c b/app/src/usb/keyboard_aoa.c index b69d6cd8..736c97b0 100644 --- a/app/src/usb/keyboard_aoa.c +++ b/app/src/usb/keyboard_aoa.c @@ -94,6 +94,7 @@ sc_keyboard_aoa_init(struct sc_keyboard_aoa *kb, struct sc_aoa *aoa) { // events are sent over AOA, so it must wait for clipboard synchronization // to be acknowledged by the device before injecting Ctrl+v. kb->key_processor.async_paste = true; + kb->key_processor.hid = true; kb->key_processor.ops = &ops; return true; diff --git a/app/tests/test_control_msg_serialize.c b/app/tests/test_control_msg_serialize.c index 0ab61153..7a978f2b 100644 --- a/app/tests/test_control_msg_serialize.c +++ b/app/tests/test_control_msg_serialize.c @@ -370,6 +370,21 @@ static void test_serialize_uhid_input(void) { assert(!memcmp(buf, expected, sizeof(expected))); } +static void test_serialize_open_hard_keyboard(void) { + struct sc_control_msg msg = { + .type = SC_CONTROL_MSG_TYPE_OPEN_HARD_KEYBOARD_SETTINGS, + }; + + uint8_t buf[SC_CONTROL_MSG_MAX_SIZE]; + size_t size = sc_control_msg_serialize(&msg, buf); + assert(size == 1); + + const uint8_t expected[] = { + SC_CONTROL_MSG_TYPE_OPEN_HARD_KEYBOARD_SETTINGS, + }; + assert(!memcmp(buf, expected, sizeof(expected))); +} + int main(int argc, char *argv[]) { (void) argc; (void) argv; @@ -390,5 +405,6 @@ int main(int argc, char *argv[]) { test_serialize_rotate_device(); test_serialize_uhid_create(); test_serialize_uhid_input(); + test_serialize_open_hard_keyboard(); return 0; } diff --git a/doc/shortcuts.md b/doc/shortcuts.md index 21bccbd9..8c402855 100644 --- a/doc/shortcuts.md +++ b/doc/shortcuts.md @@ -48,6 +48,7 @@ _[Super] is typically the Windows or Cmd key._ | Cut to clipboard⁵ | MOD+x | Synchronize clipboards and paste⁵ | MOD+v | Inject computer clipboard text | MOD+Shift+v + | Open keyboard settings (HID keyboard only) | MOD+k | Enable/disable FPS counter (on stdout) | MOD+i | Pinch-to-zoom/rotate | Ctrl+_click-and-move_ | Tilt (slide vertically with 2 fingers) | Shift+_click-and-move_ diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java index 74bf5610..bcbacb4b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java @@ -19,6 +19,7 @@ public final class ControlMessage { public static final int TYPE_ROTATE_DEVICE = 11; public static final int TYPE_UHID_CREATE = 12; public static final int TYPE_UHID_INPUT = 13; + public static final int TYPE_OPEN_HARD_KEYBOARD_SETTINGS = 14; public static final long SEQUENCE_INVALID = 0; diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java index 24aa73c0..1761d228 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java @@ -86,6 +86,7 @@ public class ControlMessageReader { case ControlMessage.TYPE_EXPAND_SETTINGS_PANEL: case ControlMessage.TYPE_COLLAPSE_PANELS: case ControlMessage.TYPE_ROTATE_DEVICE: + case ControlMessage.TYPE_OPEN_HARD_KEYBOARD_SETTINGS: msg = ControlMessage.createEmpty(type); break; case ControlMessage.TYPE_UHID_CREATE: diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index fd320d3f..87faf8ba 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -1,7 +1,9 @@ package com.genymobile.scrcpy; import com.genymobile.scrcpy.wrappers.InputManager; +import com.genymobile.scrcpy.wrappers.ServiceManager; +import android.content.Intent; import android.os.Build; import android.os.SystemClock; import android.view.InputDevice; @@ -208,6 +210,9 @@ public class Controller implements AsyncProcessor { case ControlMessage.TYPE_UHID_INPUT: getUhidManager().writeInput(msg.getId(), msg.getData()); break; + case ControlMessage.TYPE_OPEN_HARD_KEYBOARD_SETTINGS: + openHardKeyboardSettings(); + break; default: // do nothing } @@ -446,4 +451,9 @@ public class Controller implements AsyncProcessor { return ok; } + + private void openHardKeyboardSettings() { + Intent intent = new Intent("android.settings.HARD_KEYBOARD_SETTINGS"); + ServiceManager.getActivityManager().startActivity(intent); + } } diff --git a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java index 7cc67c3e..0c8086f7 100644 --- a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java +++ b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java @@ -366,6 +366,22 @@ public class ControlMessageReaderTest { Assert.assertArrayEquals(data, event.getData()); } + @Test + public void testParseOpenHardKeyboardSettings() throws IOException { + ControlMessageReader reader = new ControlMessageReader(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + dos.writeByte(ControlMessage.TYPE_OPEN_HARD_KEYBOARD_SETTINGS); + + byte[] packet = bos.toByteArray(); + + reader.readFrom(new ByteArrayInputStream(packet)); + ControlMessage event = reader.next(); + + Assert.assertEquals(ControlMessage.TYPE_OPEN_HARD_KEYBOARD_SETTINGS, event.getType()); + } + @Test public void testMultiEvents() throws IOException { ControlMessageReader reader = new ControlMessageReader(); From 6a103c809f4a208c25d7fd5d019bc6bc5b3046b4 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 25 Feb 2024 15:43:36 +0100 Subject: [PATCH 63/79] Add UHID mouse support Use the following command: scrcpy --mouse=uhid PR #4473 --- app/data/bash-completion/scrcpy | 2 +- app/data/zsh-completion/_scrcpy | 2 +- app/meson.build | 1 + app/scrcpy.1 | 5 +- app/src/cli.c | 16 ++++-- app/src/options.h | 1 + app/src/scrcpy.c | 8 +++ app/src/uhid/mouse_uhid.c | 89 +++++++++++++++++++++++++++++++++ app/src/uhid/mouse_uhid.h | 19 +++++++ 9 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 app/src/uhid/mouse_uhid.c create mode 100644 app/src/uhid/mouse_uhid.h diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 904ccdeb..8cc0b157 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -120,7 +120,7 @@ _scrcpy() { return ;; --mouse) - COMPREPLY=($(compgen -W 'disabled sdk aoa' -- "$cur")) + COMPREPLY=($(compgen -W 'disabled sdk uhid aoa' -- "$cur")) return ;; --orientation|--display-orientation) diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index f81d2b22..1cf2ae41 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -44,7 +44,7 @@ arguments=( '--lock-video-orientation=[Lock video orientation]:orientation:(unlocked initial 0 90 180 270)' {-m,--max-size=}'[Limit both the width and height of the video to value]' '--max-fps=[Limit the frame rate of screen capture]' - '--mouse[Set the mouse input mode]:mode:(disabled sdk aoa)' + '--mouse[Set the mouse input mode]:mode:(disabled sdk uhid aoa)' {-n,--no-control}'[Disable device control \(mirror the device in read only\)]' {-N,--no-playback}'[Disable video and audio playback]' '--no-audio[Disable audio forwarding]' diff --git a/app/meson.build b/app/meson.build index 3695e0f9..b0a6aadb 100644 --- a/app/meson.build +++ b/app/meson.build @@ -36,6 +36,7 @@ src = [ 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/uhid/keyboard_uhid.c', + 'src/uhid/mouse_uhid.c', 'src/uhid/uhid_output.c', 'src/util/acksync.c', 'src/util/audiobuf.c', diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 7e856664..8c0c4cc6 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -240,13 +240,14 @@ Limit the framerate of screen capture (officially supported since Android 10, bu .BI "\-\-mouse " mode Select how to send mouse inputs to the device. -Possible values are "disabled", "sdk" and "aoa": +Possible values are "disabled", "sdk", "uhid" and "aoa": - "disabled" does not send mouse inputs to the device. - "sdk" uses the Android system API to deliver mouse events to applications. + - "uhid" simulates a physical HID mouse using the Linux HID kernel module on the device. - "aoa" simulates a physical mouse using the AOAv2 protocol. It may only work over USB. -In "aoa" mode, the computer mouse is captured to control the device directly (relative mouse mode). +In "uhid" and "aoa" modes, the computer mouse is captured to control the device directly (relative mouse mode). LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer. diff --git a/app/src/cli.c b/app/src/cli.c index c1c68e92..dceb8fff 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -462,14 +462,17 @@ static const struct sc_option options[] = { .longopt = "mouse", .argdesc = "mode", .text = "Select how to send mouse inputs to the device.\n" - "Possible values are \"disabled\", \"sdk\" and \"aoa\".\n" + "Possible values are \"disabled\", \"sdk\", \"uhid\" and " + "\"aoa\".\n" "\"disabled\" does not send mouse inputs to the device.\n" "\"sdk\" uses the Android system API to deliver mouse events" "to applications.\n" + "\"uhid\" simulates a physical HID mouse using the Linux UHID " + "kernel module on the device.\n" "\"aoa\" simulates a physical mouse using the AOAv2 protocol. " "It may only work over USB.\n" - "In \"aoa\" mode, the computer mouse is captured to control " - "the device directly (relative mouse mode).\n" + "In \"uhid\" and \"aoa\" modes, the computer mouse is captured " + "to control the device directly (relative mouse mode).\n" "LAlt, LSuper or RSuper toggle the capture mode, to give " "control of the mouse back to the computer.\n" "Also see --keyboard.", @@ -1979,6 +1982,11 @@ parse_mouse(const char *optarg, enum sc_mouse_input_mode *mode) { return true; } + if (!strcmp(optarg, "uhid")) { + *mode = SC_MOUSE_INPUT_MODE_UHID; + return true; + } + if (!strcmp(optarg, "aoa")) { #ifdef HAVE_USB *mode = SC_MOUSE_INPUT_MODE_AOA; @@ -1989,7 +1997,7 @@ parse_mouse(const char *optarg, enum sc_mouse_input_mode *mode) { #endif } - LOGE("Unsupported mouse: %s (expected disabled, sdk or aoa)", optarg); + LOGE("Unsupported mouse: %s (expected disabled, sdk, uhid or aoa)", optarg); return false; } diff --git a/app/src/options.h b/app/src/options.h index 6d62fac0..5445e7c8 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -151,6 +151,7 @@ enum sc_mouse_input_mode { SC_MOUSE_INPUT_MODE_AUTO, SC_MOUSE_INPUT_MODE_DISABLED, SC_MOUSE_INPUT_MODE_SDK, + SC_MOUSE_INPUT_MODE_UHID, SC_MOUSE_INPUT_MODE_AOA, }; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 7b1a6d5c..a40a4dec 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -26,6 +26,7 @@ #include "screen.h" #include "server.h" #include "uhid/keyboard_uhid.h" +#include "uhid/mouse_uhid.h" #ifdef HAVE_USB # include "usb/aoa_hid.h" # include "usb/keyboard_aoa.h" @@ -73,6 +74,7 @@ struct scrcpy { }; union { struct sc_mouse_sdk mouse_sdk; + struct sc_mouse_uhid mouse_uhid; #ifdef HAVE_USB struct sc_mouse_aoa mouse_aoa; #endif @@ -682,6 +684,12 @@ aoa_hid_end: if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_SDK) { sc_mouse_sdk_init(&s->mouse_sdk, &s->controller); mp = &s->mouse_sdk.mouse_processor; + } else if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_UHID) { + bool ok = sc_mouse_uhid_init(&s->mouse_uhid, &s->controller); + if (!ok) { + goto end; + } + mp = &s->mouse_uhid.mouse_processor; } sc_controller_configure(&s->controller, acksync, uhid_devices); diff --git a/app/src/uhid/mouse_uhid.c b/app/src/uhid/mouse_uhid.c new file mode 100644 index 00000000..77446f9e --- /dev/null +++ b/app/src/uhid/mouse_uhid.c @@ -0,0 +1,89 @@ +#include "mouse_uhid.h" + +#include "hid/hid_mouse.h" +#include "input_events.h" +#include "util/log.h" + +/** Downcast mouse processor to mouse_uhid */ +#define DOWNCAST(MP) container_of(MP, struct sc_mouse_uhid, mouse_processor) + +#define UHID_MOUSE_ID 2 + +static void +sc_mouse_uhid_send_input(struct sc_mouse_uhid *mouse, + const struct sc_hid_event *event, const char *name) { + struct sc_control_msg msg; + msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT; + msg.uhid_input.id = UHID_MOUSE_ID; + + assert(event->size <= SC_HID_MAX_SIZE); + memcpy(msg.uhid_input.data, event->data, event->size); + msg.uhid_input.size = event->size; + + if (!sc_controller_push_msg(mouse->controller, &msg)) { + LOGE("Could not send UHID_INPUT message (%s)", name); + } +} + +static void +sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, + const struct sc_mouse_motion_event *event) { + struct sc_mouse_uhid *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + sc_hid_mouse_event_from_motion(&hid_event, event); + + sc_mouse_uhid_send_input(mouse, &hid_event, "mouse motion"); +} + +static void +sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, + const struct sc_mouse_click_event *event) { + struct sc_mouse_uhid *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + sc_hid_mouse_event_from_click(&hid_event, event); + + sc_mouse_uhid_send_input(mouse, &hid_event, "mouse click"); +} + +static void +sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, + const struct sc_mouse_scroll_event *event) { + struct sc_mouse_uhid *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + sc_hid_mouse_event_from_scroll(&hid_event, event); + + sc_mouse_uhid_send_input(mouse, &hid_event, "mouse scroll"); +} + +bool +sc_mouse_uhid_init(struct sc_mouse_uhid *mouse, + struct sc_controller *controller) { + mouse->controller = controller; + + static const struct sc_mouse_processor_ops ops = { + .process_mouse_motion = sc_mouse_processor_process_mouse_motion, + .process_mouse_click = sc_mouse_processor_process_mouse_click, + .process_mouse_scroll = sc_mouse_processor_process_mouse_scroll, + // Touch events not supported (coordinates are not relative) + .process_touch = NULL, + }; + + mouse->mouse_processor.ops = &ops; + + mouse->mouse_processor.relative_mode = true; + + struct sc_control_msg msg; + msg.type = SC_CONTROL_MSG_TYPE_UHID_CREATE; + msg.uhid_create.id = UHID_MOUSE_ID; + msg.uhid_create.report_desc = SC_HID_MOUSE_REPORT_DESC; + msg.uhid_create.report_desc_size = SC_HID_MOUSE_REPORT_DESC_LEN; + if (!sc_controller_push_msg(controller, &msg)) { + LOGE("Could not send UHID_CREATE message (mouse)"); + return false; + } + + return true; +} diff --git a/app/src/uhid/mouse_uhid.h b/app/src/uhid/mouse_uhid.h new file mode 100644 index 00000000..f117ba97 --- /dev/null +++ b/app/src/uhid/mouse_uhid.h @@ -0,0 +1,19 @@ +#ifndef SC_MOUSE_UHID_H +#define SC_MOUSE_UHID_H + +#include + +#include "controller.h" +#include "trait/mouse_processor.h" + +struct sc_mouse_uhid { + struct sc_mouse_processor mouse_processor; // mouse processor trait + + struct sc_controller *controller; +}; + +bool +sc_mouse_uhid_init(struct sc_mouse_uhid *mouse, + struct sc_controller *controller); + +#endif From 1c5ad0e8131c6e051e940c29acdc81a500df4673 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 25 Feb 2024 18:35:52 +0100 Subject: [PATCH 64/79] Reassign -K and -M to UHID keyboard and mouse The options were deprecated, but for convenience, reassign them to aliases for --keyboard=uhid and --mouse=uhid respectively. Their long version (--hid-keyboard and --hid-mouse) remain deprecated. PR #4473 --- app/data/bash-completion/scrcpy | 2 ++ app/data/zsh-completion/_scrcpy | 2 ++ app/scrcpy.1 | 8 +++++++ app/src/cli.c | 41 +++++++++++++++++++-------------- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 8cc0b157..e6b2c91a 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -27,6 +27,7 @@ _scrcpy() { --force-adb-forward --forward-all-clicks -h --help + -K --keyboard= --kill-adb-on-close --legacy-paste @@ -37,6 +38,7 @@ _scrcpy() { --lock-video-orientation --lock-video-orientation= -m --max-size= + -M --max-fps= --mouse= -n --no-control diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 1cf2ae41..a23240ec 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -34,6 +34,7 @@ arguments=( '--force-adb-forward[Do not attempt to use \"adb reverse\" to connect to the device]' '--forward-all-clicks[Forward clicks to device]' {-h,--help}'[Print the help]' + '-K[Use UHID keyboard (same as --keyboard=uhid)]' '--keyboard[Set the keyboard input mode]:mode:(disabled sdk uhid aoa)' '--kill-adb-on-close[Kill adb when scrcpy terminates]' '--legacy-paste[Inject computer clipboard text as a sequence of key events on Ctrl+v]' @@ -43,6 +44,7 @@ arguments=( '--list-encoders[List video and audio encoders available on the device]' '--lock-video-orientation=[Lock video orientation]:orientation:(unlocked initial 0 90 180 270)' {-m,--max-size=}'[Limit both the width and height of the video to value]' + '-M[Use UHID mouse (same as --mouse=uhid)]' '--max-fps=[Limit the frame rate of screen capture]' '--mouse[Set the mouse input mode]:mode:(disabled sdk uhid aoa)' {-n,--no-control}'[Disable device control \(mirror the device in read only\)]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 8c0c4cc6..13ad28f9 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -171,6 +171,10 @@ By default, right-click triggers BACK (or POWER on) and middle-click triggers HO .B \-h, \-\-help Print this help. +.TP +.B \-K +Same as \fB\-\-keyboard=uhid\fR. + .TP .BI "\-\-keyboard " mode Select how to send keyboard inputs to the device. @@ -232,6 +236,10 @@ Limit both the width and height of the video to \fIvalue\fR. The other dimension Default is 0 (unlimited). +.TP +.B \-M +Same as \fB\-\-mouse=uhid\fR. + .TP .BI "\-\-max\-fps " value Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions). diff --git a/app/src/cli.c b/app/src/cli.c index dceb8fff..cb5be008 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -95,6 +95,8 @@ enum { OPT_ORIENTATION, OPT_KEYBOARD, OPT_MOUSE, + OPT_HID_KEYBOARD_DEPRECATED, + OPT_HID_MOUSE_DEPRECATED, }; struct sc_option { @@ -360,6 +362,10 @@ static const struct sc_option options[] = { .longopt = "help", .text = "Print this help.", }, + { + .shortopt = 'K', + .text = "Same as --keyboard=uhid.", + }, { .longopt_id = OPT_KEYBOARD, .longopt = "keyboard", @@ -391,7 +397,8 @@ static const struct sc_option options[] = { }, { // deprecated - .shortopt = 'K', + //.shortopt = 'K', // old, reassigned + .longopt_id = OPT_HID_KEYBOARD_DEPRECATED, .longopt = "hid-keyboard", }, { @@ -447,9 +454,14 @@ static const struct sc_option options[] = { }, { // deprecated - .shortopt = 'M', + //.shortopt = 'M', // old, reassigned + .longopt_id = OPT_HID_MOUSE_DEPRECATED, .longopt = "hid-mouse", }, + { + .shortopt = 'M', + .text = "Same as --mouse=uhid.", + }, { .longopt_id = OPT_MAX_FPS, .longopt = "max-fps", @@ -2089,20 +2101,17 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], args->help = true; break; case 'K': -#ifdef HAVE_USB - LOGW("-K/--hid-keyboard is deprecated, use --keyboard=aoa " - "instead."); - opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_AOA; + opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_UHID; break; -#else - LOGE("HID over AOA (-K/--hid-keyboard) is disabled."); - return false; -#endif case OPT_KEYBOARD: if (!parse_keyboard(optarg, &opts->keyboard_input_mode)) { return false; } break; + case OPT_HID_KEYBOARD_DEPRECATED: + LOGE("--hid-keyboard has been removed, use --keyboard=aoa or " + "--keyboard=uhid instead."); + return false; case OPT_MAX_FPS: if (!parse_max_fps(optarg, &opts->max_fps)) { return false; @@ -2114,19 +2123,17 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } break; case 'M': -#ifdef HAVE_USB - LOGW("-M/--hid-mouse is deprecated, use --mouse=aoa instead."); - opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_AOA; + opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_UHID; break; -#else - LOGE("HID over AOA (-M/--hid-mouse) is disabled."); - return false; -#endif case OPT_MOUSE: if (!parse_mouse(optarg, &opts->mouse_input_mode)) { return false; } break; + case OPT_HID_MOUSE_DEPRECATED: + LOGE("--hid-mouse has been removed, use --mouse=aoa or " + "--mouse=uhid instead."); + return false; case OPT_LOCK_VIDEO_ORIENTATION: if (!parse_lock_video_orientation(optarg, &opts->lock_video_orientation)) { From 5f12132c47178d88ca73f17e90de6891aee33f2d Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 28 Feb 2024 22:46:48 +0100 Subject: [PATCH 65/79] Do not fallback keyboard mode if AOA fails Initially, if AOA initialization failed, default injection method was used, in order to use the same command/shortcut when the device is connected via USB or via TCP/IP, without changing the arguments. Now that there are 3 keyboard modes, it seems unexpected to switch to another specific mode if AOA fails (and it is inconsistent). If the user explicitly requests AOA, then use AOA or fail. Refs #2632 comment PR #4473 --- app/src/scrcpy.c | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index a40a4dec..c63a95c2 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -570,7 +570,7 @@ scrcpy(struct scrcpy_options *options) { if (!ok) { LOGE("Failed to initialize USB"); sc_acksync_destroy(&s->acksync); - goto aoa_hid_end; + goto end; } assert(serial); @@ -578,7 +578,7 @@ scrcpy(struct scrcpy_options *options) { ok = sc_usb_select_device(&s->usb, serial, &usb_device); if (!ok) { sc_usb_destroy(&s->usb); - goto aoa_hid_end; + goto end; } LOGI("USB device: %s (%04" PRIx16 ":%04" PRIx16 ") %s %s", @@ -591,7 +591,7 @@ scrcpy(struct scrcpy_options *options) { LOGE("Failed to connect to USB device %s", serial); sc_usb_destroy(&s->usb); sc_acksync_destroy(&s->acksync); - goto aoa_hid_end; + goto end; } ok = sc_aoa_init(&s->aoa, &s->usb, &s->acksync); @@ -600,7 +600,7 @@ scrcpy(struct scrcpy_options *options) { sc_usb_disconnect(&s->usb); sc_usb_destroy(&s->usb); sc_acksync_destroy(&s->acksync); - goto aoa_hid_end; + goto end; } if (use_keyboard_aoa) { @@ -628,41 +628,18 @@ scrcpy(struct scrcpy_options *options) { sc_usb_disconnect(&s->usb); sc_usb_destroy(&s->usb); sc_aoa_destroy(&s->aoa); - goto aoa_hid_end; + goto end; } acksync = &s->acksync; aoa_hid_initialized = true; - -aoa_hid_end: - if (!aoa_hid_initialized) { - if (keyboard_aoa_initialized) { - sc_keyboard_aoa_destroy(&s->keyboard_aoa); - keyboard_aoa_initialized = false; - } - if (mouse_aoa_initialized) { - sc_mouse_aoa_destroy(&s->mouse_aoa); - mouse_aoa_initialized = false; - } - } - - if (use_keyboard_aoa && !keyboard_aoa_initialized) { - LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)"); - options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_SDK; - } - - if (use_mouse_aoa && !mouse_aoa_initialized) { - LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)"); - options->mouse_input_mode = SC_MOUSE_INPUT_MODE_SDK; - } } #else assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_AOA); assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_AOA); #endif - // keyboard_input_mode may have been reset if AOA mode failed if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_SDK) { sc_keyboard_sdk_init(&s->keyboard_sdk, &s->controller, options->key_inject_mode, @@ -680,7 +657,6 @@ aoa_hid_end: kp = &s->keyboard_uhid.key_processor; } - // mouse_input_mode may have been reset if AOA mode failed if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_SDK) { sc_mouse_sdk_init(&s->mouse_sdk, &s->controller); mp = &s->mouse_sdk.mouse_processor; From dd479ed17613e8f7d7c2cf2447f57045815192b8 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 29 Feb 2024 08:49:18 +0100 Subject: [PATCH 66/79] Check options specific to SDK keyboard Fail if an option specific to --keyboard=sdk is passed with another keyboard input mode. PR #4473 --- app/src/cli.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app/src/cli.c b/app/src/cli.c index cb5be008..daa041cf 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -2611,6 +2611,23 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } } + if (opts->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_SDK) { + if (opts->key_inject_mode == SC_KEY_INJECT_MODE_TEXT) { + LOGE("--prefer-text is specific to --keyboard=sdk"); + return false; + } + + if (opts->key_inject_mode == SC_KEY_INJECT_MODE_RAW) { + LOGE("--raw-key-events is specific to --keyboard=sdk"); + return false; + } + + if (!opts->forward_key_repeat) { + LOGE("--no-key-repeat is specific to --keyboard=sdk"); + return false; + } + } + if ((opts->tunnel_host || opts->tunnel_port) && !opts->force_adb_forward) { LOGI("Tunnel host/port is set, " "--force-adb-forward automatically enabled."); From b9d244b4c9eaca05f9202126519200603ebd0cbe Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 29 Feb 2024 10:00:56 +0100 Subject: [PATCH 67/79] Document UHID Rework the documentation to present the keyboard and mouse input modes. PR #4473 --- FAQ.md | 9 ++-- README.md | 14 +++-- doc/control.md | 51 ++---------------- doc/develop.md | 2 +- doc/hid-otg.md | 112 --------------------------------------- doc/keyboard.md | 136 ++++++++++++++++++++++++++++++++++++++++++++++++ doc/mouse.md | 70 +++++++++++++++++++++++++ doc/otg.md | 37 +++++++++++++ 8 files changed, 262 insertions(+), 169 deletions(-) delete mode 100644 doc/hid-otg.md create mode 100644 doc/keyboard.md create mode 100644 doc/mouse.md create mode 100644 doc/otg.md diff --git a/FAQ.md b/FAQ.md index 6d02361b..5f089cd7 100644 --- a/FAQ.md +++ b/FAQ.md @@ -133,9 +133,9 @@ Try with another USB cable or plug it into another USB port. See [#281] and [#283]: https://github.com/Genymobile/scrcpy/issues/283 -## HID/OTG issues on Windows +## OTG issues on Windows -On Windows, if `scrcpy --otg` (or `--hid-keyboard`/`--hid-mouse`) results in: +On Windows, if `scrcpy --otg` (or `--keyboard=aoa`/`--mouse=aoa`) results in: > ERROR: Could not find any USB device @@ -170,12 +170,13 @@ The default text injection method is [limited to ASCII characters][text-input]. A trick allows to also inject some [accented characters][accented-characters], but that's all. See [#37]. -It is also possible to simulate a [physical keyboard][hid] (HID). +To avoid the problem, [change the keyboard mode to simulate a physical +keyboard][hid]. [text-input]: https://github.com/Genymobile/scrcpy/issues?q=is%3Aopen+is%3Aissue+label%3Aunicode [accented-characters]: https://blog.rom1v.com/2018/03/introducing-scrcpy/#handle-accented-characters [#37]: https://github.com/Genymobile/scrcpy/issues/37 -[hid]: doc/hid-otg.md +[hid]: doc/keyboard.md#physical-keyboard-simulation ## Client issues diff --git a/README.md b/README.md index 8fabd556..7a671018 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,13 @@ Its features include: - [configurable quality](doc/video.md) - [camera mirroring](doc/camera.md) (Android 12+) - [mirroring as a webcam (V4L2)](doc/v4l2.md) (Linux-only) - - [physical keyboard/mouse simulation (HID)](doc/hid-otg.md) - - [OTG mode](doc/hid-otg.md#otg) + - physical [keyboard][hid-keyboard] and [mouse][hid-mouse] simulation (HID) + - [OTG mode](doc/otg.md) - and more… +[hid-keyboard]: doc/keyboard.md#physical-keyboard-simulation +[hid-mouse]: doc/mouse.md#physical-mouse-simulation + ## Prerequisites The Android device requires at least API 21 (Android 5.0). @@ -53,8 +56,7 @@ this option is set. [control]: https://github.com/Genymobile/scrcpy/issues/70#issuecomment-373286323 -Note that USB debugging is not required to run scrcpy in [OTG -mode](doc/hid-otg.md#otg). +Note that USB debugging is not required to run scrcpy in [OTG mode](doc/otg.md). ## Get the app @@ -73,11 +75,13 @@ documented in the following pages: - [Video](doc/video.md) - [Audio](doc/audio.md) - [Control](doc/control.md) + - [Keyboard](doc/keyboard.md) + - [Mouse](doc/mouse.md) - [Device](doc/device.md) - [Window](doc/window.md) - [Recording](doc/recording.md) - [Tunnels](doc/tunnels.md) - - [HID/OTG](doc/hid-otg.md) + - [OTG](doc/otg.md) - [Camera](doc/camera.md) - [Video4Linux](doc/v4l2.md) - [Shortcuts](doc/shortcuts.md) diff --git a/doc/control.md b/doc/control.md index 595e910e..d6d1265c 100644 --- a/doc/control.md +++ b/doc/control.md @@ -10,36 +10,9 @@ scrcpy --no-control scrcpy -n # short version ``` +## Keyboard and mouse -## Text injection preference - -Two kinds of [events][textevents] are generated when typing text: - - _key events_, signaling that a key is pressed or released; - - _text events_, signaling that a text has been entered. - -By default, letters are injected using key events, so that the keyboard behaves -as expected in games (typically for WASD keys). - -But this may [cause issues][prefertext]. If you encounter such a problem, you -can avoid it by: - -```bash -scrcpy --prefer-text -``` - -(but this will break keyboard behavior in games) - -On the contrary, you could force to always inject raw key events: - -```bash -scrcpy --raw-key-events -``` - -These options have no effect on HID keyboard (all key events are sent as -scancodes in this mode). - -[textevents]: https://blog.rom1v.com/2018/03/introducing-scrcpy/#handle-text-input -[prefertext]: https://github.com/Genymobile/scrcpy/issues/650#issuecomment-512945343 +Read [keyboard](keyboard.md) and [mouse](mouse.md). ## Copy-paste @@ -85,6 +58,7 @@ way as MOD+Shift+v). To disable automatic clipboard synchronization, use `--no-clipboard-autosync`. + ## Pinch-to-zoom, rotate and tilt simulation To simulate "pinch-to-zoom": Ctrl+_click-and-move_. @@ -100,20 +74,7 @@ at a location inverted through the center of the screen. When pressing Ctrl the x and y coordinates are inverted. Using Shift only inverts x. - -## Key repeat - -By default, holding a key down generates repeated key events. This can cause -performance problems in some games, where these events are useless anyway. - -To avoid forwarding repeated key events: - -```bash -scrcpy --no-key-repeat -``` - -This option has no effect on HID keyboard (key repeat is handled by Android -directly in this mode). +This only works for the default mouse mode (`--mouse=sdk`). ## Right-click and middle-click @@ -147,7 +108,3 @@ The target directory can be changed on start: ```bash scrcpy --push-target=/sdcard/Movies/ ``` - -## Physical keyboard and mouse simulation - -See the dedicated [HID/OTG](hid-otg.md) page. diff --git a/doc/develop.md b/doc/develop.md index 67d7f9b0..e5274783 100644 --- a/doc/develop.md +++ b/doc/develop.md @@ -234,7 +234,7 @@ The video and audio streams are decoded by [FFmpeg]. The client parses the command line arguments, then [runs one of two code paths][run]: - scrcpy in "normal" mode ([`scrcpy.c`]) - - scrcpy in [OTG mode](hid-otg.md) ([`scrcpy_otg.c`]) + - scrcpy in [OTG mode](otg.md) ([`scrcpy_otg.c`]) [run]: https://github.com/Genymobile/scrcpy/blob/a3cdf1a6b86ea22786e1f7d09b9c202feabc6949/app/src/main.c#L81-L82 [`scrcpy.c`]: https://github.com/Genymobile/scrcpy/blob/a3cdf1a6b86ea22786e1f7d09b9c202feabc6949/app/src/scrcpy.c#L292-L293 diff --git a/doc/hid-otg.md b/doc/hid-otg.md deleted file mode 100644 index 7dfc60fc..00000000 --- a/doc/hid-otg.md +++ /dev/null @@ -1,112 +0,0 @@ -# HID/OTG - -By default, _scrcpy_ injects input events at the Android API level. As an -alternative, when connected over USB, it is possible to send HID events, so that -scrcpy behaves as if it was a physical keyboard and/or mouse connected to the -Android device. - -A special [OTG](#otg) mode allows to control the device without mirroring (and -without USB debugging). - - -## Physical keyboard simulation - -By default, _scrcpy_ uses Android key or text injection. It works everywhere, -but is limited to ASCII. - -Instead, it can simulate a physical USB keyboard on Android to provide a better -input experience (using [USB HID over AOAv2][hid-aoav2]): the virtual keyboard -is disabled and it works for all characters and IME. - -[hid-aoav2]: https://source.android.com/devices/accessories/aoa2#hid-support - -However, it only works if the device is connected via USB. - -Note: On Windows, it may only work in [OTG mode](#otg), not while mirroring (it -is not possible to open a USB device if it is already open by another process -like the _adb daemon_). - -To enable this mode: - -```bash -scrcpy --hid-keyboard -scrcpy -K # short version -``` - -If it fails for some reason (for example because the device is not connected via -USB), it automatically fallbacks to the default mode (with a log in the -console). This allows using the same command line options when connected over -USB and TCP/IP. - -In this mode, raw key events (scancodes) are sent to the device, independently -of the host key mapping. Therefore, if your keyboard layout does not match, it -must be configured on the Android device, in Settings → System → Languages and -input → [Physical keyboard]. - -This settings page can be started directly: - -```bash -adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS -``` - -However, the option is only available when the HID keyboard is enabled (or when -a physical keyboard is connected). - -[Physical keyboard]: https://github.com/Genymobile/scrcpy/pull/2632#issuecomment-923756915 - - -## Physical mouse simulation - -By default, _scrcpy_ uses Android mouse events injection with absolute -coordinates. By simulating a physical mouse, a mouse pointer appears on the -Android device, and relative mouse motion, clicks and scrolls are injected. - -To enable this mode: - -```bash -scrcpy --hid-mouse -scrcpy -M # short version -``` - -When this mode is enabled, the computer mouse is "captured" (the mouse pointer -disappears from the computer and appears on the Android device instead). - -Special capture keys, either Alt or Super, toggle -(disable or enable) the mouse capture. Use one of them to give the control of -the mouse back to the computer. - - -## OTG - -It is possible to run _scrcpy_ with only physical keyboard and mouse simulation -(HID), as if the computer keyboard and mouse were plugged directly to the device -via an OTG cable. - -In this mode, `adb` (USB debugging) is not necessary, and mirroring is disabled. - -This is similar to `--hid-keyboard --hid-mouse`, but without mirroring. - -To enable OTG mode: - -```bash -scrcpy --otg -# Pass the serial if several USB devices are available -scrcpy --otg -s 0123456789abcdef -``` - -It is possible to enable only HID keyboard or HID mouse: - -```bash -scrcpy --otg --hid-keyboard # keyboard only -scrcpy --otg --hid-mouse # mouse only -scrcpy --otg --hid-keyboard --hid-mouse # keyboard and mouse -# for convenience, enable both by default -scrcpy --otg # keyboard and mouse -``` - -Like `--hid-keyboard` and `--hid-mouse`, it only works if the device is -connected over USB. - -## HID/OTG issues on Windows - -See [FAQ](/FAQ.md#hidotg-issues-on-windows). diff --git a/doc/keyboard.md b/doc/keyboard.md new file mode 100644 index 00000000..80dfe070 --- /dev/null +++ b/doc/keyboard.md @@ -0,0 +1,136 @@ +# Keyboard + +Several keyboard input modes are available: + + - `--keyboard=sdk` (default) + - `--keyboard=uhid` (or `-K`): simulates a physical HID keyboard using the UHID + kernel module on the device + - `--keyboard=aoa`: simulates a physical HID keyboard using the AOAv2 protocol + - `--keyboard=disabled` + +By default, `sdk` is used, but if you use scrcpy regularly, it is recommended to +use [`uhid`](#uhid) and configure the keyboard layout once and for all. + + +## SDK keyboard + +In this mode (`--keyboard=sdk`, or if the parameter is omitted), keyboard input +events are injected at the Android API level. It works everywhere, but it is +limited to ASCII and some other characters. + +Note that on some devices, an additional option must be enabled in developer +options for this keyboard mode to work. See +[prerequisites](/README.md#prerequisites). + +Additional parameters (specific to `--keyboard=sdk`) described below allow to +customize the behavior. + + +### Text injection preference + +Two kinds of [events][textevents] are generated when typing text: + - _key events_, signaling that a key is pressed or released; + - _text events_, signaling that a text has been entered. + +By default, numbers and "special characters" are inserted using text events, but +letters are injected using key events, so that the keyboard behaves as expected +in games (typically for WASD keys). + +But this may [cause issues][prefertext]. If you encounter such a problem, you +can inject letters as text (or just switch to [UHID](#uhid)): + +```bash +scrcpy --prefer-text +``` + +(but this will break keyboard behavior in games) + +On the contrary, you could force to always inject raw key events: + +```bash +scrcpy --raw-key-events +``` + +[textevents]: https://blog.rom1v.com/2018/03/introducing-scrcpy/#handle-text-input +[prefertext]: https://github.com/Genymobile/scrcpy/issues/650#issuecomment-512945343 + + +### Key repeat + +By default, holding a key down generates repeated key events. Ths can cause +performance problems in some games, where these events are useless anyway. + +To avoid forwarding repeated key events: + +```bash +scrcpy --no-key-repeat +``` + + +## Physical keyboard simulation + +Two modes allow to simulate a physical HID keyboard on the device. + +To work properly, it is necessary to configure (once and for all) the keyboard +layout on the device to match that of the computer. + +The configuration page can be opened in one of the following ways: + - from the scrcpy window (when `uhid` or `aoa` is used), by pressing + MOD+k (see [shortcuts](shortcuts.md)) + - from the device, in Settings → System → Languages and input → Physical + devices + - from a terminal on the computer, by executing `adb shell am start -a + android.settings.HARD_KEYBOARD_SETTINGS` + +From this configuration page, it is also possible to enable or disable on-screen +keyboard. + + +### UHID + +This mode simulates a physical HID keyboard using the [UHID] kernel module on the +device. + +[UHID]: https://kernel.org/doc/Documentation/hid/uhid.txt + +To enable UHID keyboard, use: + +```bash +scrcpy --keyboard=uhid +scrcpy -K # short version +``` + +Once the keyboard layout is configured (see above), it is the best mode for +using the keyboard while mirroring: + + - it works for all characters and IME (contrary to `--keyboard=sdk`) + - the on-screen keyboard can be disabled (contrary to `--keyboard=sdk`) + - it works over TCP/IP (wirelessly) (contrary to `--keyboard=aoa`) + - there are no issues on Windows (contrary to `--keyboard=aoa`) + +One drawback is that it may not work on old Android versions due to permission +errors. + + +### AOA + +This mode simulates a physical HID keyboard using the [AOAv2] protocol. + +[AOAv2]: https://source.android.com/devices/accessories/aoa2#hid-support + +To enable AOA keyboard, use: + +```bash +scrcpy --keyboard=aoa +``` + +Contrary to the other modes, it works at the USB level directly (so it only +works over USB). + +It does not use the scrcpy server, and does not require `adb` (USB debugging). +Therefore, it is possible to control the device (but not mirror) even with USB +debugging disabled (see [OTG](otg.md)). + +Note: On Windows, it may only work in [OTG mode](otg.md), not while mirroring +(it is not possible to open a USB device if it is already open by another +process like the _adb daemon_). diff --git a/doc/mouse.md b/doc/mouse.md new file mode 100644 index 00000000..d0342954 --- /dev/null +++ b/doc/mouse.md @@ -0,0 +1,70 @@ +# Mouse + +Several mouse input modes are available: + + - `--mouse=sdk` (default) + - `--mouse=uhid` (or `-M`): simulates a physical HID mouse using the UHID + kernel module on the device + - `--mouse=aoa`: simulates a physical HID mouse using the AOAv2 protocol + - `--mouse=disabled` + + +## SDK mouse + +In this mode (`--mouse=sdk`, or if the parameter is omitted), mouse input events +are injected at the Android API level with absolute coordinates. + +Note that on some devices, an additional option must be enabled in developer +options for this mouse mode to work. See +[prerequisites](/README.md#prerequisites). + + +## Physical mouse simulation + +Two modes allow to simulate a physical HID mouse on the device. + +In these modes, the computer mouse is "captured": the mouse pointer disappears +from the computer and appears on the Android device instead. + +Special capture keys, either Alt or Super, toggle +(disable or enable) the mouse capture. Use one of them to give the control of +the mouse back to the computer. + + +### UHID + +This mode simulates a physical HID mouse using the [UHID] kernel module on the +device. + +[UHID]: https://kernel.org/doc/Documentation/hid/uhid.txt + +To enable UHID mouse, use: + +```bash +scrcpy --mouse=uhid +scrcpy -M # short version +``` + + +### AOA + +This mode simulates a physical HID mouse using the [AOAv2] protocol. + +[AOAv2]: https://source.android.com/devices/accessories/aoa2#hid-support + +To enable AOA mouse, use: + +```bash +scrcpy --mouse=aoa +``` + +Contrary to the other modes, it works at the USB level directly (so it only +works over USB). + +It does not use the scrcpy server, and does not require `adb` (USB debugging). +Therefore, it is possible to control the device (but not mirror) even with USB +debugging disabled (see [OTG](otg.md)). + +Note: On Windows, it may only work in [OTG mode](otg.md), not while mirroring +(it is not possible to open a USB device if it is already open by another +process like the _adb daemon_). diff --git a/doc/otg.md b/doc/otg.md new file mode 100644 index 00000000..3c7ed467 --- /dev/null +++ b/doc/otg.md @@ -0,0 +1,37 @@ +# OTG + +By default, _scrcpy_ injects input events at the Android API level. As an +alternative, when connected over USB, it is possible to send HID events, so that +scrcpy behaves as if it was a physical keyboard and/or mouse connected to the +Android device. + +A special mode allows to control the device without mirroring, using AOA +[keyboard](keyboard.md#aoa) and [mouse](mouse.md#aoa). Therefore, it is possible +to run _scrcpy_ with only physical keyboard and mouse simulation (HID), as if +the computer keyboard and mouse were plugged directly to the device via an OTG +cable. + +In this mode, `adb` (USB debugging) is not necessary, and mirroring is disabled. + +This is similar to `--keyboard=aoa --mouse=aoa`, but without mirroring. + +To enable OTG mode: + +```bash +scrcpy --otg +# Pass the serial if several USB devices are available +scrcpy --otg -s 0123456789abcdef +``` + +It is possible to disable HID keyboard or HID mouse: + +```bash +scrcpy --otg --keyboard=disabled +scrcpy --otg --mouse=disabled +``` + +It only works if the device is connected over USB. + +## OTG issues on Windows + +See [FAQ](/FAQ.md#otg-issues-on-windows). From bf069bd37bb064fb49b7f75c6ea665706b599784 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 29 Feb 2024 09:01:25 +0100 Subject: [PATCH 68/79] Document usage examples This exposes several common options on the front page. --- README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/README.md b/README.md index 7a671018..701bb075 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,41 @@ Note that USB debugging is not required to run scrcpy in [OTG mode](doc/otg.md). - [macOS](doc/macos.md) +## Usage examples + +There are a lot of options, [documented](#user-documentation) in separate pages. +Here are just some common examples. + + - Capture the screen in H.265 (better quality), limit the size to 1920, limit + the frame rate to 60fps, disable audio, and control the device by simulating + a physical keyboard: + + ```bash + scrcpy --video-codec=h265 --max-size=1920 --max-fps=60 --no-audio --keyboard=uhid + scrcpy --video-codec=h265 -m1920 --max-fps=60 --no-audio -K # short version + ``` + + - Record the device camera in H.265 at 1920x1080 (and microphone) to an MP4 + file: + + ```bash + scrcpy --video-source=camera --video-codec=h265 --camera-size=1920x1080 --record=file.mp4 + ``` + + - Capture the device front camera and expose it as a webcam on the computer (on + Linux): + + ```bash + scrcpy --video-source=camera --camera-size=1920x1080 --camera-facing=front --v4l2-sink=/dev/video2 --no-playback + ``` + + - Control the device without mirroring by simulating a physical keyboard and + mouse (USB debugging not required): + + ```bash + scrcpy --otg + ``` + ## User documentation The application provides a lot of features and configuration options. They are From cdf09805c042cc760397b08d9e7cf58fbf8f76a4 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 29 Feb 2024 19:37:05 +0100 Subject: [PATCH 69/79] Add missing initialization --- app/src/display.c | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/display.c b/app/src/display.c index ba15cd14..c8df615d 100644 --- a/app/src/display.c +++ b/app/src/display.c @@ -62,6 +62,7 @@ sc_display_init(struct sc_display *display, SDL_Window *window, bool mipmaps) { LOGD("Trilinear filtering disabled (not an OpenGL renderer)"); } + display->texture = NULL; display->pending.flags = 0; display->pending.frame = NULL; From fd0f432e877153d83ed435474fb7b04e41de4269 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 29 Feb 2024 19:37:14 +0100 Subject: [PATCH 70/79] Detect missing initializations Write invalid data in memory to easily detect missing initializations in debug mode. --- app/src/scrcpy.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index c63a95c2..eb9cd201 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -312,6 +312,10 @@ scrcpy_generate_scid(void) { enum scrcpy_exit_code scrcpy(struct scrcpy_options *options) { static struct scrcpy scrcpy; +#ifndef NDEBUG + // Detect missing initializations + memset(&scrcpy, 42, sizeof(scrcpy)); +#endif struct scrcpy *s = &scrcpy; // Minimal SDL initialization From 4dca08cfe3eadd4438bf235bd62050059aec1801 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 1 Mar 2024 09:54:31 +0100 Subject: [PATCH 71/79] Set SDL hints before creating any thread To avoid race conditions in SDL (reported by TSAN). --- app/src/scrcpy.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index eb9cd201..961f6202 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -409,6 +409,12 @@ scrcpy(struct scrcpy_options *options) { return SCRCPY_EXIT_FAILURE; } + if (options->video_playback) { + // Set hints before starting the server thread to avoid race conditions + // in SDL + sdl_set_hints(options->render_driver); + } + if (!sc_server_start(&s->server)) { goto end; } @@ -425,10 +431,6 @@ scrcpy(struct scrcpy_options *options) { assert(!options->video_playback || options->video); assert(!options->audio_playback || options->audio); - if (options->video_playback) { - sdl_set_hints(options->render_driver); - } - if (options->video_playback || (options->control && options->clipboard_autosync)) { // Initialize the video subsystem even if --no-video or From 36189b90ea815d8fced961c36c80f146d5952324 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 1 Mar 2024 09:55:32 +0100 Subject: [PATCH 72/79] Remove spurious line --- app/src/scrcpy.c | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 961f6202..f43af35e 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -106,7 +106,6 @@ static BOOL WINAPI windows_ctrl_handler(DWORD ctrl_type) { static void sdl_set_hints(const char *render_driver) { - if (render_driver && !SDL_SetHint(SDL_HINT_RENDER_DRIVER, render_driver)) { LOGW("Could not set render driver"); } From 125b1103e1cdfe676d66a9223df82c423c5e75bf Mon Sep 17 00:00:00 2001 From: inson1 <75314629+inson1@users.noreply.github.com> Date: Sat, 2 Mar 2024 15:39:56 +0100 Subject: [PATCH 73/79] Happy new year 2024! PR #4716 Signed-off-by: Romain Vimont --- LICENSE | 2 +- README.md | 2 +- app/scrcpy.1 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE b/LICENSE index 55f96811..d9326a74 100644 --- a/LICENSE +++ b/LICENSE @@ -188,7 +188,7 @@ identification within third-party archives. Copyright (C) 2018 Genymobile - Copyright (C) 2018-2023 Romain Vimont + Copyright (C) 2018-2024 Romain Vimont Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 0a3d03cb..5d9f04a9 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ work][donate]: ## Licence Copyright (C) 2018 Genymobile - Copyright (C) 2018-2023 Romain Vimont + Copyright (C) 2018-2024 Romain Vimont Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 0c34b4e2..eed1f355 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -689,7 +689,7 @@ Report bugs to . .SH COPYRIGHT Copyright \(co 2018 Genymobile -Copyright \(co 2018\-2023 Romain Vimont +Copyright \(co 2018\-2024 Romain Vimont Licensed under the Apache License, Version 2.0. From 8d87b91f692914ada1c146bd911ab4623552174b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 1 Mar 2024 20:02:00 +0100 Subject: [PATCH 74/79] Build dependencies from sources The project has 3 build dependencies: - SDL - FFmpeg - libusb For Windows, the release script downloaded pre-built build dependencies (either from upstream, or from the scrcpy-deps repository). Instead, download the source releases and build locally. This offers more flexibility. The official adb release is still downloaded and included as is in the release archive (it is not a build dependency). Also upgrade FFmpeg to 6.1.1 and libusb to 1.0.27. PR #4713 --- app/deps/.gitignore | 1 + app/deps/README | 27 ++++++ app/deps/adb.sh | 32 ++++++++ app/deps/common | 55 +++++++++++++ app/deps/ffmpeg.sh | 91 +++++++++++++++++++++ app/deps/libusb.sh | 44 ++++++++++ app/deps/patches/ffmpeg-6.1-fix-build.patch | 27 ++++++ app/deps/sdl.sh | 47 +++++++++++ app/prebuilt-deps/.gitignore | 1 - app/prebuilt-deps/common | 22 ----- app/prebuilt-deps/prepare-adb.sh | 32 -------- app/prebuilt-deps/prepare-ffmpeg.sh | 30 ------- app/prebuilt-deps/prepare-libusb.sh | 37 --------- app/prebuilt-deps/prepare-sdl.sh | 34 -------- release.mk | 52 ++++++------ 15 files changed, 348 insertions(+), 184 deletions(-) create mode 100644 app/deps/.gitignore create mode 100644 app/deps/README create mode 100755 app/deps/adb.sh create mode 100644 app/deps/common create mode 100755 app/deps/ffmpeg.sh create mode 100755 app/deps/libusb.sh create mode 100644 app/deps/patches/ffmpeg-6.1-fix-build.patch create mode 100755 app/deps/sdl.sh delete mode 100644 app/prebuilt-deps/.gitignore delete mode 100755 app/prebuilt-deps/common delete mode 100755 app/prebuilt-deps/prepare-adb.sh delete mode 100755 app/prebuilt-deps/prepare-ffmpeg.sh delete mode 100755 app/prebuilt-deps/prepare-libusb.sh delete mode 100755 app/prebuilt-deps/prepare-sdl.sh diff --git a/app/deps/.gitignore b/app/deps/.gitignore new file mode 100644 index 00000000..ccf6a49e --- /dev/null +++ b/app/deps/.gitignore @@ -0,0 +1 @@ +/work diff --git a/app/deps/README b/app/deps/README new file mode 100644 index 00000000..9cfb5c06 --- /dev/null +++ b/app/deps/README @@ -0,0 +1,27 @@ +This directory (app/deps/) contains: + +*.sh : shell scripts to download and build dependencies + +patches/ : patches to fix dependencies (used by scripts) + +work/sources/ : downloaded tarballs and extracted folders + ffmpeg-6.1.1.tar.xz + ffmpeg-6.1.1/ + libusb-1.0.27.tar.gz + libusb-1.0.27/ + ... +work/build/ : build dirs for each dependency/version/architecture + ffmpeg-6.1.1/win32/ + ffmpeg-6.1.1/win64/ + libusb-1.0.27/win32/ + libusb-1.0.27/win64/ + ... +work/install/ : install dirs for each architexture + win32/bin/ + win32/include/ + win32/lib/ + win32/share/ + win64/bin/ + win64/include/ + win64/lib/ + win64/share/ diff --git a/app/deps/adb.sh b/app/deps/adb.sh new file mode 100755 index 00000000..e2408216 --- /dev/null +++ b/app/deps/adb.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -ex +DEPS_DIR=$(dirname ${BASH_SOURCE[0]}) +cd "$DEPS_DIR" +. common + +VERSION=34.0.5 +FILENAME=platform-tools_r$VERSION-windows.zip +PROJECT_DIR=platform-tools-$VERSION +SHA256SUM=3f8320152704377de150418a3c4c9d07d16d80a6c0d0d8f7289c22c499e33571 + +cd "$SOURCES_DIR" + +if [[ -d "$PROJECT_DIR" ]] +then + echo "$PWD/$PROJECT_DIR" found +else + get_file "https://dl.google.com/android/repository/$FILENAME" "$FILENAME" "$SHA256SUM" + mkdir -p "$PROJECT_DIR" + cd "$PROJECT_DIR" + ZIP_PREFIX=platform-tools + unzip "../$FILENAME" \ + "$ZIP_PREFIX"/AdbWinApi.dll \ + "$ZIP_PREFIX"/AdbWinUsbApi.dll \ + "$ZIP_PREFIX"/adb.exe + mv "$ZIP_PREFIX"/* . + rmdir "$ZIP_PREFIX" +fi + +mkdir -p "$INSTALL_DIR/$HOST/bin" +cd "$INSTALL_DIR/$HOST/bin" +cp -r "$SOURCES_DIR/$PROJECT_DIR"/. "$INSTALL_DIR/$HOST/bin/" diff --git a/app/deps/common b/app/deps/common new file mode 100644 index 00000000..c1cc7729 --- /dev/null +++ b/app/deps/common @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# This file is intended to be sourced by other scripts, not executed + +if [[ $# != 1 ]] +then + # : win32 or win64 + echo "Syntax: $0 " >&2 + exit 1 +fi + +HOST="$1" + +if [[ "$HOST" = win32 ]] +then + HOST_TRIPLET=i686-w64-mingw32 +elif [[ "$HOST" = win64 ]] +then + HOST_TRIPLET=x86_64-w64-mingw32 +else + echo "Unsupported host: $HOST" >&2 + exit 1 +fi + +DEPS_DIR=$(dirname ${BASH_SOURCE[0]}) +cd "$DEPS_DIR" + +PATCHES_DIR="$PWD/patches" + +WORK_DIR="$PWD/work" +SOURCES_DIR="$WORK_DIR/sources" +BUILD_DIR="$WORK_DIR/build" +INSTALL_DIR="$WORK_DIR/install" + +mkdir -p "$INSTALL_DIR" "$SOURCES_DIR" "$WORK_DIR" + +checksum() { + local file="$1" + local sum="$2" + echo "$file: verifying checksum..." + echo "$sum $file" | sha256sum -c +} + +get_file() { + local url="$1" + local file="$2" + local sum="$3" + if [[ -f "$file" ]] + then + echo "$file: found" + else + echo "$file: not found, downloading..." + wget "$url" -O "$file" + fi + checksum "$file" "$sum" +} diff --git a/app/deps/ffmpeg.sh b/app/deps/ffmpeg.sh new file mode 100755 index 00000000..19fb2991 --- /dev/null +++ b/app/deps/ffmpeg.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -ex +DEPS_DIR=$(dirname ${BASH_SOURCE[0]}) +cd "$DEPS_DIR" +. common + +VERSION=6.1.1 +FILENAME=ffmpeg-$VERSION.tar.xz +PROJECT_DIR=ffmpeg-$VERSION +SHA256SUM=8684f4b00f94b85461884c3719382f1261f0d9eb3d59640a1f4ac0873616f968 + +cd "$SOURCES_DIR" + +if [[ -d "$PROJECT_DIR" ]] +then + echo "$PWD/$PROJECT_DIR" found +else + get_file "https://ffmpeg.org/releases/$FILENAME" "$FILENAME" "$SHA256SUM" + tar xf "$FILENAME" # First level directory is "$PROJECT_DIR" + patch -d "$PROJECT_DIR" -p1 < "$PATCHES_DIR"/ffmpeg-6.1-fix-build.patch +fi + +mkdir -p "$BUILD_DIR/$PROJECT_DIR" +cd "$BUILD_DIR/$PROJECT_DIR" + +if [[ "$HOST" = win32 ]] +then + ARCH=x86 +elif [[ "$HOST" = win64 ]] +then + ARCH=x86_64 +else + echo "Unsupported host: $HOST" >&2 + exit 1 +fi + +# -static-libgcc to avoid missing libgcc_s_dw2-1.dll +# -static to avoid dynamic dependency to zlib +export CFLAGS='-static-libgcc -static' +export CXXFLAGS="$CFLAGS" +export LDFLAGS='-static-libgcc -static' + +if [[ -d "$HOST" ]] +then + echo "'$PWD/$HOST' already exists, not reconfigured" + cd "$HOST" +else + mkdir "$HOST" + cd "$HOST" + + "$SOURCES_DIR/$PROJECT_DIR"/configure \ + --prefix="$INSTALL_DIR/$HOST" \ + --enable-cross-compile \ + --target-os=mingw32 \ + --arch="$ARCH" \ + --cross-prefix="${HOST_TRIPLET}-" \ + --cc="${HOST_TRIPLET}-gcc" \ + --extra-cflags="-O2 -fPIC" \ + --enable-shared \ + --disable-static \ + --disable-programs \ + --disable-doc \ + --disable-swscale \ + --disable-postproc \ + --disable-avfilter \ + --disable-avdevice \ + --disable-network \ + --disable-everything \ + --enable-swresample \ + --enable-decoder=h264 \ + --enable-decoder=hevc \ + --enable-decoder=av1 \ + --enable-decoder=pcm_s16le \ + --enable-decoder=opus \ + --enable-decoder=aac \ + --enable-decoder=flac \ + --enable-decoder=png \ + --enable-protocol=file \ + --enable-demuxer=image2 \ + --enable-parser=png \ + --enable-zlib \ + --enable-muxer=matroska \ + --enable-muxer=mp4 \ + --enable-muxer=opus \ + --enable-muxer=flac \ + --enable-muxer=wav \ + --disable-vulkan +fi + +make -j +make install diff --git a/app/deps/libusb.sh b/app/deps/libusb.sh new file mode 100755 index 00000000..97fc3c72 --- /dev/null +++ b/app/deps/libusb.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -ex +DEPS_DIR=$(dirname ${BASH_SOURCE[0]}) +cd "$DEPS_DIR" +. common + +VERSION=1.0.27 +FILENAME=libusb-$VERSION.tar.bz2 +PROJECT_DIR=libusb-$VERSION +SHA256SUM=ffaa41d741a8a3bee244ac8e54a72ea05bf2879663c098c82fc5757853441575 + +cd "$SOURCES_DIR" + +if [[ -d "$PROJECT_DIR" ]] +then + echo "$PWD/$PROJECT_DIR" found +else + get_file "https://github.com/libusb/libusb/releases/download/v$VERSION/libusb-$VERSION.tar.bz2" "$FILENAME" "$SHA256SUM" + tar xf "$FILENAME" # First level directory is "$PROJECT_DIR" +fi + +mkdir -p "$BUILD_DIR/$PROJECT_DIR" +cd "$BUILD_DIR/$PROJECT_DIR" + +export CFLAGS='-O2' +export CXXFLAGS="$CFLAGS" + +if [[ -d "$HOST" ]] +then + echo "'$PWD/$HOST' already exists, not reconfigured" + cd "$HOST" +else + mkdir "$HOST" + cd "$HOST" + + "$SOURCES_DIR/$PROJECT_DIR"/configure \ + --prefix="$INSTALL_DIR/$HOST" \ + --host="$HOST_TRIPLET" \ + --enable-shared \ + --disable-static +fi + +make -j +make install-strip diff --git a/app/deps/patches/ffmpeg-6.1-fix-build.patch b/app/deps/patches/ffmpeg-6.1-fix-build.patch new file mode 100644 index 00000000..ed4df48d --- /dev/null +++ b/app/deps/patches/ffmpeg-6.1-fix-build.patch @@ -0,0 +1,27 @@ +From 03c80197afb324da38c9b70254231e3fdcfa68fc Mon Sep 17 00:00:00 2001 +From: Romain Vimont +Date: Sun, 12 Nov 2023 17:58:50 +0100 +Subject: [PATCH] Fix FFmpeg 6.1 build + +Build failed on tag n6.1 With --enable-decoder=av1 but without +--enable-muxer=av1. +--- + libavcodec/Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/libavcodec/Makefile b/libavcodec/Makefile +index 580a8d6b54..aff19b670c 100644 +--- a/libavcodec/Makefile ++++ b/libavcodec/Makefile +@@ -249,7 +249,7 @@ OBJS-$(CONFIG_ATRAC3PAL_DECODER) += atrac3plusdec.o atrac3plus.o \ + OBJS-$(CONFIG_ATRAC9_DECODER) += atrac9dec.o + OBJS-$(CONFIG_AURA_DECODER) += cyuv.o + OBJS-$(CONFIG_AURA2_DECODER) += aura.o +-OBJS-$(CONFIG_AV1_DECODER) += av1dec.o ++OBJS-$(CONFIG_AV1_DECODER) += av1dec.o av1_parse.o + OBJS-$(CONFIG_AV1_CUVID_DECODER) += cuviddec.o + OBJS-$(CONFIG_AV1_MEDIACODEC_DECODER) += mediacodecdec.o + OBJS-$(CONFIG_AV1_MEDIACODEC_ENCODER) += mediacodecenc.o +-- +2.42.0 + diff --git a/app/deps/sdl.sh b/app/deps/sdl.sh new file mode 100755 index 00000000..36c7ab1c --- /dev/null +++ b/app/deps/sdl.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -ex +DEPS_DIR=$(dirname ${BASH_SOURCE[0]}) +cd "$DEPS_DIR" +. common + +VERSION=2.28.5 +FILENAME=SDL-$VERSION.tar.gz +PROJECT_DIR=SDL-release-$VERSION +SHA256SUM=9f0556e4a24ef5b267010038ad9e9948b62f236d5bcc4b22179f95ef62d84023 + +cd "$SOURCES_DIR" + +if [[ -d "$PROJECT_DIR" ]] +then + echo "$PWD/$PROJECT_DIR" found +else + get_file "https://github.com/libsdl-org/SDL/archive/refs/tags/release-$VERSION.tar.gz" "$FILENAME" "$SHA256SUM" + tar xf "$FILENAME" # First level directory is "$PROJECT_DIR" +fi + +mkdir -p "$BUILD_DIR/$PROJECT_DIR" +cd "$BUILD_DIR/$PROJECT_DIR" + +export CFLAGS='-O2' +export CXXFLAGS="$CFLAGS" + +if [[ -d "$HOST" ]] +then + echo "'$PWD/$HOST' already exists, not reconfigured" + cd "$HOST" +else + mkdir "$HOST" + cd "$HOST" + + "$SOURCES_DIR/$PROJECT_DIR"/configure \ + --prefix="$INSTALL_DIR/$HOST" \ + --host="$HOST_TRIPLET" \ + --enable-shared \ + --disable-static +fi + +make -j +# There is no "make install-strip" +make install +# Strip manually +${HOST_TRIPLET}-strip "$INSTALL_DIR/$HOST/bin/SDL2.dll" diff --git a/app/prebuilt-deps/.gitignore b/app/prebuilt-deps/.gitignore deleted file mode 100644 index 3af0ccb6..00000000 --- a/app/prebuilt-deps/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/data diff --git a/app/prebuilt-deps/common b/app/prebuilt-deps/common deleted file mode 100755 index c97f7de4..00000000 --- a/app/prebuilt-deps/common +++ /dev/null @@ -1,22 +0,0 @@ -PREBUILT_DATA_DIR=data - -checksum() { - local file="$1" - local sum="$2" - echo "$file: verifying checksum..." - echo "$sum $file" | sha256sum -c -} - -get_file() { - local url="$1" - local file="$2" - local sum="$3" - if [[ -f "$file" ]] - then - echo "$file: found" - else - echo "$file: not found, downloading..." - wget "$url" -O "$file" - fi - checksum "$file" "$sum" -} diff --git a/app/prebuilt-deps/prepare-adb.sh b/app/prebuilt-deps/prepare-adb.sh deleted file mode 100755 index 4fb6fd7d..00000000 --- a/app/prebuilt-deps/prepare-adb.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -e -DIR=$(dirname ${BASH_SOURCE[0]}) -cd "$DIR" -. common -mkdir -p "$PREBUILT_DATA_DIR" -cd "$PREBUILT_DATA_DIR" - -DEP_DIR=platform-tools-34.0.5 - -FILENAME=platform-tools_r34.0.5-windows.zip -SHA256SUM=3f8320152704377de150418a3c4c9d07d16d80a6c0d0d8f7289c22c499e33571 - -if [[ -d "$DEP_DIR" ]] -then - echo "$DEP_DIR" found - exit 0 -fi - -get_file "https://dl.google.com/android/repository/$FILENAME" \ - "$FILENAME" "$SHA256SUM" - -mkdir "$DEP_DIR" -cd "$DEP_DIR" - -ZIP_PREFIX=platform-tools -unzip "../$FILENAME" \ - "$ZIP_PREFIX"/AdbWinApi.dll \ - "$ZIP_PREFIX"/AdbWinUsbApi.dll \ - "$ZIP_PREFIX"/adb.exe -mv "$ZIP_PREFIX"/* . -rmdir "$ZIP_PREFIX" diff --git a/app/prebuilt-deps/prepare-ffmpeg.sh b/app/prebuilt-deps/prepare-ffmpeg.sh deleted file mode 100755 index 19840afb..00000000 --- a/app/prebuilt-deps/prepare-ffmpeg.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -e -DIR=$(dirname ${BASH_SOURCE[0]}) -cd "$DIR" -. common -mkdir -p "$PREBUILT_DATA_DIR" -cd "$PREBUILT_DATA_DIR" - -VERSION=6.1-scrcpy-3 -DEP_DIR="ffmpeg-$VERSION" - -FILENAME="$DEP_DIR".7z -SHA256SUM=b646d18a3d543a4e4c46881568213499f22e4454a464e1552f03f2ac9cc3a05a - -if [[ -d "$DEP_DIR" ]] -then - echo "$DEP_DIR" found - exit 0 -fi - -get_file "https://github.com/rom1v/scrcpy-deps/releases/download/$VERSION/$FILENAME" \ - "$FILENAME" "$SHA256SUM" - -mkdir "$DEP_DIR" -cd "$DEP_DIR" - -ZIP_PREFIX=ffmpeg -7z x "../$FILENAME" -mv "$ZIP_PREFIX"/* . -rmdir "$ZIP_PREFIX" diff --git a/app/prebuilt-deps/prepare-libusb.sh b/app/prebuilt-deps/prepare-libusb.sh deleted file mode 100755 index b31c45eb..00000000 --- a/app/prebuilt-deps/prepare-libusb.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -e -DIR=$(dirname ${BASH_SOURCE[0]}) -cd "$DIR" -. common -mkdir -p "$PREBUILT_DATA_DIR" -cd "$PREBUILT_DATA_DIR" - -VERSION=1.0.26 -DEP_DIR="libusb-$VERSION" - -FILENAME="libusb-$VERSION-binaries.7z" -SHA256SUM=9c242696342dbde9cdc47239391f71833939bf9f7aa2bbb28cdaabe890465ec5 - -if [[ -d "$DEP_DIR" ]] -then - echo "$DEP_DIR" found - exit 0 -fi - -get_file "https://github.com/libusb/libusb/releases/download/v$VERSION/$FILENAME" \ - "$FILENAME" "$SHA256SUM" - -mkdir "$DEP_DIR" -cd "$DEP_DIR" - -7z x "../$FILENAME" \ - "libusb-$VERSION-binaries/libusb-MinGW-Win32/" \ - "libusb-$VERSION-binaries/libusb-MinGW-x64/" - -mv "libusb-$VERSION-binaries/libusb-MinGW-Win32" . -mv "libusb-$VERSION-binaries/libusb-MinGW-x64" . -rm -rf "libusb-$VERSION-binaries" - -# Rename the dll to get the same library name on all platforms -mv libusb-MinGW-Win32/bin/msys-usb-1.0.dll libusb-MinGW-Win32/bin/libusb-1.0.dll -mv libusb-MinGW-x64/bin/msys-usb-1.0.dll libusb-MinGW-x64/bin/libusb-1.0.dll diff --git a/app/prebuilt-deps/prepare-sdl.sh b/app/prebuilt-deps/prepare-sdl.sh deleted file mode 100755 index 7569744f..00000000 --- a/app/prebuilt-deps/prepare-sdl.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -e -DIR=$(dirname ${BASH_SOURCE[0]}) -cd "$DIR" -. common -mkdir -p "$PREBUILT_DATA_DIR" -cd "$PREBUILT_DATA_DIR" - -VERSION=2.28.5 -DEP_DIR="SDL2-$VERSION" - -FILENAME="SDL2-devel-$VERSION-mingw.tar.gz" -SHA256SUM=3c0c655c2ebf67cad48fead72761d1601740ded30808952c3274ba223d226c21 - -if [[ -d "$DEP_DIR" ]] -then - echo "$DEP_DIR" found - exit 0 -fi - -get_file "https://github.com/libsdl-org/SDL/releases/download/release-$VERSION/$FILENAME" \ - "$FILENAME" "$SHA256SUM" - -mkdir "$DEP_DIR" -cd "$DEP_DIR" - -TAR_PREFIX="$DEP_DIR" # root directory inside the tar has the same name -tar xf "../$FILENAME" --strip-components=1 \ - "$TAR_PREFIX"/i686-w64-mingw32/bin/SDL2.dll \ - "$TAR_PREFIX"/i686-w64-mingw32/include/ \ - "$TAR_PREFIX"/i686-w64-mingw32/lib/ \ - "$TAR_PREFIX"/x86_64-w64-mingw32/bin/SDL2.dll \ - "$TAR_PREFIX"/x86_64-w64-mingw32/include/ \ - "$TAR_PREFIX"/x86_64-w64-mingw32/lib/ \ diff --git a/release.mk b/release.mk index fd969e5a..89f3da21 100644 --- a/release.mk +++ b/release.mk @@ -62,38 +62,38 @@ build-server: meson setup "$(SERVER_BUILD_DIR)" --buildtype release -Dcompile_app=false ) ninja -C "$(SERVER_BUILD_DIR)" -prepare-deps: - @app/prebuilt-deps/prepare-adb.sh - @app/prebuilt-deps/prepare-sdl.sh - @app/prebuilt-deps/prepare-ffmpeg.sh - @app/prebuilt-deps/prepare-libusb.sh - -build-win32: prepare-deps +prepare-deps-win32: + @app/deps/adb.sh win32 + @app/deps/sdl.sh win32 + @app/deps/ffmpeg.sh win32 + @app/deps/libusb.sh win32 + +prepare-deps-win64: + @app/deps/adb.sh win64 + @app/deps/sdl.sh win64 + @app/deps/ffmpeg.sh win64 + @app/deps/libusb.sh win64 + +build-win32: prepare-deps-win32 rm -rf "$(WIN32_BUILD_DIR)" mkdir -p "$(WIN32_BUILD_DIR)/local" - cp -r app/prebuilt-deps/data/ffmpeg-6.1-scrcpy-3/win32/. "$(WIN32_BUILD_DIR)/local/" - cp -r app/prebuilt-deps/data/SDL2-2.28.5/i686-w64-mingw32/. "$(WIN32_BUILD_DIR)/local/" - cp -r app/prebuilt-deps/data/libusb-1.0.26/libusb-MinGW-Win32/. "$(WIN32_BUILD_DIR)/local/" meson setup "$(WIN32_BUILD_DIR)" \ - --pkg-config-path="$(WIN32_BUILD_DIR)/local/lib/pkgconfig" \ - -Dc_args="-I$(PWD)/$(WIN32_BUILD_DIR)/local/include" \ - -Dc_link_args="-L$(PWD)/$(WIN32_BUILD_DIR)/local/lib" \ + --pkg-config-path="app/deps/work/install/win32/lib/pkgconfig" \ + -Dc_args="-I$(PWD)/app/deps/work/install/win32/include" \ + -Dc_link_args="-L$(PWD)/app/deps/work/install/win32/lib" \ --cross-file=cross_win32.txt \ --buildtype=release --strip -Db_lto=true \ -Dcompile_server=false \ -Dportable=true ninja -C "$(WIN32_BUILD_DIR)" -build-win64: prepare-deps +build-win64: prepare-deps-win64 rm -rf "$(WIN64_BUILD_DIR)" mkdir -p "$(WIN64_BUILD_DIR)/local" - cp -r app/prebuilt-deps/data/ffmpeg-6.1-scrcpy-3/win64/. "$(WIN64_BUILD_DIR)/local/" - cp -r app/prebuilt-deps/data/SDL2-2.28.5/x86_64-w64-mingw32/. "$(WIN64_BUILD_DIR)/local/" - cp -r app/prebuilt-deps/data/libusb-1.0.26/libusb-MinGW-x64/. "$(WIN64_BUILD_DIR)/local/" meson setup "$(WIN64_BUILD_DIR)" \ - --pkg-config-path="$(WIN64_BUILD_DIR)/local/lib/pkgconfig" \ - -Dc_args="-I$(PWD)/$(WIN64_BUILD_DIR)/local/include" \ - -Dc_link_args="-L$(PWD)/$(WIN64_BUILD_DIR)/local/lib" \ + --pkg-config-path="app/deps/work/install/win64/lib/pkgconfig" \ + -Dc_args="-I$(PWD)/app/deps/work/install/win64/include" \ + -Dc_link_args="-L$(PWD)/app/deps/work/install/win64/lib" \ --cross-file=cross_win64.txt \ --buildtype=release --strip -Db_lto=true \ -Dcompile_server=false \ @@ -108,10 +108,8 @@ dist-win32: build-server build-win32 cp app/data/scrcpy-noconsole.vbs "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/data/icon.png "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/data/open_a_terminal_here.bat "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/platform-tools-34.0.5/adb.exe "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/platform-tools-34.0.5/AdbWinApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/platform-tools-34.0.5/AdbWinUsbApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp "$(WIN32_BUILD_DIR)"/local/bin/*.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/deps/work/install/win32/bin/*.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/deps/work/install/win32/bin/adb.exe "$(DIST)/$(WIN32_TARGET_DIR)/" dist-win64: build-server build-win64 mkdir -p "$(DIST)/$(WIN64_TARGET_DIR)" @@ -121,10 +119,8 @@ dist-win64: build-server build-win64 cp app/data/scrcpy-noconsole.vbs "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/data/icon.png "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/data/open_a_terminal_here.bat "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/platform-tools-34.0.5/adb.exe "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/platform-tools-34.0.5/AdbWinApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/platform-tools-34.0.5/AdbWinUsbApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp "$(WIN64_BUILD_DIR)"/local/bin/*.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/deps/work/install/win64/bin/*.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/deps/work/install/win64/bin/adb.exe "$(DIST)/$(WIN64_TARGET_DIR)/" zip-win32: dist-win32 cd "$(DIST)"; \ From af573090741e73c66c4a543dcd94fd771c51b7be Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 2 Mar 2024 23:22:09 +0100 Subject: [PATCH 75/79] Bump version to 2.4 --- app/scrcpy-windows.rc | 2 +- meson.build | 2 +- server/build.gradle | 4 ++-- server/build_without_gradle.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/scrcpy-windows.rc b/app/scrcpy-windows.rc index 895b9c93..059e91d4 100644 --- a/app/scrcpy-windows.rc +++ b/app/scrcpy-windows.rc @@ -13,7 +13,7 @@ BEGIN VALUE "LegalCopyright", "Romain Vimont, Genymobile" VALUE "OriginalFilename", "scrcpy.exe" VALUE "ProductName", "scrcpy" - VALUE "ProductVersion", "2.3.1" + VALUE "ProductVersion", "2.4" END END BLOCK "VarFileInfo" diff --git a/meson.build b/meson.build index 4ae91f69..22d0f4ef 100644 --- a/meson.build +++ b/meson.build @@ -1,5 +1,5 @@ project('scrcpy', 'c', - version: '2.3.1', + version: '2.4', meson_version: '>= 0.48', default_options: [ 'c_std=c11', diff --git a/server/build.gradle b/server/build.gradle index 1a18d997..6a1b09df 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -7,8 +7,8 @@ android { applicationId "com.genymobile.scrcpy" minSdkVersion 21 targetSdkVersion 34 - versionCode 20301 - versionName "2.3.1" + versionCode 20400 + versionName "2.4" testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" } buildTypes { diff --git a/server/build_without_gradle.sh b/server/build_without_gradle.sh index 69d85679..7f7d7921 100755 --- a/server/build_without_gradle.sh +++ b/server/build_without_gradle.sh @@ -12,7 +12,7 @@ set -e SCRCPY_DEBUG=false -SCRCPY_VERSION_NAME=2.3.1 +SCRCPY_VERSION_NAME=2.4 PLATFORM=${ANDROID_PLATFORM:-34} BUILD_TOOLS=${ANDROID_BUILD_TOOLS:-34.0.0} From 0c94b75eefee510d3de6bc724eaeedfc600faadc Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 3 Mar 2024 00:00:24 +0100 Subject: [PATCH 76/79] Update links to 2.4 --- README.md | 2 +- doc/build.md | 6 +++--- doc/windows.md | 12 ++++++------ install_release.sh | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 30fc0a04..a672b327 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ source for the project. Do not download releases from random websites, even if their name contains `scrcpy`.** -# scrcpy (v2.3.1) +# scrcpy (v2.4) scrcpy diff --git a/doc/build.md b/doc/build.md index 7e3c84e9..751cf831 100644 --- a/doc/build.md +++ b/doc/build.md @@ -233,10 +233,10 @@ install` must be run as root)._ #### Option 2: Use prebuilt server - - [`scrcpy-server-v2.3.1`][direct-scrcpy-server] - SHA-256: `f6814822fc308a7a532f253485c9038183c6296a6c5df470a9e383b4f8e7605b` + - [`scrcpy-server-v2.4`][direct-scrcpy-server] + SHA-256: `93c272b7438605c055e127f7444064ed78fa9ca49f81156777fd201e79ce7ba3` -[direct-scrcpy-server]: https://github.com/Genymobile/scrcpy/releases/download/v2.3.1/scrcpy-server-v2.3.1 +[direct-scrcpy-server]: https://github.com/Genymobile/scrcpy/releases/download/v2.4/scrcpy-server-v2.4 Download the prebuilt server somewhere, and specify its path during the Meson configuration: diff --git a/doc/windows.md b/doc/windows.md index 60fd7986..a3711f26 100644 --- a/doc/windows.md +++ b/doc/windows.md @@ -4,14 +4,14 @@ Download the [latest release]: - - [`scrcpy-win64-v2.3.1.zip`][direct-win64] (64-bit) - SHA-256: `f1f78ac98214078425804e524a1bed515b9d4b8a05b78d210a4ced2b910b262d` - - [`scrcpy-win32-v2.3.1.zip`][direct-win32] (32-bit) - SHA-256: `5dffc2d432e9b8b5b0e16f12e71428c37c70d9124cfbe7620df0b41b7efe91ff` + - [`scrcpy-win64-v2.4.zip`][direct-win64] (64-bit) + SHA-256: `9dc56f21bfa455352ec0c58b40feaf2fb02d67372910a4235e298ece286ff3a9` + - [`scrcpy-win32-v2.4.zip`][direct-win32] (32-bit) + SHA-256: `cf92acc45eef37c6ee2db819f92e420ced3bc50f1348dd57f7d6ca1fc80f6116` [latest release]: https://github.com/Genymobile/scrcpy/releases/latest -[direct-win64]: https://github.com/Genymobile/scrcpy/releases/download/v2.3.1/scrcpy-win64-v2.3.1.zip -[direct-win32]: https://github.com/Genymobile/scrcpy/releases/download/v2.3.1/scrcpy-win32-v2.3.1.zip +[direct-win64]: https://github.com/Genymobile/scrcpy/releases/download/v2.4/scrcpy-win64-v2.4.zip +[direct-win32]: https://github.com/Genymobile/scrcpy/releases/download/v2.4/scrcpy-win32-v2.4.zip and extract it. diff --git a/install_release.sh b/install_release.sh index d8dbd951..0be5675c 100755 --- a/install_release.sh +++ b/install_release.sh @@ -2,8 +2,8 @@ set -e BUILDDIR=build-auto -PREBUILT_SERVER_URL=https://github.com/Genymobile/scrcpy/releases/download/v2.3.1/scrcpy-server-v2.3.1 -PREBUILT_SERVER_SHA256=f6814822fc308a7a532f253485c9038183c6296a6c5df470a9e383b4f8e7605b +PREBUILT_SERVER_URL=https://github.com/Genymobile/scrcpy/releases/download/v2.4/scrcpy-server-v2.4 +PREBUILT_SERVER_SHA256=93c272b7438605c055e127f7444064ed78fa9ca49f81156777fd201e79ce7ba3 echo "[scrcpy] Downloading prebuilt server..." wget "$PREBUILT_SERVER_URL" -O scrcpy-server From cc7719079ab6301e6ab42b7cd078a1da5acfa5b2 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 3 Mar 2024 00:05:26 +0100 Subject: [PATCH 77/79] Italicize coordinates letters in documentation --- doc/control.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/control.md b/doc/control.md index d6d1265c..abc9d1bf 100644 --- a/doc/control.md +++ b/doc/control.md @@ -71,8 +71,8 @@ To simulate a tilt gesture: Shift+_click-and-move-up-or-down_. Technically, _scrcpy_ generates additional touch events from a "virtual finger" at a location inverted through the center of the screen. When pressing -Ctrl the x and y coordinates are inverted. Using Shift -only inverts x. +Ctrl the _x_ and _y_ coordinates are inverted. Using Shift +only inverts _x_. This only works for the default mouse mode (`--mouse=sdk`). From 7f23ff3f2ca64dae6eb8b0ca9a881230184ae756 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 3 Mar 2024 00:06:54 +0100 Subject: [PATCH 78/79] Add videos for pinch-to-zoom and tilt A video is worth a thousand words. --- doc/control.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/control.md b/doc/control.md index abc9d1bf..e9fd9e9b 100644 --- a/doc/control.md +++ b/doc/control.md @@ -67,8 +67,12 @@ More precisely, hold down Ctrl while pressing the left-click button. Until the left-click button is released, all mouse movements scale and rotate the content (if supported by the app) relative to the center of the screen. +https://github.com/Genymobile/scrcpy/assets/543275/26c4a920-9805-43f1-8d4c-608752d04767 + To simulate a tilt gesture: Shift+_click-and-move-up-or-down_. +https://github.com/Genymobile/scrcpy/assets/543275/1e252341-4a90-4b29-9d11-9153b324669f + Technically, _scrcpy_ generates additional touch events from a "virtual finger" at a location inverted through the center of the screen. When pressing Ctrl the _x_ and _y_ coordinates are inverted. Using Shift From 79968a0ae63179c39d5f1d4f4d97da020c9e07dd Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 11 Mar 2024 18:05:27 +0100 Subject: [PATCH 79/79] Reorder documentation Present the --tcpip option without arguments first. --- doc/connection.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/connection.md b/doc/connection.md index 90ced010..17efbbdc 100644 --- a/doc/connection.md +++ b/doc/connection.md @@ -67,14 +67,6 @@ computer. An option `--tcpip` allows to configure the connection automatically. There are two variants. -If the device (accessible at 192.168.1.1 in this example) already listens on a -port (typically 5555) for incoming _adb_ connections, then run: - -```bash -scrcpy --tcpip=192.168.1.1 # default port is 5555 -scrcpy --tcpip=192.168.1.1:5555 -``` - If _adb_ TCP/IP mode is disabled on the device (or if you don't know the IP address), connect the device over USB, then run: @@ -85,6 +77,14 @@ scrcpy --tcpip # without arguments It will automatically find the device IP address and adb port, enable TCP/IP mode if necessary, then connect to the device before starting. +If the device (accessible at 192.168.1.1 in this example) already listens on a +port (typically 5555) for incoming _adb_ connections, then run: + +```bash +scrcpy --tcpip=192.168.1.1 # default port is 5555 +scrcpy --tcpip=192.168.1.1:5555 +``` + ### Manual