Introduce --keyboard and --mouse

Until now, there was two modes for keyboard and mouse:
 - event injection using the Android system API (default)
 - HID/AOA over USB

For this reason, the options were exposed as simple flags:
 - -K or --hid-keyboard to enable physical keyboard simulation (AOA)
 - -M or --hid-mouse to enable physical mouse simulation (AOA)

Replace them by explicit --keyboard and --mouse options, with 3 possible
values:
 - disabled
 - sdk (default)
 - aoa

This will allow to add a new mode (uhid).

PR #4473 <https://github.com/Genymobile/scrcpy/pull/4473>

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>
uhid.38
Simon Chan 5 months ago committed by Romain Vimont
parent 35add3daee
commit ea98d49bae

@ -27,8 +27,8 @@ _scrcpy() {
--force-adb-forward
--forward-all-clicks
-h --help
--keyboard=
--kill-adb-on-close
-K --hid-keyboard
--legacy-paste
--list-camera-sizes
--list-cameras
@ -37,8 +37,8 @@ _scrcpy() {
--lock-video-orientation
--lock-video-orientation=
-m --max-size=
-M --hid-mouse
--max-fps=
--mouse=
-n --no-control
-N --no-playback
--no-audio
@ -115,6 +115,14 @@ _scrcpy() {
COMPREPLY=($(compgen -W 'front back external' -- "$cur"))
return
;;
--keyboard)
COMPREPLY=($(compgen -W 'disabled sdk aoa' -- "$cur"))
return
;;
--mouse)
COMPREPLY=($(compgen -W 'disabled sdk aoa' -- "$cur"))
return
;;
--orientation|--display-orientation)
COMPREPLY=($(compgen -W '0 90 180 270 flip0 flip90 flip180 flip270' -- "$cur"))
return

@ -34,8 +34,8 @@ arguments=(
'--force-adb-forward[Do not attempt to use \"adb reverse\" to connect to the device]'
'--forward-all-clicks[Forward clicks to device]'
{-h,--help}'[Print the help]'
'--keyboard[Set the keyboard input mode]:mode:(disabled sdk aoa)'
'--kill-adb-on-close[Kill adb when scrcpy terminates]'
{-K,--hid-keyboard}'[Simulate a physical keyboard by using HID over AOAv2]'
'--legacy-paste[Inject computer clipboard text as a sequence of key events on Ctrl+v]'
'--list-camera-sizes[List the valid camera capture sizes]'
'--list-cameras[List cameras available on the device]'
@ -43,8 +43,8 @@ arguments=(
'--list-encoders[List video and audio encoders available on the device]'
'--lock-video-orientation=[Lock video orientation]:orientation:(unlocked initial 0 90 180 270)'
{-m,--max-size=}'[Limit both the width and height of the video to value]'
{-M,--hid-mouse}'[Simulate a physical mouse by using HID over AOAv2]'
'--max-fps=[Limit the frame rate of screen capture]'
'--mouse[Set the mouse input mode]:mode:(disabled sdk aoa)'
{-n,--no-control}'[Disable device control \(mirror the device in read only\)]'
{-N,--no-playback}'[Disable video and audio playback]'
'--no-audio[Disable audio forwarding]'

@ -172,24 +172,26 @@ By default, right-click triggers BACK (or POWER on) and middle-click triggers HO
Print this help.
.TP
.B \-\-kill\-adb\-on\-close
Kill adb when scrcpy terminates.
.BI "\-\-keyboard " mode
Select how to send keyboard inputs to the device.
.TP
.B \-K, \-\-hid\-keyboard
Simulate a physical keyboard by using HID over AOAv2.
Possible values are "disabled", "sdk" and "aoa":
This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method.
It may only work over USB.
- "disabled" does not send keyboard inputs to the device.
- "sdk" uses the Android system API to deliver keyboard events to applications.
- "aoa" simulates a physical keyboard using the AOAv2 protocol. It may only work over USB.
The keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly:
For "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly:
adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS
However, the option is only available when the HID keyboard is enabled (or a physical keyboard is connected).
This option is only available when the HID keyboard is enabled (or a physical keyboard is connected).
Also see \fB\-\-hid\-mouse\fR.
Also see \fB\-\-mouse\fR.
.TP
.B \-\-kill\-adb\-on\-close
Kill adb when scrcpy terminates.
.TP
.B \-\-legacy\-paste
@ -230,20 +232,25 @@ Limit both the width and height of the video to \fIvalue\fR. The other dimension
Default is 0 (unlimited).
.TP
.B \-M, \-\-hid\-mouse
Simulate a physical mouse by using HID over AOAv2.
.BI "\-\-max\-fps " value
Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions).
.TP
.BI "\-\-mouse " mode
Select how to send mouse inputs to the device.
In this mode, the computer mouse is captured to control the device directly (relative mouse mode).
Possible values are "disabled", "sdk" and "aoa":
LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer.
- "disabled" does not send mouse inputs to the device.
- "sdk" uses the Android system API to deliver mouse events to applications.
- "aoa" simulates a physical mouse using the AOAv2 protocol. It may only work over USB.
It may only work over USB.
In "aoa" mode, the computer mouse is captured to control the device directly (relative mouse mode).
Also see \fB\-\-hid\-keyboard\fR.
LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer.
Also see \fB\-\-keyboard\fR.
.TP
.BI "\-\-max\-fps " value
Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions).
.TP
.B \-n, \-\-no\-control

@ -93,6 +93,8 @@ enum {
OPT_DISPLAY_ORIENTATION,
OPT_RECORD_ORIENTATION,
OPT_ORIENTATION,
OPT_KEYBOARD,
OPT_MOUSE,
};
struct sc_option {
@ -358,27 +360,35 @@ static const struct sc_option options[] = {
.longopt = "help",
.text = "Print this help.",
},
{
.longopt_id = OPT_KEYBOARD,
.longopt = "keyboard",
.argdesc = "mode",
.text = "Select how to send keyboard inputs to the device.\n"
"Possible values are \"disabled\", \"sdk\" and \"aoa\".\n"
"\"disabled\" does not send keyboard inputs to the device.\n"
"\"sdk\" uses the Android system API to deliver keyboard "
"events to applications.\n"
"\"aoa\" simulates a physical keyboard using the AOAv2 "
"protocol. It may only work over USB.\n"
"For \"aoa\", the keyboard layout must be configured (once and "
"for all) on the device, via Settings -> System -> Languages "
"and input -> Physical keyboard. This settings page can be "
"started directly: `adb shell am start -a "
"android.settings.HARD_KEYBOARD_SETTINGS`.\n"
"This option is only available when the HID keyboard is "
"enabled (or a physical keyboard is connected).\n"
"Also see --mouse.",
},
{
.longopt_id = OPT_KILL_ADB_ON_CLOSE,
.longopt = "kill-adb-on-close",
.text = "Kill adb when scrcpy terminates.",
},
{
// deprecated
.shortopt = 'K',
.longopt = "hid-keyboard",
.text = "Simulate a physical keyboard by using HID over AOAv2.\n"
"It provides a better experience for IME users, and allows to "
"generate non-ASCII characters, contrary to the default "
"injection method.\n"
"It may only work over USB.\n"
"The keyboard layout must be configured (once and for all) on "
"the device, via Settings -> System -> Languages and input -> "
"Physical keyboard. This settings page can be started "
"directly: `adb shell am start -a "
"android.settings.HARD_KEYBOARD_SETTINGS`.\n"
"However, the option is only available when the HID keyboard "
"is enabled (or a physical keyboard is connected).\n"
"Also see --hid-mouse.",
},
{
.longopt_id = OPT_LEGACY_PASTE,
@ -432,15 +442,9 @@ static const struct sc_option options[] = {
"Default is 0 (unlimited).",
},
{
// deprecated
.shortopt = 'M',
.longopt = "hid-mouse",
.text = "Simulate a physical mouse by using HID over AOAv2.\n"
"In this mode, the computer mouse is captured to control the "
"device directly (relative mouse mode).\n"
"LAlt, LSuper or RSuper toggle the capture mode, to give "
"control of the mouse back to the computer.\n"
"It may only work over USB.\n"
"Also see --hid-keyboard.",
},
{
.longopt_id = OPT_MAX_FPS,
@ -449,6 +453,23 @@ static const struct sc_option options[] = {
.text = "Limit the frame rate of screen capture (officially supported "
"since Android 10, but may work on earlier versions).",
},
{
.longopt_id = OPT_MOUSE,
.longopt = "mouse",
.argdesc = "mode",
.text = "Select how to send mouse inputs to the device.\n"
"Possible values are \"disabled\", \"sdk\" and \"aoa\".\n"
"\"disabled\" does not send mouse inputs to the device.\n"
"\"sdk\" uses the Android system API to deliver mouse events"
"to applications.\n"
"\"aoa\" simulates a physical mouse using the AOAv2 protocol. "
"It may only work over USB.\n"
"In \"aoa\" mode, the computer mouse is captured to control "
"the device directly (relative mouse mode).\n"
"LAlt, LSuper or RSuper toggle the capture mode, to give "
"control of the mouse back to the computer.\n"
"Also see --keyboard.",
},
{
.shortopt = 'n',
.longopt = "no-control",
@ -543,10 +564,10 @@ static const struct sc_option options[] = {
"mirroring is disabled.\n"
"LAlt, LSuper or RSuper toggle the mouse capture mode, to give "
"control of the mouse back to the computer.\n"
"If any of --hid-keyboard or --hid-mouse is set, only enable "
"keyboard or mouse respectively, otherwise enable both.\n"
"Keyboard and mouse may be disabled separately using"
"--keyboard=disabled and --mouse=disabled.\n"
"It may only work over USB.\n"
"See --hid-keyboard and --hid-mouse.",
"See --keyboard and --mouse.",
},
{
.shortopt = 'p',
@ -1906,6 +1927,58 @@ parse_camera_fps(const char *s, uint16_t *camera_fps) {
return true;
}
static bool
parse_keyboard(const char *optarg, enum sc_keyboard_input_mode *mode) {
if (!strcmp(optarg, "disabled")) {
*mode = SC_KEYBOARD_INPUT_MODE_DISABLED;
return true;
}
if (!strcmp(optarg, "sdk")) {
*mode = SC_KEYBOARD_INPUT_MODE_SDK;
return true;
}
if (!strcmp(optarg, "aoa")) {
#ifdef HAVE_USB
*mode = SC_KEYBOARD_INPUT_MODE_AOA;
return true;
#else
LOGE("--keyboard=aoa is disabled.");
return false;
#endif
}
LOGE("Unsupported keyboard: %s (expected disabled, sdk or aoa)", optarg);
return false;
}
static bool
parse_mouse(const char *optarg, enum sc_mouse_input_mode *mode) {
if (!strcmp(optarg, "disabled")) {
*mode = SC_MOUSE_INPUT_MODE_DISABLED;
return true;
}
if (!strcmp(optarg, "sdk")) {
*mode = SC_MOUSE_INPUT_MODE_SDK;
return true;
}
if (!strcmp(optarg, "aoa")) {
#ifdef HAVE_USB
*mode = SC_MOUSE_INPUT_MODE_AOA;
return true;
#else
LOGE("--mouse=aoa is disabled.");
return false;
#endif
}
LOGE("Unsupported mouse: %s (expected disabled, sdk or aoa)", optarg);
return false;
}
static bool
parse_time_limit(const char *s, sc_tick *tick) {
long value;
@ -1995,12 +2068,19 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
break;
case 'K':
#ifdef HAVE_USB
opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;
LOGW("-K/--hid-keyboard is deprecated, use --keyboard=aoa "
"instead.");
opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_AOA;
break;
#else
LOGE("HID over AOA (-K/--hid-keyboard) is disabled.");
return false;
#endif
case OPT_KEYBOARD:
if (!parse_keyboard(optarg, &opts->keyboard_input_mode)) {
return false;
}
break;
case OPT_MAX_FPS:
if (!parse_max_fps(optarg, &opts->max_fps)) {
return false;
@ -2013,12 +2093,18 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
break;
case 'M':
#ifdef HAVE_USB
opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_HID;
LOGW("-M/--hid-mouse is deprecated, use --mouse=aoa instead.");
opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_AOA;
break;
#else
LOGE("HID over AOA (-M/--hid-mouse) is disabled.");
return false;
#endif
case OPT_MOUSE:
if (!parse_mouse(optarg, &opts->mouse_input_mode)) {
return false;
}
break;
case OPT_LOCK_VIDEO_ORIENTATION:
if (!parse_lock_video_orientation(optarg,
&opts->lock_video_orientation)) {
@ -2465,6 +2551,37 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
}
#endif
if (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AUTO) {
opts->keyboard_input_mode = otg ? SC_KEYBOARD_INPUT_MODE_AOA
: SC_KEYBOARD_INPUT_MODE_SDK;
}
if (opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_AUTO) {
opts->mouse_input_mode = otg ? SC_MOUSE_INPUT_MODE_AOA
: SC_MOUSE_INPUT_MODE_SDK;
}
if (otg) {
enum sc_keyboard_input_mode kmode = opts->keyboard_input_mode;
if (kmode != SC_KEYBOARD_INPUT_MODE_AOA
&& kmode != SC_KEYBOARD_INPUT_MODE_DISABLED) {
LOGE("In OTG mode, --keyboard only supports aoa or disabled.");
return false;
}
enum sc_mouse_input_mode mmode = opts->mouse_input_mode;
if (mmode != SC_MOUSE_INPUT_MODE_AOA
&& mmode != SC_MOUSE_INPUT_MODE_DISABLED) {
LOGE("In OTG mode, --mouse only supports aoa or disabled.");
return false;
}
if (kmode == SC_KEYBOARD_INPUT_MODE_DISABLED
&& mmode == SC_MOUSE_INPUT_MODE_DISABLED) {
LOGE("Could not disable both keyboard and mouse in OTG mode.");
return false;
}
}
if ((opts->tunnel_host || opts->tunnel_port) && !opts->force_adb_forward) {
LOGI("Tunnel host/port is set, "
"--force-adb-forward automatically enabled.");
@ -2625,12 +2742,12 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
}
# ifdef _WIN32
if (!otg && (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID
|| opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID)) {
if (!otg && (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA
|| opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA)) {
LOGE("On Windows, it is not possible to open a USB device already open "
"by another process (like adb).");
LOGE("Therefore, -K/--hid-keyboard and -M/--hid-mouse may only work in "
"OTG mode (--otg).");
LOGE("Therefore, --keyboard=aoa and --mouse=aoa may only work in OTG"
"mode (--otg).");
return false;
}
# endif

@ -21,8 +21,8 @@ const struct scrcpy_options scrcpy_options_default = {
.video_source = SC_VIDEO_SOURCE_DISPLAY,
.audio_source = SC_AUDIO_SOURCE_AUTO,
.record_format = SC_RECORD_FORMAT_AUTO,
.keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT,
.mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT,
.keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_AUTO,
.mouse_input_mode = SC_MOUSE_INPUT_MODE_AUTO,
.camera_facing = SC_CAMERA_FACING_ANY,
.port_range = {
.first = DEFAULT_LOCAL_PORT_RANGE_FIRST,

@ -140,13 +140,17 @@ enum sc_lock_video_orientation {
};
enum sc_keyboard_input_mode {
SC_KEYBOARD_INPUT_MODE_INJECT,
SC_KEYBOARD_INPUT_MODE_HID,
SC_KEYBOARD_INPUT_MODE_AUTO,
SC_KEYBOARD_INPUT_MODE_DISABLED,
SC_KEYBOARD_INPUT_MODE_SDK,
SC_KEYBOARD_INPUT_MODE_AOA,
};
enum sc_mouse_input_mode {
SC_MOUSE_INPUT_MODE_INJECT,
SC_MOUSE_INPUT_MODE_HID,
SC_MOUSE_INPUT_MODE_AUTO,
SC_MOUSE_INPUT_MODE_DISABLED,
SC_MOUSE_INPUT_MODE_SDK,
SC_MOUSE_INPUT_MODE_AOA,
};
enum sc_key_inject_mode {

@ -543,11 +543,11 @@ scrcpy(struct scrcpy_options *options) {
if (options->control) {
#ifdef HAVE_USB
bool use_hid_keyboard =
options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID;
bool use_hid_mouse =
options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID;
if (use_hid_keyboard || use_hid_mouse) {
bool use_aoa_keyboard =
options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA;
bool use_aoa_mouse =
options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA;
if (use_aoa_keyboard || use_aoa_mouse) {
bool ok = sc_acksync_init(&s->acksync);
if (!ok) {
goto end;
@ -590,7 +590,7 @@ scrcpy(struct scrcpy_options *options) {
goto aoa_hid_end;
}
if (use_hid_keyboard) {
if (use_aoa_keyboard) {
if (sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {
hid_keyboard_initialized = true;
kp = &s->keyboard_hid.key_processor;
@ -599,7 +599,7 @@ scrcpy(struct scrcpy_options *options) {
}
}
if (use_hid_mouse) {
if (use_aoa_mouse) {
if (sc_hid_mouse_init(&s->mouse_hid, &s->aoa)) {
hid_mouse_initialized = true;
mp = &s->mouse_hid.mouse_processor;
@ -634,25 +634,23 @@ aoa_hid_end:
}
}
if (use_hid_keyboard && !hid_keyboard_initialized) {
LOGE("Fallback to default keyboard injection method "
"(-K/--hid-keyboard ignored)");
options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
if (use_aoa_keyboard && !hid_keyboard_initialized) {
LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)");
options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_SDK;
}
if (use_hid_mouse && !hid_mouse_initialized) {
LOGE("Fallback to default mouse injection method "
"(-M/--hid-mouse ignored)");
options->mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT;
if (use_aoa_mouse && !hid_mouse_initialized) {
LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)");
options->mouse_input_mode = SC_MOUSE_INPUT_MODE_SDK;
}
}
#else
assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID);
assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID);
assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_AOA);
assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_AOA);
#endif
// keyboard_input_mode may have been reset if HID mode failed
if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {
if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_SDK) {
sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,
options->key_inject_mode,
options->forward_key_repeat);
@ -660,7 +658,7 @@ aoa_hid_end:
}
// mouse_input_mode may have been reset if HID mode failed
if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_INJECT) {
if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_SDK) {
sc_mouse_inject_init(&s->mouse_inject, &s->controller);
mp = &s->mouse_inject.mouse_processor;
}

@ -117,16 +117,15 @@ scrcpy_otg(struct scrcpy_options *options) {
}
aoa_initialized = true;
assert(options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA
|| options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_DISABLED);
assert(options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA
|| options->mouse_input_mode == SC_MOUSE_INPUT_MODE_DISABLED);
bool enable_keyboard =
options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID;
options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA;
bool enable_mouse =
options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID;
// If neither --hid-keyboard or --hid-mouse is passed, enable both
if (!enable_keyboard && !enable_mouse) {
enable_keyboard = true;
enable_mouse = true;
}
options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA;
if (enable_keyboard) {
ok = sc_hid_keyboard_init(&s->keyboard, &s->aoa);

Loading…
Cancel
Save