From 332acab04ccff20f7df0e6bfd4cfdfda96bad4cf Mon Sep 17 00:00:00 2001 From: Rob Wu Date: Sun, 31 Dec 2023 19:36:36 +0100 Subject: [PATCH 1/2] Add --hid-capture & --hid-replay options --- app/meson.build | 2 + app/scrcpy.1 | 17 + app/src/cli.c | 51 +++ app/src/options.c | 2 + app/src/options.h | 2 + app/src/scrcpy.c | 65 ++++ app/src/usb/aoa_hid.c | 16 +- app/src/usb/aoa_hid.h | 5 + app/src/usb/hid_event_serializer.c | 323 +++++++++++++++++++ app/src/usb/hid_event_serializer.h | 95 ++++++ app/src/usb/hid_replay.c | 481 +++++++++++++++++++++++++++++ app/src/usb/hid_replay.h | 63 ++++ app/src/usb/scrcpy_otg.c | 46 +++ app/src/util/tick.h | 1 + doc/hid-otg.md | 32 ++ doc/recording.md | 6 + 16 files changed, 1206 insertions(+), 1 deletion(-) create mode 100644 app/src/usb/hid_event_serializer.c create mode 100644 app/src/usb/hid_event_serializer.h create mode 100644 app/src/usb/hid_replay.c create mode 100644 app/src/usb/hid_replay.h diff --git a/app/meson.build b/app/meson.build index 88e2df9a..04b9dfde 100644 --- a/app/meson.build +++ b/app/meson.build @@ -88,8 +88,10 @@ usb_support = get_option('usb') if usb_support src += [ 'src/usb/aoa_hid.c', + 'src/usb/hid_event_serializer.c', 'src/usb/hid_keyboard.c', 'src/usb/hid_mouse.c', + 'src/usb/hid_replay.c', 'src/usb/scrcpy_otg.c', 'src/usb/screen_otg.c', 'src/usb/usb.c', diff --git a/app/scrcpy.1 b/app/scrcpy.1 index beaa99ab..3ebef87e 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -171,6 +171,23 @@ By default, right-click triggers BACK (or POWER on) and middle-click triggers HO .B \-h, \-\-help Print this help. +.TP +.BI "\-\-hid-record " output.log +Record HID input to a file. + +Only HID input is recorded, which requires the \fB\-\-hid\-keyboard\fR, \fB\-\-hid\-mouse\fR and/or \fB\-\-otg\fR options. +When \fB\-\-hid\-replay\fR is used simulatenously, any replayed input is also recorded to the target specified by \fB\-\-hid\-replay\fR. + +To mirror input to multiple devices, specify a named pipe as the filename (see mkfifo for UNIX) and pass the same filename to \fB\-\-hid\-replay\fR of a second (parallel) scrcpy instance. + +.TP +.BI "\-\-hid-replay " input.log +Replay HID input from a file created by \fB\-\-hid-record\fR. + +Events are only replayed when HID is used to simulate input, which requires the \fB\-\-hid\-keyboard\fR, \fB\-\-hid\-mouse\fR and/or \fB\-\-otg\fR options. + +See \fB\-\-hid\-record\fR for more information. + .TP .B \-\-kill\-adb\-on\-close Kill adb when scrcpy terminates. diff --git a/app/src/cli.c b/app/src/cli.c index c580c959..881d4372 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -58,6 +58,8 @@ enum { OPT_RAW_KEY_EVENTS, OPT_NO_DOWNSIZE_ON_ERROR, OPT_OTG, + OPT_HID_RECORD, + OPT_HID_REPLAY, OPT_NO_CLEANUP, OPT_PRINT_FPS, OPT_NO_POWER_ON, @@ -358,6 +360,29 @@ static const struct sc_option options[] = { .longopt = "help", .text = "Print this help.", }, + { + .longopt_id = OPT_HID_RECORD, + .longopt = "hid-record", + .argdesc = "output.log", + .text = "Record HID input to a file.\n" + "Only HID input is recorded, which requires the --hid-keyboard," + "--hid-mouse and/or --otg options.\n" + "When --hid-replay is used simulatenously, any replayed input " + "is also recorded to the target specified by --hid-replay.\n" + "To mirror input to multiple devices, specify a named pipe as " + "the filename (see mkfifo for UNIX) and pass the same filename " + "to --hid-replay of a second (parallel) scrcpy instance.", + }, + { + .longopt_id = OPT_HID_REPLAY, + .longopt = "hid-replay", + .argdesc = "input.log", + .text = "Replay HID input from a file created by --hid-record.\n" + "Events are only replayed when HID is used to simulate input," + "which requires the --hid-keyboard, --hid-mouse and/or --otg " + "options.\n" + "See --hid-record for more information.", + }, { .longopt_id = OPT_KILL_ADB_ON_CLOSE, .longopt = "kill-adb-on-close", @@ -2267,6 +2292,22 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], #else LOGE("OTG mode (--otg) is disabled."); return false; +#endif + case OPT_HID_RECORD: +#ifdef HAVE_USB + opts->hid_record_filename = optarg; + break; +#else + LOGE("HID recording (--hid-record) is disabled."); + return false; +#endif + case OPT_HID_REPLAY: +#ifdef HAVE_USB + opts->hid_replay_filename = optarg; + break; +#else + LOGE("HID replay (--hid-replay) is disabled."); + return false; #endif case OPT_V4L2_SINK: #ifdef HAVE_V4L2 @@ -2628,6 +2669,16 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } # endif +#ifdef HAVE_USB + if (opts->hid_replay_filename || opts->hid_record_filename) { + if (!otg && opts->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID + && opts->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID) { + LOGE("--hid-record and --hid-replay only works if --hid-keyboard, " + "--hid-mouse and/or --otg are set."); + return false; + } + } +#endif if (otg) { // OTG mode is compatible with only very few options. diff --git a/app/src/options.c b/app/src/options.c index a13df585..8c4191b1 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -56,6 +56,8 @@ const struct scrcpy_options scrcpy_options_default = { #endif #ifdef HAVE_USB .otg = false, + .hid_record_filename = NULL, + .hid_replay_filename = NULL, #endif .show_touches = false, .fullscreen = false, diff --git a/app/src/options.h b/app/src/options.h index 11e64fa1..0e2decab 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -236,6 +236,8 @@ struct scrcpy_options { #endif #ifdef HAVE_USB bool otg; + const char *hid_record_filename; + const char *hid_replay_filename; #endif bool show_touches; bool fullscreen; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index cf2e7e47..32ae5d4c 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -29,6 +29,7 @@ # include "usb/aoa_hid.h" # include "usb/hid_keyboard.h" # include "usb/hid_mouse.h" +# include "usb/hid_replay.h" # include "usb/usb.h" #endif #include "util/acksync.h" @@ -59,6 +60,7 @@ struct scrcpy { #ifdef HAVE_USB struct sc_usb usb; struct sc_aoa aoa; + struct sc_hidr hidr; // sequence/ack helper to synchronize clipboard and Ctrl+v via HID struct sc_acksync acksync; #endif @@ -332,6 +334,11 @@ scrcpy(struct scrcpy_options *options) { bool aoa_hid_initialized = false; bool hid_keyboard_initialized = false; bool hid_mouse_initialized = false; + bool hidr_initialized = false; + bool hidr_replay_started = false; + bool hidr_record_started = false; + bool need_hidr = options->hid_record_filename || + options->hid_replay_filename; #endif bool controller_initialized = false; bool controller_started = false; @@ -609,6 +616,28 @@ scrcpy(struct scrcpy_options *options) { } bool need_aoa = hid_keyboard_initialized || hid_mouse_initialized; + if (need_hidr) { + if (!need_aoa) { + goto end; + } + ok = sc_hidr_init(&s->hidr, &s->aoa, + options->hid_record_filename, + options->hid_replay_filename, hid_keyboard_initialized, + hid_mouse_initialized); + if (!ok) { + goto end; + } + hidr_initialized = true; + } + if (options->hid_record_filename) { + // Set up hidr record BEFORE starting aoa, to ensure that hidr + // can listen to events from aoa before the aoa thread starts. + ok = sc_hidr_start_record(&s->hidr); + if (!ok) { + goto end; + } + hidr_record_started = true; + } if (!need_aoa || !sc_aoa_start(&s->aoa)) { sc_acksync_destroy(&s->acksync); @@ -632,6 +661,9 @@ aoa_hid_end: sc_hid_mouse_destroy(&s->mouse_hid); hid_mouse_initialized = false; } + if (need_hidr) { + goto end; + } } if (use_hid_keyboard && !hid_keyboard_initialized) { @@ -681,6 +713,12 @@ aoa_hid_end: // There is a controller if and only if control is enabled assert(options->control == !!controller); +#ifdef HAVE_USB + if (need_hidr && !aoa_hid_initialized) { + goto end; + } +#endif + if (options->video_playback) { const char *window_title = options->window_title ? options->window_title : info->device_name; @@ -799,6 +837,17 @@ aoa_hid_end: timeout_started = true; } +#ifdef HAVE_USB + if (hidr_initialized && options->hid_replay_filename) { + assert(aoa_hid_initialized); + bool ok = sc_hidr_start_replay(&s->hidr); + if (!ok) { + goto end; + } + hidr_replay_started = true; + } +#endif + ret = event_loop(s); LOGD("quit..."); @@ -814,6 +863,13 @@ end: // The demuxer is not stopped explicitly, because it will stop by itself on // end-of-stream #ifdef HAVE_USB + if (need_hidr && !aoa_hid_initialized) { + LOGE("HID input unavailable, cannot use --hid-record / --hid-replay!"); + } + if (hidr_replay_started) { + sc_hidr_stop_replay(&s->hidr); + // hidr will not call aoa at this point. + } if (aoa_hid_initialized) { if (hid_keyboard_initialized) { sc_hid_keyboard_destroy(&s->keyboard_hid); @@ -877,6 +933,15 @@ end: sc_usb_disconnect(&s->usb); sc_usb_destroy(&s->usb); } + if (hidr_record_started) { + // Because aoa has been destroyed, aoa won't call hidr at this point. + sc_hidr_stop_record(&s->hidr); + } + if (hidr_initialized) { + // aoa has been destroyed, there are no circular references between + // hidr and aoa, so we can finally destroy hidr. + sc_hidr_destroy(&s->hidr); + } #endif // Destroy the screen only after the video demuxer is guaranteed to be diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index fb64e57c..01150d0b 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -4,7 +4,7 @@ #include #include "aoa_hid.h" -#include "util/log.h" +#include "hid_replay.h" // See . #define ACCESSORY_REGISTER_HID 54 @@ -40,6 +40,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, hid_event->buffer = buffer; hid_event->size = buffer_size; hid_event->ack_to_wait = SC_SEQUENCE_INVALID; + hid_event->timestamp = 0; } void @@ -70,6 +71,9 @@ sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb, aoa->stopped = false; aoa->acksync = acksync; aoa->usb = usb; + // If needed, will be initialized by sc_hidr_start_record, + // before a new thread is created by sc_aoa_start. + aoa->hidr_to_notify = NULL; return true; } @@ -219,6 +223,9 @@ sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) { sc_hid_event_log(event); } + if (aoa->hidr_to_notify) { + sc_hidr_observe_event_for_record(aoa->hidr_to_notify, event); + } sc_mutex_lock(&aoa->mutex); bool full = sc_vecdeque_is_full(&aoa->queue); @@ -232,6 +239,13 @@ sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { // Otherwise (if the queue is full), the event is discarded sc_mutex_unlock(&aoa->mutex); + if (full) { + // In the not-full case, a copy is made of the struct, which includes the + // heap-allocated event->buffer member. The receiver of that data will + // take care of cleaning it up. + // We need to clean up similarly when the data is dropped. + sc_hid_event_destroy((struct sc_hid_event *)event); + } return !full; } diff --git a/app/src/usb/aoa_hid.h b/app/src/usb/aoa_hid.h index 8803c1d9..27f91c4c 100644 --- a/app/src/usb/aoa_hid.h +++ b/app/src/usb/aoa_hid.h @@ -17,6 +17,7 @@ struct sc_hid_event { unsigned char *buffer; uint16_t size; uint64_t ack_to_wait; + sc_tick timestamp; // Only used by hid_replay.c & hid_event_serializer.c }; // Takes ownership of buffer @@ -29,6 +30,9 @@ sc_hid_event_destroy(struct sc_hid_event *hid_event); struct sc_hid_event_queue SC_VECDEQUE(struct sc_hid_event); +// Forward declare sc_hidr to avoid circular dependency on hid_replay.h. +struct sc_hidr; + struct sc_aoa { struct sc_usb *usb; sc_thread thread; @@ -38,6 +42,7 @@ struct sc_aoa { struct sc_hid_event_queue queue; struct sc_acksync *acksync; + struct sc_hidr *hidr_to_notify; }; bool diff --git a/app/src/usb/hid_event_serializer.c b/app/src/usb/hid_event_serializer.c new file mode 100644 index 00000000..83f514d0 --- /dev/null +++ b/app/src/usb/hid_event_serializer.c @@ -0,0 +1,323 @@ +#include "util/log.h" + +#include +#include +#include +#include + +#include "hid_event_serializer.h" +#include "util/tick.h" + +#define LOG_REPLAY_PARSE_ERROR(fmt, ...) \ + LOGE("Failed to parse HID replay at line %d in %s: " fmt, \ + hep->line, hep->source_name, ## __VA_ARGS__) + +static void +sc_hid_event_parser_parse_all_data_internal(struct sc_hid_event_parser *hep) { + assert(hep->parser_status == SC_HID_EVENT_PARSER_STATE_GOOD); + assert(hep->data[hep->data_len] == '\x00'); // Required for strchr/sscanf. + + char *data_iter = hep->data + hep->data_offset; + + assert(data_iter <= hep->data + hep->data_len); + for (;;) { + // Parse: [timestamp] [accessory_id] [buffer in hex, space-separated]\n + char * const data_end_of_line = strchr(data_iter, '\n'); + if (!data_end_of_line) { + // Cannot find end of line. Pause parser until we have more. + break; + } + + // Ignore empty lines and lines starting with #, to make manual editing + // easier. + if (data_iter == data_end_of_line || *data_iter == '#') { + if (!strchr(data_iter, '\n')) { + // Cannot find end of line. Pause parser until we have more. + break; + } + data_iter = data_end_of_line + 1; + ++hep->line; + continue; + } + + // Part 1 : timestamp + sc_tick timestamp; + if (sscanf(data_iter, "%" SCNtick " ", ×tamp) != 1) { + hep->parser_status = SC_HID_EVENT_PARSER_STATE_CORRUPT; + LOG_REPLAY_PARSE_ERROR("Line must start with a numeric timestamp"); + break; + } + data_iter = strchr(data_iter, ' '); + assert(data_iter); // We have already verified that there is a space. + ++data_iter; // Eat space. + if (*data_iter == '\x00') { + hep->parser_status = SC_HID_EVENT_PARSER_STATE_CORRUPT; + LOG_REPLAY_PARSE_ERROR("Unexpected end of data"); + break; + } + + // Part 2 : accessory_id + uint16_t accessory_id; + if (sscanf(data_iter, "%" SCNu16 " ", &accessory_id) != 1) { + hep->parser_status = SC_HID_EVENT_PARSER_STATE_CORRUPT; + LOG_REPLAY_PARSE_ERROR("accessory_id must be a number"); + break; + } + data_iter = strchr(data_iter, ' '); + assert(data_iter); // We have already verified that there is a space. + + // Part 3: buffer (repetition of space + 2 hex chars) + assert(data_end_of_line > data_iter); // Only digits/space, no \n yet. + size_t event_buffer_size = (data_end_of_line - data_iter) / 3; + if (!event_buffer_size) { + // Missing one or two characters after the space. + hep->parser_status = SC_HID_EVENT_PARSER_STATE_CORRUPT; + LOG_REPLAY_PARSE_ERROR("Unexpected %ld characters at end of line", + data_end_of_line - data_iter); + break; + } + unsigned char *event_buffer = malloc(event_buffer_size); + if (!event_buffer) { + hep->parser_status = SC_HID_EVENT_PARSER_STATE_CORRUPT; + LOG_OOM(); + break; + } + for (size_t i = 0; i < event_buffer_size; ++i) { + if (sscanf(data_iter, " %" SCNx8, &event_buffer[i]) != 1) { + break; + } + data_iter += 3; // Skip space and 2 hexdigits. + } + if (data_iter != data_end_of_line) { + hep->parser_status = SC_HID_EVENT_PARSER_STATE_CORRUPT; + LOG_REPLAY_PARSE_ERROR("Unexpected %ld characters at end of line", + data_end_of_line - data_iter); + free(event_buffer); + break; + } + assert(*data_iter == '\n'); // because data_iter == data_end_of_line + ++data_iter; // Skip '\n'. + ++hep->line; + + struct sc_hid_event *event = malloc(sizeof(struct sc_hid_event)); + if (!event) { + hep->parser_status = SC_HID_EVENT_PARSER_STATE_CORRUPT; + LOG_OOM(); + free(event_buffer); + break; + } + sc_hid_event_init(event, accessory_id, event_buffer, + event_buffer_size); + event->timestamp = timestamp; + + sc_vecdeque_push(&hep->parsed_events, event); + } + hep->data_offset = data_iter - hep->data; + assert(hep->data_offset <= hep->data_len); + assert(hep->data_offset < hep->data_buffer_size); +} + +void +sc_hid_event_parser_init(struct sc_hid_event_parser *hep, + const char *source_name) { + sc_vecdeque_init(&hep->parsed_events); + hep->source_name = source_name; + hep->line = 1; + hep->data = NULL; + hep->data_offset = 0; + hep->data_len = 0; + hep->data_buffer_size = 0; + hep->parser_status = SC_HID_EVENT_PARSER_STATE_GOOD; +} + +void +sc_hid_event_parser_destroy(struct sc_hid_event_parser *hep) { + if (hep->data) { + free(hep->data); + } + while (!sc_vecdeque_is_empty(&hep->parsed_events)) { + struct sc_hid_event *event = sc_vecdeque_pop(&hep->parsed_events); + assert(event); + sc_hid_event_destroy(event); + free(event); + } + sc_vecdeque_destroy(&hep->parsed_events); +} + +bool +sc_hid_event_parser_append_data(struct sc_hid_event_parser *hep, + const char *data, size_t data_len) { + if (hep->parser_status != SC_HID_EVENT_PARSER_STATE_GOOD) { + // We are in the fatal SC_HID_EVENT_PARSER_STATE_CORRUPT state. + return false; + } + if (!data_len) { + return true; + } + assert(data); + // Invariants: + assert(hep->data_len >= hep->data_offset); + // > not >= because data_buffer_size counts trailing NUL, data_len does not: + assert(!hep->data || hep->data_buffer_size > hep->data_len); + assert(!hep->data || hep->data[hep->data_len] == '\x00'); + + size_t space_head = hep->data_offset; + size_t space_middle = hep->data_len - hep->data_offset; + // -1 to reserve space for NUL byte. Initially 0 because no data. + size_t one_if_nul_tail = hep->data ? 1 : 0; + size_t space_tail = hep->data_buffer_size - hep->data_len - one_if_nul_tail; + size_t space_available = space_head + space_tail; + // +1 to reserve space for NUL byte. + size_t space_needed = space_middle + data_len + 1; + if (space_needed < data_len) { + // Integer overflowed. How did so much data fit in RAM...? + hep->input_data_was_corrupt = true; + LOG_OOM(); + return false; + } + if (data_len <= space_tail) { + // Enough space at the end, simply insert at the end. + assert(hep->data); + memcpy(hep->data + hep->data_len, data, data_len); + hep->data_len += data_len; + } else if (space_needed <= space_available) { + // There is space left at the start, move data and re-use buffer. + assert(hep->data); + memmove(hep->data, hep->data + hep->data_offset, space_middle); + memcpy(hep->data + space_middle, data, data_len); + hep->data_offset = 0; + hep->data_len = space_middle + data_len; + } else { + // No space left, create a new buffer to replace the old one. + char *new_data = malloc(space_needed); + if (!new_data) { + hep->input_data_was_corrupt = true; + LOG_OOM(); + return false; + } + if (hep->data) { + memcpy(new_data, hep->data + hep->data_offset, space_middle); + free(hep->data); + } else { + assert(space_middle == 0); + } + memcpy(new_data + space_middle, data, data_len); + hep->data = new_data; + hep->data_offset = 0; + hep->data_len = space_middle + data_len; + hep->data_buffer_size = space_needed; + } + hep->data[hep->data_len] = '\x00'; + + char *new_data_start = hep->data + hep->data_offset + space_middle; + if (strlen(new_data_start) != data_len) { + // The format does not expect NUL bytes. Disallow them to make sure that + // we can use C string functions such as strchr without it tripping. + hep->input_data_was_corrupt = true; + LOGE("Unexpected NUL byte found in input data."); + return false; + } + + return true; +} + +void sc_hid_event_parser_parse_all_data(struct sc_hid_event_parser *hep) { + if (hep->input_data_was_corrupt) { + hep->parser_status = SC_HID_EVENT_PARSER_STATE_CORRUPT; + } + if (hep->data && !sc_hid_event_parser_has_error(hep)) { + sc_hid_event_parser_parse_all_data_internal(hep); + } +} + +bool +sc_hid_event_parser_has_next(struct sc_hid_event_parser *hep) { + return !sc_vecdeque_is_empty(&hep->parsed_events); +} + +struct sc_hid_event* +sc_hid_event_parser_get_next(struct sc_hid_event_parser *hep) { + if (!sc_vecdeque_is_empty(&hep->parsed_events)) { + return sc_vecdeque_pop(&hep->parsed_events); + } + return NULL; +} + +bool +sc_hid_event_parser_has_error(struct sc_hid_event_parser *hep) { + // Whether hep->status is SC_HID_EVENT_PARSER_STATE_CORRUPT. + return hep->parser_status != SC_HID_EVENT_PARSER_STATE_GOOD; +} + +bool +sc_hid_event_parser_has_unparsed_data(struct sc_hid_event_parser *hep) { + return hep->data_offset != hep->data_len; +} + +// sc_hid_event_serializer: + +void +sc_hid_event_serializer_init(struct sc_hid_event_serializer *hes) { + hes->data = NULL; + hes->data_len = 0; + hes->data_buffer_size = 0; +} + +void +sc_hid_event_serializer_destroy(struct sc_hid_event_serializer *hes) { + if (hes->data) { + free(hes->data); + } +} + +bool +sc_hid_event_serializer_update(struct sc_hid_event_serializer *hes, + struct sc_hid_event *event) { + // Write: [timestamp] [accessory_id] [buffer in hex]\n + // ^^^^^^^^^^^^^^^^^^^^^^^^^^ + // This will be written first. + + // Summation of bytes: + // 1 ~ 20 : int64_t timestamp (sc_tick is alias for int64_t). + // 1 ~ 1 : space separator + // 1 ~ 5 : uint16_t accessory_id + // 3N ~ 3N : number of events * 3: space + 2 hex characters. + // 1 ~ 1 : '\n' + // 1 ~ 1 : NUL (not counted for data_len). + // = bytes needed ranges from 5+3N to 28+3N. Allocate at least that many: + size_t needed_event_size = 28 + event->size * 3; + size_t needed_total_size = needed_event_size + hes->data_len; + if (needed_total_size > hes->data_buffer_size) { + hes->data = realloc(hes->data, needed_total_size); + if (!hes->data) { + LOG_OOM(); + return false; + } + hes->data_buffer_size = needed_total_size; + } + + size_t data_len = hes->data_len; + int start_size = snprintf( + // Append after whatever that was written before: + hes->data + data_len, + // The previous realloc logic ensures that this is within bounds: + needed_event_size, + "%" PRItick " %" PRIu16, event->timestamp, event->accessory_id); + assert(start_size > 0); // not -1 because our params are correct. + assert((size_t)start_size < needed_event_size); + data_len += start_size; + for (unsigned i = 0; i < event->size; ++i) { + snprintf(hes->data + data_len, 4, " %02x", event->buffer[i]); + data_len += 3; + } + snprintf(hes->data + data_len, 2, "\n"); + ++data_len; + + hes->data_len = data_len; + return true; +} + +void +sc_hid_event_serializer_mark_as_read(struct sc_hid_event_serializer *hes) { + hes->data_len = 0; +} diff --git a/app/src/usb/hid_event_serializer.h b/app/src/usb/hid_event_serializer.h new file mode 100644 index 00000000..b792803b --- /dev/null +++ b/app/src/usb/hid_event_serializer.h @@ -0,0 +1,95 @@ +#ifndef SC_HID_EVENT_SERIALIZER_H +#define SC_HID_EVENT_SERIALIZER_H + +#include "common.h" + +#include + +#include "aoa_hid.h" +#include "util/vecdeque.h" + +// hid_event_parser: convert from bytes to sc_hid_event + +enum sc_hid_event_parser_status { + SC_HID_EVENT_PARSER_STATE_GOOD, + SC_HID_EVENT_PARSER_STATE_CORRUPT, +}; +struct sc_hid_event_ptr_queue SC_VECDEQUE(struct sc_hid_event*); + +struct sc_hid_event_parser { + const char *source_name; // Used in log messages, e.g. filename. + int line; // Line that is being parsed. + + char *data; // A zero-terminated string with string length data_len. + size_t data_offset; // Offset in data where we should start parsing. + size_t data_len; // Length of string, excluding NUL byte. + size_t data_buffer_size; // Size of |data|, including NUL and unused bytes. + + struct sc_hid_event_ptr_queue parsed_events; + enum sc_hid_event_parser_status parser_status; + + // Adding input and parsing input are separate, and receiving the input + // without parsing could already fail (e.g. due to OOM). This failure is + // stored separately, and eventually propagates when the parser starts. + bool input_data_was_corrupt; +}; + +void +sc_hid_event_parser_init(struct sc_hid_event_parser *hep, + const char *source_name); + +void +sc_hid_event_parser_destroy(struct sc_hid_event_parser *hep); + +// Appends data without taking ownership of |data|. |data| contains |data_len| +// bytes. +bool +sc_hid_event_parser_append_data(struct sc_hid_event_parser *hep, + const char *data, size_t data_len); + +// Parse all data that has been appended so far. In multi-threaded situations, +// this must be mutually exclusive with all other sc_hid_event_parser* methods. +void +sc_hid_event_parser_parse_all_data(struct sc_hid_event_parser *hep); + +// Check if a parsed event is available. This only includes events that were +// parsed until the most recent call to sc_hid_event_parser_parse_all_data. +bool +sc_hid_event_parser_has_next(struct sc_hid_event_parser *hep); + +// Retrieve the next event, if available. This only includes events that were +// parsed until the most recent call to sc_hid_event_parser_parse_all_data. +struct sc_hid_event* +sc_hid_event_parser_get_next(struct sc_hid_event_parser *hep); + +bool +sc_hid_event_parser_has_error(struct sc_hid_event_parser *hep); + +bool +sc_hid_event_parser_has_unparsed_data(struct sc_hid_event_parser *hep); + +// hid_event_serializer: convert from sc_hid_event to bytes + +struct sc_hid_event_serializer { + char *data; + size_t data_len; // Length of string, excluding NUL byte. + size_t data_buffer_size; // Size of |data|, including NUL and unused bytes. +}; + +void +sc_hid_event_serializer_init(struct sc_hid_event_serializer *hes); + +void +sc_hid_event_serializer_destroy(struct sc_hid_event_serializer *hes); + +// Appends serialization of |event| to |hes->data|. +bool +sc_hid_event_serializer_update(struct sc_hid_event_serializer *hes, + struct sc_hid_event *event); + +// To minimize allocations, callers can directly copy |hes->data_len| bytes to +// their destination from |hes->data|. After that, mark the data as read to +// allow the space to be freed for further data. +void +sc_hid_event_serializer_mark_as_read(struct sc_hid_event_serializer *hes); +#endif diff --git a/app/src/usb/hid_replay.c b/app/src/usb/hid_replay.c new file mode 100644 index 00000000..2e54021d --- /dev/null +++ b/app/src/usb/hid_replay.c @@ -0,0 +1,481 @@ +#include "util/log.h" + +#include +#include +#include +#include +#include // for file I/O helpers. + +#include "hid_event_serializer.h" +#include "hid_replay.h" +#include "util/tick.h" + +// There are three threads of interest: +// - The thread where the HID event is emitted (aoa). +// - The thread where the replay is run (run_hid_replay). This may sleep +// occasionally as part of the event replay. When there are no events to +// replay, it will wait for the io_thread to provide new data. +// - The I/O thread where the data is read, to feed the replay thread. +struct sc_hidr_replay_thread_state { + struct sc_hidr *hidr; + struct sc_hid_event_parser hep; + sc_mutex io_mutex; // guards access to hep, io_thread_stopped and io_cond. + sc_cond io_cond; + sc_thread io_thread; + bool io_thread_stopped; +}; + +static bool +sc_hidr_thread_and_queue_init(struct sc_hidr_thread_and_queue *taq, + const char *filename) { + taq->filename = filename; + if (!filename) { + return true; + } + sc_vecdeque_init(&taq->queue); + + bool ok = sc_mutex_init(&taq->mutex); + if (!ok) { + sc_vecdeque_destroy(&taq->queue); + return false; + } + + ok = sc_cond_init(&taq->event_cond); + if (!ok) { + sc_mutex_destroy(&taq->mutex); + sc_vecdeque_destroy(&taq->queue); + return false; + } + taq->stopped = false; + return true; +} + +static void +sc_hidr_thread_and_queue_destroy(struct sc_hidr_thread_and_queue *taq) { + if (!taq->filename) { + return; + } + // Sanity check: once started, sc_hidr_thread_and_queue_destroy must only + // be called after sc_hidr_thread_and_queue_stop has returned. That implies + // that taq->thread has terminated, and that there is no concurrent access + // to the mutex/queue any more. + assert(taq->stopped); + sc_cond_destroy(&taq->event_cond); + sc_mutex_destroy(&taq->mutex); + while (!sc_vecdeque_is_empty(&taq->queue)) { + struct sc_hid_event *event = sc_vecdeque_pop(&taq->queue); + assert(event); + sc_hid_event_destroy(event); + free(event); + } + sc_vecdeque_destroy(&taq->queue); +} + +static void +sc_hidr_thread_and_queue_stop(struct sc_hidr_thread_and_queue *taq) { + assert(taq->filename); // mutex etc only initialized when filename is set. + taq->stopped = true; + sc_mutex_lock(&taq->mutex); + sc_cond_signal(&taq->event_cond); + sc_mutex_unlock(&taq->mutex); + sc_thread_join(&taq->thread, NULL); +} + +static bool +sc_hidr_is_accepted_hid_event(struct sc_hidr *hidr, + const struct sc_hid_event *event) { + // 1 is HID_KEYBOARD_ACCESSORY_ID from hid_keyboard.c + if (event->accessory_id == 1) { + return hidr->enable_keyboard; + } + // 2 is HID_MOUSE_ACCESSORY_ID from hid_mouse.c + if (event->accessory_id == 2) { + return hidr->enable_mouse; + } + LOGD("Unrecognized accessory_id: %" PRIu16, event->accessory_id); + return false; +} + +bool +sc_hidr_init(struct sc_hidr *hidr, struct sc_aoa *aoa, + const char *record_filename, const char *replay_filename, + bool enable_keyboard, bool enable_mouse) { + if (record_filename && replay_filename && + !strcmp(record_filename, replay_filename)) { + // TODO: Add more comprehensive check that accounts for equivalent + // file paths, symlinks, etc. Like C++'s std::filesystem::equivalent. + LOGE("--hid-record and --hid-replay are set to the same file!"); + LOGE("Exiting early to avoid an infinite feedback loop."); + return false; + } + if (!sc_hidr_thread_and_queue_init(&hidr->taq_replay, replay_filename)) { + return false; + } + if (!sc_hidr_thread_and_queue_init(&hidr->taq_record, record_filename)) { + sc_hidr_thread_and_queue_destroy(&hidr->taq_replay); + return false; + } + hidr->aoa = aoa; + hidr->enable_mouse = enable_mouse; + hidr->enable_keyboard = enable_keyboard; + return true; +} + +void +sc_hidr_destroy(struct sc_hidr *hidr) { + sc_hidr_thread_and_queue_destroy(&hidr->taq_replay); + sc_hidr_thread_and_queue_destroy(&hidr->taq_record); +} + +static void +run_hid_record_to_file(struct sc_hidr *hidr) { + struct sc_hidr_thread_and_queue *taq = &hidr->taq_record; + SDL_RWops *io = SDL_RWFromFile(taq->filename, "wb"); + if (!io) { + LOGE("Unable to open file for HID recording: %s", taq->filename); + return; + } + + struct sc_hid_event_serializer hes; + sc_hid_event_serializer_init(&hes); + for (;;) { + sc_mutex_lock(&taq->mutex); + while (!taq->stopped && sc_vecdeque_is_empty(&taq->queue)) { + sc_cond_wait(&taq->event_cond, &taq->mutex); + } + if (taq->stopped) { + sc_mutex_unlock(&taq->mutex); + break; + } + + bool ok = true; + assert(!sc_vecdeque_is_empty(&taq->queue)); + while (!sc_vecdeque_is_empty(&taq->queue) && ok) { + struct sc_hid_event *event = sc_vecdeque_pop(&taq->queue); + ok = sc_hid_event_serializer_update(&hes, event); + sc_hid_event_destroy(event); + free(event); // balances sc_hidr_observe_event_for_record. + } + sc_mutex_unlock(&taq->mutex); + + if (!ok) { + LOGE("Failed to serialize for HID recording to %s", taq->filename); + break; + } + + assert(hes.data_len); // Non-zero because at least one event was seen. + size_t written_size = SDL_RWwrite(io, hes.data, 1, hes.data_len); + if (written_size != hes.data_len) { + LOGE("Failed to write line for HID recording to %s: %s" + " (expected to write %zu bytes, but written %zu instead)", + taq->filename, SDL_GetError(), hes.data_len, written_size); + break; + } + sc_hid_event_serializer_mark_as_read(&hes); + } + sc_hid_event_serializer_destroy(&hes); + + SDL_RWclose(io); + LOGI("Finished HID recording to: %s", taq->filename); +} + +static int +run_hid_replay_read_input(void *rts_data) { + struct sc_hidr_replay_thread_state *rts = rts_data; + struct sc_hidr *hidr = rts->hidr; + struct sc_hidr_thread_and_queue *taq = &hidr->taq_replay; + const char *filename = taq->filename; + struct sc_hid_event_parser *hep = &rts->hep; + + SDL_RWops *io = SDL_RWFromFile(filename, "rb"); + if (!io) { + LOGE("Unable to read HID replay from %s: %s", filename, SDL_GetError()); + sc_mutex_lock(&rts->io_mutex); + rts->io_thread_stopped = true; + sc_cond_signal(&rts->io_cond); + sc_mutex_unlock(&rts->io_mutex); + return 0; + } + // When the size can be determined upfront, assume that we can read all + // data at once. Do so, so we can replay without worry about slow disks + // resulting in events being replayed too late. + bool want_all_at_once = SDL_RWsize(io) != -1; + if (want_all_at_once) { + LOGD("Starting to read all data for HID replay from %s", filename); + size_t size; + char *data = SDL_LoadFile_RW(io, &size, 1); // = reads & closes file. + if (!data) { + LOGE("Unable to read HID replay from file: %s", filename); + } else { + LOGD("Read %zu bytes from %s", size, filename); + sc_mutex_lock(&rts->io_mutex); + if (!sc_hid_event_parser_append_data(hep, data, size)) { + LOGE("Failed to initialize HID event parser from %s", filename); + } + sc_mutex_unlock(&rts->io_mutex); + SDL_free(data); + } + } else { + LOGD("Starting to stream data for HID replay from %s", filename); + size_t data_buffer_size = 1024; + char data[1024]; + for (;;) { + size_t size_read = SDL_RWread(io, data, 1, data_buffer_size); + if (!size_read) { + LOGD("End of data stream for HID replay from %s", filename); + break; + } + sc_mutex_lock(&rts->io_mutex); + bool ok = sc_hid_event_parser_append_data(hep, data, size_read); + if (ok) { + sc_cond_signal(&rts->io_cond); + } + sc_mutex_unlock(&rts->io_mutex); + if (!ok) { + LOGE("Failed to copy HID replay data from %s", filename); + break; + } + } + SDL_RWclose(io); + } + + sc_mutex_lock(&rts->io_mutex); + rts->io_thread_stopped = true; + sc_cond_signal(&rts->io_cond); + sc_mutex_unlock(&rts->io_mutex); + return 0; +} + +static void +run_hid_replay_from_input(struct sc_hidr *hidr, + struct sc_hid_event_parser *hep, + bool *had_any_event, + sc_tick *last_timestamp_p) { + assert(sc_hid_event_parser_has_next(hep)); + + struct sc_hidr_thread_and_queue *taq = &hidr->taq_replay; + sc_mutex_lock(&taq->mutex); + struct sc_hid_event *hid_event; + while ((hid_event = sc_hid_event_parser_get_next(hep)) != NULL) { + if (taq->stopped) { + sc_hid_event_destroy(hid_event); + free(hid_event); + break; + } + if (!sc_hidr_is_accepted_hid_event(hidr, hid_event)) { + sc_hid_event_destroy(hid_event); + free(hid_event); + continue; + } + sc_tick ms_to_sleep = + *had_any_event ? hid_event->timestamp - *last_timestamp_p : 0; + if (ms_to_sleep < 0) { + LOGD("HID replay tried to back in time with timestamp: %" PRItick, + hid_event->timestamp); + ms_to_sleep = 0; + } + + if (ms_to_sleep) { + sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(ms_to_sleep); + bool ok = true; + while (!taq->stopped && ok) { + ok = sc_cond_timedwait(&taq->event_cond, &taq->mutex, deadline); + } + } + + if (taq->stopped) { + sc_hid_event_destroy(hid_event); + free(hid_event); + break; + } + + *had_any_event = true; + *last_timestamp_p = hid_event->timestamp; + + sc_mutex_unlock(&taq->mutex); + sc_hidr_trigger_event_for_replay(hidr, hid_event); + sc_mutex_lock(&taq->mutex); + } + sc_mutex_unlock(&taq->mutex); +} + +static int +run_hid_record(void *data) { + struct sc_hidr *hidr = data; + run_hid_record_to_file(hidr); + return 0; +} + +static int +run_hid_replay(void *data) { + struct sc_hidr *hidr = data; + + struct sc_hidr_replay_thread_state rts; + rts.hidr = hidr; + if (!sc_mutex_init(&rts.io_mutex)) { + LOGE("Failed to initialize mutex for HID replay"); + return 0; + } + if (!sc_cond_init(&rts.io_cond)) { + LOGE("Failed to initialize cond for HID replay"); + sc_mutex_destroy(&rts.io_mutex); + return 0; + } + sc_hid_event_parser_init(&rts.hep, hidr->taq_replay.filename); + rts.io_thread_stopped = false; + + // Start thread to read input. + if (!sc_thread_create(&rts.io_thread, run_hid_replay_read_input, + "scrcpyHIDinp", &rts)) { + LOGE("Failed to start thread to read input for HID replay"); + sc_hid_event_parser_destroy(&rts.hep); + sc_cond_destroy(&rts.io_cond); + sc_mutex_destroy(&rts.io_mutex); + return 0; + } + + // Receive data from input thread and forward events to aoa. + LOGD("Waiting for input to commence HID replay."); + bool had_any_event = false; + sc_tick last_timestamp = 0; + sc_mutex_lock(&rts.io_mutex); + while (!hidr->taq_replay.stopped && + !sc_hid_event_parser_has_error(&rts.hep)) { + if (!sc_hid_event_parser_has_next(&rts.hep)) { + if (!sc_hid_event_parser_has_unparsed_data(&rts.hep)) { + if (rts.io_thread_stopped) { + break; + } + sc_cond_wait(&rts.io_cond, &rts.io_mutex); + } + sc_hid_event_parser_parse_all_data(&rts.hep); + continue; + } + // Unlock IO mutex because we're going to potentially be blocked by the + // hidr.taq_replay->mutex, and don't want that to block the IO thread. + sc_mutex_unlock(&rts.io_mutex); + run_hid_replay_from_input(hidr, &rts.hep, &had_any_event, + &last_timestamp); + sc_mutex_lock(&rts.io_mutex); + } + sc_mutex_unlock(&rts.io_mutex); + + // Print diagnostic information. + LOGD("End of input for HID replay from %s", hidr->taq_replay.filename); + if (sc_hid_event_parser_has_error(&rts.hep)) { + LOGE("Invalid HID replay data in %s", hidr->taq_replay.filename); + } else if (sc_hid_event_parser_has_unparsed_data(&rts.hep) || + sc_hid_event_parser_has_next(&rts.hep)) { + LOGE("Did not finish replay of %s", hidr->taq_replay.filename); + } else if (!had_any_event) { + LOGE("Did not find any replay data in %s", hidr->taq_replay.filename); + } else { + LOGD("Successfully replayed all data in %s", hidr->taq_replay.filename); + } + + // Clean up when everything is done. + sc_thread_join(&rts.io_thread, NULL); + sc_hid_event_parser_destroy(&rts.hep); + sc_cond_destroy(&rts.io_cond); + sc_mutex_destroy(&rts.io_mutex); + LOGI("Finished HID replay from %s", hidr->taq_replay.filename); + return 0; +} + +bool +sc_hidr_start_record(struct sc_hidr *hidr) { + // sc_hidr_start_record is called before sc_aoa_start is called (which + // starts the thread that will access hidr_to_notify). Therefore we can + // safely modify aoa->hidr_to_notify here. + hidr->aoa->hidr_to_notify = hidr; + bool ok = sc_thread_create(&hidr->taq_record.thread, run_hid_record, + "scrcpyHIDrecord", hidr); + if (!ok) { + LOGE("Could not start HID recorder thread"); + return false; + } + LOGI("Recording HID input to: %s", hidr->taq_record.filename); + return true; +} + +bool +sc_hidr_start_replay(struct sc_hidr *hidr) { + bool ok = sc_thread_create(&hidr->taq_replay.thread, run_hid_replay, + "scrcpyHIDreplay", hidr); + if (!ok) { + LOGE("Could not start HID replay thread"); + return false; + } + LOGI("Replaying HID input from: %s", hidr->taq_replay.filename); + return true; +} + +void +sc_hidr_observe_event_for_record(struct sc_hidr *hidr, + const struct sc_hid_event *event) { + assert(hidr->taq_record.filename); + if (!sc_hidr_is_accepted_hid_event(hidr, event)) { + return; + } + // event is not owned, so we need to make a copy first. + struct sc_hid_event *hid_event = malloc(sizeof(struct sc_hid_event)); + unsigned char *buffer = malloc(event->size); + if (!buffer || !hid_event) { + LOG_OOM(); + free(buffer); + free(hid_event); + return; + } + memcpy(buffer, event->buffer, event->size); + sc_hid_event_init(hid_event, event->accessory_id, buffer, event->size); + hid_event->timestamp = SC_TICK_TO_MS(sc_tick_now()); + + sc_mutex_lock(&hidr->taq_record.mutex); + bool ok = false; + if (!hidr->taq_record.stopped) { + bool was_empty = sc_vecdeque_is_empty(&hidr->taq_record.queue); + ok = sc_vecdeque_push(&hidr->taq_record.queue, hid_event); + if (!ok) { + LOG_OOM(); + } else if (was_empty) { + sc_cond_signal(&hidr->taq_record.event_cond); + } + } + sc_mutex_unlock(&hidr->taq_record.mutex); + + if (!ok) { + sc_hid_event_destroy(hid_event); + free(hid_event); + } +} + +void +sc_hidr_trigger_event_for_replay(struct sc_hidr *hidr, + struct sc_hid_event *event) { + assert(hidr->taq_replay.filename); + // We should already have filtered unwanted events earlier: + assert(sc_hidr_is_accepted_hid_event(hidr, event)); + if (hidr->taq_replay.stopped) { + sc_hid_event_destroy(event); + } else { + // Note: may indirectly trigger sc_hidr_observe_event_for_record. + // To avoid deadlocks we avoid unnecessary use of mutexes here and + // among callers. + sc_aoa_push_hid_event(hidr->aoa, event); + } + // Most callers of sc_aoa_push_hid_event pass a stack-allocated event. + // |event| here is heap-allocated by sc_hid_event_parser in run_hid_replay. + free(event); // balances sc_hid_event_parser_parse_all_data. +} + +void +sc_hidr_stop_record(struct sc_hidr *hidr) { + assert(hidr->taq_record.filename); + sc_hidr_thread_and_queue_stop(&hidr->taq_record); +} + +void +sc_hidr_stop_replay(struct sc_hidr *hidr) { + assert(hidr->taq_replay.filename); + sc_hidr_thread_and_queue_stop(&hidr->taq_replay); +} diff --git a/app/src/usb/hid_replay.h b/app/src/usb/hid_replay.h new file mode 100644 index 00000000..4897a4a1 --- /dev/null +++ b/app/src/usb/hid_replay.h @@ -0,0 +1,63 @@ +#ifndef SC_HID_REPLAY_H +#define SC_HID_REPLAY_H + +#include "common.h" + +#include + +#include "aoa_hid.h" +#include "hid_event_serializer.h" +#include "hid_keyboard.h" +#include "hid_mouse.h" +#include "util/thread.h" + +struct sc_hidr_thread_and_queue { + const char *filename; + sc_thread thread; + sc_mutex mutex; // guards queue access. + sc_cond event_cond; + atomic_bool stopped; + struct sc_hid_event_ptr_queue queue; +}; + +struct sc_hidr { + struct sc_hidr_thread_and_queue taq_replay; + struct sc_hidr_thread_and_queue taq_record; + + struct sc_aoa *aoa; + bool enable_keyboard; + bool enable_mouse; +}; + +bool +sc_hidr_init(struct sc_hidr *hidr, struct sc_aoa *aoa, + const char *record_filename, const char *replay_filename, + bool enable_keyboard, bool enable_mouse); + +void +sc_hidr_destroy(struct sc_hidr *hidr); + +bool +sc_hidr_start_record(struct sc_hidr *hidr); + +bool +sc_hidr_start_replay(struct sc_hidr *hidr); + +// Can be called from any thread, after sc_hidr_start_record(). +void +sc_hidr_observe_event_for_record(struct sc_hidr *hidr, + const struct sc_hid_event *event); + +// Can be called from any thread, after sc_hidr_start_replay(). +// Takes ownership of the |event| pointee. +void +sc_hidr_trigger_event_for_replay(struct sc_hidr *hidr, + struct sc_hid_event *event); + +void +sc_hidr_stop_record(struct sc_hidr *hidr); + +void +sc_hidr_stop_replay(struct sc_hidr *hidr); + +#endif diff --git a/app/src/usb/scrcpy_otg.c b/app/src/usb/scrcpy_otg.c index 6a7fd79b..b45a22ee 100644 --- a/app/src/usb/scrcpy_otg.c +++ b/app/src/usb/scrcpy_otg.c @@ -4,6 +4,7 @@ #include "adb/adb.h" #include "events.h" +#include "hid_replay.h" #include "screen_otg.h" #include "util/log.h" @@ -14,6 +15,7 @@ struct scrcpy_otg { struct sc_hid_mouse mouse; struct sc_screen_otg screen_otg; + struct sc_hidr hidr; }; static void @@ -79,6 +81,9 @@ scrcpy_otg(struct scrcpy_options *options) { bool usb_connected = false; bool aoa_started = false; bool aoa_initialized = false; + bool hidr_initialized = false; + bool hidr_replay_started = false; + bool hidr_record_started = false; #ifdef _WIN32 // On Windows, only one process could open a USB device @@ -144,6 +149,25 @@ scrcpy_otg(struct scrcpy_options *options) { mouse = &s->mouse; } + if (options->hid_replay_filename || options->hid_record_filename) { + ok = sc_hidr_init(&s->hidr, &s->aoa, options->hid_record_filename, + options->hid_replay_filename, enable_keyboard, + enable_mouse); + if (!ok) { + goto end; + } + hidr_initialized = true; + } + if (options->hid_record_filename) { + // Set up hidr record BEFORE starting aoa, to ensure that hidr can + // subscribe to events from aoa before the aoa thread starts. + ok = sc_hidr_start_record(&s->hidr); + if (!ok) { + goto end; + } + hidr_record_started = true; + } + ok = sc_aoa_start(&s->aoa); if (!ok) { goto end; @@ -176,10 +200,23 @@ scrcpy_otg(struct scrcpy_options *options) { sc_usb_device_destroy(&usb_device); usb_device_initialized = false; + if (options->hid_replay_filename) { + ok = sc_hidr_start_replay(&s->hidr); + if (!ok) { + goto end; + } + hidr_replay_started = true; + } + ret = event_loop(s); LOGD("quit..."); end: + if (hidr_replay_started) { + sc_hidr_stop_replay(&s->hidr); + // hidr will not call aoa at this point. + } + if (aoa_started) { sc_aoa_stop(&s->aoa); } @@ -196,6 +233,15 @@ end: sc_aoa_join(&s->aoa); sc_aoa_destroy(&s->aoa); } + if (hidr_record_started) { + // Because aoa has been destroyed, aoa won't call hidr at this point. + sc_hidr_stop_record(&s->hidr); + } + if (hidr_initialized) { + // aoa has been destroyed, there are no circular references between + // hidr and aoa, so we can finally destroy hidr. + sc_hidr_destroy(&s->hidr); + } sc_usb_join(&s->usb); diff --git a/app/src/util/tick.h b/app/src/util/tick.h index 2d941f23..d7b29fc6 100644 --- a/app/src/util/tick.h +++ b/app/src/util/tick.h @@ -7,6 +7,7 @@ typedef int64_t sc_tick; #define PRItick PRIi64 +#define SCNtick SCNi64 #define SC_TICK_FREQ 1000000 // microsecond // To be adapted if SC_TICK_FREQ changes diff --git a/doc/hid-otg.md b/doc/hid-otg.md index 7dfc60fc..8eec363a 100644 --- a/doc/hid-otg.md +++ b/doc/hid-otg.md @@ -110,3 +110,35 @@ connected over USB. ## HID/OTG issues on Windows See [FAQ](/FAQ.md#hidotg-issues-on-windows). + + +## Recording + +When `--hid-keyboard`, `--hid-mouse` or `--otg` are used, the interactions with +the physical keyboard and/or mouse can be recorded to a file. These recorded +events can be replayed later to trigger the same sequence of events. + +```bash +scrcpy --otg --hid-record=recording.log +scrcpy --otg --hid-replay=recording.log +``` + +`scrcpy` can record input while replaying another session. This feature can be +used to create recordings in multiple takes rather than at once: + +```bash +scrcpy --otg --hid-record=first.log +scrcpy --otg --hid-replay=first.log --hid-record=second.log +scrcpy --otg --hid-replay=second.log +``` + + +On Linux, it is possible to control two devices simultaneously, by recording +the input to a special fifo file. The fifo file serves as a named pipe to +enable both `scrcpy` instances to communicate with each other. + +```bash +mkfifo my_named_pipe +scrcpy --otg --hid-replay=my_named_pipe -s id_of_target_device & +scrcpy --otg --hid-record=my_named_pipe -s id_of_source_device +``` diff --git a/doc/recording.md b/doc/recording.md index 216542e9..1dbbab16 100644 --- a/doc/recording.md +++ b/doc/recording.md @@ -30,6 +30,12 @@ course, not if you capture your scrcpy window and audio output on the computer). [packet delay variation]: https://en.wikipedia.org/wiki/Packet_delay_variation +## Keyboard / mouse recording and replay + +The keyboard and mouse can also be recorded and replayed, independently of +video / audio. See [HID/OTG recording](hid-otg.md#recording). + + ## Format The video and audio streams are encoded on the device, but are muxed on the From 5ff9849765f0d02a1f25d5506793e5d2fa891edf Mon Sep 17 00:00:00 2001 From: Rob Wu Date: Sun, 31 Dec 2023 19:59:33 +0100 Subject: [PATCH 2/2] Add tests for hid_event_parser & serializer This also moves the `sc_hid_event` type to a separate file, to enable the other classes to use it without requiring the several other object files to be linked. --- app/meson.build | 11 + app/src/usb/aoa_hid.c | 15 -- app/src/usb/aoa_hid.h | 18 +- app/src/usb/hid_event.c | 17 ++ app/src/usb/hid_event.h | 24 +++ app/src/usb/hid_event_serializer.h | 2 +- app/tests/test_hid_event_serializer.c | 290 ++++++++++++++++++++++++++ 7 files changed, 344 insertions(+), 33 deletions(-) create mode 100644 app/src/usb/hid_event.c create mode 100644 app/src/usb/hid_event.h create mode 100644 app/tests/test_hid_event_serializer.c diff --git a/app/meson.build b/app/meson.build index 04b9dfde..d2ae6dd3 100644 --- a/app/meson.build +++ b/app/meson.build @@ -88,6 +88,7 @@ usb_support = get_option('usb') if usb_support src += [ 'src/usb/aoa_hid.c', + 'src/usb/hid_event.c', 'src/usb/hid_event_serializer.c', 'src/usb/hid_keyboard.c', 'src/usb/hid_mouse.c', @@ -259,6 +260,16 @@ if get_option('buildtype') == 'debug' 'tests/test_vector.c', ]], ] + if usb_support + tests += [ + ['test_hid_event_serializer', [ + 'tests/test_hid_event_serializer.c', + 'src/usb/hid_event.c', + 'src/usb/hid_event_serializer.c', + 'src/util/memory.c', + ]], + ] + endif foreach t : tests sources = t[1] + ['src/compat.c'] diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index 01150d0b..5b1eb6a8 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -33,21 +33,6 @@ sc_hid_event_log(const struct sc_hid_event *event) { free(buffer); } -void -sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, - unsigned char *buffer, uint16_t buffer_size) { - hid_event->accessory_id = accessory_id; - hid_event->buffer = buffer; - hid_event->size = buffer_size; - hid_event->ack_to_wait = SC_SEQUENCE_INVALID; - hid_event->timestamp = 0; -} - -void -sc_hid_event_destroy(struct sc_hid_event *hid_event) { - free(hid_event->buffer); -} - bool sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb, struct sc_acksync *acksync) { diff --git a/app/src/usb/aoa_hid.h b/app/src/usb/aoa_hid.h index 27f91c4c..95b07918 100644 --- a/app/src/usb/aoa_hid.h +++ b/app/src/usb/aoa_hid.h @@ -6,28 +6,12 @@ #include +#include "hid_event.h" #include "usb.h" #include "util/acksync.h" #include "util/thread.h" -#include "util/tick.h" #include "util/vecdeque.h" -struct sc_hid_event { - uint16_t accessory_id; - unsigned char *buffer; - uint16_t size; - uint64_t ack_to_wait; - sc_tick timestamp; // Only used by hid_replay.c & hid_event_serializer.c -}; - -// Takes ownership of buffer -void -sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, - unsigned char *buffer, uint16_t buffer_size); - -void -sc_hid_event_destroy(struct sc_hid_event *hid_event); - struct sc_hid_event_queue SC_VECDEQUE(struct sc_hid_event); // Forward declare sc_hidr to avoid circular dependency on hid_replay.h. diff --git a/app/src/usb/hid_event.c b/app/src/usb/hid_event.c new file mode 100644 index 00000000..330108a5 --- /dev/null +++ b/app/src/usb/hid_event.c @@ -0,0 +1,17 @@ +#include "hid_event.h" +#include "util/acksync.h" + +void +sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, + unsigned char *buffer, uint16_t buffer_size) { + hid_event->accessory_id = accessory_id; + hid_event->buffer = buffer; + hid_event->size = buffer_size; + hid_event->ack_to_wait = SC_SEQUENCE_INVALID; + hid_event->timestamp = 0; +} + +void +sc_hid_event_destroy(struct sc_hid_event *hid_event) { + free(hid_event->buffer); +} diff --git a/app/src/usb/hid_event.h b/app/src/usb/hid_event.h new file mode 100644 index 00000000..2b052cc0 --- /dev/null +++ b/app/src/usb/hid_event.h @@ -0,0 +1,24 @@ +#ifndef SC_HID_EVENT_H +#define SC_HID_EVENT_H + +#include "common.h" + +#include +#include "util/tick.h" + +struct sc_hid_event { + uint16_t accessory_id; + unsigned char *buffer; + uint16_t size; + uint64_t ack_to_wait; + sc_tick timestamp; // Only used by hid_replay.c & hid_event_serializer.c +}; + +// Takes ownership of buffer +void +sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, + unsigned char *buffer, uint16_t buffer_size); + +void +sc_hid_event_destroy(struct sc_hid_event *hid_event); +#endif diff --git a/app/src/usb/hid_event_serializer.h b/app/src/usb/hid_event_serializer.h index b792803b..b8d3d63f 100644 --- a/app/src/usb/hid_event_serializer.h +++ b/app/src/usb/hid_event_serializer.h @@ -5,7 +5,7 @@ #include -#include "aoa_hid.h" +#include "hid_event.h" #include "util/vecdeque.h" // hid_event_parser: convert from bytes to sc_hid_event diff --git a/app/tests/test_hid_event_serializer.c b/app/tests/test_hid_event_serializer.c new file mode 100644 index 00000000..9e805f09 --- /dev/null +++ b/app/tests/test_hid_event_serializer.c @@ -0,0 +1,290 @@ +#include "common.h" + +#include + +#include "usb/hid_event_serializer.h" + +static void test_hid_event_serializer(void) { + struct sc_hid_event hid_event; + uint16_t accessory_id = 1337; + unsigned char *buffer = malloc(5); + buffer[0] = '\xDE'; + buffer[1] = '\xEA'; + buffer[2] = '\xBE'; + buffer[3] = '\xEF'; + buffer[4] = '\x00'; + uint16_t buffer_size = 5; + sc_hid_event_init(&hid_event, accessory_id, buffer, buffer_size); + assert(hid_event.timestamp == 0); + + struct sc_hid_event_serializer hes; + sc_hid_event_serializer_init(&hes); + + assert(hes.data_len == 0); + + sc_hid_event_serializer_update(&hes, &hid_event); + assert(strlen("0 1337 de ea be ef 00\n") == hes.data_len); + assert(hes.data_len < hes.data_buffer_size); // Need room for NUL. + assert(!strncmp("0 1337 de ea be ef 00\n", hes.data, hes.data_len + 1)); + + hid_event.timestamp = 9001; + sc_hid_event_serializer_update(&hes, &hid_event); + assert(!strncmp("0 1337 de ea be ef 00\n9001 1337 de ea be ef 00\n", + hes.data, hes.data_len + 1)); + + sc_hid_event_serializer_mark_as_read(&hes); + assert(hes.data_len == 0); + + sc_hid_event_serializer_update(&hes, &hid_event); + assert(!strncmp("9001 1337 de ea be ef 00\n", hes.data, hes.data_len + 1)); + + sc_hid_event_serializer_destroy(&hes); + free(buffer); +} + +static void test_hid_event_serializer_only_init_and_destroy(void) { + struct sc_hid_event_serializer hes; + sc_hid_event_serializer_init(&hes); + sc_hid_event_serializer_destroy(&hes); +} + +static void test_hid_event_serializer_minimum_length(void) { + struct sc_hid_event hid_event; + sc_hid_event_init(&hid_event, 2, calloc(1, 1), 1); + assert(hid_event.timestamp == 0); + + struct sc_hid_event_serializer hes; + sc_hid_event_serializer_init(&hes); + sc_hid_event_serializer_update(&hes, &hid_event); + + assert(hes.data_len < hes.data_buffer_size); // Need room for NUL. + assert(strlen("0 2 00\n") == hes.data_len); + assert(!strncmp("0 2 00\n", hes.data, hes.data_len + 1)); + + sc_hid_event_serializer_destroy(&hes); + free(hid_event.buffer); +} + +static void test_hid_event_serializer_maximum_length(void) { + struct sc_hid_event hid_event; + sc_hid_event_init(&hid_event, 65535, calloc(1, 1), 1); + // As the type is a signed 64-bit integer, the largest length is its lowest + // value. It is unlikely for such a timestamp to be seen in practice. + hid_event.timestamp = -9223372036854775807; // = -(2^63-1) + + struct sc_hid_event_serializer hes; + sc_hid_event_serializer_init(&hes); + sc_hid_event_serializer_update(&hes, &hid_event); + + // Now perform an exact check instead of a "data_len < buffer_size". The + // minimum buffer size is carefully chosen to fit the longest values. + assert(hes.data_len + 1 == hes.data_buffer_size); + + assert(hes.data_len < hes.data_buffer_size); // Need room for NUL. + assert(strlen("-9223372036854775807 65535 00\n") == hes.data_len); + assert(!strncmp("-9223372036854775807 65535 00\n", hes.data, + hes.data_len + 1)); + + sc_hid_event_serializer_destroy(&hes); + free(hid_event.buffer); +} + +static void test_hid_event_parser(void) { + struct sc_hid_event_parser hep; + sc_hid_event_parser_init(&hep, "source_name"); + + const char input_str[] = "1 1023 f0 0d\n"; + int input_len = strlen(input_str); + // Note: allocate just enough to hold input_str, without trailing NUL byte, + // to show that the NUL byte is not required (even though C strings will + // always end with a NUL). + char *data = malloc(input_len); + memcpy(data, input_str, input_len); + bool ok = sc_hid_event_parser_append_data(&hep, data, input_len); + assert(ok); + free(data); // Free immediately, further operations should not trigger UAF. + + sc_hid_event_parser_parse_all_data(&hep); + assert(sc_hid_event_parser_has_next(&hep)); + struct sc_hid_event *parsed_event = sc_hid_event_parser_get_next(&hep); + assert(parsed_event); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(!sc_hid_event_parser_has_unparsed_data(&hep)); + + assert(parsed_event->timestamp == 1); + assert(parsed_event->accessory_id == 1023); + assert(parsed_event->size == 2); + assert(!strncmp((const char*)parsed_event->buffer, "\xf0\x0d", 2)); + + // Clean up. + sc_hid_event_destroy(parsed_event); + free(parsed_event); + + // Another one, partial. + ok = sc_hid_event_parser_append_data(&hep, "7", 1); + assert(ok); + sc_hid_event_parser_parse_all_data(&hep); + assert(!sc_hid_event_parser_has_next(&hep)); + parsed_event = sc_hid_event_parser_get_next(&hep); + assert(!parsed_event); // Incomplete. + assert(!sc_hid_event_parser_has_error(&hep)); + assert(sc_hid_event_parser_has_unparsed_data(&hep)); + + // Append part of the original line (minus '\n') + ok = sc_hid_event_parser_append_data(&hep, input_str, input_len - 1); + assert(ok); + // Append the new ending. + ok = sc_hid_event_parser_append_data(&hep, " DE ED\n", strlen(" DE ED\n")); + assert(ok); + + // The event is not seen until sc_hid_event_parser_parse_all_data() is run: + parsed_event = sc_hid_event_parser_get_next(&hep); + assert(!parsed_event); + + sc_hid_event_parser_parse_all_data(&hep); + parsed_event = sc_hid_event_parser_get_next(&hep); + assert(parsed_event); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(!sc_hid_event_parser_has_unparsed_data(&hep)); + + assert(parsed_event->timestamp == 71); + assert(parsed_event->accessory_id == 1023); + assert(parsed_event->size == 4); + assert(!strncmp((const char*)parsed_event->buffer, "\xf0\x0d\xde\xed", 4)); + + sc_hid_event_parser_destroy(&hep); + + // Clean up once more, now after hep has been destroyed to prove that the + // parsed_event outlives the parser. + sc_hid_event_destroy(parsed_event); + free(parsed_event); +} + +static void test_hid_event_parser_only_init_and_destroy(void) { + struct sc_hid_event_parser hep; + sc_hid_event_parser_init(&hep, "source_name"); + sc_hid_event_parser_destroy(&hep); +} + +static void test_hid_event_parser_reject_null_in_input(void) { + struct sc_hid_event_parser hep; + sc_hid_event_parser_init(&hep, "invalid_embedded_nulls"); + + const size_t input_len = 14; + char input_with_nul[15] = {0}; + memcpy(input_with_nul + 1, "1 1023 f0 0d\n", 14); + + // Note: data is not empty but data_len is 0, so the \x00 should be ignored. + bool ok = sc_hid_event_parser_append_data(&hep, input_with_nul, 0); + assert(ok); // No data to append, all right! + sc_hid_event_parser_parse_all_data(&hep); + assert(!sc_hid_event_parser_get_next(&hep)); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(!sc_hid_event_parser_has_unparsed_data(&hep)); + + ok = sc_hid_event_parser_append_data(&hep, input_with_nul, input_len); + assert(!ok); // Invalid due to null. + // The error is not propagated until sc_hid_event_parser_parse_all_data(): + assert(!sc_hid_event_parser_has_error(&hep)); + // The "has unparsed data" status is immediately updated. + assert(sc_hid_event_parser_has_unparsed_data(&hep)); + + sc_hid_event_parser_parse_all_data(&hep); + + // Confirm that once an error is reached, that parsing fails too. + assert(!sc_hid_event_parser_get_next(&hep)); + assert(sc_hid_event_parser_has_error(&hep)); + assert(sc_hid_event_parser_has_unparsed_data(&hep)); + + sc_hid_event_parser_destroy(&hep); +} + +static void test_hid_event_parser_invalid_data(void) { + struct sc_hid_event_parser hep; + sc_hid_event_parser_init(&hep, "invalid_source_data"); + + bool ok = sc_hid_event_parser_append_data(&hep, "", 0); + assert(ok); // No data to append, all right! + sc_hid_event_parser_parse_all_data(&hep); + assert(!sc_hid_event_parser_get_next(&hep)); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(!sc_hid_event_parser_has_unparsed_data(&hep)); + + ok = sc_hid_event_parser_append_data(&hep, "", 0); + assert(ok); // No data to append, still all right! + sc_hid_event_parser_parse_all_data(&hep); + assert(!sc_hid_event_parser_get_next(&hep)); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(!sc_hid_event_parser_has_unparsed_data(&hep)); + + ok = sc_hid_event_parser_append_data(&hep, "Clearly bogus\n", 14); + assert(ok); // Garbage accepted - append does not validate. + sc_hid_event_parser_parse_all_data(&hep); + assert(!sc_hid_event_parser_get_next(&hep)); + assert(sc_hid_event_parser_has_error(&hep)); + assert(sc_hid_event_parser_has_unparsed_data(&hep)); + + ok = sc_hid_event_parser_append_data(&hep, "", 0); + assert(!ok); // No more data accepted when garbage is encountered. + + sc_hid_event_parser_destroy(&hep); +} + +static void test_hid_event_parser_skips_comments_and_lines(void) { + struct sc_hid_event_parser hep; + sc_hid_event_parser_init(&hep, "source_name"); + + bool ok = sc_hid_event_parser_append_data(&hep, "\n\n\n\n\n", 5); + assert(ok); + assert(hep.line == 1); // Not parsed yet. + sc_hid_event_parser_parse_all_data(&hep); + assert(hep.line == 6); // Skipped all blank lines. + assert(!sc_hid_event_parser_get_next(&hep)); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(!sc_hid_event_parser_has_unparsed_data(&hep)); + + ok = sc_hid_event_parser_append_data(&hep, "#", 1); + assert(ok); + sc_hid_event_parser_parse_all_data(&hep); + assert(hep.line == 6); // Line not changed. + assert(!sc_hid_event_parser_get_next(&hep)); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(sc_hid_event_parser_has_unparsed_data(&hep)); + + ok = sc_hid_event_parser_append_data(&hep, "xxx\n", 4); + assert(ok); + assert(hep.line == 6); // Not parsed yet. + sc_hid_event_parser_parse_all_data(&hep); + assert(hep.line == 7); // Line consumed & ignored. + assert(!sc_hid_event_parser_get_next(&hep)); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(!sc_hid_event_parser_has_unparsed_data(&hep)); + + ok = sc_hid_event_parser_append_data(&hep, "#\n", 2); + assert(ok); + sc_hid_event_parser_parse_all_data(&hep); + assert(hep.line == 8); // # immediately followed by \n is also ignored. + assert(!sc_hid_event_parser_get_next(&hep)); + assert(!sc_hid_event_parser_has_error(&hep)); + assert(!sc_hid_event_parser_has_unparsed_data(&hep)); + + sc_hid_event_parser_destroy(&hep); +} + +int main(int argc, char *argv[]) { + (void) argc; + (void) argv; + + test_hid_event_serializer(); + test_hid_event_serializer_only_init_and_destroy(); + test_hid_event_serializer_minimum_length(); + test_hid_event_serializer_maximum_length(); + + test_hid_event_parser(); + test_hid_event_parser_only_init_and_destroy(); + test_hid_event_parser_reject_null_in_input(); + test_hid_event_parser_invalid_data(); + test_hid_event_parser_skips_comments_and_lines(); + + return 0; +}