Handle UHID output

Use UHID output reports to synchronize CapsLock and VerrNum states.

PR #4473 <https://github.com/Genymobile/scrcpy/pull/4473>

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>
uhid.38
Simon Chan 4 months ago committed by Romain Vimont
parent 021c5d371a
commit 87da68ee0d

@ -36,6 +36,7 @@ src = [
'src/trait/frame_source.c',
'src/trait/packet_source.c',
'src/uhid/keyboard_uhid.c',
'src/uhid/uhid_output.c',
'src/util/acksync.c',
'src/util/audiobuf.c',
'src/util/average.c',

@ -43,9 +43,11 @@ sc_controller_init(struct sc_controller *controller, sc_socket control_socket) {
}
void
sc_controller_set_acksync(struct sc_controller *controller,
struct sc_acksync *acksync) {
sc_controller_configure(struct sc_controller *controller,
struct sc_acksync *acksync,
struct sc_uhid_devices *uhid_devices) {
controller->receiver.acksync = acksync;
controller->receiver.uhid_devices = uhid_devices;
}
void

@ -28,8 +28,9 @@ bool
sc_controller_init(struct sc_controller *controller, sc_socket control_socket);
void
sc_controller_set_acksync(struct sc_controller *controller,
struct sc_acksync *acksync);
sc_controller_configure(struct sc_controller *controller,
struct sc_acksync *acksync,
struct sc_uhid_devices *uhid_devices);
void
sc_controller_destroy(struct sc_controller *controller);

@ -46,6 +46,31 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len,
msg->ack_clipboard.sequence = sequence;
return 9;
}
case DEVICE_MSG_TYPE_UHID_OUTPUT: {
if (len < 5) {
// at least id + size
return 0; // not available
}
uint16_t id = sc_read16be(&buf[1]);
size_t size = sc_read16be(&buf[3]);
if (size < len - 5) {
return 0; // not available
}
uint8_t *data = malloc(size);
if (!data) {
LOG_OOM();
return -1;
}
if (size) {
memcpy(data, &buf[5], size);
}
msg->uhid_output.id = id;
msg->uhid_output.size = size;
msg->uhid_output.data = data;
return 5 + size;
}
default:
LOGW("Unknown device message type: %d", (int) msg->type);
return -1; // error, we cannot recover
@ -54,7 +79,15 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len,
void
sc_device_msg_destroy(struct sc_device_msg *msg) {
if (msg->type == DEVICE_MSG_TYPE_CLIPBOARD) {
free(msg->clipboard.text);
switch (msg->type) {
case DEVICE_MSG_TYPE_CLIPBOARD:
free(msg->clipboard.text);
break;
case DEVICE_MSG_TYPE_UHID_OUTPUT:
free(msg->uhid_output.data);
break;
default:
// nothing to do
break;
}
}

@ -14,6 +14,7 @@
enum sc_device_msg_type {
DEVICE_MSG_TYPE_CLIPBOARD,
DEVICE_MSG_TYPE_ACK_CLIPBOARD,
DEVICE_MSG_TYPE_UHID_OUTPUT,
};
struct sc_device_msg {
@ -25,6 +26,11 @@ struct sc_device_msg {
struct {
uint64_t sequence;
} ack_clipboard;
struct {
uint16_t id;
uint16_t size;
uint8_t *data; // owned, to be freed by free()
} uhid_output;
};
};

@ -1,11 +1,13 @@
#include "receiver.h"
#include <assert.h>
#include <inttypes.h>
#include <stdint.h>
#include <SDL2/SDL_clipboard.h>
#include "device_msg.h"
#include "util/log.h"
#include "util/str.h"
bool
sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) {
@ -16,6 +18,7 @@ sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) {
receiver->control_socket = control_socket;
receiver->acksync = NULL;
receiver->uhid_devices = NULL;
return true;
}
@ -57,6 +60,41 @@ process_msg(struct sc_receiver *receiver, struct sc_device_msg *msg) {
sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);
break;
case DEVICE_MSG_TYPE_UHID_OUTPUT:
if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {
char *hex = sc_str_to_hex_string(msg->uhid_output.data,
msg->uhid_output.size);
if (hex) {
LOGV("UHID output [%" PRIu16 "] %s",
msg->uhid_output.id, hex);
free(hex);
} else {
LOGV("UHID output [%" PRIu16 "] size=%" PRIu16,
msg->uhid_output.id, msg->uhid_output.size);
}
}
// This is a programming error to receive this message if there is
// no uhid_devices instance
assert(receiver->uhid_devices);
// Also check at runtime (do not trust the server)
if (!receiver->uhid_devices) {
LOGE("Received unexpected HID output message");
return;
}
struct sc_uhid_receiver *uhid_receiver =
sc_uhid_devices_get_receiver(receiver->uhid_devices,
msg->uhid_output.id);
if (uhid_receiver) {
uhid_receiver->ops->process_output(uhid_receiver,
msg->uhid_output.data,
msg->uhid_output.size);
} else {
LOGW("No UHID receiver for id %" PRIu16, msg->uhid_output.id);
}
break;
}
}

@ -5,6 +5,7 @@
#include <stdbool.h>
#include "uhid/uhid_output.h"
#include "util/acksync.h"
#include "util/net.h"
#include "util/thread.h"
@ -17,6 +18,7 @@ struct sc_receiver {
sc_mutex mutex;
struct sc_acksync *acksync;
struct sc_uhid_devices *uhid_devices;
};
bool

@ -62,6 +62,7 @@ struct scrcpy {
struct sc_aoa aoa;
// sequence/ack helper to synchronize clipboard and Ctrl+v via HID
struct sc_acksync acksync;
struct sc_uhid_devices uhid_devices;
#endif
union {
struct sc_keyboard_sdk keyboard_sdk;
@ -342,6 +343,7 @@ scrcpy(struct scrcpy_options *options) {
bool timeout_started = false;
struct sc_acksync *acksync = NULL;
struct sc_uhid_devices *uhid_devices = NULL;
uint32_t scid = scrcpy_generate_scid();
@ -666,10 +668,13 @@ aoa_hid_end:
kp = &s->keyboard_sdk.key_processor;
} else if (options->keyboard_input_mode
== SC_KEYBOARD_INPUT_MODE_UHID) {
bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller);
sc_uhid_devices_init(&s->uhid_devices);
bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller,
&s->uhid_devices);
if (!ok) {
goto end;
}
uhid_devices = &s->uhid_devices;
kp = &s->keyboard_uhid.key_processor;
}
@ -679,7 +684,7 @@ aoa_hid_end:
mp = &s->mouse_sdk.mouse_processor;
}
sc_controller_set_acksync(&s->controller, acksync);
sc_controller_configure(&s->controller, acksync, uhid_devices);
if (!sc_controller_start(&s->controller)) {
goto end;

@ -5,8 +5,52 @@
/** Downcast key processor to keyboard_uhid */
#define DOWNCAST(KP) container_of(KP, struct sc_keyboard_uhid, key_processor)
/** Downcast uhid_receiver to keyboard_uhid */
#define DOWNCAST_RECEIVER(UR) \
container_of(UR, struct sc_keyboard_uhid, uhid_receiver)
#define UHID_KEYBOARD_ID 1
static void
sc_keyboard_uhid_send_input(struct sc_keyboard_uhid *kb,
const struct sc_hid_event *event) {
struct sc_control_msg msg;
msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT;
msg.uhid_input.id = UHID_KEYBOARD_ID;
assert(event->size <= SC_HID_MAX_SIZE);
memcpy(msg.uhid_input.data, event->data, event->size);
msg.uhid_input.size = event->size;
if (!sc_controller_push_msg(kb->controller, &msg)) {
LOGE("Could not send UHID_INPUT message (key)");
}
}
static void
sc_keyboard_uhid_synchronize_mod(struct sc_keyboard_uhid *kb) {
SDL_Keymod sdl_mod = SDL_GetModState();
uint16_t mod = sc_mods_state_from_sdl(sdl_mod) & (SC_MOD_CAPS | SC_MOD_NUM);
uint16_t device_mod =
atomic_load_explicit(&kb->device_mod, memory_order_relaxed);
uint16_t diff = mod ^ device_mod;
if (diff) {
// Inherently racy (the HID output reports arrive asynchronously in
// response to key presses), but will re-synchronize on next key press
// or HID output anyway
atomic_store_explicit(&kb->device_mod, mod, memory_order_relaxed);
struct sc_hid_event hid_event;
sc_hid_keyboard_event_from_mods(&hid_event, diff);
LOGV("HID keyboard state synchronized");
sc_keyboard_uhid_send_input(kb, &hid_event);
}
}
static void
sc_key_processor_process_key(struct sc_key_processor *kp,
const struct sc_key_event *event,
@ -25,26 +69,63 @@ sc_key_processor_process_key(struct sc_key_processor *kp,
// Not all keys are supported, just ignore unsupported keys
if (sc_hid_keyboard_event_from_key(&kb->hid, &hid_event, event)) {
struct sc_control_msg msg;
msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT;
msg.uhid_input.id = UHID_KEYBOARD_ID;
if (event->scancode == SC_SCANCODE_CAPSLOCK) {
atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_CAPS,
memory_order_relaxed);
} else if (event->scancode == SC_SCANCODE_NUMLOCK) {
atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_NUM,
memory_order_relaxed);
} else {
// Synchronize modifiers (only if the scancode itself does not
// change the modifiers)
sc_keyboard_uhid_synchronize_mod(kb);
}
sc_keyboard_uhid_send_input(kb, &hid_event);
}
}
assert(hid_event.size <= SC_HID_MAX_SIZE);
memcpy(msg.uhid_input.data, hid_event.data, hid_event.size);
msg.uhid_input.size = hid_event.size;
static unsigned
sc_keyboard_uhid_to_sc_mod(uint8_t hid_led) {
// <https://www.usb.org/sites/default/files/documents/hut1_12v2.pdf>
// (chapter 11: LED page)
unsigned mod = 0;
if (hid_led & 0x01) {
mod |= SC_MOD_NUM;
}
if (hid_led & 0x02) {
mod |= SC_MOD_CAPS;
}
return mod;
}
if (!sc_controller_push_msg(kb->controller, &msg)) {
LOGE("Could not send UHID_INPUT message (key)");
}
static void
sc_uhid_receiver_process_output(struct sc_uhid_receiver *receiver,
const uint8_t *data, size_t len) {
// Called from the thread receiving device messages
assert(len);
// Also check at runtime (do not trust the server)
if (!len) {
LOGE("Unexpected empty HID output message");
return;
}
struct sc_keyboard_uhid *kb = DOWNCAST_RECEIVER(receiver);
uint8_t hid_led = data[0];
uint16_t device_mod = sc_keyboard_uhid_to_sc_mod(hid_led);
atomic_store_explicit(&kb->device_mod, device_mod, memory_order_relaxed);
}
bool
sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
struct sc_controller *controller) {
struct sc_controller *controller,
struct sc_uhid_devices *uhid_devices) {
sc_hid_keyboard_init(&kb->hid);
kb->controller = controller;
atomic_init(&kb->device_mod, 0);
static const struct sc_key_processor_ops ops = {
.process_key = sc_key_processor_process_key,
@ -58,6 +139,14 @@ sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
kb->key_processor.async_paste = false;
kb->key_processor.ops = &ops;
static const struct sc_uhid_receiver_ops uhid_receiver_ops = {
.process_output = sc_uhid_receiver_process_output,
};
kb->uhid_receiver.id = UHID_KEYBOARD_ID;
kb->uhid_receiver.ops = &uhid_receiver_ops;
sc_uhid_devices_add_receiver(uhid_devices, &kb->uhid_receiver);
struct sc_control_msg msg;
msg.type = SC_CONTROL_MSG_TYPE_UHID_CREATE;
msg.uhid_create.id = UHID_KEYBOARD_ID;

@ -7,17 +7,21 @@
#include "controller.h"
#include "hid/hid_keyboard.h"
#include "uhid/uhid_output.h"
#include "trait/key_processor.h"
struct sc_keyboard_uhid {
struct sc_key_processor key_processor; // key processor trait
struct sc_uhid_receiver uhid_receiver;
struct sc_hid_keyboard hid;
struct sc_controller *controller;
atomic_uint_least16_t device_mod;
};
bool
sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
struct sc_controller *controller);
struct sc_controller *controller,
struct sc_uhid_devices *uhid_devices);
#endif

@ -0,0 +1,25 @@
#include "uhid_output.h"
#include <assert.h>
void
sc_uhid_devices_init(struct sc_uhid_devices *devices) {
devices->count = 0;
}
void
sc_uhid_devices_add_receiver(struct sc_uhid_devices *devices,
struct sc_uhid_receiver *receiver) {
assert(devices->count < SC_UHID_MAX_RECEIVERS);
devices->receivers[devices->count++] = receiver;
}
struct sc_uhid_receiver *
sc_uhid_devices_get_receiver(struct sc_uhid_devices *devices, uint16_t id) {
for (size_t i = 0; i < devices->count; ++i) {
if (devices->receivers[i]->id == id) {
return devices->receivers[i];
}
}
return NULL;
}

@ -0,0 +1,45 @@
#ifndef SC_UHID_OUTPUT_H
#define SC_UHID_OUTPUT_H
#include "common.h"
#include <stdbool.h>
#include <stdint.h>
/**
* The communication with UHID devices is bidirectional.
*
* This component manages the registration of receivers to handle UHID output
* messages (sent from the device to the computer).
*/
struct sc_uhid_receiver {
uint16_t id;
const struct sc_uhid_receiver_ops *ops;
};
struct sc_uhid_receiver_ops {
void
(*process_output)(struct sc_uhid_receiver *receiver,
const uint8_t *data, size_t len);
};
#define SC_UHID_MAX_RECEIVERS 1
struct sc_uhid_devices {
struct sc_uhid_receiver *receivers[SC_UHID_MAX_RECEIVERS];
unsigned count;
};
void
sc_uhid_devices_init(struct sc_uhid_devices *devices);
void
sc_uhid_devices_add_receiver(struct sc_uhid_devices *devices,
struct sc_uhid_receiver *receiver);
struct sc_uhid_receiver *
sc_uhid_devices_get_receiver(struct sc_uhid_devices *devices, uint16_t id);
#endif

@ -61,6 +61,28 @@ static void test_deserialize_ack_set_clipboard(void) {
assert(msg.ack_clipboard.sequence == UINT64_C(0x0102030405060708));
}
static void test_deserialize_uhid_output(void) {
const uint8_t input[] = {
DEVICE_MSG_TYPE_UHID_OUTPUT,
0, 42, // id
0, 5, // size
0x01, 0x02, 0x03, 0x04, 0x05, // data
};
struct sc_device_msg msg;
ssize_t r = sc_device_msg_deserialize(input, sizeof(input), &msg);
assert(r == 10);
assert(msg.type == DEVICE_MSG_TYPE_UHID_OUTPUT);
assert(msg.uhid_output.id == 42);
assert(msg.uhid_output.size == 5);
uint8_t expected[] = {1, 2, 3, 4, 5};
assert(!memcmp(msg.uhid_output.data, expected, sizeof(expected)));
sc_device_msg_destroy(&msg);
}
int main(int argc, char *argv[]) {
(void) argc;
(void) argv;
@ -68,5 +90,6 @@ int main(int argc, char *argv[]) {
test_deserialize_clipboard();
test_deserialize_clipboard_big();
test_deserialize_ack_set_clipboard();
test_deserialize_uhid_output();
return 0;
}

@ -52,7 +52,7 @@ public class Controller implements AsyncProcessor {
this.powerOn = powerOn;
initPointers();
sender = new DeviceMessageSender(controlChannel);
uhidManager = new UhidManager();
uhidManager = new UhidManager(sender);
}
private void initPointers() {

@ -4,10 +4,13 @@ public final class DeviceMessage {
public static final int TYPE_CLIPBOARD = 0;
public static final int TYPE_ACK_CLIPBOARD = 1;
public static final int TYPE_UHID_OUTPUT = 2;
private int type;
private String text;
private long sequence;
private int id;
private byte[] data;
private DeviceMessage() {
}
@ -26,6 +29,14 @@ public final class DeviceMessage {
return event;
}
public static DeviceMessage createUhidOutput(int id, byte[] data) {
DeviceMessage event = new DeviceMessage();
event.type = TYPE_UHID_OUTPUT;
event.id = id;
event.data = data;
return event;
}
public int getType() {
return type;
}
@ -37,4 +48,12 @@ public final class DeviceMessage {
public long getSequence() {
return sequence;
}
public int getId() {
return id;
}
public byte[] getData() {
return data;
}
}

@ -29,6 +29,13 @@ public class DeviceMessageWriter {
buffer.putLong(msg.getSequence());
output.write(rawBuffer, 0, buffer.position());
break;
case DeviceMessage.TYPE_UHID_OUTPUT:
buffer.putShort((short) msg.getId());
byte[] data = msg.getData();
buffer.putShort((short) data.length);
buffer.put(data);
output.write(rawBuffer, 0, buffer.position());
break;
default:
Ln.w("Unknown device message: " + msg.getType());
break;

@ -1,5 +1,8 @@
package com.genymobile.scrcpy;
import android.os.Build;
import android.os.HandlerThread;
import android.os.MessageQueue;
import android.system.ErrnoException;
import android.system.Os;
import android.system.OsConstants;
@ -7,6 +10,7 @@ import android.util.ArrayMap;
import java.io.FileDescriptor;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
@ -14,13 +18,31 @@ import java.nio.charset.StandardCharsets;
public final class UhidManager {
// Linux: include/uapi/linux/uhid.h
private static final int UHID_OUTPUT = 6;
private static final int UHID_CREATE2 = 11;
private static final int UHID_INPUT2 = 12;
// Linux: include/uapi/linux/input.h
private static final short BUS_VIRTUAL = 0x06;
private static final int SIZE_OF_UHID_EVENT = 4380; // sizeof(struct uhid_event)
private final ArrayMap<Integer, FileDescriptor> fds = new ArrayMap<>();
private final ByteBuffer buffer = ByteBuffer.allocate(SIZE_OF_UHID_EVENT).order(ByteOrder.nativeOrder());
private final DeviceMessageSender sender;
private final HandlerThread thread = new HandlerThread("UHidManager");
private final MessageQueue queue;
public UhidManager(DeviceMessageSender sender) {
this.sender = sender;
thread.start();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
queue = thread.getLooper().getQueue();
} else {
queue = null;
}
}
public void open(int id, byte[] reportDesc) throws IOException {
try {
@ -34,6 +56,8 @@ public final class UhidManager {
byte[] req = buildUhidCreate2Req(reportDesc);
Os.write(fd, req, 0, req.length);
registerUhidListener(id, fd);
} catch (Exception e) {
close(fd);
throw e;
@ -43,6 +67,62 @@ public final class UhidManager {
}
}
private void registerUhidListener(int id, FileDescriptor fd) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
queue.addOnFileDescriptorEventListener(fd, MessageQueue.OnFileDescriptorEventListener.EVENT_INPUT, (fd2, events) -> {
try {
buffer.clear();
int r = Os.read(fd2, buffer);
buffer.flip();
if (r > 0) {
int type = buffer.getInt();
if (type == UHID_OUTPUT) {
byte[] data = extractHidOutputData(buffer);
if (data != null) {
DeviceMessage msg = DeviceMessage.createUhidOutput(id, data);
sender.send(msg);
}
}
}
} catch (ErrnoException | InterruptedIOException e) {
Ln.e("Failed to read UHID output", e);
return 0;
}
return events;
});
}
}
private static byte[] extractHidOutputData(ByteBuffer buffer) {
/*
* #define UHID_DATA_MAX 4096
* struct uhid_event {
* uint32_t type;
* union {
* // ...
* struct uhid_output_req {
* __u8 data[UHID_DATA_MAX];
* __u16 size;
* __u8 rtype;
* };
* };
* } __attribute__((__packed__));
*/
if (buffer.remaining() < 4099) {
Ln.w("Incomplete HID output");
return null;
}
int size = buffer.getShort(buffer.position() + 4096) & 0xFFFF;
if (size > 4096) {
Ln.w("Incorrect HID output size: " + size);
return null;
}
byte[] data = new byte[size];
buffer.get(data);
return data;
}
public void writeInput(int id, byte[] data) throws IOException {
FileDescriptor fd = fds.get(id);
if (fd == null) {

@ -52,4 +52,27 @@ public class DeviceMessageWriterTest {
Assert.assertArrayEquals(expected, actual);
}
@Test
public void testSerializeUhidOutput() throws IOException {
DeviceMessageWriter writer = new DeviceMessageWriter();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bos);
dos.writeByte(DeviceMessage.TYPE_UHID_OUTPUT);
dos.writeShort(42); // id
byte[] data = {1, 2, 3, 4, 5};
dos.writeShort(data.length);
dos.write(data);
byte[] expected = bos.toByteArray();
DeviceMessage msg = DeviceMessage.createUhidOutput(42, data);
bos = new ByteArrayOutputStream();
writer.writeTo(msg, bos);
byte[] actual = bos.toByteArray();
Assert.assertArrayEquals(expected, actual);
}
}

Loading…
Cancel
Save