Archives/scrcpy
2
0
Fork 0
mirror of https://github.com/Genymobile/scrcpy synced 2024-11-17 03:25:38 +00:00
Code Issues Projects Releases Wiki Activity

Handle UHID output

Browse Source 
Use UHID output reports to synchronize CapsLock and VerrNum states.

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>
This commit is contained in:
Simon Chan 2024-01-12 23:32:30 +08:00 committed by Romain Vimont
parent 80644d9133
commit 033f7f1f31
16 changed files with 336 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/meson.build
View File

@ -36,6 +36,7 @@ src = [
'src/trait/frame_source.c',
'src/trait/packet_source.c',
'src/uhid/keyboard_uhid.c',
'src/uhid/uhid_output.c',
'src/util/acksync.c',
'src/util/audiobuf.c',
'src/util/average.c',

6
app/src/controller.c
View File

@ -43,9 +43,11 @@ sc_controller_init(struct sc_controller *controller, sc_socket control_socket) {
}
void
sc_controller_set_acksync(struct sc_controller *controller,
struct sc_acksync *acksync) {
sc_controller_configure(struct sc_controller *controller,
struct sc_acksync *acksync,
struct sc_uhid_devices *uhid_devices) {
controller->receiver.acksync = acksync;
controller->receiver.uhid_devices = uhid_devices;
}
void

5
app/src/controller.h
View File

@ -28,8 +28,9 @@ bool
sc_controller_init(struct sc_controller *controller, sc_socket control_socket);
void
sc_controller_set_acksync(struct sc_controller *controller,
struct sc_acksync *acksync);
sc_controller_configure(struct sc_controller *controller,
struct sc_acksync *acksync,
struct sc_uhid_devices *uhid_devices);
void
sc_controller_destroy(struct sc_controller *controller);

35
app/src/device_msg.c
View File

@ -46,6 +46,31 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len,
msg->ack_clipboard.sequence = sequence;
return 9;
}
case DEVICE_MSG_TYPE_UHID_OUTPUT: {
if (len < 5) {
// at least id + size
return 0; // not available
}
uint16_t id = sc_read16be(&buf[1]);
size_t size = sc_read16be(&buf[3]);
if (size < len - 5) {
return 0; // not available
}
uint8_t *data = malloc(size);
if (!data) {
LOG_OOM();
return -1;
}
if (size) {
memcpy(data, &buf[5], size);
}
msg->uhid_output.id = id;
msg->uhid_output.size = size;
msg->uhid_output.data = data;
return 5 + size;
}
default:
LOGW("Unknown device message type: %d", (int) msg->type);
return -1; // error, we cannot recover
@ -54,7 +79,15 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len,
void
sc_device_msg_destroy(struct sc_device_msg *msg) {
if (msg->type == DEVICE_MSG_TYPE_CLIPBOARD) {
switch (msg->type) {
case DEVICE_MSG_TYPE_CLIPBOARD:
free(msg->clipboard.text);
break;
case DEVICE_MSG_TYPE_UHID_OUTPUT:
free(msg->uhid_output.data);
break;
default:
// nothing to do
break;
}
}

6
app/src/device_msg.h
View File

@ -14,6 +14,7 @@
enum sc_device_msg_type {
DEVICE_MSG_TYPE_CLIPBOARD,
DEVICE_MSG_TYPE_ACK_CLIPBOARD,
DEVICE_MSG_TYPE_UHID_OUTPUT,
};
struct sc_device_msg {
@ -25,6 +26,11 @@ struct sc_device_msg {
struct {
uint64_t sequence;
} ack_clipboard;
struct {
uint16_t id;
uint16_t size;
uint8_t *data; // owned, to be freed by free()
} uhid_output;
};
};

28
app/src/receiver.c
View File

@ -1,11 +1,13 @@
#include "receiver.h"
#include <assert.h>
#include <inttypes.h>
#include <stdint.h>
#include <SDL2/SDL_clipboard.h>
#include "device_msg.h"
#include "util/log.h"
#include "util/str.h"
bool
sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) {
@ -16,6 +18,7 @@ sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) {
receiver->control_socket = control_socket;
receiver->acksync = NULL;
receiver->uhid_devices = NULL;
return true;
}
@ -47,6 +50,31 @@ process_msg(struct sc_receiver *receiver, struct sc_device_msg *msg) {
msg->ack_clipboard.sequence);
sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);
break;
case DEVICE_MSG_TYPE_UHID_OUTPUT:
if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {
char *hex = sc_str_to_hex_string(msg->uhid_output.data,
msg->uhid_output.size);
if (hex) {
LOGV("UHID output [%" PRIu16 "] %s",
msg->uhid_output.id, hex);
free(hex);
} else {
LOGV("UHID output [%" PRIu16 "] size=%" PRIu16,
msg->uhid_output.id, msg->uhid_output.size);
}
}
assert(receiver->uhid_devices);
struct sc_uhid_receiver *uhid_receiver =
sc_uhid_devices_get_receiver(receiver->uhid_devices,
msg->uhid_output.id);
if (uhid_receiver) {
uhid_receiver->ops->process_output(uhid_receiver,
msg->uhid_output.data,
msg->uhid_output.size);
} else {
LOGW("No UHID receiver for id %" PRIu16, msg->uhid_output.id);
}
break;
}
}

2
app/src/receiver.h
View File

@ -5,6 +5,7 @@
#include <stdbool.h>
#include "uhid/uhid_output.h"
#include "util/acksync.h"
#include "util/net.h"
#include "util/thread.h"
@ -17,6 +18,7 @@ struct sc_receiver {
sc_mutex mutex;
struct sc_acksync *acksync;
struct sc_uhid_devices *uhid_devices;
};
bool

8
app/src/scrcpy.c
View File

@ -62,6 +62,7 @@ struct scrcpy {
struct sc_aoa aoa;
// sequence/ack helper to synchronize clipboard and Ctrl+v via HID
struct sc_acksync acksync;
struct sc_uhid_devices uhid_devices;
#endif
union {
struct sc_keyboard_sdk keyboard_sdk;
@ -342,6 +343,7 @@ scrcpy(struct scrcpy_options *options) {
bool timeout_started = false;
struct sc_acksync *acksync = NULL;
struct sc_uhid_devices *uhid_devices = NULL;
uint32_t scid = scrcpy_generate_scid();
@ -666,10 +668,12 @@ aoa_hid_end:
kp = &s->keyboard_sdk.key_processor;
} else if (options->keyboard_input_mode
== SC_KEYBOARD_INPUT_MODE_UHID) {
bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller);
bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller,
&s->uhid_devices);
if (!ok) {
goto end;
}
uhid_devices = &s->uhid_devices;
kp = &s->keyboard_uhid.key_processor;
}
@ -679,7 +683,7 @@ aoa_hid_end:
mp = &s->mouse_sdk.mouse_processor;
}
sc_controller_set_acksync(&s->controller, acksync);
sc_controller_configure(&s->controller, acksync, uhid_devices);
if (!sc_controller_start(&s->controller)) {
goto end;

104
app/src/uhid/keyboard_uhid.c
View File

@ -5,8 +5,52 @@
/** Downcast key processor to keyboard_uhid */
#define DOWNCAST(KP) container_of(KP, struct sc_keyboard_uhid, key_processor)
/** Downcast uhid_receiver to keyboard_uhid */
#define DOWNCAST_RECEIVER(UR) \
container_of(UR, struct sc_keyboard_uhid, uhid_receiver)
#define UHID_KEYBOARD_ID 1
static void
sc_keyboard_uhid_send_input(struct sc_keyboard_uhid *kb,
const struct sc_hid_event *event) {
struct sc_control_msg msg;
msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT;
msg.uhid_input.id = UHID_KEYBOARD_ID;
assert(event->size <= SC_HID_MAX_SIZE);
memcpy(msg.uhid_input.data, event->data, event->size);
msg.uhid_input.size = event->size;
if (!sc_controller_push_msg(kb->controller, &msg)) {
LOGE("Could not send UHID_INPUT message (key)");
}
}
static void
sc_keyboard_uhid_synchronize_mod(struct sc_keyboard_uhid *kb) {
SDL_Keymod sdl_mod = SDL_GetModState();
uint16_t mod = sc_mods_state_from_sdl(sdl_mod) & (SC_MOD_CAPS | SC_MOD_NUM);
uint16_t device_mod =
atomic_load_explicit(&kb->device_mod, memory_order_relaxed);
uint16_t diff = mod ^ device_mod;
if (diff) {
// Inherently racy (the HID output reports arrive asynchronously in
// response to key presses), but will re-synchronize on next key press
// or HID output anyway
atomic_store_explicit(&kb->device_mod, mod, memory_order_relaxed);
struct sc_hid_event hid_event;
sc_hid_keyboard_event_from_mods(&hid_event, diff);
LOGV("HID keyboard state synchronized");
sc_keyboard_uhid_send_input(kb, &hid_event);
}
}
static void
sc_key_processor_process_key(struct sc_key_processor *kp,
const struct sc_key_event *event,
@ -25,26 +69,56 @@ sc_key_processor_process_key(struct sc_key_processor *kp,
// Not all keys are supported, just ignore unsupported keys
if (sc_hid_keyboard_event_from_key(&kb->hid, &hid_event, event)) {
struct sc_control_msg msg;
msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT;
msg.uhid_input.id = UHID_KEYBOARD_ID;
assert(hid_event.size <= SC_HID_MAX_SIZE);
memcpy(msg.uhid_input.data, hid_event.data, hid_event.size);
msg.uhid_input.size = hid_event.size;
if (!sc_controller_push_msg(kb->controller, &msg)) {
LOGE("Could not send UHID_INPUT message (key)");
if (event->scancode == SC_SCANCODE_CAPSLOCK) {
atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_CAPS,
memory_order_relaxed);
} else if (event->scancode == SC_SCANCODE_NUMLOCK) {
atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_NUM,
memory_order_relaxed);
} else {
// Synchronize modifiers (only if the scancode itself does not
// change the modifiers)
sc_keyboard_uhid_synchronize_mod(kb);
}
sc_keyboard_uhid_send_input(kb, &hid_event);
}
}
static unsigned
sc_keyboard_uhid_to_sc_mod(uint8_t hid_led) {
// <https://www.usb.org/sites/default/files/documents/hut1_12v2.pdf>
// (chapter 11: LED page)
unsigned mod = 0;
if (hid_led & 0x01) {
mod |= SC_MOD_NUM;
}
if (hid_led & 0x02) {
mod |= SC_MOD_CAPS;
}
return mod;
}
static void
sc_uhid_receiver_process_output(struct sc_uhid_receiver *receiver,
const uint8_t *data, size_t len) {
// Called from the thread receiving device messages
assert(len);
(void) len;
struct sc_keyboard_uhid *kb = DOWNCAST_RECEIVER(receiver);
uint8_t hid_led = data[0];
uint16_t device_mod = sc_keyboard_uhid_to_sc_mod(hid_led);
atomic_store_explicit(&kb->device_mod, device_mod, memory_order_relaxed);
}
bool
sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
struct sc_controller *controller) {
struct sc_controller *controller,
struct sc_uhid_devices *uhid_devices) {
sc_hid_keyboard_init(&kb->hid);
kb->controller = controller;
atomic_init(&kb->device_mod, 0);
static const struct sc_key_processor_ops ops = {
.process_key = sc_key_processor_process_key,
@ -58,6 +132,14 @@ sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
kb->key_processor.async_paste = false;
kb->key_processor.ops = &ops;
static const struct sc_uhid_receiver_ops uhid_receiver_ops = {
.process_output = sc_uhid_receiver_process_output,
};
kb->uhid_receiver.id = UHID_KEYBOARD_ID;
kb->uhid_receiver.ops = &uhid_receiver_ops;
sc_uhid_devices_add_receiver(uhid_devices, &kb->uhid_receiver);
struct sc_control_msg msg;
msg.type = SC_CONTROL_MSG_TYPE_UHID_CREATE;
msg.uhid_create.id = UHID_KEYBOARD_ID;

6
app/src/uhid/keyboard_uhid.h
View File

@ -7,17 +7,21 @@
#include "controller.h"
#include "hid/hid_keyboard.h"
#include "uhid/uhid_output.h"
#include "trait/key_processor.h"
struct sc_keyboard_uhid {
struct sc_key_processor key_processor; // key processor trait
struct sc_uhid_receiver uhid_receiver;
struct sc_hid_keyboard hid;
struct sc_controller *controller;
atomic_uint_least16_t device_mod;
};
bool
sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
struct sc_controller *controller);
struct sc_controller *controller,
struct sc_uhid_devices *uhid_devices);
#endif

23
app/tests/test_device_msg_deserialize.c
View File

@ -61,6 +61,28 @@ static void test_deserialize_ack_set_clipboard(void) {
assert(msg.ack_clipboard.sequence == UINT64_C(0x0102030405060708));
}
static void test_deserialize_uhid_output(void) {
const uint8_t input[] = {
DEVICE_MSG_TYPE_UHID_OUTPUT,
0, 42, // id
0, 5, // size
0x01, 0x02, 0x03, 0x04, 0x05, // data
};
struct sc_device_msg msg;
ssize_t r = sc_device_msg_deserialize(input, sizeof(input), &msg);
assert(r == 10);
assert(msg.type == DEVICE_MSG_TYPE_UHID_OUTPUT);
assert(msg.uhid_output.id == 42);
assert(msg.uhid_output.size == 5);
uint8_t expected[] = {1, 2, 3, 4, 5};
assert(!memcmp(msg.uhid_output.data, expected, sizeof(expected)));
sc_device_msg_destroy(&msg);
}
int main(int argc, char *argv[]) {
(void) argc;
(void) argv;
@ -68,5 +90,6 @@ int main(int argc, char *argv[]) {
test_deserialize_clipboard();
test_deserialize_clipboard_big();
test_deserialize_ack_set_clipboard();
test_deserialize_uhid_output();
return 0;
}

2
server/src/main/java/com/genymobile/scrcpy/Controller.java
View File

@ -52,7 +52,7 @@ public class Controller implements AsyncProcessor {
this.powerOn = powerOn;
initPointers();
sender = new DeviceMessageSender(controlChannel);
uhidManager = new UhidManager();
uhidManager = new UhidManager(sender);
}
private void initPointers() {

19
server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java
View File

@ -4,10 +4,13 @@ public final class DeviceMessage {
public static final int TYPE_CLIPBOARD = 0;
public static final int TYPE_ACK_CLIPBOARD = 1;
public static final int TYPE_UHID_OUTPUT = 2;
private int type;
private String text;
private long sequence;
private int id;
private byte[] data;
private DeviceMessage() {
}
@ -26,6 +29,14 @@ public final class DeviceMessage {
return event;
}
public static DeviceMessage createUhidOutput(int id, byte[] data) {
DeviceMessage event = new DeviceMessage();
event.type = TYPE_UHID_OUTPUT;
event.id = id;
event.data = data;
return event;
}
public int getType() {
return type;
}
@ -37,4 +48,12 @@ public final class DeviceMessage {
public long getSequence() {
return sequence;
}
public int getId() {
return id;
}
public byte[] getData() {
return data;
}
}

7
server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java
View File

@ -29,6 +29,13 @@ public class DeviceMessageWriter {
buffer.putLong(msg.getSequence());
output.write(rawBuffer, 0, buffer.position());
break;
case DeviceMessage.TYPE_UHID_OUTPUT:
buffer.putShort((short) msg.getId());
byte[] data = msg.getData();
buffer.putShort((short) data.length);
buffer.put(data);
output.write(rawBuffer, 0, buffer.position());
break;
default:
Ln.w("Unknown device message: " + msg.getType());
break;

80
server/src/main/java/com/genymobile/scrcpy/UhidManager.java
View File

@ -1,5 +1,8 @@
package com.genymobile.scrcpy;
import android.os.Build;
import android.os.HandlerThread;
import android.os.MessageQueue;
import android.system.ErrnoException;
import android.system.Os;
import android.system.OsConstants;
@ -7,6 +10,7 @@ import android.util.ArrayMap;
import java.io.FileDescriptor;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
@ -14,13 +18,31 @@ import java.nio.charset.StandardCharsets;
public final class UhidManager {
// Linux: include/uapi/linux/uhid.h
private static final int UHID_OUTPUT = 6;
private static final int UHID_CREATE2 = 11;
private static final int UHID_INPUT2 = 12;
// Linux: include/uapi/linux/input.h
private static final short BUS_VIRTUAL = 0x06;
private static final int SIZE_OF_UHID_EVENT = 4380; // sizeof(struct uhid_event)
private final ArrayMap<Integer, FileDescriptor> fds = new ArrayMap<>();
private final ByteBuffer buffer = ByteBuffer.allocate(SIZE_OF_UHID_EVENT).order(ByteOrder.nativeOrder());
private final DeviceMessageSender sender;
private final HandlerThread thread = new HandlerThread("UHidManager");
private final MessageQueue queue;
public UhidManager(DeviceMessageSender sender) {
this.sender = sender;
thread.start();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
queue = thread.getLooper().getQueue();
} else {
queue = null;
}
}
public void open(int id, byte[] reportDesc) throws IOException {
try {
@ -34,6 +56,8 @@ public final class UhidManager {
byte[] req = buildUhidCreate2Req(reportDesc);
Os.write(fd, req, 0, req.length);
registerUhidListener(id, fd);
} catch (Exception e) {
close(fd);
throw e;
@ -43,6 +67,62 @@ public final class UhidManager {
}
}
private void registerUhidListener(int id, FileDescriptor fd) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
queue.addOnFileDescriptorEventListener(fd, MessageQueue.OnFileDescriptorEventListener.EVENT_INPUT, (fd2, events) -> {
try {
buffer.clear();
int r = Os.read(fd2, buffer);
buffer.flip();
if (r > 0) {
int type = buffer.getInt();
if (type == UHID_OUTPUT) {
byte[] data = extractHidOutputData(buffer);
if (data != null) {
DeviceMessage msg = DeviceMessage.createUhidOutput(id, data);
sender.send(msg);
}
}
}
} catch (ErrnoException | InterruptedIOException e) {
Ln.e("Failed to read UHID output", e);
return 0;
}
return events;
});
}
}
private static byte[] extractHidOutputData(ByteBuffer buffer) {
/*
* #define UHID_DATA_MAX 4096
* struct uhid_event {
* uint32_t type;
* union {
* // ...
* struct uhid_output_req {
* __u8 data[UHID_DATA_MAX];
* __u16 size;
* __u8 rtype;
* };
* };
* } __attribute__((__packed__));
*/
if (buffer.remaining() < 4099) {
Ln.w("Incomplete HID output");
return null;
}
int size = buffer.getShort(buffer.position() + 4096) & 0xFFFF;
if (size > 4096) {
Ln.w("Incorrect HID output size: " + size);
return null;
}
byte[] data = new byte[size];
buffer.get(data);
return data;
}
public void writeInput(int id, byte[] data) throws IOException {
FileDescriptor fd = fds.get(id);
if (fd == null) {

23
server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java
View File

@ -52,4 +52,27 @@ public class DeviceMessageWriterTest {
Assert.assertArrayEquals(expected, actual);
}
@Test
public void testSerializeUhidOutput() throws IOException {
DeviceMessageWriter writer = new DeviceMessageWriter();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bos);
dos.writeByte(DeviceMessage.TYPE_UHID_OUTPUT);
dos.writeShort(42); // id
byte[] data = {1, 2, 3, 4, 5};
dos.writeShort(data.length);
dos.write(data);
byte[] expected = bos.toByteArray();
DeviceMessage msg = DeviceMessage.createUhidOutput(42, data);
bos = new ByteArrayOutputStream();
writer.writeTo(msg, bos);
byte[] actual = bos.toByteArray();
Assert.assertArrayEquals(expected, actual);
}
}