Support USB HID over AoAv2 mode for keyboard input

This provides a better input experience via
simulate physical keyboard, it converts
SDL_KeyboardEvent to proper HID events and send it
via HID over AoAv2.

This is a rewriting and bugfix of the origin code
from [@amosbird](https://github.com/amosbird).

Make sdl_keymod_to_hid_modifiers() more readable

Support MOD keys in HID mode

Enable Ctrl+V on HID mode

Support to send media events from hid_keyboard

Use existing --serial to replace --usb

Use explict option for input mode

Move HID keyboard setup code into functions

Send HID events in separated thread

Let libusb handle max package size

Fix HID keyboard report desc
This commit is contained in:
Alynx Zhou
2021-09-10 18:57:35 +08:00
committed by Romain Vimont
parent 0be11fa2e2
commit 99acc3e910
11 changed files with 933 additions and 13 deletions

View File

@@ -88,11 +88,12 @@ Install the required packages from your package manager.
```bash
# runtime dependencies
sudo apt install ffmpeg libsdl2-2.0-0 adb
sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0
# client build dependencies
sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \
libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev\
libusb-dev
# server build dependencies
sudo apt install openjdk-11-jdk
@@ -114,7 +115,7 @@ pip3 install meson
sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm
# client build dependencies
sudo dnf install SDL2-devel ffms2-devel meson gcc make
sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make
# server build dependencies
sudo dnf install java-devel
@@ -159,7 +160,8 @@ install the required packages:
```bash
# runtime dependencies
pacman -S mingw-w64-x86_64-SDL2 \
mingw-w64-x86_64-ffmpeg
mingw-w64-x86_64-ffmpeg \
mingw-w64-x86_64-libusb
# client build dependencies
pacman -S mingw-w64-x86_64-make \
@@ -173,7 +175,8 @@ For a 32 bits version, replace `x86_64` by `i686`:
```bash
# runtime dependencies
pacman -S mingw-w64-i686-SDL2 \
mingw-w64-i686-ffmpeg
mingw-w64-i686-ffmpeg \
mingw-w64-i686-libusb
# client build dependencies
pacman -S mingw-w64-i686-make \
@@ -197,7 +200,7 @@ Install the packages with [Homebrew]:
```bash
# runtime dependencies
brew install sdl2 ffmpeg
brew install sdl2 ffmpeg libusb
# client build dependencies
brew install pkg-config meson

View File

@@ -207,6 +207,29 @@ scrcpy --crop 1224:1440:0:0 # 1224x1440 at offset (0,0)
If `--max-size` is also specified, resizing is applied after cropping.
#### USB HID over AOAv2
Scrcpy can simulate a USB physical keyboard on Android to provide better input
experience, you need to connect your device via USB, not wireless.
However, due to some limitation of libusb and WinUSB driver, you cannot use HID
over AOAv2 on Windows.
Currently a USB serial number is needed to use HID over AOAv2.
```bash
scrcpy --serial XXXXXXXXXXXXXXXX # don't use HID
scrcpy --serial XXXXXXXXXXXXXXXX --input-mode inject # don't use HID
scrcpy --serial XXXXXXXXXXXXXXXX --input-mode hid # try HID and exit if failed
```
Serial number can be found by `adb get-serialno`.
If you are a non-QWERTY keyboard user and using HID mode, please remember to set
correct physical keyboard layout manually in Android settings, because scrcpy
just forwards scancodes to Android device and Android system is responsible for
converting scancodes to correct keycode on Android device (your system does this
on your PC).
#### Lock video orientation

View File

@@ -42,6 +42,14 @@ if v4l2_support
src += [ 'src/v4l2_sink.c' ]
endif
aoa_hid_support = host_machine.system() == 'linux'
if aoa_hid_support
src += [
'src/aoa_hid.c',
'src/hid_keyboard.c',
]
endif
check_functions = [
'strdup'
]
@@ -62,8 +70,11 @@ if not get_option('crossbuild_windows')
dependencies += dependency('libavdevice')
endif
else
if aoa_hid_support
dependencies += dependency('libusb-1.0')
endif
else
# cross-compile mingw32 build (from Linux to Windows)
prebuilt_sdl2 = meson.get_cross_property('prebuilt_sdl2')
sdl2_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_sdl2 + '/bin'
@@ -140,6 +151,9 @@ conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == '
# enable V4L2 support (linux only)
conf.set('HAVE_V4L2', v4l2_support)
# enable HID over AOA support (linux only)
conf.set('HAVE_AOA_HID', aoa_hid_support)
configure_file(configuration: conf, output: 'config.h')
src_dir = include_directories('src')

View File

@@ -82,6 +82,24 @@ Start in fullscreen.
.B \-h, \-\-help
Print this help.
.TP
.B \-K, \-\-keyboard\-hid
Simulate a physical keyboard by using HID over AOAv2.
This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method.
It may only work over USB, and is currently only supported on Linux.
.TP
.B \-i, \-\-input\-mode mode
Select input mode for keyboard events.
Possible values are "hid" and "inject".
"hid" uses Android's USB HID over AOAv2 feature to simulate physical keyboard's events, which provides better experience for IME users if supported by you device.
"inject" is the legacy scrcpy way by injecting keycode events on Android, works on most devices and is the default.
.TP
.B \-\-legacy\-paste
Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).

362
app/src/aoa_hid.c Normal file
View File

@@ -0,0 +1,362 @@
#include "util/log.h"
#include <assert.h>
#include <stdio.h>
#include "aoa_hid.h"
// See <https://source.android.com/devices/accessories/aoa2#hid-support>.
#define ACCESSORY_REGISTER_HID 54
#define ACCESSORY_SET_HID_REPORT_DESC 56
#define ACCESSORY_SEND_HID_EVENT 57
#define ACCESSORY_UNREGISTER_HID 55
#define DEFAULT_TIMEOUT 1000
static void
hid_event_log(const struct hid_event *event) {
// HID Event: [00] FF FF FF FF...
assert(event->size);
unsigned buffer_size = event->size * 3 + 1;
char *buffer = malloc(buffer_size);
if (!buffer) {
return;
}
for (unsigned i = 0; i < event->size; ++i) {
snprintf(buffer + i * 3, 4, " %02x", event->buffer[i]);
}
LOGV("HID Event: [%d]%s", event->from_accessory_id, buffer);
free(buffer);
}
static void
hid_event_destroy(struct hid_event *event) {
free(event->buffer);
}
static void
log_libusb_error(enum libusb_error errcode) {
LOGW("libusb error: %s", libusb_strerror(errcode));
}
static bool
accept_device(libusb_device *device, const char *serial) {
// do not log any USB error in this function, it is expected that many USB
// devices available on the computer have permission restrictions
struct libusb_device_descriptor desc;
libusb_get_device_descriptor(device, &desc);
if (!desc.iSerialNumber) {
return false;
}
libusb_device_handle *handle;
int result = libusb_open(device, &handle);
if (result < 0) {
return false;
}
char buffer[128];
result = libusb_get_string_descriptor_ascii(handle, desc.iSerialNumber,
(unsigned char *) buffer,
sizeof(buffer));
libusb_close(handle);
if (result < 0) {
return false;
}
buffer[sizeof(buffer) - 1] = '\0'; // just in case
// accept the device if its serial matches
return !strcmp(buffer, serial);
}
static libusb_device *
aoa_find_usb_device(const char *serial) {
if (!serial) {
return NULL;
}
libusb_device **list;
libusb_device *result = NULL;
ssize_t count = libusb_get_device_list(NULL, &list);
if (count < 0) {
log_libusb_error((enum libusb_error) count);
return NULL;
}
for (ssize_t i = 0; i < count; ++i) {
libusb_device *device = list[i];
if (accept_device(device, serial)) {
result = libusb_ref_device(device);
break;
}
}
libusb_free_device_list(list, 1);
return result;
}
static int
aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {
int result = libusb_open(device, handle);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return result;
}
return 0;
}
bool
aoa_init(struct aoa *aoa, const char *serial) {
cbuf_init(&aoa->queue);
if (!sc_mutex_init(&aoa->mutex)) {
return false;
}
if (!sc_cond_init(&aoa->event_cond)) {
sc_mutex_destroy(&aoa->mutex);
return false;
}
if (libusb_init(&aoa->usb_context) != LIBUSB_SUCCESS) {
sc_cond_destroy(&aoa->event_cond);
sc_mutex_destroy(&aoa->mutex);
return false;
}
aoa->usb_device = aoa_find_usb_device(serial);
if (!aoa->usb_device) {
LOGW("USB device of serial %s not found", serial);
libusb_exit(aoa->usb_context);
sc_mutex_destroy(&aoa->mutex);
sc_cond_destroy(&aoa->event_cond);
return false;
}
if (aoa_open_usb_handle(aoa->usb_device, &aoa->usb_handle) < 0) {
LOGW("Open USB handle failed");
libusb_unref_device(aoa->usb_device);
libusb_exit(aoa->usb_context);
sc_cond_destroy(&aoa->event_cond);
sc_mutex_destroy(&aoa->mutex);
return false;
}
aoa->stopped = false;
return true;
}
void
aoa_destroy(struct aoa *aoa) {
// Destroy remaining events
struct hid_event event;
while (cbuf_take(&aoa->queue, &event)) {
hid_event_destroy(&event);
}
libusb_close(aoa->usb_handle);
libusb_unref_device(aoa->usb_device);
libusb_exit(aoa->usb_context);
sc_cond_destroy(&aoa->event_cond);
sc_mutex_destroy(&aoa->mutex);
}
static bool
aoa_register_hid(struct aoa *aoa, uint16_t accessory_id,
uint16_t report_desc_size) {
uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
uint8_t request = ACCESSORY_REGISTER_HID;
// <https://source.android.com/devices/accessories/aoa2.html#hid-support>
// value (arg0): accessory assigned ID for the HID device
// index (arg1): total length of the HID report descriptor
uint16_t value = accessory_id;
uint16_t index = report_desc_size;
unsigned char *buffer = NULL;
uint16_t length = 0;
int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
value, index, buffer, length,
DEFAULT_TIMEOUT);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return false;
}
return true;
}
static bool
aoa_set_hid_report_desc(struct aoa *aoa, uint16_t accessory_id,
const unsigned char *report_desc,
uint16_t report_desc_size) {
uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
uint8_t request = ACCESSORY_SET_HID_REPORT_DESC;
/**
* If the HID descriptor is longer than the endpoint zero max packet size,
* the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC
* commands. The data for the descriptor must be sent sequentially
* if multiple packets are needed.
* <https://source.android.com/devices/accessories/aoa2.html#hid-support>
*
* libusb handles packet abstraction internally, so we don't need to care
* about bMaxPacketSize0 here.
*
* See <https://libusb.sourceforge.io/api-1.0/libusb_packetoverflow.html>
*/
// value (arg0): accessory assigned ID for the HID device
// index (arg1): offset of data (buffer) in descriptor
uint16_t value = accessory_id;
uint16_t index = 0;
// libusb_control_transfer expects a pointer to non-const
unsigned char *buffer = (unsigned char *) report_desc;
uint16_t length = report_desc_size;
int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
value, index, buffer, length,
DEFAULT_TIMEOUT);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return false;
}
return true;
}
bool
aoa_setup_hid(struct aoa *aoa, uint16_t accessory_id,
const unsigned char *report_desc, uint16_t report_desc_size) {
bool ok = aoa_register_hid(aoa, accessory_id, report_desc_size);
if (!ok) {
return false;
}
ok = aoa_set_hid_report_desc(aoa, accessory_id, report_desc,
report_desc_size);
if (!ok) {
if (!aoa_unregister_hid(aoa, accessory_id)) {
LOGW("Could not unregister HID");
}
return false;
}
return true;
}
static bool
aoa_send_hid_event(struct aoa *aoa, const struct hid_event *event) {
uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
uint8_t request = ACCESSORY_SEND_HID_EVENT;
// <https://source.android.com/devices/accessories/aoa2.html#hid-support>
// value (arg0): accessory assigned ID for the HID device
// index (arg1): 0 (unused)
uint16_t value = event->from_accessory_id;
uint16_t index = 0;
unsigned char *buffer = event->buffer;
uint16_t length = event->size;
int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
value, index, buffer, length,
DEFAULT_TIMEOUT);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return false;
}
return true;
}
bool
aoa_unregister_hid(struct aoa *aoa, const uint16_t accessory_id) {
uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
uint8_t request = ACCESSORY_UNREGISTER_HID;
// <https://source.android.com/devices/accessories/aoa2.html#hid-support>
// value (arg0): accessory assigned ID for the HID device
// index (arg1): 0
uint16_t value = accessory_id;
uint16_t index = 0;
unsigned char *buffer = NULL;
uint16_t length = 0;
int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
value, index, buffer, length,
DEFAULT_TIMEOUT);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return false;
}
return true;
}
bool
aoa_push_hid_event(struct aoa *aoa, const struct hid_event *event) {
hid_event_log(event);
sc_mutex_lock(&aoa->mutex);
bool was_empty = cbuf_is_empty(&aoa->queue);
bool res = cbuf_push(&aoa->queue, *event);
if (was_empty) {
sc_cond_signal(&aoa->event_cond);
}
sc_mutex_unlock(&aoa->mutex);
return res;
}
static inline bool
process_hid_event(struct aoa *aoa, const struct hid_event *event) {
return aoa_send_hid_event(aoa, event);
}
static int
run_aoa_thread(void *data) {
struct aoa *aoa = data;
for (;;) {
sc_mutex_lock(&aoa->mutex);
while (!aoa->stopped && cbuf_is_empty(&aoa->queue)) {
sc_cond_wait(&aoa->event_cond, &aoa->mutex);
}
if (aoa->stopped) {
// Stop immediately, do not process further events
sc_mutex_unlock(&aoa->mutex);
break;
}
struct hid_event event;
bool non_empty = cbuf_take(&aoa->queue, &event);
assert(non_empty);
(void) non_empty;
sc_mutex_unlock(&aoa->mutex);
bool ok = process_hid_event(aoa, &event);
hid_event_destroy(&event);
if (!ok) {
LOGW("Could not send HID event to USB device");
}
}
return 0;
}
bool
aoa_start(struct aoa *aoa) {
LOGD("Starting AOA thread");
bool ok = sc_thread_create(&aoa->thread, run_aoa_thread, "aoa_thread", aoa);
if (!ok) {
LOGC("Could not start AOA thread");
return false;
}
return true;
}
void
aoa_stop(struct aoa *aoa) {
sc_mutex_lock(&aoa->mutex);
aoa->stopped = true;
sc_cond_signal(&aoa->event_cond);
sc_mutex_unlock(&aoa->mutex);
}
void
aoa_join(struct aoa *aoa) {
sc_thread_join(&aoa->thread, NULL);
}

57
app/src/aoa_hid.h Normal file
View File

@@ -0,0 +1,57 @@
#ifndef AOA_HID_H
#define AOA_HID_H
#include <stdint.h>
#include <stdbool.h>
#include <libusb-1.0/libusb.h>
#include "scrcpy.h"
#include "util/cbuf.h"
#include "util/thread.h"
struct hid_event {
uint16_t from_accessory_id;
unsigned char *buffer;
uint16_t size;
};
struct hid_event_queue CBUF(struct hid_event, 64);
struct aoa {
libusb_context *usb_context;
libusb_device *usb_device;
libusb_device_handle *usb_handle;
sc_thread thread;
sc_mutex mutex;
sc_cond event_cond;
bool stopped;
struct hid_event_queue queue;
};
bool
aoa_init(struct aoa *aoa, const char *serial);
void
aoa_destroy(struct aoa *aoa);
bool
aoa_start(struct aoa *aoa);
void
aoa_stop(struct aoa *aoa);
void
aoa_join(struct aoa *aoa);
bool
aoa_setup_hid(struct aoa *aoa, uint16_t accessory_id,
const unsigned char *report_desc, uint16_t report_desc_size);
bool
aoa_unregister_hid(struct aoa *aoa, uint16_t accessory_id);
bool
aoa_push_hid_event(struct aoa *aoa, const struct hid_event *event);
#endif

View File

@@ -76,6 +76,14 @@ scrcpy_print_usage(const char *arg0) {
" -f, --fullscreen\n"
" Start in fullscreen.\n"
"\n"
" -K, --keyboard-hid\n"
" Simulate a physical keyboard by using HID over AOAv2.\n"
" It provides a better experience for IME users, and allows to\n"
" generate non-ASCII characters, contrary to the default\n"
" injection method.\n"
" It may only work over USB, and is currently only supported\n"
" on Linux.\n"
"\n"
" -h, --help\n"
" Print this help.\n"
"\n"
@@ -738,6 +746,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
OPT_FORWARD_ALL_CLICKS},
{"fullscreen", no_argument, NULL, 'f'},
{"help", no_argument, NULL, 'h'},
{"keyboard-hid", no_argument, NULL, 'K'},
{"legacy-paste", no_argument, NULL, OPT_LEGACY_PASTE},
{"lock-video-orientation", optional_argument, NULL,
OPT_LOCK_VIDEO_ORIENTATION},
@@ -784,7 +793,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
optind = 0; // reset to start from the first argument in tests
int c;
while ((c = getopt_long(argc, argv, "b:c:fF:hm:nNp:r:s:StTvV:w",
while ((c = getopt_long(argc, argv, "b:c:fF:hKm:nNp:r:s:StTvV:w",
long_options, NULL)) != -1) {
switch (c) {
case 'b':
@@ -817,6 +826,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
case 'h':
args->help = true;
break;
case 'K':
opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;
break;
case OPT_MAX_FPS:
if (!parse_max_fps(optarg, &opts->max_fps)) {
return false;

310
app/src/hid_keyboard.c Normal file
View File

@@ -0,0 +1,310 @@
#include "hid_keyboard.h"
#include <assert.h>
#include <SDL2/SDL_events.h>
#include "util/log.h"
/** Downcast key processor to hid_keyboard */
#define DOWNCAST(KP) \
container_of(KP, struct hid_keyboard, key_processor)
#define HID_KEYBOARD_ACCESSORY_ID 1
#define HID_KEYBOARD_MODIFIER_NONE 0x00
#define HID_KEYBOARD_MODIFIER_LEFT_CONTROL (1 << 0)
#define HID_KEYBOARD_MODIFIER_LEFT_SHIFT (1 << 1)
#define HID_KEYBOARD_MODIFIER_LEFT_ALT (1 << 2)
#define HID_KEYBOARD_MODIFIER_LEFT_GUI (1 << 3)
#define HID_KEYBOARD_MODIFIER_RIGHT_CONTROL (1 << 4)
#define HID_KEYBOARD_MODIFIER_RIGHT_SHIFT (1 << 5)
#define HID_KEYBOARD_MODIFIER_RIGHT_ALT (1 << 6)
#define HID_KEYBOARD_MODIFIER_RIGHT_GUI (1 << 7)
#define HID_KEYBOARD_INDEX_MODIFIER 0
#define HID_KEYBOARD_INDEX_KEYS 2
// USB HID protocol says 6 keys in an event is the requirement for BIOS
// keyboard support, though OS could support more keys via modifying the report
// desc. 6 should be enough for scrcpy.
#define HID_KEYBOARD_MAX_KEYS 6
#define HID_KEYBOARD_EVENT_SIZE (2 + HID_KEYBOARD_MAX_KEYS)
#define HID_KEYBOARD_RESERVED 0x00
#define HID_KEYBOARD_ERROR_ROLL_OVER 0x01
/**
* For HID over AOAv2, only report descriptor is needed.
*
* The specification is available here:
* <https://www.usb.org/sites/default/files/hid1_11.pdf>
*
* In particular, read:
* - 6.2.2 Report Descriptor
* - Appendix B.1 Protocol 1 (Keyboard)
* - Appendix C: Keyboard Implementation
*
* Normally a basic HID keyboard uses 8 bytes:
* Modifier Reserved Key Key Key Key Key Key
*
* You can dump your device's report descriptor with:
*
* sudo usbhid-dump -m vid:pid -e descriptor
*
* (change vid:pid' to your device's vendor ID and product ID).
*/
static const unsigned char keyboard_report_desc[] = {
// Usage Page (Generic Desktop)
0x05, 0x01,
// Usage (Keyboard)
0x09, 0x06,
// Collection (Application)
0xA1, 0x01,
// Usage Page (Key Codes)
0x05, 0x07,
// Usage Minimum (224)
0x19, 0xE0,
// Usage Maximum (231)
0x29, 0xE7,
// Logical Minimum (0)
0x15, 0x00,
// Logical Maximum (1)
0x25, 0x01,
// Report Size (1)
0x75, 0x01,
// Report Count (8)
0x95, 0x08,
// Input (Data, Variable, Absolute): Modifier byte
0x81, 0x02,
// Report Size (8)
0x75, 0x08,
// Report Count (1)
0x95, 0x01,
// Input (Constant): Reserved byte
0x81, 0x01,
// Usage Page (LEDs)
0x05, 0x08,
// Usage Minimum (1)
0x19, 0x01,
// Usage Maximum (5)
0x29, 0x05,
// Report Size (1)
0x75, 0x01,
// Report Count (5)
0x95, 0x05,
// Output (Data, Variable, Absolute): LED report
0x91, 0x02,
// Report Size (3)
0x75, 0x03,
// Report Count (1)
0x95, 0x01,
// Output (Constant): LED report padding
0x91, 0x01,
// Usage Page (Key Codes)
0x05, 0x07,
// Usage Minimum (0)
0x19, 0x00,
// Usage Maximum (101)
0x29, HID_KEYBOARD_KEYS - 1,
// Logical Minimum (0)
0x15, 0x00,
// Logical Maximum(101)
0x25, HID_KEYBOARD_KEYS - 1,
// Report Size (8)
0x75, 0x08,
// Report Count (6)
0x95, HID_KEYBOARD_MAX_KEYS,
// Input (Data, Array): Keys
0x81, 0x00,
// End Collection
0xC0
};
static unsigned char *
create_hid_keyboard_event(void) {
unsigned char *buffer = malloc(HID_KEYBOARD_EVENT_SIZE);
if (!buffer) {
return NULL;
}
buffer[HID_KEYBOARD_INDEX_MODIFIER] = HID_KEYBOARD_MODIFIER_NONE;
buffer[1] = HID_KEYBOARD_RESERVED;
memset(&buffer[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS);
return buffer;
}
static unsigned char
sdl_keymod_to_hid_modifiers(SDL_Keymod mod) {
unsigned char modifiers = HID_KEYBOARD_MODIFIER_NONE;
if (mod & KMOD_LCTRL) {
modifiers |= HID_KEYBOARD_MODIFIER_LEFT_CONTROL;
}
if (mod & KMOD_LSHIFT) {
modifiers |= HID_KEYBOARD_MODIFIER_LEFT_SHIFT;
}
if (mod & KMOD_LALT) {
modifiers |= HID_KEYBOARD_MODIFIER_LEFT_ALT;
}
if (mod & KMOD_LGUI) {
modifiers |= HID_KEYBOARD_MODIFIER_LEFT_GUI;
}
if (mod & KMOD_RCTRL) {
modifiers |= HID_KEYBOARD_MODIFIER_RIGHT_CONTROL;
}
if (mod & KMOD_RSHIFT) {
modifiers |= HID_KEYBOARD_MODIFIER_RIGHT_SHIFT;
}
if (mod & KMOD_RALT) {
modifiers |= HID_KEYBOARD_MODIFIER_RIGHT_ALT;
}
if (mod & KMOD_RGUI) {
modifiers |= HID_KEYBOARD_MODIFIER_RIGHT_GUI;
}
return modifiers;
}
static bool
convert_hid_keyboard_event(struct hid_keyboard *kb, struct hid_event *hid_event,
const SDL_KeyboardEvent *event) {
hid_event->buffer = create_hid_keyboard_event();
if (!hid_event->buffer) {
return false;
}
hid_event->size = HID_KEYBOARD_EVENT_SIZE;
unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->keysym.mod);
SDL_Scancode scancode = event->keysym.scancode;
assert(scancode >= 0);
if (scancode < HID_KEYBOARD_KEYS) {
// Pressed is true and released is false
kb->keys[scancode] = (event->type == SDL_KEYDOWN);
LOGV("keys[%02x] = %s", scancode,
kb->keys[scancode] ? "true" : "false");
}
hid_event->buffer[HID_KEYBOARD_INDEX_MODIFIER] = modifiers;
// Re-calculate pressed keys every time
int keys_pressed_count = 0;
for (int i = 0; i < HID_KEYBOARD_KEYS; ++i) {
if (kb->keys[i]) {
// USB HID protocol says that if keys exceeds report count, a
// phantom state should be reported
if (keys_pressed_count >= HID_KEYBOARD_MAX_KEYS) {
// Pantom state:
// - Modifiers
// - Reserved
// - ErrorRollOver * HID_KEYBOARD_MAX_KEYS
memset(&hid_event->buffer[HID_KEYBOARD_INDEX_KEYS],
HID_KEYBOARD_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS);
return true;
}
hid_event->buffer[HID_KEYBOARD_INDEX_KEYS + keys_pressed_count] = i;
++keys_pressed_count;
}
}
return true;
}
static inline bool
scancode_is_modifier(SDL_Scancode scancode) {
return scancode >= SDL_SCANCODE_LCTRL && scancode <= SDL_SCANCODE_RGUI;
}
static bool
hid_keyboard_convert_event(struct hid_keyboard *kb, struct hid_event *hid_event,
const SDL_KeyboardEvent *event) {
LOGV(
"Type: %s, Repeat: %s, Modifiers: %02x, Key: %02x",
event->type == SDL_KEYDOWN ? "down" : "up",
event->repeat != 0 ? "true" : "false",
sdl_keymod_to_hid_modifiers(event->keysym.mod),
event->keysym.scancode
);
hid_event->from_accessory_id = HID_KEYBOARD_ACCESSORY_ID;
SDL_Scancode scancode = event->keysym.scancode;
assert(scancode >= 0);
// SDL also generates events when only modifiers are pressed, we cannot
// ignore them totally, for example press 'a' first then press 'Control',
// if we ignore 'Control' event, only 'a' is sent.
if (scancode < HID_KEYBOARD_KEYS || scancode_is_modifier(scancode)) {
return convert_hid_keyboard_event(kb, hid_event, event);
}
return false;
}
static void
sc_key_processor_process_key(struct sc_key_processor *kp,
const SDL_KeyboardEvent *event) {
if (event->repeat) {
// In USB HID protocol, key repeat is handled by the host (Android), so
// just ignore key repeat here.
return;
}
struct hid_keyboard *kh = DOWNCAST(kp);
struct hid_event hid_event;
// Not all keys are supported, just ignore unsupported keys
if (hid_keyboard_convert_event(kh, &hid_event, event)) {
if (!aoa_push_hid_event(kh->aoa, &hid_event)) {
LOGW("Could request HID event");
}
}
}
static void
sc_key_processor_process_text(struct sc_key_processor *kp,
const SDL_TextInputEvent *event) {
(void) kp;
(void) event;
// Never forward text input via HID (all the keys are injected separately)
}
bool
hid_keyboard_init(struct hid_keyboard *kb, struct aoa *aoa) {
kb->aoa = aoa;
bool ok = aoa_setup_hid(aoa, HID_KEYBOARD_ACCESSORY_ID,
keyboard_report_desc,
ARRAY_LEN(keyboard_report_desc));
if (!ok) {
LOGW("Register HID for keyboard failed");
return false;
}
// Reset all states
memset(kb->keys, false, HID_KEYBOARD_KEYS);
static const struct sc_key_processor_ops ops = {
.process_key = sc_key_processor_process_key,
.process_text = sc_key_processor_process_text,
};
kb->key_processor.ops = &ops;
return true;
}
void
hid_keyboard_destroy(struct hid_keyboard *kb) {
// Unregister HID keyboard so the soft keyboard shows again on Android
bool ok = aoa_unregister_hid(kb->aoa, HID_KEYBOARD_ACCESSORY_ID);
if (!ok) {
LOGW("Could not unregister HID");
}
}

44
app/src/hid_keyboard.h Normal file
View File

@@ -0,0 +1,44 @@
#ifndef HID_KEYBOARD_H
#define HID_KEYBOARD_H
#include "common.h"
#include <stdbool.h>
#include <SDL2/SDL.h>
#include "aoa_hid.h"
#include "trait/key_processor.h"
// See "SDL2/SDL_scancode.h".
// Maybe SDL_Keycode is used by most people, but SDL_Scancode is taken from USB
// HID protocol.
// 0x65 is Application, typically AT-101 Keyboard ends here.
#define HID_KEYBOARD_KEYS 0x66
/**
* HID keyboard events are sequence-based, every time keyboard state changes
* it sends an array of currently pressed keys, the host is responsible for
* compare events and determine which key becomes pressed and which key becomes
* released. In order to convert SDL_KeyboardEvent to HID events, we first use
* an array of keys to save each keys' state. And when a SDL_KeyboardEvent was
* emitted, we updated our state, and then we use a loop to generate HID
* events. The sequence of array elements is unimportant and when too much keys
* pressed at the same time (more than report count), we should generate
* phantom state. Don't forget that modifiers should be updated too, even for
* phantom state.
*/
struct hid_keyboard {
struct sc_key_processor key_processor; // key processor trait
struct aoa *aoa;
bool keys[HID_KEYBOARD_KEYS];
};
bool
hid_keyboard_init(struct hid_keyboard *kb, struct aoa *aoa);
void
hid_keyboard_destroy(struct hid_keyboard *kb);
#endif

View File

@@ -18,6 +18,9 @@
#include "events.h"
#include "file_handler.h"
#include "input_manager.h"
#ifdef HAVE_AOA_HID
# include "hid_keyboard.h"
#endif
#include "keyboard_inject.h"
#include "mouse_inject.h"
#include "recorder.h"
@@ -42,7 +45,15 @@ struct scrcpy {
#endif
struct controller controller;
struct file_handler file_handler;
struct sc_keyboard_inject keyboard_inject;
#ifdef HAVE_AOA_HID
struct aoa aoa;
#endif
union {
struct sc_keyboard_inject keyboard_inject;
#ifdef HAVE_AOA_HID
struct hid_keyboard keyboard_hid;
#endif
};
struct sc_mouse_inject mouse_inject;
struct input_manager input_manager;
};
@@ -244,7 +255,7 @@ stream_on_eos(struct stream *stream, void *userdata) {
}
bool
scrcpy(const struct scrcpy_options *options) {
scrcpy(struct scrcpy_options *options) {
static struct scrcpy scrcpy;
struct scrcpy *s = &scrcpy;
@@ -261,6 +272,9 @@ scrcpy(const struct scrcpy_options *options) {
bool v4l2_sink_initialized = false;
#endif
bool stream_started = false;
#ifdef HAVE_AOA_HID
bool aoa_hid_initialized = false;
#endif
bool controller_initialized = false;
bool controller_started = false;
bool screen_initialized = false;
@@ -420,8 +434,51 @@ scrcpy(const struct scrcpy_options *options) {
struct sc_mouse_processor *mp = NULL;
if (options->control) {
sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, options);
kp = &s->keyboard_inject.key_processor;
if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
#ifdef HAVE_AOA_HID
bool aoa_hid_ok = false;
if (!aoa_init(&s->aoa, options->serial)) {
goto aoa_hid_end;
}
if (!aoa_start(&s->aoa)) {
aoa_destroy(&s->aoa);
goto aoa_hid_end;
}
if (!hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {
aoa_join(&s->aoa);
aoa_stop(&s->aoa);
aoa_destroy(&s->aoa);
goto aoa_hid_end;
}
aoa_hid_ok = true;
kp = &s->keyboard_hid.key_processor;
aoa_hid_initialized = true;
aoa_hid_end:
if (!aoa_hid_ok) {
LOGE("Failed to enable HID over AOA, "
"fallback to default keyboard injection method "
"(-K/--keyboard-hid ignored)");
options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
}
#else
LOGE("HID over AOA is not supported on this platform, "
"fallback to default keyboard injection method "
"(-K/--keyboard-hid ignored)");
options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
#endif
}
// keyboard_input_mode may have been reset if HID mode failed
if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {
sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,
options);
kp = &s->keyboard_inject.key_processor;
}
sc_mouse_inject_init(&s->mouse_inject, &s->controller, &s->screen);
mp = &s->mouse_inject.mouse_processor;
@@ -440,6 +497,12 @@ scrcpy(const struct scrcpy_options *options) {
end:
// The stream is not stopped explicitly, because it will stop by itself on
// end-of-stream
#ifdef HAVE_AOA_HID
if (aoa_hid_initialized) {
hid_keyboard_destroy(&s->keyboard_hid);
aoa_stop(&s->aoa);
}
#endif
if (controller_started) {
controller_stop(&s->controller);
}
@@ -467,6 +530,13 @@ end:
}
#endif
#ifdef HAVE_AOA_HID
if (aoa_hid_initialized) {
aoa_join(&s->aoa);
aoa_destroy(&s->aoa);
}
#endif
// Destroy the screen only after the stream is guaranteed to be finished,
// because otherwise the screen could receive new frames after destruction
if (screen_initialized) {

View File

@@ -33,6 +33,11 @@ enum sc_lock_video_orientation {
SC_LOCK_VIDEO_ORIENTATION_3,
};
enum sc_keyboard_input_mode {
SC_KEYBOARD_INPUT_MODE_INJECT,
SC_KEYBOARD_INPUT_MODE_HID,
};
#define SC_MAX_SHORTCUT_MODS 8
enum sc_shortcut_mod {
@@ -68,6 +73,7 @@ struct scrcpy_options {
const char *v4l2_device;
enum sc_log_level log_level;
enum sc_record_format record_format;
enum sc_keyboard_input_mode keyboard_input_mode;
struct sc_port_range port_range;
struct sc_shortcut_mods shortcut_mods;
uint16_t max_size;
@@ -112,6 +118,7 @@ struct scrcpy_options {
.v4l2_device = NULL, \
.log_level = SC_LOG_LEVEL_INFO, \
.record_format = SC_RECORD_FORMAT_AUTO, \
.keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, \
.port_range = { \
.first = DEFAULT_LOCAL_PORT_RANGE_FIRST, \
.last = DEFAULT_LOCAL_PORT_RANGE_LAST, \
@@ -151,6 +158,6 @@ struct scrcpy_options {
}
bool
scrcpy(const struct scrcpy_options *options);
scrcpy(struct scrcpy_options *options);
#endif