Compare commits

..

2 Commits

Author SHA1 Message Date
Romain Vimont
c2d0dcd002 Add multitouch simulation
While the Ctrl key is held, the first left-click adds an immobile
"virtual finger" at this position.

Moving the mouse moves another cursor. The left click may be released
and pressed as necessary to control the second finger.

Releasing Ctrl releases the virtual finger.
2019-10-10 21:45:22 +02:00
Romain Vimont
9448eae8a4 Initialize input manager dynamically
To simulate multitouch, we will need more fields in input_manager, which
will be initialized dynamically.
2019-10-06 21:36:06 +02:00
13 changed files with 140 additions and 285 deletions

View File

@@ -16,6 +16,7 @@
(3 + CONTROL_MSG_CLIPBOARD_TEXT_MAX_LENGTH)
#define POINTER_ID_MOUSE UINT64_C(-1);
#define POINTER_ID_VIRTUAL_FINGER UINT64_C(-2);
enum control_msg_type {
CONTROL_MSG_TYPE_INJECT_KEYCODE,

View File

@@ -7,6 +7,19 @@
#include "lock_util.h"
#include "log.h"
void
input_manager_init(struct input_manager *input_manager,
struct controller *controller,
struct video_buffer *video_buffer,
struct screen *screen) {
input_manager->controller = controller;
input_manager->video_buffer = video_buffer;
input_manager->screen = screen;
input_manager->ctrl_down = false;
input_manager->vfinger.down = false;
}
// Convert window coordinates (as provided by SDL_GetMouseState() to renderer
// coordinates (as provided in SDL mouse events)
//
@@ -211,6 +224,26 @@ clipboard_paste(struct controller *controller) {
}
}
static void
simulate_virtual_finger(struct input_manager *input_manager, bool down,
struct position *position) {
SDL_assert(input_manager->vfinger.down != down);
input_manager->vfinger.down = down;
struct control_msg msg;
msg.type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
msg.inject_touch_event.action = down ? AMOTION_EVENT_ACTION_DOWN
: AMOTION_EVENT_ACTION_UP;
msg.inject_touch_event.pointer_id = POINTER_ID_VIRTUAL_FINGER;
msg.inject_touch_event.position = *position;
msg.inject_touch_event.pressure = 1.f;
msg.inject_touch_event.buttons = 0;
if (!controller_push_msg(input_manager->controller, &msg)) {
LOGW("Could not request 'inject virtual finger event'");
}
}
void
input_manager_process_text_input(struct input_manager *input_manager,
const SDL_TextInputEvent *event) {
@@ -244,6 +277,14 @@ input_manager_process_key(struct input_manager *input_manager,
bool alt = event->keysym.mod & (KMOD_LALT | KMOD_RALT);
bool meta = event->keysym.mod & (KMOD_LGUI | KMOD_RGUI);
// store the Ctrl state to modify mouse events
input_manager->ctrl_down = ctrl;
if (input_manager->vfinger.down && !ctrl) {
simulate_virtual_finger(input_manager, false,
&input_manager->vfinger.position);
}
// use Cmd on macOS, Ctrl on other platforms
#ifdef __APPLE__
bool cmd = !ctrl && meta;
@@ -427,6 +468,7 @@ input_manager_process_mouse_button(struct input_manager *input_manager,
// simulated from touch events, so it's a duplicate
return;
}
if (event->type == SDL_MOUSEBUTTONDOWN) {
if (control && event->button == SDL_BUTTON_RIGHT) {
press_back_or_turn_screen_on(input_manager->controller);
@@ -437,12 +479,24 @@ input_manager_process_mouse_button(struct input_manager *input_manager,
return;
}
// double-click on black borders resize to fit the device screen
if (event->button == SDL_BUTTON_LEFT && event->clicks == 2) {
bool outside =
is_outside_device_screen(input_manager, event->x, event->y);
if (outside) {
screen_resize_to_fit(input_manager->screen);
return;
if (event->button == SDL_BUTTON_LEFT) {
if (event->clicks >= 2) {
bool outside =
is_outside_device_screen(input_manager, event->x, event->y);
if (outside) {
screen_resize_to_fit(input_manager->screen);
return;
}
}
struct virtual_finger *vfinger = &input_manager->vfinger;
if (input_manager->ctrl_down && !vfinger->down) {
vfinger->position.point.x = event->x;
vfinger->position.point.y = event->y;
vfinger->position.screen_size =
input_manager->screen->frame_size,
simulate_virtual_finger(input_manager, true,
&vfinger->position);
}
}
// otherwise, send the click event to the device

View File

@@ -14,8 +14,21 @@ struct input_manager {
struct controller *controller;
struct video_buffer *video_buffer;
struct screen *screen;
bool ctrl_down;
struct virtual_finger {
bool down;
struct position position;
} vfinger;
};
void
input_manager_init(struct input_manager *input_manager,
struct controller *controller,
struct video_buffer *video_buffer,
struct screen *screen);
void
input_manager_process_text_input(struct input_manager *input_manager,
const SDL_TextInputEvent *event);

View File

@@ -37,12 +37,7 @@ static struct decoder decoder;
static struct recorder recorder;
static struct controller controller;
static struct file_handler file_handler;
static struct input_manager input_manager = {
.controller = &controller,
.video_buffer = &video_buffer,
.screen = &screen,
};
static struct input_manager input_manager;
// init SDL and set appropriate hints
static bool
@@ -311,6 +306,8 @@ scrcpy(const struct scrcpy_options *options) {
goto end;
}
input_manager_init(&input_manager, &controller, &video_buffer, &screen);
if (!server_connect_to(&server)) {
goto end;
}

View File

@@ -16,7 +16,7 @@
// get the window size in a struct size
static struct size
get_window_size(SDL_Window *window) {
get_native_window_size(SDL_Window *window) {
int width;
int height;
SDL_GetWindowSize(window, &width, &height);
@@ -29,11 +29,11 @@ get_window_size(SDL_Window *window) {
// get the windowed window size
static struct size
get_windowed_window_size(const struct screen *screen) {
get_window_size(const struct screen *screen) {
if (screen->fullscreen) {
return screen->windowed_window_size;
}
return get_window_size(screen->window);
return get_native_window_size(screen->window);
}
// set the window size to be applied when fullscreen is disabled
@@ -112,8 +112,8 @@ get_optimal_size(struct size current_size, struct size frame_size) {
// same as get_optimal_size(), but read the current size from the window
static inline struct size
get_optimal_window_size(const struct screen *screen, struct size frame_size) {
struct size windowed_size = get_windowed_window_size(screen);
return get_optimal_size(windowed_size, frame_size);
struct size current_size = get_window_size(screen);
return get_optimal_size(current_size, frame_size);
}
// initially, there is no current size, so use the frame size as current size
@@ -229,11 +229,11 @@ prepare_for_frame(struct screen *screen, struct size new_frame_size) {
// frame dimension changed, destroy texture
SDL_DestroyTexture(screen->texture);
struct size windowed_size = get_windowed_window_size(screen);
struct size current_size = get_window_size(screen);
struct size target_size = {
(uint32_t) windowed_size.width * new_frame_size.width
(uint32_t) current_size.width * new_frame_size.width
/ screen->frame_size.width,
(uint32_t) windowed_size.height * new_frame_size.height
(uint32_t) current_size.height * new_frame_size.height
/ screen->frame_size.height,
};
target_size = get_optimal_size(target_size, new_frame_size);
@@ -289,7 +289,7 @@ void
screen_switch_fullscreen(struct screen *screen) {
if (!screen->fullscreen) {
// going to fullscreen, store the current windowed window size
screen->windowed_window_size = get_window_size(screen->window);
screen->windowed_window_size = get_native_window_size(screen->window);
}
uint32_t new_mode = screen->fullscreen ? 0 : SDL_WINDOW_FULLSCREEN_DESKTOP;
if (SDL_SetWindowFullscreen(screen->window, new_mode)) {

View File

@@ -1,62 +0,0 @@
#!/bin/bash
#
# This script generates the scrcpy binary "manually" (without gradle).
#
# Adapt Android platform and build tools versions (via ANDROID_PLATFORM and
# ANDROID_BUILD_TOOLS environment variables).
#
# Then execute:
#
# BUILD_DIR=my_build_dir ./build_without_gradle.sh
set -e
PLATFORM=${ANDROID_PLATFORM:-29}
BUILD_TOOLS=${ANDROID_BUILD_TOOLS:-29.0.2}
BUILD_DIR="$(realpath ${BUILD_DIR:-build_manual})"
CLASSES_DIR="$BUILD_DIR/classes"
SERVER_DIR=$(dirname "$0")
SERVER_BINARY=scrcpy-server.jar
echo "Platform: android-$PLATFORM"
echo "Build-tools: $BUILD_TOOLS"
echo "Build dir: $BUILD_DIR"
rm -rf "$CLASSES_DIR" "$BUILD_DIR/$SERVER_BINARY" classes.dex
mkdir -p "$CLASSES_DIR/com/genymobile/scrcpy"
<< EOF cat > "$CLASSES_DIR/com/genymobile/scrcpy/BuildConfig.java"
package com.genymobile.scrcpy;
public final class BuildConfig {
public static final boolean DEBUG = false;
}
EOF
echo "Generating java from aidl..."
cd "$SERVER_DIR/src/main/aidl"
"$ANDROID_HOME/build-tools/$BUILD_TOOLS/aidl" -o "$CLASSES_DIR" \
android/view/IRotationWatcher.aidl
echo "Compiling java sources..."
cd ../java
javac -bootclasspath "$ANDROID_HOME/platforms/android-$PLATFORM/android.jar" \
-cp "$CLASSES_DIR" -d "$CLASSES_DIR" -source 1.8 -target 1.8 \
com/genymobile/scrcpy/*.java \
com/genymobile/scrcpy/wrappers/*.java
echo "Dexing..."
cd "$CLASSES_DIR"
"$ANDROID_HOME/build-tools/$BUILD_TOOLS/dx" --dex \
--output "$BUILD_DIR/classes.dex" \
android/view/*.class \
com/genymobile/scrcpy/*.class \
com/genymobile/scrcpy/wrappers/*.class
echo "Archiving..."
cd "$BUILD_DIR"
jar cvf "$SERVER_BINARY" classes.dex
rm -rf classes.dex classes
echo "Server generated in $BUILD_DIR/scrcpy-server.jar"

View File

@@ -82,7 +82,7 @@ public class Controller {
injectText(msg.getText());
break;
case ControlMessage.TYPE_INJECT_TOUCH_EVENT:
injectTouch(msg.getAction(), msg.getPointerId(), msg.getPosition(), msg.getPressure(), msg.getButtons());
injectTouch(msg.getAction(), msg.getPointerId(), msg.getPosition(), msg.getPressure(), 0);
break;
case ControlMessage.TYPE_INJECT_SCROLL_EVENT:
injectScroll(msg.getPosition(), msg.getHScroll(), msg.getVScroll());
@@ -177,7 +177,7 @@ public class Controller {
}
MotionEvent event = MotionEvent.obtain(lastTouchDown, now, action, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, 0, 0, InputDevice.SOURCE_TOUCHSCREEN, 0);
pointerCoords, 0, 0, 1f, 1f, 0, 0, InputDevice.SOURCE_TOUCHSCREEN, 0);
return injectEvent(event);
}

View File

@@ -161,11 +161,7 @@ public final class Device {
* @param mode one of the {@code SCREEN_POWER_MODE_*} constants
*/
public void setScreenPowerMode(int mode) {
IBinder d = SurfaceControl.getBuiltInDisplay();
if (d == null) {
Ln.e("Could not get built-in display");
return;
}
IBinder d = SurfaceControl.getBuiltInDisplay(0);
SurfaceControl.setDisplayPowerMode(d, mode);
Ln.i("Device screen turned " + (mode == Device.POWER_MODE_OFF ? "off" : "on"));
}

View File

@@ -1,102 +1,44 @@
package com.genymobile.scrcpy.wrappers;
import com.genymobile.scrcpy.Ln;
import android.content.ClipData;
import android.os.Build;
import android.os.IInterface;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
public class ClipboardManager {
private static final String PACKAGE_NAME = "com.android.shell";
private static final int USER_ID = 0;
private final IInterface manager;
private Method getPrimaryClipMethod;
private Method setPrimaryClipMethod;
private final Method getPrimaryClipMethod;
private final Method setPrimaryClipMethod;
public ClipboardManager(IInterface manager) {
this.manager = manager;
}
private Method getGetPrimaryClipMethod() {
if (getPrimaryClipMethod == null) {
try {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class);
} else {
getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, int.class);
}
} catch (NoSuchMethodException e) {
Ln.e("Could not find method", e);
}
}
return getPrimaryClipMethod;
}
private Method getSetPrimaryClipMethod() {
if (setPrimaryClipMethod == null) {
try {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
setPrimaryClipMethod = manager.getClass().getMethod("setPrimaryClip", ClipData.class, String.class);
} else {
setPrimaryClipMethod = manager.getClass().getMethod("setPrimaryClip", ClipData.class,
String.class, int.class);
}
} catch (NoSuchMethodException e) {
Ln.e("Could not find method", e);
}
}
return setPrimaryClipMethod;
}
private static ClipData getPrimaryClip(Method method, IInterface manager) throws InvocationTargetException,
IllegalAccessException {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
return (ClipData) method.invoke(manager, PACKAGE_NAME);
}
return (ClipData) method.invoke(manager, PACKAGE_NAME, USER_ID);
}
private static void setPrimaryClip(Method method, IInterface manager, ClipData clipData) throws InvocationTargetException,
IllegalAccessException {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
method.invoke(manager, clipData, PACKAGE_NAME);
} else {
method.invoke(manager, clipData, PACKAGE_NAME, USER_ID);
try {
getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class);
setPrimaryClipMethod = manager.getClass().getMethod("setPrimaryClip", ClipData.class, String.class);
} catch (NoSuchMethodException e) {
throw new AssertionError(e);
}
}
public CharSequence getText() {
Method method = getGetPrimaryClipMethod();
if (method == null) {
return null;
}
try {
ClipData clipData = getPrimaryClip(method, manager);
ClipData clipData = (ClipData) getPrimaryClipMethod.invoke(manager, "com.android.shell");
if (clipData == null || clipData.getItemCount() == 0) {
return null;
}
return clipData.getItemAt(0).getText();
} catch (InvocationTargetException | IllegalAccessException e) {
Ln.e("Could not invoke " + method.getName(), e);
return null;
throw new AssertionError(e);
}
}
public void setText(CharSequence text) {
Method method = getSetPrimaryClipMethod();
if (method == null) {
return;
}
ClipData clipData = ClipData.newPlainText(null, text);
try {
setPrimaryClip(method, manager, clipData);
setPrimaryClipMethod.invoke(manager, clipData, "com.android.shell");
} catch (InvocationTargetException | IllegalAccessException e) {
Ln.e("Could not invoke " + method.getName(), e);
throw new AssertionError(e);
}
}
}

View File

@@ -1,7 +1,5 @@
package com.genymobile.scrcpy.wrappers;
import com.genymobile.scrcpy.Ln;
import android.os.IInterface;
import android.view.InputEvent;
@@ -15,33 +13,22 @@ public final class InputManager {
public static final int INJECT_INPUT_EVENT_MODE_WAIT_FOR_FINISH = 2;
private final IInterface manager;
private Method injectInputEventMethod;
private final Method injectInputEventMethod;
public InputManager(IInterface manager) {
this.manager = manager;
}
private Method getInjectInputEventMethod() {
if (injectInputEventMethod == null) {
try {
injectInputEventMethod = manager.getClass().getMethod("injectInputEvent", InputEvent.class, int.class);
} catch (NoSuchMethodException e) {
Ln.e("Could not find method", e);
}
try {
injectInputEventMethod = manager.getClass().getMethod("injectInputEvent", InputEvent.class, int.class);
} catch (NoSuchMethodException e) {
throw new AssertionError(e);
}
return injectInputEventMethod;
}
public boolean injectInputEvent(InputEvent inputEvent, int mode) {
Method method = getInjectInputEventMethod();
if (method == null) {
return false;
}
try {
return (Boolean) method.invoke(manager, inputEvent, mode);
return (Boolean) injectInputEventMethod.invoke(manager, inputEvent, mode);
} catch (InvocationTargetException | IllegalAccessException e) {
Ln.e("Could not invoke " + method.getName(), e);
return false;
throw new AssertionError(e);
}
}
}

View File

@@ -1,7 +1,5 @@
package com.genymobile.scrcpy.wrappers;
import com.genymobile.scrcpy.Ln;
import android.annotation.SuppressLint;
import android.os.Build;
import android.os.IInterface;
@@ -11,35 +9,24 @@ import java.lang.reflect.Method;
public final class PowerManager {
private final IInterface manager;
private Method isScreenOnMethod;
private final Method isScreenOnMethod;
public PowerManager(IInterface manager) {
this.manager = manager;
}
private Method getIsScreenOnMethod() {
if (isScreenOnMethod == null) {
try {
@SuppressLint("ObsoleteSdkInt") // we may lower minSdkVersion in the future
String methodName = Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT_WATCH ? "isInteractive" : "isScreenOn";
isScreenOnMethod = manager.getClass().getMethod(methodName);
} catch (NoSuchMethodException e) {
Ln.e("Could not find method", e);
}
try {
@SuppressLint("ObsoleteSdkInt") // we may lower minSdkVersion in the future
String methodName = Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT_WATCH ? "isInteractive" : "isScreenOn";
isScreenOnMethod = manager.getClass().getMethod(methodName);
} catch (NoSuchMethodException e) {
throw new AssertionError(e);
}
return isScreenOnMethod;
}
public boolean isScreenOn() {
Method method = getIsScreenOnMethod();
if (method == null) {
return false;
}
try {
return (Boolean) method.invoke(manager);
return (Boolean) isScreenOnMethod.invoke(manager);
} catch (InvocationTargetException | IllegalAccessException e) {
Ln.e("Could not invoke " + method.getName(), e);
return false;
throw new AssertionError(e);
}
}
}

View File

@@ -17,49 +17,35 @@ public class StatusBarManager {
this.manager = manager;
}
private Method getExpandNotificationsPanelMethod() {
public void expandNotificationsPanel() {
if (expandNotificationsPanelMethod == null) {
try {
expandNotificationsPanelMethod = manager.getClass().getMethod("expandNotificationsPanel");
} catch (NoSuchMethodException e) {
Ln.e("Could not find method", e);
Ln.e("ServiceBarManager.expandNotificationsPanel() is not available on this device");
return;
}
}
return expandNotificationsPanelMethod;
}
private Method getCollapsePanelsMethod() {
if (collapsePanelsMethod == null) {
try {
collapsePanelsMethod = manager.getClass().getMethod("collapsePanels");
} catch (NoSuchMethodException e) {
Ln.e("Could not find method", e);
}
}
return collapsePanelsMethod;
}
public void expandNotificationsPanel() {
Method method = getExpandNotificationsPanelMethod();
if (method == null) {
return;
}
try {
method.invoke(manager);
expandNotificationsPanelMethod.invoke(manager);
} catch (InvocationTargetException | IllegalAccessException e) {
Ln.e("Could not invoke " + method.getName(), e);
Ln.e("Could not invoke ServiceBarManager.expandNotificationsPanel()", e);
}
}
public void collapsePanels() {
Method method = getCollapsePanelsMethod();
if (method == null) {
return;
if (collapsePanelsMethod == null) {
try {
collapsePanelsMethod = manager.getClass().getMethod("collapsePanels");
} catch (NoSuchMethodException e) {
Ln.e("ServiceBarManager.collapsePanels() is not available on this device");
return;
}
}
try {
method.invoke(manager);
collapsePanelsMethod.invoke(manager);
} catch (InvocationTargetException | IllegalAccessException e) {
Ln.e("Could not invoke " + method.getName(), e);
Ln.e("Could not invoke ServiceBarManager.collapsePanels()", e);
}
}
}

View File

@@ -1,16 +1,11 @@
package com.genymobile.scrcpy.wrappers;
import com.genymobile.scrcpy.Ln;
import android.annotation.SuppressLint;
import android.graphics.Rect;
import android.os.Build;
import android.os.IBinder;
import android.view.Surface;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@SuppressLint("PrivateApi")
public final class SurfaceControl {
@@ -28,9 +23,6 @@ public final class SurfaceControl {
}
}
private static Method getBuiltInDisplayMethod;
private static Method setDisplayPowerModeMethod;
private SurfaceControl() {
// only static methods
}
@@ -84,62 +76,24 @@ public final class SurfaceControl {
}
}
private static Method getGetBuiltInDisplayMethod() {
if (getBuiltInDisplayMethod == null) {
try {
// the method signature has changed in Android Q
// <https://github.com/Genymobile/scrcpy/issues/586>
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
getBuiltInDisplayMethod = CLASS.getMethod("getBuiltInDisplay", int.class);
} else {
getBuiltInDisplayMethod = CLASS.getMethod("getInternalDisplayToken");
}
} catch (NoSuchMethodException e) {
Ln.e("Could not find method", e);
}
}
return getBuiltInDisplayMethod;
}
public static IBinder getBuiltInDisplay() {
Method method = getGetBuiltInDisplayMethod();
if (method == null) {
return null;
}
public static IBinder getBuiltInDisplay(int builtInDisplayId) {
try {
// the method signature has changed in Android Q
// <https://github.com/Genymobile/scrcpy/issues/586>
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
// call getBuiltInDisplay(0)
return (IBinder) method.invoke(null, 0);
return (IBinder) CLASS.getMethod("getBuiltInDisplay", int.class).invoke(null, builtInDisplayId);
}
// call getInternalDisplayToken()
return (IBinder) method.invoke(null);
} catch (InvocationTargetException | IllegalAccessException e) {
Ln.e("Could not invoke " + method.getName(), e);
return null;
return (IBinder) CLASS.getMethod("getPhysicalDisplayToken", long.class).invoke(null, builtInDisplayId);
} catch (Exception e) {
throw new AssertionError(e);
}
}
private static Method getSetDisplayPowerModeMethod() {
if (setDisplayPowerModeMethod == null) {
try {
setDisplayPowerModeMethod = CLASS.getMethod("setDisplayPowerMode", IBinder.class, int.class);
} catch (NoSuchMethodException e) {
Ln.e("Could not find method", e);
}
}
return setDisplayPowerModeMethod;
}
public static void setDisplayPowerMode(IBinder displayToken, int mode) {
Method method = getSetDisplayPowerModeMethod();
if (method == null) {
return;
}
try {
method.invoke(null, displayToken, mode);
} catch (InvocationTargetException | IllegalAccessException e) {
Ln.e("Could not invoke " + method.getName(), e);
CLASS.getMethod("setDisplayPowerMode", IBinder.class, int.class).invoke(null, displayToken, mode);
} catch (Exception e) {
throw new AssertionError(e);
}
}