Add multitouch simulation

While the Ctrl key is held, the first left-click adds an immobile
"virtual finger" at this position.

Moving the mouse moves another cursor. The left click may be released
and pressed as necessary to control the second finger.

Releasing Ctrl releases the virtual finger.

app/src/control_msg.h

@@ -16,6 +16,7 @@
     (3 + CONTROL_MSG_CLIPBOARD_TEXT_MAX_LENGTH)
 
 #define POINTER_ID_MOUSE UINT64_C(-1);
+#define POINTER_ID_VIRTUAL_FINGER UINT64_C(-2);
 
 enum control_msg_type {
     CONTROL_MSG_TYPE_INJECT_KEYCODE,

app/src/event_converter.c

@@ -5,7 +5,7 @@
 #define MAP(FROM, TO) case FROM: *to = TO; return true
 #define FAIL default: return false
 
-bool
+static bool
 convert_keycode_action(SDL_EventType from, enum android_keyevent_action *to) {
     switch (from) {
         MAP(SDL_KEYDOWN, AKEY_EVENT_ACTION_DOWN);
@@ -33,7 +33,7 @@ autocomplete_metastate(enum android_metastate metastate) {
     return metastate;
 }
 
-enum android_metastate
+static enum android_metastate
 convert_meta_state(SDL_Keymod mod) {
     enum android_metastate metastate = 0;
     if (mod & KMOD_LSHIFT) {
@@ -74,7 +74,7 @@ convert_meta_state(SDL_Keymod mod) {
     return autocomplete_metastate(metastate);
 }
 
-bool
+static bool
 convert_keycode(SDL_Keycode from, enum android_keycode *to, uint16_t mod) {
     switch (from) {
         MAP(SDLK_RETURN,       AKEYCODE_ENTER);
@@ -128,7 +128,7 @@ convert_keycode(SDL_Keycode from, enum android_keycode *to, uint16_t mod) {
     }
 }
 
-enum android_motionevent_buttons
+static enum android_motionevent_buttons
 convert_mouse_buttons(uint32_t state) {
     enum android_motionevent_buttons buttons = 0;
     if (state & SDL_BUTTON_LMASK) {
@@ -150,6 +150,24 @@ convert_mouse_buttons(uint32_t state) {
 }
 
 bool
+convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to) {
+    to->type = CONTROL_MSG_TYPE_INJECT_KEYCODE;
+
+    if (!convert_keycode_action(from->type, &to->inject_keycode.action)) {
+        return false;
+    }
+
+    uint16_t mod = from->keysym.mod;
+    if (!convert_keycode(from->keysym.sym, &to->inject_keycode.keycode, mod)) {
+        return false;
+    }
+
+    to->inject_keycode.metastate = convert_meta_state(mod);
+
+    return true;
+}
+
+static bool
 convert_mouse_action(SDL_EventType from, enum android_motionevent_action *to) {
     switch (from) {
         MAP(SDL_MOUSEBUTTONDOWN, AMOTION_EVENT_ACTION_DOWN);
@@ -159,6 +177,41 @@ convert_mouse_action(SDL_EventType from, enum android_motionevent_action *to) {
 }
 
 bool
+convert_mouse_button(const SDL_MouseButtonEvent *from, struct size screen_size,
+                     struct control_msg *to) {
+    to->type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
+
+    if (!convert_mouse_action(from->type, &to->inject_touch_event.action)) {
+        return false;
+    }
+
+    to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
+    to->inject_touch_event.position.screen_size = screen_size;
+    to->inject_touch_event.position.point.x = from->x;
+    to->inject_touch_event.position.point.y = from->y;
+    to->inject_touch_event.pressure = 1.f;
+    to->inject_touch_event.buttons =
+        convert_mouse_buttons(SDL_BUTTON(from->button));
+
+    return true;
+}
+
+bool
+convert_mouse_motion(const SDL_MouseMotionEvent *from, struct size screen_size,
+                     struct control_msg *to) {
+    to->type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
+    to->inject_touch_event.action = AMOTION_EVENT_ACTION_MOVE;
+    to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
+    to->inject_touch_event.position.screen_size = screen_size;
+    to->inject_touch_event.position.point.x = from->x;
+    to->inject_touch_event.position.point.y = from->y;
+    to->inject_touch_event.pressure = 1.f;
+    to->inject_touch_event.buttons = convert_mouse_buttons(from->state);
+
+    return true;
+}
+
+static bool
 convert_touch_action(SDL_EventType from, enum android_motionevent_action *to) {
     switch (from) {
         MAP(SDL_FINGERMOTION, AMOTION_EVENT_ACTION_MOVE);
@@ -167,3 +220,39 @@ convert_touch_action(SDL_EventType from, enum android_motionevent_action *to) {
         FAIL;
     }
 }
+
+bool
+convert_touch(const SDL_TouchFingerEvent *from, struct size screen_size,
+              struct control_msg *to) {
+    to->type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
+
+    if (!convert_touch_action(from->type, &to->inject_touch_event.action)) {
+        return false;
+    }
+
+    to->inject_touch_event.pointer_id = from->fingerId;
+    to->inject_touch_event.position.screen_size = screen_size;
+    // SDL touch event coordinates are normalized in the range [0; 1]
+    to->inject_touch_event.position.point.x = from->x * screen_size.width;
+    to->inject_touch_event.position.point.y = from->y * screen_size.height;
+    to->inject_touch_event.pressure = from->pressure;
+    to->inject_touch_event.buttons = 0;
+    return true;
+}
+
+bool
+convert_mouse_wheel(const SDL_MouseWheelEvent *from, struct position position,
+                    struct control_msg *to) {
+    to->type = CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT;
+
+    to->inject_scroll_event.position = position;
+
+    int mul = from->direction == SDL_MOUSEWHEEL_NORMAL ? 1 : -1;
+    // SDL behavior seems inconsistent between horizontal and vertical scrolling
+    // so reverse the horizontal
+    // <https://wiki.libsdl.org/SDL_MouseWheelEvent#Remarks>
+    to->inject_scroll_event.hscroll = -mul * from->x;
+    to->inject_scroll_event.vscroll = mul * from->y;
+
+    return true;
+}

app/src/event_converter.h

@@ -7,22 +7,36 @@
 #include "config.h"
 #include "control_msg.h"
 
-bool
+struct complete_mouse_motion_event {
-convert_keycode_action(SDL_EventType from, enum android_keyevent_action *to);
+    SDL_MouseMotionEvent *mouse_motion_event;
+    struct size screen_size;
+};
 
-enum android_metastate
+struct complete_mouse_wheel_event {
-convert_meta_state(SDL_Keymod mod);
+    SDL_MouseWheelEvent *mouse_wheel_event;
+    struct point position;
+};
 
 bool
-convert_keycode(SDL_Keycode from, enum android_keycode *to, uint16_t mod);
+convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to);
-
-enum android_motionevent_buttons
-convert_mouse_buttons(uint32_t state);
 
 bool
-convert_mouse_action(SDL_EventType from, enum android_motionevent_action *to);
+convert_mouse_button(const SDL_MouseButtonEvent *from, struct size screen_size,
+                     struct control_msg *to);
+
+// the video size may be different from the real device size, so we need the
+// size to which the absolute position apply, to scale it accordingly
+bool
+convert_mouse_motion(const SDL_MouseMotionEvent *from, struct size screen_size,
+                     struct control_msg *to);
 
 bool
-convert_touch_action(SDL_EventType from, enum android_motionevent_action *to);
+convert_touch(const SDL_TouchFingerEvent *from, struct size screen_size,
+              struct control_msg *to);
+
+// on Android, a scroll event requires the current mouse position
+bool
+convert_mouse_wheel(const SDL_MouseWheelEvent *from, struct position position,
+                    struct control_msg *to);
 
 #endif

app/src/input_manager.c

@@ -7,6 +7,46 @@
 #include "lock_util.h"
 #include "log.h"
 
+void
+input_manager_init(struct input_manager *input_manager,
+                   struct controller *controller,
+                   struct video_buffer *video_buffer,
+                   struct screen *screen) {
+    input_manager->controller = controller;
+    input_manager->video_buffer = video_buffer;
+    input_manager->screen = screen;
+
+    input_manager->ctrl_down = false;
+    input_manager->vfinger.down = false;
+}
+
+// Convert window coordinates (as provided by SDL_GetMouseState() to renderer
+// coordinates (as provided in SDL mouse events)
+//
+// See my question:
+// <https://stackoverflow.com/questions/49111054/how-to-get-mouse-position-on-mouse-wheel-event>
+static void
+convert_to_renderer_coordinates(SDL_Renderer *renderer, int *x, int *y) {
+    SDL_Rect viewport;
+    float scale_x, scale_y;
+    SDL_RenderGetViewport(renderer, &viewport);
+    SDL_RenderGetScale(renderer, &scale_x, &scale_y);
+    *x = (int) (*x / scale_x) - viewport.x;
+    *y = (int) (*y / scale_y) - viewport.y;
+}
+
+static struct point
+get_mouse_point(struct screen *screen) {
+    int x;
+    int y;
+    SDL_GetMouseState(&x, &y);
+    convert_to_renderer_coordinates(screen->renderer, &x, &y);
+    return (struct point) {
+        .x = x,
+        .y = y,
+    };
+}
+
 static const int ACTION_DOWN = 1;
 static const int ACTION_UP = 1 << 1;
 
@@ -184,8 +224,28 @@ clipboard_paste(struct controller *controller) {
     }
 }
 
+static void
+simulate_virtual_finger(struct input_manager *input_manager, bool down,
+                        struct position *position) {
+    SDL_assert(input_manager->vfinger.down != down);
+    input_manager->vfinger.down = down;
+
+    struct control_msg msg;
+    msg.type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
+    msg.inject_touch_event.action = down ? AMOTION_EVENT_ACTION_DOWN
+                                         : AMOTION_EVENT_ACTION_UP;
+    msg.inject_touch_event.pointer_id = POINTER_ID_VIRTUAL_FINGER;
+    msg.inject_touch_event.position = *position;
+    msg.inject_touch_event.pressure = 1.f;
+    msg.inject_touch_event.buttons = 0;
+
+    if (!controller_push_msg(input_manager->controller, &msg)) {
+        LOGW("Could not request 'inject virtual finger event'");
+    }
+}
+
 void
-input_manager_process_text_input(struct input_manager *im,
+input_manager_process_text_input(struct input_manager *input_manager,
                                  const SDL_TextInputEvent *event) {
     char c = event->text[0];
     if (isalpha(c) || c == ' ') {
@@ -200,32 +260,14 @@ input_manager_process_text_input(struct input_manager *im,
         LOGW("Could not strdup input text");
         return;
     }
-    if (!controller_push_msg(im->controller, &msg)) {
+    if (!controller_push_msg(input_manager->controller, &msg)) {
         SDL_free(msg.inject_text.text);
         LOGW("Could not request 'inject text'");
     }
 }
 
-static bool
-convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to) {
-    to->type = CONTROL_MSG_TYPE_INJECT_KEYCODE;
-
-    if (!convert_keycode_action(from->type, &to->inject_keycode.action)) {
-        return false;
-    }
-
-    uint16_t mod = from->keysym.mod;
-    if (!convert_keycode(from->keysym.sym, &to->inject_keycode.keycode, mod)) {
-        return false;
-    }
-
-    to->inject_keycode.metastate = convert_meta_state(mod);
-
-    return true;
-}
-
 void
-input_manager_process_key(struct input_manager *im,
+input_manager_process_key(struct input_manager *input_manager,
                           const SDL_KeyboardEvent *event,
                           bool control) {
     // control: indicates the state of the command-line option --no-control
@@ -235,6 +277,14 @@ input_manager_process_key(struct input_manager *im,
     bool alt = event->keysym.mod & (KMOD_LALT | KMOD_RALT);
     bool meta = event->keysym.mod & (KMOD_LGUI | KMOD_RGUI);
 
+    // store the Ctrl state to modify mouse events
+    input_manager->ctrl_down = ctrl;
+
+    if (input_manager->vfinger.down && !ctrl) {
+        simulate_virtual_finger(input_manager, false,
+                                &input_manager->vfinger.position);
+    }
+
     // use Cmd on macOS, Ctrl on other platforms
 #ifdef __APPLE__
     bool cmd = !ctrl && meta;
@@ -252,7 +302,7 @@ input_manager_process_key(struct input_manager *im,
         return;
     }
 
-    struct controller *controller = im->controller;
+    struct controller *controller = input_manager->controller;
 
     // capture all Ctrl events
     if (ctrl || cmd) {
@@ -327,23 +377,23 @@ input_manager_process_key(struct input_manager *im,
                 return;
             case SDLK_f:
                 if (!shift && cmd && !repeat && down) {
-                    screen_switch_fullscreen(im->screen);
+                    screen_switch_fullscreen(input_manager->screen);
                 }
                 return;
             case SDLK_x:
                 if (!shift && cmd && !repeat && down) {
-                    screen_resize_to_fit(im->screen);
+                    screen_resize_to_fit(input_manager->screen);
                 }
                 return;
             case SDLK_g:
                 if (!shift && cmd && !repeat && down) {
-                    screen_resize_to_pixel_perfect(im->screen);
+                    screen_resize_to_pixel_perfect(input_manager->screen);
                 }
                 return;
             case SDLK_i:
                 if (!shift && cmd && !repeat && down) {
                     struct fps_counter *fps_counter =
-                        im->video_buffer->fps_counter;
+                        input_manager->video_buffer->fps_counter;
                     switch_fps_counter_state(fps_counter);
                 }
                 return;
@@ -373,23 +423,8 @@ input_manager_process_key(struct input_manager *im,
     }
 }
 
-static bool
-convert_mouse_motion(const SDL_MouseMotionEvent *from, struct screen *screen,
-                     struct control_msg *to) {
-    to->type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
-    to->inject_touch_event.action = AMOTION_EVENT_ACTION_MOVE;
-    to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
-    to->inject_touch_event.position.screen_size = screen->frame_size;
-    to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, from->x, from->y);
-    to->inject_touch_event.pressure = 1.f;
-    to->inject_touch_event.buttons = convert_mouse_buttons(from->state);
-
-    return true;
-}
-
 void
-input_manager_process_mouse_motion(struct input_manager *im,
+input_manager_process_mouse_motion(struct input_manager *input_manager,
                                    const SDL_MouseMotionEvent *event) {
     if (!event->state) {
         // do not send motion events when no button is pressed
@@ -400,98 +435,68 @@ input_manager_process_mouse_motion(struct input_manager *im,
         return;
     }
     struct control_msg msg;
-    if (convert_mouse_motion(event, im->screen, &msg)) {
+    if (convert_mouse_motion(event, input_manager->screen->frame_size, &msg)) {
-        if (!controller_push_msg(im->controller, &msg)) {
+        if (!controller_push_msg(input_manager->controller, &msg)) {
             LOGW("Could not request 'inject mouse motion event'");
         }
     }
 }
 
-static bool
-convert_touch(const SDL_TouchFingerEvent *from, struct screen *screen,
-              struct control_msg *to) {
-    to->type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
-
-    if (!convert_touch_action(from->type, &to->inject_touch_event.action)) {
-        return false;
-    }
-
-    struct size frame_size = screen->frame_size;
-    // SDL touch event coordinates are normalized in the range [0; 1]
-    float x = from->x * frame_size.width;
-    float y = from->y * frame_size.height;
-
-    to->inject_touch_event.pointer_id = from->fingerId;
-    to->inject_touch_event.position.screen_size = frame_size;
-    to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, x, y);
-    to->inject_touch_event.pressure = from->pressure;
-    to->inject_touch_event.buttons = 0;
-    return true;
-}
-
 void
-input_manager_process_touch(struct input_manager *im,
+input_manager_process_touch(struct input_manager *input_manager,
                             const SDL_TouchFingerEvent *event) {
     struct control_msg msg;
-    if (convert_touch(event, im->screen, &msg)) {
+    if (convert_touch(event, input_manager->screen->frame_size, &msg)) {
-        if (!controller_push_msg(im->controller, &msg)) {
+        if (!controller_push_msg(input_manager->controller, &msg)) {
             LOGW("Could not request 'inject touch event'");
         }
     }
 }
 
 static bool
-is_outside_device_screen(struct input_manager *im, int x, int y)
+is_outside_device_screen(struct input_manager *input_manager, int x, int y)
 {
-    return x < 0 || x >= im->screen->frame_size.width ||
+    return x < 0 || x >= input_manager->screen->frame_size.width ||
-           y < 0 || y >= im->screen->frame_size.height;
+           y < 0 || y >= input_manager->screen->frame_size.height;
-}
-
-static bool
-convert_mouse_button(const SDL_MouseButtonEvent *from, struct screen *screen,
-                     struct control_msg *to) {
-    to->type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
-
-    if (!convert_mouse_action(from->type, &to->inject_touch_event.action)) {
-        return false;
-    }
-
-    to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
-    to->inject_touch_event.position.screen_size = screen->frame_size;
-    to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, from->x, from->y);
-    to->inject_touch_event.pressure = 1.f;
-    to->inject_touch_event.buttons =
-        convert_mouse_buttons(SDL_BUTTON(from->button));
-
-    return true;
 }
 
 void
-input_manager_process_mouse_button(struct input_manager *im,
+input_manager_process_mouse_button(struct input_manager *input_manager,
                                    const SDL_MouseButtonEvent *event,
                                    bool control) {
     if (event->which == SDL_TOUCH_MOUSEID) {
         // simulated from touch events, so it's a duplicate
         return;
     }
+
     if (event->type == SDL_MOUSEBUTTONDOWN) {
         if (control && event->button == SDL_BUTTON_RIGHT) {
-            press_back_or_turn_screen_on(im->controller);
+            press_back_or_turn_screen_on(input_manager->controller);
             return;
         }
         if (control && event->button == SDL_BUTTON_MIDDLE) {
-            action_home(im->controller, ACTION_DOWN | ACTION_UP);
+            action_home(input_manager->controller, ACTION_DOWN | ACTION_UP);
             return;
         }
         // double-click on black borders resize to fit the device screen
-        if (event->button == SDL_BUTTON_LEFT && event->clicks == 2) {
+        if (event->button == SDL_BUTTON_LEFT) {
-            bool outside =
+            if (event->clicks >= 2) {
-                is_outside_device_screen(im, event->x, event->y);
+                bool outside =
-            if (outside) {
+                    is_outside_device_screen(input_manager, event->x, event->y);
-                screen_resize_to_fit(im->screen);
+                if (outside) {
-                return;
+                    screen_resize_to_fit(input_manager->screen);
+                    return;
+                }
+            }
+
+            struct virtual_finger *vfinger = &input_manager->vfinger;
+            if (input_manager->ctrl_down && !vfinger->down) {
+                vfinger->position.point.x = event->x;
+                vfinger->position.point.y = event->y;
+                vfinger->position.screen_size =
+                    input_manager->screen->frame_size,
+                simulate_virtual_finger(input_manager, true,
+                                        &vfinger->position);
             }
         }
         // otherwise, send the click event to the device
@@ -502,47 +507,23 @@ input_manager_process_mouse_button(struct input_manager *im,
     }
 
     struct control_msg msg;
-    if (convert_mouse_button(event, im->screen, &msg)) {
+    if (convert_mouse_button(event, input_manager->screen->frame_size, &msg)) {
-        if (!controller_push_msg(im->controller, &msg)) {
+        if (!controller_push_msg(input_manager->controller, &msg)) {
             LOGW("Could not request 'inject mouse button event'");
         }
     }
 }
 
-static bool
-convert_mouse_wheel(const SDL_MouseWheelEvent *from, struct screen *screen,
-                    struct control_msg *to) {
-
-    // mouse_x and mouse_y are expressed in pixels relatice to the window
-    int mouse_x;
-    int mouse_y;
-    SDL_GetMouseState(&mouse_x, &mouse_y);
-
-    struct position position = {
-        .screen_size = screen->frame_size,
-        .point = screen_convert_to_frame_coords(screen, mouse_x, mouse_y),
-    };
-
-    to->type = CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT;
-
-    to->inject_scroll_event.position = position;
-
-    int mul = from->direction == SDL_MOUSEWHEEL_NORMAL ? 1 : -1;
-    // SDL behavior seems inconsistent between horizontal and vertical scrolling
-    // so reverse the horizontal
-    // <https://wiki.libsdl.org/SDL_MouseWheelEvent#Remarks>
-    to->inject_scroll_event.hscroll = -mul * from->x;
-    to->inject_scroll_event.vscroll = mul * from->y;
-
-    return true;
-}
-
 void
-input_manager_process_mouse_wheel(struct input_manager *im,
+input_manager_process_mouse_wheel(struct input_manager *input_manager,
                                   const SDL_MouseWheelEvent *event) {
+    struct position position = {
+        .screen_size = input_manager->screen->frame_size,
+        .point = get_mouse_point(input_manager->screen),
+    };
     struct control_msg msg;
-    if (convert_mouse_wheel(event, im->screen, &msg)) {
+    if (convert_mouse_wheel(event, position, &msg)) {
-        if (!controller_push_msg(im->controller, &msg)) {
+        if (!controller_push_msg(input_manager->controller, &msg)) {
             LOGW("Could not request 'inject mouse wheel event'");
         }
     }

app/src/input_manager.h

@@ -14,32 +14,45 @@ struct input_manager {
     struct controller *controller;
     struct video_buffer *video_buffer;
     struct screen *screen;
+
+    bool ctrl_down;
+
+    struct virtual_finger {
+        bool down;
+        struct position position;
+    } vfinger;
 };
 
 void
-input_manager_process_text_input(struct input_manager *im,
+input_manager_init(struct input_manager *input_manager,
+                   struct controller *controller,
+                   struct video_buffer *video_buffer,
+                   struct screen *screen);
+
+void
+input_manager_process_text_input(struct input_manager *input_manager,
                                  const SDL_TextInputEvent *event);
 
 void
-input_manager_process_key(struct input_manager *im,
+input_manager_process_key(struct input_manager *input_manager,
                           const SDL_KeyboardEvent *event,
                           bool control);
 
 void
-input_manager_process_mouse_motion(struct input_manager *im,
+input_manager_process_mouse_motion(struct input_manager *input_manager,
                                    const SDL_MouseMotionEvent *event);
 
 void
-input_manager_process_touch(struct input_manager *im,
+input_manager_process_touch(struct input_manager *input_manager,
                             const SDL_TouchFingerEvent *event);
 
 void
-input_manager_process_mouse_button(struct input_manager *im,
+input_manager_process_mouse_button(struct input_manager *input_manager,
                                    const SDL_MouseButtonEvent *event,
                                    bool control);
 
 void
-input_manager_process_mouse_wheel(struct input_manager *im,
+input_manager_process_mouse_wheel(struct input_manager *input_manager,
                                   const SDL_MouseWheelEvent *event);
 
 #endif

app/src/scrcpy.c

@@ -37,12 +37,7 @@ static struct decoder decoder;
 static struct recorder recorder;
 static struct controller controller;
 static struct file_handler file_handler;
-
+static struct input_manager input_manager;
-static struct input_manager input_manager = {
-    .controller = &controller,
-    .video_buffer = &video_buffer,
-    .screen = &screen,
-};
 
 // init SDL and set appropriate hints
 static bool
@@ -145,10 +140,8 @@ handle_event(SDL_Event *event, bool control) {
         case SDL_WINDOWEVENT:
             switch (event->window.event) {
                 case SDL_WINDOWEVENT_EXPOSED:
-                    screen_render(&screen);
-                    break;
                 case SDL_WINDOWEVENT_SIZE_CHANGED:
-                    screen_window_resized(&screen);
+                    screen_render(&screen);
                     break;
             }
             break;
@@ -313,6 +306,8 @@ scrcpy(const struct scrcpy_options *options) {
         goto end;
     }
 
+    input_manager_init(&input_manager, &controller, &video_buffer, &screen);
+
     if (!server_connect_to(&server)) {
         goto end;
     }

app/src/screen.c

@@ -16,7 +16,7 @@
 
 // get the window size in a struct size
 static struct size
-get_window_size(SDL_Window *window) {
+get_native_window_size(SDL_Window *window) {
     int width;
     int height;
     SDL_GetWindowSize(window, &width, &height);
@@ -29,11 +29,11 @@ get_window_size(SDL_Window *window) {
 
 // get the windowed window size
 static struct size
-get_windowed_window_size(const struct screen *screen) {
+get_window_size(const struct screen *screen) {
     if (screen->fullscreen) {
         return screen->windowed_window_size;
     }
-    return get_window_size(screen->window);
+    return get_native_window_size(screen->window);
 }
 
 // set the window size to be applied when fullscreen is disabled
@@ -112,8 +112,8 @@ get_optimal_size(struct size current_size, struct size frame_size) {
 // same as get_optimal_size(), but read the current size from the window
 static inline struct size
 get_optimal_window_size(const struct screen *screen, struct size frame_size) {
-    struct size windowed_size = get_windowed_window_size(screen);
+    struct size current_size = get_window_size(screen);
-    return get_optimal_size(windowed_size, frame_size);
+    return get_optimal_size(current_size, frame_size);
 }
 
 // initially, there is no current size, so use the frame size as current size
@@ -122,32 +122,6 @@ get_initial_optimal_size(struct size frame_size) {
     return get_optimal_size(frame_size, frame_size);
 }
 
-static void
-update_frame_rect(struct screen *screen) {
-    int ww;
-    int wh;
-    SDL_GL_GetDrawableSize(screen->window, &ww, &wh);
-
-    // 32 bits because we need to multiply two 16 bits values
-    uint32_t fw = screen->frame_size.width;
-    uint32_t fh = screen->frame_size.height;
-
-    SDL_Rect *rect = &screen->rect;
-
-    bool keep_width = fw * wh > fh * ww;
-    if (keep_width) {
-        rect->x = 0;
-        rect->w = ww;
-        rect->h = ww * fh / fw;
-        rect->y = (wh - rect->h) / 2;
-    } else {
-        rect->y = 0;
-        rect->h = wh;
-        rect->w = wh * fw / fh;
-        rect->x = (ww - rect->w) / 2;
-    }
-}
-
 void
 screen_init(struct screen *screen) {
     *screen = (struct screen) SCREEN_INITIALIZER;
@@ -196,6 +170,13 @@ screen_init_rendering(struct screen *screen, const char *window_title,
         return false;
     }
 
+    if (SDL_RenderSetLogicalSize(screen->renderer, frame_size.width,
+                                 frame_size.height)) {
+        LOGE("Could not set renderer logical size: %s", SDL_GetError());
+        screen_destroy(screen);
+        return false;
+    }
+
     SDL_Surface *icon = read_xpm(icon_xpm);
     if (icon) {
         SDL_SetWindowIcon(screen->window, icon);
@@ -239,14 +220,20 @@ static bool
 prepare_for_frame(struct screen *screen, struct size new_frame_size) {
     if (screen->frame_size.width != new_frame_size.width
             || screen->frame_size.height != new_frame_size.height) {
+        if (SDL_RenderSetLogicalSize(screen->renderer, new_frame_size.width,
+                                     new_frame_size.height)) {
+            LOGE("Could not set renderer logical size: %s", SDL_GetError());
+            return false;
+        }
+
         // frame dimension changed, destroy texture
         SDL_DestroyTexture(screen->texture);
 
-        struct size windowed_size = get_windowed_window_size(screen);
+        struct size current_size = get_window_size(screen);
         struct size target_size = {
-            (uint32_t) windowed_size.width * new_frame_size.width
+            (uint32_t) current_size.width * new_frame_size.width
                     / screen->frame_size.width,
-            (uint32_t) windowed_size.height * new_frame_size.height
+            (uint32_t) current_size.height * new_frame_size.height
                     / screen->frame_size.height,
         };
         target_size = get_optimal_size(target_size, new_frame_size);
@@ -284,7 +271,6 @@ screen_update_frame(struct screen *screen, struct video_buffer *vb) {
         mutex_unlock(vb->mutex);
         return false;
     }
-    update_frame_rect(screen);
     update_texture(screen, frame);
     mutex_unlock(vb->mutex);
 
@@ -292,16 +278,10 @@ screen_update_frame(struct screen *screen, struct video_buffer *vb) {
     return true;
 }
 
-void
-screen_window_resized(struct screen *screen) {
-    update_frame_rect(screen);
-    screen_render(screen);
-}
-
 void
 screen_render(struct screen *screen) {
     SDL_RenderClear(screen->renderer);
-    SDL_RenderCopy(screen->renderer, screen->texture, NULL, &screen->rect);
+    SDL_RenderCopy(screen->renderer, screen->texture, NULL, NULL);
     SDL_RenderPresent(screen->renderer);
 }
 
@@ -309,7 +289,7 @@ void
 screen_switch_fullscreen(struct screen *screen) {
     if (!screen->fullscreen) {
         // going to fullscreen, store the current windowed window size
-        screen->windowed_window_size = get_window_size(screen->window);
+        screen->windowed_window_size = get_native_window_size(screen->window);
     }
     uint32_t new_mode = screen->fullscreen ? 0 : SDL_WINDOW_FULLSCREEN_DESKTOP;
     if (SDL_SetWindowFullscreen(screen->window, new_mode)) {
@@ -347,11 +327,3 @@ screen_resize_to_pixel_perfect(struct screen *screen) {
         LOGD("Resized to pixel-perfect");
     }
 }
-
-struct point
-screen_convert_to_frame_coords(struct screen *screen, float x, float y) {
-    struct point out;
-    out.x = (x - screen->rect.x) * screen->frame_size.width / screen->rect.w;
-    out.y = (y - screen->rect.y) * screen->frame_size.height / screen->rect.h;
-    return out;
-}

app/src/screen.h

@@ -17,7 +17,6 @@ struct screen {
     struct size frame_size;
     //used only in fullscreen mode to know the windowed window size
     struct size windowed_window_size;
-    struct SDL_Rect rect;
     bool has_frame;
     bool fullscreen;
     bool no_window;
@@ -35,15 +34,9 @@ struct screen {
         .width = 0,           \
         .height = 0,          \
     },                        \
-    .rect = {                 \
+    .has_frame = false,   \
-        .x = 0,               \
+    .fullscreen = false,  \
-        .y = 0,               \
+    .no_window = false,   \
-        .w = 0,               \
-        .h = 0,               \
-    },                        \
-    .has_frame = false,       \
-    .fullscreen = false,      \
-    .no_window = false,       \
 }
 
 // initialize default values
@@ -67,10 +60,6 @@ screen_destroy(struct screen *screen);
 bool
 screen_update_frame(struct screen *screen, struct video_buffer *vb);
 
-// update content after window resizing
-void
-screen_window_resized(struct screen *screen);
-
 // render the texture to the renderer
 void
 screen_render(struct screen *screen);
@@ -87,9 +76,4 @@ screen_resize_to_fit(struct screen *screen);
 void
 screen_resize_to_pixel_perfect(struct screen *screen);
 
-// convert point from window coordinates to frame coordinates
-// x and y are expressed in pixels (float to allow partial pixels)
-struct point
-screen_convert_to_frame_coords(struct screen *screen, float x, float y);
-
 #endif

server/src/main/java/com/genymobile/scrcpy/Device.java

@@ -161,11 +161,7 @@ public final class Device {
      * @param mode one of the {@code SCREEN_POWER_MODE_*} constants
      */
     public void setScreenPowerMode(int mode) {
-        IBinder d = SurfaceControl.getBuiltInDisplay();
+        IBinder d = SurfaceControl.getBuiltInDisplay(0);
-        if (d == null) {
-            Ln.e("Could not get built-in display");
-            return;
-        }
         SurfaceControl.setDisplayPowerMode(d, mode);
         Ln.i("Device screen turned " + (mode == Device.POWER_MODE_OFF ? "off" : "on"));
     }

server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java

@@ -1,102 +1,44 @@
 package com.genymobile.scrcpy.wrappers;
 
-import com.genymobile.scrcpy.Ln;
-
 import android.content.ClipData;
-import android.os.Build;
 import android.os.IInterface;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 
 public class ClipboardManager {
-
-    private static final String PACKAGE_NAME = "com.android.shell";
-    private static final int USER_ID = 0;
-
     private final IInterface manager;
-    private Method getPrimaryClipMethod;
+    private final Method getPrimaryClipMethod;
-    private Method setPrimaryClipMethod;
+    private final Method setPrimaryClipMethod;
 
     public ClipboardManager(IInterface manager) {
         this.manager = manager;
-    }
+        try {
-
+            getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class);
-    private Method getGetPrimaryClipMethod() {
+            setPrimaryClipMethod = manager.getClass().getMethod("setPrimaryClip", ClipData.class, String.class);
-        if (getPrimaryClipMethod == null) {
+        } catch (NoSuchMethodException e) {
-            try {
+            throw new AssertionError(e);
-                if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-                    getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class);
-                } else {
-                    getPrimaryClipMethod = manager.getClass().getMethod("getPrimaryClip", String.class, int.class);
-                }
-            } catch (NoSuchMethodException e) {
-                Ln.e("Could not find method", e);
-            }
-        }
-        return getPrimaryClipMethod;
-    }
-
-    private Method getSetPrimaryClipMethod() {
-        if (setPrimaryClipMethod == null) {
-            try {
-                if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-                    setPrimaryClipMethod = manager.getClass().getMethod("setPrimaryClip", ClipData.class, String.class);
-                } else {
-                    setPrimaryClipMethod = manager.getClass().getMethod("setPrimaryClip", ClipData.class,
-                            String.class, int.class);
-                }
-            } catch (NoSuchMethodException e) {
-                Ln.e("Could not find method", e);
-            }
-        }
-        return setPrimaryClipMethod;
-    }
-
-    private static ClipData getPrimaryClip(Method method, IInterface manager) throws InvocationTargetException,
-            IllegalAccessException {
-        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-            return (ClipData) method.invoke(manager, PACKAGE_NAME);
-        }
-        return (ClipData) method.invoke(manager, PACKAGE_NAME, USER_ID);
-    }
-
-    private static void setPrimaryClip(Method method, IInterface manager, ClipData clipData) throws InvocationTargetException,
-            IllegalAccessException {
-        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-            method.invoke(manager, clipData, PACKAGE_NAME);
-        } else {
-            method.invoke(manager, clipData, PACKAGE_NAME, USER_ID);
         }
     }
 
     public CharSequence getText() {
-        Method method = getGetPrimaryClipMethod();
-        if (method == null) {
-            return null;
-        }
         try {
-            ClipData clipData = getPrimaryClip(method, manager);
+            ClipData clipData = (ClipData) getPrimaryClipMethod.invoke(manager, "com.android.shell");
             if (clipData == null || clipData.getItemCount() == 0) {
                 return null;
             }
             return clipData.getItemAt(0).getText();
         } catch (InvocationTargetException | IllegalAccessException e) {
-            Ln.e("Could not invoke " + method.getName(), e);
+            throw new AssertionError(e);
-            return null;
         }
     }
 
     public void setText(CharSequence text) {
-        Method method = getSetPrimaryClipMethod();
-        if (method == null) {
-            return;
-        }
         ClipData clipData = ClipData.newPlainText(null, text);
         try {
-            setPrimaryClip(method, manager, clipData);
+            setPrimaryClipMethod.invoke(manager, clipData, "com.android.shell");
         } catch (InvocationTargetException | IllegalAccessException e) {
-            Ln.e("Could not invoke " + method.getName(), e);
+            throw new AssertionError(e);
         }
     }
 }

server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java

@@ -1,7 +1,5 @@
 package com.genymobile.scrcpy.wrappers;
 
-import com.genymobile.scrcpy.Ln;
-
 import android.os.IInterface;
 import android.view.InputEvent;
 
@@ -15,33 +13,22 @@ public final class InputManager {
     public static final int INJECT_INPUT_EVENT_MODE_WAIT_FOR_FINISH = 2;
 
     private final IInterface manager;
-    private Method injectInputEventMethod;
+    private final Method injectInputEventMethod;
 
     public InputManager(IInterface manager) {
         this.manager = manager;
-    }
+        try {
-
+            injectInputEventMethod = manager.getClass().getMethod("injectInputEvent", InputEvent.class, int.class);
-    private Method getInjectInputEventMethod() {
+        } catch (NoSuchMethodException e) {
-        if (injectInputEventMethod == null) {
+            throw new AssertionError(e);
-            try {
-                injectInputEventMethod = manager.getClass().getMethod("injectInputEvent", InputEvent.class, int.class);
-            } catch (NoSuchMethodException e) {
-                Ln.e("Could not find method", e);
-            }
         }
-        return injectInputEventMethod;
     }
 
     public boolean injectInputEvent(InputEvent inputEvent, int mode) {
-        Method method = getInjectInputEventMethod();
-        if (method == null) {
-            return false;
-        }
         try {
-            return (Boolean) method.invoke(manager, inputEvent, mode);
+            return (Boolean) injectInputEventMethod.invoke(manager, inputEvent, mode);
         } catch (InvocationTargetException | IllegalAccessException e) {
-            Ln.e("Could not invoke " + method.getName(), e);
+            throw new AssertionError(e);
-            return false;
         }
     }
 }

server/src/main/java/com/genymobile/scrcpy/wrappers/PowerManager.java

@@ -1,7 +1,5 @@
 package com.genymobile.scrcpy.wrappers;
 
-import com.genymobile.scrcpy.Ln;
-
 import android.annotation.SuppressLint;
 import android.os.Build;
 import android.os.IInterface;
@@ -11,35 +9,24 @@ import java.lang.reflect.Method;
 
 public final class PowerManager {
     private final IInterface manager;
-    private Method isScreenOnMethod;
+    private final Method isScreenOnMethod;
 
     public PowerManager(IInterface manager) {
         this.manager = manager;
-    }
+        try {
-
+            @SuppressLint("ObsoleteSdkInt") // we may lower minSdkVersion in the future
-    private Method getIsScreenOnMethod() {
+            String methodName = Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT_WATCH ? "isInteractive" : "isScreenOn";
-        if (isScreenOnMethod == null) {
+            isScreenOnMethod = manager.getClass().getMethod(methodName);
-            try {
+        } catch (NoSuchMethodException e) {
-                @SuppressLint("ObsoleteSdkInt") // we may lower minSdkVersion in the future
+            throw new AssertionError(e);
-                        String methodName = Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT_WATCH ? "isInteractive" : "isScreenOn";
-                isScreenOnMethod = manager.getClass().getMethod(methodName);
-            } catch (NoSuchMethodException e) {
-                Ln.e("Could not find method", e);
-            }
         }
-        return isScreenOnMethod;
     }
 
     public boolean isScreenOn() {
-        Method method = getIsScreenOnMethod();
-        if (method == null) {
-            return false;
-        }
         try {
-            return (Boolean) method.invoke(manager);
+            return (Boolean) isScreenOnMethod.invoke(manager);
         } catch (InvocationTargetException | IllegalAccessException e) {
-            Ln.e("Could not invoke " + method.getName(), e);
+            throw new AssertionError(e);
-            return false;
         }
     }
 }

server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java

@@ -17,49 +17,35 @@ public class StatusBarManager {
         this.manager = manager;
     }
 
-    private Method getExpandNotificationsPanelMethod() {
+    public void expandNotificationsPanel() {
         if (expandNotificationsPanelMethod == null) {
             try {
                 expandNotificationsPanelMethod = manager.getClass().getMethod("expandNotificationsPanel");
             } catch (NoSuchMethodException e) {
-                Ln.e("Could not find method", e);
+                Ln.e("ServiceBarManager.expandNotificationsPanel() is not available on this device");
+                return;
             }
         }
-        return expandNotificationsPanelMethod;
-    }
-
-    private Method getCollapsePanelsMethod() {
-        if (collapsePanelsMethod == null) {
-            try {
-                collapsePanelsMethod = manager.getClass().getMethod("collapsePanels");
-            } catch (NoSuchMethodException e) {
-                Ln.e("Could not find method", e);
-            }
-        }
-        return collapsePanelsMethod;
-    }
-
-    public void expandNotificationsPanel() {
-        Method method = getExpandNotificationsPanelMethod();
-        if (method == null) {
-            return;
-        }
         try {
-            method.invoke(manager);
+            expandNotificationsPanelMethod.invoke(manager);
         } catch (InvocationTargetException | IllegalAccessException e) {
-            Ln.e("Could not invoke " + method.getName(), e);
+            Ln.e("Could not invoke ServiceBarManager.expandNotificationsPanel()", e);
         }
     }
 
     public void collapsePanels() {
-        Method method = getCollapsePanelsMethod();
+        if (collapsePanelsMethod == null) {
-        if (method == null) {
+            try {
-            return;
+                collapsePanelsMethod = manager.getClass().getMethod("collapsePanels");
+            } catch (NoSuchMethodException e) {
+                Ln.e("ServiceBarManager.collapsePanels() is not available on this device");
+                return;
+            }
         }
         try {
-            method.invoke(manager);
+            collapsePanelsMethod.invoke(manager);
         } catch (InvocationTargetException | IllegalAccessException e) {
-            Ln.e("Could not invoke " + method.getName(), e);
+            Ln.e("Could not invoke ServiceBarManager.collapsePanels()", e);
         }
     }
 }

server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java

@@ -1,16 +1,11 @@
 package com.genymobile.scrcpy.wrappers;
 
-import com.genymobile.scrcpy.Ln;
-
 import android.annotation.SuppressLint;
 import android.graphics.Rect;
 import android.os.Build;
 import android.os.IBinder;
 import android.view.Surface;
 
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-
 @SuppressLint("PrivateApi")
 public final class SurfaceControl {
 
@@ -28,9 +23,6 @@ public final class SurfaceControl {
         }
     }
 
-    private static Method getBuiltInDisplayMethod;
-    private static Method setDisplayPowerModeMethod;
-
     private SurfaceControl() {
         // only static methods
     }
@@ -84,62 +76,24 @@ public final class SurfaceControl {
         }
     }
 
-    private static Method getGetBuiltInDisplayMethod() {
+    public static IBinder getBuiltInDisplay(int builtInDisplayId) {
-        if (getBuiltInDisplayMethod == null) {
-            try {
-                // the method signature has changed in Android Q
-                // <https://github.com/Genymobile/scrcpy/issues/586>
-                if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-                    getBuiltInDisplayMethod = CLASS.getMethod("getBuiltInDisplay", int.class);
-                } else {
-                    getBuiltInDisplayMethod = CLASS.getMethod("getInternalDisplayToken");
-                }
-            } catch (NoSuchMethodException e) {
-                Ln.e("Could not find method", e);
-            }
-        }
-        return getBuiltInDisplayMethod;
-    }
-
-    public static IBinder getBuiltInDisplay() {
-        Method method = getGetBuiltInDisplayMethod();
-        if (method == null) {
-            return null;
-        }
         try {
+            // the method signature has changed in Android Q
+            // <https://github.com/Genymobile/scrcpy/issues/586>
             if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-                // call getBuiltInDisplay(0)
+                return (IBinder) CLASS.getMethod("getBuiltInDisplay", int.class).invoke(null, builtInDisplayId);
-                return (IBinder) method.invoke(null, 0);
             }
-
+            return (IBinder) CLASS.getMethod("getPhysicalDisplayToken", long.class).invoke(null, builtInDisplayId);
-            // call getInternalDisplayToken()
+        } catch (Exception e) {
-            return (IBinder) method.invoke(null);
+            throw new AssertionError(e);
-        } catch (InvocationTargetException | IllegalAccessException e) {
-            Ln.e("Could not invoke " + method.getName(), e);
-            return null;
         }
     }
 
-    private static Method getSetDisplayPowerModeMethod() {
-        if (setDisplayPowerModeMethod == null) {
-            try {
-                setDisplayPowerModeMethod = CLASS.getMethod("setDisplayPowerMode", IBinder.class, int.class);
-            } catch (NoSuchMethodException e) {
-                Ln.e("Could not find method", e);
-            }
-        }
-        return setDisplayPowerModeMethod;
-    }
-
     public static void setDisplayPowerMode(IBinder displayToken, int mode) {
-        Method method = getSetDisplayPowerModeMethod();
-        if (method == null) {
-            return;
-        }
         try {
-            method.invoke(null, displayToken, mode);
+            CLASS.getMethod("setDisplayPowerMode", IBinder.class, int.class).invoke(null, displayToken, mode);
-        } catch (InvocationTargetException | IllegalAccessException e) {
+        } catch (Exception e) {
-            Ln.e("Could not invoke " + method.getName(), e);
+            throw new AssertionError(e);
         }
     }