Disable input events when necessary

Disable input events on secondary displays before Android 10, even if
FLAG_PRESENTATION is not set.

Ref #1288 <https://github.com/Genymobile/scrcpy/issues/1288>

app/src/input_manager.c

@@ -7,6 +7,33 @@
 #include "util/lock.h"
 #include "util/log.h"
 
+// Convert window coordinates (as provided by SDL_GetMouseState() to renderer
+// coordinates (as provided in SDL mouse events)
+//
+// See my question:
+// <https://stackoverflow.com/questions/49111054/how-to-get-mouse-position-on-mouse-wheel-event>
+static void
+convert_to_renderer_coordinates(SDL_Renderer *renderer, int *x, int *y) {
+    SDL_Rect viewport;
+    float scale_x, scale_y;
+    SDL_RenderGetViewport(renderer, &viewport);
+    SDL_RenderGetScale(renderer, &scale_x, &scale_y);
+    *x = (int) (*x / scale_x) - viewport.x;
+    *y = (int) (*y / scale_y) - viewport.y;
+}
+
+static struct point
+get_mouse_point(struct screen *screen) {
+    int x;
+    int y;
+    SDL_GetMouseState(&x, &y);
+    convert_to_renderer_coordinates(screen->renderer, &x, &y);
+    return (struct point) {
+        .x = x,
+        .y = y,
+    };
+}
+
 static const int ACTION_DOWN = 1;
 static const int ACTION_UP = 1 << 1;
 
@@ -415,6 +442,36 @@ input_manager_process_key(struct input_manager *im,
     }
 }
 
+static struct point
+rotate_position(struct screen *screen, int32_t x, int32_t y) {
+    unsigned rotation = screen->rotation;
+    assert(rotation < 4);
+
+    int32_t w = screen->content_size.width;
+    int32_t h = screen->content_size.height;
+    struct point result;
+    switch (rotation) {
+        case 0:
+            result.x = x;
+            result.y = y;
+            break;
+        case 1:
+            result.x = h - y;
+            result.y = x;
+            break;
+        case 2:
+            result.x = w - x;
+            result.y = h - y;
+            break;
+        default:
+            assert(rotation == 3);
+            result.x = y;
+            result.y = w - x;
+            break;
+    }
+    return result;
+}
+
 static bool
 convert_mouse_motion(const SDL_MouseMotionEvent *from, struct screen *screen,
                      struct control_msg *to) {
@@ -423,7 +480,7 @@ convert_mouse_motion(const SDL_MouseMotionEvent *from, struct screen *screen,
     to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
     to->inject_touch_event.position.screen_size = screen->frame_size;
     to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, from->x, from->y);
+        rotate_position(screen, from->x, from->y);
     to->inject_touch_event.pressure = 1.f;
     to->inject_touch_event.buttons = convert_mouse_buttons(from->state);
 
@@ -458,19 +515,14 @@ convert_touch(const SDL_TouchFingerEvent *from, struct screen *screen,
         return false;
     }
 
+    struct size frame_size = screen->frame_size;
+
     to->inject_touch_event.pointer_id = from->fingerId;
-    to->inject_touch_event.position.screen_size = screen->frame_size;
-
-    int ww;
-    int wh;
-    SDL_GL_GetDrawableSize(screen->window, &ww, &wh);
-
+    to->inject_touch_event.position.screen_size = frame_size;
     // SDL touch event coordinates are normalized in the range [0; 1]
-    int32_t x = from->x * ww;
-    int32_t y = from->y * wh;
-    to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, x, y);
-
+    float x = from->x * frame_size.width;
+    float y = from->y * frame_size.height;
+    to->inject_touch_event.position.point = rotate_position(screen, x, y);
     to->inject_touch_event.pressure = from->pressure;
     to->inject_touch_event.buttons = 0;
     return true;
@@ -487,6 +539,13 @@ input_manager_process_touch(struct input_manager *im,
     }
 }
 
+static bool
+is_outside_device_screen(struct input_manager *im, int x, int y)
+{
+    return x < 0 || x >= im->screen->content_size.width ||
+           y < 0 || y >= im->screen->content_size.height;
+}
+
 static bool
 convert_mouse_button(const SDL_MouseButtonEvent *from, struct screen *screen,
                      struct control_msg *to) {
@@ -499,7 +558,7 @@ convert_mouse_button(const SDL_MouseButtonEvent *from, struct screen *screen,
     to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
     to->inject_touch_event.position.screen_size = screen->frame_size;
     to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, from->x, from->y);
+        rotate_position(screen, from->x, from->y);
     to->inject_touch_event.pressure = 1.f;
     to->inject_touch_event.buttons =
         convert_mouse_buttons(SDL_BUTTON(from->button));
@@ -524,14 +583,10 @@ input_manager_process_mouse_button(struct input_manager *im,
             action_home(im->controller, ACTION_DOWN | ACTION_UP);
             return;
         }
-
         // double-click on black borders resize to fit the device screen
         if (event->button == SDL_BUTTON_LEFT && event->clicks == 2) {
-            int x = event->x;
-            int y = event->y;
-            SDL_Rect *r = &im->screen->rect;
-            bool outside = x < r->x || x >= r->x + r->w
-                        || y < r->y || y >= r->y + r->h;
+            bool outside =
+                is_outside_device_screen(im, event->x, event->y);
             if (outside) {
                 screen_resize_to_fit(im->screen);
                 return;
@@ -555,15 +610,9 @@ input_manager_process_mouse_button(struct input_manager *im,
 static bool
 convert_mouse_wheel(const SDL_MouseWheelEvent *from, struct screen *screen,
                     struct control_msg *to) {
-
-    // mouse_x and mouse_y are expressed in pixels relative to the window
-    int mouse_x;
-    int mouse_y;
-    SDL_GetMouseState(&mouse_x, &mouse_y);
-
     struct position position = {
         .screen_size = screen->frame_size,
-        .point = screen_convert_to_frame_coords(screen, mouse_x, mouse_y),
+        .point = get_mouse_point(screen),
     };
 
     to->type = CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT;

app/src/screen.c

@@ -87,26 +87,6 @@ get_preferred_display_bounds(struct size *bounds) {
     return true;
 }
 
-// Indicate if the width should be kept on computing the optimal rectangle size
-// to display the content in the window.
-// Return true if the width should be kept, false if the height should be kept.
-static bool
-should_keep_width(struct size window_size, struct size content_size) {
-    // 32 bits because we need to multiply two 16 bits values
-    uint32_t ww = window_size.width;
-    uint32_t wh = window_size.height;
-    uint32_t cw = content_size.width;
-    uint32_t ch = content_size.height;
-
-    // To avoid keeping alternatively width and height on successive resizes
-    // due to rounding to integer, always prefer height (arbitrarily) if in the
-    // error range.
-
-    // err = ceil of content aspect ratio
-    unsigned err = (cw + ch - 1) / ch;
-    return cw * wh > ch * (ww + err);
-}
-
 // return the optimal size of the window, with the following constraints:
 //  - it attempts to keep at least one dimension of the current_size (i.e. it
 //    crops the black borders)
@@ -119,33 +99,33 @@ get_optimal_size(struct size current_size, struct size content_size) {
         return current_size;
     }
 
-    struct size window_size;
-
     struct size display_size;
+    // 32 bits because we need to multiply two 16 bits values
+    uint32_t w;
+    uint32_t h;
+
     if (!get_preferred_display_bounds(&display_size)) {
         // could not get display bounds, do not constraint the size
-        window_size.width = current_size.width;
-        window_size.height = current_size.height;
+        w = current_size.width;
+        h = current_size.height;
     } else {
-        window_size.width = MIN(current_size.width, display_size.width);
-        window_size.height = MIN(current_size.height, display_size.height);
+        w = MIN(current_size.width, display_size.width);
+        h = MIN(current_size.height, display_size.height);
     }
 
-    bool keep_width = should_keep_width(window_size, content_size);
+    bool keep_width = content_size.width * h > content_size.height * w;
     if (keep_width) {
         // remove black borders on top and bottom
-        window_size.height = content_size.height * window_size.width
-                           / content_size.width;
+        h = content_size.height * w / content_size.width;
     } else {
         // remove black borders on left and right (or none at all if it already
         // fits)
-        window_size.width = content_size.width * window_size.height
-                          / content_size.height;
+        w = content_size.width * h / content_size.height;
     }
 
-    // width and height must fit into 16 bits
-    assert(window_size.width < 0x10000 && window_size.height < 0x10000);
-    return window_size;
+    // w and h must fit into 16 bits
+    assert(w < 0x10000 && h < 0x10000);
+    return (struct size) {w, h};
 }
 
 // same as get_optimal_size(), but read the current size from the window
@@ -182,34 +162,6 @@ get_initial_optimal_size(struct size content_size, uint16_t req_width,
     return window_size;
 }
 
-static void
-update_content_rect(struct screen *screen) {
-    int w;
-    int h;
-    SDL_GL_GetDrawableSize(screen->window, &w, &h);
-
-    struct size window_size = {w, h};
-    uint16_t ww = window_size.width;
-    uint16_t wh = window_size.height;
-    uint16_t cw = screen->content_size.width;
-    uint16_t ch = screen->content_size.height;
-
-    SDL_Rect *rect = &screen->rect;
-
-    bool keep_width = should_keep_width(window_size, screen->content_size);
-    if (keep_width) {
-        rect->x = 0;
-        rect->w = ww;
-        rect->h = ww * ch / cw;
-        rect->y = (wh - rect->h) / 2;
-    } else {
-        rect->y = 0;
-        rect->h = wh;
-        rect->w = wh * cw / ch;
-        rect->x = (ww - rect->w) / 2;
-    }
-}
-
 void
 screen_init(struct screen *screen) {
     *screen = (struct screen) SCREEN_INITIALIZER;
@@ -299,6 +251,13 @@ screen_init_rendering(struct screen *screen, const char *window_title,
     const char *renderer_name = r ? NULL : renderer_info.name;
     LOGI("Renderer: %s", renderer_name ? renderer_name : "(unknown)");
 
+    if (SDL_RenderSetLogicalSize(screen->renderer, content_size.width,
+                                 content_size.height)) {
+        LOGE("Could not set renderer logical size: %s", SDL_GetError());
+        screen_destroy(screen);
+        return false;
+    }
+
     // stats with "opengl"
     screen->use_opengl = renderer_name && !strncmp(renderer_name, "opengl", 6);
     if (screen->use_opengl) {
@@ -342,8 +301,6 @@ screen_init_rendering(struct screen *screen, const char *window_title,
         return false;
     }
 
-    update_content_rect(screen);
-
     screen->windowed_window_size = window_size;
 
     return true;
@@ -378,6 +335,13 @@ screen_set_rotation(struct screen *screen, unsigned rotation) {
     struct size new_content_size =
         get_rotated_size(screen->frame_size, rotation);
 
+    if (SDL_RenderSetLogicalSize(screen->renderer,
+                                 new_content_size.width,
+                                 new_content_size.height)) {
+        LOGE("Could not set renderer logical size: %s", SDL_GetError());
+        return;
+    }
+
     struct size windowed_size = get_windowed_window_size(screen);
     struct size target_size = {
         .width = (uint32_t) windowed_size.width * new_content_size.width
@@ -392,7 +356,6 @@ screen_set_rotation(struct screen *screen, unsigned rotation) {
     screen->rotation = rotation;
     LOGI("Display rotation set to %u", rotation);
 
-    update_content_rect(screen);
     screen_render(screen);
 }
 
@@ -401,11 +364,18 @@ static bool
 prepare_for_frame(struct screen *screen, struct size new_frame_size) {
     if (screen->frame_size.width != new_frame_size.width
             || screen->frame_size.height != new_frame_size.height) {
+        struct size new_content_size =
+            get_rotated_size(new_frame_size, screen->rotation);
+        if (SDL_RenderSetLogicalSize(screen->renderer,
+                                     new_content_size.width,
+                                     new_content_size.height)) {
+            LOGE("Could not set renderer logical size: %s", SDL_GetError());
+            return false;
+        }
+
         // frame dimension changed, destroy texture
         SDL_DestroyTexture(screen->texture);
 
-        struct size new_content_size =
-            get_rotated_size(new_frame_size, screen->rotation);
         struct size content_size = screen->content_size;
         struct size windowed_size = get_windowed_window_size(screen);
         struct size target_size = {
@@ -419,7 +389,6 @@ prepare_for_frame(struct screen *screen, struct size new_frame_size) {
 
         screen->frame_size = new_frame_size;
         screen->content_size = new_content_size;
-        update_content_rect(screen);
 
         LOGI("New texture: %" PRIu16 "x%" PRIu16,
                      screen->frame_size.width, screen->frame_size.height);
@@ -465,17 +434,11 @@ screen_update_frame(struct screen *screen, struct video_buffer *vb) {
     return true;
 }
 
-void
-screen_window_resized(struct screen *screen) {
-    update_content_rect(screen);
-    screen_render(screen);
-}
-
 void
 screen_render(struct screen *screen) {
     SDL_RenderClear(screen->renderer);
     if (screen->rotation == 0) {
-        SDL_RenderCopy(screen->renderer, screen->texture, NULL, &screen->rect);
+        SDL_RenderCopy(screen->renderer, screen->texture, NULL, NULL);
     } else {
         // rotation in RenderCopyEx() is clockwise, while screen->rotation is
         // counterclockwise (to be consistent with --lock-video-orientation)
@@ -485,14 +448,12 @@ screen_render(struct screen *screen) {
         SDL_Rect *dstrect = NULL;
         SDL_Rect rect;
         if (screen->rotation & 1) {
-            rect.x = screen->rect.x + (screen->rect.w - screen->rect.h) / 2;
-            rect.y = screen->rect.y + (screen->rect.h - screen->rect.w) / 2;
-            rect.w = screen->rect.h;
-            rect.h = screen->rect.w;
+            struct size size = screen->content_size;
+            rect.x = (size.width - size.height) / 2;
+            rect.y = (size.height - size.width) / 2;
+            rect.w = size.height;
+            rect.h = size.width;
             dstrect = &rect;
-        } else {
-            assert(screen->rotation == 2);
-            dstrect = &screen->rect;
         }
 
         SDL_RenderCopyEx(screen->renderer, screen->texture, NULL, dstrect,
@@ -530,8 +491,7 @@ screen_resize_to_fit(struct screen *screen) {
     struct size optimal_size =
         get_optimal_window_size(screen, screen->content_size);
     SDL_SetWindowSize(screen->window, optimal_size.width, optimal_size.height);
-    LOGD("Resized to optimal size: %ux%u", optimal_size.width,
-                                           optimal_size.height);
+    LOGD("Resized to optimal size");
 }
 
 void
@@ -547,8 +507,7 @@ screen_resize_to_pixel_perfect(struct screen *screen) {
 
     struct size content_size = screen->content_size;
     SDL_SetWindowSize(screen->window, content_size.width, content_size.height);
-    LOGD("Resized to pixel-perfect: %ux%u", content_size.width,
-                                            content_size.height);
+    LOGD("Resized to pixel-perfect");
 }
 
 void
@@ -571,7 +530,7 @@ screen_handle_window_event(struct screen *screen,
                 // window is maximized or fullscreen is enabled.
                 screen->windowed_window_size = get_window_size(screen->window);
             }
-            screen_window_resized(screen);
+            screen_render(screen);
             break;
         case SDL_WINDOWEVENT_MAXIMIZED:
             // The backup size must be non-nul.
@@ -592,44 +551,3 @@ screen_handle_window_event(struct screen *screen,
             break;
     }
 }
-
-struct point
-screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y) {
-    unsigned rotation = screen->rotation;
-    assert(rotation < 4);
-
-    int32_t w = screen->content_size.width;
-    int32_t h = screen->content_size.height;
-
-    // take the HiDPI scaling (dw/ww and dh/wh) into account
-    int ww, wh, dw, dh;
-    SDL_GetWindowSize(screen->window, &ww, &wh);
-    SDL_GL_GetDrawableSize(screen->window, &dw, &dh);
-
-    // scale (64 bits for intermediate multiplications)
-    x = (int64_t) (x - screen->rect.x) * w * dw / (screen->rect.w * ww);
-    y = (int64_t) (y - screen->rect.y) * h * dh / (screen->rect.h * wh);
-
-    // rotate
-    struct point result;
-    switch (rotation) {
-        case 0:
-            result.x = x;
-            result.y = y;
-            break;
-        case 1:
-            result.x = h - y;
-            result.y = x;
-            break;
-        case 2:
-            result.x = w - x;
-            result.y = h - y;
-            break;
-        default:
-            assert(rotation == 3);
-            result.x = y;
-            result.y = w - x;
-            break;
-    }
-    return result;
-}

app/src/screen.h

@@ -28,8 +28,6 @@ struct screen {
     struct size windowed_window_size_backup;
     // client rotation: 0, 1, 2 or 3 (x90 degrees counterclockwise)
     unsigned rotation;
-    // rectangle of the content (excluding black borders)
-    struct SDL_Rect rect;
     bool has_frame;
     bool fullscreen;
     bool maximized;
@@ -60,12 +58,6 @@ struct screen {
         .height = 0, \
     }, \
     .rotation = 0, \
-    .rect = { \
-        .x = 0, \
-        .y = 0, \
-        .w = 0, \
-        .h = 0, \
-    }, \
     .has_frame = false, \
     .fullscreen = false, \
     .maximized = false, \
@@ -98,10 +90,6 @@ screen_destroy(struct screen *screen);
 bool
 screen_update_frame(struct screen *screen, struct video_buffer *vb);
 
-// update content after window resizing
-void
-screen_window_resized(struct screen *screen);
-
 // render the texture to the renderer
 void
 screen_render(struct screen *screen);
@@ -126,9 +114,4 @@ screen_set_rotation(struct screen *screen, unsigned rotation);
 void
 screen_handle_window_event(struct screen *screen, const SDL_WindowEvent *event);
 
-// convert point from window coordinates to frame coordinates
-// x and y are expressed in pixels
-struct point
-screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y);
-
 #endif

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -136,12 +136,12 @@ public class ScreenEncoder implements Device.RotationListener {
     }
 
     private static MediaCodec createCodec() throws IOException {
-        return MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
+        return MediaCodec.createEncoderByType("video/avc");
     }
 
     private static MediaFormat createFormat(int bitRate, int maxFps, int iFrameInterval) {
         MediaFormat format = new MediaFormat();
-        format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_AVC);
+        format.setString(MediaFormat.KEY_MIME, "video/avc");
         format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
         // must be present to configure the encoder, but does not impact the actual frame rate, which is variable
         format.setInteger(MediaFormat.KEY_FRAME_RATE, 60);

server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java

@@ -53,7 +53,8 @@ public final class InputManager {
             method.invoke(inputEvent, displayId);
             return true;
         } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
-            Ln.e("Cannot associate a display id to the input event", e);
+            // just a warning, it might happen on old devices
+            Ln.w("Cannot associate a display id to the input event");
             return false;
         }
     }