relative-mouse-mode

This will allow the mouse processor to handle relative motion easily.

app/src/control_msg.c

@@ -119,7 +119,8 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
                              (uint32_t) msg->inject_scroll_event.hscroll);
             buffer_write32be(&buf[17],
                              (uint32_t) msg->inject_scroll_event.vscroll);
-            return 21;
+            buffer_write32be(&buf[21], msg->inject_scroll_event.buttons);
+            return 25;
         case CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON:
             buf[1] = msg->inject_keycode.action;
             return 2;
@@ -192,11 +193,12 @@ control_msg_log(const struct control_msg *msg) {
         }
         case CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT:
             LOG_CMSG("scroll position=%" PRIi32 ",%" PRIi32 " hscroll=%" PRIi32
-                         " vscroll=%" PRIi32,
+                         " vscroll=%" PRIi32 " buttons=%06lx",
                      msg->inject_scroll_event.position.point.x,
                      msg->inject_scroll_event.position.point.y,
                      msg->inject_scroll_event.hscroll,
-                     msg->inject_scroll_event.vscroll);
+                     msg->inject_scroll_event.vscroll,
+                     (long) msg->inject_scroll_event.buttons);
             break;
         case CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON:
             LOG_CMSG("back-or-screen-on %s",

app/src/control_msg.h

@@ -70,6 +70,7 @@ struct control_msg {
             struct sc_position position;
             int32_t hscroll;
             int32_t vscroll;
+            enum android_motionevent_buttons buttons;
         } inject_scroll_event;
         struct {
             enum android_keyevent_action action; // action for the BACK key

app/src/hid_mouse.c

@@ -43,24 +43,24 @@ static const unsigned char mouse_report_desc[]  = {
 
     // Usage Minimum (1)
     0x19, 0x01,
-     // Usage Maximum (3)
-    0x29, 0x03,
+     // Usage Maximum (5)
+    0x29, 0x05,
     // Logical Minimum (0)
     0x15, 0x00,
     // Logical Maximum (1)
     0x25, 0x01,
-    // Report Count (3)
-    0x95, 0x03,
+    // Report Count (5)
+    0x95, 0x05,
     // Report Size (1)
     0x75, 0x01,
-    // Input (Data, Variable, Absolute): 3 buttons bits
+    // Input (Data, Variable, Absolute): 5 buttons bits
     0x81, 0x02,
 
     // Report Count (1)
     0x95, 0x01,
-    // Report Size (5)
-    0x75, 0x05,
-    // Input (Constant): 5 bits padding
+    // Report Size (3)
+    0x75, 0x03,
+    // Input (Constant): 3 bits padding
     0x81, 0x01,
 
     // Usage Page (Generic Desktop)
@@ -98,12 +98,14 @@ static const unsigned char mouse_report_desc[]  = {
  *
  *                   7 6 5 4 3 2 1 0
  *                  +---------------+
- *         byte 0:  |0 0 0 0 0 . . .| buttons state
+ *         byte 0:  |0 0 0 . . . . .| buttons state
  *                  +---------------+
- *                             ^ ^ ^
- *                             | | `- left button
- *                             | `--- right button
- *                             `----- middle button
+ *                         ^ ^ ^ ^ ^
+ *                         | | | | `- left button
+ *                         | | | `--- right button
+ *                         | | `----- middle button
+ *                         | `------- button 4
+ *                         `--------- button 5
  *
  *                  +---------------+
  *         byte 1:  |. . . . . . . .| relative x motion
@@ -152,7 +154,12 @@ buttons_state_to_hid_buttons(uint8_t buttons_state) {
     if (buttons_state & SC_MOUSE_BUTTON_MIDDLE) {
         c |= 1 << 2;
     }
-    // TODO buttons 4 and 5?
+    if (buttons_state & SC_MOUSE_BUTTON_X1) {
+        c |= 1 << 3;
+    }
+    if (buttons_state & SC_MOUSE_BUTTON_X2) {
+        c |= 1 << 4;
+    }
     return c;
 }
 

app/src/input_events.h

@@ -361,6 +361,7 @@ struct sc_mouse_scroll_event {
     struct sc_position position;
     int32_t hscroll;
     int32_t vscroll;
+    uint8_t buttons_state; // bitwise-OR of sc_mouse_button values
 };
 
 struct sc_mouse_motion_event {

app/src/input_manager.c

@@ -4,6 +4,7 @@
 #include <SDL2/SDL_keycode.h>
 
 #include "input_events.h"
+#include "screen.h"
 #include "util/log.h"
 
 static inline uint16_t
@@ -121,24 +122,22 @@ is_shortcut_mod(struct input_manager *im, uint16_t sdl_mod) {
 }
 
 void
-input_manager_init(struct input_manager *im, struct controller *controller,
-                   struct screen *screen, struct sc_key_processor *kp,
-                   struct sc_mouse_processor *mp,
-                   const struct scrcpy_options *options) {
-    assert(!options->control || (kp && kp->ops));
-    assert(!options->control || (mp && mp->ops));
+input_manager_init(struct input_manager *im,
+                   const struct input_manager_params *params) {
+    assert(!params->control || (params->kp && params->kp->ops));
+    assert(!params->control || (params->mp && params->mp->ops));
 
-    im->controller = controller;
-    im->screen = screen;
-    im->kp = kp;
-    im->mp = mp;
+    im->controller = params->controller;
+    im->screen = params->screen;
+    im->kp = params->kp;
+    im->mp = params->mp;
 
-    im->control = options->control;
-    im->forward_all_clicks = options->forward_all_clicks;
-    im->legacy_paste = options->legacy_paste;
-    im->clipboard_autosync = options->clipboard_autosync;
+    im->control = params->control;
+    im->forward_all_clicks = params->forward_all_clicks;
+    im->legacy_paste = params->legacy_paste;
+    im->clipboard_autosync = params->clipboard_autosync;
 
-    const struct sc_shortcut_mods *shortcut_mods = &options->shortcut_mods;
+    const struct sc_shortcut_mods *shortcut_mods = params->shortcut_mods;
     assert(shortcut_mods->count);
     assert(shortcut_mods->count < SC_MAX_SHORTCUT_MODS);
     for (unsigned i = 0; i < shortcut_mods->count; ++i) {
@@ -777,6 +776,12 @@ input_manager_process_mouse_button(struct input_manager *im,
     assert(im->mp->ops->process_mouse_click);
     im->mp->ops->process_mouse_click(im->mp, &evt);
 
+    if (im->mp->relative_mode) {
+        assert(!im->vfinger_down); // vfinger must not be used in relative mode
+        // No pinch-to-zoom simulation
+        return;
+    }
+
     // Pinch-to-zoom simulation.
     //
     // If Ctrl is hold when the left-click button is pressed, then
@@ -814,7 +819,7 @@ input_manager_process_mouse_wheel(struct input_manager *im,
     // mouse_x and mouse_y are expressed in pixels relative to the window
     int mouse_x;
     int mouse_y;
-    SDL_GetMouseState(&mouse_x, &mouse_y);
+    uint32_t buttons = SDL_GetMouseState(&mouse_x, &mouse_y);
 
     struct sc_mouse_scroll_event evt = {
         .position = {
@@ -824,6 +829,8 @@ input_manager_process_mouse_wheel(struct input_manager *im,
         },
         .hscroll = event->x,
         .vscroll = event->y,
+        .buttons_state =
+            sc_mouse_buttons_state_from_sdl(buttons, im->forward_all_clicks),
     };
 
     im->mp->ops->process_mouse_scroll(im->mp, &evt);

app/src/input_manager.h

@@ -10,10 +10,11 @@
 #include "controller.h"
 #include "fps_counter.h"
 #include "options.h"
-#include "screen.h"
 #include "trait/key_processor.h"
 #include "trait/mouse_processor.h"
 
+struct screen; // forward declaration to avoid cycle
+
 struct input_manager {
     struct controller *controller;
     struct screen *screen;
@@ -43,11 +44,22 @@ struct input_manager {
     uint64_t next_sequence; // used for request acknowledgements
 };
 
+struct input_manager_params {
+    struct controller *controller;
+    struct screen *screen;
+    struct sc_key_processor *kp;
+    struct sc_mouse_processor *mp;
+
+    bool control;
+    bool forward_all_clicks;
+    bool legacy_paste;
+    bool clipboard_autosync;
+    const struct sc_shortcut_mods *shortcut_mods;
+};
+
 void
-input_manager_init(struct input_manager *im, struct controller *controller,
-                   struct screen *screen, struct sc_key_processor *kp,
-                   struct sc_mouse_processor *mp,
-                   const struct scrcpy_options *options);
+input_manager_init(struct input_manager *im,
+                   const struct input_manager_params *params);
 
 bool
 input_manager_handle_event(struct input_manager *im, SDL_Event *event);

app/src/mouse_inject.c

@@ -108,6 +108,7 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp,
             .position = event->position,
             .hscroll = event->hscroll,
             .vscroll = event->vscroll,
+            .buttons = convert_mouse_buttons(event->buttons_state),
         },
     };
 
@@ -150,4 +151,6 @@ sc_mouse_inject_init(struct sc_mouse_inject *mi,
     };
 
     mi->mouse_processor.ops = &ops;
+
+    mi->mouse_processor.relative_mode = false;
 }

app/src/scrcpy.c

@@ -17,7 +17,6 @@
 #include "decoder.h"
 #include "events.h"
 #include "file_handler.h"
-#include "input_manager.h"
 #ifdef HAVE_AOA_HID
 # include "hid_keyboard.h"
 # include "hid_mouse.h"
@@ -63,7 +62,6 @@ struct scrcpy {
         struct sc_hid_mouse mouse_hid;
 #endif
     };
-    struct input_manager input_manager;
 };
 
 static inline void
@@ -195,11 +193,6 @@ handle_event(struct scrcpy *s, const struct scrcpy_options *options,
     }
 
     bool consumed = screen_handle_event(&s->screen, event);
-    if (consumed) {
-        goto end;
-    }
-
-    consumed = input_manager_handle_event(&s->input_manager, event);
     (void) consumed;
 
 end:
@@ -456,6 +449,9 @@ scrcpy(struct scrcpy_options *options) {
         stream_add_sink(&s->stream, &rec->packet_sink);
     }
 
+    struct sc_key_processor *kp = NULL;
+    struct sc_mouse_processor *mp = NULL;
+
     if (options->control) {
 #ifdef HAVE_AOA_HID
         if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
@@ -487,59 +483,7 @@ scrcpy(struct scrcpy_options *options) {
                 LOGW("Could not request 'set screen power mode'");
             }
         }
-    }
 
-    if (options->display) {
-        const char *window_title =
-            options->window_title ? options->window_title : info->device_name;
-
-        struct screen_params screen_params = {
-            .window_title = window_title,
-            .frame_size = info->frame_size,
-            .always_on_top = options->always_on_top,
-            .window_x = options->window_x,
-            .window_y = options->window_y,
-            .window_width = options->window_width,
-            .window_height = options->window_height,
-            .window_borderless = options->window_borderless,
-            .rotation = options->rotation,
-            .mipmaps = options->mipmaps,
-            .fullscreen = options->fullscreen,
-            .buffering_time = options->display_buffer,
-        };
-
-        if (!screen_init(&s->screen, &screen_params)) {
-            goto end;
-        }
-        screen_initialized = true;
-
-        decoder_add_sink(&s->decoder, &s->screen.frame_sink);
-    }
-
-#ifdef HAVE_V4L2
-    if (options->v4l2_device) {
-        if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device,
-                               info->frame_size, options->v4l2_buffer)) {
-            goto end;
-        }
-
-        decoder_add_sink(&s->decoder, &s->v4l2_sink.frame_sink);
-
-        v4l2_sink_initialized = true;
-    }
-#endif
-
-    // now we consumed the header values, the socket receives the video stream
-    // start the stream
-    if (!stream_start(&s->stream)) {
-        goto end;
-    }
-    stream_started = true;
-
-    struct sc_key_processor *kp = NULL;
-    struct sc_mouse_processor *mp = NULL;
-
-    if (options->control) {
         if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
 #ifdef HAVE_AOA_HID
             bool aoa_hid_ok = false;
@@ -592,8 +536,60 @@ aoa_hid_end:
         mp = &s->mouse_hid.mouse_processor;
     }
 
-    input_manager_init(&s->input_manager, &s->controller, &s->screen, kp, mp,
-                       options);
+    if (options->display) {
+        const char *window_title =
+            options->window_title ? options->window_title : info->device_name;
+
+        struct screen_params screen_params = {
+            .controller = &s->controller,
+            .kp = kp,
+            .mp = mp,
+            .control = options->control,
+            .forward_all_clicks = options->forward_all_clicks,
+            .legacy_paste = options->legacy_paste,
+            .clipboard_autosync = options->clipboard_autosync,
+            .shortcut_mods = &options->shortcut_mods,
+            .window_title = window_title,
+            .frame_size = info->frame_size,
+            .always_on_top = options->always_on_top,
+            .window_x = options->window_x,
+            .window_y = options->window_y,
+            .window_width = options->window_width,
+            .window_height = options->window_height,
+            .window_borderless = options->window_borderless,
+            .rotation = options->rotation,
+            .mipmaps = options->mipmaps,
+            .fullscreen = options->fullscreen,
+            .buffering_time = options->display_buffer,
+        };
+
+        if (!screen_init(&s->screen, &screen_params)) {
+            goto end;
+        }
+        screen_initialized = true;
+
+        decoder_add_sink(&s->decoder, &s->screen.frame_sink);
+    }
+
+#ifdef HAVE_V4L2
+    if (options->v4l2_device) {
+        if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device,
+                               info->frame_size, options->v4l2_buffer)) {
+            goto end;
+        }
+
+        decoder_add_sink(&s->decoder, &s->v4l2_sink.frame_sink);
+
+        v4l2_sink_initialized = true;
+    }
+#endif
+
+    // now we consumed the header values, the socket receives the video stream
+    // start the stream
+    if (!stream_start(&s->stream)) {
+        goto end;
+    }
+    stream_started = true;
 
     ret = event_loop(s, options);
     LOGD("quit...");

app/src/screen.c

@@ -470,6 +470,20 @@ screen_init(struct screen *screen, const struct screen_params *params) {
         goto error_destroy_texture;
     }
 
+    struct input_manager_params im_params = {
+        .controller = params->controller,
+        .screen = screen,
+        .kp = params->kp,
+        .mp = params->mp,
+        .control = params->control,
+        .forward_all_clicks = params->forward_all_clicks,
+        .legacy_paste = params->legacy_paste,
+        .clipboard_autosync = params->clipboard_autosync,
+        .shortcut_mods = params->shortcut_mods,
+    };
+
+    input_manager_init(&screen->im, &im_params);
+
     // Reset the window size to trigger a SIZE_CHANGED event, to workaround
     // HiDPI issues with some SDL renderers when several displays having
     // different HiDPI scaling are connected
@@ -777,7 +791,7 @@ screen_handle_event(struct screen *screen, SDL_Event *event) {
             return true;
     }
 
-    return false;
+    return input_manager_handle_event(&screen->im, event);
 }
 
 struct sc_point

app/src/screen.h

@@ -7,10 +7,14 @@
 #include <SDL2/SDL.h>
 #include <libavformat/avformat.h>
 
+#include "controller.h"
 #include "coords.h"
 #include "fps_counter.h"
+#include "input_manager.h"
 #include "opengl.h"
+#include "trait/key_processor.h"
 #include "trait/frame_sink.h"
+#include "trait/mouse_processor.h"
 #include "video_buffer.h"
 
 struct screen {
@@ -20,6 +24,7 @@ struct screen {
     bool open; // track the open/close state to assert correct behavior
 #endif
 
+    struct input_manager im;
     struct sc_video_buffer vb;
     struct fps_counter fps_counter;
 
@@ -50,6 +55,16 @@ struct screen {
 };
 
 struct screen_params {
+    struct controller *controller;
+    struct sc_key_processor *kp;
+    struct sc_mouse_processor *mp;
+
+    bool control;
+    bool forward_all_clicks;
+    bool legacy_paste;
+    bool clipboard_autosync;
+    const struct sc_shortcut_mods *shortcut_mods;
+
     const char *window_title;
     struct sc_size frame_size;
     bool always_on_top;

app/src/trait/mouse_processor.h

@@ -16,6 +16,13 @@
  */
 struct sc_mouse_processor {
     const struct sc_mouse_processor_ops *ops;
+
+    /**
+     * If set, the mouse processor works in relative mode (the absolute
+     * position is irrelevant). In particular, it indicates that the mouse
+     * pointer must be "captured" by the UI.
+     */
+    bool relative_mode;
 };
 
 struct sc_mouse_processor_ops {

app/tests/test_adb_parser.c

@@ -11,6 +11,7 @@ static void test_get_ip_single_line() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.12.34"));
+    free(ip);
 }
 
 static void test_get_ip_single_line_without_eol() {
@@ -20,6 +21,7 @@ static void test_get_ip_single_line_without_eol() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.12.34"));
+    free(ip);
 }
 
 static void test_get_ip_single_line_with_trailing_space() {
@@ -29,6 +31,7 @@ static void test_get_ip_single_line_with_trailing_space() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.12.34"));
+    free(ip);
 }
 
 static void test_get_ip_multiline_first_ok() {
@@ -40,6 +43,7 @@ static void test_get_ip_multiline_first_ok() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.1.2"));
+    free(ip);
 }
 
 static void test_get_ip_multiline_second_ok() {
@@ -51,6 +55,7 @@ static void test_get_ip_multiline_second_ok() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.1.3"));
+    free(ip);
 }
 
 static void test_get_ip_no_wlan() {

app/tests/test_control_msg_serialize.c

@@ -126,12 +126,13 @@ static void test_serialize_inject_scroll_event(void) {
             },
             .hscroll = 1,
             .vscroll = -1,
+            .buttons = 1,
         },
     };
 
     unsigned char buf[CONTROL_MSG_MAX_SIZE];
     size_t size = control_msg_serialize(&msg, buf);
-    assert(size == 21);
+    assert(size == 25);
 
     const unsigned char expected[] = {
         CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT,
@@ -139,6 +140,7 @@ static void test_serialize_inject_scroll_event(void) {
         0x04, 0x38, 0x07, 0x80, // 1080 1920
         0x00, 0x00, 0x00, 0x01, // 1
         0xFF, 0xFF, 0xFF, 0xFF, // -1
+        0x00, 0x00, 0x00, 0x01, // 1
     };
     assert(!memcmp(buf, expected, sizeof(expected)));
 }

server/src/main/java/com/genymobile/scrcpy/ControlMessage.java

@@ -71,12 +71,13 @@ public final class ControlMessage {
         return msg;
     }
 
-    public static ControlMessage createInjectScrollEvent(Position position, int hScroll, int vScroll) {
+    public static ControlMessage createInjectScrollEvent(Position position, int hScroll, int vScroll, int buttons) {
         ControlMessage msg = new ControlMessage();
         msg.type = TYPE_INJECT_SCROLL_EVENT;
         msg.position = position;
         msg.hScroll = hScroll;
         msg.vScroll = vScroll;
+        msg.buttons = buttons;
         return msg;
     }
 

server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java

@@ -10,7 +10,7 @@ public class ControlMessageReader {
 
     static final int INJECT_KEYCODE_PAYLOAD_LENGTH = 13;
     static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 27;
-    static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
+    static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 24;
     static final int BACK_OR_SCREEN_ON_LENGTH = 1;
     static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
     static final int GET_CLIPBOARD_LENGTH = 1;
@@ -154,7 +154,8 @@ public class ControlMessageReader {
         Position position = readPosition(buffer);
         int hScroll = buffer.getInt();
         int vScroll = buffer.getInt();
-        return ControlMessage.createInjectScrollEvent(position, hScroll, vScroll);
+        int buttons = buffer.getInt();
+        return ControlMessage.createInjectScrollEvent(position, hScroll, vScroll, buttons);
     }
 
     private ControlMessage parseBackOrScreenOnEvent() {

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -98,7 +98,7 @@ public class Controller {
                 break;
             case ControlMessage.TYPE_INJECT_SCROLL_EVENT:
                 if (device.supportsInputEvents()) {
-                    injectScroll(msg.getPosition(), msg.getHScroll(), msg.getVScroll());
+                    injectScroll(msg.getPosition(), msg.getHScroll(), msg.getVScroll(), msg.getButtons());
                 }
                 break;
             case ControlMessage.TYPE_BACK_OR_SCREEN_ON:
@@ -221,7 +221,7 @@ public class Controller {
         return device.injectEvent(event, Device.INJECT_MODE_ASYNC);
     }
 
-    private boolean injectScroll(Position position, int hScroll, int vScroll) {
+    private boolean injectScroll(Position position, int hScroll, int vScroll, int buttons) {
         long now = SystemClock.uptimeMillis();
         Point point = device.getPhysicalPoint(position);
         if (point == null) {
@@ -239,7 +239,7 @@ public class Controller {
         coords.setAxisValue(MotionEvent.AXIS_VSCROLL, vScroll);
 
         MotionEvent event = MotionEvent
-                .obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, 0, 1f, 1f, DEFAULT_DEVICE_ID, 0,
+                .obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0,
                         InputDevice.SOURCE_MOUSE, 0);
         return device.injectEvent(event, Device.INJECT_MODE_ASYNC);
     }

server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java

@@ -128,6 +128,7 @@ public class ControlMessageReaderTest {
         dos.writeShort(1920);
         dos.writeInt(1);
         dos.writeInt(-1);
+        dos.writeInt(1);
 
         byte[] packet = bos.toByteArray();
 
@@ -144,6 +145,7 @@ public class ControlMessageReaderTest {
         Assert.assertEquals(1920, event.getPosition().getScreenSize().getHeight());
         Assert.assertEquals(1, event.getHScroll());
         Assert.assertEquals(-1, event.getVScroll());
+        Assert.assertEquals(1, event.getButtons());
     }
 
     @Test