Add mouse shortcut to expand settings panel

Double-click on extra mouse button to open the settings panel (a
single-click opens the notification panel).

This is consistent with the keyboard shortcut MOD+n+n.

PR #2264 <https://github.com/Genymobile/scrcpy/pull/2264>

Signed-off-by: Romain Vimont <rom@rom1v.com>

README.md

@@ -491,6 +491,18 @@ scrcpy -Sw
 ```
 
 
+#### Render expired frames
+
+By default, to minimize latency, _scrcpy_ always renders the last decoded frame
+available, and drops any previous one.
+
+To force the rendering of all frames (at a cost of a possible increased
+latency), use:
+
+```bash
+scrcpy --render-expired-frames
+```
+
 #### Show touches
 
 For presentations, it may be useful to show physical touches (on the physical

app/scrcpy.1

@@ -155,6 +155,10 @@ Supported names are currently "direct3d", "opengl", "opengles2", "opengles", "me
 .UR https://wiki.libsdl.org/SDL_HINT_RENDER_DRIVER
 .UE
 
+.TP
+.B \-\-render\-expired\-frames
+By default, to minimize latency, scrcpy always renders the last available decoded frame, and drops any previous ones. This flag forces to render all frames, at a cost of a possible increased latency.
+
 .TP
 .BI "\-\-rotation " value
 Set the initial display rotation. Possibles values are 0, 1, 2 and 3. Each increment adds a 90 degrees rotation counterclockwise.
@@ -183,16 +187,16 @@ Enable "show touches" on start, restore the initial value on exit.
 
 It only shows physical touches (not clicks from scrcpy).
 
-.TP
-.B \-v, \-\-version
-Print the version of scrcpy.
-
 .TP
 .BI "\-V, \-\-verbosity " value
 Set the log level ("debug", "info", "warn" or "error").
 
 Default is "info" for release builds, "debug" for debug builds.
 
+.TP
+.B \-v, \-\-version
+Print the version of scrcpy.
+
 .TP
 .B \-w, \-\-stay-awake
 Keep the device on while scrcpy is running, when the device is plugged in.

app/src/cli.c

@@ -143,6 +143,12 @@ scrcpy_print_usage(const char *arg0) {
         "        \"opengles2\", \"opengles\", \"metal\" and \"software\".\n"
         "        <https://wiki.libsdl.org/SDL_HINT_RENDER_DRIVER>\n"
         "\n"
+        "    --render-expired-frames\n"
+        "        By default, to minimize latency, scrcpy always renders the\n"
+        "        last available decoded frame, and drops any previous ones.\n"
+        "        This flag forces to render all frames, at a cost of a\n"
+        "        possible increased latency.\n"
+        "\n"
         "    --rotation value\n"
         "        Set the initial display rotation.\n"
         "        Possibles values are 0, 1, 2 and 3. Each increment adds a 90\n"
@@ -173,9 +179,6 @@ scrcpy_print_usage(const char *arg0) {
         "        on exit.\n"
         "        It only shows physical touches (not clicks from scrcpy).\n"
         "\n"
-        "    -v, --version\n"
-        "        Print the version of scrcpy.\n"
-        "\n"
         "    -V, --verbosity value\n"
         "        Set the log level (debug, info, warn or error).\n"
 #ifndef NDEBUG
@@ -183,6 +186,9 @@ scrcpy_print_usage(const char *arg0) {
 #else
         "        Default is info.\n"
 #endif
+        "\n"
+        "    -v, --version\n"
+        "        Print the version of scrcpy.\n"
         "\n"
         "    -w, --stay-awake\n"
         "        Keep the device on while scrcpy is running, when the device\n"
@@ -810,8 +816,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
                 opts->stay_awake = true;
                 break;
             case OPT_RENDER_EXPIRED_FRAMES:
-                LOGW("Option --render-expired-frames has been removed. This "
-                     "flag has been ignored.");
+                opts->render_expired_frames = true;
                 break;
             case OPT_WINDOW_TITLE:
                 opts->window_title = optarg;

app/src/common.h

@@ -8,7 +8,4 @@
 #define MIN(X,Y) (X) < (Y) ? (X) : (Y)
 #define MAX(X,Y) (X) > (Y) ? (X) : (Y)
 
-#define container_of(ptr, type, member) \
-    ((type *) (((char *) (ptr)) - offsetof(type, member)))
-
 #endif

app/src/compat.h

@@ -8,9 +8,20 @@
 # define _DARWIN_C_SOURCE
 #endif
 
+#include <libavcodec/version.h>
 #include <libavformat/version.h>
 #include <SDL2/SDL_version.h>
 
+// In ffmpeg/doc/APIchanges:
+// 2016-04-11 - 6f69f7a / 9200514 - lavf 57.33.100 / 57.5.0 - avformat.h
+//   Add AVStream.codecpar, deprecate AVStream.codec.
+#if    (LIBAVFORMAT_VERSION_MICRO >= 100 /* FFmpeg */ && \
+        LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 33, 100)) \
+    || (LIBAVFORMAT_VERSION_MICRO < 100 && /* Libav */ \
+        LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 5, 0))
+# define SCRCPY_LAVF_HAS_NEW_CODEC_PARAMS_API
+#endif
+
 // In ffmpeg/doc/APIchanges:
 // 2018-02-06 - 0694d87024 - lavf 58.9.100 - avformat.h
 //   Deprecate use of av_register_input_format(), av_register_output_format(),
@@ -22,6 +33,15 @@
 # define SCRCPY_LAVF_REQUIRES_REGISTER_ALL
 #endif
 
+// In ffmpeg/doc/APIchanges:
+// 2016-04-21 - 7fc329e - lavc 57.37.100 - avcodec.h
+//   Add a new audio/video encoding and decoding API with decoupled input
+//   and output -- avcodec_send_packet(), avcodec_receive_frame(),
+//   avcodec_send_frame() and avcodec_receive_packet().
+#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100)
+# define SCRCPY_LAVF_HAS_NEW_ENCODING_DECODING_API
+#endif
+
 #if SDL_VERSION_ATLEAST(2, 0, 5)
 // <https://wiki.libsdl.org/SDL_HINT_MOUSE_FOCUS_CLICKTHROUGH>
 # define SCRCPY_SDL_HAS_HINT_MOUSE_FOCUS_CLICKTHROUGH

app/src/control_msg.c

@@ -67,6 +67,9 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
             buffer_write32be(&buf[17],
                              (uint32_t) msg->inject_scroll_event.vscroll);
             return 21;
+        case CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON:
+            buf[1] = msg->inject_keycode.action;
+            return 2;
         case CONTROL_MSG_TYPE_SET_CLIPBOARD: {
             buf[1] = !!msg->set_clipboard.paste;
             size_t len = write_string(msg->set_clipboard.text,
@@ -77,9 +80,9 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
         case CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE:
             buf[1] = msg->set_screen_power_mode.mode;
             return 2;
-        case CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON:
         case CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL:
-        case CONTROL_MSG_TYPE_COLLAPSE_NOTIFICATION_PANEL:
+        case CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL:
+        case CONTROL_MSG_TYPE_COLLAPSE_PANELS:
         case CONTROL_MSG_TYPE_GET_CLIPBOARD:
         case CONTROL_MSG_TYPE_ROTATE_DEVICE:
             // no additional data

app/src/control_msg.h

@@ -27,7 +27,8 @@ enum control_msg_type {
     CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT,
     CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON,
     CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL,
-    CONTROL_MSG_TYPE_COLLAPSE_NOTIFICATION_PANEL,
+    CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL,
+    CONTROL_MSG_TYPE_COLLAPSE_PANELS,
     CONTROL_MSG_TYPE_GET_CLIPBOARD,
     CONTROL_MSG_TYPE_SET_CLIPBOARD,
     CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE,
@@ -64,6 +65,10 @@ struct control_msg {
             int32_t hscroll;
             int32_t vscroll;
         } inject_scroll_event;
+        struct {
+            enum android_keyevent_action action; // action for the BACK key
+            // screen may only be turned on on ACTION_DOWN
+        } back_or_screen_on;
         struct {
             char *text; // owned, to be freed by free()
             bool paste;

app/src/decoder.c

@@ -4,40 +4,14 @@
 
 #include "events.h"
 #include "video_buffer.h"
-#include "trait/frame_sink.h"
 #include "util/log.h"
 
-/** Downcast packet_sink to decoder */
-#define DOWNCAST(SINK) container_of(SINK, struct decoder, packet_sink)
-
-static void
-decoder_close_first_sinks(struct decoder *decoder, unsigned count) {
-    while (count) {
-        struct sc_frame_sink *sink = decoder->sinks[--count];
-        sink->ops->close(sink);
-    }
+void
+decoder_init(struct decoder *decoder, struct video_buffer *vb) {
+    decoder->video_buffer = vb;
 }
 
-static inline void
-decoder_close_sinks(struct decoder *decoder) {
-    decoder_close_first_sinks(decoder, decoder->sink_count);
-}
-
-static bool
-decoder_open_sinks(struct decoder *decoder) {
-    for (unsigned i = 0; i < decoder->sink_count; ++i) {
-        struct sc_frame_sink *sink = decoder->sinks[i];
-        if (!sink->ops->open(sink)) {
-            LOGE("Could not open frame sink %d", i);
-            decoder_close_first_sinks(decoder, i);
-            return false;
-        }
-    }
-
-    return true;
-}
-
-static bool
+bool
 decoder_open(struct decoder *decoder, const AVCodec *codec) {
     decoder->codec_ctx = avcodec_alloc_context3(codec);
     if (!decoder->codec_ctx) {
@@ -51,105 +25,52 @@ decoder_open(struct decoder *decoder, const AVCodec *codec) {
         return false;
     }
 
-    decoder->frame = av_frame_alloc();
-    if (!decoder->frame) {
-        LOGE("Could not create decoder frame");
-        avcodec_close(decoder->codec_ctx);
-        avcodec_free_context(&decoder->codec_ctx);
-        return false;
-    }
-
-    if (!decoder_open_sinks(decoder)) {
-        LOGE("Could not open decoder sinks");
-        av_frame_free(&decoder->frame);
-        avcodec_close(decoder->codec_ctx);
-        avcodec_free_context(&decoder->codec_ctx);
-    }
-
     return true;
 }
 
-static void
+void
 decoder_close(struct decoder *decoder) {
-    decoder_close_sinks(decoder);
-    av_frame_free(&decoder->frame);
     avcodec_close(decoder->codec_ctx);
     avcodec_free_context(&decoder->codec_ctx);
 }
 
-static bool
-push_frame_to_sinks(struct decoder *decoder, const AVFrame *frame) {
-    for (unsigned i = 0; i < decoder->sink_count; ++i) {
-        struct sc_frame_sink *sink = decoder->sinks[i];
-        if (!sink->ops->push(sink, frame)) {
-            LOGE("Could not send frame to sink %d", i);
-            return false;
-        }
-    }
-
-    return true;
-}
-
-static bool
+bool
 decoder_push(struct decoder *decoder, const AVPacket *packet) {
-    bool is_config = packet->pts == AV_NOPTS_VALUE;
-    if (is_config) {
-        // nothing to do
-        return true;
-    }
-
+// the new decoding/encoding API has been introduced by:
+// <http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=7fc329e2dd6226dfecaa4a1d7adf353bf2773726>
+#ifdef SCRCPY_LAVF_HAS_NEW_ENCODING_DECODING_API
     int ret;
     if ((ret = avcodec_send_packet(decoder->codec_ctx, packet)) < 0) {
         LOGE("Could not send video packet: %d", ret);
         return false;
     }
-    ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame);
+    ret = avcodec_receive_frame(decoder->codec_ctx,
+                                decoder->video_buffer->producer_frame);
     if (!ret) {
         // a frame was received
-        bool ok = push_frame_to_sinks(decoder, decoder->frame);
-        // A frame lost should not make the whole pipeline fail. The error, if
-        // any, is already logged.
-        (void) ok;
+        video_buffer_producer_offer_frame(decoder->video_buffer);
     } else if (ret != AVERROR(EAGAIN)) {
         LOGE("Could not receive video frame: %d", ret);
         return false;
     }
+#else
+    int got_picture;
+    int len = avcodec_decode_video2(decoder->codec_ctx,
+                                    decoder->video_buffer->producer_frame,
+                                    &got_picture,
+                                    packet);
+    if (len < 0) {
+        LOGE("Could not decode video packet: %d", len);
+        return false;
+    }
+    if (got_picture) {
+        video_buffer_producer_offer_frame(decoder->video_buffer);
+    }
+#endif
     return true;
 }
 
-static bool
-decoder_packet_sink_open(struct sc_packet_sink *sink, const AVCodec *codec) {
-    struct decoder *decoder = DOWNCAST(sink);
-    return decoder_open(decoder, codec);
-}
-
-static void
-decoder_packet_sink_close(struct sc_packet_sink *sink) {
-    struct decoder *decoder = DOWNCAST(sink);
-    decoder_close(decoder);
-}
-
-static bool
-decoder_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) {
-    struct decoder *decoder = DOWNCAST(sink);
-    return decoder_push(decoder, packet);
-}
-
 void
-decoder_init(struct decoder *decoder) {
-    static const struct sc_packet_sink_ops ops = {
-        .open = decoder_packet_sink_open,
-        .close = decoder_packet_sink_close,
-        .push = decoder_packet_sink_push,
-    };
-
-    decoder->packet_sink.ops = &ops;
-}
-
-void
-decoder_add_sink(struct decoder *decoder, struct sc_frame_sink *sink) {
-    assert(decoder->sink_count < DECODER_MAX_SINKS);
-    assert(sink);
-    assert(sink->ops);
-    decoder->sinks[decoder->sink_count++] = sink;
+decoder_interrupt(struct decoder *decoder) {
+    video_buffer_interrupt(decoder->video_buffer);
 }

app/src/decoder.h

@@ -3,27 +3,30 @@
 
 #include "common.h"
 
-#include "trait/packet_sink.h"
-
 #include <stdbool.h>
 #include <libavformat/avformat.h>
 
-#define DECODER_MAX_SINKS 1
+struct video_buffer;
 
 struct decoder {
-    struct sc_packet_sink packet_sink; // packet sink trait
-
-    struct sc_frame_sink *sinks[DECODER_MAX_SINKS];
-    unsigned sink_count;
+    struct video_buffer *video_buffer;
 
     AVCodecContext *codec_ctx;
-    AVFrame *frame;
 };
 
 void
-decoder_init(struct decoder *decoder);
+decoder_init(struct decoder *decoder, struct video_buffer *vb);
+
+bool
+decoder_open(struct decoder *decoder, const AVCodec *codec);
 
 void
-decoder_add_sink(struct decoder *decoder, struct sc_frame_sink *sink);
+decoder_close(struct decoder *decoder);
+
+bool
+decoder_push(struct decoder *decoder, const AVPacket *packet);
+
+void
+decoder_interrupt(struct decoder *decoder);
 
 #endif

app/src/input_manager.c

@@ -72,6 +72,10 @@ input_manager_init(struct input_manager *im,
     im->sdl_shortcut_mods.count = shortcut_mods->count;
 
     im->vfinger_down = false;
+
+    im->last_keycode = SDLK_UNKNOWN;
+    im->last_mod = 0;
+    im->key_repeat = 0;
 }
 
 static void
@@ -146,13 +150,25 @@ action_cut(struct controller *controller, int actions) {
 }
 
 // turn the screen on if it was off, press BACK otherwise
+// If the screen is off, it is turned on only on ACTION_DOWN
 static void
-press_back_or_turn_screen_on(struct controller *controller) {
+press_back_or_turn_screen_on(struct controller *controller, int actions) {
     struct control_msg msg;
     msg.type = CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON;
 
-    if (!controller_push_msg(controller, &msg)) {
-        LOGW("Could not request 'press back or turn screen on'");
+    if (actions & ACTION_DOWN) {
+        msg.back_or_screen_on.action = AKEY_EVENT_ACTION_DOWN;
+        if (!controller_push_msg(controller, &msg)) {
+            LOGW("Could not request 'press back or turn screen on'");
+            return;
+        }
+    }
+
+    if (actions & ACTION_UP) {
+        msg.back_or_screen_on.action = AKEY_EVENT_ACTION_UP;
+        if (!controller_push_msg(controller, &msg)) {
+            LOGW("Could not request 'press back or turn screen on'");
+        }
     }
 }
 
@@ -167,9 +183,19 @@ expand_notification_panel(struct controller *controller) {
 }
 
 static void
-collapse_notification_panel(struct controller *controller) {
+expand_settings_panel(struct controller *controller) {
     struct control_msg msg;
-    msg.type = CONTROL_MSG_TYPE_COLLAPSE_NOTIFICATION_PANEL;
+    msg.type = CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL;
+
+    if (!controller_push_msg(controller, &msg)) {
+        LOGW("Could not request 'expand settings panel'");
+    }
+}
+
+static void
+collapse_panels(struct controller *controller) {
+    struct control_msg msg;
+    msg.type = CONTROL_MSG_TYPE_COLLAPSE_PANELS;
 
     if (!controller_push_msg(controller, &msg)) {
         LOGW("Could not request 'collapse notification panel'");
@@ -372,16 +398,27 @@ input_manager_process_key(struct input_manager *im,
     // control: indicates the state of the command-line option --no-control
     bool control = im->control;
 
-    bool smod = is_shortcut_mod(im, event->keysym.mod);
-
     struct controller *controller = im->controller;
 
     SDL_Keycode keycode = event->keysym.sym;
+    uint16_t mod = event->keysym.mod;
     bool down = event->type == SDL_KEYDOWN;
     bool ctrl = event->keysym.mod & KMOD_CTRL;
     bool shift = event->keysym.mod & KMOD_SHIFT;
     bool repeat = event->repeat;
 
+    bool smod = is_shortcut_mod(im, mod);
+
+    if (down && !repeat) {
+        if (keycode == im->last_keycode && mod == im->last_mod) {
+            ++im->key_repeat;
+        } else {
+            im->key_repeat = 0;
+            im->last_keycode = keycode;
+            im->last_mod = mod;
+        }
+    }
+
     // The shortcut modifier is pressed
     if (smod) {
         int action = down ? ACTION_DOWN : ACTION_UP;
@@ -486,9 +523,11 @@ input_manager_process_key(struct input_manager *im,
             case SDLK_n:
                 if (control && !repeat && down) {
                     if (shift) {
-                        collapse_notification_panel(controller);
-                    } else {
+                        collapse_panels(controller);
+                    } else if (im->key_repeat == 0) {
                         expand_notification_panel(controller);
+                    } else {
+                        expand_settings_panel(controller);
                     }
                 }
                 return;
@@ -646,13 +685,27 @@ input_manager_process_mouse_button(struct input_manager *im,
     }
 
     bool down = event->type == SDL_MOUSEBUTTONDOWN;
-    if (!im->forward_all_clicks && down) {
+    if (!im->forward_all_clicks) {
+        int action = down ? ACTION_DOWN : ACTION_UP;
+
+        if (control && event->button == SDL_BUTTON_X1) {
+            action_app_switch(im->controller, action);
+            return;
+        }
+        if (control && event->button == SDL_BUTTON_X2 && down) {
+            if (event->clicks < 2) {
+                expand_notification_panel(im->controller);
+            } else {
+                expand_settings_panel(im->controller);
+            }
+            return;
+        }
         if (control && event->button == SDL_BUTTON_RIGHT) {
-            press_back_or_turn_screen_on(im->controller);
+            press_back_or_turn_screen_on(im->controller, action);
             return;
         }
         if (control && event->button == SDL_BUTTON_MIDDLE) {
-            action_home(im->controller, ACTION_DOWN | ACTION_UP);
+            action_home(im->controller, action);
             return;
         }
 
@@ -665,7 +718,9 @@ input_manager_process_mouse_button(struct input_manager *im,
             bool outside = x < r->x || x >= r->x + r->w
                         || y < r->y || y >= r->y + r->h;
             if (outside) {
-                screen_resize_to_fit(im->screen);
+                if (down) {
+                    screen_resize_to_fit(im->screen);
+                }
                 return;
             }
         }

app/src/input_manager.h

@@ -33,6 +33,13 @@ struct input_manager {
     } sdl_shortcut_mods;
 
     bool vfinger_down;
+
+    // Tracks the number of identical consecutive shortcut key down events.
+    // Not to be confused with event->repeat, which counts the number of
+    // system-generated repeated key presses.
+    unsigned key_repeat;
+    SDL_Keycode last_keycode;
+    uint16_t last_mod;
 };
 
 void

app/src/recorder.c

@@ -5,9 +5,6 @@
 
 #include "util/log.h"
 
-/** Downcast packet_sink to recorder */
-#define DOWNCAST(SINK) container_of(SINK, struct recorder, packet_sink)
-
 static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us
 
 static const AVOutputFormat *
@@ -60,6 +57,50 @@ recorder_queue_clear(struct recorder_queue *queue) {
     }
 }
 
+bool
+recorder_init(struct recorder *recorder,
+              const char *filename,
+              enum sc_record_format format,
+              struct size declared_frame_size) {
+    recorder->filename = strdup(filename);
+    if (!recorder->filename) {
+        LOGE("Could not strdup filename");
+        return false;
+    }
+
+    bool ok = sc_mutex_init(&recorder->mutex);
+    if (!ok) {
+        LOGC("Could not create mutex");
+        free(recorder->filename);
+        return false;
+    }
+
+    ok = sc_cond_init(&recorder->queue_cond);
+    if (!ok) {
+        LOGC("Could not create cond");
+        sc_mutex_destroy(&recorder->mutex);
+        free(recorder->filename);
+        return false;
+    }
+
+    queue_init(&recorder->queue);
+    recorder->stopped = false;
+    recorder->failed = false;
+    recorder->format = format;
+    recorder->declared_frame_size = declared_frame_size;
+    recorder->header_written = false;
+    recorder->previous = NULL;
+
+    return true;
+}
+
+void
+recorder_destroy(struct recorder *recorder) {
+    sc_cond_destroy(&recorder->queue_cond);
+    sc_mutex_destroy(&recorder->mutex);
+    free(recorder->filename);
+}
+
 static const char *
 recorder_get_format_name(enum sc_record_format format) {
     switch (format) {
@@ -69,6 +110,88 @@ recorder_get_format_name(enum sc_record_format format) {
     }
 }
 
+bool
+recorder_open(struct recorder *recorder, const AVCodec *input_codec) {
+    const char *format_name = recorder_get_format_name(recorder->format);
+    assert(format_name);
+    const AVOutputFormat *format = find_muxer(format_name);
+    if (!format) {
+        LOGE("Could not find muxer");
+        return false;
+    }
+
+    recorder->ctx = avformat_alloc_context();
+    if (!recorder->ctx) {
+        LOGE("Could not allocate output context");
+        return false;
+    }
+
+    // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate()
+    // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat
+    // still expects a pointer-to-non-const (it has not be updated accordingly)
+    // <https://github.com/FFmpeg/FFmpeg/commit/0694d8702421e7aff1340038559c438b61bb30dd>
+    recorder->ctx->oformat = (AVOutputFormat *) format;
+
+    av_dict_set(&recorder->ctx->metadata, "comment",
+                "Recorded by scrcpy " SCRCPY_VERSION, 0);
+
+    AVStream *ostream = avformat_new_stream(recorder->ctx, input_codec);
+    if (!ostream) {
+        avformat_free_context(recorder->ctx);
+        return false;
+    }
+
+#ifdef SCRCPY_LAVF_HAS_NEW_CODEC_PARAMS_API
+    ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
+    ostream->codecpar->codec_id = input_codec->id;
+    ostream->codecpar->format = AV_PIX_FMT_YUV420P;
+    ostream->codecpar->width = recorder->declared_frame_size.width;
+    ostream->codecpar->height = recorder->declared_frame_size.height;
+#else
+    ostream->codec->codec_type = AVMEDIA_TYPE_VIDEO;
+    ostream->codec->codec_id = input_codec->id;
+    ostream->codec->pix_fmt = AV_PIX_FMT_YUV420P;
+    ostream->codec->width = recorder->declared_frame_size.width;
+    ostream->codec->height = recorder->declared_frame_size.height;
+#endif
+
+    int ret = avio_open(&recorder->ctx->pb, recorder->filename,
+                        AVIO_FLAG_WRITE);
+    if (ret < 0) {
+        LOGE("Failed to open output file: %s", recorder->filename);
+        // ostream will be cleaned up during context cleaning
+        avformat_free_context(recorder->ctx);
+        return false;
+    }
+
+    LOGI("Recording started to %s file: %s", format_name, recorder->filename);
+
+    return true;
+}
+
+void
+recorder_close(struct recorder *recorder) {
+    if (recorder->header_written) {
+        int ret = av_write_trailer(recorder->ctx);
+        if (ret < 0) {
+            LOGE("Failed to write trailer to %s", recorder->filename);
+            recorder->failed = true;
+        }
+    } else {
+        // the recorded file is empty
+        recorder->failed = true;
+    }
+    avio_close(recorder->ctx->pb);
+    avformat_free_context(recorder->ctx);
+
+    if (recorder->failed) {
+        LOGE("Recording failed to %s", recorder->filename);
+    } else {
+        const char *format_name = recorder_get_format_name(recorder->format);
+        LOGI("Recording complete to %s file: %s", format_name, recorder->filename);
+    }
+}
+
 static bool
 recorder_write_header(struct recorder *recorder, const AVPacket *packet) {
     AVStream *ostream = recorder->ctx->streams[0];
@@ -82,8 +205,13 @@ recorder_write_header(struct recorder *recorder, const AVPacket *packet) {
     // copy the first packet to the extra data
     memcpy(extradata, packet->data, packet->size);
 
+#ifdef SCRCPY_LAVF_HAS_NEW_CODEC_PARAMS_API
     ostream->codecpar->extradata = extradata;
     ostream->codecpar->extradata_size = packet->size;
+#else
+    ostream->codec->extradata = extradata;
+    ostream->codec->extradata_size = packet->size;
+#endif
 
     int ret = avformat_write_header(recorder->ctx, NULL);
     if (ret < 0) {
@@ -189,26 +317,7 @@ run_recorder(void *data) {
             sc_mutex_unlock(&recorder->mutex);
             break;
         }
-    }
 
-    if (!recorder->failed) {
-        if (recorder->header_written) {
-            int ret = av_write_trailer(recorder->ctx);
-            if (ret < 0) {
-                LOGE("Failed to write trailer to %s", recorder->filename);
-                recorder->failed = true;
-            }
-        } else {
-            // the recorded file is empty
-            recorder->failed = true;
-        }
-    }
-
-    if (recorder->failed) {
-        LOGE("Recording failed to %s", recorder->filename);
-    } else {
-        const char *format_name = recorder_get_format_name(recorder->format);
-        LOGI("Recording complete to %s file: %s", format_name, recorder->filename);
     }
 
     LOGD("Recorder thread ended");
@@ -216,80 +325,34 @@ run_recorder(void *data) {
     return 0;
 }
 
-static bool
-recorder_open(struct recorder *recorder, const AVCodec *input_codec) {
-    const char *format_name = recorder_get_format_name(recorder->format);
-    assert(format_name);
-    const AVOutputFormat *format = find_muxer(format_name);
-    if (!format) {
-        LOGE("Could not find muxer");
-        return false;
-    }
-
-    recorder->ctx = avformat_alloc_context();
-    if (!recorder->ctx) {
-        LOGE("Could not allocate output context");
-        return false;
-    }
-
-    // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate()
-    // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat
-    // still expects a pointer-to-non-const (it has not be updated accordingly)
-    // <https://github.com/FFmpeg/FFmpeg/commit/0694d8702421e7aff1340038559c438b61bb30dd>
-    recorder->ctx->oformat = (AVOutputFormat *) format;
-
-    av_dict_set(&recorder->ctx->metadata, "comment",
-                "Recorded by scrcpy " SCRCPY_VERSION, 0);
-
-    AVStream *ostream = avformat_new_stream(recorder->ctx, input_codec);
-    if (!ostream) {
-        avformat_free_context(recorder->ctx);
-        return false;
-    }
-
-    ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
-    ostream->codecpar->codec_id = input_codec->id;
-    ostream->codecpar->format = AV_PIX_FMT_YUV420P;
-    ostream->codecpar->width = recorder->declared_frame_size.width;
-    ostream->codecpar->height = recorder->declared_frame_size.height;
-
-    int ret = avio_open(&recorder->ctx->pb, recorder->filename,
-                        AVIO_FLAG_WRITE);
-    if (ret < 0) {
-        LOGE("Failed to open output file: %s", recorder->filename);
-        // ostream will be cleaned up during context cleaning
-        avformat_free_context(recorder->ctx);
-        return false;
-    }
-
+bool
+recorder_start(struct recorder *recorder) {
     LOGD("Starting recorder thread");
+
     bool ok = sc_thread_create(&recorder->thread, run_recorder, "recorder",
                                recorder);
     if (!ok) {
         LOGC("Could not start recorder thread");
-        avformat_free_context(recorder->ctx);
         return false;
     }
 
-    LOGI("Recording started to %s file: %s", format_name, recorder->filename);
-
     return true;
 }
 
-static void
-recorder_close(struct recorder *recorder) {
+void
+recorder_stop(struct recorder *recorder) {
     sc_mutex_lock(&recorder->mutex);
     recorder->stopped = true;
     sc_cond_signal(&recorder->queue_cond);
     sc_mutex_unlock(&recorder->mutex);
-
-    sc_thread_join(&recorder->thread, NULL);
-
-    avio_close(recorder->ctx->pb);
-    avformat_free_context(recorder->ctx);
 }
 
-static bool
+void
+recorder_join(struct recorder *recorder) {
+    sc_thread_join(&recorder->thread, NULL);
+}
+
+bool
 recorder_push(struct recorder *recorder, const AVPacket *packet) {
     sc_mutex_lock(&recorder->mutex);
     assert(!recorder->stopped);
@@ -313,73 +376,3 @@ recorder_push(struct recorder *recorder, const AVPacket *packet) {
     sc_mutex_unlock(&recorder->mutex);
     return true;
 }
-
-static bool
-recorder_packet_sink_open(struct sc_packet_sink *sink, const AVCodec *codec) {
-    struct recorder *recorder = DOWNCAST(sink);
-    return recorder_open(recorder, codec);
-}
-
-static void
-recorder_packet_sink_close(struct sc_packet_sink *sink) {
-    struct recorder *recorder = DOWNCAST(sink);
-    recorder_close(recorder);
-}
-
-static bool
-recorder_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) {
-    struct recorder *recorder = DOWNCAST(sink);
-    return recorder_push(recorder, packet);
-}
-
-bool
-recorder_init(struct recorder *recorder,
-              const char *filename,
-              enum sc_record_format format,
-              struct size declared_frame_size) {
-    recorder->filename = strdup(filename);
-    if (!recorder->filename) {
-        LOGE("Could not strdup filename");
-        return false;
-    }
-
-    bool ok = sc_mutex_init(&recorder->mutex);
-    if (!ok) {
-        LOGC("Could not create mutex");
-        free(recorder->filename);
-        return false;
-    }
-
-    ok = sc_cond_init(&recorder->queue_cond);
-    if (!ok) {
-        LOGC("Could not create cond");
-        sc_mutex_destroy(&recorder->mutex);
-        free(recorder->filename);
-        return false;
-    }
-
-    queue_init(&recorder->queue);
-    recorder->stopped = false;
-    recorder->failed = false;
-    recorder->format = format;
-    recorder->declared_frame_size = declared_frame_size;
-    recorder->header_written = false;
-    recorder->previous = NULL;
-
-    static const struct sc_packet_sink_ops ops = {
-        .open = recorder_packet_sink_open,
-        .close = recorder_packet_sink_close,
-        .push = recorder_packet_sink_push,
-    };
-
-    recorder->packet_sink.ops = &ops;
-
-    return true;
-}
-
-void
-recorder_destroy(struct recorder *recorder) {
-    sc_cond_destroy(&recorder->queue_cond);
-    sc_mutex_destroy(&recorder->mutex);
-    free(recorder->filename);
-}

app/src/recorder.h

@@ -8,7 +8,6 @@
 
 #include "coords.h"
 #include "scrcpy.h"
-#include "trait/packet_sink.h"
 #include "util/queue.h"
 #include "util/thread.h"
 
@@ -20,8 +19,6 @@ struct record_packet {
 struct recorder_queue QUEUE(struct record_packet);
 
 struct recorder {
-    struct sc_packet_sink packet_sink; // packet sink trait
-
     char *filename;
     enum sc_record_format format;
     AVFormatContext *ctx;
@@ -31,7 +28,7 @@ struct recorder {
     sc_thread thread;
     sc_mutex mutex;
     sc_cond queue_cond;
-    bool stopped; // set on recorder_close()
+    bool stopped; // set on recorder_stop() by the stream reader
     bool failed; // set on packet write failure
     struct recorder_queue queue;
 
@@ -49,4 +46,22 @@ recorder_init(struct recorder *recorder, const char *filename,
 void
 recorder_destroy(struct recorder *recorder);
 
+bool
+recorder_open(struct recorder *recorder, const AVCodec *input_codec);
+
+void
+recorder_close(struct recorder *recorder);
+
+bool
+recorder_start(struct recorder *recorder);
+
+void
+recorder_stop(struct recorder *recorder);
+
+void
+recorder_join(struct recorder *recorder);
+
+bool
+recorder_push(struct recorder *recorder, const AVPacket *packet);
+
 #endif

app/src/scrcpy.c

@@ -25,12 +25,14 @@
 #include "server.h"
 #include "stream.h"
 #include "tiny_xpm.h"
+#include "video_buffer.h"
 #include "util/log.h"
 #include "util/net.h"
 
 static struct server server;
 static struct screen screen;
 static struct fps_counter fps_counter;
+static struct video_buffer video_buffer;
 static struct stream stream;
 static struct decoder decoder;
 static struct recorder recorder;
@@ -126,30 +128,6 @@ sdl_init_and_configure(bool display, const char *render_driver,
     return true;
 }
 
-
-#if defined(__APPLE__) || defined(__WINDOWS__)
-# define CONTINUOUS_RESIZING_WORKAROUND
-#endif
-
-#ifdef CONTINUOUS_RESIZING_WORKAROUND
-// On Windows and MacOS, resizing blocks the event loop, so resizing events are
-// not triggered. As a workaround, handle them in an event handler.
-//
-// <https://bugzilla.libsdl.org/show_bug.cgi?id=2077>
-// <https://stackoverflow.com/a/40693139/1987178>
-static int
-event_watcher(void *data, SDL_Event *event) {
-    (void) data;
-    if (event->type == SDL_WINDOWEVENT
-            && event->window.event == SDL_WINDOWEVENT_RESIZED) {
-        // In practice, it seems to always be called from the same thread in
-        // that specific case. Anyway, it's just a workaround.
-        screen_render(&screen, true);
-    }
-    return 0;
-}
-#endif
-
 static bool
 is_apk(const char *file) {
     const char *ext = strrchr(file, '.');
@@ -189,7 +167,7 @@ handle_event(SDL_Event *event, const struct scrcpy_options *options) {
                 action = ACTION_PUSH_FILE;
             }
             file_handler_request(&file_handler, action, file);
-            break;
+            goto end;
         }
     }
 
@@ -207,11 +185,6 @@ end:
 
 static bool
 event_loop(const struct scrcpy_options *options) {
-#ifdef CONTINUOUS_RESIZING_WORKAROUND
-    if (options->display) {
-        SDL_AddEventWatch(event_watcher, NULL);
-    }
-#endif
     SDL_Event event;
     while (SDL_WaitEvent(&event)) {
         enum event_result result = handle_event(&event, options);
@@ -274,6 +247,7 @@ scrcpy(const struct scrcpy_options *options) {
 
     bool server_started = false;
     bool fps_counter_initialized = false;
+    bool video_buffer_initialized = false;
     bool file_handler_initialized = false;
     bool recorder_initialized = false;
     bool stream_started = false;
@@ -331,6 +305,11 @@ scrcpy(const struct scrcpy_options *options) {
         }
         fps_counter_initialized = true;
 
+        if (!video_buffer_init(&video_buffer, options->render_expired_frames)) {
+            goto end;
+        }
+        video_buffer_initialized = true;
+
         if (options->control) {
             if (!file_handler_init(&file_handler, server.serial,
                                    options->push_target)) {
@@ -339,7 +318,7 @@ scrcpy(const struct scrcpy_options *options) {
             file_handler_initialized = true;
         }
 
-        decoder_init(&decoder);
+        decoder_init(&decoder, &video_buffer);
         dec = &decoder;
     }
 
@@ -357,15 +336,7 @@ scrcpy(const struct scrcpy_options *options) {
 
     av_log_set_callback(av_log_callback);
 
-    stream_init(&stream, server.video_socket);
-
-    if (dec) {
-        stream_add_sink(&stream, &dec->packet_sink);
-    }
-
-    if (rec) {
-        stream_add_sink(&stream, &rec->packet_sink);
-    }
+    stream_init(&stream, server.video_socket, dec, rec);
 
     if (options->display) {
         if (options->control) {
@@ -394,15 +365,15 @@ scrcpy(const struct scrcpy_options *options) {
             .window_borderless = options->window_borderless,
             .rotation = options->rotation,
             .mipmaps = options->mipmaps,
+            .fullscreen = options->fullscreen,
         };
 
-        if (!screen_init(&screen, &fps_counter, &screen_params)) {
+        if (!screen_init(&screen, &video_buffer, &fps_counter,
+                         &screen_params)) {
             goto end;
         }
         screen_initialized = true;
 
-        decoder_add_sink(&decoder, &screen.frame_sink);
-
         if (options->turn_screen_off) {
             struct control_msg msg;
             msg.type = CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE;
@@ -412,10 +383,6 @@ scrcpy(const struct scrcpy_options *options) {
                 LOGW("Could not request 'set screen power mode'");
             }
         }
-
-        if (options->fullscreen) {
-            screen_switch_fullscreen(&screen);
-        }
     }
 
     // now we consumed the header values, the socket receives the video stream
@@ -431,8 +398,11 @@ scrcpy(const struct scrcpy_options *options) {
     LOGD("quit...");
 
 end:
-    // The stream is not stopped explicitly, because it will stop by itself on
-    // end-of-stream
+    // stop stream and controller so that they don't continue once their socket
+    // is shutdown
+    if (stream_started) {
+        stream_stop(&stream);
+    }
     if (controller_started) {
         controller_stop(&controller);
     }
@@ -476,6 +446,10 @@ end:
         file_handler_destroy(&file_handler);
     }
 
+    if (video_buffer_initialized) {
+        video_buffer_destroy(&video_buffer);
+    }
+
     if (fps_counter_initialized) {
         fps_counter_join(&fps_counter);
         fps_counter_destroy(&fps_counter);

app/src/scrcpy.h

@@ -72,6 +72,7 @@ struct scrcpy_options {
     bool control;
     bool display;
     bool turn_screen_off;
+    bool render_expired_frames;
     bool prefer_text;
     bool window_borderless;
     bool mipmaps;
@@ -119,6 +120,7 @@ struct scrcpy_options {
     .control = true, \
     .display = true, \
     .turn_screen_off = false, \
+    .render_expired_frames = false, \
     .prefer_text = false, \
     .window_borderless = false, \
     .mipmaps = true, \

app/src/screen.c

@@ -13,8 +13,6 @@
 
 #define DISPLAY_MARGINS 96
 
-#define DOWNCAST(SINK) container_of(SINK, struct screen, frame_sink)
-
 static inline struct size
 get_rotated_size(struct size size, int rotation) {
     struct size rotated_size;
@@ -193,6 +191,27 @@ screen_update_content_rect(struct screen *screen) {
     }
 }
 
+static void
+on_frame_available(struct video_buffer *vb, void *userdata) {
+    (void) vb;
+    (void) userdata;
+
+    static SDL_Event new_frame_event = {
+        .type = EVENT_NEW_FRAME,
+    };
+
+    // Post the event on the UI thread
+    SDL_PushEvent(&new_frame_event);
+}
+
+static void
+on_frame_skipped(struct video_buffer *vb, void *userdata) {
+    (void) vb;
+
+    struct screen *screen = userdata;
+    fps_counter_add_skipped_frame(screen->fps_counter);
+}
+
 static inline SDL_Texture *
 create_texture(struct screen *screen) {
     SDL_Renderer *renderer = screen->renderer;
@@ -220,58 +239,34 @@ create_texture(struct screen *screen) {
     return texture;
 }
 
-static bool
-screen_frame_sink_open(struct sc_frame_sink *sink) {
-    struct screen *screen = DOWNCAST(sink);
-    (void) screen;
-#ifndef NDEBUG
-    screen->open = true;
+#if defined(__APPLE__) || defined(__WINDOWS__)
+# define CONTINUOUS_RESIZING_WORKAROUND
 #endif
 
-    // nothing to do, the screen is already open on the main thread
-    return true;
+#ifdef CONTINUOUS_RESIZING_WORKAROUND
+// On Windows and MacOS, resizing blocks the event loop, so resizing events are
+// not triggered. As a workaround, handle them in an event handler.
+//
+// <https://bugzilla.libsdl.org/show_bug.cgi?id=2077>
+// <https://stackoverflow.com/a/40693139/1987178>
+static int
+event_watcher(void *data, SDL_Event *event) {
+    struct screen *screen = data;
+    if (event->type == SDL_WINDOWEVENT
+            && event->window.event == SDL_WINDOWEVENT_RESIZED) {
+        // In practice, it seems to always be called from the same thread in
+        // that specific case. Anyway, it's just a workaround.
+        screen_render(screen, true);
+    }
+    return 0;
 }
-
-static void
-screen_frame_sink_close(struct sc_frame_sink *sink) {
-    struct screen *screen = DOWNCAST(sink);
-    (void) screen;
-#ifndef NDEBUG
-    screen->open = false;
 #endif
 
-    // nothing to do, the screen lifecycle is not managed by the frame producer
-}
-
-static bool
-screen_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {
-    struct screen *screen = DOWNCAST(sink);
-
-    bool previous_frame_skipped;
-    bool ok = video_buffer_push(&screen->vb, frame, &previous_frame_skipped);
-    if (!ok) {
-        return false;
-    }
-
-    if (previous_frame_skipped) {
-        fps_counter_add_skipped_frame(screen->fps_counter);
-        // The EVENT_NEW_FRAME triggered for the previous frame will consume
-        // this new frame instead
-    } else {
-        static SDL_Event new_frame_event = {
-            .type = EVENT_NEW_FRAME,
-        };
-
-        // Post the event on the UI thread
-        SDL_PushEvent(&new_frame_event);
-    }
-
-    return true;
-}
-
 bool
-screen_init(struct screen *screen, struct fps_counter *fps_counter,
+screen_init(struct screen *screen, struct video_buffer *vb,
+            struct fps_counter *fps_counter,
             const struct screen_params *params) {
+    screen->vb = vb;
     screen->fps_counter = fps_counter;
 
     screen->resize_pending = false;
@@ -279,11 +274,11 @@ screen_init(struct screen *screen, struct fps_counter *fps_counter,
     screen->fullscreen = false;
     screen->maximized = false;
 
-    bool ok = video_buffer_init(&screen->vb);
-    if (!ok) {
-        LOGE("Could not initialize video buffer");
-        return false;
-    }
+    static const struct video_buffer_callbacks cbs = {
+        .on_frame_available = on_frame_available,
+        .on_frame_skipped = on_frame_skipped,
+    };
+    video_buffer_set_consumer_callbacks(vb, &cbs, screen);
 
     screen->frame_size = params->frame_size;
     screen->rotation = params->rotation;
@@ -329,7 +324,6 @@ screen_init(struct screen *screen, struct fps_counter *fps_counter,
     if (!screen->renderer) {
         LOGC("Could not create renderer: %s", SDL_GetError());
         SDL_DestroyWindow(screen->window);
-        video_buffer_destroy(&screen->vb);
         return false;
     }
 
@@ -381,17 +375,6 @@ screen_init(struct screen *screen, struct fps_counter *fps_counter,
         LOGC("Could not create texture: %s", SDL_GetError());
         SDL_DestroyRenderer(screen->renderer);
         SDL_DestroyWindow(screen->window);
-        video_buffer_destroy(&screen->vb);
-        return false;
-    }
-
-    screen->frame = av_frame_alloc();
-    if (!screen->frame) {
-        LOGC("Could not create screen frame");
-        SDL_DestroyTexture(screen->texture);
-        SDL_DestroyRenderer(screen->renderer);
-        SDL_DestroyWindow(screen->window);
-        video_buffer_destroy(&screen->vb);
         return false;
     }
 
@@ -402,36 +385,27 @@ screen_init(struct screen *screen, struct fps_counter *fps_counter,
 
     screen_update_content_rect(screen);
 
-    static const struct sc_frame_sink_ops ops = {
-        .open = screen_frame_sink_open,
-        .close = screen_frame_sink_close,
-        .push = screen_frame_sink_push,
-    };
+    if (params->fullscreen) {
+        screen_switch_fullscreen(screen);
+    }
 
-    screen->frame_sink.ops = &ops;
-
-#ifndef NDEBUG
-    screen->open = false;
+#ifdef CONTINUOUS_RESIZING_WORKAROUND
+    SDL_AddEventWatch(event_watcher, screen);
 #endif
 
     return true;
 }
 
-void
+static void
 screen_show_window(struct screen *screen) {
     SDL_ShowWindow(screen->window);
 }
 
 void
 screen_destroy(struct screen *screen) {
-#ifndef NDEBUG
-    assert(!screen->open);
-#endif
-    av_frame_free(&screen->frame);
     SDL_DestroyTexture(screen->texture);
     SDL_DestroyRenderer(screen->renderer);
     SDL_DestroyWindow(screen->window);
-    video_buffer_destroy(&screen->vb);
 }
 
 static void
@@ -536,9 +510,7 @@ update_texture(struct screen *screen, const AVFrame *frame) {
 
 static bool
 screen_update_frame(struct screen *screen) {
-    av_frame_unref(screen->frame);
-    video_buffer_consume(&screen->vb, screen->frame);
-    AVFrame *frame = screen->frame;
+    const AVFrame *frame = video_buffer_consumer_take_frame(screen->vb);
 
     fps_counter_add_rendered_frame(screen->fps_counter);
 

app/src/screen.h

@@ -9,19 +9,11 @@
 
 #include "coords.h"
 #include "opengl.h"
-#include "trait/frame_sink.h"
-#include "video_buffer.h"
 
 struct video_buffer;
 
 struct screen {
-    struct sc_frame_sink frame_sink; // frame sink trait
-
-#ifndef NDEBUG
-    bool open; // track the open/close state to assert correct behavior
-#endif
-
-    struct video_buffer vb;
+    struct video_buffer *vb;
     struct fps_counter *fps_counter;
 
     SDL_Window *window;
@@ -44,8 +36,6 @@ struct screen {
     bool fullscreen;
     bool maximized;
     bool mipmaps;
-
-    AVFrame *frame;
 };
 
 struct screen_params {
@@ -62,17 +52,16 @@ struct screen_params {
 
     uint8_t rotation;
     bool mipmaps;
+
+    bool fullscreen;
 };
 
 // initialize screen, create window, renderer and texture (window is hidden)
 bool
-screen_init(struct screen *screen, struct fps_counter *fps_counter,
+screen_init(struct screen *screen, struct video_buffer *vb,
+            struct fps_counter *fps_counter,
             const struct screen_params *params);
 
-// show the window
-void
-screen_show_window(struct screen *screen);
-
 // destroy window, renderer and texture (if any)
 void
 screen_destroy(struct screen *screen);

app/src/stream.c

@@ -66,11 +66,25 @@ notify_stopped(void) {
 }
 
 static bool
-push_packet_to_sinks(struct stream *stream, const AVPacket *packet) {
-    for (unsigned i = 0; i < stream->sink_count; ++i) {
-        struct sc_packet_sink *sink = stream->sinks[i];
-        if (!sink->ops->push(sink, packet)) {
-            LOGE("Could not send config packet to sink %d", i);
+process_config_packet(struct stream *stream, AVPacket *packet) {
+    if (stream->recorder && !recorder_push(stream->recorder, packet)) {
+        LOGE("Could not send config packet to recorder");
+        return false;
+    }
+    return true;
+}
+
+static bool
+process_frame(struct stream *stream, AVPacket *packet) {
+    if (stream->decoder && !decoder_push(stream->decoder, packet)) {
+        return false;
+    }
+
+    if (stream->recorder) {
+        packet->dts = packet->pts;
+
+        if (!recorder_push(stream->recorder, packet)) {
+            LOGE("Could not send packet to recorder");
             return false;
         }
     }
@@ -97,11 +111,9 @@ stream_parse(struct stream *stream, AVPacket *packet) {
         packet->flags |= AV_PKT_FLAG_KEY;
     }
 
-    packet->dts = packet->pts;
-
-    bool ok = push_packet_to_sinks(stream, packet);
+    bool ok = process_frame(stream, packet);
     if (!ok) {
-        LOGE("Could not process packet");
+        LOGE("Could not process frame");
         return false;
     }
 
@@ -144,7 +156,7 @@ stream_push_packet(struct stream *stream, AVPacket *packet) {
 
     if (is_config) {
         // config packet
-        bool ok = push_packet_to_sinks(stream, packet);
+        bool ok = process_config_packet(stream, packet);
         if (!ok) {
             return false;
         }
@@ -165,33 +177,6 @@ stream_push_packet(struct stream *stream, AVPacket *packet) {
     return true;
 }
 
-static void
-stream_close_first_sinks(struct stream *stream, unsigned count) {
-    while (count) {
-        struct sc_packet_sink *sink = stream->sinks[--count];
-        sink->ops->close(sink);
-    }
-}
-
-static inline void
-stream_close_sinks(struct stream *stream) {
-    stream_close_first_sinks(stream, stream->sink_count);
-}
-
-static bool
-stream_open_sinks(struct stream *stream, const AVCodec *codec) {
-    for (unsigned i = 0; i < stream->sink_count; ++i) {
-        struct sc_packet_sink *sink = stream->sinks[i];
-        if (!sink->ops->open(sink, codec)) {
-            LOGE("Could not open packet sink %d", i);
-            stream_close_first_sinks(stream, i);
-            return false;
-        }
-    }
-
-    return true;
-}
-
 static int
 run_stream(void *data) {
     struct stream *stream = data;
@@ -208,15 +193,27 @@ run_stream(void *data) {
         goto end;
     }
 
-    if (!stream_open_sinks(stream, codec)) {
-        LOGE("Could not open stream sinks");
+    if (stream->decoder && !decoder_open(stream->decoder, codec)) {
+        LOGE("Could not open decoder");
         goto finally_free_codec_ctx;
     }
 
+    if (stream->recorder) {
+        if (!recorder_open(stream->recorder, codec)) {
+            LOGE("Could not open recorder");
+            goto finally_close_decoder;
+        }
+
+        if (!recorder_start(stream->recorder)) {
+            LOGE("Could not start recorder");
+            goto finally_close_recorder;
+        }
+    }
+
     stream->parser = av_parser_init(AV_CODEC_ID_H264);
     if (!stream->parser) {
         LOGE("Could not initialize parser");
-        goto finally_close_sinks;
+        goto finally_stop_and_join_recorder;
     }
 
     // We must only pass complete frames to av_parser_parse2()!
@@ -246,8 +243,20 @@ run_stream(void *data) {
     }
 
     av_parser_close(stream->parser);
-finally_close_sinks:
-    stream_close_sinks(stream);
+finally_stop_and_join_recorder:
+    if (stream->recorder) {
+        recorder_stop(stream->recorder);
+        LOGI("Finishing recording...");
+        recorder_join(stream->recorder);
+    }
+finally_close_recorder:
+    if (stream->recorder) {
+        recorder_close(stream->recorder);
+    }
+finally_close_decoder:
+    if (stream->decoder) {
+        decoder_close(stream->decoder);
+    }
 finally_free_codec_ctx:
     avcodec_free_context(&stream->codec_ctx);
 end:
@@ -256,18 +265,12 @@ end:
 }
 
 void
-stream_init(struct stream *stream, socket_t socket) {
+stream_init(struct stream *stream, socket_t socket,
+            struct decoder *decoder, struct recorder *recorder) {
     stream->socket = socket;
+    stream->decoder = decoder,
+    stream->recorder = recorder;
     stream->has_pending = false;
-    stream->sink_count = 0;
-}
-
-void
-stream_add_sink(struct stream *stream, struct sc_packet_sink *sink) {
-    assert(stream->sink_count < STREAM_MAX_SINKS);
-    assert(sink);
-    assert(sink->ops);
-    stream->sinks[stream->sink_count++] = sink;
 }
 
 bool
@@ -282,6 +285,13 @@ stream_start(struct stream *stream) {
     return true;
 }
 
+void
+stream_stop(struct stream *stream) {
+    if (stream->decoder) {
+        decoder_interrupt(stream->decoder);
+    }
+}
+
 void
 stream_join(struct stream *stream) {
     sc_thread_join(&stream->thread, NULL);

app/src/stream.h

@@ -8,19 +8,14 @@
 #include <libavformat/avformat.h>
 #include <SDL2/SDL_atomic.h>
 
-#include "trait/packet_sink.h"
 #include "util/net.h"
 #include "util/thread.h"
 
-#define STREAM_MAX_SINKS 2
-
 struct stream {
     socket_t socket;
     sc_thread thread;
-
-    struct sc_packet_sink *sinks[STREAM_MAX_SINKS];
-    unsigned sink_count;
-
+    struct decoder *decoder;
+    struct recorder *recorder;
     AVCodecContext *codec_ctx;
     AVCodecParserContext *parser;
     // successive packets may need to be concatenated, until a non-config
@@ -30,14 +25,15 @@ struct stream {
 };
 
 void
-stream_init(struct stream *stream, socket_t socket);
-
-void
-stream_add_sink(struct stream *stream, struct sc_packet_sink *sink);
+stream_init(struct stream *stream, socket_t socket,
+            struct decoder *decoder, struct recorder *recorder);
 
 bool
 stream_start(struct stream *stream);
 
+void
+stream_stop(struct stream *stream);
+
 void
 stream_join(struct stream *stream);
 

app/src/trait/frame_sink.h

@@ -1,24 +0,0 @@
-#ifndef SC_FRAME_SINK
-#define SC_FRAME_SINK
-
-#include "common.h"
-
-#include <assert.h>
-#include <stdbool.h>
-
-/**
- * Frame sink trait.
- *
- * Component able to receive AVFrames should implement this trait.
- */
-struct sc_frame_sink {
-    const struct sc_frame_sink_ops *ops;
-};
-
-struct sc_frame_sink_ops {
-    bool (*open)(struct sc_frame_sink *sink);
-    void (*close)(struct sc_frame_sink *sink);
-    bool (*push)(struct sc_frame_sink *sink, const AVFrame *frame);
-};
-
-#endif

app/src/trait/packet_sink.h

@@ -1,25 +0,0 @@
-#ifndef SC_PACKET_SINK
-#define SC_PACKET_SINK
-
-#include "common.h"
-
-#include <assert.h>
-#include <stdbool.h>
-#include <libavcodec/avcodec.h>
-
-/**
- * Packet sink trait.
- *
- * Component able to receive AVPackets should implement this trait.
- */
-struct sc_packet_sink {
-    const struct sc_packet_sink_ops *ops;
-};
-
-struct sc_packet_sink_ops {
-    bool (*open)(struct sc_packet_sink *sink, const AVCodec *codec);
-    void (*close)(struct sc_packet_sink *sink);
-    bool (*push)(struct sc_packet_sink *sink, const AVPacket *packet);
-};
-
-#endif

app/src/video_buffer.c

@@ -7,36 +7,67 @@
 #include "util/log.h"
 
 bool
-video_buffer_init(struct video_buffer *vb) {
-    vb->pending_frame = av_frame_alloc();
-    if (!vb->pending_frame) {
-        return false;
+video_buffer_init(struct video_buffer *vb, bool wait_consumer) {
+    vb->producer_frame = av_frame_alloc();
+    if (!vb->producer_frame) {
+        goto error_0;
     }
 
-    vb->tmp_frame = av_frame_alloc();
-    if (!vb->tmp_frame) {
-        av_frame_free(&vb->pending_frame);
-        return false;
+    vb->pending_frame = av_frame_alloc();
+    if (!vb->pending_frame) {
+        goto error_1;
+    }
+
+    vb->consumer_frame = av_frame_alloc();
+    if (!vb->consumer_frame) {
+        goto error_2;
     }
 
     bool ok = sc_mutex_init(&vb->mutex);
     if (!ok) {
-        av_frame_free(&vb->pending_frame);
-        av_frame_free(&vb->tmp_frame);
-        return false;
+        goto error_3;
+    }
+
+    vb->wait_consumer = wait_consumer;
+    if (wait_consumer) {
+        ok = sc_cond_init(&vb->pending_frame_consumed_cond);
+        if (!ok) {
+            sc_mutex_destroy(&vb->mutex);
+            goto error_2;
+        }
+        // interrupted is not used if wait_consumer is disabled since offering
+        // a frame will never block
+        vb->interrupted = false;
     }
 
     // there is initially no frame, so consider it has already been consumed
     vb->pending_frame_consumed = true;
 
+    // The callbacks must be set by the consumer via
+    // video_buffer_set_consumer_callbacks()
+    vb->cbs = NULL;
+
     return true;
+
+error_3:
+    av_frame_free(&vb->consumer_frame);
+error_2:
+    av_frame_free(&vb->pending_frame);
+error_1:
+    av_frame_free(&vb->producer_frame);
+error_0:
+    return false;
 }
 
 void
 video_buffer_destroy(struct video_buffer *vb) {
+    if (vb->wait_consumer) {
+        sc_cond_destroy(&vb->pending_frame_consumed_cond);
+    }
     sc_mutex_destroy(&vb->mutex);
+    av_frame_free(&vb->consumer_frame);
     av_frame_free(&vb->pending_frame);
-    av_frame_free(&vb->tmp_frame);
+    av_frame_free(&vb->producer_frame);
 }
 
 static inline void
@@ -46,43 +77,71 @@ swap_frames(AVFrame **lhs, AVFrame **rhs) {
     *rhs = tmp;
 }
 
-bool
-video_buffer_push(struct video_buffer *vb, const AVFrame *frame,
-                  bool *previous_frame_skipped) {
+void
+video_buffer_set_consumer_callbacks(struct video_buffer *vb,
+                                    const struct video_buffer_callbacks *cbs,
+                                    void *cbs_userdata) {
+    assert(!vb->cbs); // must be set only once
+    assert(cbs);
+    assert(cbs->on_frame_available);
+    vb->cbs = cbs;
+    vb->cbs_userdata = cbs_userdata;
+}
+
+void
+video_buffer_producer_offer_frame(struct video_buffer *vb) {
+    assert(vb->cbs);
+
     sc_mutex_lock(&vb->mutex);
-
-    // Use a temporary frame to preserve pending_frame in case of error.
-    // tmp_frame is an empty frame, no need to call av_frame_unref() beforehand.
-    int r = av_frame_ref(vb->tmp_frame, frame);
-    if (r) {
-        LOGE("Could not ref frame: %d", r);
-        return false;
+    if (vb->wait_consumer) {
+        // wait for the current (expired) frame to be consumed
+        while (!vb->pending_frame_consumed && !vb->interrupted) {
+            sc_cond_wait(&vb->pending_frame_consumed_cond, &vb->mutex);
+        }
     }
 
-    // Now that av_frame_ref() succeeded, we can replace the previous
-    // pending_frame
-    swap_frames(&vb->pending_frame, &vb->tmp_frame);
-    av_frame_unref(vb->tmp_frame);
+    av_frame_unref(vb->pending_frame);
+    swap_frames(&vb->producer_frame, &vb->pending_frame);
 
-    if (previous_frame_skipped) {
-        *previous_frame_skipped = !vb->pending_frame_consumed;
-    }
+    bool skipped = !vb->pending_frame_consumed;
     vb->pending_frame_consumed = false;
 
     sc_mutex_unlock(&vb->mutex);
 
-    return true;
+    if (skipped) {
+        if (vb->cbs->on_frame_skipped)
+            vb->cbs->on_frame_skipped(vb, vb->cbs_userdata);
+    } else {
+        vb->cbs->on_frame_available(vb, vb->cbs_userdata);
+    }
 }
 
-void
-video_buffer_consume(struct video_buffer *vb, AVFrame *dst) {
+const AVFrame *
+video_buffer_consumer_take_frame(struct video_buffer *vb) {
     sc_mutex_lock(&vb->mutex);
     assert(!vb->pending_frame_consumed);
     vb->pending_frame_consumed = true;
 
-    av_frame_move_ref(dst, vb->pending_frame);
-    // av_frame_move_ref() resets its source frame, so no need to call
-    // av_frame_unref()
+    swap_frames(&vb->consumer_frame, &vb->pending_frame);
+    av_frame_unref(vb->pending_frame);
 
+    if (vb->wait_consumer) {
+        // unblock video_buffer_offer_decoded_frame()
+        sc_cond_signal(&vb->pending_frame_consumed_cond);
+    }
     sc_mutex_unlock(&vb->mutex);
+
+    // consumer_frame is only written from this thread, no need to lock
+    return vb->consumer_frame;
+}
+
+void
+video_buffer_interrupt(struct video_buffer *vb) {
+    if (vb->wait_consumer) {
+        sc_mutex_lock(&vb->mutex);
+        vb->interrupted = true;
+        sc_mutex_unlock(&vb->mutex);
+        // wake up blocking wait
+        sc_cond_signal(&vb->pending_frame_consumed_cond);
+    }
 }

app/src/video_buffer.h

@@ -12,39 +12,71 @@
 typedef struct AVFrame AVFrame;
 
 /**
- * A video buffer holds 1 pending frame, which is the last frame received from
- * the producer (typically, the decoder).
+ * There are 3 frames in memory:
+ *  - one frame is held by the producer (producer_frame)
+ *  - one frame is held by the consumer (consumer_frame)
+ *  - one frame is shared between the producer and the consumer (pending_frame)
  *
- * If a pending frame has not been consumed when the producer pushes a new
- * frame, then it is lost. The intent is to always provide access to the very
- * last frame to minimize latency.
+ * The producer generates a frame into the producer_frame (it may takes time).
  *
- * The producer and the consumer typically do not live in the same thread.
- * That's the reason why the callback on_frame_available() does not provide the
- * frame as parameter: the consumer might post an event to its own thread to
- * retrieve the pending frame from there, and that frame may have changed since
- * the callback if producer pushed a new one in between.
+ * Once the frame is produced, it calls video_buffer_producer_offer_frame(),
+ * which swaps the producer and pending frames.
+ *
+ * When the consumer is notified that a new frame is available, it calls
+ * video_buffer_consumer_take_frame() to retrieve it, which swaps the pending
+ * and consumer frames. The frame is valid until the next call, without
+ * blocking the producer.
  */
 
 struct video_buffer {
+    AVFrame *producer_frame;
     AVFrame *pending_frame;
-    AVFrame *tmp_frame; // To preserve the pending frame on error
+    AVFrame *consumer_frame;
 
     sc_mutex mutex;
+    bool wait_consumer; // never overwrite a pending frame if it is not consumed
+    bool interrupted;
 
+    sc_cond pending_frame_consumed_cond;
     bool pending_frame_consumed;
+
+    const struct video_buffer_callbacks *cbs;
+    void *cbs_userdata;
+};
+
+struct video_buffer_callbacks {
+    // Called when a new frame can be consumed by
+    // video_buffer_consumer_take_frame(vb)
+    // This callback is mandatory (it must not be NULL).
+    void (*on_frame_available)(struct video_buffer *vb, void *userdata);
+
+    // Called when a pending frame has been overwritten by the producer
+    // This callback is optional (it may be NULL).
+    void (*on_frame_skipped)(struct video_buffer *vb, void *userdata);
 };
 
 bool
-video_buffer_init(struct video_buffer *vb);
+video_buffer_init(struct video_buffer *vb, bool wait_consumer);
 
 void
 video_buffer_destroy(struct video_buffer *vb);
 
-bool
-video_buffer_push(struct video_buffer *vb, const AVFrame *frame, bool *skipped);
-
 void
-video_buffer_consume(struct video_buffer *vb, AVFrame *dst);
+video_buffer_set_consumer_callbacks(struct video_buffer *vb,
+                                    const struct video_buffer_callbacks *cbs,
+                                    void *cbs_userdata);
+
+// set the producer frame as ready for consuming
+void
+video_buffer_producer_offer_frame(struct video_buffer *vb);
+
+// mark the consumer frame as consumed and return it
+// the frame is valid until the next call to this function
+const AVFrame *
+video_buffer_consumer_take_frame(struct video_buffer *vb);
+
+// wake up and avoid any blocking call
+void
+video_buffer_interrupt(struct video_buffer *vb);
 
 #endif

app/tests/test_cli.c

@@ -58,6 +58,7 @@ static void test_options(void) {
         "--push-target", "/sdcard/Movies",
         "--record", "file",
         "--record-format", "mkv",
+        "--render-expired-frames",
         "--serial", "0123456789abcdef",
         "--show-touches",
         "--turn-screen-off",
@@ -86,6 +87,7 @@ static void test_options(void) {
     assert(!strcmp(opts->push_target, "/sdcard/Movies"));
     assert(!strcmp(opts->record_filename, "file"));
     assert(opts->record_format == SC_RECORD_FORMAT_MKV);
+    assert(opts->render_expired_frames);
     assert(!strcmp(opts->serial, "0123456789abcdef"));
     assert(opts->show_touches);
     assert(opts->turn_screen_off);

app/tests/test_control_msg_serialize.c

@@ -146,14 +146,18 @@ static void test_serialize_inject_scroll_event(void) {
 static void test_serialize_back_or_screen_on(void) {
     struct control_msg msg = {
         .type = CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON,
+        .back_or_screen_on = {
+            .action = AKEY_EVENT_ACTION_UP,
+        },
     };
 
     unsigned char buf[CONTROL_MSG_MAX_SIZE];
     size_t size = control_msg_serialize(&msg, buf);
-    assert(size == 1);
+    assert(size == 2);
 
     const unsigned char expected[] = {
         CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON,
+        0x01, // AKEY_EVENT_ACTION_UP
     };
     assert(!memcmp(buf, expected, sizeof(expected)));
 }
@@ -173,9 +177,9 @@ static void test_serialize_expand_notification_panel(void) {
     assert(!memcmp(buf, expected, sizeof(expected)));
 }
 
-static void test_serialize_collapse_notification_panel(void) {
+static void test_serialize_expand_settings_panel(void) {
     struct control_msg msg = {
-        .type = CONTROL_MSG_TYPE_COLLAPSE_NOTIFICATION_PANEL,
+        .type = CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL,
     };
 
     unsigned char buf[CONTROL_MSG_MAX_SIZE];
@@ -183,7 +187,22 @@ static void test_serialize_collapse_notification_panel(void) {
     assert(size == 1);
 
     const unsigned char expected[] = {
-        CONTROL_MSG_TYPE_COLLAPSE_NOTIFICATION_PANEL,
+        CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL,
+    };
+    assert(!memcmp(buf, expected, sizeof(expected)));
+}
+
+static void test_serialize_collapse_panels(void) {
+    struct control_msg msg = {
+        .type = CONTROL_MSG_TYPE_COLLAPSE_PANELS,
+    };
+
+    unsigned char buf[CONTROL_MSG_MAX_SIZE];
+    size_t size = control_msg_serialize(&msg, buf);
+    assert(size == 1);
+
+    const unsigned char expected[] = {
+        CONTROL_MSG_TYPE_COLLAPSE_PANELS,
     };
     assert(!memcmp(buf, expected, sizeof(expected)));
 }
@@ -270,7 +289,8 @@ int main(int argc, char *argv[]) {
     test_serialize_inject_scroll_event();
     test_serialize_back_or_screen_on();
     test_serialize_expand_notification_panel();
-    test_serialize_collapse_notification_panel();
+    test_serialize_expand_settings_panel();
+    test_serialize_collapse_panels();
     test_serialize_get_clipboard();
     test_serialize_set_clipboard();
     test_serialize_set_screen_power_mode();

server/src/main/java/com/genymobile/scrcpy/ControlMessage.java

@@ -11,11 +11,12 @@ public final class ControlMessage {
     public static final int TYPE_INJECT_SCROLL_EVENT = 3;
     public static final int TYPE_BACK_OR_SCREEN_ON = 4;
     public static final int TYPE_EXPAND_NOTIFICATION_PANEL = 5;
-    public static final int TYPE_COLLAPSE_NOTIFICATION_PANEL = 6;
-    public static final int TYPE_GET_CLIPBOARD = 7;
-    public static final int TYPE_SET_CLIPBOARD = 8;
-    public static final int TYPE_SET_SCREEN_POWER_MODE = 9;
-    public static final int TYPE_ROTATE_DEVICE = 10;
+    public static final int TYPE_EXPAND_SETTINGS_PANEL = 6;
+    public static final int TYPE_COLLAPSE_PANELS = 7;
+    public static final int TYPE_GET_CLIPBOARD = 8;
+    public static final int TYPE_SET_CLIPBOARD = 9;
+    public static final int TYPE_SET_SCREEN_POWER_MODE = 10;
+    public static final int TYPE_ROTATE_DEVICE = 11;
 
     private int type;
     private String text;
@@ -71,6 +72,13 @@ public final class ControlMessage {
         return msg;
     }
 
+    public static ControlMessage createBackOrScreenOn(int action) {
+        ControlMessage msg = new ControlMessage();
+        msg.type = TYPE_BACK_OR_SCREEN_ON;
+        msg.action = action;
+        return msg;
+    }
+
     public static ControlMessage createSetClipboard(String text, boolean paste) {
         ControlMessage msg = new ControlMessage();
         msg.type = TYPE_SET_CLIPBOARD;

server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java

@@ -11,6 +11,7 @@ public class ControlMessageReader {
     static final int INJECT_KEYCODE_PAYLOAD_LENGTH = 13;
     static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 27;
     static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
+    static final int BACK_OR_SCREEN_ON_LENGTH = 1;
     static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
     static final int SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH = 1;
 
@@ -66,15 +67,18 @@ public class ControlMessageReader {
             case ControlMessage.TYPE_INJECT_SCROLL_EVENT:
                 msg = parseInjectScrollEvent();
                 break;
+            case ControlMessage.TYPE_BACK_OR_SCREEN_ON:
+                msg = parseBackOrScreenOnEvent();
+                break;
             case ControlMessage.TYPE_SET_CLIPBOARD:
                 msg = parseSetClipboard();
                 break;
             case ControlMessage.TYPE_SET_SCREEN_POWER_MODE:
                 msg = parseSetScreenPowerMode();
                 break;
-            case ControlMessage.TYPE_BACK_OR_SCREEN_ON:
             case ControlMessage.TYPE_EXPAND_NOTIFICATION_PANEL:
-            case ControlMessage.TYPE_COLLAPSE_NOTIFICATION_PANEL:
+            case ControlMessage.TYPE_EXPAND_SETTINGS_PANEL:
+            case ControlMessage.TYPE_COLLAPSE_PANELS:
             case ControlMessage.TYPE_GET_CLIPBOARD:
             case ControlMessage.TYPE_ROTATE_DEVICE:
                 msg = ControlMessage.createEmpty(type);
@@ -150,6 +154,14 @@ public class ControlMessageReader {
         return ControlMessage.createInjectScrollEvent(position, hScroll, vScroll);
     }
 
+    private ControlMessage parseBackOrScreenOnEvent() {
+        if (buffer.remaining() < BACK_OR_SCREEN_ON_LENGTH) {
+            return null;
+        }
+        int action = toUnsigned(buffer.get());
+        return ControlMessage.createBackOrScreenOn(action);
+    }
+
     private ControlMessage parseSetClipboard() {
         if (buffer.remaining() < SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH) {
             return null;

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -101,13 +101,16 @@ public class Controller {
                 break;
             case ControlMessage.TYPE_BACK_OR_SCREEN_ON:
                 if (device.supportsInputEvents()) {
-                    pressBackOrTurnScreenOn();
+                    pressBackOrTurnScreenOn(msg.getAction());
                 }
                 break;
             case ControlMessage.TYPE_EXPAND_NOTIFICATION_PANEL:
                 Device.expandNotificationPanel();
                 break;
-            case ControlMessage.TYPE_COLLAPSE_NOTIFICATION_PANEL:
+            case ControlMessage.TYPE_EXPAND_SETTINGS_PANEL:
+                Device.expandSettingsPanel();
+                break;
+            case ControlMessage.TYPE_COLLAPSE_PANELS:
                 Device.collapsePanels();
                 break;
             case ControlMessage.TYPE_GET_CLIPBOARD:
@@ -255,12 +258,22 @@ public class Controller {
         }, 200, TimeUnit.MILLISECONDS);
     }
 
-    private boolean pressBackOrTurnScreenOn() {
-        int keycode = Device.isScreenOn() ? KeyEvent.KEYCODE_BACK : KeyEvent.KEYCODE_POWER;
-        if (keepPowerModeOff && keycode == KeyEvent.KEYCODE_POWER) {
+    private boolean pressBackOrTurnScreenOn(int action) {
+        if (Device.isScreenOn()) {
+            return device.injectKeyEvent(action, KeyEvent.KEYCODE_BACK, 0, 0);
+        }
+
+        // Screen is off
+        // Only press POWER on ACTION_DOWN
+        if (action != KeyEvent.ACTION_DOWN) {
+            // do nothing,
+            return true;
+        }
+
+        if (keepPowerModeOff) {
             schedulePowerModeOff();
         }
-        return device.injectKeycode(keycode);
+        return device.injectKeycode(KeyEvent.KEYCODE_POWER);
     }
 
     private boolean setClipboard(String text, boolean paste) {

server/src/main/java/com/genymobile/scrcpy/Device.java

@@ -227,6 +227,10 @@ public final class Device {
         SERVICE_MANAGER.getStatusBarManager().expandNotificationsPanel();
     }
 
+    public static void expandSettingsPanel() {
+        SERVICE_MANAGER.getStatusBarManager().expandSettingsPanel();
+    }
+
     public static void collapsePanels() {
         SERVICE_MANAGER.getStatusBarManager().collapsePanels();
     }

server/src/main/java/com/genymobile/scrcpy/wrappers/StatusBarManager.java

@@ -11,6 +11,8 @@ public class StatusBarManager {
 
     private final IInterface manager;
     private Method expandNotificationsPanelMethod;
+    private Method expandSettingsPanelMethod;
+    private boolean expandSettingsPanelMethodLegacy;
     private Method collapsePanelsMethod;
 
     public StatusBarManager(IInterface manager) {
@@ -24,6 +26,20 @@ public class StatusBarManager {
         return expandNotificationsPanelMethod;
     }
 
+    private Method getExpandSettingsPanel() throws NoSuchMethodException {
+        if (expandSettingsPanelMethod == null) {
+            try {
+                // Since Android 7: https://android.googlesource.com/platform/frameworks/base.git/+/a9927325eda025504d59bb6594fee8e240d95b01%5E%21/
+                expandSettingsPanelMethod = manager.getClass().getMethod("expandSettingsPanel", String.class);
+            } catch (NoSuchMethodException e) {
+                // old version
+                expandSettingsPanelMethod = manager.getClass().getMethod("expandSettingsPanel");
+                expandSettingsPanelMethodLegacy = true;
+            }
+        }
+        return expandSettingsPanelMethod;
+    }
+
     private Method getCollapsePanelsMethod() throws NoSuchMethodException {
         if (collapsePanelsMethod == null) {
             collapsePanelsMethod = manager.getClass().getMethod("collapsePanels");
@@ -40,6 +56,21 @@ public class StatusBarManager {
         }
     }
 
+    public void expandSettingsPanel() {
+        try {
+            Method method = getExpandSettingsPanel();
+            if (!expandSettingsPanelMethodLegacy) {
+                // new version
+                method.invoke(manager, (Object) null);
+            } else {
+                // old version
+                method.invoke(manager);
+            }
+        } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
+            Ln.e("Could not invoke method", e);
+        }
+    }
+
     public void collapsePanels() {
         try {
             Method method = getCollapsePanelsMethod();

server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java

@@ -154,6 +154,7 @@ public class ControlMessageReaderTest {
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
         DataOutputStream dos = new DataOutputStream(bos);
         dos.writeByte(ControlMessage.TYPE_BACK_OR_SCREEN_ON);
+        dos.writeByte(KeyEvent.ACTION_UP);
 
         byte[] packet = bos.toByteArray();
 
@@ -161,6 +162,7 @@ public class ControlMessageReaderTest {
         ControlMessage event = reader.next();
 
         Assert.assertEquals(ControlMessage.TYPE_BACK_OR_SCREEN_ON, event.getType());
+        Assert.assertEquals(KeyEvent.ACTION_UP, event.getAction());
     }
 
     @Test
@@ -180,19 +182,35 @@ public class ControlMessageReaderTest {
     }
 
     @Test
-    public void testParseCollapseNotificationPanelEvent() throws IOException {
+    public void testParseExpandSettingsPanelEvent() throws IOException {
         ControlMessageReader reader = new ControlMessageReader();
 
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
         DataOutputStream dos = new DataOutputStream(bos);
-        dos.writeByte(ControlMessage.TYPE_COLLAPSE_NOTIFICATION_PANEL);
+        dos.writeByte(ControlMessage.TYPE_EXPAND_SETTINGS_PANEL);
 
         byte[] packet = bos.toByteArray();
 
         reader.readFrom(new ByteArrayInputStream(packet));
         ControlMessage event = reader.next();
 
-        Assert.assertEquals(ControlMessage.TYPE_COLLAPSE_NOTIFICATION_PANEL, event.getType());
+        Assert.assertEquals(ControlMessage.TYPE_EXPAND_SETTINGS_PANEL, event.getType());
+    }
+
+    @Test
+    public void testParseCollapsePanelsEvent() throws IOException {
+        ControlMessageReader reader = new ControlMessageReader();
+
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        DataOutputStream dos = new DataOutputStream(bos);
+        dos.writeByte(ControlMessage.TYPE_COLLAPSE_PANELS);
+
+        byte[] packet = bos.toByteArray();
+
+        reader.readFrom(new ByteArrayInputStream(packet));
+        ControlMessage event = reader.next();
+
+        Assert.assertEquals(ControlMessage.TYPE_COLLAPSE_PANELS, event.getType());
     }
 
     @Test