wip_everything_broken

The video buffer held 3 frames:
 - the producer frame (for decoding)
 - the pending frame (to exchange between the producer and consumer)
 - the consumer frame (for rendering)

It worked well, but it prevented video buffers to be chained, because
the consumer frame of the first video buffer must be the same as the
producer frame of the second video buffer.

To solve this problem, make the decoder handle its decoding frame, and
keep only the pending and consumer frames in the video_buffer.

    decoder -> pending -> consumer -> pending -> consumer
             |---------------------||---------------------|
                  video_buffer 1         video_buffer 2

This paves the way to support asynchronous swscale.

app/meson.build

@@ -15,6 +15,7 @@ src = [
     'src/opengl.c',
     'src/receiver.c',
     'src/recorder.c',
+    'src/resizer.c',
     'src/scrcpy.c',
     'src/screen.c',
     'src/server.c',
@@ -46,6 +47,7 @@ if not get_option('crossbuild_windows')
         dependency('libavformat'),
         dependency('libavcodec'),
         dependency('libavutil'),
+        dependency('libswscale'),
         dependency('sdl2'),
     ]
 
@@ -114,16 +116,6 @@ conf.set('PORTABLE', get_option('portable'))
 conf.set('DEFAULT_LOCAL_PORT_RANGE_FIRST', '27183')
 conf.set('DEFAULT_LOCAL_PORT_RANGE_LAST', '27199')
 
-# the default max video size for both dimensions, in pixels
-# overridden by option --max-size
-conf.set('DEFAULT_MAX_SIZE', '0')  # 0: unlimited
-
-# the default video orientation
-# natural device orientation is 0 and each increment adds 90 degrees
-# counterclockwise
-# overridden by option --lock-video-orientation
-conf.set('DEFAULT_LOCK_VIDEO_ORIENTATION', '-1')  # -1: unlocked
-
 # the default video bitrate, in bits/second
 # overridden by option --bit-rate
 conf.set('DEFAULT_BIT_RATE', '8000000')  # 8Mbps

app/scrcpy.1

@@ -167,6 +167,13 @@ Set the initial display rotation. Possibles values are 0, 1, 2 and 3. Each incre
 .BI "\-s, \-\-serial " number
 The device serial number. Mandatory only if several devices are connected to adb.
 
+.TP
+.BI "\-\-scale\-filter filter
+Supported filters are "none" and "trilinear".
+
+Trilinear filtering is only available if the renderer is OpenGL 3.0+ or OpenGL
+ES 2.0+.
+
 .TP
 .BI "\-\-shortcut\-mod " key[+...]][,...]
 Specify the modifiers to use for scrcpy shortcuts. Possible keys are "lctrl", "rctrl", "lalt", "ralt", "lsuper" and "rsuper".

app/src/cli.c

@@ -10,6 +10,9 @@
 #include "util/log.h"
 #include "util/str_util.h"
 
+#define STR_IMPL_(x) #x
+#define STR(x) STR_IMPL_(x)
+
 void
 scrcpy_print_usage(const char *arg0) {
     fprintf(stderr,
@@ -23,7 +26,7 @@ scrcpy_print_usage(const char *arg0) {
         "    -b, --bit-rate value\n"
         "        Encode the video at the given bit-rate, expressed in bits/s.\n"
         "        Unit suffixes are supported: 'K' (x1000) and 'M' (x1000000).\n"
-        "        Default is %d.\n"
+        "        Default is " STR(DEFAULT_BIT_RATE) ".\n"
         "\n"
         "    --codec-options key[:type]=value[,...]\n"
         "        Set a list of comma-separated key:type=value options for the\n"
@@ -81,7 +84,7 @@ scrcpy_print_usage(const char *arg0) {
         "        Possible values are -1 (unlocked), 0, 1, 2 and 3.\n"
         "        Natural device orientation is 0, and each increment adds a\n"
         "        90 degrees rotation counterclockwise.\n"
-        "        Default is %d%s.\n"
+        "        Default is -1 (unlocked).\n"
         "\n"
         "    --max-fps value\n"
         "        Limit the frame rate of screen capture (officially supported\n"
@@ -91,7 +94,7 @@ scrcpy_print_usage(const char *arg0) {
         "        Limit both the width and height of the video to value. The\n"
         "        other dimension is computed so that the device aspect-ratio\n"
         "        is preserved.\n"
-        "        Default is %d%s.\n"
+        "        Default is 0 (unlimited).\n"
         "\n"
         "    -n, --no-control\n"
         "        Disable device control (mirror the device in read-only).\n"
@@ -103,14 +106,10 @@ scrcpy_print_usage(const char *arg0) {
         "    --no-key-repeat\n"
         "        Do not forward repeated key events when a key is held down.\n"
         "\n"
-        "    --no-mipmaps\n"
-        "        If the renderer is OpenGL 3.0+ or OpenGL ES 2.0+, then\n"
-        "        mipmaps are automatically generated to improve downscaling\n"
-        "        quality. This option disables the generation of mipmaps.\n"
-        "\n"
         "    -p, --port port[:port]\n"
         "        Set the TCP port (range) used by the client to listen.\n"
-        "        Default is %d:%d.\n"
+        "        Default is " STR(DEFAULT_LOCAL_PORT_RANGE_FIRST) ":"
+                              STR(DEFAULT_LOCAL_PORT_RANGE_LAST) ".\n"
         "\n"
         "    --prefer-text\n"
         "        Inject alpha characters and space as text events instead of\n"
@@ -154,6 +153,11 @@ scrcpy_print_usage(const char *arg0) {
         "        The device serial number. Mandatory only if several devices\n"
         "        are connected to adb.\n"
         "\n"
+        "    --scale-filter filter\n"
+        "        Supported filters are \"none\" and \"trilinear\".\n"
+        "        Trilinear filtering is only available if the renderer is\n"
+        "        OpenGL 3.0+ or OpenGL ES 2.0+.\n"
+        "\n"
         "    --shortcut-mod key[+...]][,...]\n"
         "        Specify the modifiers to use for scrcpy shortcuts.\n"
         "        Possible keys are \"lctrl\", \"rctrl\", \"lalt\", \"ralt\",\n"
@@ -297,12 +301,7 @@ scrcpy_print_usage(const char *arg0) {
         "\n"
         "    Drag & drop APK file\n"
         "        Install APK from computer\n"
-        "\n",
-        arg0,
-        DEFAULT_BIT_RATE,
-        DEFAULT_LOCK_VIDEO_ORIENTATION, DEFAULT_LOCK_VIDEO_ORIENTATION >= 0 ? "" : " (unlocked)",
-        DEFAULT_MAX_SIZE, DEFAULT_MAX_SIZE ? "" : " (unlimited)",
-        DEFAULT_LOCAL_PORT_RANGE_FIRST, DEFAULT_LOCAL_PORT_RANGE_LAST);
+        "\n", arg0);
 }
 
 static bool
@@ -641,6 +640,63 @@ guess_record_format(const char *filename) {
     return 0;
 }
 
+static bool
+parse_scale_filter(const char *optarg, enum sc_scale_filter *filter) {
+    // TODO store in a map and loop over the entries instead
+    if (!strcmp(optarg, "none")) {
+        *filter = SC_SCALE_FILTER_NONE;
+        return true;
+    }
+    if (!strcmp(optarg, "trilinear")) {
+        *filter = SC_SCALE_FILTER_TRILINEAR;
+        return true;
+    }
+    if (!strcmp(optarg, "bilinear")) {
+        *filter = SC_SCALE_FILTER_BILINEAR;
+        return true;
+    }
+    if (!strcmp(optarg, "bicubic")) {
+        *filter = SC_SCALE_FILTER_BICUBIC;
+        return true;
+    }
+    if (!strcmp(optarg, "x")) {
+        *filter = SC_SCALE_FILTER_X;
+        return true;
+    }
+    if (!strcmp(optarg, "point")) {
+        *filter = SC_SCALE_FILTER_POINT;
+        return true;
+    }
+    if (!strcmp(optarg, "area")) {
+        *filter = SC_SCALE_FILTER_AREA;
+        return true;
+    }
+    if (!strcmp(optarg, "bicublin")) {
+        *filter = SC_SCALE_FILTER_BICUBLIN;
+        return true;
+    }
+    if (!strcmp(optarg, "gauss")) {
+        *filter = SC_SCALE_FILTER_GAUSS;
+        return true;
+    }
+    if (!strcmp(optarg, "sinc")) {
+        *filter = SC_SCALE_FILTER_SINC;
+        return true;
+    }
+    if (!strcmp(optarg, "lanczos")) {
+        *filter = SC_SCALE_FILTER_LANCZOS;
+        return true;
+    }
+    if (!strcmp(optarg, "spline")) {
+        *filter = SC_SCALE_FILTER_SPLINE;
+        return true;
+    }
+
+    LOGE("Unsupported scale filter: %s "
+         "(expected one of [TODO])", optarg);
+    return false;
+}
+
 #define OPT_RENDER_EXPIRED_FRAMES  1000
 #define OPT_WINDOW_TITLE           1001
 #define OPT_PUSH_TARGET            1002
@@ -667,6 +723,7 @@ guess_record_format(const char *filename) {
 #define OPT_FORWARD_ALL_CLICKS     1023
 #define OPT_LEGACY_PASTE           1024
 #define OPT_ENCODER_NAME           1025
+#define OPT_SCALE_FILTER           1026
 
 bool
 scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
@@ -703,6 +760,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
         {"render-expired-frames",  no_argument,       NULL,
                                                   OPT_RENDER_EXPIRED_FRAMES},
         {"rotation",               required_argument, NULL, OPT_ROTATION},
+        {"scale-filter",           required_argument, NULL, OPT_SCALE_FILTER},
         {"serial",                 required_argument, NULL, 's'},
         {"shortcut-mod",           required_argument, NULL, OPT_SHORTCUT_MOD},
         {"show-touches",           no_argument,       NULL, 't'},
@@ -857,7 +915,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
                 opts->render_driver = optarg;
                 break;
             case OPT_NO_MIPMAPS:
-                opts->mipmaps = false;
+                LOGW("Deprecated option --no-mipmaps. "
+                     "Use --scale-filter=none instead.");
+                opts->scale_filter = SC_SCALE_FILTER_NONE;
                 break;
             case OPT_NO_KEY_REPEAT:
                 opts->forward_key_repeat = false;
@@ -885,6 +945,11 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
             case OPT_LEGACY_PASTE:
                 opts->legacy_paste = true;
                 break;
+            case OPT_SCALE_FILTER:
+                if (!parse_scale_filter(optarg, &opts->scale_filter)) {
+                    return false;
+                }
+                break;
             default:
                 // getopt prints the error message on stderr
                 return false;

app/src/decoder.c

@@ -11,12 +11,6 @@
 #include "util/buffer_util.h"
 #include "util/log.h"
 
-// set the decoded frame as ready for rendering, and notify
-static void
-push_frame(struct decoder *decoder) {
-    video_buffer_producer_offer_frame(decoder->video_buffer);
-}
-
 void
 decoder_init(struct decoder *decoder, struct video_buffer *vb) {
     decoder->video_buffer = vb;
@@ -36,11 +30,19 @@ decoder_open(struct decoder *decoder, const AVCodec *codec) {
         return false;
     }
 
+    decoder->frame = av_frame_alloc();
+    if (!decoder->frame) {
+        avcodec_close(decoder->codec_ctx);
+        avcodec_free_context(&decoder->codec_ctx);
+        return false;
+    }
+
     return true;
 }
 
 void
 decoder_close(struct decoder *decoder) {
+    av_frame_free(&decoder->frame);
     avcodec_close(decoder->codec_ctx);
     avcodec_free_context(&decoder->codec_ctx);
 }
@@ -55,11 +57,11 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) {
         LOGE("Could not send video packet: %d", ret);
         return false;
     }
-    ret = avcodec_receive_frame(decoder->codec_ctx,
-                                decoder->video_buffer->producer_frame);
+    ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame);
     if (!ret) {
         // a frame was received
-        push_frame(decoder);
+        video_buffer_producer_offer_frame(decoder->video_buffer,
+                                          &decoder->frame);
     } else if (ret != AVERROR(EAGAIN)) {
         LOGE("Could not receive video frame: %d", ret);
         return false;
@@ -67,7 +69,7 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) {
 #else
     int got_picture;
     int len = avcodec_decode_video2(decoder->codec_ctx,
-                                    decoder->video_buffer->decoding_frame,
+                                    decoder->frame,
                                     &got_picture,
                                     packet);
     if (len < 0) {

app/src/decoder.h

@@ -10,7 +10,9 @@ struct video_buffer;
 
 struct decoder {
     struct video_buffer *video_buffer;
+
     AVCodecContext *codec_ctx;
+    AVFrame *frame;
 };
 
 void

app/src/fps_counter.c

@@ -168,7 +168,7 @@ fps_counter_add_rendered_frame(struct fps_counter *counter) {
 }
 
 void
-fps_counter_add_skipped_frames(struct fps_counter *counter, unsigned n) {
+fps_counter_add_skipped_frame(struct fps_counter *counter) {
     if (!is_started(counter)) {
         return;
     }
@@ -176,6 +176,6 @@ fps_counter_add_skipped_frames(struct fps_counter *counter, unsigned n) {
     sc_mutex_lock(&counter->mutex);
     uint32_t now = SDL_GetTicks();
     check_interval_expired(counter, now);
-    counter->nr_skipped += n;
+    ++counter->nr_skipped;
     sc_mutex_unlock(&counter->mutex);
 }

app/src/fps_counter.h

@@ -54,6 +54,6 @@ void
 fps_counter_add_rendered_frame(struct fps_counter *counter);
 
 void
-fps_counter_add_skipped_frames(struct fps_counter *counter, unsigned n);
+fps_counter_add_skipped_frame(struct fps_counter *counter);
 
 #endif

app/src/resizer.c

@@ -0,0 +1,207 @@
+#include "resizer.h"
+
+#include <assert.h>
+#include <libswscale/swscale.h>
+
+#include "util/log.h"
+
+bool
+sc_resizer_init(struct sc_resizer *resizer, struct video_buffer *vb_in,
+                enum sc_scale_filter scale_filter, struct size size) {
+    bool ok = sc_mutex_init(&resizer->mutex);
+    if (!ok) {
+        return false;
+    }
+
+    ok = sc_cond_init(&resizer->req_cond);
+    if (!ok) {
+        sc_mutex_destroy(&resizer->mutex);
+        return false;
+    }
+
+    ok = video_buffer_init(&resizer->vb_out, false); // FIXME wait_consumer
+    if (!ok) {
+        sc_cond_destroy(&resizer->req_cond);
+        sc_mutex_destroy(&resizer->mutex);
+        return false;
+    }
+
+    resizer->resized_frame = av_frame_alloc();
+    if (!resizer->resized_frame) {
+        video_buffer_destroy(&resizer->vb_out);
+        sc_cond_destroy(&resizer->req_cond);
+        sc_mutex_destroy(&resizer->mutex);
+        return false;
+    }
+
+    resizer->vb_in = vb_in;
+    resizer->scale_filter = scale_filter;
+    resizer->size = size;
+
+    resizer->input_frame = NULL;
+    resizer->has_request = false;
+    resizer->has_new_frame = false;
+    resizer->interrupted = false;
+
+    return true;
+}
+
+void
+sc_resizer_destroy(struct sc_resizer *resizer) {
+    av_frame_free(&resizer->resized_frame);
+    sc_cond_destroy(&resizer->req_cond);
+    sc_mutex_destroy(&resizer->mutex);
+}
+
+static int
+to_sws_filter(enum sc_scale_filter scale_filter) {
+    switch (scale_filter) {
+        case SC_SCALE_FILTER_BILINEAR:  return SWS_BILINEAR;
+        case SC_SCALE_FILTER_BICUBIC:   return SWS_BICUBIC;
+        case SC_SCALE_FILTER_X:         return SWS_X;
+        case SC_SCALE_FILTER_POINT:     return SWS_POINT;
+        case SC_SCALE_FILTER_AREA:      return SWS_AREA;
+        case SC_SCALE_FILTER_BICUBLIN:  return SWS_BICUBLIN;
+        case SC_SCALE_FILTER_GAUSS:     return SWS_GAUSS;
+        case SC_SCALE_FILTER_SINC:      return SWS_SINC;
+        case SC_SCALE_FILTER_LANCZOS:   return SWS_LANCZOS;
+        case SC_SCALE_FILTER_SPLINE:    return SWS_SPLINE;
+        default: assert(!"unsupported filter");
+    }
+}
+
+static bool
+sc_resizer_swscale(struct sc_resizer *resizer) {
+    assert(!resizer->resized_frame->buf[0]); // The frame must be "empty"
+    assert(resizer->size.width);
+    assert(resizer->size.height);
+
+    const AVFrame *in = resizer->input_frame;
+    struct size size = resizer->size;
+
+    AVFrame *out = resizer->resized_frame;
+    out->format = AV_PIX_FMT_RGB24;
+    out->width = size.width;
+    out->height = size.height;
+
+    int ret = av_frame_get_buffer(out, 32);
+    if (ret < 0) {
+        return false;
+    }
+
+    int flags = to_sws_filter(resizer->scale_filter);
+    struct SwsContext *ctx =
+        sws_getContext(in->width, in->height, AV_PIX_FMT_YUV420P,
+                       size.width, size.height, AV_PIX_FMT_RGB24, flags, NULL,
+                       NULL, NULL);
+    if (!ctx) {
+        av_frame_unref(out);
+        return false;
+    }
+
+    sws_scale(ctx, (const uint8_t *const *) in->data, in->linesize, 0,
+              in->height, out->data, out->linesize);
+    sws_freeContext(ctx);
+
+    return true;
+}
+
+static int
+run_resizer(void *data) {
+    struct sc_resizer *resizer = data;
+
+    sc_mutex_lock(&resizer->mutex);
+    while (!resizer->interrupted) {
+        while (!resizer->interrupted && !resizer->has_request) {
+            sc_cond_wait(&resizer->req_cond, &resizer->mutex);
+        }
+
+        if (resizer->has_new_frame) {
+            resizer->input_frame =
+                video_buffer_consumer_take_frame(resizer->vb_in);
+
+            resizer->has_new_frame = false;
+        }
+
+        resizer->has_request = false;
+        sc_mutex_unlock(&resizer->mutex);
+
+        // Do the actual work without mutex
+        sc_resizer_swscale(resizer);
+
+        sc_mutex_lock(&resizer->mutex);
+
+        // Update the original size of the resized frame
+        resizer->original_size.width = resizer->input_frame->width;
+        resizer->original_size.height = resizer->input_frame->height;
+        assert(resizer->original_size.width);
+        assert(resizer->original_size.height);
+
+        video_buffer_producer_offer_frame(&resizer->vb_out,
+                                          &resizer->resized_frame);
+    }
+    sc_mutex_unlock(&resizer->mutex);
+
+    return 0;
+}
+
+bool
+sc_resizer_start(struct sc_resizer *resizer) {
+    LOGD("Starting resizer thread");
+
+    bool ok = sc_thread_create(&resizer->thread, run_resizer, "resizer",
+                               resizer);
+    if (!ok) {
+        LOGE("Could not start resizer thread");
+        return false;
+    }
+
+    return true;
+}
+
+void
+sc_resizer_stop(struct sc_resizer *resizer) {
+    sc_mutex_lock(&resizer->mutex);
+    resizer->interrupted = true;
+    sc_cond_signal(&resizer->req_cond);
+    sc_mutex_unlock(&resizer->mutex);
+
+    video_buffer_interrupt(&resizer->vb_out);
+}
+
+void
+sc_resizer_join(struct sc_resizer *resizer) {
+    sc_thread_join(&resizer->thread, NULL);
+}
+
+const AVFrame *
+sc_resizer_consumer_take_frame(struct sc_resizer *resizer,
+                               struct size *out_original_size) {
+    // Locking the mutex is necessary to ensure that the size corresponds to the correct frame
+    sc_mutex_lock(&resizer->mutex);
+
+    const AVFrame *frame = video_buffer_consumer_take_frame(&resizer->vb_out);
+    *out_original_size = resizer->original_size;
+
+    sc_mutex_unlock(&resizer->mutex);
+
+    return frame;
+}
+
+void
+sc_resizer_process_new_frame(struct sc_resizer *resizer) {
+    sc_mutex_lock(&resizer->mutex);
+    resizer->has_request = true;
+    resizer->has_new_frame = true;
+    sc_cond_signal(&resizer->req_cond);
+    sc_mutex_unlock(&resizer->mutex);
+}
+
+void
+sc_resizer_process_new_size(struct sc_resizer *resizer, struct size size) {
+    sc_mutex_lock(&resizer->mutex);
+    resizer->size = size;
+    resizer->has_request = true;
+    sc_cond_signal(&resizer->req_cond);
+    sc_mutex_unlock(&resizer->mutex);
+}

app/src/resizer.h

@@ -0,0 +1,62 @@
+#ifndef SC_RESIZER
+#define SC_RESIZER
+
+#include "common.h"
+
+#include <stdbool.h>
+#include <libavformat/avformat.h>
+
+#include "coords.h"
+#include "scrcpy.h"
+#include "util/thread.h"
+#include "video_buffer.h"
+
+struct sc_resizer {
+    struct video_buffer *vb_in;
+    struct video_buffer vb_out;
+    enum sc_scale_filter scale_filter;
+    struct size size;
+
+    // valid until the next call to video_buffer_consumer_take_frame(vb_in)
+    const AVFrame *input_frame;
+    AVFrame *resized_frame;
+
+    // original size of the available (resized) frame in vb_out
+    struct size original_size;
+
+    sc_thread thread;
+    sc_mutex mutex;
+    sc_cond req_cond;
+
+    bool has_request;
+    bool has_new_frame;
+    bool interrupted;
+};
+
+bool
+sc_resizer_init(struct sc_resizer *resizer, struct video_buffer *vb_in,
+                enum sc_scale_filter scale_filter, struct size size);
+
+void
+sc_resizer_destroy(struct sc_resizer *resizer);
+
+bool
+sc_resizer_start(struct sc_resizer *resizer);
+
+void
+sc_resizer_stop(struct sc_resizer *resizer);
+
+void
+sc_resizer_join(struct sc_resizer *resizer);
+
+const AVFrame *
+sc_resizer_consumer_take_frame(struct sc_resizer *resizer,
+                               struct size *out_original_size);
+
+void
+sc_resizer_process_new_frame(struct sc_resizer *resizer);
+
+void
+sc_resizer_process_new_size(struct sc_resizer *resizer, struct size size);
+
+#endif

app/src/scrcpy.c

@@ -267,14 +267,6 @@ av_log_callback(void *avcl, int level, const char *fmt, va_list vl) {
     free(local_fmt);
 }
 
-static void
-video_buffer_on_frame_available(struct video_buffer *vb, void *userdata) {
-    (void) vb;
-    (void) userdata;
-
-    screen_on_frame_available(&screen);
-}
-
 bool
 scrcpy(const struct scrcpy_options *options) {
     if (!server_init(&server)) {
@@ -341,12 +333,7 @@ scrcpy(const struct scrcpy_options *options) {
         }
         fps_counter_initialized = true;
 
-        static const struct video_buffer_callbacks video_buffer_cbs = {
-            .on_frame_available = video_buffer_on_frame_available,
-        };
-
-        if (!video_buffer_init(&video_buffer, options->render_expired_frames,
-                               &video_buffer_cbs, NULL)) {
+        if (!video_buffer_init(&video_buffer, options->render_expired_frames)) {
             goto end;
         }
         video_buffer_initialized = true;
@@ -402,7 +389,7 @@ scrcpy(const struct scrcpy_options *options) {
                                    options->window_y, options->window_width,
                                    options->window_height,
                                    options->window_borderless,
-                                   options->rotation, options->mipmaps)) {
+                                   options->rotation, options->scale_filter)) {
             goto end;
         }
 

app/src/scrcpy.h

@@ -41,6 +41,22 @@ struct sc_port_range {
     uint16_t last;
 };
 
+enum sc_scale_filter {
+    SC_SCALE_FILTER_NONE,
+    SC_SCALE_FILTER_TRILINEAR, // mipmaps
+    SC_SCALE_FILTER_BILINEAR,
+    SC_SCALE_FILTER_BICUBIC,
+    SC_SCALE_FILTER_X,
+    SC_SCALE_FILTER_POINT,
+    SC_SCALE_FILTER_AREA,
+    SC_SCALE_FILTER_BICUBLIN,
+    SC_SCALE_FILTER_GAUSS,
+    SC_SCALE_FILTER_SINC,
+    SC_SCALE_FILTER_LANCZOS,
+    SC_SCALE_FILTER_SPLINE,
+    SC_SCALE_FILTER__COUNT,
+};
+
 #define SC_WINDOW_POSITION_UNDEFINED (-0x8000)
 
 struct scrcpy_options {
@@ -56,6 +72,7 @@ struct scrcpy_options {
     enum sc_record_format record_format;
     struct sc_port_range port_range;
     struct sc_shortcut_mods shortcut_mods;
+    enum sc_scale_filter scale_filter;
     uint16_t max_size;
     uint32_t bit_rate;
     uint16_t max_fps;
@@ -75,7 +92,6 @@ struct scrcpy_options {
     bool render_expired_frames;
     bool prefer_text;
     bool window_borderless;
-    bool mipmaps;
     bool stay_awake;
     bool force_adb_forward;
     bool disable_screensaver;
@@ -103,10 +119,11 @@ struct scrcpy_options {
         .data = {SC_MOD_LALT, SC_MOD_LSUPER}, \
         .count = 2, \
     }, \
-    .max_size = DEFAULT_MAX_SIZE, \
+    .scale_filter = SC_SCALE_FILTER_TRILINEAR, \
+    .max_size = 0, \
     .bit_rate = DEFAULT_BIT_RATE, \
     .max_fps = 0, \
-    .lock_video_orientation = DEFAULT_LOCK_VIDEO_ORIENTATION, \
+    .lock_video_orientation = -1, \
     .rotation = 0, \
     .window_x = SC_WINDOW_POSITION_UNDEFINED, \
     .window_y = SC_WINDOW_POSITION_UNDEFINED, \
@@ -122,7 +139,6 @@ struct scrcpy_options {
     .render_expired_frames = false, \
     .prefer_text = false, \
     .window_borderless = false, \
-    .mipmaps = true, \
     .stay_awake = false, \
     .force_adb_forward = false, \
     .disable_screensaver = false, \

app/src/screen.c

@@ -6,7 +6,6 @@
 
 #include "events.h"
 #include "icon.xpm"
-#include "scrcpy.h"
 #include "tiny_xpm.h"
 #include "video_buffer.h"
 #include "util/log.h"
@@ -191,26 +190,85 @@ screen_update_content_rect(struct screen *screen) {
     }
 }
 
+static void
+on_frame_available(struct video_buffer *vb, void *userdata) {
+    (void) vb;
+
+    struct screen *screen = userdata;
+
+    if (screen->use_swscale) {
+        sc_resizer_process_new_frame(&screen->resizer);
+    } else {
+        static SDL_Event new_frame_event = {
+            .type = EVENT_NEW_FRAME,
+        };
+
+        // Post the event on the UI thread
+        SDL_PushEvent(&new_frame_event);
+    }
+}
+
+static void
+on_frame_skipped(struct video_buffer *vb, void *userdata) {
+    (void) vb;
+
+    struct screen *screen = userdata;
+    fps_counter_add_skipped_frame(screen->fps_counter);
+}
+
+static void
+resizer_on_frame_available(struct video_buffer *vb, void *userdata) {
+    (void) vb;
+    (void) userdata;
+
+    static SDL_Event new_frame_event = {
+        .type = EVENT_NEW_FRAME,
+    };
+
+    // Post the event on the UI thread
+    SDL_PushEvent(&new_frame_event);
+}
+
+static void
+resizer_on_frame_skipped(struct video_buffer *vb, void *userdata) {
+    // Count skipped frames from decoder or resizer the same way
+    on_frame_skipped(vb, userdata);
+}
+
 void
 screen_init(struct screen *screen, struct video_buffer *vb,
             struct fps_counter *fps_counter) {
-    *screen = (struct screen) SCREEN_INITIALIZER;
     screen->vb = vb;
     screen->fps_counter = fps_counter;
+
+    screen->resize_pending = false;
+    screen->has_frame = false;
+    screen->fullscreen = false;
+    screen->maximized = false;
+
+    static const struct video_buffer_callbacks cbs = {
+        .on_frame_available = on_frame_available,
+        .on_frame_skipped = on_frame_skipped,
+    };
+
+    video_buffer_set_consumer_callbacks(vb, &cbs, screen);
 }
 
 static inline SDL_Texture *
 create_texture(struct screen *screen) {
     SDL_Renderer *renderer = screen->renderer;
     struct size size = screen->frame_size;
-    SDL_Texture *texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12,
+    SDL_PixelFormatEnum fmt = screen->use_swscale ? SDL_PIXELFORMAT_RGB24
+                                                  : SDL_PIXELFORMAT_YV12;
+    SDL_Texture *texture = SDL_CreateTexture(renderer, fmt,
                                              SDL_TEXTUREACCESS_STREAMING,
                                              size.width, size.height);
     if (!texture) {
         return NULL;
     }
 
-    if (screen->mipmaps) {
+    if (screen->scale_filter == SC_SCALE_FILTER_TRILINEAR) {
+        assert(!screen->use_swscale);
         struct sc_opengl *gl = &screen->gl;
 
         SDL_GL_BindTexture(texture, NULL, NULL);
@@ -226,12 +284,18 @@ create_texture(struct screen *screen) {
     return texture;
 }
 
+static inline bool
+is_swscale(enum sc_scale_filter scale_filter) {
+    return scale_filter != SC_SCALE_FILTER_NONE
+        && scale_filter != SC_SCALE_FILTER_TRILINEAR;
+}
+
 bool
 screen_init_rendering(struct screen *screen, const char *window_title,
                       struct size frame_size, bool always_on_top,
                       int16_t window_x, int16_t window_y, uint16_t window_width,
                       uint16_t window_height, bool window_borderless,
-                      uint8_t rotation, bool mipmaps) {
+                      uint8_t rotation, enum sc_scale_filter scale_filter) {
     screen->frame_size = frame_size;
     screen->rotation = rotation;
     if (rotation) {
@@ -285,6 +349,7 @@ screen_init_rendering(struct screen *screen, const char *window_title,
 
     // starts with "opengl"
     bool use_opengl = renderer_name && !strncmp(renderer_name, "opengl", 6);
+    bool mipmaps = scale_filter == SC_SCALE_FILTER_TRILINEAR;
     if (use_opengl) {
         struct sc_opengl *gl = &screen->gl;
         sc_opengl_init(gl);
@@ -297,10 +362,10 @@ screen_init_rendering(struct screen *screen, const char *window_title,
                                                2, 0  /* OpenGL ES 2.0+ */);
             if (supports_mipmaps) {
                 LOGI("Trilinear filtering enabled");
-                screen->mipmaps = true;
             } else {
                 LOGW("Trilinear filtering disabled "
                      "(OpenGL 3.0+ or ES 2.0+ required)");
+                scale_filter = SC_SCALE_FILTER_NONE;
             }
         } else {
             LOGI("Trilinear filtering disabled");
@@ -309,6 +374,46 @@ screen_init_rendering(struct screen *screen, const char *window_title,
         LOGD("Trilinear filtering disabled (not an OpenGL renderer)");
     }
 
+    screen->use_swscale = is_swscale(scale_filter);
+    if (screen->use_swscale) {
+        bool ok = video_buffer_init(&screen->resizer_vb, false);
+        if (!ok) {
+            LOGE("Could not create resizer video buffer");
+            SDL_DestroyRenderer(screen->renderer);
+            SDL_DestroyWindow(screen->window);
+            return false;
+        }
+
+        ok = sc_resizer_init(&screen->resizer, screen->vb, scale_filter,
+                             window_size);
+        if (!ok) {
+            LOGE("Could not create resizer");
+            video_buffer_destroy(&screen->resizer_vb);
+            SDL_DestroyRenderer(screen->renderer);
+            SDL_DestroyWindow(screen->window);
+            return false;
+        }
+
+        static const struct video_buffer_callbacks cbs = {
+            .on_frame_available = resizer_on_frame_available,
+            .on_frame_skipped = resizer_on_frame_skipped,
+        };
+
+        video_buffer_set_consumer_callbacks(&screen->resizer.vb_out, &cbs,
+                                            screen);
+
+        ok = sc_resizer_start(&screen->resizer);
+        if (!ok) {
+            LOGE("Could not start resizer");
+            sc_resizer_destroy(&screen->resizer);
+            video_buffer_destroy(&screen->resizer_vb);
+            SDL_DestroyRenderer(screen->renderer);
+            SDL_DestroyWindow(screen->window);
+        }
+    }
+
+    screen->scale_filter = scale_filter;
+
     SDL_Surface *icon = read_xpm(icon_xpm);
     if (icon) {
         SDL_SetWindowIcon(screen->window, icon);
@@ -324,7 +429,6 @@ screen_init_rendering(struct screen *screen, const char *window_title,
         LOGC("Could not create texture: %s", SDL_GetError());
         SDL_DestroyRenderer(screen->renderer);
         SDL_DestroyWindow(screen->window);
-        screen_destroy(screen);
         return false;
     }
 
@@ -345,6 +449,11 @@ screen_show_window(struct screen *screen) {
 
 void
 screen_destroy(struct screen *screen) {
+    if (screen->use_swscale) {
+        sc_resizer_stop(&screen->resizer);
+        sc_resizer_join(&screen->resizer);
+        sc_resizer_destroy(&screen->resizer);
+    }
     if (screen->texture) {
         SDL_DestroyTexture(screen->texture);
     }
@@ -411,16 +520,17 @@ screen_set_rotation(struct screen *screen, unsigned rotation) {
 
 // recreate the texture and resize the window if the frame size has changed
 static bool
-prepare_for_frame(struct screen *screen, struct size new_frame_size) {
-    if (screen->frame_size.width != new_frame_size.width
-            || screen->frame_size.height != new_frame_size.height) {
+prepare_for_frame(struct screen *screen, struct size original_frame_size,
+                  struct size frame_size) {
+    if (screen->frame_size.width != original_frame_size.width
+            || screen->frame_size.height != original_frame_size.height) {
         // frame dimension changed, destroy texture
         SDL_DestroyTexture(screen->texture);
 
-        screen->frame_size = new_frame_size;
+        screen->frame_size = original_frame_size;
 
         struct size new_content_size =
-            get_rotated_size(new_frame_size, screen->rotation);
+            get_rotated_size(original_frame_size, screen->rotation);
         set_content_size(screen, new_content_size);
 
         screen_update_content_rect(screen);
@@ -440,29 +550,41 @@ prepare_for_frame(struct screen *screen, struct size new_frame_size) {
 // write the frame into the texture
 static void
 update_texture(struct screen *screen, const AVFrame *frame) {
-    SDL_UpdateYUVTexture(screen->texture, NULL,
-            frame->data[0], frame->linesize[0],
-            frame->data[1], frame->linesize[1],
-            frame->data[2], frame->linesize[2]);
+    if (screen->use_swscale) {
+        SDL_UpdateTexture(screen->texture, NULL, frame->data[0],
+                          frame->linesize[0]);
+    } else {
+        SDL_UpdateYUVTexture(screen->texture, NULL,
+                frame->data[0], frame->linesize[0],
+                frame->data[1], frame->linesize[1],
+                frame->data[2], frame->linesize[2]);
 
-    if (screen->mipmaps) {
-        SDL_GL_BindTexture(screen->texture, NULL, NULL);
-        screen->gl.GenerateMipmap(GL_TEXTURE_2D);
-        SDL_GL_UnbindTexture(screen->texture);
+        if (screen->scale_filter == SC_SCALE_FILTER_TRILINEAR) {
+            SDL_GL_BindTexture(screen->texture, NULL, NULL);
+            screen->gl.GenerateMipmap(GL_TEXTURE_2D);
+            SDL_GL_UnbindTexture(screen->texture);
+        }
     }
 }
 
 static bool
 screen_update_frame(struct screen *screen) {
-    unsigned skipped;
-    const AVFrame *frame =
-        video_buffer_consumer_take_frame(screen->vb, &skipped);
+    const AVFrame *frame;
+    struct size original_frame_size;
+
+    if (screen->use_swscale) {
+        frame = sc_resizer_consumer_take_frame(&screen->resizer,
+                                               &original_frame_size);
+    } else {
+        frame = video_buffer_consumer_take_frame(screen->vb);
+        original_frame_size.width = frame->width;
+        original_frame_size.height = frame->height;
+    }
+    struct size frame_size = {frame->width, frame->height};
 
-    fps_counter_add_skipped_frames(screen->fps_counter, skipped);
     fps_counter_add_rendered_frame(screen->fps_counter);
 
-    struct size new_frame_size = {frame->width, frame->height};
-    if (!prepare_for_frame(screen, new_frame_size)) {
+    if (!prepare_for_frame(screen, original_frame_size, frame_size)) {
         return false;
     }
     update_texture(screen, frame);
@@ -655,15 +777,3 @@ screen_hidpi_scale_coords(struct screen *screen, int32_t *x, int32_t *y) {
     *x = (int64_t) *x * dw / ww;
     *y = (int64_t) *y * dh / wh;
 }
-
-void
-screen_on_frame_available(struct screen *screen) {
-    (void) screen;
-
-    static SDL_Event new_frame_event = {
-        .type = EVENT_NEW_FRAME,
-    };
-
-    // Post the event on the UI thread
-    SDL_PushEvent(&new_frame_event);
-}

app/src/screen.h

@@ -9,6 +9,9 @@
 
 #include "coords.h"
 #include "opengl.h"
+#include "resizer.h"
+#include "scrcpy.h"
+#include "video_buffer.h"
 
 struct video_buffer;
 
@@ -35,43 +38,15 @@ struct screen {
     bool has_frame;
     bool fullscreen;
     bool maximized;
-    bool no_window;
     bool mipmaps;
-};
 
-#define SCREEN_INITIALIZER { \
-    .vb = NULL, \
-    .fps_counter = NULL, \
-    .window = NULL, \
-    .renderer = NULL, \
-    .texture = NULL, \
-    .gl = {0}, \
-    .frame_size = { \
-        .width = 0, \
-        .height = 0, \
-    }, \
-    .content_size = { \
-        .width = 0, \
-        .height = 0, \
-    }, \
-    .resize_pending = false, \
-    .windowed_content_size = { \
-        .width = 0, \
-        .height = 0, \
-    }, \
-    .rotation = 0, \
-    .rect = { \
-        .x = 0, \
-        .y = 0, \
-        .w = 0, \
-        .h = 0, \
-    }, \
-    .has_frame = false, \
-    .fullscreen = false, \
-    .maximized = false, \
-    .no_window = false, \
-    .mipmaps = false, \
-}
+    enum sc_scale_filter scale_filter;
+    bool use_swscale;
+
+    // For swscale resizing
+    struct video_buffer resizer_vb;
+    struct sc_resizer resizer;
+};
 
 // initialize default values
 void
@@ -85,7 +60,7 @@ screen_init_rendering(struct screen *screen, const char *window_title,
                       struct size frame_size, bool always_on_top,
                       int16_t window_x, int16_t window_y, uint16_t window_width,
                       uint16_t window_height, bool window_borderless,
-                      uint8_t rotation, bool mipmaps);
+                      uint8_t rotation, enum sc_scale_filter scale_filter);
 
 // show the window
 void
@@ -141,9 +116,4 @@ screen_convert_drawable_to_frame_coords(struct screen *screen,
 void
 screen_hidpi_scale_coords(struct screen *screen, int32_t *x, int32_t *y);
 
-// Notify the screen that a new frame is available in the video_buffer.
-// Called from a separate thread.
-void
-screen_on_frame_available(struct screen *screen);
-
 #endif

app/src/video_buffer.c

@@ -7,27 +7,20 @@
 #include "util/log.h"
 
 bool
-video_buffer_init(struct video_buffer *vb, bool wait_consumer,
-                  const struct video_buffer_callbacks *cbs,
-                  void *cbs_userdata) {
-    vb->producer_frame = av_frame_alloc();
-    if (!vb->producer_frame) {
-        goto error_0;
-    }
-
+video_buffer_init(struct video_buffer *vb, bool wait_consumer) {
     vb->pending_frame = av_frame_alloc();
     if (!vb->pending_frame) {
-        goto error_1;
+        goto error_0;
     }
 
     vb->consumer_frame = av_frame_alloc();
     if (!vb->consumer_frame) {
-        goto error_2;
+        goto error_1;
     }
 
     bool ok = sc_mutex_init(&vb->mutex);
     if (!ok) {
-        goto error_3;
+        goto error_2;
     }
 
     vb->wait_consumer = wait_consumer;
@@ -37,30 +30,24 @@ video_buffer_init(struct video_buffer *vb, bool wait_consumer,
             sc_mutex_destroy(&vb->mutex);
             goto error_2;
         }
-        // interrupted is not used if expired frames are not rendered
-        // since offering a frame will never block
+        // interrupted is not used if wait_consumer is disabled since offering
+        // a frame will never block
         vb->interrupted = false;
     }
 
-    // there is initially no rendering frame, so consider it has already been
-    // consumed
+    // there is initially no frame, so consider it has already been consumed
     vb->pending_frame_consumed = true;
 
-    vb->skipped = 0;
-
-    assert(cbs);
-    assert(cbs->on_frame_available);
-    vb->cbs = cbs;
-    vb->cbs_userdata = cbs_userdata;
+    // The callbacks must be set by the consumer via
+    // video_buffer_set_consumer_callbacks()
+    vb->cbs = NULL;
 
     return true;
 
-error_3:
-    av_frame_free(&vb->consumer_frame);
 error_2:
-    av_frame_free(&vb->pending_frame);
+    av_frame_free(&vb->consumer_frame);
 error_1:
-    av_frame_free(&vb->producer_frame);
+    av_frame_free(&vb->pending_frame);
 error_0:
     return false;
 }
@@ -73,27 +60,30 @@ video_buffer_destroy(struct video_buffer *vb) {
     sc_mutex_destroy(&vb->mutex);
     av_frame_free(&vb->consumer_frame);
     av_frame_free(&vb->pending_frame);
-    av_frame_free(&vb->producer_frame);
 }
 
-static void
-video_buffer_swap_producer_frame(struct video_buffer *vb) {
-    sc_mutex_assert(&vb->mutex);
-    AVFrame *tmp = vb->producer_frame;
-    vb->producer_frame = vb->pending_frame;
-    vb->pending_frame = tmp;
-}
-
-static void
-video_buffer_swap_consumer_frame(struct video_buffer *vb) {
-    sc_mutex_assert(&vb->mutex);
-    AVFrame *tmp = vb->consumer_frame;
-    vb->consumer_frame = vb->pending_frame;
-    vb->pending_frame = tmp;
+static inline void
+swap_frames(AVFrame **lhs, AVFrame **rhs) {
+    AVFrame *tmp = *lhs;
+    *lhs = *rhs;
+    *rhs = tmp;
 }
 
 void
-video_buffer_producer_offer_frame(struct video_buffer *vb) {
+video_buffer_set_consumer_callbacks(struct video_buffer *vb,
+                                    const struct video_buffer_callbacks *cbs,
+                                    void *cbs_userdata) {
+    assert(!vb->cbs); // must be set only once
+    assert(cbs);
+    assert(cbs->on_frame_available);
+    vb->cbs = cbs;
+    vb->cbs_userdata = cbs_userdata;
+}
+
+void
+video_buffer_producer_offer_frame(struct video_buffer *vb, AVFrame **pframe) {
+    assert(vb->cbs);
+
     sc_mutex_lock(&vb->mutex);
     if (vb->wait_consumer) {
         // wait for the current (expired) frame to be consumed
@@ -102,42 +92,35 @@ video_buffer_producer_offer_frame(struct video_buffer *vb) {
         }
     }
 
-    video_buffer_swap_producer_frame(vb);
+    av_frame_unref(vb->pending_frame);
+    swap_frames(pframe, &vb->pending_frame);
 
     bool skipped = !vb->pending_frame_consumed;
-    if (skipped) {
-        ++vb->skipped;
-    }
-
     vb->pending_frame_consumed = false;
 
     sc_mutex_unlock(&vb->mutex);
 
-    if (!skipped) {
-        // If skipped, then the previous call will consume this frame, the
-        // callback must not be called
+    if (skipped) {
+        if (vb->cbs->on_frame_skipped)
+            vb->cbs->on_frame_skipped(vb, vb->cbs_userdata);
+    } else {
         vb->cbs->on_frame_available(vb, vb->cbs_userdata);
     }
 }
 
 const AVFrame *
-video_buffer_consumer_take_frame(struct video_buffer *vb, unsigned *skipped) {
+video_buffer_consumer_take_frame(struct video_buffer *vb) {
     sc_mutex_lock(&vb->mutex);
     assert(!vb->pending_frame_consumed);
     vb->pending_frame_consumed = true;
 
-    video_buffer_swap_consumer_frame(vb);
+    swap_frames(&vb->consumer_frame, &vb->pending_frame);
+    av_frame_unref(vb->pending_frame);
 
     if (vb->wait_consumer) {
         // unblock video_buffer_offer_decoded_frame()
         sc_cond_signal(&vb->pending_frame_consumed_cond);
     }
-
-    if (skipped) {
-        *skipped = vb->skipped;
-    }
-    vb->skipped = 0; // reset
-
     sc_mutex_unlock(&vb->mutex);
 
     // consumer_frame is only written from this thread, no need to lock

app/src/video_buffer.h

@@ -29,7 +29,6 @@ typedef struct AVFrame AVFrame;
  */
 
 struct video_buffer {
-    AVFrame *producer_frame;
     AVFrame *pending_frame;
     AVFrame *consumer_frame;
 
@@ -40,8 +39,6 @@ struct video_buffer {
     sc_cond pending_frame_consumed_cond;
     bool pending_frame_consumed;
 
-    unsigned skipped;
-
     const struct video_buffer_callbacks *cbs;
     void *cbs_userdata;
 };
@@ -49,26 +46,34 @@ struct video_buffer {
 struct video_buffer_callbacks {
     // Called when a new frame can be consumed by
     // video_buffer_consumer_take_frame(vb)
+    // This callback is mandatory (it must not be NULL).
     void (*on_frame_available)(struct video_buffer *vb, void *userdata);
+
+    // Called when a pending frame has been overwritten by the producer
+    // This callback is optional (it may be NULL).
+    void (*on_frame_skipped)(struct video_buffer *vb, void *userdata);
 };
 
 bool
-video_buffer_init(struct video_buffer *vb, bool wait_consumer,
-                  const struct video_buffer_callbacks *cbs, void *cbs_userdata);
+video_buffer_init(struct video_buffer *vb, bool wait_consumer);
 
 void
 video_buffer_destroy(struct video_buffer *vb);
 
-// set the producer frame as ready for consuming
 void
-video_buffer_producer_offer_frame(struct video_buffer *vb);
+video_buffer_set_consumer_callbacks(struct video_buffer *vb,
+                                    const struct video_buffer_callbacks *cbs,
+                                    void *cbs_userdata);
+
+// set the producer frame as ready for consuming
+// the produced frame is exchanged with an unused allocated frame
+void
+video_buffer_producer_offer_frame(struct video_buffer *vb, AVFrame **pframe);
 
 // mark the consumer frame as consumed and return it
 // the frame is valid until the next call to this function
-// the output parameter "skipped" indicates how many produced frames have been
-// skipped
 const AVFrame *
-video_buffer_consumer_take_frame(struct video_buffer *vb, unsigned *skipped);
+video_buffer_consumer_take_frame(struct video_buffer *vb);
 
 // wake up and avoid any blocking call
 void