wip_everything_broken

The video buffer held 3 frames:
 - the producer frame (for decoding)
 - the pending frame (to exchange between the producer and consumer)
 - the consumer frame (for rendering)

It worked well, but it prevented video buffers to be chained, because
the consumer frame of the first video buffer must be the same as the
producer frame of the second video buffer.

To solve this problem, make the decoder handle its decoding frame, and
keep only the pending and consumer frames in the video_buffer.

    decoder -> pending -> consumer -> pending -> consumer
             |---------------------||---------------------|
                  video_buffer 1         video_buffer 2

This paves the way to support asynchronous swscale.

app/meson.build

@@ -47,6 +47,7 @@ if not get_option('crossbuild_windows')
         dependency('libavformat'),
         dependency('libavcodec'),
         dependency('libavutil'),
+        dependency('libswscale'),
         dependency('sdl2'),
     ]
 

app/src/cli.c

@@ -642,6 +642,7 @@ guess_record_format(const char *filename) {
 
 static bool
 parse_scale_filter(const char *optarg, enum sc_scale_filter *filter) {
+    // TODO store in a map and loop over the entries instead
     if (!strcmp(optarg, "none")) {
         *filter = SC_SCALE_FILTER_NONE;
         return true;
@@ -650,8 +651,49 @@ parse_scale_filter(const char *optarg, enum sc_scale_filter *filter) {
         *filter = SC_SCALE_FILTER_TRILINEAR;
         return true;
     }
+    if (!strcmp(optarg, "bilinear")) {
+        *filter = SC_SCALE_FILTER_BILINEAR;
+        return true;
+    }
+    if (!strcmp(optarg, "bicubic")) {
+        *filter = SC_SCALE_FILTER_BICUBIC;
+        return true;
+    }
+    if (!strcmp(optarg, "x")) {
+        *filter = SC_SCALE_FILTER_X;
+        return true;
+    }
+    if (!strcmp(optarg, "point")) {
+        *filter = SC_SCALE_FILTER_POINT;
+        return true;
+    }
+    if (!strcmp(optarg, "area")) {
+        *filter = SC_SCALE_FILTER_AREA;
+        return true;
+    }
+    if (!strcmp(optarg, "bicublin")) {
+        *filter = SC_SCALE_FILTER_BICUBLIN;
+        return true;
+    }
+    if (!strcmp(optarg, "gauss")) {
+        *filter = SC_SCALE_FILTER_GAUSS;
+        return true;
+    }
+    if (!strcmp(optarg, "sinc")) {
+        *filter = SC_SCALE_FILTER_SINC;
+        return true;
+    }
+    if (!strcmp(optarg, "lanczos")) {
+        *filter = SC_SCALE_FILTER_LANCZOS;
+        return true;
+    }
+    if (!strcmp(optarg, "spline")) {
+        *filter = SC_SCALE_FILTER_SPLINE;
+        return true;
+    }
+
     LOGE("Unsupported scale filter: %s "
-         "(expected \"none\" or \"trilinear\")", optarg);
+         "(expected one of [TODO])", optarg);
     return false;
 }
 

app/src/decoder.c

@@ -32,7 +32,6 @@ decoder_open(struct decoder *decoder, const AVCodec *codec) {
 
     decoder->frame = av_frame_alloc();
     if (!decoder->frame) {
-        LOGE("Could not create decoder frame");
         avcodec_close(decoder->codec_ctx);
         avcodec_free_context(&decoder->codec_ctx);
         return false;

app/src/resizer.c

@@ -1,12 +1,13 @@
 #include "resizer.h"
 
 #include <assert.h>
+#include <libswscale/swscale.h>
 
 #include "util/log.h"
 
 bool
 sc_resizer_init(struct sc_resizer *resizer, struct video_buffer *vb_in,
-                struct video_buffer *vb_out, struct size size) {
+                enum sc_scale_filter scale_filter, struct size size) {
     bool ok = sc_mutex_init(&resizer->mutex);
     if (!ok) {
         return false;
@@ -18,15 +19,23 @@ sc_resizer_init(struct sc_resizer *resizer, struct video_buffer *vb_in,
         return false;
     }
 
+    ok = video_buffer_init(&resizer->vb_out, false); // FIXME wait_consumer
+    if (!ok) {
+        sc_cond_destroy(&resizer->req_cond);
+        sc_mutex_destroy(&resizer->mutex);
+        return false;
+    }
+
     resizer->resized_frame = av_frame_alloc();
     if (!resizer->resized_frame) {
+        video_buffer_destroy(&resizer->vb_out);
         sc_cond_destroy(&resizer->req_cond);
         sc_mutex_destroy(&resizer->mutex);
         return false;
     }
 
     resizer->vb_in = vb_in;
-    resizer->vb_out = vb_out;
+    resizer->scale_filter = scale_filter;
     resizer->size = size;
 
     resizer->input_frame = NULL;
@@ -44,11 +53,57 @@ sc_resizer_destroy(struct sc_resizer *resizer) {
     sc_mutex_destroy(&resizer->mutex);
 }
 
+static int
+to_sws_filter(enum sc_scale_filter scale_filter) {
+    switch (scale_filter) {
+        case SC_SCALE_FILTER_BILINEAR:  return SWS_BILINEAR;
+        case SC_SCALE_FILTER_BICUBIC:   return SWS_BICUBIC;
+        case SC_SCALE_FILTER_X:         return SWS_X;
+        case SC_SCALE_FILTER_POINT:     return SWS_POINT;
+        case SC_SCALE_FILTER_AREA:      return SWS_AREA;
+        case SC_SCALE_FILTER_BICUBLIN:  return SWS_BICUBLIN;
+        case SC_SCALE_FILTER_GAUSS:     return SWS_GAUSS;
+        case SC_SCALE_FILTER_SINC:      return SWS_SINC;
+        case SC_SCALE_FILTER_LANCZOS:   return SWS_LANCZOS;
+        case SC_SCALE_FILTER_SPLINE:    return SWS_SPLINE;
+        default: assert(!"unsupported filter");
+    }
+}
+
 static bool
 sc_resizer_swscale(struct sc_resizer *resizer) {
     assert(!resizer->resized_frame->buf[0]); // The frame must be "empty"
+    assert(resizer->size.width);
+    assert(resizer->size.height);
 
-    return false;
+    const AVFrame *in = resizer->input_frame;
+    struct size size = resizer->size;
+
+    AVFrame *out = resizer->resized_frame;
+    out->format = AV_PIX_FMT_RGB24;
+    out->width = size.width;
+    out->height = size.height;
+
+    int ret = av_frame_get_buffer(out, 32);
+    if (ret < 0) {
+        return false;
+    }
+
+    int flags = to_sws_filter(resizer->scale_filter);
+    struct SwsContext *ctx =
+        sws_getContext(in->width, in->height, AV_PIX_FMT_YUV420P,
+                       size.width, size.height, AV_PIX_FMT_RGB24, flags, NULL,
+                       NULL, NULL);
+    if (!ctx) {
+        av_frame_unref(out);
+        return false;
+    }
+
+    sws_scale(ctx, (const uint8_t *const *) in->data, in->linesize, 0,
+              in->height, out->data, out->linesize);
+    sws_freeContext(ctx);
+
+    return true;
 }
 
 static int
@@ -74,10 +129,16 @@ run_resizer(void *data) {
         // Do the actual work without mutex
         sc_resizer_swscale(resizer);
 
-        video_buffer_producer_offer_frame(resizer->vb_out,
-                                          &resizer->resized_frame);
-
         sc_mutex_lock(&resizer->mutex);
+
+        // Update the original size of the resized frame
+        resizer->original_size.width = resizer->input_frame->width;
+        resizer->original_size.height = resizer->input_frame->height;
+        assert(resizer->original_size.width);
+        assert(resizer->original_size.height);
+
+        video_buffer_producer_offer_frame(&resizer->vb_out,
+                                          &resizer->resized_frame);
     }
     sc_mutex_unlock(&resizer->mutex);
 
@@ -105,7 +166,7 @@ sc_resizer_stop(struct sc_resizer *resizer) {
     sc_cond_signal(&resizer->req_cond);
     sc_mutex_unlock(&resizer->mutex);
 
-    video_buffer_interrupt(resizer->vb_out);
+    video_buffer_interrupt(&resizer->vb_out);
 }
 
 void
@@ -113,6 +174,20 @@ sc_resizer_join(struct sc_resizer *resizer) {
     sc_thread_join(&resizer->thread, NULL);
 }
 
+const AVFrame *
+sc_resizer_consumer_take_frame(struct sc_resizer *resizer,
+                               struct size *out_original_size) {
+    // Locking the mutex is necessary to ensure that the size corresponds to the correct frame
+    sc_mutex_lock(&resizer->mutex);
+
+    const AVFrame *frame = video_buffer_consumer_take_frame(&resizer->vb_out);
+    *out_original_size = resizer->original_size;
+
+    sc_mutex_unlock(&resizer->mutex);
+
+    return frame;
+}
+
 void
 sc_resizer_process_new_frame(struct sc_resizer *resizer) {
     sc_mutex_lock(&resizer->mutex);

app/src/resizer.h

@@ -7,18 +7,23 @@
 #include <libavformat/avformat.h>
 
 #include "coords.h"
+#include "scrcpy.h"
 #include "util/thread.h"
 #include "video_buffer.h"
 
 struct sc_resizer {
     struct video_buffer *vb_in;
-    struct video_buffer *vb_out;
+    struct video_buffer vb_out;
+    enum sc_scale_filter scale_filter;
     struct size size;
 
     // valid until the next call to video_buffer_consumer_take_frame(vb_in)
     const AVFrame *input_frame;
     AVFrame *resized_frame;
 
+    // original size of the available (resized) frame in vb_out
+    struct size original_size;
+
     sc_thread thread;
     sc_mutex mutex;
     sc_cond req_cond;
@@ -30,7 +35,7 @@ struct sc_resizer {
 
 bool
 sc_resizer_init(struct sc_resizer *resizer, struct video_buffer *vb_in,
-                struct video_buffer *vb_out, struct size initial_size);
+                enum sc_scale_filter scale_filter, struct size size);
 
 void
 sc_resizer_destroy(struct sc_resizer *resizer);
@@ -44,6 +49,10 @@ sc_resizer_stop(struct sc_resizer *resizer);
 void
 sc_resizer_join(struct sc_resizer *resizer);
 
+const AVFrame *
+sc_resizer_consumer_take_frame(struct sc_resizer *resizer,
+                               struct size *out_original_size);
+
 void
 sc_resizer_process_new_frame(struct sc_resizer *resizer);
 

app/src/scrcpy.c

@@ -382,21 +382,14 @@ scrcpy(const struct scrcpy_options *options) {
         const char *window_title =
             options->window_title ? options->window_title : device_name;
 
-        struct screen_params screen_params = {
-            .window_title = window_title,
-            .frame_size = frame_size,
-            .always_on_top = options->always_on_top,
-            .window_x = options->window_x,
-            .window_y = options->window_y,
-            .window_width = options->window_width,
-            .window_height = options->window_height,
-            .window_borderless = options->window_borderless,
-            .rotation = options->rotation,
-            .scale_filter = options->scale_filter,
-        };
+        screen_init(&screen, &video_buffer, &fps_counter);
 
-        if (!screen_init(&screen, &video_buffer, &fps_counter,
-                         &screen_params)) {
+        if (!screen_init_rendering(&screen, window_title, frame_size,
+                                   options->always_on_top, options->window_x,
+                                   options->window_y, options->window_width,
+                                   options->window_height,
+                                   options->window_borderless,
+                                   options->rotation, options->scale_filter)) {
             goto end;
         }
 

app/src/scrcpy.h

@@ -44,6 +44,17 @@ struct sc_port_range {
 enum sc_scale_filter {
     SC_SCALE_FILTER_NONE,
     SC_SCALE_FILTER_TRILINEAR, // mipmaps
+    SC_SCALE_FILTER_BILINEAR,
+    SC_SCALE_FILTER_BICUBIC,
+    SC_SCALE_FILTER_X,
+    SC_SCALE_FILTER_POINT,
+    SC_SCALE_FILTER_AREA,
+    SC_SCALE_FILTER_BICUBLIN,
+    SC_SCALE_FILTER_GAUSS,
+    SC_SCALE_FILTER_SINC,
+    SC_SCALE_FILTER_LANCZOS,
+    SC_SCALE_FILTER_SPLINE,
+    SC_SCALE_FILTER__COUNT,
 };
 
 #define SC_WINDOW_POSITION_UNDEFINED (-0x8000)

app/src/screen.c

@@ -193,6 +193,32 @@ screen_update_content_rect(struct screen *screen) {
 static void
 on_frame_available(struct video_buffer *vb, void *userdata) {
     (void) vb;
+
+    struct screen *screen = userdata;
+
+    if (screen->use_swscale) {
+        sc_resizer_process_new_frame(&screen->resizer);
+    } else {
+        static SDL_Event new_frame_event = {
+            .type = EVENT_NEW_FRAME,
+        };
+
+        // Post the event on the UI thread
+        SDL_PushEvent(&new_frame_event);
+    }
+}
+
+static void
+on_frame_skipped(struct video_buffer *vb, void *userdata) {
+    (void) vb;
+
+    struct screen *screen = userdata;
+    fps_counter_add_skipped_frame(screen->fps_counter);
+}
+
+static void
+resizer_on_frame_available(struct video_buffer *vb, void *userdata) {
+    (void) vb;
     (void) userdata;
 
     static SDL_Event new_frame_event = {
@@ -204,25 +230,45 @@ on_frame_available(struct video_buffer *vb, void *userdata) {
 }
 
 static void
-on_frame_skipped(struct video_buffer *vb, void *userdata) {
-    (void) vb;
+resizer_on_frame_skipped(struct video_buffer *vb, void *userdata) {
+    // Count skipped frames from decoder or resizer the same way
+    on_frame_skipped(vb, userdata);
+}
 
-    struct screen *screen = userdata;
-    fps_counter_add_skipped_frame(screen->fps_counter);
+void
+screen_init(struct screen *screen, struct video_buffer *vb,
+            struct fps_counter *fps_counter) {
+    screen->vb = vb;
+    screen->fps_counter = fps_counter;
+
+    screen->resize_pending = false;
+    screen->has_frame = false;
+    screen->fullscreen = false;
+    screen->maximized = false;
+
+    static const struct video_buffer_callbacks cbs = {
+        .on_frame_available = on_frame_available,
+        .on_frame_skipped = on_frame_skipped,
+    };
+
+    video_buffer_set_consumer_callbacks(vb, &cbs, screen);
 }
 
 static inline SDL_Texture *
 create_texture(struct screen *screen) {
     SDL_Renderer *renderer = screen->renderer;
     struct size size = screen->frame_size;
-    SDL_Texture *texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12,
+    SDL_PixelFormatEnum fmt = screen->use_swscale ? SDL_PIXELFORMAT_RGB24
+                                                  : SDL_PIXELFORMAT_YV12;
+    SDL_Texture *texture = SDL_CreateTexture(renderer, fmt,
                                              SDL_TEXTUREACCESS_STREAMING,
                                              size.width, size.height);
     if (!texture) {
         return NULL;
     }
 
-    if (screen->mipmaps) {
+    if (screen->scale_filter == SC_SCALE_FILTER_TRILINEAR) {
+        assert(!screen->use_swscale);
         struct sc_opengl *gl = &screen->gl;
 
         SDL_GL_BindTexture(texture, NULL, NULL);
@@ -238,41 +284,33 @@ create_texture(struct screen *screen) {
     return texture;
 }
 
+static inline bool
+is_swscale(enum sc_scale_filter scale_filter) {
+    return scale_filter != SC_SCALE_FILTER_NONE
+        && scale_filter != SC_SCALE_FILTER_TRILINEAR;
+}
+
 bool
-screen_init(struct screen *screen, struct video_buffer *vb,
-            struct fps_counter *fps_counter,
-            const struct screen_params *params) {
-    screen->vb = vb;
-    screen->fps_counter = fps_counter;
-
-    screen->resize_pending = false;
-    screen->has_frame = false;
-    screen->fullscreen = false;
-    screen->maximized = false;
-
-    static const struct video_buffer_callbacks cbs = {
-        .on_frame_available = on_frame_available,
-        .on_frame_skipped = on_frame_skipped,
-    };
-    video_buffer_set_consumer_callbacks(vb, &cbs, screen);
-
-    screen->frame_size = params->frame_size;
-    screen->rotation = params->rotation;
-    if (screen->rotation) {
-        LOGI("Initial display rotation set to %u", screen->rotation);
+screen_init_rendering(struct screen *screen, const char *window_title,
+                      struct size frame_size, bool always_on_top,
+                      int16_t window_x, int16_t window_y, uint16_t window_width,
+                      uint16_t window_height, bool window_borderless,
+                      uint8_t rotation, enum sc_scale_filter scale_filter) {
+    screen->frame_size = frame_size;
+    screen->rotation = rotation;
+    if (rotation) {
+        LOGI("Initial display rotation set to %u", rotation);
     }
-    struct size content_size =
-        get_rotated_size(screen->frame_size, screen->rotation);
+    struct size content_size = get_rotated_size(frame_size, screen->rotation);
     screen->content_size = content_size;
 
-    struct size window_size = get_initial_optimal_size(content_size,
-                                                       params->window_width,
-                                                       params->window_height);
+    struct size window_size =
+        get_initial_optimal_size(content_size, window_width, window_height);
     uint32_t window_flags = SDL_WINDOW_HIDDEN | SDL_WINDOW_RESIZABLE;
 #ifdef HIDPI_SUPPORT
     window_flags |= SDL_WINDOW_ALLOW_HIGHDPI;
 #endif
-    if (params->always_on_top) {
+    if (always_on_top) {
 #ifdef SCRCPY_SDL_HAS_WINDOW_ALWAYS_ON_TOP
         window_flags |= SDL_WINDOW_ALWAYS_ON_TOP;
 #else
@@ -280,15 +318,15 @@ screen_init(struct screen *screen, struct video_buffer *vb,
              "(compile with SDL >= 2.0.5 to enable it)");
 #endif
     }
-    if (params->window_borderless) {
+    if (window_borderless) {
         window_flags |= SDL_WINDOW_BORDERLESS;
     }
 
-    int x = params->window_x != SC_WINDOW_POSITION_UNDEFINED
-          ? params->window_x : (int) SDL_WINDOWPOS_UNDEFINED;
-    int y = params->window_y != SC_WINDOW_POSITION_UNDEFINED
-          ? params->window_y : (int) SDL_WINDOWPOS_UNDEFINED;
-    screen->window = SDL_CreateWindow(params->window_title, x, y,
+    int x = window_x != SC_WINDOW_POSITION_UNDEFINED
+          ? window_x : (int) SDL_WINDOWPOS_UNDEFINED;
+    int y = window_y != SC_WINDOW_POSITION_UNDEFINED
+          ? window_y : (int) SDL_WINDOWPOS_UNDEFINED;
+    screen->window = SDL_CreateWindow(window_title, x, y,
                                       window_size.width, window_size.height,
                                       window_flags);
     if (!screen->window) {
@@ -309,11 +347,9 @@ screen_init(struct screen *screen, struct video_buffer *vb,
     const char *renderer_name = r ? NULL : renderer_info.name;
     LOGI("Renderer: %s", renderer_name ? renderer_name : "(unknown)");
 
-    screen->mipmaps = false;
-
     // starts with "opengl"
     bool use_opengl = renderer_name && !strncmp(renderer_name, "opengl", 6);
-    bool mipmaps = params->scale_filter == SC_SCALE_FILTER_TRILINEAR;
+    bool mipmaps = scale_filter == SC_SCALE_FILTER_TRILINEAR;
     if (use_opengl) {
         struct sc_opengl *gl = &screen->gl;
         sc_opengl_init(gl);
@@ -326,10 +362,10 @@ screen_init(struct screen *screen, struct video_buffer *vb,
                                                2, 0  /* OpenGL ES 2.0+ */);
             if (supports_mipmaps) {
                 LOGI("Trilinear filtering enabled");
-                screen->mipmaps = true;
             } else {
                 LOGW("Trilinear filtering disabled "
                      "(OpenGL 3.0+ or ES 2.0+ required)");
+                scale_filter = SC_SCALE_FILTER_NONE;
             }
         } else {
             LOGI("Trilinear filtering disabled");
@@ -338,6 +374,46 @@ screen_init(struct screen *screen, struct video_buffer *vb,
         LOGD("Trilinear filtering disabled (not an OpenGL renderer)");
     }
 
+    screen->use_swscale = is_swscale(scale_filter);
+    if (screen->use_swscale) {
+        bool ok = video_buffer_init(&screen->resizer_vb, false);
+        if (!ok) {
+            LOGE("Could not create resizer video buffer");
+            SDL_DestroyRenderer(screen->renderer);
+            SDL_DestroyWindow(screen->window);
+            return false;
+        }
+
+        ok = sc_resizer_init(&screen->resizer, screen->vb, scale_filter,
+                             window_size);
+        if (!ok) {
+            LOGE("Could not create resizer");
+            video_buffer_destroy(&screen->resizer_vb);
+            SDL_DestroyRenderer(screen->renderer);
+            SDL_DestroyWindow(screen->window);
+            return false;
+        }
+
+        static const struct video_buffer_callbacks cbs = {
+            .on_frame_available = resizer_on_frame_available,
+            .on_frame_skipped = resizer_on_frame_skipped,
+        };
+
+        video_buffer_set_consumer_callbacks(&screen->resizer.vb_out, &cbs,
+                                            screen);
+
+        ok = sc_resizer_start(&screen->resizer);
+        if (!ok) {
+            LOGE("Could not start resizer");
+            sc_resizer_destroy(&screen->resizer);
+            video_buffer_destroy(&screen->resizer_vb);
+            SDL_DestroyRenderer(screen->renderer);
+            SDL_DestroyWindow(screen->window);
+        }
+    }
+
+    screen->scale_filter = scale_filter;
+
     SDL_Surface *icon = read_xpm(icon_xpm);
     if (icon) {
         SDL_SetWindowIcon(screen->window, icon);
@@ -346,8 +422,8 @@ screen_init(struct screen *screen, struct video_buffer *vb,
         LOGW("Could not load icon");
     }
 
-    LOGI("Initial texture: %" PRIu16 "x%" PRIu16, params->frame_size.width,
-                                                  params->frame_size.height);
+    LOGI("Initial texture: %" PRIu16 "x%" PRIu16, frame_size.width,
+                                                  frame_size.height);
     screen->texture = create_texture(screen);
     if (!screen->texture) {
         LOGC("Could not create texture: %s", SDL_GetError());
@@ -356,15 +432,6 @@ screen_init(struct screen *screen, struct video_buffer *vb,
         return false;
     }
 
-    screen->frame = av_frame_alloc();
-    if (!screen->frame) {
-        LOGC("Could not create screen frame");
-        SDL_DestroyTexture(screen->texture);
-        SDL_DestroyRenderer(screen->renderer);
-        SDL_DestroyWindow(screen->window);
-        return false;
-    }
-
     // Reset the window size to trigger a SIZE_CHANGED event, to workaround
     // HiDPI issues with some SDL renderers when several displays having
     // different HiDPI scaling are connected
@@ -382,8 +449,14 @@ screen_show_window(struct screen *screen) {
 
 void
 screen_destroy(struct screen *screen) {
-    av_frame_free(&screen->frame);
-    SDL_DestroyTexture(screen->texture);
+    if (screen->use_swscale) {
+        sc_resizer_stop(&screen->resizer);
+        sc_resizer_join(&screen->resizer);
+        sc_resizer_destroy(&screen->resizer);
+    }
+    if (screen->texture) {
+        SDL_DestroyTexture(screen->texture);
+    }
     SDL_DestroyRenderer(screen->renderer);
     SDL_DestroyWindow(screen->window);
 }
@@ -447,16 +520,17 @@ screen_set_rotation(struct screen *screen, unsigned rotation) {
 
 // recreate the texture and resize the window if the frame size has changed
 static bool
-prepare_for_frame(struct screen *screen, struct size new_frame_size) {
-    if (screen->frame_size.width != new_frame_size.width
-            || screen->frame_size.height != new_frame_size.height) {
+prepare_for_frame(struct screen *screen, struct size original_frame_size,
+                  struct size frame_size) {
+    if (screen->frame_size.width != original_frame_size.width
+            || screen->frame_size.height != original_frame_size.height) {
         // frame dimension changed, destroy texture
         SDL_DestroyTexture(screen->texture);
 
-        screen->frame_size = new_frame_size;
+        screen->frame_size = original_frame_size;
 
         struct size new_content_size =
-            get_rotated_size(new_frame_size, screen->rotation);
+            get_rotated_size(original_frame_size, screen->rotation);
         set_content_size(screen, new_content_size);
 
         screen_update_content_rect(screen);
@@ -476,27 +550,41 @@ prepare_for_frame(struct screen *screen, struct size new_frame_size) {
 // write the frame into the texture
 static void
 update_texture(struct screen *screen, const AVFrame *frame) {
-    SDL_UpdateYUVTexture(screen->texture, NULL,
-            frame->data[0], frame->linesize[0],
-            frame->data[1], frame->linesize[1],
-            frame->data[2], frame->linesize[2]);
+    if (screen->use_swscale) {
+        SDL_UpdateTexture(screen->texture, NULL, frame->data[0],
+                          frame->linesize[0]);
+    } else {
+        SDL_UpdateYUVTexture(screen->texture, NULL,
+                frame->data[0], frame->linesize[0],
+                frame->data[1], frame->linesize[1],
+                frame->data[2], frame->linesize[2]);
 
-    if (screen->mipmaps) {
-        SDL_GL_BindTexture(screen->texture, NULL, NULL);
-        screen->gl.GenerateMipmap(GL_TEXTURE_2D);
-        SDL_GL_UnbindTexture(screen->texture);
+        if (screen->scale_filter == SC_SCALE_FILTER_TRILINEAR) {
+            SDL_GL_BindTexture(screen->texture, NULL, NULL);
+            screen->gl.GenerateMipmap(GL_TEXTURE_2D);
+            SDL_GL_UnbindTexture(screen->texture);
+        }
     }
 }
 
 static bool
 screen_update_frame(struct screen *screen) {
-    video_buffer_consumer_take_frame(screen->vb, &screen->frame);
-    AVFrame *frame = screen->frame;
+    const AVFrame *frame;
+    struct size original_frame_size;
+
+    if (screen->use_swscale) {
+        frame = sc_resizer_consumer_take_frame(&screen->resizer,
+                                               &original_frame_size);
+    } else {
+        frame = video_buffer_consumer_take_frame(screen->vb);
+        original_frame_size.width = frame->width;
+        original_frame_size.height = frame->height;
+    }
+    struct size frame_size = {frame->width, frame->height};
 
     fps_counter_add_rendered_frame(screen->fps_counter);
 
-    struct size new_frame_size = {frame->width, frame->height};
-    if (!prepare_for_frame(screen, new_frame_size)) {
+    if (!prepare_for_frame(screen, original_frame_size, frame_size)) {
         return false;
     }
     update_texture(screen, frame);

app/src/screen.h

@@ -9,7 +9,9 @@
 
 #include "coords.h"
 #include "opengl.h"
+#include "resizer.h"
 #include "scrcpy.h"
+#include "video_buffer.h"
 
 struct video_buffer;
 
@@ -38,30 +40,27 @@ struct screen {
     bool maximized;
     bool mipmaps;
 
-    AVFrame *frame;
-};
-
-struct screen_params {
-    const char *window_title;
-    struct size frame_size;
-    bool always_on_top;
-
-    int16_t window_x;
-    int16_t window_y;
-    uint16_t window_width;  // accepts SC_WINDOW_POSITION_UNDEFINED
-    uint16_t window_height; // accepts SC_WINDOW_POSITION_UNDEFINED
-
-    bool window_borderless;
-
-    uint8_t rotation;
     enum sc_scale_filter scale_filter;
+    bool use_swscale;
+
+    // For swscale resizing
+    struct video_buffer resizer_vb;
+    struct sc_resizer resizer;
 };
 
+// initialize default values
+void
+screen_init(struct screen *screen, struct video_buffer *vb,
+            struct fps_counter *fps_counter);
+
 // initialize screen, create window, renderer and texture (window is hidden)
+// window_x and window_y accept SC_WINDOW_POSITION_UNDEFINED
 bool
-screen_init(struct screen *screen, struct video_buffer *vb,
-            struct fps_counter *fps_counter,
-            const struct screen_params *params);
+screen_init_rendering(struct screen *screen, const char *window_title,
+                      struct size frame_size, bool always_on_top,
+                      int16_t window_x, int16_t window_y, uint16_t window_width,
+                      uint16_t window_height, bool window_borderless,
+                      uint8_t rotation, enum sc_scale_filter scale_filter);
 
 // show the window
 void

app/src/video_buffer.c

@@ -13,9 +13,14 @@ video_buffer_init(struct video_buffer *vb, bool wait_consumer) {
         goto error_0;
     }
 
+    vb->consumer_frame = av_frame_alloc();
+    if (!vb->consumer_frame) {
+        goto error_1;
+    }
+
     bool ok = sc_mutex_init(&vb->mutex);
     if (!ok) {
-        goto error_1;
+        goto error_2;
     }
 
     vb->wait_consumer = wait_consumer;
@@ -23,7 +28,7 @@ video_buffer_init(struct video_buffer *vb, bool wait_consumer) {
         ok = sc_cond_init(&vb->pending_frame_consumed_cond);
         if (!ok) {
             sc_mutex_destroy(&vb->mutex);
-            goto error_1;
+            goto error_2;
         }
         // interrupted is not used if wait_consumer is disabled since offering
         // a frame will never block
@@ -39,6 +44,8 @@ video_buffer_init(struct video_buffer *vb, bool wait_consumer) {
 
     return true;
 
+error_2:
+    av_frame_free(&vb->consumer_frame);
 error_1:
     av_frame_free(&vb->pending_frame);
 error_0:
@@ -51,6 +58,7 @@ video_buffer_destroy(struct video_buffer *vb) {
         sc_cond_destroy(&vb->pending_frame_consumed_cond);
     }
     sc_mutex_destroy(&vb->mutex);
+    av_frame_free(&vb->consumer_frame);
     av_frame_free(&vb->pending_frame);
 }
 
@@ -100,13 +108,13 @@ video_buffer_producer_offer_frame(struct video_buffer *vb, AVFrame **pframe) {
     }
 }
 
-void
-video_buffer_consumer_take_frame(struct video_buffer *vb, AVFrame **pframe) {
+const AVFrame *
+video_buffer_consumer_take_frame(struct video_buffer *vb) {
     sc_mutex_lock(&vb->mutex);
     assert(!vb->pending_frame_consumed);
     vb->pending_frame_consumed = true;
 
-    swap_frames(pframe, &vb->pending_frame);
+    swap_frames(&vb->consumer_frame, &vb->pending_frame);
     av_frame_unref(vb->pending_frame);
 
     if (vb->wait_consumer) {
@@ -114,6 +122,9 @@ video_buffer_consumer_take_frame(struct video_buffer *vb, AVFrame **pframe) {
         sc_cond_signal(&vb->pending_frame_consumed_cond);
     }
     sc_mutex_unlock(&vb->mutex);
+
+    // consumer_frame is only written from this thread, no need to lock
+    return vb->consumer_frame;
 }
 
 void

app/src/video_buffer.h

@@ -30,6 +30,7 @@ typedef struct AVFrame AVFrame;
 
 struct video_buffer {
     AVFrame *pending_frame;
+    AVFrame *consumer_frame;
 
     sc_mutex mutex;
     bool wait_consumer; // never overwrite a pending frame if it is not consumed
@@ -69,10 +70,10 @@ video_buffer_set_consumer_callbacks(struct video_buffer *vb,
 void
 video_buffer_producer_offer_frame(struct video_buffer *vb, AVFrame **pframe);
 
-// mark the consumer frame as consumed and exchange it with an unused allocated
-// frame
-void
-video_buffer_consumer_take_frame(struct video_buffer *vb, AVFrame **pframe);
+// mark the consumer frame as consumed and return it
+// the frame is valid until the next call to this function
+const AVFrame *
+video_buffer_consumer_take_frame(struct video_buffer *vb);
 
 // wake up and avoid any blocking call
 void