Compensate slope on start

The first frames are typically received and decoded with more delay than
the others, causing a wrong slope estimation on start.

To compensate, assume an initial slope of 1, then progressively use the
estimated slope.

README.md

@@ -321,6 +321,24 @@ For example, you could capture the video within [OBS].
 [OBS]: https://obsproject.com/fr
 
 
+#### Buffering
+
+It is possible to add buffering. This increases latency but reduces jitter (see
+#2464).
+
+The option is available for display buffering:
+
+```bash
+scrcpy --display-buffer=50  # add 50 ms buffering for display
+```
+
+and V4L2 sink:
+
+```bash
+scrcpy --v4l2-buffer=500    # add 500 ms buffering for v4l2 sink
+```
+
+
 ### Connection
 
 #### Wireless

app/meson.build

@@ -2,6 +2,7 @@ src = [
     'src/main.c',
     'src/adb.c',
     'src/cli.c',
+    'src/clock.c',
     'src/compat.c',
     'src/control_msg.c',
     'src/controller.c',
@@ -26,6 +27,7 @@ src = [
     'src/util/process.c',
     'src/util/str_util.c',
     'src/util/thread.c',
+    'src/util/tick.c',
 ]
 
 if host_machine.system() == 'windows'
@@ -166,6 +168,10 @@ if get_option('buildtype') == 'debug'
             'src/cli.c',
             'src/util/str_util.c',
         ]],
+        ['test_clock', [
+            'tests/test_clock.c',
+            'src/clock.c',
+        ]],
         ['test_control_msg_serialize', [
             'tests/test_control_msg_serialize.c',
             'src/control_msg.c',

app/scrcpy.1

@@ -56,6 +56,12 @@ The list of possible display ids can be listed by "adb shell dumpsys display"
 
 Default is 0.
 
+.TP
+.BI "\-\-display\-buffer ms
+Add a buffering delay (in milliseconds) before displaying. This increases latency to compensate for jitter.
+
+Default is 0 (no buffering).
+
 .TP
 .BI "\-\-encoder " name
 Use a specific MediaCodec encoder (must be a H.264 encoder).
@@ -83,7 +89,7 @@ Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+S
 This is a workaround for some devices not behaving as expected when setting the device clipboard programmatically.
 
 .TP
-.BI "\-\-lock\-video\-orientation " [value]
+.BI "\-\-lock\-video\-orientation[=value]
 Lock video orientation to \fIvalue\fR. Possible values are "unlocked", "initial" (locked to the initial orientation), 0, 1, 2 and 3. Natural device orientation is 0, and each increment adds a 90 degrees otation counterclockwise.
 
 Default is "unlocked".
@@ -189,7 +195,13 @@ It only shows physical touches (not clicks from scrcpy).
 .BI "\-\-v4l2-sink " /dev/videoN
 Output to v4l2loopback device.
 
-It requires to lock the video orientation (see --lock-video-orientation).
+It requires to lock the video orientation (see \fB\-\-lock\-video\-orientation\fR).
+
+.TP
+.BI "\-\-v4l2-buffer " ms
+Add a buffering delay (in milliseconds) before pushing frames. This increases latency to compensate for jitter.
+
+This option is similar to \fB\-\-display\-buffer\fR, but specific to V4L2 sink.
 
 .TP
 .BI "\-V, \-\-verbosity " value
@@ -240,7 +252,7 @@ Default is 0 (automatic).
 .SH SHORTCUTS
 
 In the following list, MOD is the shortcut modifier. By default, it's (left)
-Alt or (left) Super, but it can be configured by \-\-shortcut-mod (see above).
+Alt or (left) Super, but it can be configured by \fB\-\-shortcut\-mod\fR (see above).
 
 .TP
 .B MOD+f

app/src/cli.c

@@ -55,6 +55,12 @@ scrcpy_print_usage(const char *arg0) {
         "\n"
         "        Default is 0.\n"
         "\n"
+        "    --display-buffer ms\n"
+        "        Add a buffering delay (in milliseconds) before displaying.\n"
+        "        This increases latency to compensate for jitter.\n"
+        "\n"
+        "        Default is 0 (no buffering).\n"
+        "\n"
         "    --encoder name\n"
         "        Use a specific MediaCodec encoder (must be a H.264 encoder).\n"
         "\n"
@@ -79,7 +85,7 @@ scrcpy_print_usage(const char *arg0) {
         "        This is a workaround for some devices not behaving as\n"
         "        expected when setting the device clipboard programmatically.\n"
         "\n"
-        "    --lock-video-orientation [value]\n"
+        "    --lock-video-orientation[=value]\n"
         "        Lock video orientation to value.\n"
         "        Possible values are \"unlocked\", \"initial\" (locked to the\n"
         "        initial orientation), 0, 1, 2 and 3.\n"
@@ -182,6 +188,15 @@ scrcpy_print_usage(const char *arg0) {
         "        It requires to lock the video orientation (see\n"
         "        --lock-video-orientation).\n"
         "\n"
+        "    --v4l2-buffer ms\n"
+        "        Add a buffering delay (in milliseconds) before pushing\n"
+        "        frames. This increases latency to compensate for jitter.\n"
+        "\n"
+        "        This option is similar to --display-buffer, but specific to\n"
+        "        V4L2 sink.\n"
+        "\n"
+        "        Default is 0 (no buffering).\n"
+        "\n"
 #endif
         "    -V, --verbosity value\n"
         "        Set the log level (verbose, debug, info, warn or error).\n"
@@ -392,6 +407,19 @@ parse_max_fps(const char *s, uint16_t *max_fps) {
     return true;
 }
 
+static bool
+parse_buffering_time(const char *s, sc_tick *tick) {
+    long value;
+    bool ok = parse_integer_arg(s, &value, false, 0, 0xFFFFFFFF,
+                                "buffering time");
+    if (!ok) {
+        return false;
+    }
+
+    *tick = SC_TICK_FROM_MS(value);
+    return true;
+}
+
 static bool
 parse_lock_video_orientation(const char *s,
                              enum sc_lock_video_orientation *lock_mode) {
@@ -689,6 +717,8 @@ guess_record_format(const char *filename) {
 #define OPT_ENCODER_NAME           1025
 #define OPT_POWER_OFF_ON_CLOSE     1026
 #define OPT_V4L2_SINK              1027
+#define OPT_DISPLAY_BUFFER         1028
+#define OPT_V4L2_BUFFER            1029
 
 bool
 scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
@@ -700,6 +730,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
         {"disable-screensaver",    no_argument,       NULL,
                                                   OPT_DISABLE_SCREENSAVER},
         {"display",                required_argument, NULL, OPT_DISPLAY_ID},
+        {"display-buffer",         required_argument, NULL, OPT_DISPLAY_BUFFER},
         {"encoder",                required_argument, NULL, OPT_ENCODER_NAME},
         {"force-adb-forward",      no_argument,       NULL,
                                                   OPT_FORCE_ADB_FORWARD},
@@ -732,6 +763,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
         {"turn-screen-off",        no_argument,       NULL, 'S'},
 #ifdef HAVE_V4L2
         {"v4l2-sink",              required_argument, NULL, OPT_V4L2_SINK},
+        {"v4l2-buffer",            required_argument, NULL, OPT_V4L2_BUFFER},
 #endif
         {"verbosity",              required_argument, NULL, 'V'},
         {"version",                no_argument,       NULL, 'v'},
@@ -917,10 +949,20 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
             case OPT_POWER_OFF_ON_CLOSE:
                 opts->power_off_on_close = true;
                 break;
+            case OPT_DISPLAY_BUFFER:
+                if (!parse_buffering_time(optarg, &opts->display_buffer)) {
+                    return false;
+                }
+                break;
 #ifdef HAVE_V4L2
             case OPT_V4L2_SINK:
                 opts->v4l2_device = optarg;
                 break;
+            case OPT_V4L2_BUFFER:
+                if (!parse_buffering_time(optarg, &opts->v4l2_buffer)) {
+                    return false;
+                }
+                break;
 #endif
             default:
                 // getopt prints the error message on stderr

app/src/clock.c

@@ -0,0 +1,111 @@
+#include "clock.h"
+
+#include "util/log.h"
+
+#define SC_CLOCK_NDEBUG // comment to debug
+
+void
+sc_clock_init(struct sc_clock *clock) {
+    clock->count = 0;
+    clock->head = 0;
+    clock->left_sum.system = 0;
+    clock->left_sum.stream = 0;
+    clock->right_sum.system = 0;
+    clock->right_sum.stream = 0;
+}
+
+// Estimate the affine function f(stream) = slope * stream + offset
+static void
+sc_clock_estimate(struct sc_clock *clock,
+                  double *out_slope, sc_tick *out_offset) {
+    assert(clock->count > 1); // two points are necessary
+
+    struct sc_clock_point left_avg = {
+        .system = clock->left_sum.system / (clock->count / 2),
+        .stream = clock->left_sum.stream / (clock->count / 2),
+    };
+    struct sc_clock_point right_avg = {
+        .system = clock->right_sum.system / ((clock->count + 1) / 2),
+        .stream = clock->right_sum.stream / ((clock->count + 1) / 2),
+    };
+
+    double slope = (double) (right_avg.system - left_avg.system)
+                 / (right_avg.stream - left_avg.stream);
+
+    if (clock->count < SC_CLOCK_RANGE) {
+        /* The first frames are typically received and decoded with more delay
+         * than the others, causing a wrong slope estimation on start. To
+         * compensate, assume an initial slope of 1, then progressively use the
+         * estimated slope. */
+        slope = (clock->count * slope + (SC_CLOCK_RANGE - clock->count))
+              / SC_CLOCK_RANGE;
+    }
+
+    struct sc_clock_point global_avg = {
+        .system = (clock->left_sum.system + clock->right_sum.system)
+                 / clock->count,
+        .stream = (clock->left_sum.stream + clock->right_sum.stream)
+                 / clock->count,
+    };
+
+    sc_tick offset = global_avg.system - (sc_tick) (global_avg.stream * slope);
+
+    *out_slope = slope;
+    *out_offset = offset;
+}
+
+void
+sc_clock_update(struct sc_clock *clock, sc_tick system, sc_tick stream) {
+    struct sc_clock_point *point = &clock->points[clock->head];
+
+    if (clock->count == SC_CLOCK_RANGE || clock->count & 1) {
+        // One point passes from the right sum to the left sum
+
+        unsigned mid;
+        if (clock->count == SC_CLOCK_RANGE) {
+            mid = (clock->head + SC_CLOCK_RANGE / 2) % SC_CLOCK_RANGE;
+        } else {
+            // Only for the first frames
+            mid = clock->count / 2;
+        }
+
+        struct sc_clock_point *mid_point = &clock->points[mid];
+        clock->left_sum.system += mid_point->system;
+        clock->left_sum.stream += mid_point->stream;
+        clock->right_sum.system -= mid_point->system;
+        clock->right_sum.stream -= mid_point->stream;
+    }
+
+    if (clock->count == SC_CLOCK_RANGE) {
+        // The current point overwrites the previous value in the circular
+        // array, update the left sum accordingly
+        clock->left_sum.system -= point->system;
+        clock->left_sum.stream -= point->stream;
+    } else {
+        ++clock->count;
+    }
+
+    point->system = system;
+    point->stream = stream;
+
+    clock->right_sum.system += system;
+    clock->right_sum.stream += stream;
+
+    clock->head = (clock->head + 1) % SC_CLOCK_RANGE;
+
+    if (clock->count > 1) {
+        // Update estimation
+        sc_clock_estimate(clock, &clock->slope, &clock->offset);
+
+#ifndef SC_CLOCK_NDEBUG
+        LOGD("Clock estimation: %g * pts + %" PRItick,
+             clock->slope, clock->offset);
+#endif
+    }
+}
+
+sc_tick
+sc_clock_to_system_time(struct sc_clock *clock, sc_tick stream) {
+    assert(clock->count > 1); // sc_clock_update() must have been called
+    return (sc_tick) (stream * clock->slope) + clock->offset;
+}

app/src/clock.h

@@ -0,0 +1,70 @@
+#ifndef SC_CLOCK_H
+#define SC_CLOCK_H
+
+#include "common.h"
+
+#include <assert.h>
+
+#include "util/tick.h"
+
+#define SC_CLOCK_RANGE 32
+static_assert(!(SC_CLOCK_RANGE & 1), "SC_CLOCK_RANGE must be even");
+
+struct sc_clock_point {
+    sc_tick system;
+    sc_tick stream;
+};
+
+/**
+ * The clock aims to estimate the affine relation between the stream (device)
+ * time and the system time:
+ *
+ *     f(stream) = slope * stream + offset
+ *
+ * To that end, it stores the SC_CLOCK_RANGE last clock points (the timestamps
+ * of a frame expressed both in stream time and system time) in a circular
+ * array.
+ *
+ * To estimate the slope, it splits the last SC_CLOCK_RANGE points into two
+ * sets of SC_CLOCK_RANGE/2 points, and compute their centroid ("average
+ * point"). The slope of the estimated affine function is that of the line
+ * passing through these two points.
+ *
+ * To estimate the offset, it computes the centroid of all the SC_CLOCK_RANGE
+ * points. The resulting affine function passes by this centroid.
+ *
+ * With a circular array, the rolling sums (and average) are quick to compute.
+ * In practice, the estimation is stable and the evolution is smooth.
+ */
+struct sc_clock {
+    // Circular array
+    struct sc_clock_point points[SC_CLOCK_RANGE];
+
+    // Number of points in the array (count <= SC_CLOCK_RANGE)
+    unsigned count;
+
+    // Index of the next point to write
+    unsigned head;
+
+    // Sum of the first count/2 points
+    struct sc_clock_point left_sum;
+
+    // Sum of the last (count+1)/2 points
+    struct sc_clock_point right_sum;
+
+    // Estimated slope and offset
+    // (computed on sc_clock_update(), used by sc_clock_to_system_time())
+    double slope;
+    sc_tick offset;
+};
+
+void
+sc_clock_init(struct sc_clock *clock);
+
+void
+sc_clock_update(struct sc_clock *clock, sc_tick system, sc_tick stream);
+
+sc_tick
+sc_clock_to_system_time(struct sc_clock *clock, sc_tick stream);
+
+#endif

app/src/fps_counter.c

@@ -4,7 +4,7 @@
 
 #include "util/log.h"
 
-#define FPS_COUNTER_INTERVAL_MS 1000
+#define FPS_COUNTER_INTERVAL SC_TICK_FROM_SEC(1)
 
 bool
 fps_counter_init(struct fps_counter *counter) {
@@ -46,7 +46,7 @@ set_started(struct fps_counter *counter, bool started) {
 static void
 display_fps(struct fps_counter *counter) {
     unsigned rendered_per_second =
-        counter->nr_rendered * 1000 / FPS_COUNTER_INTERVAL_MS;
+        counter->nr_rendered * SC_TICK_FREQ / FPS_COUNTER_INTERVAL;
     if (counter->nr_skipped) {
         LOGI("%u fps (+%u frames skipped)", rendered_per_second,
                                             counter->nr_skipped);
@@ -67,8 +67,8 @@ check_interval_expired(struct fps_counter *counter, uint32_t now) {
     counter->nr_skipped = 0;
     // add a multiple of the interval
     uint32_t elapsed_slices =
-        (now - counter->next_timestamp) / FPS_COUNTER_INTERVAL_MS + 1;
-    counter->next_timestamp += FPS_COUNTER_INTERVAL_MS * elapsed_slices;
+        (now - counter->next_timestamp) / FPS_COUNTER_INTERVAL + 1;
+    counter->next_timestamp += FPS_COUNTER_INTERVAL * elapsed_slices;
 }
 
 static int
@@ -96,7 +96,7 @@ run_fps_counter(void *data) {
 bool
 fps_counter_start(struct fps_counter *counter) {
     sc_mutex_lock(&counter->mutex);
-    counter->next_timestamp = sc_tick_now() + FPS_COUNTER_INTERVAL_MS;
+    counter->next_timestamp = sc_tick_now() + FPS_COUNTER_INTERVAL;
     counter->nr_rendered = 0;
     counter->nr_skipped = 0;
     sc_mutex_unlock(&counter->mutex);

app/src/scrcpy.c

@@ -381,6 +381,7 @@ scrcpy(const struct scrcpy_options *options) {
             .rotation = options->rotation,
             .mipmaps = options->mipmaps,
             .fullscreen = options->fullscreen,
+            .buffering_time = options->display_buffer,
         };
 
         if (!screen_init(&s->screen, &screen_params)) {
@@ -393,7 +394,8 @@ scrcpy(const struct scrcpy_options *options) {
 
 #ifdef HAVE_V4L2
     if (options->v4l2_device) {
-        if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device, frame_size)) {
+        if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device, frame_size,
+                               options->v4l2_buffer)) {
             goto end;
         }
 

app/src/scrcpy.h

@@ -7,6 +7,8 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "util/tick.h"
+
 enum sc_log_level {
     SC_LOG_LEVEL_VERBOSE,
     SC_LOG_LEVEL_DEBUG,
@@ -78,6 +80,8 @@ struct scrcpy_options {
     uint16_t window_width;
     uint16_t window_height;
     uint32_t display_id;
+    sc_tick display_buffer;
+    sc_tick v4l2_buffer;
     bool show_touches;
     bool fullscreen;
     bool always_on_top;
@@ -126,6 +130,8 @@ struct scrcpy_options {
     .window_width = 0, \
     .window_height = 0, \
     .display_id = 0, \
+    .display_buffer = 0, \
+    .v4l2_buffer = 0, \
     .show_touches = false, \
     .fullscreen = false, \
     .always_on_top = false, \

app/src/screen.c

@@ -308,7 +308,8 @@ screen_init(struct screen *screen, const struct screen_params *params) {
         .on_new_frame = sc_video_buffer_on_new_frame,
     };
 
-    bool ok = sc_video_buffer_init(&screen->vb, 2000, &cbs, screen);
+    bool ok = sc_video_buffer_init(&screen->vb, params->buffering_time, &cbs,
+                                   screen);
     if (!ok) {
         LOGE("Could not initialize video buffer");
         return false;

app/src/screen.h

@@ -63,6 +63,8 @@ struct screen_params {
     bool mipmaps;
 
     bool fullscreen;
+
+    sc_tick buffering_time;
 };
 
 // initialize screen, create window, renderer and texture (window is hidden)

app/src/server.c

@@ -554,10 +554,10 @@ server_stop(struct server *server) {
     sc_mutex_lock(&server->mutex);
     bool signaled = false;
     if (!server->process_terminated) {
-#define WATCHDOG_DELAY_MS 1000
+#define WATCHDOG_DELAY SC_TICK_FROM_SEC(1)
         signaled = sc_cond_timedwait(&server->process_terminated_cond,
                                      &server->mutex,
-                                     sc_tick_now() + WATCHDOG_DELAY_MS);
+                                     sc_tick_now() + WATCHDOG_DELAY);
     }
     sc_mutex_unlock(&server->mutex);
 

app/src/util/thread.c

@@ -2,7 +2,6 @@
 
 #include <assert.h>
 #include <SDL2/SDL_thread.h>
-#include <SDL2/SDL_timer.h>
 
 #include "log.h"
 
@@ -130,8 +129,8 @@ sc_cond_timedwait(sc_cond *cond, sc_mutex *mutex, sc_tick deadline) {
         return false; // timeout
     }
 
-    sc_tick delay = deadline - now;
-    int r = SDL_CondWaitTimeout(cond->cond, mutex->mutex, delay);
+    uint32_t ms = SC_TICK_TO_MS(deadline - now);
+    int r = SDL_CondWaitTimeout(cond->cond, mutex->mutex, ms);
 #ifndef NDEBUG
     if (r < 0) {
         LOGC("Could not wait on condition with timeout: %s", SDL_GetError());
@@ -170,11 +169,3 @@ sc_cond_broadcast(sc_cond *cond) {
     (void) r;
 #endif
 }
-
-sc_tick
-sc_tick_now(void) {
-    // SDL ticks is an unsigned 32 bits, but this is an implementation detail.
-    // It wraps if the program runs for more than ~49 days, but in practice we
-    // can assume it does not.
-    return (sc_tick) SDL_GetTicks();
-}

app/src/util/thread.h

@@ -5,7 +5,8 @@
 
 #include <stdatomic.h>
 #include <stdbool.h>
-#include <stdint.h>
+
+#include "tick.h"
 
 /* Forward declarations */
 typedef struct SDL_Thread SDL_Thread;
@@ -16,8 +17,6 @@ typedef int sc_thread_fn(void *);
 typedef unsigned sc_thread_id;
 typedef atomic_uint sc_atomic_thread_id;
 
-typedef int64_t sc_tick;
-
 typedef struct sc_thread {
     SDL_Thread *thread;
 } sc_thread;
@@ -82,7 +81,4 @@ sc_cond_signal(sc_cond *cond);
 void
 sc_cond_broadcast(sc_cond *cond);
 
-sc_tick
-sc_tick_now(void);
-
 #endif

app/src/util/tick.c

@@ -0,0 +1,16 @@
+#include "tick.h"
+
+#include <SDL2/SDL_timer.h>
+
+sc_tick
+sc_tick_now(void) {
+    // SDL_GetTicks() resolution is in milliseconds, but sc_tick are expressed
+    // in microseconds to avoid loosing precision on PTS.
+    //
+    // SDL_GetPerformanceCounter()/SDL_GetPerformanceFrequency() could be used,
+    // but:
+    //  - the conversions (to avoid overflow) are not zero-cost, since the
+    //    frequency is not known at compile time;
+    //  - in practice, we don't need more precision for now.
+    return (sc_tick) SDL_GetTicks() * 1000;
+}

app/src/util/tick.h

@@ -0,0 +1,21 @@
+#ifndef SC_TICK_H
+#define SC_TICK_H
+
+#include <stdint.h>
+
+typedef int64_t sc_tick;
+#define PRItick PRIi64
+#define SC_TICK_FREQ 1000000 // microsecond
+
+// To be adapted if SC_TICK_FREQ changes
+#define SC_TICK_TO_US(tick) (tick)
+#define SC_TICK_TO_MS(tick) ((tick) / 1000)
+#define SC_TICK_TO_SEC(tick) ((tick) / 1000000)
+#define SC_TICK_FROM_US(us) (us)
+#define SC_TICK_FROM_MS(ms) ((ms) * 1000)
+#define SC_TICK_FROM_SEC(sec) ((sec) * 1000000)
+
+sc_tick
+sc_tick_now(void);
+
+#endif

app/src/v4l2_sink.c

@@ -159,7 +159,7 @@ sc_v4l2_sink_open(struct sc_v4l2_sink *vs) {
         .on_new_frame = sc_video_buffer_on_new_frame,
     };
 
-    bool ok = sc_video_buffer_init(&vs->vb, 1, &cbs, vs);
+    bool ok = sc_video_buffer_init(&vs->vb, vs->buffering_time, &cbs, vs);
     if (!ok) {
         LOGE("Could not initialize video buffer");
         return false;
@@ -356,7 +356,7 @@ sc_v4l2_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {
 
 bool
 sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
-                  struct size frame_size) {
+                  struct size frame_size, sc_tick buffering_time) {
     vs->device_name = strdup(device_name);
     if (!vs->device_name) {
         LOGE("Could not strdup v4l2 device name");
@@ -364,6 +364,7 @@ sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
     }
 
     vs->frame_size = frame_size;
+    vs->buffering_time = buffering_time;
 
     static const struct sc_frame_sink_ops ops = {
         .open = sc_v4l2_frame_sink_open,

app/src/v4l2_sink.h

@@ -6,6 +6,7 @@
 #include "coords.h"
 #include "trait/frame_sink.h"
 #include "video_buffer.h"
+#include "util/tick.h"
 
 #include <libavformat/avformat.h>
 
@@ -18,6 +19,7 @@ struct sc_v4l2_sink {
 
     char *device_name;
     struct size frame_size;
+    sc_tick buffering_time;
 
     sc_thread thread;
     sc_mutex mutex;
@@ -32,7 +34,7 @@ struct sc_v4l2_sink {
 
 bool
 sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
-                  struct size frame_size);
+                  struct size frame_size, sc_tick buffering_time);
 
 void
 sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs);

app/src/video_buffer.c

@@ -8,58 +8,7 @@
 
 #include "util/log.h"
 
-#define SC_CLOCK_AVERAGE_RANGE 32
-
-static void
-sc_clock_init(struct sc_clock *clock) {
-    clock->coeff = 1;
-    clock->offset = 0;
-    clock->weight = 0;
-
-    clock->last.system = 0;
-    clock->last.stream = 0;
-}
-
-static sc_tick
-sc_clock_to_system_ts(struct sc_clock *clock, sc_tick stream_ts) {
-    assert(clock->weight); // sc_clock_update() must have been called
-    return (sc_tick) (stream_ts * clock->coeff) + clock->offset;
-}
-
-static void
-sc_clock_update(struct sc_clock *clock, sc_tick now, sc_tick stream_ts) {
-    double instant_coeff;
-    sc_tick mad;
-    if (clock->weight) {
-        sc_tick system_delta = now - clock->last.system;
-        sc_tick stream_delta = stream_ts - clock->last.stream;
-        instant_coeff = (double) system_delta / stream_delta;
-        sc_tick system_pts = sc_clock_to_system_ts(clock, stream_ts);
-        mad = llabs(now - system_pts);
-    } else {
-        // This is the first update, we cannot compute delta
-        instant_coeff = 1;
-        mad = 0;
-    }
-
-    if (clock->weight < SC_CLOCK_AVERAGE_RANGE) {
-        ++clock->weight;
-    }
-
-    // (1-t) * avg + t * new
-    clock->coeff = ((clock->weight - 1) * clock->coeff + instant_coeff)
-                 / clock->weight;
-
-    // FIXME it cannot change at every frame!
-    clock->offset = now - (sc_tick) (stream_ts * clock->coeff);
-
-    LOGD("%g x + %ld", clock->coeff, clock->offset);
-
-    clock->mad = ((clock->weight - 1) * clock->mad + mad) / clock->weight;
-
-    clock->last.system = now;
-    clock->last.stream = stream_ts;
-}
+#define SC_BUFFERING_NDEBUG // comment to debug
 
 static struct sc_video_buffer_frame *
 sc_video_buffer_frame_new(const AVFrame *frame) {
@@ -106,7 +55,7 @@ static int
 run_buffering(void *data) {
     struct sc_video_buffer *vb = data;
 
-    assert(vb->buffering_ms);
+    assert(vb->buffering_time > 0);
 
     for (;;) {
         sc_mutex_lock(&vb->b.mutex);
@@ -123,22 +72,18 @@ run_buffering(void *data) {
         struct sc_video_buffer_frame *vb_frame;
         sc_queue_take(&vb->b.queue, next, &vb_frame);
 
-        sc_tick now = sc_tick_now();
-        // FIXME time_base units
-        int64_t pts = vb_frame->frame->pts; // micros to millis
-        LOGD("==== pts = %ld", pts);
-        sc_tick system_pts = sc_clock_to_system_ts(&vb->b.clock, pts);
-        if (now + vb->buffering_ms < system_pts) {
-            system_pts = now + vb->buffering_ms;
-        }
+        sc_tick max_deadline = sc_tick_now() + vb->buffering_time;
+        // PTS (written by the server) are expressed in microseconds
+        sc_tick pts = SC_TICK_TO_US(vb_frame->frame->pts);
 
-        sc_tick deadline = system_pts + vb->buffering_ms;
-
-        LOGD("==== %ld %ld %ld\n", now, system_pts, deadline);
-
-        LOGD("[WAITING FOR] %ld ", deadline-now);
         bool timed_out = false;
         while (!vb->b.stopped && !timed_out) {
+            sc_tick deadline = sc_clock_to_system_time(&vb->b.clock, pts)
+                             + vb->buffering_time;
+            if (deadline > max_deadline) {
+                deadline = max_deadline;
+            }
+
             timed_out =
                 !sc_cond_timedwait(&vb->b.wait_cond, &vb->b.mutex, deadline);
         }
@@ -151,6 +96,11 @@ run_buffering(void *data) {
 
         sc_mutex_unlock(&vb->b.mutex);
 
+#ifndef SC_BUFFERING_NDEBUG
+        LOGD("Buffering: %" PRItick ";%" PRItick ";%" PRItick,
+             pts, vb_frame->push_date, sc_tick_now());
+#endif
+
         sc_video_buffer_offer(vb, vb_frame->frame);
 
         sc_video_buffer_frame_delete(vb_frame);
@@ -170,7 +120,7 @@ stopped:
 }
 
 bool
-sc_video_buffer_init(struct sc_video_buffer *vb, unsigned buffering_ms,
+sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick buffering_time,
                      const struct sc_video_buffer_callbacks *cbs,
                      void *cbs_userdata) {
     bool ok = sc_frame_buffer_init(&vb->fb);
@@ -178,7 +128,8 @@ sc_video_buffer_init(struct sc_video_buffer *vb, unsigned buffering_ms,
         return false;
     }
 
-    if (buffering_ms) {
+    assert(buffering_time >= 0);
+    if (buffering_time) {
         ok = sc_mutex_init(&vb->b.mutex);
         if (!ok) {
             LOGC("Could not create mutex");
@@ -210,7 +161,7 @@ sc_video_buffer_init(struct sc_video_buffer *vb, unsigned buffering_ms,
     assert(cbs);
     assert(cbs->on_new_frame);
 
-    vb->buffering_ms = buffering_ms;
+    vb->buffering_time = buffering_time;
     vb->cbs = cbs;
     vb->cbs_userdata = cbs_userdata;
     return true;
@@ -218,7 +169,7 @@ sc_video_buffer_init(struct sc_video_buffer *vb, unsigned buffering_ms,
 
 bool
 sc_video_buffer_start(struct sc_video_buffer *vb) {
-    if (vb->buffering_ms) {
+    if (vb->buffering_time) {
         bool ok =
             sc_thread_create(&vb->b.thread, run_buffering, "buffering", vb);
         if (!ok) {
@@ -232,7 +183,7 @@ sc_video_buffer_start(struct sc_video_buffer *vb) {
 
 void
 sc_video_buffer_stop(struct sc_video_buffer *vb) {
-    if (vb->buffering_ms) {
+    if (vb->buffering_time) {
         sc_mutex_lock(&vb->b.mutex);
         vb->b.stopped = true;
         sc_cond_signal(&vb->b.queue_cond);
@@ -243,7 +194,7 @@ sc_video_buffer_stop(struct sc_video_buffer *vb) {
 
 void
 sc_video_buffer_join(struct sc_video_buffer *vb) {
-    if (vb->buffering_ms) {
+    if (vb->buffering_time) {
         sc_thread_join(&vb->b.thread, NULL);
     }
 }
@@ -251,7 +202,7 @@ sc_video_buffer_join(struct sc_video_buffer *vb) {
 void
 sc_video_buffer_destroy(struct sc_video_buffer *vb) {
     sc_frame_buffer_destroy(&vb->fb);
-    if (vb->buffering_ms) {
+    if (vb->buffering_time) {
         sc_cond_destroy(&vb->b.wait_cond);
         sc_cond_destroy(&vb->b.queue_cond);
         sc_mutex_destroy(&vb->b.mutex);
@@ -260,22 +211,39 @@ sc_video_buffer_destroy(struct sc_video_buffer *vb) {
 
 bool
 sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame) {
-    if (!vb->buffering_ms) {
-        // no buffering
+    if (!vb->buffering_time) {
+        // No buffering
+        return sc_video_buffer_offer(vb, frame);
+    }
+
+    sc_mutex_lock(&vb->b.mutex);
+
+    sc_tick pts = SC_TICK_FROM_US(frame->pts);
+    sc_clock_update(&vb->b.clock, sc_tick_now(), pts);
+    sc_cond_signal(&vb->b.wait_cond);
+
+    if (vb->b.clock.count == 1) {
+        sc_mutex_unlock(&vb->b.mutex);
+        // First frame, offer it immediately, for two reasons:
+        //  - not to delay the opening of the scrcpy window
+        //  - the buffering estimation needs at least two clock points, so it
+        //  could not handle the first frame
         return sc_video_buffer_offer(vb, frame);
     }
 
     struct sc_video_buffer_frame *vb_frame = sc_video_buffer_frame_new(frame);
     if (!vb_frame) {
+        sc_mutex_unlock(&vb->b.mutex);
         LOGE("Could not allocate frame");
         return false;
     }
 
-    sc_clock_update(&vb->b.clock, sc_tick_now(), vb_frame->frame->pts);
-
-    sc_mutex_lock(&vb->b.mutex);
+#ifndef SC_BUFFERING_NDEBUG
+    vb_frame->push_date = sc_tick_now();
+#endif
     sc_queue_push(&vb->b.queue, next, vb_frame);
     sc_cond_signal(&vb->b.queue_cond);
+
     sc_mutex_unlock(&vb->b.mutex);
 
     return true;

app/src/video_buffer.h

@@ -5,38 +5,29 @@
 
 #include <stdbool.h>
 
+#include "clock.h"
 #include "frame_buffer.h"
 #include "util/queue.h"
 #include "util/thread.h"
+#include "util/tick.h"
 
 // forward declarations
 typedef struct AVFrame AVFrame;
 
 struct sc_video_buffer_frame {
     AVFrame *frame;
-    sc_tick system_pts;
     struct sc_video_buffer_frame *next;
+#ifndef NDEBUG
+    sc_tick push_date;
+#endif
 };
 
 struct sc_video_buffer_frame_queue SC_QUEUE(struct sc_video_buffer_frame);
 
-struct sc_clock {
-    double coeff;
-    sc_tick offset;
-    unsigned weight;
-
-    struct {
-        sc_tick system;
-        sc_tick stream;
-    } last;
-
-    sc_tick mad; // mean absolute difference
-};
-
 struct sc_video_buffer {
     struct sc_frame_buffer fb;
 
-    unsigned buffering_ms;
+    sc_tick buffering_time;
 
     // only if buffering_ms > 0
     struct {
@@ -60,7 +51,7 @@ struct sc_video_buffer_callbacks {
 };
 
 bool
-sc_video_buffer_init(struct sc_video_buffer *vb, unsigned buffering_ms,
+sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick buffering_time,
                      const struct sc_video_buffer_callbacks *cbs,
                      void *cbs_userdata);
 

app/tests/test_clock.c

@@ -0,0 +1,79 @@
+#include "common.h"
+
+#include <assert.h>
+
+#include "clock.h"
+
+void test_small_rolling_sum(void) {
+    struct sc_clock clock;
+    sc_clock_init(&clock);
+
+    assert(clock.count == 0);
+    assert(clock.left_sum.system == 0);
+    assert(clock.left_sum.stream == 0);
+    assert(clock.right_sum.system == 0);
+    assert(clock.right_sum.stream == 0);
+
+    sc_clock_update(&clock, 2, 3);
+    assert(clock.count == 1);
+    assert(clock.left_sum.system == 0);
+    assert(clock.left_sum.stream == 0);
+    assert(clock.right_sum.system == 2);
+    assert(clock.right_sum.stream == 3);
+
+    sc_clock_update(&clock, 10, 20);
+    assert(clock.count == 2);
+    assert(clock.left_sum.system == 2);
+    assert(clock.left_sum.stream == 3);
+    assert(clock.right_sum.system == 10);
+    assert(clock.right_sum.stream == 20);
+
+    sc_clock_update(&clock, 40, 80);
+    assert(clock.count == 3);
+    assert(clock.left_sum.system == 2);
+    assert(clock.left_sum.stream == 3);
+    assert(clock.right_sum.system == 50);
+    assert(clock.right_sum.stream == 100);
+
+    sc_clock_update(&clock, 400, 800);
+    assert(clock.count == 4);
+    assert(clock.left_sum.system == 12);
+    assert(clock.left_sum.stream == 23);
+    assert(clock.right_sum.system == 440);
+    assert(clock.right_sum.stream == 880);
+}
+
+void test_large_rolling_sum(void) {
+    const unsigned half_range = SC_CLOCK_RANGE / 2;
+
+    struct sc_clock clock1;
+    sc_clock_init(&clock1);
+    for (unsigned i = 0; i < 5 * half_range; ++i) {
+        sc_clock_update(&clock1, i, 2 * i + 1);
+    }
+
+    struct sc_clock clock2;
+    sc_clock_init(&clock2);
+    for (unsigned i = 3 * half_range; i < 5 * half_range; ++i) {
+        sc_clock_update(&clock2, i, 2 * i + 1);
+    }
+
+    assert(clock1.count == SC_CLOCK_RANGE);
+    assert(clock2.count == SC_CLOCK_RANGE);
+
+    // The values before the last SC_CLOCK_RANGE points in clock1 should have
+    // no impact
+    assert(clock1.left_sum.system == clock2.left_sum.system);
+    assert(clock1.left_sum.stream == clock2.left_sum.stream);
+    assert(clock1.right_sum.system == clock2.right_sum.system);
+    assert(clock1.right_sum.stream == clock2.right_sum.stream);
+}
+
+int main(int argc, char *argv[]) {
+    (void) argc;
+    (void) argv;
+
+    test_small_rolling_sum();
+    test_large_rolling_sum();
+    return 0;
+};