Document audio player

Add some high-level documentation on the audio player implementation.

app/scrcpy.1

@@ -23,13 +23,13 @@ Make scrcpy window always on top (above other windows).
 .BI "\-\-audio\-bit\-rate " value
 Encode the audio at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000).
 
-Default is 128K (128000).
+Default is 196K (196000).
 
 .TP
 .BI "\-\-audio\-buffer ms
 Configure the audio buffering delay (in milliseconds).
 
-Lower values decrease the latency, but increase the likelyhood of buffer underrun (causing audio glitches).
+Lower values decrease the latency, but increase the likelyhood of buffer underruns (causing audio glitches).
 
 Default is 50.
 

app/src/audio_player.c

@@ -6,6 +6,53 @@
 
 #define SC_AUDIO_PLAYER_NDEBUG // comment to debug
 
+/**
+ * Real-time audio player with configurable latency
+ *
+ * As input, the player regularly receives AVFrames of decoded audio samples.
+ * As output, an SDL callback regularly requests audio samples to be played.
+ * In the middle, an audio buffer stores the samples produced but not consumed
+ * yet.
+ *
+ * The goal of the player is to feed the audio output with a latency as low as
+ * possible while avoiding buffer underrun (i.e. not being able to provide
+ * samples when requested).
+ *
+ * For that purpose, it attempts to keep the average buffering (the number of
+ * samples present in the buffer) around a target value. If this target
+ * buffering is too low, then buffer underrun will occur often. If it is too
+ * high, then latency will be increased (this can be used on purpose to delay
+ * the audio playback). This value can be configured (in milliseconds) via the
+ * scrcpy option --audio-buffer.
+ *
+ * The player could not adjust the sample input rate (it receives samples
+ * produced in real-time) nor the sample output rate (it must provide samples
+ * as requested by the audio output callback). Therefore, it may only apply
+ * compensation by resampling (convert m input samples to n output samples).
+ *
+ * The compensation itself is applied by swresample (FFmpeg). It is configured
+ * by swr_set_compensation(). An important work for the player is thus to
+ * estimate regularly the compensation value to configure.
+ *
+ * Basically, the player wants to estimate the current buffering level: it is
+ * the result of an average of the "natural" buffering (samples are produced
+ * and consumed by blocks, so it must be smoothed), and by instant adjustments
+ * resulting of its own actions (explicit compensation and silence insertion on
+ * underflow), which are not smoothed.
+ *
+ * Buffer underflow events may occur when packets arrive too late. In that
+ * case, the player is inserting silence. Once the packets finally arrive
+ * (late), one strategy could be to drop the samples that were replaced by
+ * silence, in order to keep a minimal latency. However, dropping samples in
+ * case of buffer underflow is not a good strategy: it would temporarily
+ * increase the underflow even more, and cause very audible audio glitches.
+ *
+ * Therefore, the player don't drop any sample on underflow, the stream is just
+ * delayed by the number of silent samples inserted. But in return, these
+ * additional samples will increase buffering (latency) when the late packets
+ * will arrive, so they will be taken into account by compensation.
+ */
+
 /** Downcast frame_sink to sc_audio_player */
 #define DOWNCAST(SINK) container_of(SINK, struct sc_audio_player, frame_sink)
 
@@ -48,8 +95,10 @@ sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) {
         // remaining (margin) last samples will be handled by compensation.
         uint32_t margin = 30 * ap->sample_rate / 1000; // 30ms
         if (buffered_samples + margin < ap->target_buffering) {
-            LOGV("[Audio] Inserting initial buffering silence: %" PRIu32
+#ifndef SC_AUDIO_PLAYER_NDEBUG
+            LOGD("[Audio] Inserting initial buffering silence: %" PRIu32
                  " samples", bytes_to_samples(ap, len));
+#endif
             // Delay playback starting to reach the target buffering. Fill the
             // whole buffer with silence (len is small compared to the
             // arbitrary margin value).
@@ -66,13 +115,14 @@ sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) {
     if (read < len) {
         size_t silence_bytes = len - read;
         uint32_t silence_samples = bytes_to_samples(ap, silence_bytes);
-        // Insert silence. In theory, the inserted silent samples replace the
-        // missing real samples, which will arrive later, so they should be
-        // dropped to keep the latency minimal. However, this would cause very
-        // audible glitches, so let the clock compensation restore the target
-        // latency.
+        // Insert silence. In theory, the inserted silent replaces the missing
+        // samples, which will arrive later, so they should be dropped to keep
+        // the latency minimal. However, this would cause very audible
+        // glitches, so let the clock compensation restore the target latency.
+#ifndef SC_AUDIO_PLAYER_NDEBUG
         LOGD("[Audio] Buffer underflow, inserting silence: %" PRIu32 " samples",
              silence_samples);
+#endif
         memset(stream + read, 0, silence_bytes);
 
         if (ap->received) {
@@ -85,7 +135,7 @@ sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) {
 }
 
 static uint8_t *
-sc_audio_player_get_swr_buf(struct sc_audio_player *ap, uint32_t min_samples) {
+sc_audio_player_get_swr_buf(struct sc_audio_player *ap, size_t min_samples) {
     size_t min_buf_size = samples_to_bytes(ap, min_samples);
     if (min_buf_size > ap->swr_buf_alloc_size) {
         size_t new_size = min_buf_size + 4096;
@@ -110,8 +160,8 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink,
     SwrContext *swr_ctx = ap->swr_ctx;
 
     int64_t swr_delay = swr_get_delay(swr_ctx, ap->sample_rate);
-    // No need to av_rescale_rnd(), input and output sample rates are the same.
-    // Add more space (256) for clock compensation.
+    // No need to av_rescale_rnd(), input and output sample rates are the same
+    // Add more space (256) for clock compensation
     int dst_nb_samples = swr_delay + frame->nb_samples + 256;
 
     uint8_t *swr_buf = sc_audio_player_get_swr_buf(ap, dst_nb_samples);
@@ -155,41 +205,18 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink,
         size_t write_avail =
             sc_bytebuf_write_available(&ap->buf) / align * align;
         if (swr_buf_size > write_avail) {
-            // Entering this branch is very unlikely, the ring-buffer (bytebuf)
-            // is allocated with a size sufficient to store 1 second more than
-            // the target buffering. If this happens, though, we have to skip
-            // old samples.
+            // Skip old samples
             size_t cap = sc_bytebuf_capacity(&ap->buf) / align * align;
             if (swr_buf_size > cap) {
-                // Very very unlikely: a single resampled frame should never
-                // exceed the ring-buffer size (or something is very wrong).
                 // Ignore the first bytes in swr_buf
                 swr_buf += swr_buf_size - cap;
                 swr_buf_size = cap;
-                // This change in samples_written will impact the
-                // instant_compensation below
-                samples_written -= bytes_to_samples(ap, swr_buf_size - cap);
             }
-
-            assert(swr_buf_size >= write_avail);
-            if (swr_buf_size > write_avail) {
+            assert(swr_buf_size > write_avail);
+            if (swr_buf_size - write_avail > 0) {
                 sc_bytebuf_skip(&ap->buf, swr_buf_size - write_avail);
-                uint32_t skip_samples =
-                    bytes_to_samples(ap, swr_buf_size - write_avail);
-                assert(buffered_samples >= skip_samples);
-                buffered_samples -= skip_samples;
-                if (ap->played) {
-                    // Dropping input samples instantly decreases buffering
-                    ap->avg_buffering.avg -= skip_samples;
-                }
             }
-
-            // It should remain exactly the expected size to write the new
-            // samples.
-            assert((sc_bytebuf_write_available(&ap->buf) / align * align)
-                    == swr_buf_size);
         }
-
         sc_bytebuf_write(&ap->buf, swr_buf, swr_buf_size);
     }
 
@@ -268,9 +295,9 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink,
             // Limit compensation rate to 2%
             int abs_max_diff = distance / 50;
             diff = CLAMP(diff, -abs_max_diff, abs_max_diff);
-            LOGV("[Audio] Buffering: target=%" PRIu32 " avg=%f cur=%" PRIu32
-                 " compensation=%d", ap->target_buffering, avg,
-                 buffered_samples, diff);
+#ifndef SC_AUDIO_PLAYER_NDEBUG
+            LOGD("[Audio] Average buffering=%f, compensation %d", avg, diff);
+#endif
             int ret = swr_set_compensation(swr_ctx, diff, distance);
             if (ret < 0) {
                 LOGW("Resampling compensation failed: %d", ret);

app/src/cli.c

@@ -119,7 +119,7 @@ static const struct sc_option options[] = {
         .argdesc = "value",
         .text = "Encode the audio at the given bit-rate, expressed in bits/s. "
                 "Unit suffixes are supported: 'K' (x1000) and 'M' (x1000000).\n"
-                "Default is 128K (128000).",
+                "Default is 196K (196000).",
     },
     {
         .longopt_id = OPT_AUDIO_BUFFER,
@@ -127,7 +127,7 @@ static const struct sc_option options[] = {
         .argdesc = "ms",
         .text = "Configure the audio buffering delay (in milliseconds).\n"
                 "Lower values decrease the latency, but increase the "
-                "likelyhood of buffer underrun (causing audio glitches).\n"
+                "likelyhood of buffer underruns (causing audio glitches).\n"
                 "Default is 50.",
     },
     {

app/src/demuxer.c

@@ -200,7 +200,7 @@ run_demuxer(void *data) {
         ok = sc_packet_source_sinks_push(&demuxer->packet_source, packet);
         av_packet_unref(packet);
         if (!ok) {
-            // The sink already logged its concrete error
+            LOGE("Demuxer '%s': could not process packet", demuxer->name);
             break;
         }
     }

server/src/main/java/com/genymobile/scrcpy/AudioCapture.java

@@ -129,8 +129,8 @@ public final class AudioCapture {
             pts = nextPts;
         }
 
-        long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
-        nextPts = pts + durationUs;
+        long durationMs = r * 1000 / CHANNELS / SAMPLE_RATE;
+        nextPts = pts + durationMs;
 
         if (previousPts != 0 && pts < previousPts) {
             // Audio PTS may come from two sources:

server/src/main/java/com/genymobile/scrcpy/Options.java

@@ -13,7 +13,7 @@ public class Options {
     private VideoCodec videoCodec = VideoCodec.H264;
     private AudioCodec audioCodec = AudioCodec.OPUS;
     private int videoBitRate = 8000000;
-    private int audioBitRate = 128000;
+    private int audioBitRate = 196000;
     private int maxFps;
     private int lockVideoOrientation = -1;
     private boolean tunnelForward;