Fix audio PTS when not monotonically increasing

Some decoders fail to guarantee that PTS must be monotonically
increasing. Fix the PTS when they're not.

Fixes #4054 <https://github.com/Genymobile/scrcpy/issues/4054>

app/src/recorder.c

@@ -96,29 +96,23 @@ sc_recorder_rescale_packet(AVStream *stream, AVPacket *packet) {
 }
 
 static bool
-sc_recorder_write_stream(struct sc_recorder *recorder,
+sc_recorder_write_stream(struct sc_recorder *recorder, int stream_index,
-                         struct sc_recorder_stream *st, AVPacket *packet) {
+                         AVPacket *packet) {
-    AVStream *stream = recorder->ctx->streams[st->index];
+    AVStream *stream = recorder->ctx->streams[stream_index];
     sc_recorder_rescale_packet(stream, packet);
-    if (st->last_pts != AV_NOPTS_VALUE && packet->pts <= st->last_pts) {
-        LOGW("Fixing PTS non monotonically increasing "
-             "(%" PRIi64 " >= %" PRIi64 ")", st->last_pts, packet->pts);
-        packet->pts = ++st->last_pts;
-        packet->dts = packet->pts;
-    } else {
-        st->last_pts = packet->pts;
-    }
     return av_interleaved_write_frame(recorder->ctx, packet) >= 0;
 }
 
 static inline bool
 sc_recorder_write_video(struct sc_recorder *recorder, AVPacket *packet) {
-    return sc_recorder_write_stream(recorder, &recorder->video_stream, packet);
+    return sc_recorder_write_stream(recorder, recorder->video_stream_index,
+                                    packet);
 }
 
 static inline bool
 sc_recorder_write_audio(struct sc_recorder *recorder, AVPacket *packet) {
-    return sc_recorder_write_stream(recorder, &recorder->audio_stream, packet);
+    return sc_recorder_write_stream(recorder, recorder->audio_stream_index,
+                                    packet);
 }
 
 static bool
@@ -184,11 +178,10 @@ static bool
 sc_recorder_process_header(struct sc_recorder *recorder) {
     sc_mutex_lock(&recorder->mutex);
 
-    while (!recorder->stopped &&
+    while (!recorder->stopped && (!recorder->video_init
-              ((recorder->video && !recorder->video_init)
+                               || !recorder->audio_init
-            || (recorder->audio && !recorder->audio_init)
+                               || sc_recorder_has_empty_queues(recorder))) {
-            || sc_recorder_has_empty_queues(recorder))) {
+        sc_cond_wait(&recorder->stream_cond, &recorder->mutex);
-        sc_cond_wait(&recorder->cond, &recorder->mutex);
     }
 
     if (recorder->video && sc_vecdeque_is_empty(&recorder->video_queue)) {
@@ -221,9 +214,9 @@ sc_recorder_process_header(struct sc_recorder *recorder) {
             goto end;
         }
 
-        assert(recorder->video_stream.index >= 0);
+        assert(recorder->video_stream_index >= 0);
         AVStream *video_stream =
-            recorder->ctx->streams[recorder->video_stream.index];
+            recorder->ctx->streams[recorder->video_stream_index];
         bool ok = sc_recorder_set_extradata(video_stream, video_pkt);
         if (!ok) {
             goto end;
@@ -236,9 +229,9 @@ sc_recorder_process_header(struct sc_recorder *recorder) {
             goto end;
         }
 
-        assert(recorder->audio_stream.index >= 0);
+        assert(recorder->audio_stream_index >= 0);
         AVStream *audio_stream =
-            recorder->ctx->streams[recorder->audio_stream.index];
+            recorder->ctx->streams[recorder->audio_stream_index];
         bool ok = sc_recorder_set_extradata(audio_stream, audio_pkt);
         if (!ok) {
             goto end;
@@ -296,7 +289,7 @@ sc_recorder_process_packets(struct sc_recorder *recorder) {
                 // A new packet may be assigned to audio_pkt and be processed
                 break;
             }
-            sc_cond_wait(&recorder->cond, &recorder->mutex);
+            sc_cond_wait(&recorder->queue_cond, &recorder->mutex);
         }
 
         // If stopped is set, continue to process the remaining events (to
@@ -511,10 +504,10 @@ sc_recorder_video_packet_sink_open(struct sc_packet_sink *sink,
         return false;
     }
 
-    recorder->video_stream.index = stream->index;
+    recorder->video_stream_index = stream->index;
 
     recorder->video_init = true;
-    sc_cond_signal(&recorder->cond);
+    sc_cond_signal(&recorder->stream_cond);
     sc_mutex_unlock(&recorder->mutex);
 
     return true;
@@ -529,7 +522,7 @@ sc_recorder_video_packet_sink_close(struct sc_packet_sink *sink) {
     sc_mutex_lock(&recorder->mutex);
     // EOS also stops the recorder
     recorder->stopped = true;
-    sc_cond_signal(&recorder->cond);
+    sc_cond_signal(&recorder->queue_cond);
     sc_mutex_unlock(&recorder->mutex);
 }
 
@@ -555,7 +548,7 @@ sc_recorder_video_packet_sink_push(struct sc_packet_sink *sink,
         return false;
     }
 
-    rec->stream_index = recorder->video_stream.index;
+    rec->stream_index = recorder->video_stream_index;
 
     bool ok = sc_vecdeque_push(&recorder->video_queue, rec);
     if (!ok) {
@@ -564,7 +557,7 @@ sc_recorder_video_packet_sink_push(struct sc_packet_sink *sink,
         return false;
     }
 
-    sc_cond_signal(&recorder->cond);
+    sc_cond_signal(&recorder->queue_cond);
 
     sc_mutex_unlock(&recorder->mutex);
     return true;
@@ -592,10 +585,10 @@ sc_recorder_audio_packet_sink_open(struct sc_packet_sink *sink,
         return false;
     }
 
-    recorder->audio_stream.index = stream->index;
+    recorder->audio_stream_index = stream->index;
 
     recorder->audio_init = true;
-    sc_cond_signal(&recorder->cond);
+    sc_cond_signal(&recorder->stream_cond);
     sc_mutex_unlock(&recorder->mutex);
 
     return true;
@@ -611,7 +604,7 @@ sc_recorder_audio_packet_sink_close(struct sc_packet_sink *sink) {
     sc_mutex_lock(&recorder->mutex);
     // EOS also stops the recorder
     recorder->stopped = true;
-    sc_cond_signal(&recorder->cond);
+    sc_cond_signal(&recorder->queue_cond);
     sc_mutex_unlock(&recorder->mutex);
 }
 
@@ -638,7 +631,7 @@ sc_recorder_audio_packet_sink_push(struct sc_packet_sink *sink,
         return false;
     }
 
-    rec->stream_index = recorder->audio_stream.index;
+    rec->stream_index = recorder->audio_stream_index;
 
     bool ok = sc_vecdeque_push(&recorder->audio_queue, rec);
     if (!ok) {
@@ -647,7 +640,7 @@ sc_recorder_audio_packet_sink_push(struct sc_packet_sink *sink,
         return false;
     }
 
-    sc_cond_signal(&recorder->cond);
+    sc_cond_signal(&recorder->queue_cond);
 
     sc_mutex_unlock(&recorder->mutex);
     return true;
@@ -665,16 +658,10 @@ sc_recorder_audio_packet_sink_disable(struct sc_packet_sink *sink) {
     sc_mutex_lock(&recorder->mutex);
     recorder->audio = false;
     recorder->audio_init = true;
-    sc_cond_signal(&recorder->cond);
+    sc_cond_signal(&recorder->stream_cond);
     sc_mutex_unlock(&recorder->mutex);
 }
 
-static void
-sc_recorder_stream_init(struct sc_recorder_stream *stream) {
-    stream->index = -1;
-    stream->last_pts = AV_NOPTS_VALUE;
-}
-
 bool
 sc_recorder_init(struct sc_recorder *recorder, const char *filename,
                  enum sc_record_format format, bool video, bool audio,
@@ -690,11 +677,16 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename,
         goto error_free_filename;
     }
 
-    ok = sc_cond_init(&recorder->cond);
+    ok = sc_cond_init(&recorder->queue_cond);
     if (!ok) {
         goto error_mutex_destroy;
     }
 
+    ok = sc_cond_init(&recorder->stream_cond);
+    if (!ok) {
+        goto error_queue_cond_destroy;
+    }
+
     assert(video || audio);
     recorder->video = video;
     recorder->audio = audio;
@@ -706,8 +698,8 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename,
     recorder->video_init = false;
     recorder->audio_init = false;
 
-    sc_recorder_stream_init(&recorder->video_stream);
+    recorder->video_stream_index = -1;
-    sc_recorder_stream_init(&recorder->audio_stream);
+    recorder->audio_stream_index = -1;
 
     recorder->format = format;
 
@@ -738,6 +730,8 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename,
 
     return true;
 
+error_queue_cond_destroy:
+    sc_cond_destroy(&recorder->queue_cond);
 error_mutex_destroy:
     sc_mutex_destroy(&recorder->mutex);
 error_free_filename:
@@ -762,7 +756,8 @@ void
 sc_recorder_stop(struct sc_recorder *recorder) {
     sc_mutex_lock(&recorder->mutex);
     recorder->stopped = true;
-    sc_cond_signal(&recorder->cond);
+    sc_cond_signal(&recorder->queue_cond);
+    sc_cond_signal(&recorder->stream_cond);
     sc_mutex_unlock(&recorder->mutex);
 }
 
@@ -773,7 +768,8 @@ sc_recorder_join(struct sc_recorder *recorder) {
 
 void
 sc_recorder_destroy(struct sc_recorder *recorder) {
-    sc_cond_destroy(&recorder->cond);
+    sc_cond_destroy(&recorder->stream_cond);
+    sc_cond_destroy(&recorder->queue_cond);
     sc_mutex_destroy(&recorder->mutex);
     free(recorder->filename);
 }

app/src/recorder.h

@@ -14,11 +14,6 @@
 
 struct sc_recorder_queue SC_VECDEQUE(AVPacket *);
 
-struct sc_recorder_stream {
-    int index;
-    int64_t last_pts;
-};
-
 struct sc_recorder {
     struct sc_packet_sink video_packet_sink;
     struct sc_packet_sink audio_packet_sink;
@@ -40,18 +35,19 @@ struct sc_recorder {
 
     sc_thread thread;
     sc_mutex mutex;
-    sc_cond cond;
+    sc_cond queue_cond;
     // set on sc_recorder_stop(), packet_sink close or recording failure
     bool stopped;
     struct sc_recorder_queue video_queue;
     struct sc_recorder_queue audio_queue;
 
     // wake up the recorder thread once the video or audio codec is known
+    sc_cond stream_cond;
     bool video_init;
     bool audio_init;
 
-    struct sc_recorder_stream video_stream;
+    int video_stream_index;
-    struct sc_recorder_stream audio_stream;
+    int audio_stream_index;
 
     const struct sc_recorder_callbacks *cbs;
     void *cbs_userdata;

server/src/main/java/com/genymobile/scrcpy/AudioCapture.java

@@ -163,7 +163,7 @@ public final class AudioCapture {
             //  - an estimation from the previous PTS and the packet size as a fallback.
             //
             // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
-            pts = previousPts + 1;
+            pts = previousPts;
         }
         previousPts = pts;
 

server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java

@@ -105,8 +105,17 @@ public final class AudioEncoder implements AsyncProcessor {
     private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException {
         streamer.writeAudioHeader();
 
+        long lastPts = 0;
         while (!Thread.currentThread().isInterrupted()) {
             OutputTask task = outputTasks.take();
+
+            if (task.bufferInfo.presentationTimeUs < lastPts) {
+                // Fix PTS if not monotonically increasing
+                task.bufferInfo.presentationTimeUs = lastPts;
+            } else {
+                lastPts = task.bufferInfo.presentationTimeUs;
+            }
+
             ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index);
             try {
                 streamer.writePacket(buffer, task.bufferInfo);