AudioEncoder WIP

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>

app/src/recorder.c

@@ -11,6 +11,8 @@
 /** Downcast packet_sink to recorder */
 #define DOWNCAST(SINK) container_of(SINK, struct sc_recorder, packet_sink)
 
+#define SC_PTS_ORIGIN_NONE UINT64_C(-1)
+
 static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us
 
 static const AVOutputFormat *
@@ -128,6 +130,8 @@ sc_recorder_write(struct sc_recorder *recorder, AVPacket *packet) {
         return true;
     }
 
+    LOGI("==== %" PRIu64, packet->pts);
+
     sc_recorder_rescale_packet(recorder, packet);
     return av_write_frame(recorder->ctx, packet) >= 0;
 }
@@ -169,6 +173,18 @@ run_recorder(void *data) {
 
         sc_mutex_unlock(&recorder->mutex);
 
+        if (recorder->pts_origin == SC_PTS_ORIGIN_NONE
+                && rec->packet->pts != AV_NOPTS_VALUE) {
+            // First PTS received
+            recorder->pts_origin = rec->packet->pts;
+        }
+
+        if (rec->packet->pts != AV_NOPTS_VALUE) {
+            // Set PTS relatve to the origin
+            rec->packet->pts -= recorder->pts_origin;
+            rec->packet->dts = rec->packet->pts;
+        }
+
         // recorder->previous is only written from this thread, no need to lock
         struct sc_record_packet *previous = recorder->previous;
         recorder->previous = rec;
@@ -243,6 +259,7 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) {
     recorder->failed = false;
     recorder->header_written = false;
     recorder->previous = NULL;
+    recorder->pts_origin = SC_PTS_ORIGIN_NONE;
 
     const char *format_name = sc_recorder_get_format_name(recorder->format);
     assert(format_name);

app/src/recorder.h

@@ -28,6 +28,8 @@ struct sc_recorder {
     struct sc_size declared_frame_size;
     bool header_written;
 
+    uint64_t pts_origin;
+
     sc_thread thread;
     sc_mutex mutex;
     sc_cond queue_cond;

server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java

@@ -0,0 +1,196 @@
+package com.genymobile.scrcpy;
+
+import com.genymobile.scrcpy.wrappers.ServiceManager;
+
+import android.annotation.SuppressLint;
+import android.annotation.TargetApi;
+import android.content.ComponentName;
+import android.content.Intent;
+import android.media.AudioFormat;
+import android.media.AudioRecord;
+import android.media.AudioTimestamp;
+import android.media.MediaCodec;
+import android.media.MediaFormat;
+import android.media.MediaRecorder;
+import android.os.Build;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.SystemClock;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public final class AudioEncoder {
+
+    private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS;
+    private static final int SAMPLE_RATE = 48000;
+    private static final int CHANNELS = 2;
+    private static final int BIT_RATE = 128000;
+
+    private static int BUFFER_MS = 15; // milliseconds
+    private static final int BUFFER_SIZE = SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000;
+
+    private AudioRecord recorder;
+    private MediaCodec mediaCodec;
+    private HandlerThread thread;
+    private final AtomicBoolean interrupted = new AtomicBoolean();
+    private final Semaphore endSemaphore = new Semaphore(0); // blocks until encoding is ended
+
+    private static AudioFormat createAudioFormat() {
+        AudioFormat.Builder builder = new AudioFormat.Builder();
+        builder.setEncoding(AudioFormat.ENCODING_PCM_16BIT);
+        builder.setSampleRate(SAMPLE_RATE);
+        builder.setChannelMask(CHANNELS == 2 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO);
+        return builder.build();
+    }
+
+    @TargetApi(Build.VERSION_CODES.M)
+    @SuppressLint({"WrongConstant", "MissingPermission"})
+    private static AudioRecord createAudioRecord() {
+        AudioRecord.Builder builder = new AudioRecord.Builder();
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+            // On older APIs, Workarounds.fillAppInfo() must be called beforehand
+            builder.setContext(FakeContext.get());
+        }
+        builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
+        builder.setAudioFormat(createAudioFormat());
+        builder.setBufferSizeInBytes(1024 * 1024);
+        return builder.build();
+    }
+
+    private static MediaFormat createFormat() {
+        MediaFormat format = new MediaFormat();
+        format.setString(MediaFormat.KEY_MIME, MIMETYPE);
+        format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
+        format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS);
+        format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE);
+        return format;
+    }
+
+
+
+    @TargetApi(Build.VERSION_CODES.M)
+    public void start() throws IOException {
+        mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException
+
+        recorder = createAudioRecord();
+
+        MediaFormat format = createFormat();
+        mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+
+        recorder.startRecording();
+
+        thread = new HandlerThread("AudioEncoder");
+        thread.start();
+
+        class AudioEncoderCallbacks extends MediaCodec.Callback {
+
+            private final AudioTimestamp timestamp = new AudioTimestamp();
+            private long nextPts;
+            private boolean eofSignaled;
+            private boolean ended;
+
+            private void notifyEnded() {
+                assert(!ended);
+                ended = true;
+                endSemaphore.release();
+            }
+
+            @TargetApi(Build.VERSION_CODES.N)
+            @Override
+            public void onInputBufferAvailable(MediaCodec codec, int index) {
+                if (eofSignaled) {
+                    return;
+                }
+
+                ByteBuffer inputBuffer = codec.getInputBuffer(index);
+                int r = recorder.read(inputBuffer, BUFFER_SIZE);
+
+                long pts;
+
+                int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
+                if (ret == AudioRecord.SUCCESS) {
+                    pts = timestamp.nanoTime / 1000;
+                } else {
+                    if (nextPts == 0) {
+                        Ln.w("Could not get any audio timestamp");
+                    }
+                    // compute from previous timestamp and packet size
+                    pts = nextPts;
+                }
+
+                long durationMs = r * 1000 / CHANNELS / SAMPLE_RATE;
+                nextPts = pts + durationMs;
+
+                int flags = 0;
+                if (interrupted.get()) {
+                    flags = flags | MediaCodec.BUFFER_FLAG_END_OF_STREAM;
+                    eofSignaled = true;
+                }
+
+                codec.queueInputBuffer(index, 0, r, pts, flags);
+            }
+
+            @Override
+            public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) {
+                if (ended) {
+                    return;
+                }
+
+                ByteBuffer codecBuffer = codec.getOutputBuffer(index);
+                try {
+                    boolean isConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0;
+                    long pts = bufferInfo.presentationTimeUs;
+                    Ln.i("Audio packet: pts=" + pts  + " " + codecBuffer.remaining() + " bytes");
+                } finally {
+                    codec.releaseOutputBuffer(index, false);
+                }
+
+                boolean eof = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
+                if (eof) {
+                    notifyEnded();
+                }
+            }
+
+            @Override
+            public void onError(MediaCodec codec, MediaCodec.CodecException e) {
+                Ln.e("MediaCodec error", e);
+                if (!ended) {
+                    notifyEnded();
+                }
+            }
+
+            @Override
+            public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
+                // ignore
+            }
+        }
+
+        mediaCodec.setCallback(new AudioEncoderCallbacks(), new Handler(thread.getLooper()));
+        mediaCodec.start();
+    }
+
+    private void waitEnded() {
+        try {
+            endSemaphore.acquire();
+        } catch (InterruptedException e) {
+            // ignore
+        }
+    }
+
+    public void stop() {
+        Ln.i("==== STOP");
+        if (thread != null) {
+            interrupted.set(true);
+            waitEnded();
+            thread.interrupt();
+            thread = null;
+            mediaCodec.stop();
+            mediaCodec.release();
+            recorder.stop();
+            Ln.i("==== STOPPED");
+        }
+    }
+}

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -24,6 +24,8 @@ public class Controller {
 
     private static final ScheduledExecutorService EXECUTOR = Executors.newSingleThreadScheduledExecutor();
 
+    private Thread thread;
+
     private final Device device;
     private final DesktopConnection connection;
     private final DeviceMessageSender sender;
@@ -62,7 +64,7 @@ public class Controller {
         }
     }
 
-    public void control() throws IOException {
+    private void control() throws IOException {
         // on start, power on the device
         if (powerOn && !Device.isScreenOn()) {
             device.pressReleaseKeycode(KeyEvent.KEYCODE_POWER, Device.INJECT_MODE_ASYNC);
@@ -82,6 +84,27 @@ public class Controller {
         }
     }
 
+    public void start() {
+        thread = new Thread(() -> {
+            try {
+                control();
+            } catch (IOException e) {
+                // this is expected on close
+                Ln.d("Controller stopped");
+            }
+        });
+        thread.start();
+        sender.start();
+    }
+
+    public void stop() {
+        if (thread != null) {
+            thread.interrupt();
+            thread = null;
+        }
+        sender.stop();
+    }
+
     public DeviceMessageSender getSender() {
         return sender;
     }

server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java

@@ -6,6 +6,8 @@ public final class DeviceMessageSender {
 
     private final DesktopConnection connection;
 
+    private Thread thread;
+
     private String clipboardText;
 
     private long ack;
@@ -24,7 +26,7 @@ public final class DeviceMessageSender {
         notify();
     }
 
-    public void loop() throws IOException, InterruptedException {
+    private void loop() throws IOException, InterruptedException {
         while (!Thread.currentThread().isInterrupted()) {
             String text;
             long sequence;
@@ -49,4 +51,22 @@ public final class DeviceMessageSender {
             }
         }
     }
+    public void start() {
+        thread = new Thread(() -> {
+            try {
+                loop();
+            } catch (IOException | InterruptedException e) {
+                // this is expected on close
+                Ln.d("Device message sender stopped");
+            }
+        });
+        thread.start();
+    }
+
+    public void stop() {
+        if (thread != null) {
+            thread.interrupt();
+            thread = null;
+        }
+    }
 }

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -42,7 +42,6 @@ public class ScreenEncoder implements Device.RotationListener {
     private final int maxFps;
     private final boolean sendFrameMeta;
     private final boolean downsizeOnError;
-    private long ptsOrigin;
 
     private boolean firstFrameSent;
     private int consecutiveErrors;
@@ -207,10 +206,7 @@ public class ScreenEncoder implements Device.RotationListener {
         if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
             pts = PACKET_FLAG_CONFIG; // non-media data packet
         } else {
-            if (ptsOrigin == 0) {
+            pts = bufferInfo.presentationTimeUs;
-                ptsOrigin = bufferInfo.presentationTimeUs;
-            }
-            pts = bufferInfo.presentationTimeUs - ptsOrigin;
             if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) {
                 pts |= PACKET_FLAG_KEY_FRAME;
             }

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -4,6 +4,7 @@ import android.graphics.Rect;
 import android.media.MediaCodecInfo;
 import android.os.BatteryManager;
 import android.os.Build;
+import android.provider.MediaStore;
 
 import java.io.IOException;
 import java.util.List;
@@ -66,18 +67,27 @@ public final class Server {
 
         Thread initThread = startInitThread(options);
 
-        Workarounds.prepareMainLooper();
-        if (Build.BRAND.equalsIgnoreCase("meizu")) {
-            // <https://github.com/Genymobile/scrcpy/issues/240>
-            // <https://github.com/Genymobile/scrcpy/issues/2656>
-            Workarounds.fillAppInfo();
-        }
-
         int uid = options.getUid();
         boolean tunnelForward = options.isTunnelForward();
         boolean control = options.getControl();
+        boolean audio = Build.VERSION.SDK_INT >= Build.VERSION_CODES.R; // TODO option
         boolean sendDummyByte = options.getSendDummyByte();
 
+        Workarounds.prepareMainLooper();
+
+        // <https://github.com/Genymobile/scrcpy/issues/240>
+        // <https://github.com/Genymobile/scrcpy/issues/2656>
+        boolean mustFillAppInfo = Build.BRAND.equalsIgnoreCase("meizu");
+
+        // Before Android 11, audio is not supported.
+        // Since Android 12, we can properly set a context on the AudioRecord.
+        // Only on Android 11 we must fill app info for the AudioRecord to work.
+        mustFillAppInfo |= audio && Build.VERSION.SDK_INT == Build.VERSION_CODES.R;
+
+        if (mustFillAppInfo) {
+            Workarounds.fillAppInfo();
+        }
+
         try (DesktopConnection connection = DesktopConnection.open(uid, tunnelForward, control, sendDummyByte)) {
             if (options.getSendDeviceMeta()) {
                 Size videoSize = device.getScreenInfo().getVideoSize();
@@ -86,16 +96,19 @@ public final class Server {
             ScreenEncoder screenEncoder = new ScreenEncoder(options.getSendFrameMeta(), options.getBitRate(), options.getMaxFps(), codecOptions,
                     options.getEncoderName(), options.getDownsizeOnError());
 
-            Thread controllerThread = null;
+            Controller controller = null;
-            Thread deviceMessageSenderThread = null;
             if (control) {
-                final Controller controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn());
+                controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn());
+                controller.start();
 
-                // asynchronous
+                final Controller controllerRef = controller;
-                controllerThread = startController(controller);
+                device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text));
-                deviceMessageSenderThread = startDeviceMessageSender(controller.getSender());
+            }
 
-                device.setClipboardListener(text -> controller.getSender().pushClipboardText(text));
+            AudioEncoder audioEncoder = null;
+            if (audio) {
+                audioEncoder = new AudioEncoder();
+                audioEncoder.start();
             }
 
             try {
@@ -106,11 +119,11 @@ public final class Server {
                 Ln.d("Screen streaming stopped");
             } finally {
                 initThread.interrupt();
-                if (controllerThread != null) {
+                if (controller != null) {
-                    controllerThread.interrupt();
+                    controller.stop();
                 }
-                if (deviceMessageSenderThread != null) {
+                if (audioEncoder != null) {
-                    deviceMessageSenderThread.interrupt();
+                    audioEncoder.stop();
                 }
             }
         }
@@ -122,32 +135,6 @@ public final class Server {
         return thread;
     }
 
-    private static Thread startController(final Controller controller) {
-        Thread thread = new Thread(() -> {
-            try {
-                controller.control();
-            } catch (IOException e) {
-                // this is expected on close
-                Ln.d("Controller stopped");
-            }
-        });
-        thread.start();
-        return thread;
-    }
-
-    private static Thread startDeviceMessageSender(final DeviceMessageSender sender) {
-        Thread thread = new Thread(() -> {
-            try {
-                sender.loop();
-            } catch (IOException | InterruptedException e) {
-                // this is expected on close
-                Ln.d("Device message sender stopped");
-            }
-        });
-        thread.start();
-        return thread;
-    }
-
     private static Options createOptions(String... args) {
         if (args.length < 1) {
             throw new IllegalArgumentException("Missing client version");

server/src/main/java/com/genymobile/scrcpy/Workarounds.java

@@ -3,6 +3,7 @@ package com.genymobile.scrcpy;
 import android.annotation.SuppressLint;
 import android.app.Application;
 import android.app.Instrumentation;
+import android.content.ContextWrapper;
 import android.content.pm.ApplicationInfo;
 import android.os.Looper;
 
@@ -60,7 +61,10 @@ public final class Workarounds {
             mBoundApplicationField.setAccessible(true);
             mBoundApplicationField.set(activityThread, appBindData);
 
-            Application app = Instrumentation.newApplication(Application.class, FakeContext.get());
+            Application app = Application.class.newInstance();
+            Field baseField = ContextWrapper.class.getDeclaredField("mBase");
+            baseField.setAccessible(true);
+            baseField.set(app, FakeContext.get());
 
             // activityThread.mInitialApplication = app;
             Field mInitialApplicationField = activityThreadClass.getDeclaredField("mInitialApplication");