Compute relative PTS on the client-side

The PTS received by MediaCodec are expressed relative to an arbitrary
clock origin. We consider the PTS of the first frame to be 0, and the
PTS of every other frame is relative to this first PTS (note that the
PTS is only used for recording, it is ignored for mirroring).

For simplicity, this relative PTS was computed on the server-side.

To prepare support for multiple stream (video and audio), send the
packet with its original PTS, and handle the PTS offset on the
client-side (by the recorder).

Since we can't know in advance which stream will produce the first
packet with the lowest PTS (a packet received later on one stream may
have a PTS lower than a packet received earlier on another stream),
computing the PTS on the server-side would require unnecessary waiting.

server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java

@@ -1,74 +0,0 @@
-package com.genymobile.scrcpy;
-
-import com.genymobile.scrcpy.wrappers.ServiceManager;
-
-import android.annotation.SuppressLint;
-import android.annotation.TargetApi;
-import android.content.ComponentName;
-import android.content.Intent;
-import android.media.AudioFormat;
-import android.media.AudioRecord;
-import android.media.MediaRecorder;
-import android.os.Build;
-import android.os.SystemClock;
-
-public final class AudioEncoder {
-
-    private static final int SAMPLE_RATE = 48000;
-    private static final int CHANNELS = 2;
-
-    private Thread thread;
-
-    private static AudioFormat createAudioFormat() {
-        AudioFormat.Builder builder = new AudioFormat.Builder();
-        builder.setEncoding(AudioFormat.ENCODING_PCM_16BIT);
-        builder.setSampleRate(SAMPLE_RATE);
-        builder.setChannelMask(CHANNELS == 2 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO);
-        return builder.build();
-    }
-
-    @TargetApi(Build.VERSION_CODES.M)
-    @SuppressLint({"WrongConstant", "MissingPermission"})
-    private static AudioRecord createAudioRecord() {
-        AudioRecord.Builder builder = new AudioRecord.Builder();
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
-            // On older APIs, Workarounds.fillAppInfo() must be called beforehand
-            builder.setContext(FakeContext.get());
-        }
-        builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
-        builder.setAudioFormat(createAudioFormat());
-        builder.setBufferSizeInBytes(1024 * 1024);
-        return builder.build();
-    }
-
-    public void start() {
-        AudioRecord recorder = createAudioRecord();
-
-        thread = new Thread(() -> {
-            recorder.startRecording();
-            try {
-                int BUFFER_MS = 15; // do not buffer more than BUFFER_MS milliseconds
-                byte[] buf = new byte[SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000];
-                while (!Thread.currentThread().isInterrupted()) {
-                    int r = recorder.read(buf, 0, buf.length);
-                    if (r > 0) {
-                        Ln.i("Audio captured: " + r + " bytes");
-                    }
-                    if (r < 0) {
-                        Ln.e("Audio capture error: " + r);
-                    }
-                }
-            } finally {
-                recorder.stop();
-            }
-        });
-        thread.start();
-    }
-
-    public void stop() {
-        if (thread != null) {
-            thread.interrupt();
-            thread = null;
-        }
-    }
-}

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -24,8 +24,6 @@ public class Controller {
 
     private static final ScheduledExecutorService EXECUTOR = Executors.newSingleThreadScheduledExecutor();
 
-    private Thread thread;
-
     private final Device device;
     private final DesktopConnection connection;
     private final DeviceMessageSender sender;
@@ -64,7 +62,7 @@ public class Controller {
         }
     }
 
-    private void control() throws IOException {
+    public void control() throws IOException {
         // on start, power on the device
         if (powerOn && !Device.isScreenOn()) {
             device.pressReleaseKeycode(KeyEvent.KEYCODE_POWER, Device.INJECT_MODE_ASYNC);
@@ -84,27 +82,6 @@ public class Controller {
         }
     }
 
-    public void start() {
-        thread = new Thread(() -> {
-            try {
-                control();
-            } catch (IOException e) {
-                // this is expected on close
-                Ln.d("Controller stopped");
-            }
-        });
-        thread.start();
-        sender.start();
-    }
-
-    public void stop() {
-        if (thread != null) {
-            thread.interrupt();
-            thread = null;
-        }
-        sender.stop();
-    }
-
     public DeviceMessageSender getSender() {
         return sender;
     }

server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java

@@ -6,8 +6,6 @@ public final class DeviceMessageSender {
 
     private final DesktopConnection connection;
 
-    private Thread thread;
-
     private String clipboardText;
 
     private long ack;
@@ -26,7 +24,7 @@ public final class DeviceMessageSender {
         notify();
     }
 
-    private void loop() throws IOException, InterruptedException {
+    public void loop() throws IOException, InterruptedException {
         while (!Thread.currentThread().isInterrupted()) {
             String text;
             long sequence;
@@ -51,22 +49,4 @@ public final class DeviceMessageSender {
             }
         }
     }
-    public void start() {
-        thread = new Thread(() -> {
-            try {
-                loop();
-            } catch (IOException | InterruptedException e) {
-                // this is expected on close
-                Ln.d("Device message sender stopped");
-            }
-        });
-        thread.start();
-    }
-
-    public void stop() {
-        if (thread != null) {
-            thread.interrupt();
-            thread = null;
-        }
-    }
 }

server/src/main/java/com/genymobile/scrcpy/FakeContext.java

@@ -1,40 +0,0 @@
-package com.genymobile.scrcpy;
-
-import android.annotation.TargetApi;
-import android.content.AttributionSource;
-import android.content.ContextWrapper;
-import android.os.Build;
-import android.os.Process;
-
-public final class FakeContext extends ContextWrapper {
-
-    public static final String PACKAGE_NAME = "com.android.shell";
-
-    private static final FakeContext INSTANCE = new FakeContext();
-
-    public static FakeContext get() {
-        return INSTANCE;
-    }
-
-    private FakeContext() {
-        super(null);
-    }
-
-    @Override
-    public String getPackageName() {
-        return PACKAGE_NAME;
-    }
-
-    @Override
-    public String getOpPackageName() {
-        return PACKAGE_NAME;
-    }
-
-    @TargetApi(Build.VERSION_CODES.S)
-    @Override
-    public AttributionSource getAttributionSource() {
-        AttributionSource.Builder builder = new AttributionSource.Builder(Process.SHELL_UID);
-        builder.setPackageName(PACKAGE_NAME);
-        return builder.build();
-    }
-}

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -66,6 +66,17 @@ public class ScreenEncoder implements Device.RotationListener {
     }
 
     public void streamScreen(Device device, FileDescriptor fd) throws IOException {
+        Workarounds.prepareMainLooper();
+        if (Build.BRAND.equalsIgnoreCase("meizu")) {
+            // <https://github.com/Genymobile/scrcpy/issues/240>
+            // <https://github.com/Genymobile/scrcpy/issues/2656>
+            Workarounds.fillAppInfo();
+        }
+
+        internalStreamScreen(device, fd);
+    }
+
+    private void internalStreamScreen(Device device, FileDescriptor fd) throws IOException {
         MediaCodec codec = createCodec(encoderName);
         MediaFormat format = createFormat(bitRate, maxFps, codecOptions);
         IBinder display = createDisplay();

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -4,7 +4,6 @@ import android.graphics.Rect;
 import android.media.MediaCodecInfo;
 import android.os.BatteryManager;
 import android.os.Build;
-import android.provider.MediaStore;
 
 import java.io.IOException;
 import java.util.List;
@@ -70,24 +69,8 @@ public final class Server {
         int uid = options.getUid();
         boolean tunnelForward = options.isTunnelForward();
         boolean control = options.getControl();
-        boolean audio = Build.VERSION.SDK_INT >= Build.VERSION_CODES.R; // TODO option
         boolean sendDummyByte = options.getSendDummyByte();
 
-        Workarounds.prepareMainLooper();
-
-        // <https://github.com/Genymobile/scrcpy/issues/240>
-        // <https://github.com/Genymobile/scrcpy/issues/2656>
-        boolean mustFillAppInfo = Build.BRAND.equalsIgnoreCase("meizu");
-
-        // Before Android 11, audio is not supported.
-        // Since Android 12, we can properly set a context on the AudioRecord.
-        // Only on Android 11 we must fill app info for the AudioRecord to work.
-        mustFillAppInfo |= audio && Build.VERSION.SDK_INT == Build.VERSION_CODES.R;
-
-        if (mustFillAppInfo) {
-            Workarounds.fillAppInfo();
-        }
-
         try (DesktopConnection connection = DesktopConnection.open(uid, tunnelForward, control, sendDummyByte)) {
             if (options.getSendDeviceMeta()) {
                 Size videoSize = device.getScreenInfo().getVideoSize();
@@ -96,19 +79,16 @@ public final class Server {
             ScreenEncoder screenEncoder = new ScreenEncoder(options.getSendFrameMeta(), options.getBitRate(), options.getMaxFps(), codecOptions,
                     options.getEncoderName(), options.getDownsizeOnError());
 
-            Controller controller = null;
+            Thread controllerThread = null;
+            Thread deviceMessageSenderThread = null;
             if (control) {
-                controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn());
-                controller.start();
+                final Controller controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn());
 
-                final Controller controllerRef = controller;
-                device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text));
-            }
+                // asynchronous
+                controllerThread = startController(controller);
+                deviceMessageSenderThread = startDeviceMessageSender(controller.getSender());
 
-            AudioEncoder audioEncoder = null;
-            if (audio) {
-                audioEncoder = new AudioEncoder();
-                audioEncoder.start();
+                device.setClipboardListener(text -> controller.getSender().pushClipboardText(text));
             }
 
             try {
@@ -119,11 +99,11 @@ public final class Server {
                 Ln.d("Screen streaming stopped");
             } finally {
                 initThread.interrupt();
-                if (controller != null) {
-                    controller.stop();
+                if (controllerThread != null) {
+                    controllerThread.interrupt();
                 }
-                if (audioEncoder != null) {
-                    audioEncoder.stop();
+                if (deviceMessageSenderThread != null) {
+                    deviceMessageSenderThread.interrupt();
                 }
             }
         }
@@ -135,6 +115,32 @@ public final class Server {
         return thread;
     }
 
+    private static Thread startController(final Controller controller) {
+        Thread thread = new Thread(() -> {
+            try {
+                controller.control();
+            } catch (IOException e) {
+                // this is expected on close
+                Ln.d("Controller stopped");
+            }
+        });
+        thread.start();
+        return thread;
+    }
+
+    private static Thread startDeviceMessageSender(final DeviceMessageSender sender) {
+        Thread thread = new Thread(() -> {
+            try {
+                sender.loop();
+            } catch (IOException | InterruptedException e) {
+                // this is expected on close
+                Ln.d("Device message sender stopped");
+            }
+        });
+        thread.start();
+        return thread;
+    }
+
     private static Options createOptions(String... args) {
         if (args.length < 1) {
             throw new IllegalArgumentException("Missing client version");

server/src/main/java/com/genymobile/scrcpy/Workarounds.java

@@ -3,12 +3,13 @@ package com.genymobile.scrcpy;
 import android.annotation.SuppressLint;
 import android.app.Application;
 import android.app.Instrumentation;
-import android.content.ContextWrapper;
+import android.content.Context;
 import android.content.pm.ApplicationInfo;
 import android.os.Looper;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 
 public final class Workarounds {
     private Workarounds() {
@@ -49,7 +50,7 @@ public final class Workarounds {
             Object appBindData = appBindDataConstructor.newInstance();
 
             ApplicationInfo applicationInfo = new ApplicationInfo();
-            applicationInfo.packageName = FakeContext.PACKAGE_NAME;
+            applicationInfo.packageName = "com.genymobile.scrcpy";
 
             // appBindData.appInfo = applicationInfo;
             Field appInfoField = appBindDataClass.getDeclaredField("appInfo");
@@ -61,10 +62,11 @@ public final class Workarounds {
             mBoundApplicationField.setAccessible(true);
             mBoundApplicationField.set(activityThread, appBindData);
 
-            Application app = Application.class.newInstance();
-            Field baseField = ContextWrapper.class.getDeclaredField("mBase");
-            baseField.setAccessible(true);
-            baseField.set(app, FakeContext.get());
+            // Context ctx = activityThread.getSystemContext();
+            Method getSystemContextMethod = activityThreadClass.getDeclaredMethod("getSystemContext");
+            Context ctx = (Context) getSystemContextMethod.invoke(activityThread);
+
+            Application app = Instrumentation.newApplication(Application.class, ctx);
 
             // activityThread.mInitialApplication = app;
             Field mInitialApplicationField = activityThreadClass.getDeclaredField("mInitialApplication");

server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java

@@ -5,7 +5,6 @@ import com.genymobile.scrcpy.Ln;
 import android.os.Binder;
 import android.os.IBinder;
 import android.os.IInterface;
-import android.os.Process;
 
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
@@ -49,10 +48,10 @@ public class ActivityManager {
             Object[] args;
             if (getContentProviderExternalMethodNewVersion) {
                 // new version
-                args = new Object[]{name, Process.ROOT_UID, token, null};
+                args = new Object[]{name, ServiceManager.USER_ID, token, null};
             } else {
                 // old version
-                args = new Object[]{name, Process.ROOT_UID, token};
+                args = new Object[]{name, ServiceManager.USER_ID, token};
             }
             // ContentProviderHolder providerHolder = getContentProviderExternal(...);
             Object providerHolder = method.invoke(manager, args);

server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java

@@ -1,13 +1,11 @@
 package com.genymobile.scrcpy.wrappers;
 
-import com.genymobile.scrcpy.FakeContext;
 import com.genymobile.scrcpy.Ln;
 
 import android.content.ClipData;
 import android.content.IOnPrimaryClipChangedListener;
 import android.os.Build;
 import android.os.IInterface;
-import android.os.Process;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -60,22 +58,22 @@ public class ClipboardManager {
     private static ClipData getPrimaryClip(Method method, boolean alternativeMethod, IInterface manager)
             throws InvocationTargetException, IllegalAccessException {
         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-            return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME);
+            return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME);
         }
         if (alternativeMethod) {
-            return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, Process.ROOT_UID);
+            return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID);
         }
-        return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, Process.ROOT_UID);
+        return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID);
     }
 
     private static void setPrimaryClip(Method method, boolean alternativeMethod, IInterface manager, ClipData clipData)
             throws InvocationTargetException, IllegalAccessException {
         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-            method.invoke(manager, clipData, FakeContext.PACKAGE_NAME);
+            method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME);
         } else if (alternativeMethod) {
-            method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, null, Process.ROOT_UID);
+            method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID);
         } else {
-            method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, Process.ROOT_UID);
+            method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID);
         }
     }
 
@@ -108,11 +106,11 @@ public class ClipboardManager {
     private static void addPrimaryClipChangedListener(Method method, boolean alternativeMethod, IInterface manager,
             IOnPrimaryClipChangedListener listener) throws InvocationTargetException, IllegalAccessException {
         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-            method.invoke(manager, listener, FakeContext.PACKAGE_NAME);
+            method.invoke(manager, listener, ServiceManager.PACKAGE_NAME);
         } else if (alternativeMethod) {
-            method.invoke(manager, listener, FakeContext.PACKAGE_NAME, null, Process.ROOT_UID);
+            method.invoke(manager, listener, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID);
         } else {
-            method.invoke(manager, listener, FakeContext.PACKAGE_NAME, Process.ROOT_UID);
+            method.invoke(manager, listener, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID);
         }
     }
 

server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java

@@ -1,15 +1,11 @@
 package com.genymobile.scrcpy.wrappers;
 
-import com.genymobile.scrcpy.FakeContext;
 import com.genymobile.scrcpy.Ln;
 import com.genymobile.scrcpy.SettingsException;
 
 import android.annotation.SuppressLint;
-import android.content.AttributionSource;
-import android.os.Build;
 import android.os.Bundle;
 import android.os.IBinder;
-import android.os.Process;
 
 import java.io.Closeable;
 import java.lang.reflect.InvocationTargetException;
@@ -55,10 +51,11 @@ public class ContentProvider implements Closeable {
     @SuppressLint("PrivateApi")
     private Method getCallMethod() throws NoSuchMethodException {
         if (callMethod == null) {
-            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
-                callMethod = provider.getClass().getMethod("call", AttributionSource.class, String.class, String.class, String.class, Bundle.class);
+            try {
+                Class<?> attributionSourceClass = Class.forName("android.content.AttributionSource");
+                callMethod = provider.getClass().getMethod("call", attributionSourceClass, String.class, String.class, String.class, Bundle.class);
                 callMethodVersion = 0;
-            } else {
+            } catch (NoSuchMethodException | ClassNotFoundException e0) {
                 // old versions
                 try {
                     callMethod = provider.getClass()
@@ -78,29 +75,40 @@ public class ContentProvider implements Closeable {
         return callMethod;
     }
 
+    @SuppressLint("PrivateApi")
+    private Object getAttributionSource()
+            throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
+        if (attributionSource == null) {
+            Class<?> cl = Class.forName("android.content.AttributionSource$Builder");
+            Object builder = cl.getConstructor(int.class).newInstance(ServiceManager.USER_ID);
+            cl.getDeclaredMethod("setPackageName", String.class).invoke(builder, ServiceManager.PACKAGE_NAME);
+            attributionSource = cl.getDeclaredMethod("build").invoke(builder);
+        }
+
+        return attributionSource;
+    }
+
     private Bundle call(String callMethod, String arg, Bundle extras)
-            throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+            throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException {
         try {
             Method method = getCallMethod();
             Object[] args;
-
-            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && callMethodVersion == 0) {
-                args = new Object[]{FakeContext.get().getAttributionSource(), "settings", callMethod, arg, extras};
-            } else {
-                switch (callMethodVersion) {
-                    case 1:
-                        args = new Object[]{FakeContext.PACKAGE_NAME, null, "settings", callMethod, arg, extras};
-                        break;
-                    case 2:
-                        args = new Object[]{FakeContext.PACKAGE_NAME, "settings", callMethod, arg, extras};
-                        break;
-                    default:
-                        args = new Object[]{FakeContext.PACKAGE_NAME, callMethod, arg, extras};
-                        break;
-                }
+            switch (callMethodVersion) {
+                case 0:
+                    args = new Object[]{getAttributionSource(), "settings", callMethod, arg, extras};
+                    break;
+                case 1:
+                    args = new Object[]{ServiceManager.PACKAGE_NAME, null, "settings", callMethod, arg, extras};
+                    break;
+                case 2:
+                    args = new Object[]{ServiceManager.PACKAGE_NAME, "settings", callMethod, arg, extras};
+                    break;
+                default:
+                    args = new Object[]{ServiceManager.PACKAGE_NAME, callMethod, arg, extras};
+                    break;
             }
             return (Bundle) method.invoke(provider, args);
-        } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
+        } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException | ClassNotFoundException | InstantiationException e) {
             Ln.e("Could not invoke method", e);
             throw e;
         }
@@ -139,7 +147,7 @@ public class ContentProvider implements Closeable {
     public String getValue(String table, String key) throws SettingsException {
         String method = getGetMethod(table);
         Bundle arg = new Bundle();
-        arg.putInt(CALL_METHOD_USER_KEY, Process.ROOT_UID);
+        arg.putInt(CALL_METHOD_USER_KEY, ServiceManager.USER_ID);
         try {
             Bundle bundle = call(method, key, arg);
             if (bundle == null) {
@@ -155,7 +163,7 @@ public class ContentProvider implements Closeable {
     public void putValue(String table, String key, String value) throws SettingsException {
         String method = getPutMethod(table);
         Bundle arg = new Bundle();
-        arg.putInt(CALL_METHOD_USER_KEY, Process.ROOT_UID);
+        arg.putInt(CALL_METHOD_USER_KEY, ServiceManager.USER_ID);
         arg.putString(NAME_VALUE_TABLE_VALUE, value);
         try {
             call(method, key, arg);

server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java

@@ -10,6 +10,9 @@ import java.lang.reflect.Method;
 @SuppressLint("PrivateApi,DiscouragedPrivateApi")
 public final class ServiceManager {
 
+    public static final String PACKAGE_NAME = "com.android.shell";
+    public static final int USER_ID = 0;
+
     private static final Method GET_SERVICE_METHOD;
     static {
         try {