From 65cc9d765d8d0feb26a41c2287b3abe13dc37d12 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 18:46:59 +0100 Subject: [PATCH] Extract audio capture The audio capture was implemented in AudioEncoder. In order to reuse it without encoding, extract it to a separate class. PR #3757 --- .../com/genymobile/scrcpy/AudioCapture.java | 148 ++++++++++++++++++ .../com/genymobile/scrcpy/AudioEncoder.java | 136 ++-------------- 2 files changed, 161 insertions(+), 123 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/AudioCapture.java diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java new file mode 100644 index 00000000..3cef7801 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java @@ -0,0 +1,148 @@ +package com.genymobile.scrcpy; + +import com.genymobile.scrcpy.wrappers.ServiceManager; + +import android.annotation.SuppressLint; +import android.annotation.TargetApi; +import android.content.ComponentName; +import android.content.Intent; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.AudioTimestamp; +import android.media.MediaCodec; +import android.media.MediaRecorder; +import android.os.Build; +import android.os.SystemClock; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public final class AudioCapture { + + public static final int SAMPLE_RATE = 48000; + public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO; + public static final int CHANNELS = 2; + public static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT; + public static final int BYTES_PER_SAMPLE = 2; + + private AudioRecord recorder; + + private final AudioTimestamp timestamp = new AudioTimestamp(); + private long previousPts = 0; + private long nextPts = 0; + + public static int millisToBytes(int millis) { + return SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * millis / 1000; + } + + private static AudioFormat createAudioFormat() { + AudioFormat.Builder builder = new AudioFormat.Builder(); + builder.setEncoding(FORMAT); + builder.setSampleRate(SAMPLE_RATE); + builder.setChannelMask(CHANNEL_CONFIG); + return builder.build(); + } + + @TargetApi(Build.VERSION_CODES.M) + @SuppressLint({"WrongConstant", "MissingPermission"}) + private static AudioRecord createAudioRecord() { + AudioRecord.Builder builder = new AudioRecord.Builder(); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + // On older APIs, Workarounds.fillAppInfo() must be called beforehand + builder.setContext(FakeContext.get()); + } + builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX); + builder.setAudioFormat(createAudioFormat()); + int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT); + // This buffer size does not impact latency + builder.setBufferSizeInBytes(8 * minBufferSize); + return builder.build(); + } + + private static void startWorkaroundAndroid11() { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + // Android 11 requires Apps to be at foreground to record audio. + // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground. + // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android + // shell ("com.android.shell"). + // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the + // foreground. + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + Intent intent = new Intent(Intent.ACTION_MAIN); + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + intent.addCategory(Intent.CATEGORY_LAUNCHER); + intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity")); + ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent); + // Wait for activity to start + SystemClock.sleep(150); + } + } + } + + private static void stopWorkaroundAndroid11() { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME); + } + } + + public void start() throws AudioCaptureForegroundException { + startWorkaroundAndroid11(); + try { + recorder = createAudioRecord(); + recorder.startRecording(); + } catch (UnsupportedOperationException e) { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + Ln.e("Failed to start audio capture"); + Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy."); + throw new AudioCaptureForegroundException(); + } + throw e; + } finally { + stopWorkaroundAndroid11(); + } + } + + public void stop() { + if (recorder != null) { + // Will call .stop() if necessary, without throwing an IllegalStateException + recorder.release(); + } + } + + @TargetApi(Build.VERSION_CODES.N) + public int read(ByteBuffer directBuffer, int size, MediaCodec.BufferInfo outBufferInfo) throws IOException { + int r = recorder.read(directBuffer, size); + if (r < 0) { + return r; + } + + long pts; + + int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC); + if (ret == AudioRecord.SUCCESS) { + pts = timestamp.nanoTime / 1000; + } else { + if (nextPts == 0) { + Ln.w("Could not get any audio timestamp"); + } + // compute from previous timestamp and packet size + pts = nextPts; + } + + long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE); + nextPts = pts + durationUs; + + if (previousPts != 0 && pts < previousPts) { + // Audio PTS may come from two sources: + // - recorder.getTimestamp() if the call works; + // - an estimation from the previous PTS and the packet size as a fallback. + // + // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it. + pts = previousPts + 1; + } + previousPts = pts; + + outBufferInfo.set(0, r, pts, 0); + return r; + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index cc786bdb..8b60d37e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -1,22 +1,12 @@ package com.genymobile.scrcpy; -import com.genymobile.scrcpy.wrappers.ServiceManager; - -import android.annotation.SuppressLint; import android.annotation.TargetApi; -import android.content.ComponentName; -import android.content.Intent; -import android.media.AudioFormat; -import android.media.AudioRecord; -import android.media.AudioTimestamp; import android.media.MediaCodec; import android.media.MediaFormat; -import android.media.MediaRecorder; import android.os.Build; import android.os.Handler; import android.os.HandlerThread; import android.os.Looper; -import android.os.SystemClock; import java.io.IOException; import java.nio.ByteBuffer; @@ -44,14 +34,11 @@ public final class AudioEncoder { } } - private static final int SAMPLE_RATE = 48000; - private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO; - private static final int CHANNELS = 2; - private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT; - private static final int BYTES_PER_SAMPLE = 2; + private static final int SAMPLE_RATE = AudioCapture.SAMPLE_RATE; + private static final int CHANNELS = AudioCapture.CHANNELS; private static final int READ_MS = 5; // milliseconds - private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000; + private static final int READ_SIZE = AudioCapture.millisToBytes(READ_MS); private final Streamer streamer; private final int bitRate; @@ -78,30 +65,6 @@ public final class AudioEncoder { this.encoderName = encoderName; } - private static AudioFormat createAudioFormat() { - AudioFormat.Builder builder = new AudioFormat.Builder(); - builder.setEncoding(FORMAT); - builder.setSampleRate(SAMPLE_RATE); - builder.setChannelMask(CHANNEL_CONFIG); - return builder.build(); - } - - @TargetApi(Build.VERSION_CODES.M) - @SuppressLint({"WrongConstant", "MissingPermission"}) - private static AudioRecord createAudioRecord() { - AudioRecord.Builder builder = new AudioRecord.Builder(); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - // On older APIs, Workarounds.fillAppInfo() must be called beforehand - builder.setContext(FakeContext.get()); - } - builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX); - builder.setAudioFormat(createAudioFormat()); - int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT); - // This buffer size does not impact latency - builder.setBufferSizeInBytes(8 * minBufferSize); - return builder.build(); - } - private static MediaFormat createFormat(String mimeType, int bitRate, List codecOptions) { MediaFormat format = new MediaFormat(); format.setString(MediaFormat.KEY_MIME, mimeType); @@ -122,47 +85,18 @@ public final class AudioEncoder { } @TargetApi(Build.VERSION_CODES.N) - private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException { - final AudioTimestamp timestamp = new AudioTimestamp(); - long previousPts = 0; - long nextPts = 0; + private void inputThread(MediaCodec mediaCodec, AudioCapture capture) throws IOException, InterruptedException { + final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); while (!Thread.currentThread().isInterrupted()) { InputTask task = inputTasks.take(); ByteBuffer buffer = mediaCodec.getInputBuffer(task.index); - int r = recorder.read(buffer, READ_SIZE); + int r = capture.read(buffer, READ_SIZE, bufferInfo); if (r < 0) { throw new IOException("Could not read audio: " + r); } - long pts; - - int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC); - if (ret == AudioRecord.SUCCESS) { - pts = timestamp.nanoTime / 1000; - } else { - if (nextPts == 0) { - Ln.w("Could not get any audio timestamp"); - } - // compute from previous timestamp and packet size - pts = nextPts; - } - - long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE); - nextPts = pts + durationUs; - - if (previousPts != 0 && pts < previousPts) { - // Audio PTS may come from two sources: - // - recorder.getTimestamp() if the call works; - // - an estimation from the previous PTS and the packet size as a fallback. - // - // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it. - pts = previousPts + 1; - } - - previousPts = pts; - - mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0); + mediaCodec.queueInputBuffer(task.index, bufferInfo.offset, bufferInfo.size, bufferInfo.presentationTimeUs, bufferInfo.flags); } } @@ -223,32 +157,6 @@ public final class AudioEncoder { } } - private static void startWorkaroundAndroid11() { - if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { - // Android 11 requires Apps to be at foreground to record audio. - // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground. - // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android - // shell ("com.android.shell"). - // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the - // foreground. - if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { - Intent intent = new Intent(Intent.ACTION_MAIN); - intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - intent.addCategory(Intent.CATEGORY_LAUNCHER); - intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity")); - ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent); - // Wait for activity to start - SystemClock.sleep(150); - } - } - } - - private static void stopWorkaroundAndroid11() { - if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { - ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME); - } - } - @TargetApi(Build.VERSION_CODES.M) public void encode() throws IOException, ConfigurationException, AudioCaptureForegroundException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { @@ -258,10 +166,9 @@ public final class AudioEncoder { } MediaCodec mediaCodec = null; - AudioRecord recorder = null; + AudioCapture capture = new AudioCapture(); boolean mediaCodecStarted = false; - boolean recorderStarted = false; try { Codec codec = streamer.getCodec(); mediaCodec = createMediaCodec(codec, encoderName); @@ -273,27 +180,13 @@ public final class AudioEncoder { mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - startWorkaroundAndroid11(); - try { - recorder = createAudioRecord(); - recorder.startRecording(); - } catch (UnsupportedOperationException e) { - if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { - Ln.e("Failed to start audio capture"); - Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy."); - throw new AudioCaptureForegroundException(); - } - throw e; - } finally { - stopWorkaroundAndroid11(); - } - recorderStarted = true; + capture.start(); final MediaCodec mediaCodecRef = mediaCodec; - final AudioRecord recorderRef = recorder; + final AudioCapture captureRef = capture; inputThread = new Thread(() -> { try { - inputThread(mediaCodecRef, recorderRef); + inputThread(mediaCodecRef, captureRef); } catch (IOException | InterruptedException e) { Ln.e("Audio capture error", e); } finally { @@ -366,11 +259,8 @@ public final class AudioEncoder { } mediaCodec.release(); } - if (recorder != null) { - if (recorderStarted) { - recorder.stop(); - } - recorder.release(); + if (capture != null) { + capture.stop(); } } }