From 0a844927c17e23b57740ce3cf4079b20e355d838 Mon Sep 17 00:00:00 2001 From: Ingo Wolf Date: Tue, 10 Nov 2026 13:12:59 +1100 Subject: [PATCH 1/2] Android: keep one output rate so joins across sample rates stay gapless ExoPlayer's DefaultAudioSink can only reuse its AudioTrack when the output format is unchanged. At a join between tracks at different sample rates it played A out to silence, released the track and built a new one, which left a short gap: between tracks, and in the middle of A's audio when AutoMix mixed across rates. MixProcessor now leaves at one rate (the first stream's when it is 44.1 or 48 kHz, else the device's), converting other streams after the mixer with a new streaming resampler in hocket-android (JNI): a Kaiser-windowed sinc with exact per-phase taps, output frame j at input j * in / out, ~1.5 ms lookahead. - A gapless join at the same format continues through the same resampler. - When the next stream is at another format, A's end of stream emits A's last frames (with the armed tail's continuation of A as lookahead), and B's resampler is primed with the tail's lead, so the switch from A to the tail has no seam. - prepare_tail decodes from 30 ms before the tail start and returns the lead and lookahead alongside the tail; it now uses the same resampler. Co-Authored-By: Claude Opus 5.5 --- .../java/app/hocket/playback/ExoBackend.kt | 23 +- .../app/hocket/playback/LoudnessRecorder.kt | 23 +- .../java/app/hocket/playback/MixProcessor.kt | 120 ++++- .../java/app/hocket/playback/NativeDsp.kt | 21 + .../java/app/hocket/playback/OutputRate.kt | 138 ++++++ .../app/hocket/playback/ExoBackendClipTest.kt | 7 +- .../app/hocket/playback/FakeResamplerDsp.kt | 89 ++++ .../hocket/playback/LoudnessRecorderTest.kt | 3 +- .../app/hocket/playback/OutputRateTest.kt | 185 ++++++++ .../hocket/playback/TransitionMixerTest.kt | 5 + crates/hocket-android/src/dsp.rs | 271 ++++++++++- crates/hocket-android/src/lib.rs | 33 +- crates/hocket-android/src/resampler.rs | 439 ++++++++++++++++++ docs/automix.md | 47 +- 14 files changed, 1343 insertions(+), 61 deletions(-) create mode 100644 android/playback/src/main/java/app/hocket/playback/OutputRate.kt create mode 100644 android/playback/src/test/java/app/hocket/playback/FakeResamplerDsp.kt create mode 100644 android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt create mode 100644 crates/hocket-android/src/resampler.rs diff --git a/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt b/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt index 2ba38e0c..2e618467 100644 --- a/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt +++ b/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt @@ -1,6 +1,7 @@ package app.hocket.playback import android.content.Context +import android.media.AudioManager import android.net.Uri import android.os.Handler import android.os.Looper @@ -164,6 +165,8 @@ class ExoBackend( private const val HANDOVER_POLL_MS = 40L /** A's tail is decoded and rendered once A is this close to its tail start. */ private const val PREPARE_AHEAD_MS = 60_000L + /** A decoded before its tail start for the tail render (more than its resampler seam needs). */ + private const val TAIL_CONTEXT_MS = 30L /** * A clipped item whose tail isn't ready this long before its clip point is unclipped: the * sink and the decoder run up to about a second ahead of what is heard. @@ -242,7 +245,11 @@ class ExoBackend( PlaybackLog.i("AutoMix: ${armed.from} -> ${armed.to} plays unmixed: $reason") } } - private val processor: MixProcessor? = dsp?.let { MixProcessor(it, mixControl, mixListener) } + private val processor: MixProcessor? = dsp?.let { MixProcessor(it, mixControl, mixListener, deviceRate()) } + + /** The device's output rate (what [OutputRate] uses when the first item isn't at 44.1 or 48 kHz). */ + private fun deviceRate(): Int = + (context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager)?.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE)?.toIntOrNull() ?: 48_000 private fun renderersFactory() = object : DefaultRenderersFactory(context) { override fun buildAudioSink(context: Context, enableFloatOutput: Boolean, enableAudioOutputPlaybackParams: Boolean): AudioSink? { @@ -1084,15 +1091,21 @@ class ExoBackend( mixControl.armed = null } - /** Decode A's tail, then DSP, resample to B's format and render it in Rust. Null on failure (logged). */ + /** + * Decode A's tail (from [TAIL_CONTEXT_MS] before it: filter history, and A's audio either side of + * the tail start for [OutputRate]), then DSP, resample to B's format and render it in Rust. Null + * on failure (logged). + */ private fun renderTail(a: CoreMediaSource, b: CoreMediaSource, armed: MixControl.Armed, settings: String?, cancelled: () -> Boolean): MixControl.Tail? = try { val started = System.nanoTime() val decoder = PcmDecoder(streams) - val pcm = decoder.decode(a.url, a.headers, armed.tailStartMs, armed.tailSourceMs + 50, cancelled) + val from = (armed.tailStartMs - TAIL_CONTEXT_MS).coerceAtLeast(0) + val context = armed.tailStartMs - from + val pcm = decoder.decode(a.url, a.headers, from, context + armed.tailSourceMs + 50, cancelled) val (rate, channels) = decoder.probe(b.url, b.headers) - val samples = ffi.prepareTail(armed.planJson, pcm, channels, rate, settings, a.gainDb) + val r = ffi.prepareTail(armed.planJson, pcm, context.toInt(), channels, rate, settings, a.gainDb) PlaybackLog.i("AutoMix: tail ${a.key} -> ${b.key} ready (${pcm.durationMs} ms of A at ${pcm.rate} Hz -> ${armed.tailLenMs} ms at $rate Hz) in ${(System.nanoTime() - started) / 1_000_000} ms") - MixControl.Tail(samples, rate, channels) + MixControl.Tail(r.tail, rate, channels, r.lead, r.after, pcm.rate, pcm.channels) } catch (e: InterruptedException) { null } catch (e: Throwable) { diff --git a/android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt b/android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt index c4911538..525fb603 100644 --- a/android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt +++ b/android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt @@ -108,12 +108,21 @@ internal class LoudnessRecorder( } } +/** + * A rendered tail ([tail], at B's format) and A's audio either side of its start: [lead] just before + * it at B's format, [after] from it at A's format (`MixControl.Tail`). + */ +class RenderedTail(val tail: FloatArray, val lead: FloatArray, val after: FloatArray) + /** The one-shot UniFFI calls of AutoMix, behind a seam (no native library in JVM tests). */ interface AutomixFfi { fun usable(record: AnalysisRecord, durationMs: Long): Boolean fun loudnessRecord(head: PcmDecoder.Pcm, tail: PcmDecoder.Pcm, tailStartMs: Long, durationMs: Long, transcoded: Boolean): AnalysisRecord? - /** The rendered tail as interleaved floats at [outRate] × [outChannels]. */ - fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): FloatArray + /** + * The rendered tail from [pcm] (A from [contextMs] before the tail start), as interleaved floats + * at [outRate] × [outChannels], with A's audio either side of the tail start. + */ + fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): RenderedTail object Native : AutomixFfi { override fun usable(record: AnalysisRecord, durationMs: Long): Boolean = @@ -128,10 +137,14 @@ interface AutomixFfi { return HocketJson.json.decodeFromString(AnalysisRecord.serializer(), json) } - override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): FloatArray { - val bytes = app.hocket.core.ffi.automixPrepareTail( - planJson, pcm.leBytes(), pcm.channels.toUInt(), pcm.rate.toUInt(), outChannels.toUInt(), outRate.toUInt(), audioSettingsJson, gainDb, + override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): RenderedTail { + val r = app.hocket.core.ffi.automixPrepareTail( + planJson, pcm.leBytes(), contextMs.toUInt(), pcm.channels.toUInt(), pcm.rate.toUInt(), outChannels.toUInt(), outRate.toUInt(), audioSettingsJson, gainDb, ) + return RenderedTail(floats(r.tail), floats(r.lead), floats(r.after)) + } + + private fun floats(bytes: ByteArray): FloatArray { val floats = java.nio.ByteBuffer.wrap(bytes).order(java.nio.ByteOrder.LITTLE_ENDIAN).asFloatBuffer() return FloatArray(floats.remaining()).also { floats.get(it) } } diff --git a/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt b/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt index 5bd482e4..36f84c07 100644 --- a/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt +++ b/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt @@ -18,8 +18,20 @@ class MixControl { /** A playlist item by media id (the queue key). */ data class Item(val gainDb: Double, val clipEndMs: Long?) - /** A rendered tail: interleaved floats at B's rate and channel count. */ - class Tail(val samples: FloatArray, val rate: Int, val channels: Int) { + /** + * A rendered tail: interleaved floats at B's rate and channel count. [lead] is A just before the + * tail (B's format) and [after] A from the tail's start (A's format, [afterRate] × + * [afterChannels]): the [OutputRate] seam when A and B are converted differently. + */ + class Tail( + val samples: FloatArray, + val rate: Int, + val channels: Int, + val lead: FloatArray = FloatArray(0), + val after: FloatArray = FloatArray(0), + val afterRate: Int = 0, + val afterChannels: Int = 0, + ) { val frames: Int get() = samples.size / channels.coerceAtLeast(1) } @@ -110,6 +122,12 @@ class TransitionMixer( val mixing: Boolean get() = mix != null + /** The running mix's tail. */ + val tail: MixControl.Tail? get() = mix?.tail + + /** The last [startItem] continued the stream before it (a gapless join, or the same item going on). */ + val joined: Boolean get() = joining + fun configure(rate: Int, channels: Int) { if (chain == 0L) { chain = dsp.chainNew(rate, channels) @@ -126,11 +144,12 @@ class TransitionMixer( * item's end without a seek (a gapless join), else a seek or a fresh start. */ fun startItem(mediaId: String?, positionUs: Long, contiguous: Boolean) { - // Until a frame comes, a flush at the item's start still belongs to the join. When B's format - // differs from A's (another sample rate) the sink can't join gaplessly: it drains A, flushes - // once still naming A (the new configuration takes over after that flush), then flushes for - // B without another end of stream. Taken as a seek, that second flush would leave B unmixed. - // A seek while the sink waits for A to play out lands elsewhere and stays a seek. + // Until a frame comes, a flush at the item's start still belongs to the join. When the sink + // can't keep its `AudioTrack` for B (another channel count; the rate is always the same) + // it drains A, flushes once still naming A (the new configuration takes over after that + // flush), then flushes for B without another end of stream. Taken as a seek, that second + // flush would leave B unmixed. A seek while the sink waits for A to play out lands + // elsewhere and stays a seek. val joined = contiguous || (joining && itemFrames == 0L && positionUs < JOIN_START_US) joining = joined if (joined && mediaId != null && mediaId == item) { @@ -298,8 +317,9 @@ class TransitionMixer( } /** - * The ExoPlayer side of [TransitionMixer] (`docs/automix.md` → Android): installed in the main - * player's `DefaultAudioSink`, on the PCM path only (no offload, 16-bit or float in, the same out). + * The ExoPlayer side of [TransitionMixer] and [OutputRate] (`docs/automix.md` → Android): installed + * in the main player's `DefaultAudioSink`, on the PCM path only (no offload, 16-bit or float in, the + * same out). * * Item boundaries come from the sink itself: at a gapless join it drains the processors * (`queueEndOfStream`) and flushes them with the new stream's [AudioProcessor.StreamMetadata], @@ -307,13 +327,26 @@ class TransitionMixer( * So the first frame after such a flush is the item's first frame, sample-exact, without counting * frames against durations. The sink then drains and flushes once more for the same item (a timing * resync after every stream change); [TransitionMixer.startItem] lets that through as the same stream. - * At a format change (B at another sample rate) the sink drains A and flushes twice before B's first - * frame, the first time still naming A and the second without an end of stream before it; - * [TransitionMixer.startItem] takes every flush before the next frame as part of the join. + * + * Every item leaves at one rate ([OutputRate]), so a join at another sample rate is a gapless join + * like any other: the sink keeps its `AudioTrack`. (A change of channel count still rebuilds it: + * the sink drains A and flushes twice before B's first frame, the first time still naming A; + * [TransitionMixer.startItem] takes every flush before the next frame as part of the join.) */ -class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) : BaseAudioProcessor() { +class MixProcessor( + dsp: DspEngine, + private val control: MixControl, + listener: MixListener, + /** The output rate when the first item isn't at 44.1 or 48 kHz: the device's. */ + fallbackRate: Int, +) : BaseAudioProcessor() { val mixer = TransitionMixer(dsp, control, listener) + val rate = OutputRate(dsp, fallbackRate) private var endedBeforeFlush = false + /** The input format the sink configured last: the next stream's, once it differs. */ + private var configured: AudioProcessor.AudioFormat? = null + /** The stream ends at this end of stream: its resampler's last frames are still to come out. */ + private var finishPending = false private val period = Timeline.Period() private val window = Timeline.Window() @@ -322,31 +355,52 @@ class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) : PlaybackLog.w("DSP bypassed for PCM encoding ${inputAudioFormat.encoding}") return AudioProcessor.AudioFormat.NOT_SET } - return inputAudioFormat + configured = inputAudioFormat + val out = rate.latch(inputAudioFormat.sampleRate) + return AudioProcessor.AudioFormat(out, inputAudioFormat.channelCount, inputAudioFormat.encoding) } override fun onQueueEndOfStream() { endedBeforeFlush = true + val next = configured ?: return + // The next stream is at another format (it was configured while this one played): this + // resampler's held-back frames are this stream's last, and must come out before the switch. + if (rate.endsBefore(next.sampleRate, next.channelCount)) finishPending = true } + override fun isEnded(): Boolean = super.isEnded() && !finishPending + override fun onFlush(streamMetadata: AudioProcessor.StreamMetadata) { + finishPending = false mixer.configure(inputAudioFormat.sampleRate, inputAudioFormat.channelCount) mixer.startItem(mediaIdOf(streamMetadata), streamMetadata.positionOffsetUs, endedBeforeFlush) + val lead = mixer.tail?.takeIf { mixer.joined }?.lead + rate.start(inputAudioFormat.sampleRate, inputAudioFormat.channelCount, mixer.joined, lead) endedBeforeFlush = false } override fun onReset() { mixer.release() + rate.release() + configured = null endedBeforeFlush = false + finishPending = false } override fun queueInput(inputBuffer: ByteBuffer) { val format = inputAudioFormat val float = format.encoding == C.ENCODING_PCM_FLOAT + if (finishPending) { + // After the end of stream the pipeline asks for more with an empty buffer: the last frames. + finishPending = false + emit(rate.output, finish(), format.channelCount, float) + return + } val frames = inputBuffer.remaining() / format.bytesPerFrame if (frames == 0) return - if (mixer.passThrough(frames)) { - // Nothing to do (no mix, nothing clipped, a passthrough chain): the input as is. + val pass = mixer.passThrough(frames) + if (pass && !rate.resampling) { + // Nothing to do (no mix, nothing clipped, a passthrough chain, the output rate): the input as is. replaceOutputBuffer(inputBuffer.remaining()).put(inputBuffer).flip() return } @@ -359,14 +413,36 @@ class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) : for (i in 0 until samples) work.putFloat(i * 4, inputBuffer.getShort(start + i * 2) / 32768f) } inputBuffer.position(inputBuffer.limit()) - val outFrames = mixer.process(frames) - val outSamples = outFrames * format.channelCount - val out = replaceOutputBuffer(outFrames * format.bytesPerFrame) - val src = mixer.output + var src = work + var n = frames + if (!pass) { + n = mixer.process(frames) + src = mixer.output + } + if (rate.resampling) { + n = rate.process(src, n) + src = rate.output + } + emit(src, n, format.channelCount, float) + } + + /** The stream's last frames, with A's continuation as lookahead when the armed mix goes on from here. */ + private fun finish(): Int { + val armed = control.armed + val tail = armed?.tail + val after = tail?.takeIf { + armed.from == mixer.item && it.afterRate == rate.inRate && it.afterChannels == rate.channels + }?.after + return rate.finish(after) + } + + private fun emit(src: ByteBuffer, frames: Int, channels: Int, float: Boolean) { + val samples = frames * channels + val out = replaceOutputBuffer(samples * if (float) 4 else 2) if (float) { - for (i in 0 until outSamples) out.putFloat(src.getFloat(i * 4)) + for (i in 0 until samples) out.putFloat(src.getFloat(i * 4)) } else { - for (i in 0 until outSamples) out.putShort(toPcm16(src.getFloat(i * 4))) + for (i in 0 until samples) out.putShort(toPcm16(src.getFloat(i * 4))) } out.flip() } diff --git a/android/playback/src/main/java/app/hocket/playback/NativeDsp.kt b/android/playback/src/main/java/app/hocket/playback/NativeDsp.kt index 53525fbc..54673035 100644 --- a/android/playback/src/main/java/app/hocket/playback/NativeDsp.kt +++ b/android/playback/src/main/java/app/hocket/playback/NativeDsp.kt @@ -25,6 +25,22 @@ interface DspEngine { fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int) /** The chain's soft limiter (−1 dB knee), for the sum of a tail and B. */ fun limit(buffer: ByteBuffer, samples: Int) + + /** + * A `StreamResampler` (`crates/hocket-android/src/resampler.rs`): one stream from [inRate] to + * [outRate], output frame `j` at input frame `j × inRate / outRate` exactly. + */ + fun resamplerNew(inRate: Int, outRate: Int, channels: Int): Long + fun resamplerFree(handle: Long) + /** Frames before the stream's start, as history only. Before any input. */ + fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) + /** The stream ends: frames that would have followed, as lookahead only (0 samples: silence). */ + fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) + /** + * Take [inSamples] from [input], then write what is ready into [output] from sample [outOffset] + * up to [outCapacity]; returns the samples written. Call again with no input while it fills it. + */ + fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int): Int } /** @@ -56,4 +72,9 @@ object NativeDsp : DspEngine { external override fun shaperFree(handle: Long) external override fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int) external override fun limit(buffer: ByteBuffer, samples: Int) + external override fun resamplerNew(inRate: Int, outRate: Int, channels: Int): Long + external override fun resamplerFree(handle: Long) + external override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) + external override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) + external override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int): Int } diff --git a/android/playback/src/main/java/app/hocket/playback/OutputRate.kt b/android/playback/src/main/java/app/hocket/playback/OutputRate.kt new file mode 100644 index 00000000..e2bcc99b --- /dev/null +++ b/android/playback/src/main/java/app/hocket/playback/OutputRate.kt @@ -0,0 +1,138 @@ +package app.hocket.playback + +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * The last stage of [MixProcessor]: everything leaves at one sample rate (`docs/automix.md` → + * Android → Output rate), so the sink never rebuilds its `AudioTrack` between items. A rebuild means + * the old track plays out to silence before the new one starts: a gap at every join between + * tracks at different rates, and in the middle of A's audio when AutoMix mixes across rates. + * + * - **The rate** is the first stream's after a reset when that is 44.1 or 48 kHz, else [fallbackRate] + * (the device's). A library at one rate is never resampled; the rest is converted to it. + * - **Streams**: [start] at every flush. A stream joined gaplessly at the same format goes on through + * the same resampler (no seam inside continuous audio); anything else starts a new one. + * - **Seams**: when the next stream is at another format, the old one [finish]es at its end of + * stream, with A's real continuation as lookahead when a mix follows; the new one is primed with + * the mix tail's lead. So a mix across rates is as seamless as one at a single rate. + * + * Free of Media3 so JVM tests can drive it; floats in direct buffers, native order. + */ +class OutputRate(private val dsp: DspEngine, private val fallbackRate: Int) { + /** The rate everything leaves at; 0 until the first [latch]. */ + var outRate = 0 + private set + var inRate = 0 + private set + var channels = 0 + private set + private var handle = 0L + + /** Whether the current stream is converted (else it passes through as is). */ + val resampling: Boolean get() = handle != 0L + + /** Output floats of the last [process] or [finish], from index 0. */ + var output: ByteBuffer = floatBuffer(4096) + private set + private var scratch: ByteBuffer = floatBuffer(1024) + + /** The output rate for a stream at [rate], fixed by the first stream after a [release]. */ + fun latch(rate: Int): Int { + if (outRate == 0) outRate = if (rate == 44_100 || rate == 48_000 || fallbackRate <= 0) rate else fallbackRate + return outRate + } + + /** + * A stream (re)starts at [rate] × [channels]: [joined] when it follows the previous one's end + * without a seek. [lead] (at this stream's format) is the audio just before it, when a mix tail + * opens it: history for the resampler, not output. + */ + fun start(rate: Int, channels: Int, joined: Boolean, lead: FloatArray?) { + if (outRate == 0) latch(rate) + if (joined && rate == inRate && channels == this.channels) return + free() + inRate = rate + this.channels = channels + if (rate == outRate || rate <= 0) return + handle = dsp.resamplerNew(rate, outRate, channels) + if (handle == 0L) { + PlaybackLog.w("no resampler for $rate Hz -> $outRate Hz: plays at the wrong speed") + return + } + if (joined && lead != null && lead.size >= channels) { + val n = lead.size / channels * channels + dsp.resamplerPrime(handle, fill(lead, n), n) + } + } + + /** Whether the stream must end before a next one at [rate] × [channels]: it is converted, and differently. */ + fun endsBefore(rate: Int, channels: Int): Boolean = handle != 0L && (rate != inRate || channels != this.channels) + + /** + * End the stream: its last frames into [output], with [lookahead] (at its format: the audio that + * would follow) informing them. Returns the frames written. The next [start] begins anew. + */ + fun finish(lookahead: FloatArray?): Int { + if (handle == 0L) return 0 + val ch = channels.coerceAtLeast(1) + val n = if (lookahead != null) lookahead.size / ch * ch else 0 + dsp.resamplerFinish(handle, fill(lookahead ?: FloatArray(0), n), n) + val frames = drain(scratch, 0, 64) + free() + inRate = 0 + return frames + } + + /** Convert [frames] input frames from [input]; returns the frames written to [output]. */ + fun process(input: ByteBuffer, frames: Int): Int = drain(input, frames * channels, frames) + + fun release() { + free() + outRate = 0 + inRate = 0 + channels = 0 + } + + private fun drain(input: ByteBuffer, inSamples: Int, inFrames: Int): Int { + val ch = channels.coerceAtLeast(1) + // What `inFrames` make plus what the kernel held back; [resamplerProcess] is called again + // while it fills the buffer, so this is only the usual case. + val want = ((inFrames.toLong() * outRate / inRate.coerceAtLeast(1)) + SLACK_FRAMES).toInt() * ch + ensureOutput(want, 0) + var written = dsp.resamplerProcess(handle, input, inSamples, output, 0, output.capacity() / 4) + while (written >= output.capacity() / 4 - ch) { + ensureOutput(output.capacity() / 4 * 2, written) + val more = dsp.resamplerProcess(handle, input, 0, output, written, output.capacity() / 4) + if (more == 0) break + written += more + } + return written / ch + } + + private fun free() { + if (handle != 0L) dsp.resamplerFree(handle) + handle = 0L + } + + private fun fill(samples: FloatArray, n: Int): ByteBuffer { + if (scratch.capacity() < n * 4) scratch = floatBuffer(n) + for (i in 0 until n) scratch.putFloat(i * 4, samples[i]) + return scratch + } + + /** Make [output] hold [samples] floats, keeping its first [keep]. */ + private fun ensureOutput(samples: Int, keep: Int) { + if (output.capacity() >= samples * 4) return + val grown = floatBuffer(samples) + for (i in 0 until keep) grown.putFloat(i * 4, output.getFloat(i * 4)) + output = grown + } + + private companion object { + /** More than the widest kernel holds back (256 input frames). */ + const val SLACK_FRAMES = 600L + + fun floatBuffer(samples: Int): ByteBuffer = ByteBuffer.allocateDirect(samples.coerceAtLeast(1) * 4).order(ByteOrder.nativeOrder()) + } +} diff --git a/android/playback/src/test/java/app/hocket/playback/ExoBackendClipTest.kt b/android/playback/src/test/java/app/hocket/playback/ExoBackendClipTest.kt index 8852e7e8..77e9ebd3 100644 --- a/android/playback/src/test/java/app/hocket/playback/ExoBackendClipTest.kt +++ b/android/playback/src/test/java/app/hocket/playback/ExoBackendClipTest.kt @@ -49,13 +49,18 @@ class ExoBackendClipTest { override fun shaperFree(handle: Long) = Unit override fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int) = Unit override fun limit(buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerNew(inRate: Int, outRate: Int, channels: Int) = 0L + override fun resamplerFree(handle: Long) = Unit + override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int) = 0 } /** Every render fails, like a tail that never gets ready. */ private object FailingFfi : AutomixFfi { override fun usable(record: AnalysisRecord, durationMs: Long) = true override fun loudnessRecord(head: PcmDecoder.Pcm, tail: PcmDecoder.Pcm, tailStartMs: Long, durationMs: Long, transcoded: Boolean): AnalysisRecord? = null - override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): FloatArray = + override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): RenderedTail = throw IllegalStateException("no tail") } diff --git a/android/playback/src/test/java/app/hocket/playback/FakeResamplerDsp.kt b/android/playback/src/test/java/app/hocket/playback/FakeResamplerDsp.kt new file mode 100644 index 00000000..19d14708 --- /dev/null +++ b/android/playback/src/test/java/app/hocket/playback/FakeResamplerDsp.kt @@ -0,0 +1,89 @@ +package app.hocket.playback + +import java.nio.ByteBuffer + +/** + * A Rust-free [DspEngine] for JVM tests: a passthrough chain, and resamplers with the native one's + * contract (`crates/hocket-android/src/resampler.rs`) on a linear kernel: output frame `j` is the + * input at `j × in / out` exactly, one frame of lookahead is held back until [resamplerFinish], and + * [resamplerPrime] / [resamplerFinish] take history / lookahead that is never output itself. + */ +class FakeResamplerDsp : DspEngine { + private class Stream(val inRate: Long, val outRate: Long, val channels: Int) { + val input = ArrayList() + /** Input frames of the stream (primed history included). */ + var end = 0L + var finished = false + /** Next output frame's position, in units of `1 / outRate` input frames. */ + var pos = 0L + } + + private val streams = HashMap() + private var next = 1L + var created = 0 + private set + val live: Int get() = streams.size + + override fun chainNew(rate: Int, channels: Int) = 1L + override fun chainFree(handle: Long) = Unit + override fun chainSetFormat(handle: Long, rate: Int, channels: Int) = Unit + override fun chainSetAudioSettings(handle: Long, json: String) = true + override fun chainSetTrackGain(handle: Long, gainDb: Double) = Unit + override fun chainReset(handle: Long) = Unit + override fun chainIsBypassed(handle: Long) = true + override fun chainProcess(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun shaperNew(planJson: String, channels: Int, rate: Int) = 1L + override fun shaperFree(handle: Long) = Unit + override fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun limit(buffer: ByteBuffer, samples: Int) = Unit + + override fun resamplerNew(inRate: Int, outRate: Int, channels: Int): Long { + created++ + return next++.also { streams[it] = Stream(inRate.toLong(), outRate.toLong(), channels) } + } + + override fun resamplerFree(handle: Long) { + check(streams.remove(handle) != null) { "freed twice" } + } + + override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) { + val s = streams.getValue(handle) + check(s.end == 0L) { "primed after input" } + append(s, buffer, samples) + s.end = (s.input.size / s.channels).toLong() + s.pos = s.end * s.outRate + } + + override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) { + val s = streams.getValue(handle) + append(s, buffer, samples) + s.finished = true + } + + override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int): Int { + val s = streams.getValue(handle) + if (!s.finished) { + append(s, input, inSamples) + s.end = (s.input.size / s.channels).toLong() + } + val have = (s.input.size / s.channels).toLong() + var at = outOffset + while (at + s.channels <= outCapacity) { + val i = s.pos / s.outRate + val ready = if (s.finished) i < s.end else i + 1 < s.end + if (!ready) break + val frac = (s.pos % s.outRate).toFloat() / s.outRate + for (c in 0 until s.channels) { + fun x(k: Long) = if (k < have) s.input[(k * s.channels + c).toInt()] else 0f + output.putFloat(at * 4, x(i) + frac * (x(i + 1) - x(i))) + at++ + } + s.pos += s.inRate + } + return at - outOffset + } + + private fun append(s: Stream, buffer: ByteBuffer, samples: Int) { + for (i in 0 until samples / s.channels * s.channels) s.input += buffer.getFloat(i * 4) + } +} diff --git a/android/playback/src/test/java/app/hocket/playback/LoudnessRecorderTest.kt b/android/playback/src/test/java/app/hocket/playback/LoudnessRecorderTest.kt index 1d2473e4..f1a7af51 100644 --- a/android/playback/src/test/java/app/hocket/playback/LoudnessRecorderTest.kt +++ b/android/playback/src/test/java/app/hocket/playback/LoudnessRecorderTest.kt @@ -19,7 +19,8 @@ class LoudnessRecorderTest { private object NoFfi : AutomixFfi { override fun usable(record: AnalysisRecord, durationMs: Long) = false override fun loudnessRecord(head: PcmDecoder.Pcm, tail: PcmDecoder.Pcm, tailStartMs: Long, durationMs: Long, transcoded: Boolean): AnalysisRecord? = null - override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double) = FloatArray(0) + override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double) = + RenderedTail(FloatArray(0), FloatArray(0), FloatArray(0)) } @Test diff --git a/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt b/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt new file mode 100644 index 00000000..39e81086 --- /dev/null +++ b/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt @@ -0,0 +1,185 @@ +package app.hocket.playback + +import androidx.media3.common.C +import androidx.media3.common.audio.AudioProcessor +import java.nio.ByteBuffer +import java.nio.ByteOrder +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +/** [OutputRate] and its place in [MixProcessor], on [FakeResamplerDsp] (plain JVM). */ +class OutputRateTest { + private val dsp = FakeResamplerDsp() + + private fun floats(values: FloatArray): ByteBuffer { + val b = ByteBuffer.allocateDirect(values.size * 4).order(ByteOrder.nativeOrder()) + values.forEachIndexed { i, v -> b.putFloat(i * 4, v) } + return b + } + + private fun OutputRate.feed(values: FloatArray): FloatArray { + val n = process(floats(values), values.size / channels) + return FloatArray(n * channels) { output.getFloat(it * 4) } + } + + private fun OutputRate.end(lookahead: FloatArray? = null): FloatArray { + val ch = channels + val n = finish(lookahead) + return FloatArray(n * ch) { output.getFloat(it * 4) } + } + + /** A ramp, so every frame is distinct and a linear kernel reproduces it exactly. */ + private fun ramp(from: Int, n: Int) = FloatArray(n) { (from + it) / 1_000f } + + /** [input] at 3 Hz converted to 4 Hz in one stream, finished with [lookahead]. */ + private fun whole(input: FloatArray, lookahead: FloatArray? = null): FloatArray { + val r = OutputRate(FakeResamplerDsp(), 4) + r.latch(4) + r.start(3, 1, joined = false, lead = null) + return r.feed(input) + r.end(lookahead) + } + + @Test + fun theFirstStreamAt44Or48FixesTheRateElseTheDevicesIs() { + assertEquals(44_100, OutputRate(dsp, 48_000).latch(44_100)) + val r = OutputRate(dsp, 48_000) + assertEquals(48_000, r.latch(96_000)) + assertEquals(48_000, r.latch(44_100)) + r.release() + assertEquals(44_100, r.latch(44_100)) + } + + @Test + fun aStreamAtTheOutputRatePassesThrough() { + val r = OutputRate(dsp, 48_000) + r.latch(44_100) + r.start(44_100, 2, joined = false, lead = null) + assertFalse(r.resampling) + assertEquals(0, dsp.created) + r.start(48_000, 2, joined = true, lead = null) + assertTrue(r.resampling) + } + + @Test + fun aGaplessJoinAtTheSameFormatGoesOnThroughTheSameResampler() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(3, 1, joined = false, lead = null) + val a = r.feed(ramp(0, 30)) + r.start(3, 1, joined = true, lead = null) + val b = r.feed(ramp(30, 30)) + assertEquals(1, dsp.created) + val all = a + b + r.end() + assertArrayEquals(whole(ramp(0, 60)), all, 0f) + assertEquals(80, all.size) + } + + @Test + fun aSeekStartsANewStream() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(3, 1, joined = false, lead = null) + r.feed(ramp(0, 30)) + r.start(3, 1, joined = false, lead = null) + assertEquals(2, dsp.created) + assertEquals(1, dsp.live) + // Output frame 0 of the new stream is its input frame 0. + assertEquals(0.5f, r.feed(ramp(500, 3))[0], 0f) + } + + @Test + fun aStreamEndsExactlyAndWithTheLookaheadLikeAnUncutOne() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(3, 1, joined = false, lead = null) + assertTrue(r.endsBefore(4, 1)) + assertFalse(r.endsBefore(3, 1)) + val out = r.feed(ramp(0, 30)) + r.end(ramp(30, 5)) + // 30 frames at 3 Hz are 40 at 4 Hz, the last one using the continuation. + assertArrayEquals(whole(ramp(0, 40)).copyOf(40), out, 1e-6f) + assertEquals(0, dsp.live) + assertFalse(r.resampling) + } + + @Test + fun aLeadPrimesTheNextStreamSoItStartsWhereTheOldOneStopped() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(4, 1, joined = false, lead = null) + assertFalse(r.resampling) + // B at 3 Hz opens with a tail whose lead is the 3 frames before it. + r.start(3, 1, joined = true, lead = ramp(27, 3)) + val out = r.feed(ramp(30, 6)) + r.end() + // Exactly the uncut stream's frames from frame 30 (output frame 40) on. + assertArrayEquals(whole(ramp(0, 36)).copyOfRange(40, 48), out, 1e-6f) + // The fake's kernel only looks ahead, so this is about position: B's first output frame is + // its first input frame, not a lead frame. + assertEquals(0.03f, out[0], 1e-6f) + } + + // -- In MixProcessor --------------------------------------------------------------------- + + private val processor = MixProcessor(dsp, MixControl(), object : MixListener { + override fun onMixStarted(armed: MixControl.Armed) = Unit + override fun onMixSkipped(armed: MixControl.Armed, reason: String) = Unit + }, fallbackRate = 48_000) + + private fun pcm16(rate: Int) = AudioProcessor.AudioFormat(rate, 1, C.ENCODING_PCM_16BIT) + + private fun queue(frames: Int): Int { + val b = ByteBuffer.allocateDirect(frames * 2).order(ByteOrder.nativeOrder()) + for (i in 0 until frames) b.putShort(i * 2, 1_000) + processor.queueInput(b) + return processor.output.remaining() / 2 + } + + @Test + fun theOutputRateStaysFixedAcrossItemsAtOtherRates() { + assertEquals(48_000, processor.configure(pcm16(48_000)).sampleRate) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + assertEquals(480, queue(480)) + assertEquals(pcm16(48_000), processor.configure(pcm16(44_100))) + assertEquals(pcm16(48_000), processor.configure(pcm16(96_000))) + } + + @Test + fun aResampledStreamComesOutWholeBeforeAFormatChange() { + processor.configure(pcm16(48_000)) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + processor.configure(pcm16(44_100)) + processor.queueEndOfStream() + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + // A at 44.1 kHz into 48 kHz: one frame held back for lookahead. + assertEquals(479, queue(441)) + // B at 48 kHz is configured while A plays; A's end of stream lets out its last frame. + processor.configure(pcm16(48_000)) + processor.queueEndOfStream() + assertFalse("the held-back frame is still to come", processor.isEnded) + processor.queueInput(ByteBuffer.allocateDirect(0)) + assertEquals(1, processor.output.remaining() / 2) + assertTrue(processor.isEnded) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + assertEquals(0, dsp.live) + assertEquals(100, queue(100)) + } + + @Test + fun anEndOfStreamAtTheSameFormatHoldsTheResamplerOpen() { + processor.configure(pcm16(48_000)) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + processor.configure(pcm16(44_100)) + processor.queueEndOfStream() + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + queue(441) + processor.queueEndOfStream() + assertTrue(processor.isEnded) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + assertEquals(1, dsp.live) + // The held-back frame comes out with the next item's first: 147 frames in, 160 out (one + // held back again). + assertEquals(160, queue(147)) + } +} diff --git a/android/playback/src/test/java/app/hocket/playback/TransitionMixerTest.kt b/android/playback/src/test/java/app/hocket/playback/TransitionMixerTest.kt index 5e3f0f60..d386a6a7 100644 --- a/android/playback/src/test/java/app/hocket/playback/TransitionMixerTest.kt +++ b/android/playback/src/test/java/app/hocket/playback/TransitionMixerTest.kt @@ -54,6 +54,11 @@ class TransitionMixerTest { override fun limit(buffer: ByteBuffer, samples: Int) { for (i in 0 until samples) buffer.putFloat(i * 4, buffer.getFloat(i * 4).coerceIn(-0.9f, 0.9f)) } + override fun resamplerNew(inRate: Int, outRate: Int, channels: Int) = 0L + override fun resamplerFree(handle: Long) = Unit + override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int) = 0 } private val dsp = FakeDsp() diff --git a/crates/hocket-android/src/dsp.rs b/crates/hocket-android/src/dsp.rs index 0c3e79e8..d29c108e 100644 --- a/crates/hocket-android/src/dsp.rs +++ b/crates/hocket-android/src/dsp.rs @@ -9,6 +9,7 @@ //! - [`Chain`]: hocket-core's [`DspChain`] (track gain incl. boost, EQ, normalisation, limiter). //! - [`IncomingShaper`]: B's side of a transition. //! - `limit`: the chain's soft limiter, run on the sum of the tail and B. +//! - [`StreamResampler`]: the output stage, to the one rate the sink plays at. //! - **One shot, UniFFI** (`lib.rs`): [`prepare_tail`] (DSP chain like A's, channel map, resample to //! B's rate, `render_tail`) and [`loudness_record`], on a background thread. //! @@ -24,6 +25,8 @@ use hocket_core::audio::resample; use hocket_core::automix::loudness::{self, HeadTail}; use hocket_core::automix::{is_usable, render_tail, IncomingShaper}; +use crate::resampler::{gcd, StreamResampler}; + /// The chain for one stream: hocket-core's [`DspChain`] with the format the processor was /// configured with. pub struct Chain { @@ -90,8 +93,12 @@ pub fn parse_audio_settings(json: &str) -> Result { /// What [`prepare_tail`] renders from. pub struct TailInput<'a> { - /// A from `tail_start_ms` (no preroll), interleaved, at `in_rate`, straight from the decoder. + /// A from `tail_start_ms − context_ms` (no preroll), interleaved, at `in_rate`, straight from + /// the decoder. pub pcm: &'a [f32], + /// How much of `pcm` comes before `tail_start_ms`: history for the chain and the resampler, + /// and where the lead comes from. + pub context_ms: u32, pub in_channels: usize, pub in_rate: u32, /// B's output format: the tail is mixed into B's frames. @@ -102,21 +109,83 @@ pub struct TailInput<'a> { pub gain_db: f64, } +/// [`prepare_tail`]'s result. +#[derive(Debug, Default)] +pub struct PreparedTail { + /// Exactly `tail_len_ms` at `out_rate` × `out_channels`, from `tail_start_ms`. + pub tail: Vec, + /// A just before `tail_start_ms` at `out_rate` × `out_channels`, its last frame the one before + /// the tail's first: history for the output resampler when B's stream starts with the tail. + pub lead: Vec, + /// A from `tail_start_ms` through A's chain, at `in_rate` × `in_channels`: lookahead for the + /// output resampler when A's stream ends at the tail. + pub after: Vec, +} + +/// Lead and lookahead length (at least the output resampler's kernel half-width). +const SEAM_MS: u32 = 10; + /// The rendered tail: A's PCM through a chain configured like A's, mapped to B's channels, -/// resampled to B's rate, then `render_tail`. Exactly `tail_len_ms` at `out_rate`. -pub fn prepare_tail(plan: &TransitionPlan, input: TailInput<'_>) -> Result, String> { +/// resampled to B's rate, then `render_tail`; plus the [`PreparedTail::lead`] and +/// [`PreparedTail::after`] that let the output resampler switch streams without a seam. +pub fn prepare_tail(plan: &TransitionPlan, input: TailInput<'_>) -> Result { let in_ch = input.in_channels.max(1); + let in_rate = input.in_rate.max(1); + let out_ch = input.out_channels.max(1); + let out_rate = input.out_rate.max(1); let mut pcm = input.pcm[..input.pcm.len() / in_ch * in_ch].to_vec(); - let mut chain = Chain::new(input.in_rate, in_ch); + let mut chain = Chain::new(in_rate, in_ch); if let Some(a) = input.audio { chain.set_audio_settings(a); } chain.set_track_gain(input.gain_db); chain.process(&mut pcm); - let out_ch = input.out_channels.max(1); - let converted = resample::convert(&pcm, in_ch, input.in_rate, out_ch, input.out_rate) - .map_err(|e| e.to_string())?; - Ok(render_tail(plan, &converted, out_ch, input.out_rate)) + let frames = pcm.len() / in_ch; + let ctx = ((u64::from(input.context_ms) * u64::from(in_rate) + 500) / 1000) as usize; + let ctx = ctx.min(frames); + let seam_in = (u64::from(SEAM_MS) * u64::from(in_rate) / 1000) as usize; + let after = pcm[ctx * in_ch..(ctx + seam_in).min(frames) * in_ch].to_vec(); + + let mut mapped = Vec::with_capacity(frames * out_ch); + resample::remap_channels(&pcm, in_ch, out_ch, &mut mapped); + let seam_out = (u64::from(SEAM_MS) * u64::from(out_rate) / 1000) as usize; + let (lead, source) = if in_rate == out_rate { + let lead_from = ctx.saturating_sub(seam_out); + ( + mapped[lead_from * out_ch..ctx * out_ch].to_vec(), + mapped[ctx * out_ch..].to_vec(), + ) + } else { + // One stream from a little before `tail_start_ms`, so that an output frame lands exactly on + // it: `lead_in` input frames make `lead` output frames when both are whole multiples of + // the reduced ratio. What comes before only primes the filter. + let g = gcd(u64::from(in_rate), u64::from(out_rate)); + let (step, den) = ( + (u64::from(in_rate) / g) as usize, + (u64::from(out_rate) / g) as usize, + ); + let mut k = seam_out.div_ceil(den).max(1); + while k > 0 && k * step > ctx { + k -= 1; + } + let (lead_out, lead_in) = (k * den, k * step); + let start = ctx - lead_in; + let mut r = StreamResampler::new(in_rate, out_rate, out_ch); + r.prime(&mapped[..start * out_ch]); + r.push(&mapped[start * out_ch..]); + r.finish(&[]); + let mut converted = vec![0f32; ((frames - start) * den).div_ceil(step) * out_ch + out_ch]; + let n = r.pull(&mut converted); + converted.truncate(n); + let split = (lead_out * out_ch).min(converted.len()); + let source = converted.split_off(split); + (converted, source) + }; + Ok(PreparedTail { + tail: render_tail(plan, &source, out_ch, out_rate), + lead, + after, + }) } /// A loudness-only record from a track's decoded head and tail, fingerprinted with the @@ -227,6 +296,11 @@ mod jni_shims { (h != 0).then(|| unsafe { &mut *(h as *mut IncomingShaper) }) } + fn resampler<'a>(h: jlong) -> Option<&'a mut StreamResampler> { + // SAFETY: as for `chain`, from `resamplerNew`. + (h != 0).then(|| unsafe { &mut *(h as *mut StreamResampler) }) + } + #[no_mangle] pub extern "system" fn Java_app_hocket_playback_NativeDsp_chainNew( _env: JNIEnv, @@ -401,6 +475,103 @@ mod jni_shims { }) } + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerNew( + _env: JNIEnv, + _this: JObject, + in_rate: jint, + out_rate: jint, + channels: jint, + ) -> jlong { + guarded(0, || { + Box::into_raw(Box::new(StreamResampler::new( + in_rate.max(1) as u32, + out_rate.max(1) as u32, + channels.max(1) as usize, + ))) as jlong + }) + } + + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerFree( + _env: JNIEnv, + _this: JObject, + h: jlong, + ) { + if h != 0 { + // SAFETY: from `resamplerNew`, freed once (Kotlin zeroes its copy). + drop(unsafe { Box::from_raw(h as *mut StreamResampler) }); + } + } + + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerPrime( + env: JNIEnv, + _this: JObject, + h: jlong, + buf: JByteBuffer, + samples: jint, + ) { + let Some(s) = floats(&env, &buf, samples) else { + return; + }; + guarded((), || { + if let Some(r) = resampler(h) { + r.prime(s); + } + }) + } + + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerFinish( + env: JNIEnv, + _this: JObject, + h: jlong, + buf: JByteBuffer, + samples: jint, + ) { + let Some(s) = floats(&env, &buf, samples) else { + return; + }; + guarded((), || { + if let Some(r) = resampler(h) { + r.finish(s); + } + }) + } + + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerProcess( + env: JNIEnv, + _this: JObject, + h: jlong, + input: JByteBuffer, + in_samples: jint, + output: JByteBuffer, + out_offset: jint, + out_capacity: jint, + ) -> jint { + let Some(inp) = floats(&env, &input, in_samples) else { + return 0; + }; + let Some(out) = floats(&env, &output, out_capacity) else { + return 0; + }; + let Ok(offset) = usize::try_from(out_offset) else { + return 0; + }; + if offset > out.len() { + return 0; + } + guarded(0, || match resampler(h) { + Some(r) => { + r.push(inp); + r.pull(&mut out[offset..]) as jint + } + None => 0, + }) + } + #[no_mangle] pub extern "system" fn Java_app_hocket_playback_NativeDsp_limit( env: JNIEnv, @@ -527,10 +698,11 @@ mod tests { fn tail_is_exact_length_at_bs_rate_and_channels() { let p = plan(); let pcm = dc(44_100, 1, 0.5); - let tail = prepare_tail( + let prepared = prepare_tail( &p, TailInput { pcm: &pcm, + context_ms: 0, in_channels: 1, in_rate: 44_100, out_channels: 2, @@ -540,12 +712,93 @@ mod tests { }, ) .unwrap(); + let tail = prepared.tail; assert_eq!(tail.len(), 48_000 * 2); // Before the overlap the tail is A at A's gain (−6 dB of 0.5). assert!((tail[4_800 * 2] - 0.25).abs() < 0.01, "{}", tail[4_800 * 2]); // It has faded out by its end. assert!(tail[tail.len() - 1].abs() < 0.01); assert!(tail.iter().all(|s| s.is_finite())); + // No context: no lead; the lookahead is A's first 10 ms at A's format. + assert!(prepared.lead.is_empty()); + assert_eq!(prepared.after.len(), 441); + } + + /// A sine at `rate`, `frames` long, from frame `from` of the same sine. + fn sine_at(rate: u32, frames: usize, from: usize, channels: usize) -> Vec { + (from..from + frames) + .flat_map(|i| { + let s = (2.0 * std::f64::consts::PI * 2_000.0 * i as f64 / f64::from(rate)).sin() + as f32 + * 0.25; + std::iter::repeat_n(s, channels) + }) + .collect() + } + + #[test] + fn tail_lead_and_after_line_up_with_the_tail_start() { + // A at 44.1 kHz from 30 ms before the tail start; B at 48 kHz. A plain fade with no + // overlap yet at the start, so the tail's first frames are A itself. + let mut p = plan(); + p.incoming_at_ms = 900; + p.handover_ms = 950; + let ctx_frames = 1_323; // 30 ms at 44.1 kHz + let pcm = sine_at(44_100, 44_100, 0, 2); + let prepared = prepare_tail( + &p, + TailInput { + pcm: &pcm, + context_ms: 30, + in_channels: 2, + in_rate: 44_100, + out_channels: 2, + out_rate: 48_000, + audio: None, + gain_db: 0.0, + }, + ) + .unwrap(); + // The tail starts exactly at the tail start: frame j is A at ctx + j × 44.1 / 48. + let want_tail = sine_at(48_000, 200, 1_440, 2); + let err = + |a: &[f32], b: &[f32]| a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())); + assert!( + err(&prepared.tail[..400], &want_tail) < 1e-3, + "tail start {}", + err(&prepared.tail[..400], &want_tail) + ); + // The lead ends on the frame before it (at least 10 ms of it, in whole 160-frame steps). + let lead_frames = prepared.lead.len() / 2; + assert_eq!(lead_frames, 480); + let want_lead = sine_at(48_000, lead_frames, 1_440 - lead_frames, 2); + assert!(err(&prepared.lead[lead_frames..], &want_lead[lead_frames..]) < 1e-3); + // The lookahead is A from the tail start at A's rate. + assert_eq!( + prepared.after, + pcm[ctx_frames * 2..(ctx_frames + 441) * 2].to_vec() + ); + } + + #[test] + fn same_rate_tail_needs_no_resampling() { + let pcm = sine_at(48_000, 48_000, 0, 1); + let prepared = prepare_tail( + &plan(), + TailInput { + pcm: &pcm, + context_ms: 20, + in_channels: 1, + in_rate: 48_000, + out_channels: 1, + out_rate: 48_000, + audio: None, + gain_db: 0.0, + }, + ) + .unwrap(); + assert_eq!(prepared.lead, pcm[480..960].to_vec()); + assert_eq!(prepared.tail[..100], pcm[960..1_060]); } #[test] diff --git a/crates/hocket-android/src/lib.rs b/crates/hocket-android/src/lib.rs index 785e0c95..32a3a7b6 100644 --- a/crates/hocket-android/src/lib.rs +++ b/crates/hocket-android/src/lib.rs @@ -259,22 +259,35 @@ pub fn artwork_layout( // One-shot calls for a background thread. PCM crosses as little-endian `f32` bytes (a Kotlin // `ByteArray`; a `Vec` would be a boxed `List`). The per-buffer calls are JNI. -/// The rendered tail of a transition (`docs/automix.md`): `pcm` is A from `tail_start_ms`, -/// `in_channels` at `in_rate`; it goes through a DSP chain like A's (`audio_settings_json`, an -/// `AudioSettings`, and A's `gain_db`), is mapped to B's channels, resampled to B's rate and -/// rendered with `render_tail`. Returns exactly `tail_len_ms` at `out_rate`, as LE `f32` bytes. +/// [`automix_prepare_tail`]'s result, each as LE `f32` bytes (see [`dsp::PreparedTail`]). +#[derive(uniffi::Record)] +pub struct PreparedTail { + /// Exactly `tail_len_ms` at `out_rate` × `out_channels`. + pub tail: Vec, + /// A just before `tail_start_ms`, at `out_rate` × `out_channels`. + pub lead: Vec, + /// A from `tail_start_ms` through A's chain, at `in_rate` × `in_channels`. + pub after: Vec, +} + +/// The rendered tail of a transition (`docs/automix.md`): `pcm` is A from `tail_start_ms − +/// context_ms`, `in_channels` at `in_rate`; it goes through a DSP chain like A's +/// (`audio_settings_json`, an `AudioSettings`, and A's `gain_db`), is mapped to B's channels, +/// resampled to B's rate and rendered with `render_tail`. Also returns A's audio either side of +/// `tail_start_ms` for a seamless switch of the output resampler. #[uniffi::export] #[allow(clippy::too_many_arguments)] pub fn automix_prepare_tail( plan_json: String, pcm: Vec, + context_ms: u32, in_channels: u32, in_rate: u32, out_channels: u32, out_rate: u32, audio_settings_json: Option, gain_db: f64, -) -> Result, HocketError> { +) -> Result { let failed = |reason: String| HocketError::Failed { reason }; let plan = dsp::parse_plan(&plan_json).map_err(failed)?; let audio = audio_settings_json @@ -283,10 +296,11 @@ pub fn automix_prepare_tail( .transpose() .map_err(failed)?; let pcm = dsp::f32_from_le(&pcm); - let tail = dsp::prepare_tail( + let prepared = dsp::prepare_tail( &plan, dsp::TailInput { pcm: &pcm, + context_ms, in_channels: in_channels as usize, in_rate, out_channels: out_channels as usize, @@ -296,7 +310,11 @@ pub fn automix_prepare_tail( }, ) .map_err(failed)?; - Ok(dsp::f32_to_le(&tail)) + Ok(PreparedTail { + tail: dsp::f32_to_le(&prepared.tail), + lead: dsp::f32_to_le(&prepared.lead), + after: dsp::f32_to_le(&prepared.after), + }) } /// A loudness-only `AnalysisRecord` (JSON, producer Device) from a track's decoded head (from 0) @@ -343,6 +361,7 @@ pub fn core_version() -> String { } pub mod dsp; +pub mod resampler; #[cfg(test)] mod fixtures; diff --git a/crates/hocket-android/src/resampler.rs b/crates/hocket-android/src/resampler.rs new file mode 100644 index 00000000..6570887c --- /dev/null +++ b/crates/hocket-android/src/resampler.rs @@ -0,0 +1,439 @@ +//! Streaming sample-rate conversion for the output of `MixProcessor` (`docs/automix.md` → +//! Android → Output rate). +//! +//! ExoPlayer's sink rebuilds its `AudioTrack` whenever the output rate changes, which leaves a gap +//! between tracks at different rates (and in the middle of an AutoMix transition). So the +//! processor keeps the output rate fixed and converts every stream at another rate with this. +//! +//! What this needs that a chunked converter (`hocket_core::audio::resample`) doesn't give: +//! +//! - **Sample-exact positions.** Output frame `j` of a stream is its input at `j × in / out`, +//! exactly (the rates are reduced to a ratio of integers): no filter delay to trim, and a stream +//! of `n` input frames is `ceil(n × out / in)` output frames. +//! - **Low latency.** An output frame is produced as soon as the kernel's lookahead (`half` input +//! frames, ~1.5 ms) has arrived, not when a 1024-frame chunk is full. Whatever is still waiting +//! at an item boundary barely moves the sink's clock. +//! - **Seams without clicks.** A stream that ends ([`StreamResampler::finish`]) can take frames +//! past its end as lookahead only, and a stream that starts ([`StreamResampler::prime`]) can +//! take frames before its start as history only. With the same audio on both sides (an AutoMix +//! tail continues A), the switch from one stream to the next is as if nothing were cut. +//! +//! The kernel is a Kaiser-windowed sinc, one row of taps per output phase (`out / gcd` of them: +//! 160 for 44.1 ↔ 48 kHz), so there is no interpolation between rows. + +use std::f64::consts::PI; + +/// Kernel half-width in input frames when upsampling (scaled up when downsampling). +const HALF_BASE: usize = 64; +/// The widest kernel half-width (192 kHz into 44.1 kHz would want 279). +const HALF_MAX: usize = 256; +/// Phases beyond this (unusual rate pairs) are quantised to this many rows. +const PHASES_MAX: u64 = 1024; +/// Cutoff as a fraction of the lower Nyquist frequency. +const CUTOFF: f64 = 0.92; +/// Kaiser β: about 80 dB of stopband. +const BETA: f64 = 8.0; +/// Consumed input is dropped from the front once this many frames have piled up. +const COMPACT_FRAMES: u64 = 4096; + +pub struct StreamResampler { + channels: usize, + /// Each output frame advances `step / den` input frames. + step: u64, + den: u64, + half: usize, + phases: u64, + /// `phases + 1` rows of `2 × half` taps; row `r` is the kernel at offset `r / phases`. + table: Vec, + /// Input frames from absolute frame `base`, interleaved. + buf: Vec, + base: u64, + /// Input frames of the stream (primed history included): outputs lie before this. + end: u64, + finished: bool, + /// The next output frame's position in input frames, in units of `1 / den`. + pos: u64, + acc: Vec, +} + +impl StreamResampler { + pub fn new(in_rate: u32, out_rate: u32, channels: usize) -> Self { + let (in_rate, out_rate) = (u64::from(in_rate.max(1)), u64::from(out_rate.max(1))); + let g = gcd(in_rate, out_rate); + let (step, den) = (in_rate / g, out_rate / g); + let down = in_rate as f64 / out_rate as f64; + let half = ((HALF_BASE as f64 * down.max(1.0)).ceil() as usize).min(HALF_MAX); + let phases = den.min(PHASES_MAX); + // Cycles per input frame, times two: the sinc's zero crossings. + let fc2 = CUTOFF * (1.0 / down).min(1.0); + let taps = 2 * half; + let mut table = vec![0f32; (phases as usize + 1) * taps]; + let i0_beta = bessel_i0(BETA); + for r in 0..=phases as usize { + let frac = r as f64 / phases as f64; + let row = &mut table[r * taps..(r + 1) * taps]; + let mut sum = 0.0; + let mut w = vec![0f64; taps]; + for (k, wk) in w.iter_mut().enumerate() { + // Tap k reads input frame `ipos - half + 1 + k`, at `t` frames from the output. + let t = k as f64 - half as f64 + 1.0 - frac; + let x = t / half as f64; + let window = if x.abs() >= 1.0 { + 0.0 + } else { + bessel_i0(BETA * (1.0 - x * x).sqrt()) / i0_beta + }; + *wk = fc2 * sinc(fc2 * t) * window; + sum += *wk; + } + // Unity gain at DC on every phase. + for (o, wk) in row.iter_mut().zip(&w) { + *o = (wk / sum) as f32; + } + } + let channels = channels.max(1); + Self { + channels, + step, + den, + half, + phases, + table, + buf: Vec::new(), + base: 0, + end: 0, + finished: false, + pos: 0, + acc: vec![0.0; channels], + } + } + + pub fn channels(&self) -> usize { + self.channels + } + + /// History before the stream's start: `frames` only inform its first outputs. Only before any + /// input; ignored after. + pub fn prime(&mut self, frames: &[f32]) { + if self.end != 0 || self.finished { + return; + } + self.append(frames); + self.end = self.buffered_end(); + self.pos = self.end * self.den; + } + + /// The stream's next input frames (interleaved). + pub fn push(&mut self, frames: &[f32]) { + if self.finished { + return; + } + self.append(frames); + self.end = self.buffered_end(); + } + + /// The stream ends here: `lookahead` (the audio that would have followed, if any) only + /// informs the last outputs; past it, silence. [`Self::pull`] then yields the rest. + pub fn finish(&mut self, lookahead: &[f32]) { + if self.finished { + return; + } + self.append(lookahead); + self.finished = true; + } + + /// Output frames into `out` (interleaved; whole frames only), as many as are ready and fit. + /// Returns the samples written. Call again while it fills `out`. + pub fn pull(&mut self, out: &mut [f32]) -> usize { + let ch = self.channels; + let taps = 2 * self.half; + let have = self.buffered_end(); + let mut written = 0; + while written + ch <= out.len() { + let ipos = self.pos / self.den; + let ready = if self.finished { + ipos < self.end + } else { + ipos + (self.half as u64) < self.end + }; + if !ready { + break; + } + let frac = self.pos % self.den; + let r = if self.phases == self.den { + frac + } else { + (frac * self.phases + self.den / 2) / self.den + } as usize; + let row = &self.table[r * taps..(r + 1) * taps]; + let first = ipos as i64 - self.half as i64 + 1; + self.acc.iter_mut().for_each(|a| *a = 0.0); + if first >= self.base as i64 && first as u64 + taps as u64 <= have { + let from = (first as u64 - self.base) as usize * ch; + let frames = &self.buf[from..from + taps * ch]; + if ch == 2 { + let (mut l, mut rr) = (0f32, 0f32); + for (f, w) in frames.as_chunks::<2>().0.iter().zip(row) { + l += f[0] * w; + rr += f[1] * w; + } + self.acc[0] = l; + self.acc[1] = rr; + } else { + for (f, w) in frames.chunks_exact(ch).zip(row) { + for (a, s) in self.acc.iter_mut().zip(f) { + *a += s * w; + } + } + } + } else { + // At the edges: silence before the first frame and past the last. + for (k, w) in row.iter().enumerate() { + let i = first + k as i64; + if i < self.base as i64 || i as u64 >= have { + continue; + } + let at = (i as u64 - self.base) as usize * ch; + for (c, a) in self.acc.iter_mut().enumerate() { + *a += self.buf[at + c] * w; + } + } + } + out[written..written + ch].copy_from_slice(&self.acc); + written += ch; + self.pos += self.step; + } + self.compact(); + written + } + + fn append(&mut self, frames: &[f32]) { + let whole = frames.len() / self.channels * self.channels; + self.buf.extend_from_slice(&frames[..whole]); + } + + fn buffered_end(&self) -> u64 { + self.base + (self.buf.len() / self.channels) as u64 + } + + /// Drop input no output will read again (the next one reads from `ipos - half + 1`). + fn compact(&mut self) { + let keep_from = (self.pos / self.den).saturating_sub(self.half as u64 - 1); + if keep_from > self.base + COMPACT_FRAMES { + let drop = (keep_from - self.base) as usize; + let drop = drop.min(self.buf.len() / self.channels); + self.buf.drain(..drop * self.channels); + self.base += drop as u64; + } + } +} + +pub(crate) fn gcd(a: u64, b: u64) -> u64 { + if b == 0 { + a.max(1) + } else { + gcd(b, a % b) + } +} + +fn sinc(x: f64) -> f64 { + if x.abs() < 1e-12 { + 1.0 + } else { + (PI * x).sin() / (PI * x) + } +} + +/// The zeroth-order modified Bessel function of the first kind (series; converges fast for β ≤ 20). +fn bessel_i0(x: f64) -> f64 { + let (mut sum, mut term) = (1.0, 1.0); + let q = x * x / 4.0; + for k in 1..64 { + term *= q / (k * k) as f64; + sum += term; + if term < sum * 1e-17 { + break; + } + } + sum +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sine(frames: usize, channels: usize, rate: f64, freq: f64, offset: usize) -> Vec { + let mut v = Vec::with_capacity(frames * channels); + for i in 0..frames { + let s = (2.0 * PI * freq * (i + offset) as f64 / rate).sin() as f32 * 0.5; + for c in 0..channels { + v.push(if c == 0 { s } else { -s }); + } + } + v + } + + fn drain(r: &mut StreamResampler, out: &mut Vec) { + let mut chunk = vec![0f32; 1000 * r.channels()]; + loop { + let n = r.pull(&mut chunk); + out.extend_from_slice(&chunk[..n]); + if n < chunk.len() { + break; + } + } + } + + /// The whole of `input`, finished with `lookahead`, in one go. + fn convert( + in_rate: u32, + out_rate: u32, + ch: usize, + input: &[f32], + lookahead: &[f32], + ) -> Vec { + let mut r = StreamResampler::new(in_rate, out_rate, ch); + r.push(input); + r.finish(lookahead); + let mut out = Vec::new(); + drain(&mut r, &mut out); + out + } + + fn max_err(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) + } + + #[test] + fn output_length_is_exact() { + for (i, o, n) in [ + (44_100, 48_000, 44_100), + (48_000, 44_100, 12_345), + (96_000, 48_000, 1_001), + (22_050, 48_000, 7), + ] { + let out = convert(i, o, 2, &vec![0.1; n * 2], &[]); + let want = (n as u64 * u64::from(o)).div_ceil(u64::from(i)) as usize; + assert_eq!(out.len(), want * 2, "{i} -> {o}, {n} frames"); + } + } + + #[test] + fn a_sine_comes_through_clean_and_in_place() { + for (i, o) in [ + (44_100u32, 48_000u32), + (48_000, 44_100), + (96_000, 44_100), + (32_000, 48_000), + ] { + let input = sine(i as usize, 2, f64::from(i), 1_000.0, 0); + let out = convert(i, o, 2, &input, &[]); + // Away from the edges, output frame j is the sine at j / out seconds: no delay. + let want = sine(out.len() / 2, 2, f64::from(o), 1_000.0, 0); + let mid = 2 * 1_000..out.len() - 2 * 1_000; + let err = max_err(&out[mid.clone()], &want[mid]); + assert!(err < 1e-3, "{i} -> {o}: {err}"); + } + } + + #[test] + fn content_above_the_output_band_is_removed() { + // 23 kHz at 48 kHz would alias to 21.1 kHz at 44.1 kHz. + let input = sine(48_000, 1, 48_000.0, 23_000.0, 0); + let out = convert(48_000, 44_100, 1, &input, &[]); + let peak = out[4_000..out.len() - 4_000] + .iter() + .fold(0f32, |m, s| m.max(s.abs())); + assert!(peak < 0.5 * 1e-3, "{peak}"); + } + + #[test] + fn chunking_does_not_change_the_output() { + let input = sine(10_000, 2, 44_100.0, 440.0, 0); + let whole = convert(44_100, 48_000, 2, &input, &[]); + let mut r = StreamResampler::new(44_100, 48_000, 2); + let mut out = Vec::new(); + let mut at = 0; + for len in [1usize, 7, 300, 2, 1_500, 64, 4_000].iter().cycle() { + if at >= 10_000 { + break; + } + let end = (at + len).min(10_000); + r.push(&input[at * 2..end * 2]); + drain(&mut r, &mut out); + at = end; + } + r.finish(&[]); + drain(&mut r, &mut out); + assert_eq!(out, whole); + } + + #[test] + fn only_lookahead_is_held_back_until_finish() { + let mut r = StreamResampler::new(44_100, 48_000, 1); + r.push(&vec![0.2; 1_000]); + let mut out = Vec::new(); + drain(&mut r, &mut out); + let total = (1_000u64 * 48_000).div_ceil(44_100) as usize; + let held = total - out.len(); + // About `half` input frames' worth (64 × 48/44.1 ≈ 70). + assert!((60..=80).contains(&held), "{held}"); + } + + #[test] + fn finish_with_the_real_continuation_matches_an_uncut_stream() { + let input = sine(6_000, 2, 44_100.0, 3_000.0, 0); + let uncut = convert(44_100, 48_000, 2, &input, &[]); + let cut = convert( + 44_100, + 48_000, + 2, + &input[..4_000 * 2], + &input[4_000 * 2..4_400 * 2], + ); + let n = cut.len(); + assert_eq!(n, (4_000u64 * 48_000).div_ceil(44_100) as usize * 2); + assert!(max_err(&cut, &uncut[..n]) < 1e-6); + // Without it the last frames are a band-limited cut to silence: well off. + let bare = convert(44_100, 48_000, 2, &input[..4_000 * 2], &[]); + assert!(max_err(&bare[n - 20..], &uncut[n - 20..n]) > 0.05); + } + + #[test] + fn priming_with_the_audio_before_matches_an_uncut_stream() { + // The cut at 1470 input frames: an output frame lands exactly on it (× 160 / 147), and the + // sine isn't at a zero crossing there. + let input = sine(6_000, 2, 44_100.0, 2_500.0, 0); + let uncut = convert(44_100, 48_000, 2, &input, &[]); + let cut_at = 1_470; + let mut r = StreamResampler::new(44_100, 48_000, 2); + r.prime(&input[..cut_at * 2]); + r.push(&input[cut_at * 2..]); + r.finish(&[]); + let mut out = Vec::new(); + drain(&mut r, &mut out); + let skip = cut_at * 48_000 / 44_100 * 2; + assert_eq!(out.len(), uncut.len() - skip); + assert!(max_err(&out, &uncut[skip..]) < 1e-6); + // Unprimed, the first frames see silence before them: audibly off (a click). + let fresh = convert(44_100, 48_000, 2, &input[cut_at * 2..], &[]); + let fresh_err = max_err(&fresh[..20], &uncut[skip..skip + 20]); + assert!(fresh_err > 0.01, "{fresh_err}"); + } + + #[test] + fn long_streams_keep_memory_bounded() { + let mut r = StreamResampler::new(48_000, 44_100, 2); + let block = sine(1_024, 2, 48_000.0, 100.0, 0); + let mut out = vec![0f32; 4_096]; + for _ in 0..2_000 { + r.push(&block); + while r.pull(&mut out) == out.len() {} + } + assert!( + r.buf.len() < (COMPACT_FRAMES as usize + 2_048) * 2, + "{}", + r.buf.len() + ); + } +} diff --git a/docs/automix.md b/docs/automix.md index 9d0c3fb2..956bb547 100644 --- a/docs/automix.md +++ b/docs/automix.md @@ -378,8 +378,9 @@ The planner is deterministic: same inputs → same plan on every device. `store_analysis_record`, which replans the follow-up. One at a time, each track tried once per session. `on_analysis_produced` uploads the record (see *Sharing records*). -- **Android** (`ExoBackend.kt`, `MixProcessor.kt`, `PcmDecoder.kt`, - `LoudnessRecorder.kt`; Rust side `crates/hocket-android/src/dsp.rs`): +- **Android** (`ExoBackend.kt`, `MixProcessor.kt`, `OutputRate.kt`, + `PcmDecoder.kt`, `LoudnessRecorder.kt`; Rust side + `crates/hocket-android/src/dsp.rs`, `resampler.rs`): - **DSP**: a Media3 `BaseAudioProcessor` in the main player's `DefaultAudioSink` (PCM path only: 16-bit or float in, the same out; no float output, no offload) runs hocket-core's `DspChain` per buffer @@ -400,12 +401,36 @@ The planner is deterministic: same inputs → same plan on every device. renderer's `handleDiscontinuity` at every stream change forces a timing resync); a contiguous flush of the item already playing is the same stream going on and must not end a mix. - When B is at another sample rate the sink can't join gaplessly: it - drains A, flushes once still naming A (the new configuration takes over - after that flush), then flushes for B with no end of stream in between. - So every flush between an end of stream and the next frame belongs to - the join; without that, every mix between a 44.1 kHz and a 48 kHz track - was dropped and A cut off at its clip point. + When B is at another channel count the sink can't keep its + `AudioTrack`: it drains A, flushes once still naming A (the new + configuration takes over after that flush), then flushes for B with no + end of stream in between. So every flush between an end of stream and + the next frame belongs to the join. + - **Output rate** (`OutputRate`): the sink rebuilds its `AudioTrack` + whenever its output format changes, and a rebuild plays the old track + out to silence before the new one starts: a gap at every join between + tracks at different rates, in the middle of A's audio when the join is + a mix. So the processor keeps one output rate: the first stream's after + a reset when it is 44.1 or 48 kHz, else the device's + (`PROPERTY_OUTPUT_SAMPLE_RATE`). A library at one rate is never + resampled. Other streams go through a `StreamResampler` (JNI; a + Kaiser-windowed sinc with one row of taps per output phase, ~80 dB + stopband), after the mixer, so the mixer's sample-exact item accounting + is untouched. Output frame `j` is input frame `j × in / out` exactly, + and only the kernel's lookahead (~1.5 ms) is held back. + - A gapless join at the same format goes on through the same resampler: + no seam inside continuous audio. What it held back comes out after + the flush, which moves B's clock by ~1.5 ms (and at the very end of + playback, with nothing after, those ~1.5 ms are dropped). + - When the next stream is at another format (the sink configures it + while A plays), A's end of stream lets out A's last frames, with the + armed tail's `after` (A from the tail start through A's chain, at A's + format) as lookahead when the mix follows. B's resampler is primed + with the tail's `lead` (A just before the tail start, at B's format; + its frames line up with the tail's, both from one resampler run). So + the switch from A to the tail is as seamless as at one rate. + - The tail render decodes A from 30 ms before the tail start for that + (and so the tail's own first frames have filter history). - **Clipping**: an item with `mix_out_ms` gets `ClippingConfiguration(endPositionMs)` (on `load` only when its `next` has `mix_in`); the processor also drops frames past the clip end @@ -422,9 +447,9 @@ The planner is deterministic: same inputs → same plan on every device. stream reader as a `MediaDataSource`, `file:` from disk, else HTTP with the source's headers — probes B's rate and channels from its container, and calls `automix_prepare_tail`: a `DspChain` like A's (A's gain, the - EQ), channel map, **rubato** resampling to B's rate (the desktop - resampler, moved to `audio::resample` behind the feature `resample`, - which `hocket-android` enables; pure Rust), then `render_tail`. + EQ), channel map, resampling to B's rate (the `StreamResampler`, primed + with the 30 ms before the tail start), then `render_tail`; plus the + `lead` and `after` above. - **Mixing**: at B's first frame after a contiguous join, if the armed pair matches and the tail is at B's format, the processor outputs the tail's first `incoming_at_ms` at once, then B (its first From f48ba920a1ff6c00adfab44027cd27bb94b393ed Mon Sep 17 00:00:00 2001 From: Ingo Wolf Date: Tue, 10 Nov 2026 13:24:55 +1100 Subject: [PATCH 2/2] Output rate review: last track comes out whole; sturdier seams - The end of the playlist (MixClockSink.playToEndOfStream) finishes a resampled stream, so its held-back last ~1.5 ms is played. - A lead still primes a resampler that hasn't had input (a change of channel count flushes twice before the mix starts). - The finish path reads the output buffer after it may have grown. - prepare_tail floors the context in frames like the decoder and the clip point, sizes its buffer in u64, and the tail render decodes 60 ms of context so short-step rate pairs still get a lead. - The resampler JNI refuses one buffer as both input and output. Co-Authored-By: Claude Opus 5.5 --- .../java/app/hocket/playback/ExoBackend.kt | 5 +-- .../java/app/hocket/playback/MixClockSink.kt | 7 ++++ .../java/app/hocket/playback/MixProcessor.kt | 20 ++++++++--- .../java/app/hocket/playback/OutputRate.kt | 26 +++++++++++---- .../app/hocket/playback/OutputRateTest.kt | 33 +++++++++++++++++++ crates/hocket-android/src/dsp.rs | 13 ++++++-- docs/automix.md | 9 ++--- 7 files changed, 93 insertions(+), 20 deletions(-) diff --git a/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt b/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt index 2e618467..3e992591 100644 --- a/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt +++ b/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt @@ -166,7 +166,7 @@ class ExoBackend( /** A's tail is decoded and rendered once A is this close to its tail start. */ private const val PREPARE_AHEAD_MS = 60_000L /** A decoded before its tail start for the tail render (more than its resampler seam needs). */ - private const val TAIL_CONTEXT_MS = 30L + private const val TAIL_CONTEXT_MS = 60L /** * A clipped item whose tail isn't ready this long before its clip point is unclipped: the * sink and the decoder run up to about a second ahead of what is heard. @@ -256,7 +256,8 @@ class ExoBackend( val p = processor ?: return super.buildAudioSink(context, enableFloatOutput, enableAudioOutputPlaybackParams) // PCM path only: no float output, no offload (both are off by default), so every buffer // reaches the processor. - return MixClockSink(DefaultAudioSink.Builder(context).setAudioProcessors(arrayOf(p)).build()).also { clockSink = it } + val sink = DefaultAudioSink.Builder(context).setAudioProcessors(arrayOf(p)).build() + return MixClockSink(sink, onPlayToEnd = p::playToEnd).also { clockSink = it } } } diff --git a/android/playback/src/main/java/app/hocket/playback/MixClockSink.kt b/android/playback/src/main/java/app/hocket/playback/MixClockSink.kt index 97a715ad..c6806e97 100644 --- a/android/playback/src/main/java/app/hocket/playback/MixClockSink.kt +++ b/android/playback/src/main/java/app/hocket/playback/MixClockSink.kt @@ -23,6 +23,8 @@ import kotlin.math.abs class MixClockSink( sink: AudioSink, private val nanoTime: () -> Long = System::nanoTime, + /** The renderer plays out the end of the whole playlist (the next end of stream is the last). */ + private val onPlayToEnd: () -> Unit = {}, ) : ForwardingAudioSink(sink) { private val requested = AtomicLong() private var correctionUs = 0L @@ -75,6 +77,11 @@ class MixClockSink( return raw - correctionUs } + override fun playToEndOfStream() { + onPlayToEnd() + super.playToEndOfStream() + } + override fun flush() { clear() super.flush() diff --git a/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt b/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt index 36f84c07..d5e37e33 100644 --- a/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt +++ b/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt @@ -347,6 +347,8 @@ class MixProcessor( private var configured: AudioProcessor.AudioFormat? = null /** The stream ends at this end of stream: its resampler's last frames are still to come out. */ private var finishPending = false + /** The sink plays out the end of the playlist: the next end of stream is the last. */ + private var lastEnd = false private val period = Timeline.Period() private val window = Timeline.Window() @@ -360,18 +362,24 @@ class MixProcessor( return AudioProcessor.AudioFormat(out, inputAudioFormat.channelCount, inputAudioFormat.encoding) } + /** The sink is asked to play out the end of the playlist (on the playback thread, before its drain). */ + fun playToEnd() { + lastEnd = true + } + override fun onQueueEndOfStream() { endedBeforeFlush = true - val next = configured ?: return - // The next stream is at another format (it was configured while this one played): this - // resampler's held-back frames are this stream's last, and must come out before the switch. - if (rate.endsBefore(next.sampleRate, next.channelCount)) finishPending = true + // The next stream is at another format (it was configured while this one played), or there + // is none: this resampler's held-back frames are this stream's last, and must come out now. + val next = configured + if (rate.resampling && (lastEnd || (next != null && rate.endsBefore(next.sampleRate, next.channelCount)))) finishPending = true } override fun isEnded(): Boolean = super.isEnded() && !finishPending override fun onFlush(streamMetadata: AudioProcessor.StreamMetadata) { finishPending = false + lastEnd = false mixer.configure(inputAudioFormat.sampleRate, inputAudioFormat.channelCount) mixer.startItem(mediaIdOf(streamMetadata), streamMetadata.positionOffsetUs, endedBeforeFlush) val lead = mixer.tail?.takeIf { mixer.joined }?.lead @@ -385,6 +393,7 @@ class MixProcessor( configured = null endedBeforeFlush = false finishPending = false + lastEnd = false } override fun queueInput(inputBuffer: ByteBuffer) { @@ -393,7 +402,8 @@ class MixProcessor( if (finishPending) { // After the end of stream the pipeline asks for more with an empty buffer: the last frames. finishPending = false - emit(rate.output, finish(), format.channelCount, float) + val n = finish() + emit(rate.output, n, format.channelCount, float) return } val frames = inputBuffer.remaining() / format.bytesPerFrame diff --git a/android/playback/src/main/java/app/hocket/playback/OutputRate.kt b/android/playback/src/main/java/app/hocket/playback/OutputRate.kt index e2bcc99b..65c82770 100644 --- a/android/playback/src/main/java/app/hocket/playback/OutputRate.kt +++ b/android/playback/src/main/java/app/hocket/playback/OutputRate.kt @@ -28,6 +28,8 @@ class OutputRate(private val dsp: DspEngine, private val fallbackRate: Int) { var channels = 0 private set private var handle = 0L + /** The resampler hasn't had input yet (a lead can still prime it). */ + private var fresh = false /** Whether the current stream is converted (else it passes through as is). */ val resampling: Boolean get() = handle != 0L @@ -50,7 +52,12 @@ class OutputRate(private val dsp: DspEngine, private val fallbackRate: Int) { */ fun start(rate: Int, channels: Int, joined: Boolean, lead: FloatArray?) { if (outRate == 0) latch(rate) - if (joined && rate == inRate && channels == this.channels) return + if (joined && rate == inRate && channels == this.channels) { + // A stream that hasn't started yet can still take a lead: at a change of channel count + // the sink flushes twice before the first frame, and only the second starts the mix. + if (handle != 0L && fresh && lead != null && lead.size >= channels) prime(lead) + return + } free() inRate = rate this.channels = channels @@ -60,10 +67,14 @@ class OutputRate(private val dsp: DspEngine, private val fallbackRate: Int) { PlaybackLog.w("no resampler for $rate Hz -> $outRate Hz: plays at the wrong speed") return } - if (joined && lead != null && lead.size >= channels) { - val n = lead.size / channels * channels - dsp.resamplerPrime(handle, fill(lead, n), n) - } + fresh = true + if (joined && lead != null && lead.size >= channels) prime(lead) + } + + private fun prime(lead: FloatArray) { + val n = lead.size / channels * channels + dsp.resamplerPrime(handle, fill(lead, n), n) + fresh = false } /** Whether the stream must end before a next one at [rate] × [channels]: it is converted, and differently. */ @@ -85,7 +96,10 @@ class OutputRate(private val dsp: DspEngine, private val fallbackRate: Int) { } /** Convert [frames] input frames from [input]; returns the frames written to [output]. */ - fun process(input: ByteBuffer, frames: Int): Int = drain(input, frames * channels, frames) + fun process(input: ByteBuffer, frames: Int): Int { + fresh = false + return drain(input, frames * channels, frames) + } fun release() { free() diff --git a/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt b/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt index 39e81086..f9c51b89 100644 --- a/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt +++ b/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt @@ -182,4 +182,37 @@ class OutputRateTest { // held back again). assertEquals(160, queue(147)) } + + @Test + fun theLastStreamOfThePlaylistComesOutWhole() { + processor.configure(pcm16(48_000)) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + processor.configure(pcm16(44_100)) + processor.queueEndOfStream() + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + assertEquals(479, queue(441)) + processor.playToEnd() + processor.queueEndOfStream() + assertFalse(processor.isEnded) + processor.queueInput(ByteBuffer.allocateDirect(0)) + assertEquals(1, processor.output.remaining() / 2) + assertTrue(processor.isEnded) + } + + @Test + fun aLeadStillPrimesAStreamThatHasNotStarted() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(4, 1, joined = false, lead = null) + // A first flush for the new format without the mix, then the one that starts it. + r.start(3, 1, joined = true, lead = null) + r.start(3, 1, joined = true, lead = ramp(27, 3)) + val out = r.feed(ramp(30, 6)) + r.end() + assertArrayEquals(whole(ramp(0, 36)).copyOfRange(40, 48), out, 1e-6f) + // Once it has had input, a lead is too late. + r.start(3, 1, joined = false, lead = null) + r.feed(ramp(0, 3)) + r.start(3, 1, joined = true, lead = ramp(27, 3)) + assertEquals(1, dsp.live) + } } diff --git a/crates/hocket-android/src/dsp.rs b/crates/hocket-android/src/dsp.rs index d29c108e..9ea6af66 100644 --- a/crates/hocket-android/src/dsp.rs +++ b/crates/hocket-android/src/dsp.rs @@ -141,7 +141,8 @@ pub fn prepare_tail(plan: &TransitionPlan, input: TailInput<'_>) -> Result) -> Result jint { + // Two views of one buffer would be aliasing `&mut` slices. + if env.get_direct_buffer_address(&input).ok() == env.get_direct_buffer_address(&output).ok() + { + return 0; + } let Some(inp) = floats(&env, &input, in_samples) else { return 0; }; @@ -772,7 +779,7 @@ mod tests { let lead_frames = prepared.lead.len() / 2; assert_eq!(lead_frames, 480); let want_lead = sine_at(48_000, lead_frames, 1_440 - lead_frames, 2); - assert!(err(&prepared.lead[lead_frames..], &want_lead[lead_frames..]) < 1e-3); + assert!(err(&prepared.lead, &want_lead) < 1e-3); // The lookahead is A from the tail start at A's rate. assert_eq!( prepared.after, diff --git a/docs/automix.md b/docs/automix.md index 956bb547..aa7b798e 100644 --- a/docs/automix.md +++ b/docs/automix.md @@ -420,8 +420,9 @@ The planner is deterministic: same inputs → same plan on every device. and only the kernel's lookahead (~1.5 ms) is held back. - A gapless join at the same format goes on through the same resampler: no seam inside continuous audio. What it held back comes out after - the flush, which moves B's clock by ~1.5 ms (and at the very end of - playback, with nothing after, those ~1.5 ms are dropped). + the flush, which moves B's clock by ~1.5 ms. At the end of the + playlist (`MixClockSink.playToEndOfStream` tells the processor) they + come out at the end of stream. - When the next stream is at another format (the sink configures it while A plays), A's end of stream lets out A's last frames, with the armed tail's `after` (A from the tail start through A's chain, at A's @@ -429,7 +430,7 @@ The planner is deterministic: same inputs → same plan on every device. with the tail's `lead` (A just before the tail start, at B's format; its frames line up with the tail's, both from one resampler run). So the switch from A to the tail is as seamless as at one rate. - - The tail render decodes A from 30 ms before the tail start for that + - The tail render decodes A from 60 ms before the tail start for that (and so the tail's own first frames have filter history). - **Clipping**: an item with `mix_out_ms` gets `ClippingConfiguration(endPositionMs)` (on `load` only when its `next` @@ -448,7 +449,7 @@ The planner is deterministic: same inputs → same plan on every device. the source's headers — probes B's rate and channels from its container, and calls `automix_prepare_tail`: a `DspChain` like A's (A's gain, the EQ), channel map, resampling to B's rate (the `StreamResampler`, primed - with the 30 ms before the tail start), then `render_tail`; plus the + with the 60 ms before the tail start), then `render_tail`; plus the `lead` and `after` above. - **Mixing**: at B's first frame after a contiguous join, if the armed pair matches and the tail is at B's format, the processor outputs the