diff --git a/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt b/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt index 2ba38e0c..3e992591 100644 --- a/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt +++ b/android/playback/src/main/java/app/hocket/playback/ExoBackend.kt @@ -1,6 +1,7 @@ package app.hocket.playback import android.content.Context +import android.media.AudioManager import android.net.Uri import android.os.Handler import android.os.Looper @@ -164,6 +165,8 @@ class ExoBackend( private const val HANDOVER_POLL_MS = 40L /** A's tail is decoded and rendered once A is this close to its tail start. */ private const val PREPARE_AHEAD_MS = 60_000L + /** A decoded before its tail start for the tail render (more than its resampler seam needs). */ + private const val TAIL_CONTEXT_MS = 60L /** * A clipped item whose tail isn't ready this long before its clip point is unclipped: the * sink and the decoder run up to about a second ahead of what is heard. @@ -242,14 +245,19 @@ class ExoBackend( PlaybackLog.i("AutoMix: ${armed.from} -> ${armed.to} plays unmixed: $reason") } } - private val processor: MixProcessor? = dsp?.let { MixProcessor(it, mixControl, mixListener) } + private val processor: MixProcessor? = dsp?.let { MixProcessor(it, mixControl, mixListener, deviceRate()) } + + /** The device's output rate (what [OutputRate] uses when the first item isn't at 44.1 or 48 kHz). */ + private fun deviceRate(): Int = + (context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager)?.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE)?.toIntOrNull() ?: 48_000 private fun renderersFactory() = object : DefaultRenderersFactory(context) { override fun buildAudioSink(context: Context, enableFloatOutput: Boolean, enableAudioOutputPlaybackParams: Boolean): AudioSink? { val p = processor ?: return super.buildAudioSink(context, enableFloatOutput, enableAudioOutputPlaybackParams) // PCM path only: no float output, no offload (both are off by default), so every buffer // reaches the processor. - return MixClockSink(DefaultAudioSink.Builder(context).setAudioProcessors(arrayOf(p)).build()).also { clockSink = it } + val sink = DefaultAudioSink.Builder(context).setAudioProcessors(arrayOf(p)).build() + return MixClockSink(sink, onPlayToEnd = p::playToEnd).also { clockSink = it } } } @@ -1084,15 +1092,21 @@ class ExoBackend( mixControl.armed = null } - /** Decode A's tail, then DSP, resample to B's format and render it in Rust. Null on failure (logged). */ + /** + * Decode A's tail (from [TAIL_CONTEXT_MS] before it: filter history, and A's audio either side of + * the tail start for [OutputRate]), then DSP, resample to B's format and render it in Rust. Null + * on failure (logged). + */ private fun renderTail(a: CoreMediaSource, b: CoreMediaSource, armed: MixControl.Armed, settings: String?, cancelled: () -> Boolean): MixControl.Tail? = try { val started = System.nanoTime() val decoder = PcmDecoder(streams) - val pcm = decoder.decode(a.url, a.headers, armed.tailStartMs, armed.tailSourceMs + 50, cancelled) + val from = (armed.tailStartMs - TAIL_CONTEXT_MS).coerceAtLeast(0) + val context = armed.tailStartMs - from + val pcm = decoder.decode(a.url, a.headers, from, context + armed.tailSourceMs + 50, cancelled) val (rate, channels) = decoder.probe(b.url, b.headers) - val samples = ffi.prepareTail(armed.planJson, pcm, channels, rate, settings, a.gainDb) + val r = ffi.prepareTail(armed.planJson, pcm, context.toInt(), channels, rate, settings, a.gainDb) PlaybackLog.i("AutoMix: tail ${a.key} -> ${b.key} ready (${pcm.durationMs} ms of A at ${pcm.rate} Hz -> ${armed.tailLenMs} ms at $rate Hz) in ${(System.nanoTime() - started) / 1_000_000} ms") - MixControl.Tail(samples, rate, channels) + MixControl.Tail(r.tail, rate, channels, r.lead, r.after, pcm.rate, pcm.channels) } catch (e: InterruptedException) { null } catch (e: Throwable) { diff --git a/android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt b/android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt index c4911538..525fb603 100644 --- a/android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt +++ b/android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt @@ -108,12 +108,21 @@ internal class LoudnessRecorder( } } +/** + * A rendered tail ([tail], at B's format) and A's audio either side of its start: [lead] just before + * it at B's format, [after] from it at A's format (`MixControl.Tail`). + */ +class RenderedTail(val tail: FloatArray, val lead: FloatArray, val after: FloatArray) + /** The one-shot UniFFI calls of AutoMix, behind a seam (no native library in JVM tests). */ interface AutomixFfi { fun usable(record: AnalysisRecord, durationMs: Long): Boolean fun loudnessRecord(head: PcmDecoder.Pcm, tail: PcmDecoder.Pcm, tailStartMs: Long, durationMs: Long, transcoded: Boolean): AnalysisRecord? - /** The rendered tail as interleaved floats at [outRate] × [outChannels]. */ - fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): FloatArray + /** + * The rendered tail from [pcm] (A from [contextMs] before the tail start), as interleaved floats + * at [outRate] × [outChannels], with A's audio either side of the tail start. + */ + fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): RenderedTail object Native : AutomixFfi { override fun usable(record: AnalysisRecord, durationMs: Long): Boolean = @@ -128,10 +137,14 @@ interface AutomixFfi { return HocketJson.json.decodeFromString(AnalysisRecord.serializer(), json) } - override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): FloatArray { - val bytes = app.hocket.core.ffi.automixPrepareTail( - planJson, pcm.leBytes(), pcm.channels.toUInt(), pcm.rate.toUInt(), outChannels.toUInt(), outRate.toUInt(), audioSettingsJson, gainDb, + override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): RenderedTail { + val r = app.hocket.core.ffi.automixPrepareTail( + planJson, pcm.leBytes(), contextMs.toUInt(), pcm.channels.toUInt(), pcm.rate.toUInt(), outChannels.toUInt(), outRate.toUInt(), audioSettingsJson, gainDb, ) + return RenderedTail(floats(r.tail), floats(r.lead), floats(r.after)) + } + + private fun floats(bytes: ByteArray): FloatArray { val floats = java.nio.ByteBuffer.wrap(bytes).order(java.nio.ByteOrder.LITTLE_ENDIAN).asFloatBuffer() return FloatArray(floats.remaining()).also { floats.get(it) } } diff --git a/android/playback/src/main/java/app/hocket/playback/MixClockSink.kt b/android/playback/src/main/java/app/hocket/playback/MixClockSink.kt index 97a715ad..c6806e97 100644 --- a/android/playback/src/main/java/app/hocket/playback/MixClockSink.kt +++ b/android/playback/src/main/java/app/hocket/playback/MixClockSink.kt @@ -23,6 +23,8 @@ import kotlin.math.abs class MixClockSink( sink: AudioSink, private val nanoTime: () -> Long = System::nanoTime, + /** The renderer plays out the end of the whole playlist (the next end of stream is the last). */ + private val onPlayToEnd: () -> Unit = {}, ) : ForwardingAudioSink(sink) { private val requested = AtomicLong() private var correctionUs = 0L @@ -75,6 +77,11 @@ class MixClockSink( return raw - correctionUs } + override fun playToEndOfStream() { + onPlayToEnd() + super.playToEndOfStream() + } + override fun flush() { clear() super.flush() diff --git a/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt b/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt index 5bd482e4..d5e37e33 100644 --- a/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt +++ b/android/playback/src/main/java/app/hocket/playback/MixProcessor.kt @@ -18,8 +18,20 @@ class MixControl { /** A playlist item by media id (the queue key). */ data class Item(val gainDb: Double, val clipEndMs: Long?) - /** A rendered tail: interleaved floats at B's rate and channel count. */ - class Tail(val samples: FloatArray, val rate: Int, val channels: Int) { + /** + * A rendered tail: interleaved floats at B's rate and channel count. [lead] is A just before the + * tail (B's format) and [after] A from the tail's start (A's format, [afterRate] × + * [afterChannels]): the [OutputRate] seam when A and B are converted differently. + */ + class Tail( + val samples: FloatArray, + val rate: Int, + val channels: Int, + val lead: FloatArray = FloatArray(0), + val after: FloatArray = FloatArray(0), + val afterRate: Int = 0, + val afterChannels: Int = 0, + ) { val frames: Int get() = samples.size / channels.coerceAtLeast(1) } @@ -110,6 +122,12 @@ class TransitionMixer( val mixing: Boolean get() = mix != null + /** The running mix's tail. */ + val tail: MixControl.Tail? get() = mix?.tail + + /** The last [startItem] continued the stream before it (a gapless join, or the same item going on). */ + val joined: Boolean get() = joining + fun configure(rate: Int, channels: Int) { if (chain == 0L) { chain = dsp.chainNew(rate, channels) @@ -126,11 +144,12 @@ class TransitionMixer( * item's end without a seek (a gapless join), else a seek or a fresh start. */ fun startItem(mediaId: String?, positionUs: Long, contiguous: Boolean) { - // Until a frame comes, a flush at the item's start still belongs to the join. When B's format - // differs from A's (another sample rate) the sink can't join gaplessly: it drains A, flushes - // once still naming A (the new configuration takes over after that flush), then flushes for - // B without another end of stream. Taken as a seek, that second flush would leave B unmixed. - // A seek while the sink waits for A to play out lands elsewhere and stays a seek. + // Until a frame comes, a flush at the item's start still belongs to the join. When the sink + // can't keep its `AudioTrack` for B (another channel count; the rate is always the same) + // it drains A, flushes once still naming A (the new configuration takes over after that + // flush), then flushes for B without another end of stream. Taken as a seek, that second + // flush would leave B unmixed. A seek while the sink waits for A to play out lands + // elsewhere and stays a seek. val joined = contiguous || (joining && itemFrames == 0L && positionUs < JOIN_START_US) joining = joined if (joined && mediaId != null && mediaId == item) { @@ -298,8 +317,9 @@ class TransitionMixer( } /** - * The ExoPlayer side of [TransitionMixer] (`docs/automix.md` → Android): installed in the main - * player's `DefaultAudioSink`, on the PCM path only (no offload, 16-bit or float in, the same out). + * The ExoPlayer side of [TransitionMixer] and [OutputRate] (`docs/automix.md` → Android): installed + * in the main player's `DefaultAudioSink`, on the PCM path only (no offload, 16-bit or float in, the + * same out). * * Item boundaries come from the sink itself: at a gapless join it drains the processors * (`queueEndOfStream`) and flushes them with the new stream's [AudioProcessor.StreamMetadata], @@ -307,13 +327,28 @@ class TransitionMixer( * So the first frame after such a flush is the item's first frame, sample-exact, without counting * frames against durations. The sink then drains and flushes once more for the same item (a timing * resync after every stream change); [TransitionMixer.startItem] lets that through as the same stream. - * At a format change (B at another sample rate) the sink drains A and flushes twice before B's first - * frame, the first time still naming A and the second without an end of stream before it; - * [TransitionMixer.startItem] takes every flush before the next frame as part of the join. + * + * Every item leaves at one rate ([OutputRate]), so a join at another sample rate is a gapless join + * like any other: the sink keeps its `AudioTrack`. (A change of channel count still rebuilds it: + * the sink drains A and flushes twice before B's first frame, the first time still naming A; + * [TransitionMixer.startItem] takes every flush before the next frame as part of the join.) */ -class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) : BaseAudioProcessor() { +class MixProcessor( + dsp: DspEngine, + private val control: MixControl, + listener: MixListener, + /** The output rate when the first item isn't at 44.1 or 48 kHz: the device's. */ + fallbackRate: Int, +) : BaseAudioProcessor() { val mixer = TransitionMixer(dsp, control, listener) + val rate = OutputRate(dsp, fallbackRate) private var endedBeforeFlush = false + /** The input format the sink configured last: the next stream's, once it differs. */ + private var configured: AudioProcessor.AudioFormat? = null + /** The stream ends at this end of stream: its resampler's last frames are still to come out. */ + private var finishPending = false + /** The sink plays out the end of the playlist: the next end of stream is the last. */ + private var lastEnd = false private val period = Timeline.Period() private val window = Timeline.Window() @@ -322,31 +357,60 @@ class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) : PlaybackLog.w("DSP bypassed for PCM encoding ${inputAudioFormat.encoding}") return AudioProcessor.AudioFormat.NOT_SET } - return inputAudioFormat + configured = inputAudioFormat + val out = rate.latch(inputAudioFormat.sampleRate) + return AudioProcessor.AudioFormat(out, inputAudioFormat.channelCount, inputAudioFormat.encoding) + } + + /** The sink is asked to play out the end of the playlist (on the playback thread, before its drain). */ + fun playToEnd() { + lastEnd = true } override fun onQueueEndOfStream() { endedBeforeFlush = true + // The next stream is at another format (it was configured while this one played), or there + // is none: this resampler's held-back frames are this stream's last, and must come out now. + val next = configured + if (rate.resampling && (lastEnd || (next != null && rate.endsBefore(next.sampleRate, next.channelCount)))) finishPending = true } + override fun isEnded(): Boolean = super.isEnded() && !finishPending + override fun onFlush(streamMetadata: AudioProcessor.StreamMetadata) { + finishPending = false + lastEnd = false mixer.configure(inputAudioFormat.sampleRate, inputAudioFormat.channelCount) mixer.startItem(mediaIdOf(streamMetadata), streamMetadata.positionOffsetUs, endedBeforeFlush) + val lead = mixer.tail?.takeIf { mixer.joined }?.lead + rate.start(inputAudioFormat.sampleRate, inputAudioFormat.channelCount, mixer.joined, lead) endedBeforeFlush = false } override fun onReset() { mixer.release() + rate.release() + configured = null endedBeforeFlush = false + finishPending = false + lastEnd = false } override fun queueInput(inputBuffer: ByteBuffer) { val format = inputAudioFormat val float = format.encoding == C.ENCODING_PCM_FLOAT + if (finishPending) { + // After the end of stream the pipeline asks for more with an empty buffer: the last frames. + finishPending = false + val n = finish() + emit(rate.output, n, format.channelCount, float) + return + } val frames = inputBuffer.remaining() / format.bytesPerFrame if (frames == 0) return - if (mixer.passThrough(frames)) { - // Nothing to do (no mix, nothing clipped, a passthrough chain): the input as is. + val pass = mixer.passThrough(frames) + if (pass && !rate.resampling) { + // Nothing to do (no mix, nothing clipped, a passthrough chain, the output rate): the input as is. replaceOutputBuffer(inputBuffer.remaining()).put(inputBuffer).flip() return } @@ -359,14 +423,36 @@ class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) : for (i in 0 until samples) work.putFloat(i * 4, inputBuffer.getShort(start + i * 2) / 32768f) } inputBuffer.position(inputBuffer.limit()) - val outFrames = mixer.process(frames) - val outSamples = outFrames * format.channelCount - val out = replaceOutputBuffer(outFrames * format.bytesPerFrame) - val src = mixer.output + var src = work + var n = frames + if (!pass) { + n = mixer.process(frames) + src = mixer.output + } + if (rate.resampling) { + n = rate.process(src, n) + src = rate.output + } + emit(src, n, format.channelCount, float) + } + + /** The stream's last frames, with A's continuation as lookahead when the armed mix goes on from here. */ + private fun finish(): Int { + val armed = control.armed + val tail = armed?.tail + val after = tail?.takeIf { + armed.from == mixer.item && it.afterRate == rate.inRate && it.afterChannels == rate.channels + }?.after + return rate.finish(after) + } + + private fun emit(src: ByteBuffer, frames: Int, channels: Int, float: Boolean) { + val samples = frames * channels + val out = replaceOutputBuffer(samples * if (float) 4 else 2) if (float) { - for (i in 0 until outSamples) out.putFloat(src.getFloat(i * 4)) + for (i in 0 until samples) out.putFloat(src.getFloat(i * 4)) } else { - for (i in 0 until outSamples) out.putShort(toPcm16(src.getFloat(i * 4))) + for (i in 0 until samples) out.putShort(toPcm16(src.getFloat(i * 4))) } out.flip() } diff --git a/android/playback/src/main/java/app/hocket/playback/NativeDsp.kt b/android/playback/src/main/java/app/hocket/playback/NativeDsp.kt index 53525fbc..54673035 100644 --- a/android/playback/src/main/java/app/hocket/playback/NativeDsp.kt +++ b/android/playback/src/main/java/app/hocket/playback/NativeDsp.kt @@ -25,6 +25,22 @@ interface DspEngine { fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int) /** The chain's soft limiter (−1 dB knee), for the sum of a tail and B. */ fun limit(buffer: ByteBuffer, samples: Int) + + /** + * A `StreamResampler` (`crates/hocket-android/src/resampler.rs`): one stream from [inRate] to + * [outRate], output frame `j` at input frame `j × inRate / outRate` exactly. + */ + fun resamplerNew(inRate: Int, outRate: Int, channels: Int): Long + fun resamplerFree(handle: Long) + /** Frames before the stream's start, as history only. Before any input. */ + fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) + /** The stream ends: frames that would have followed, as lookahead only (0 samples: silence). */ + fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) + /** + * Take [inSamples] from [input], then write what is ready into [output] from sample [outOffset] + * up to [outCapacity]; returns the samples written. Call again with no input while it fills it. + */ + fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int): Int } /** @@ -56,4 +72,9 @@ object NativeDsp : DspEngine { external override fun shaperFree(handle: Long) external override fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int) external override fun limit(buffer: ByteBuffer, samples: Int) + external override fun resamplerNew(inRate: Int, outRate: Int, channels: Int): Long + external override fun resamplerFree(handle: Long) + external override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) + external override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) + external override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int): Int } diff --git a/android/playback/src/main/java/app/hocket/playback/OutputRate.kt b/android/playback/src/main/java/app/hocket/playback/OutputRate.kt new file mode 100644 index 00000000..65c82770 --- /dev/null +++ b/android/playback/src/main/java/app/hocket/playback/OutputRate.kt @@ -0,0 +1,152 @@ +package app.hocket.playback + +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * The last stage of [MixProcessor]: everything leaves at one sample rate (`docs/automix.md` → + * Android → Output rate), so the sink never rebuilds its `AudioTrack` between items. A rebuild means + * the old track plays out to silence before the new one starts: a gap at every join between + * tracks at different rates, and in the middle of A's audio when AutoMix mixes across rates. + * + * - **The rate** is the first stream's after a reset when that is 44.1 or 48 kHz, else [fallbackRate] + * (the device's). A library at one rate is never resampled; the rest is converted to it. + * - **Streams**: [start] at every flush. A stream joined gaplessly at the same format goes on through + * the same resampler (no seam inside continuous audio); anything else starts a new one. + * - **Seams**: when the next stream is at another format, the old one [finish]es at its end of + * stream, with A's real continuation as lookahead when a mix follows; the new one is primed with + * the mix tail's lead. So a mix across rates is as seamless as one at a single rate. + * + * Free of Media3 so JVM tests can drive it; floats in direct buffers, native order. + */ +class OutputRate(private val dsp: DspEngine, private val fallbackRate: Int) { + /** The rate everything leaves at; 0 until the first [latch]. */ + var outRate = 0 + private set + var inRate = 0 + private set + var channels = 0 + private set + private var handle = 0L + /** The resampler hasn't had input yet (a lead can still prime it). */ + private var fresh = false + + /** Whether the current stream is converted (else it passes through as is). */ + val resampling: Boolean get() = handle != 0L + + /** Output floats of the last [process] or [finish], from index 0. */ + var output: ByteBuffer = floatBuffer(4096) + private set + private var scratch: ByteBuffer = floatBuffer(1024) + + /** The output rate for a stream at [rate], fixed by the first stream after a [release]. */ + fun latch(rate: Int): Int { + if (outRate == 0) outRate = if (rate == 44_100 || rate == 48_000 || fallbackRate <= 0) rate else fallbackRate + return outRate + } + + /** + * A stream (re)starts at [rate] × [channels]: [joined] when it follows the previous one's end + * without a seek. [lead] (at this stream's format) is the audio just before it, when a mix tail + * opens it: history for the resampler, not output. + */ + fun start(rate: Int, channels: Int, joined: Boolean, lead: FloatArray?) { + if (outRate == 0) latch(rate) + if (joined && rate == inRate && channels == this.channels) { + // A stream that hasn't started yet can still take a lead: at a change of channel count + // the sink flushes twice before the first frame, and only the second starts the mix. + if (handle != 0L && fresh && lead != null && lead.size >= channels) prime(lead) + return + } + free() + inRate = rate + this.channels = channels + if (rate == outRate || rate <= 0) return + handle = dsp.resamplerNew(rate, outRate, channels) + if (handle == 0L) { + PlaybackLog.w("no resampler for $rate Hz -> $outRate Hz: plays at the wrong speed") + return + } + fresh = true + if (joined && lead != null && lead.size >= channels) prime(lead) + } + + private fun prime(lead: FloatArray) { + val n = lead.size / channels * channels + dsp.resamplerPrime(handle, fill(lead, n), n) + fresh = false + } + + /** Whether the stream must end before a next one at [rate] × [channels]: it is converted, and differently. */ + fun endsBefore(rate: Int, channels: Int): Boolean = handle != 0L && (rate != inRate || channels != this.channels) + + /** + * End the stream: its last frames into [output], with [lookahead] (at its format: the audio that + * would follow) informing them. Returns the frames written. The next [start] begins anew. + */ + fun finish(lookahead: FloatArray?): Int { + if (handle == 0L) return 0 + val ch = channels.coerceAtLeast(1) + val n = if (lookahead != null) lookahead.size / ch * ch else 0 + dsp.resamplerFinish(handle, fill(lookahead ?: FloatArray(0), n), n) + val frames = drain(scratch, 0, 64) + free() + inRate = 0 + return frames + } + + /** Convert [frames] input frames from [input]; returns the frames written to [output]. */ + fun process(input: ByteBuffer, frames: Int): Int { + fresh = false + return drain(input, frames * channels, frames) + } + + fun release() { + free() + outRate = 0 + inRate = 0 + channels = 0 + } + + private fun drain(input: ByteBuffer, inSamples: Int, inFrames: Int): Int { + val ch = channels.coerceAtLeast(1) + // What `inFrames` make plus what the kernel held back; [resamplerProcess] is called again + // while it fills the buffer, so this is only the usual case. + val want = ((inFrames.toLong() * outRate / inRate.coerceAtLeast(1)) + SLACK_FRAMES).toInt() * ch + ensureOutput(want, 0) + var written = dsp.resamplerProcess(handle, input, inSamples, output, 0, output.capacity() / 4) + while (written >= output.capacity() / 4 - ch) { + ensureOutput(output.capacity() / 4 * 2, written) + val more = dsp.resamplerProcess(handle, input, 0, output, written, output.capacity() / 4) + if (more == 0) break + written += more + } + return written / ch + } + + private fun free() { + if (handle != 0L) dsp.resamplerFree(handle) + handle = 0L + } + + private fun fill(samples: FloatArray, n: Int): ByteBuffer { + if (scratch.capacity() < n * 4) scratch = floatBuffer(n) + for (i in 0 until n) scratch.putFloat(i * 4, samples[i]) + return scratch + } + + /** Make [output] hold [samples] floats, keeping its first [keep]. */ + private fun ensureOutput(samples: Int, keep: Int) { + if (output.capacity() >= samples * 4) return + val grown = floatBuffer(samples) + for (i in 0 until keep) grown.putFloat(i * 4, output.getFloat(i * 4)) + output = grown + } + + private companion object { + /** More than the widest kernel holds back (256 input frames). */ + const val SLACK_FRAMES = 600L + + fun floatBuffer(samples: Int): ByteBuffer = ByteBuffer.allocateDirect(samples.coerceAtLeast(1) * 4).order(ByteOrder.nativeOrder()) + } +} diff --git a/android/playback/src/test/java/app/hocket/playback/ExoBackendClipTest.kt b/android/playback/src/test/java/app/hocket/playback/ExoBackendClipTest.kt index 8852e7e8..77e9ebd3 100644 --- a/android/playback/src/test/java/app/hocket/playback/ExoBackendClipTest.kt +++ b/android/playback/src/test/java/app/hocket/playback/ExoBackendClipTest.kt @@ -49,13 +49,18 @@ class ExoBackendClipTest { override fun shaperFree(handle: Long) = Unit override fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int) = Unit override fun limit(buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerNew(inRate: Int, outRate: Int, channels: Int) = 0L + override fun resamplerFree(handle: Long) = Unit + override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int) = 0 } /** Every render fails, like a tail that never gets ready. */ private object FailingFfi : AutomixFfi { override fun usable(record: AnalysisRecord, durationMs: Long) = true override fun loudnessRecord(head: PcmDecoder.Pcm, tail: PcmDecoder.Pcm, tailStartMs: Long, durationMs: Long, transcoded: Boolean): AnalysisRecord? = null - override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): FloatArray = + override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): RenderedTail = throw IllegalStateException("no tail") } diff --git a/android/playback/src/test/java/app/hocket/playback/FakeResamplerDsp.kt b/android/playback/src/test/java/app/hocket/playback/FakeResamplerDsp.kt new file mode 100644 index 00000000..19d14708 --- /dev/null +++ b/android/playback/src/test/java/app/hocket/playback/FakeResamplerDsp.kt @@ -0,0 +1,89 @@ +package app.hocket.playback + +import java.nio.ByteBuffer + +/** + * A Rust-free [DspEngine] for JVM tests: a passthrough chain, and resamplers with the native one's + * contract (`crates/hocket-android/src/resampler.rs`) on a linear kernel: output frame `j` is the + * input at `j × in / out` exactly, one frame of lookahead is held back until [resamplerFinish], and + * [resamplerPrime] / [resamplerFinish] take history / lookahead that is never output itself. + */ +class FakeResamplerDsp : DspEngine { + private class Stream(val inRate: Long, val outRate: Long, val channels: Int) { + val input = ArrayList() + /** Input frames of the stream (primed history included). */ + var end = 0L + var finished = false + /** Next output frame's position, in units of `1 / outRate` input frames. */ + var pos = 0L + } + + private val streams = HashMap() + private var next = 1L + var created = 0 + private set + val live: Int get() = streams.size + + override fun chainNew(rate: Int, channels: Int) = 1L + override fun chainFree(handle: Long) = Unit + override fun chainSetFormat(handle: Long, rate: Int, channels: Int) = Unit + override fun chainSetAudioSettings(handle: Long, json: String) = true + override fun chainSetTrackGain(handle: Long, gainDb: Double) = Unit + override fun chainReset(handle: Long) = Unit + override fun chainIsBypassed(handle: Long) = true + override fun chainProcess(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun shaperNew(planJson: String, channels: Int, rate: Int) = 1L + override fun shaperFree(handle: Long) = Unit + override fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun limit(buffer: ByteBuffer, samples: Int) = Unit + + override fun resamplerNew(inRate: Int, outRate: Int, channels: Int): Long { + created++ + return next++.also { streams[it] = Stream(inRate.toLong(), outRate.toLong(), channels) } + } + + override fun resamplerFree(handle: Long) { + check(streams.remove(handle) != null) { "freed twice" } + } + + override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) { + val s = streams.getValue(handle) + check(s.end == 0L) { "primed after input" } + append(s, buffer, samples) + s.end = (s.input.size / s.channels).toLong() + s.pos = s.end * s.outRate + } + + override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) { + val s = streams.getValue(handle) + append(s, buffer, samples) + s.finished = true + } + + override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int): Int { + val s = streams.getValue(handle) + if (!s.finished) { + append(s, input, inSamples) + s.end = (s.input.size / s.channels).toLong() + } + val have = (s.input.size / s.channels).toLong() + var at = outOffset + while (at + s.channels <= outCapacity) { + val i = s.pos / s.outRate + val ready = if (s.finished) i < s.end else i + 1 < s.end + if (!ready) break + val frac = (s.pos % s.outRate).toFloat() / s.outRate + for (c in 0 until s.channels) { + fun x(k: Long) = if (k < have) s.input[(k * s.channels + c).toInt()] else 0f + output.putFloat(at * 4, x(i) + frac * (x(i + 1) - x(i))) + at++ + } + s.pos += s.inRate + } + return at - outOffset + } + + private fun append(s: Stream, buffer: ByteBuffer, samples: Int) { + for (i in 0 until samples / s.channels * s.channels) s.input += buffer.getFloat(i * 4) + } +} diff --git a/android/playback/src/test/java/app/hocket/playback/LoudnessRecorderTest.kt b/android/playback/src/test/java/app/hocket/playback/LoudnessRecorderTest.kt index 1d2473e4..f1a7af51 100644 --- a/android/playback/src/test/java/app/hocket/playback/LoudnessRecorderTest.kt +++ b/android/playback/src/test/java/app/hocket/playback/LoudnessRecorderTest.kt @@ -19,7 +19,8 @@ class LoudnessRecorderTest { private object NoFfi : AutomixFfi { override fun usable(record: AnalysisRecord, durationMs: Long) = false override fun loudnessRecord(head: PcmDecoder.Pcm, tail: PcmDecoder.Pcm, tailStartMs: Long, durationMs: Long, transcoded: Boolean): AnalysisRecord? = null - override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double) = FloatArray(0) + override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double) = + RenderedTail(FloatArray(0), FloatArray(0), FloatArray(0)) } @Test diff --git a/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt b/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt new file mode 100644 index 00000000..f9c51b89 --- /dev/null +++ b/android/playback/src/test/java/app/hocket/playback/OutputRateTest.kt @@ -0,0 +1,218 @@ +package app.hocket.playback + +import androidx.media3.common.C +import androidx.media3.common.audio.AudioProcessor +import java.nio.ByteBuffer +import java.nio.ByteOrder +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +/** [OutputRate] and its place in [MixProcessor], on [FakeResamplerDsp] (plain JVM). */ +class OutputRateTest { + private val dsp = FakeResamplerDsp() + + private fun floats(values: FloatArray): ByteBuffer { + val b = ByteBuffer.allocateDirect(values.size * 4).order(ByteOrder.nativeOrder()) + values.forEachIndexed { i, v -> b.putFloat(i * 4, v) } + return b + } + + private fun OutputRate.feed(values: FloatArray): FloatArray { + val n = process(floats(values), values.size / channels) + return FloatArray(n * channels) { output.getFloat(it * 4) } + } + + private fun OutputRate.end(lookahead: FloatArray? = null): FloatArray { + val ch = channels + val n = finish(lookahead) + return FloatArray(n * ch) { output.getFloat(it * 4) } + } + + /** A ramp, so every frame is distinct and a linear kernel reproduces it exactly. */ + private fun ramp(from: Int, n: Int) = FloatArray(n) { (from + it) / 1_000f } + + /** [input] at 3 Hz converted to 4 Hz in one stream, finished with [lookahead]. */ + private fun whole(input: FloatArray, lookahead: FloatArray? = null): FloatArray { + val r = OutputRate(FakeResamplerDsp(), 4) + r.latch(4) + r.start(3, 1, joined = false, lead = null) + return r.feed(input) + r.end(lookahead) + } + + @Test + fun theFirstStreamAt44Or48FixesTheRateElseTheDevicesIs() { + assertEquals(44_100, OutputRate(dsp, 48_000).latch(44_100)) + val r = OutputRate(dsp, 48_000) + assertEquals(48_000, r.latch(96_000)) + assertEquals(48_000, r.latch(44_100)) + r.release() + assertEquals(44_100, r.latch(44_100)) + } + + @Test + fun aStreamAtTheOutputRatePassesThrough() { + val r = OutputRate(dsp, 48_000) + r.latch(44_100) + r.start(44_100, 2, joined = false, lead = null) + assertFalse(r.resampling) + assertEquals(0, dsp.created) + r.start(48_000, 2, joined = true, lead = null) + assertTrue(r.resampling) + } + + @Test + fun aGaplessJoinAtTheSameFormatGoesOnThroughTheSameResampler() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(3, 1, joined = false, lead = null) + val a = r.feed(ramp(0, 30)) + r.start(3, 1, joined = true, lead = null) + val b = r.feed(ramp(30, 30)) + assertEquals(1, dsp.created) + val all = a + b + r.end() + assertArrayEquals(whole(ramp(0, 60)), all, 0f) + assertEquals(80, all.size) + } + + @Test + fun aSeekStartsANewStream() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(3, 1, joined = false, lead = null) + r.feed(ramp(0, 30)) + r.start(3, 1, joined = false, lead = null) + assertEquals(2, dsp.created) + assertEquals(1, dsp.live) + // Output frame 0 of the new stream is its input frame 0. + assertEquals(0.5f, r.feed(ramp(500, 3))[0], 0f) + } + + @Test + fun aStreamEndsExactlyAndWithTheLookaheadLikeAnUncutOne() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(3, 1, joined = false, lead = null) + assertTrue(r.endsBefore(4, 1)) + assertFalse(r.endsBefore(3, 1)) + val out = r.feed(ramp(0, 30)) + r.end(ramp(30, 5)) + // 30 frames at 3 Hz are 40 at 4 Hz, the last one using the continuation. + assertArrayEquals(whole(ramp(0, 40)).copyOf(40), out, 1e-6f) + assertEquals(0, dsp.live) + assertFalse(r.resampling) + } + + @Test + fun aLeadPrimesTheNextStreamSoItStartsWhereTheOldOneStopped() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(4, 1, joined = false, lead = null) + assertFalse(r.resampling) + // B at 3 Hz opens with a tail whose lead is the 3 frames before it. + r.start(3, 1, joined = true, lead = ramp(27, 3)) + val out = r.feed(ramp(30, 6)) + r.end() + // Exactly the uncut stream's frames from frame 30 (output frame 40) on. + assertArrayEquals(whole(ramp(0, 36)).copyOfRange(40, 48), out, 1e-6f) + // The fake's kernel only looks ahead, so this is about position: B's first output frame is + // its first input frame, not a lead frame. + assertEquals(0.03f, out[0], 1e-6f) + } + + // -- In MixProcessor --------------------------------------------------------------------- + + private val processor = MixProcessor(dsp, MixControl(), object : MixListener { + override fun onMixStarted(armed: MixControl.Armed) = Unit + override fun onMixSkipped(armed: MixControl.Armed, reason: String) = Unit + }, fallbackRate = 48_000) + + private fun pcm16(rate: Int) = AudioProcessor.AudioFormat(rate, 1, C.ENCODING_PCM_16BIT) + + private fun queue(frames: Int): Int { + val b = ByteBuffer.allocateDirect(frames * 2).order(ByteOrder.nativeOrder()) + for (i in 0 until frames) b.putShort(i * 2, 1_000) + processor.queueInput(b) + return processor.output.remaining() / 2 + } + + @Test + fun theOutputRateStaysFixedAcrossItemsAtOtherRates() { + assertEquals(48_000, processor.configure(pcm16(48_000)).sampleRate) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + assertEquals(480, queue(480)) + assertEquals(pcm16(48_000), processor.configure(pcm16(44_100))) + assertEquals(pcm16(48_000), processor.configure(pcm16(96_000))) + } + + @Test + fun aResampledStreamComesOutWholeBeforeAFormatChange() { + processor.configure(pcm16(48_000)) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + processor.configure(pcm16(44_100)) + processor.queueEndOfStream() + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + // A at 44.1 kHz into 48 kHz: one frame held back for lookahead. + assertEquals(479, queue(441)) + // B at 48 kHz is configured while A plays; A's end of stream lets out its last frame. + processor.configure(pcm16(48_000)) + processor.queueEndOfStream() + assertFalse("the held-back frame is still to come", processor.isEnded) + processor.queueInput(ByteBuffer.allocateDirect(0)) + assertEquals(1, processor.output.remaining() / 2) + assertTrue(processor.isEnded) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + assertEquals(0, dsp.live) + assertEquals(100, queue(100)) + } + + @Test + fun anEndOfStreamAtTheSameFormatHoldsTheResamplerOpen() { + processor.configure(pcm16(48_000)) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + processor.configure(pcm16(44_100)) + processor.queueEndOfStream() + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + queue(441) + processor.queueEndOfStream() + assertTrue(processor.isEnded) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + assertEquals(1, dsp.live) + // The held-back frame comes out with the next item's first: 147 frames in, 160 out (one + // held back again). + assertEquals(160, queue(147)) + } + + @Test + fun theLastStreamOfThePlaylistComesOutWhole() { + processor.configure(pcm16(48_000)) + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + processor.configure(pcm16(44_100)) + processor.queueEndOfStream() + processor.flush(AudioProcessor.StreamMetadata.DEFAULT) + assertEquals(479, queue(441)) + processor.playToEnd() + processor.queueEndOfStream() + assertFalse(processor.isEnded) + processor.queueInput(ByteBuffer.allocateDirect(0)) + assertEquals(1, processor.output.remaining() / 2) + assertTrue(processor.isEnded) + } + + @Test + fun aLeadStillPrimesAStreamThatHasNotStarted() { + val r = OutputRate(dsp, 4) + r.latch(4) + r.start(4, 1, joined = false, lead = null) + // A first flush for the new format without the mix, then the one that starts it. + r.start(3, 1, joined = true, lead = null) + r.start(3, 1, joined = true, lead = ramp(27, 3)) + val out = r.feed(ramp(30, 6)) + r.end() + assertArrayEquals(whole(ramp(0, 36)).copyOfRange(40, 48), out, 1e-6f) + // Once it has had input, a lead is too late. + r.start(3, 1, joined = false, lead = null) + r.feed(ramp(0, 3)) + r.start(3, 1, joined = true, lead = ramp(27, 3)) + assertEquals(1, dsp.live) + } +} diff --git a/android/playback/src/test/java/app/hocket/playback/TransitionMixerTest.kt b/android/playback/src/test/java/app/hocket/playback/TransitionMixerTest.kt index 5e3f0f60..d386a6a7 100644 --- a/android/playback/src/test/java/app/hocket/playback/TransitionMixerTest.kt +++ b/android/playback/src/test/java/app/hocket/playback/TransitionMixerTest.kt @@ -54,6 +54,11 @@ class TransitionMixerTest { override fun limit(buffer: ByteBuffer, samples: Int) { for (i in 0 until samples) buffer.putFloat(i * 4, buffer.getFloat(i * 4).coerceIn(-0.9f, 0.9f)) } + override fun resamplerNew(inRate: Int, outRate: Int, channels: Int) = 0L + override fun resamplerFree(handle: Long) = Unit + override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int) = Unit + override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int) = 0 } private val dsp = FakeDsp() diff --git a/crates/hocket-android/src/dsp.rs b/crates/hocket-android/src/dsp.rs index 0c3e79e8..9ea6af66 100644 --- a/crates/hocket-android/src/dsp.rs +++ b/crates/hocket-android/src/dsp.rs @@ -9,6 +9,7 @@ //! - [`Chain`]: hocket-core's [`DspChain`] (track gain incl. boost, EQ, normalisation, limiter). //! - [`IncomingShaper`]: B's side of a transition. //! - `limit`: the chain's soft limiter, run on the sum of the tail and B. +//! - [`StreamResampler`]: the output stage, to the one rate the sink plays at. //! - **One shot, UniFFI** (`lib.rs`): [`prepare_tail`] (DSP chain like A's, channel map, resample to //! B's rate, `render_tail`) and [`loudness_record`], on a background thread. //! @@ -24,6 +25,8 @@ use hocket_core::audio::resample; use hocket_core::automix::loudness::{self, HeadTail}; use hocket_core::automix::{is_usable, render_tail, IncomingShaper}; +use crate::resampler::{gcd, StreamResampler}; + /// The chain for one stream: hocket-core's [`DspChain`] with the format the processor was /// configured with. pub struct Chain { @@ -90,8 +93,12 @@ pub fn parse_audio_settings(json: &str) -> Result { /// What [`prepare_tail`] renders from. pub struct TailInput<'a> { - /// A from `tail_start_ms` (no preroll), interleaved, at `in_rate`, straight from the decoder. + /// A from `tail_start_ms − context_ms` (no preroll), interleaved, at `in_rate`, straight from + /// the decoder. pub pcm: &'a [f32], + /// How much of `pcm` comes before `tail_start_ms`: history for the chain and the resampler, + /// and where the lead comes from. + pub context_ms: u32, pub in_channels: usize, pub in_rate: u32, /// B's output format: the tail is mixed into B's frames. @@ -102,21 +109,85 @@ pub struct TailInput<'a> { pub gain_db: f64, } +/// [`prepare_tail`]'s result. +#[derive(Debug, Default)] +pub struct PreparedTail { + /// Exactly `tail_len_ms` at `out_rate` × `out_channels`, from `tail_start_ms`. + pub tail: Vec, + /// A just before `tail_start_ms` at `out_rate` × `out_channels`, its last frame the one before + /// the tail's first: history for the output resampler when B's stream starts with the tail. + pub lead: Vec, + /// A from `tail_start_ms` through A's chain, at `in_rate` × `in_channels`: lookahead for the + /// output resampler when A's stream ends at the tail. + pub after: Vec, +} + +/// Lead and lookahead length (at least the output resampler's kernel half-width). +const SEAM_MS: u32 = 10; + /// The rendered tail: A's PCM through a chain configured like A's, mapped to B's channels, -/// resampled to B's rate, then `render_tail`. Exactly `tail_len_ms` at `out_rate`. -pub fn prepare_tail(plan: &TransitionPlan, input: TailInput<'_>) -> Result, String> { +/// resampled to B's rate, then `render_tail`; plus the [`PreparedTail::lead`] and +/// [`PreparedTail::after`] that let the output resampler switch streams without a seam. +pub fn prepare_tail(plan: &TransitionPlan, input: TailInput<'_>) -> Result { let in_ch = input.in_channels.max(1); + let in_rate = input.in_rate.max(1); + let out_ch = input.out_channels.max(1); + let out_rate = input.out_rate.max(1); let mut pcm = input.pcm[..input.pcm.len() / in_ch * in_ch].to_vec(); - let mut chain = Chain::new(input.in_rate, in_ch); + let mut chain = Chain::new(in_rate, in_ch); if let Some(a) = input.audio { chain.set_audio_settings(a); } chain.set_track_gain(input.gain_db); chain.process(&mut pcm); - let out_ch = input.out_channels.max(1); - let converted = resample::convert(&pcm, in_ch, input.in_rate, out_ch, input.out_rate) - .map_err(|e| e.to_string())?; - Ok(render_tail(plan, &converted, out_ch, input.out_rate)) + let frames = pcm.len() / in_ch; + // Floored, like the decoder's start trim and the processor's clip point. + let ctx = (u64::from(input.context_ms) * u64::from(in_rate) / 1000) as usize; + let ctx = ctx.min(frames); + let seam_in = (u64::from(SEAM_MS) * u64::from(in_rate) / 1000) as usize; + let after = pcm[ctx * in_ch..(ctx + seam_in).min(frames) * in_ch].to_vec(); + + let mut mapped = Vec::with_capacity(frames * out_ch); + resample::remap_channels(&pcm, in_ch, out_ch, &mut mapped); + let seam_out = (u64::from(SEAM_MS) * u64::from(out_rate) / 1000) as usize; + let (lead, source) = if in_rate == out_rate { + let lead_from = ctx.saturating_sub(seam_out); + ( + mapped[lead_from * out_ch..ctx * out_ch].to_vec(), + mapped[ctx * out_ch..].to_vec(), + ) + } else { + // One stream from a little before `tail_start_ms`, so that an output frame lands exactly on + // it: `lead_in` input frames make `lead` output frames when both are whole multiples of + // the reduced ratio. What comes before only primes the filter. + let g = gcd(u64::from(in_rate), u64::from(out_rate)); + let (step, den) = ( + (u64::from(in_rate) / g) as usize, + (u64::from(out_rate) / g) as usize, + ); + let mut k = seam_out.div_ceil(den).max(1); + while k > 0 && k * step > ctx { + k -= 1; + } + let (lead_out, lead_in) = (k * den, k * step); + let start = ctx - lead_in; + let mut r = StreamResampler::new(in_rate, out_rate, out_ch); + r.prime(&mapped[..start * out_ch]); + r.push(&mapped[start * out_ch..]); + r.finish(&[]); + let out_frames = ((frames - start) as u64 * den as u64).div_ceil(step as u64) as usize; + let mut converted = vec![0f32; out_frames * out_ch + out_ch]; + let n = r.pull(&mut converted); + converted.truncate(n); + let split = (lead_out * out_ch).min(converted.len()); + let source = converted.split_off(split); + (converted, source) + }; + Ok(PreparedTail { + tail: render_tail(plan, &source, out_ch, out_rate), + lead, + after, + }) } /// A loudness-only record from a track's decoded head and tail, fingerprinted with the @@ -227,6 +298,11 @@ mod jni_shims { (h != 0).then(|| unsafe { &mut *(h as *mut IncomingShaper) }) } + fn resampler<'a>(h: jlong) -> Option<&'a mut StreamResampler> { + // SAFETY: as for `chain`, from `resamplerNew`. + (h != 0).then(|| unsafe { &mut *(h as *mut StreamResampler) }) + } + #[no_mangle] pub extern "system" fn Java_app_hocket_playback_NativeDsp_chainNew( _env: JNIEnv, @@ -401,6 +477,108 @@ mod jni_shims { }) } + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerNew( + _env: JNIEnv, + _this: JObject, + in_rate: jint, + out_rate: jint, + channels: jint, + ) -> jlong { + guarded(0, || { + Box::into_raw(Box::new(StreamResampler::new( + in_rate.max(1) as u32, + out_rate.max(1) as u32, + channels.max(1) as usize, + ))) as jlong + }) + } + + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerFree( + _env: JNIEnv, + _this: JObject, + h: jlong, + ) { + if h != 0 { + // SAFETY: from `resamplerNew`, freed once (Kotlin zeroes its copy). + drop(unsafe { Box::from_raw(h as *mut StreamResampler) }); + } + } + + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerPrime( + env: JNIEnv, + _this: JObject, + h: jlong, + buf: JByteBuffer, + samples: jint, + ) { + let Some(s) = floats(&env, &buf, samples) else { + return; + }; + guarded((), || { + if let Some(r) = resampler(h) { + r.prime(s); + } + }) + } + + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerFinish( + env: JNIEnv, + _this: JObject, + h: jlong, + buf: JByteBuffer, + samples: jint, + ) { + let Some(s) = floats(&env, &buf, samples) else { + return; + }; + guarded((), || { + if let Some(r) = resampler(h) { + r.finish(s); + } + }) + } + + #[no_mangle] + pub extern "system" fn Java_app_hocket_playback_NativeDsp_resamplerProcess( + env: JNIEnv, + _this: JObject, + h: jlong, + input: JByteBuffer, + in_samples: jint, + output: JByteBuffer, + out_offset: jint, + out_capacity: jint, + ) -> jint { + // Two views of one buffer would be aliasing `&mut` slices. + if env.get_direct_buffer_address(&input).ok() == env.get_direct_buffer_address(&output).ok() + { + return 0; + } + let Some(inp) = floats(&env, &input, in_samples) else { + return 0; + }; + let Some(out) = floats(&env, &output, out_capacity) else { + return 0; + }; + let Ok(offset) = usize::try_from(out_offset) else { + return 0; + }; + if offset > out.len() { + return 0; + } + guarded(0, || match resampler(h) { + Some(r) => { + r.push(inp); + r.pull(&mut out[offset..]) as jint + } + None => 0, + }) + } + #[no_mangle] pub extern "system" fn Java_app_hocket_playback_NativeDsp_limit( env: JNIEnv, @@ -527,10 +705,11 @@ mod tests { fn tail_is_exact_length_at_bs_rate_and_channels() { let p = plan(); let pcm = dc(44_100, 1, 0.5); - let tail = prepare_tail( + let prepared = prepare_tail( &p, TailInput { pcm: &pcm, + context_ms: 0, in_channels: 1, in_rate: 44_100, out_channels: 2, @@ -540,12 +719,93 @@ mod tests { }, ) .unwrap(); + let tail = prepared.tail; assert_eq!(tail.len(), 48_000 * 2); // Before the overlap the tail is A at A's gain (−6 dB of 0.5). assert!((tail[4_800 * 2] - 0.25).abs() < 0.01, "{}", tail[4_800 * 2]); // It has faded out by its end. assert!(tail[tail.len() - 1].abs() < 0.01); assert!(tail.iter().all(|s| s.is_finite())); + // No context: no lead; the lookahead is A's first 10 ms at A's format. + assert!(prepared.lead.is_empty()); + assert_eq!(prepared.after.len(), 441); + } + + /// A sine at `rate`, `frames` long, from frame `from` of the same sine. + fn sine_at(rate: u32, frames: usize, from: usize, channels: usize) -> Vec { + (from..from + frames) + .flat_map(|i| { + let s = (2.0 * std::f64::consts::PI * 2_000.0 * i as f64 / f64::from(rate)).sin() + as f32 + * 0.25; + std::iter::repeat_n(s, channels) + }) + .collect() + } + + #[test] + fn tail_lead_and_after_line_up_with_the_tail_start() { + // A at 44.1 kHz from 30 ms before the tail start; B at 48 kHz. A plain fade with no + // overlap yet at the start, so the tail's first frames are A itself. + let mut p = plan(); + p.incoming_at_ms = 900; + p.handover_ms = 950; + let ctx_frames = 1_323; // 30 ms at 44.1 kHz + let pcm = sine_at(44_100, 44_100, 0, 2); + let prepared = prepare_tail( + &p, + TailInput { + pcm: &pcm, + context_ms: 30, + in_channels: 2, + in_rate: 44_100, + out_channels: 2, + out_rate: 48_000, + audio: None, + gain_db: 0.0, + }, + ) + .unwrap(); + // The tail starts exactly at the tail start: frame j is A at ctx + j × 44.1 / 48. + let want_tail = sine_at(48_000, 200, 1_440, 2); + let err = + |a: &[f32], b: &[f32]| a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())); + assert!( + err(&prepared.tail[..400], &want_tail) < 1e-3, + "tail start {}", + err(&prepared.tail[..400], &want_tail) + ); + // The lead ends on the frame before it (at least 10 ms of it, in whole 160-frame steps). + let lead_frames = prepared.lead.len() / 2; + assert_eq!(lead_frames, 480); + let want_lead = sine_at(48_000, lead_frames, 1_440 - lead_frames, 2); + assert!(err(&prepared.lead, &want_lead) < 1e-3); + // The lookahead is A from the tail start at A's rate. + assert_eq!( + prepared.after, + pcm[ctx_frames * 2..(ctx_frames + 441) * 2].to_vec() + ); + } + + #[test] + fn same_rate_tail_needs_no_resampling() { + let pcm = sine_at(48_000, 48_000, 0, 1); + let prepared = prepare_tail( + &plan(), + TailInput { + pcm: &pcm, + context_ms: 20, + in_channels: 1, + in_rate: 48_000, + out_channels: 1, + out_rate: 48_000, + audio: None, + gain_db: 0.0, + }, + ) + .unwrap(); + assert_eq!(prepared.lead, pcm[480..960].to_vec()); + assert_eq!(prepared.tail[..100], pcm[960..1_060]); } #[test] diff --git a/crates/hocket-android/src/lib.rs b/crates/hocket-android/src/lib.rs index 785e0c95..32a3a7b6 100644 --- a/crates/hocket-android/src/lib.rs +++ b/crates/hocket-android/src/lib.rs @@ -259,22 +259,35 @@ pub fn artwork_layout( // One-shot calls for a background thread. PCM crosses as little-endian `f32` bytes (a Kotlin // `ByteArray`; a `Vec` would be a boxed `List`). The per-buffer calls are JNI. -/// The rendered tail of a transition (`docs/automix.md`): `pcm` is A from `tail_start_ms`, -/// `in_channels` at `in_rate`; it goes through a DSP chain like A's (`audio_settings_json`, an -/// `AudioSettings`, and A's `gain_db`), is mapped to B's channels, resampled to B's rate and -/// rendered with `render_tail`. Returns exactly `tail_len_ms` at `out_rate`, as LE `f32` bytes. +/// [`automix_prepare_tail`]'s result, each as LE `f32` bytes (see [`dsp::PreparedTail`]). +#[derive(uniffi::Record)] +pub struct PreparedTail { + /// Exactly `tail_len_ms` at `out_rate` × `out_channels`. + pub tail: Vec, + /// A just before `tail_start_ms`, at `out_rate` × `out_channels`. + pub lead: Vec, + /// A from `tail_start_ms` through A's chain, at `in_rate` × `in_channels`. + pub after: Vec, +} + +/// The rendered tail of a transition (`docs/automix.md`): `pcm` is A from `tail_start_ms − +/// context_ms`, `in_channels` at `in_rate`; it goes through a DSP chain like A's +/// (`audio_settings_json`, an `AudioSettings`, and A's `gain_db`), is mapped to B's channels, +/// resampled to B's rate and rendered with `render_tail`. Also returns A's audio either side of +/// `tail_start_ms` for a seamless switch of the output resampler. #[uniffi::export] #[allow(clippy::too_many_arguments)] pub fn automix_prepare_tail( plan_json: String, pcm: Vec, + context_ms: u32, in_channels: u32, in_rate: u32, out_channels: u32, out_rate: u32, audio_settings_json: Option, gain_db: f64, -) -> Result, HocketError> { +) -> Result { let failed = |reason: String| HocketError::Failed { reason }; let plan = dsp::parse_plan(&plan_json).map_err(failed)?; let audio = audio_settings_json @@ -283,10 +296,11 @@ pub fn automix_prepare_tail( .transpose() .map_err(failed)?; let pcm = dsp::f32_from_le(&pcm); - let tail = dsp::prepare_tail( + let prepared = dsp::prepare_tail( &plan, dsp::TailInput { pcm: &pcm, + context_ms, in_channels: in_channels as usize, in_rate, out_channels: out_channels as usize, @@ -296,7 +310,11 @@ pub fn automix_prepare_tail( }, ) .map_err(failed)?; - Ok(dsp::f32_to_le(&tail)) + Ok(PreparedTail { + tail: dsp::f32_to_le(&prepared.tail), + lead: dsp::f32_to_le(&prepared.lead), + after: dsp::f32_to_le(&prepared.after), + }) } /// A loudness-only `AnalysisRecord` (JSON, producer Device) from a track's decoded head (from 0) @@ -343,6 +361,7 @@ pub fn core_version() -> String { } pub mod dsp; +pub mod resampler; #[cfg(test)] mod fixtures; diff --git a/crates/hocket-android/src/resampler.rs b/crates/hocket-android/src/resampler.rs new file mode 100644 index 00000000..6570887c --- /dev/null +++ b/crates/hocket-android/src/resampler.rs @@ -0,0 +1,439 @@ +//! Streaming sample-rate conversion for the output of `MixProcessor` (`docs/automix.md` → +//! Android → Output rate). +//! +//! ExoPlayer's sink rebuilds its `AudioTrack` whenever the output rate changes, which leaves a gap +//! between tracks at different rates (and in the middle of an AutoMix transition). So the +//! processor keeps the output rate fixed and converts every stream at another rate with this. +//! +//! What this needs that a chunked converter (`hocket_core::audio::resample`) doesn't give: +//! +//! - **Sample-exact positions.** Output frame `j` of a stream is its input at `j × in / out`, +//! exactly (the rates are reduced to a ratio of integers): no filter delay to trim, and a stream +//! of `n` input frames is `ceil(n × out / in)` output frames. +//! - **Low latency.** An output frame is produced as soon as the kernel's lookahead (`half` input +//! frames, ~1.5 ms) has arrived, not when a 1024-frame chunk is full. Whatever is still waiting +//! at an item boundary barely moves the sink's clock. +//! - **Seams without clicks.** A stream that ends ([`StreamResampler::finish`]) can take frames +//! past its end as lookahead only, and a stream that starts ([`StreamResampler::prime`]) can +//! take frames before its start as history only. With the same audio on both sides (an AutoMix +//! tail continues A), the switch from one stream to the next is as if nothing were cut. +//! +//! The kernel is a Kaiser-windowed sinc, one row of taps per output phase (`out / gcd` of them: +//! 160 for 44.1 ↔ 48 kHz), so there is no interpolation between rows. + +use std::f64::consts::PI; + +/// Kernel half-width in input frames when upsampling (scaled up when downsampling). +const HALF_BASE: usize = 64; +/// The widest kernel half-width (192 kHz into 44.1 kHz would want 279). +const HALF_MAX: usize = 256; +/// Phases beyond this (unusual rate pairs) are quantised to this many rows. +const PHASES_MAX: u64 = 1024; +/// Cutoff as a fraction of the lower Nyquist frequency. +const CUTOFF: f64 = 0.92; +/// Kaiser β: about 80 dB of stopband. +const BETA: f64 = 8.0; +/// Consumed input is dropped from the front once this many frames have piled up. +const COMPACT_FRAMES: u64 = 4096; + +pub struct StreamResampler { + channels: usize, + /// Each output frame advances `step / den` input frames. + step: u64, + den: u64, + half: usize, + phases: u64, + /// `phases + 1` rows of `2 × half` taps; row `r` is the kernel at offset `r / phases`. + table: Vec, + /// Input frames from absolute frame `base`, interleaved. + buf: Vec, + base: u64, + /// Input frames of the stream (primed history included): outputs lie before this. + end: u64, + finished: bool, + /// The next output frame's position in input frames, in units of `1 / den`. + pos: u64, + acc: Vec, +} + +impl StreamResampler { + pub fn new(in_rate: u32, out_rate: u32, channels: usize) -> Self { + let (in_rate, out_rate) = (u64::from(in_rate.max(1)), u64::from(out_rate.max(1))); + let g = gcd(in_rate, out_rate); + let (step, den) = (in_rate / g, out_rate / g); + let down = in_rate as f64 / out_rate as f64; + let half = ((HALF_BASE as f64 * down.max(1.0)).ceil() as usize).min(HALF_MAX); + let phases = den.min(PHASES_MAX); + // Cycles per input frame, times two: the sinc's zero crossings. + let fc2 = CUTOFF * (1.0 / down).min(1.0); + let taps = 2 * half; + let mut table = vec![0f32; (phases as usize + 1) * taps]; + let i0_beta = bessel_i0(BETA); + for r in 0..=phases as usize { + let frac = r as f64 / phases as f64; + let row = &mut table[r * taps..(r + 1) * taps]; + let mut sum = 0.0; + let mut w = vec![0f64; taps]; + for (k, wk) in w.iter_mut().enumerate() { + // Tap k reads input frame `ipos - half + 1 + k`, at `t` frames from the output. + let t = k as f64 - half as f64 + 1.0 - frac; + let x = t / half as f64; + let window = if x.abs() >= 1.0 { + 0.0 + } else { + bessel_i0(BETA * (1.0 - x * x).sqrt()) / i0_beta + }; + *wk = fc2 * sinc(fc2 * t) * window; + sum += *wk; + } + // Unity gain at DC on every phase. + for (o, wk) in row.iter_mut().zip(&w) { + *o = (wk / sum) as f32; + } + } + let channels = channels.max(1); + Self { + channels, + step, + den, + half, + phases, + table, + buf: Vec::new(), + base: 0, + end: 0, + finished: false, + pos: 0, + acc: vec![0.0; channels], + } + } + + pub fn channels(&self) -> usize { + self.channels + } + + /// History before the stream's start: `frames` only inform its first outputs. Only before any + /// input; ignored after. + pub fn prime(&mut self, frames: &[f32]) { + if self.end != 0 || self.finished { + return; + } + self.append(frames); + self.end = self.buffered_end(); + self.pos = self.end * self.den; + } + + /// The stream's next input frames (interleaved). + pub fn push(&mut self, frames: &[f32]) { + if self.finished { + return; + } + self.append(frames); + self.end = self.buffered_end(); + } + + /// The stream ends here: `lookahead` (the audio that would have followed, if any) only + /// informs the last outputs; past it, silence. [`Self::pull`] then yields the rest. + pub fn finish(&mut self, lookahead: &[f32]) { + if self.finished { + return; + } + self.append(lookahead); + self.finished = true; + } + + /// Output frames into `out` (interleaved; whole frames only), as many as are ready and fit. + /// Returns the samples written. Call again while it fills `out`. + pub fn pull(&mut self, out: &mut [f32]) -> usize { + let ch = self.channels; + let taps = 2 * self.half; + let have = self.buffered_end(); + let mut written = 0; + while written + ch <= out.len() { + let ipos = self.pos / self.den; + let ready = if self.finished { + ipos < self.end + } else { + ipos + (self.half as u64) < self.end + }; + if !ready { + break; + } + let frac = self.pos % self.den; + let r = if self.phases == self.den { + frac + } else { + (frac * self.phases + self.den / 2) / self.den + } as usize; + let row = &self.table[r * taps..(r + 1) * taps]; + let first = ipos as i64 - self.half as i64 + 1; + self.acc.iter_mut().for_each(|a| *a = 0.0); + if first >= self.base as i64 && first as u64 + taps as u64 <= have { + let from = (first as u64 - self.base) as usize * ch; + let frames = &self.buf[from..from + taps * ch]; + if ch == 2 { + let (mut l, mut rr) = (0f32, 0f32); + for (f, w) in frames.as_chunks::<2>().0.iter().zip(row) { + l += f[0] * w; + rr += f[1] * w; + } + self.acc[0] = l; + self.acc[1] = rr; + } else { + for (f, w) in frames.chunks_exact(ch).zip(row) { + for (a, s) in self.acc.iter_mut().zip(f) { + *a += s * w; + } + } + } + } else { + // At the edges: silence before the first frame and past the last. + for (k, w) in row.iter().enumerate() { + let i = first + k as i64; + if i < self.base as i64 || i as u64 >= have { + continue; + } + let at = (i as u64 - self.base) as usize * ch; + for (c, a) in self.acc.iter_mut().enumerate() { + *a += self.buf[at + c] * w; + } + } + } + out[written..written + ch].copy_from_slice(&self.acc); + written += ch; + self.pos += self.step; + } + self.compact(); + written + } + + fn append(&mut self, frames: &[f32]) { + let whole = frames.len() / self.channels * self.channels; + self.buf.extend_from_slice(&frames[..whole]); + } + + fn buffered_end(&self) -> u64 { + self.base + (self.buf.len() / self.channels) as u64 + } + + /// Drop input no output will read again (the next one reads from `ipos - half + 1`). + fn compact(&mut self) { + let keep_from = (self.pos / self.den).saturating_sub(self.half as u64 - 1); + if keep_from > self.base + COMPACT_FRAMES { + let drop = (keep_from - self.base) as usize; + let drop = drop.min(self.buf.len() / self.channels); + self.buf.drain(..drop * self.channels); + self.base += drop as u64; + } + } +} + +pub(crate) fn gcd(a: u64, b: u64) -> u64 { + if b == 0 { + a.max(1) + } else { + gcd(b, a % b) + } +} + +fn sinc(x: f64) -> f64 { + if x.abs() < 1e-12 { + 1.0 + } else { + (PI * x).sin() / (PI * x) + } +} + +/// The zeroth-order modified Bessel function of the first kind (series; converges fast for β ≤ 20). +fn bessel_i0(x: f64) -> f64 { + let (mut sum, mut term) = (1.0, 1.0); + let q = x * x / 4.0; + for k in 1..64 { + term *= q / (k * k) as f64; + sum += term; + if term < sum * 1e-17 { + break; + } + } + sum +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sine(frames: usize, channels: usize, rate: f64, freq: f64, offset: usize) -> Vec { + let mut v = Vec::with_capacity(frames * channels); + for i in 0..frames { + let s = (2.0 * PI * freq * (i + offset) as f64 / rate).sin() as f32 * 0.5; + for c in 0..channels { + v.push(if c == 0 { s } else { -s }); + } + } + v + } + + fn drain(r: &mut StreamResampler, out: &mut Vec) { + let mut chunk = vec![0f32; 1000 * r.channels()]; + loop { + let n = r.pull(&mut chunk); + out.extend_from_slice(&chunk[..n]); + if n < chunk.len() { + break; + } + } + } + + /// The whole of `input`, finished with `lookahead`, in one go. + fn convert( + in_rate: u32, + out_rate: u32, + ch: usize, + input: &[f32], + lookahead: &[f32], + ) -> Vec { + let mut r = StreamResampler::new(in_rate, out_rate, ch); + r.push(input); + r.finish(lookahead); + let mut out = Vec::new(); + drain(&mut r, &mut out); + out + } + + fn max_err(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) + } + + #[test] + fn output_length_is_exact() { + for (i, o, n) in [ + (44_100, 48_000, 44_100), + (48_000, 44_100, 12_345), + (96_000, 48_000, 1_001), + (22_050, 48_000, 7), + ] { + let out = convert(i, o, 2, &vec![0.1; n * 2], &[]); + let want = (n as u64 * u64::from(o)).div_ceil(u64::from(i)) as usize; + assert_eq!(out.len(), want * 2, "{i} -> {o}, {n} frames"); + } + } + + #[test] + fn a_sine_comes_through_clean_and_in_place() { + for (i, o) in [ + (44_100u32, 48_000u32), + (48_000, 44_100), + (96_000, 44_100), + (32_000, 48_000), + ] { + let input = sine(i as usize, 2, f64::from(i), 1_000.0, 0); + let out = convert(i, o, 2, &input, &[]); + // Away from the edges, output frame j is the sine at j / out seconds: no delay. + let want = sine(out.len() / 2, 2, f64::from(o), 1_000.0, 0); + let mid = 2 * 1_000..out.len() - 2 * 1_000; + let err = max_err(&out[mid.clone()], &want[mid]); + assert!(err < 1e-3, "{i} -> {o}: {err}"); + } + } + + #[test] + fn content_above_the_output_band_is_removed() { + // 23 kHz at 48 kHz would alias to 21.1 kHz at 44.1 kHz. + let input = sine(48_000, 1, 48_000.0, 23_000.0, 0); + let out = convert(48_000, 44_100, 1, &input, &[]); + let peak = out[4_000..out.len() - 4_000] + .iter() + .fold(0f32, |m, s| m.max(s.abs())); + assert!(peak < 0.5 * 1e-3, "{peak}"); + } + + #[test] + fn chunking_does_not_change_the_output() { + let input = sine(10_000, 2, 44_100.0, 440.0, 0); + let whole = convert(44_100, 48_000, 2, &input, &[]); + let mut r = StreamResampler::new(44_100, 48_000, 2); + let mut out = Vec::new(); + let mut at = 0; + for len in [1usize, 7, 300, 2, 1_500, 64, 4_000].iter().cycle() { + if at >= 10_000 { + break; + } + let end = (at + len).min(10_000); + r.push(&input[at * 2..end * 2]); + drain(&mut r, &mut out); + at = end; + } + r.finish(&[]); + drain(&mut r, &mut out); + assert_eq!(out, whole); + } + + #[test] + fn only_lookahead_is_held_back_until_finish() { + let mut r = StreamResampler::new(44_100, 48_000, 1); + r.push(&vec![0.2; 1_000]); + let mut out = Vec::new(); + drain(&mut r, &mut out); + let total = (1_000u64 * 48_000).div_ceil(44_100) as usize; + let held = total - out.len(); + // About `half` input frames' worth (64 × 48/44.1 ≈ 70). + assert!((60..=80).contains(&held), "{held}"); + } + + #[test] + fn finish_with_the_real_continuation_matches_an_uncut_stream() { + let input = sine(6_000, 2, 44_100.0, 3_000.0, 0); + let uncut = convert(44_100, 48_000, 2, &input, &[]); + let cut = convert( + 44_100, + 48_000, + 2, + &input[..4_000 * 2], + &input[4_000 * 2..4_400 * 2], + ); + let n = cut.len(); + assert_eq!(n, (4_000u64 * 48_000).div_ceil(44_100) as usize * 2); + assert!(max_err(&cut, &uncut[..n]) < 1e-6); + // Without it the last frames are a band-limited cut to silence: well off. + let bare = convert(44_100, 48_000, 2, &input[..4_000 * 2], &[]); + assert!(max_err(&bare[n - 20..], &uncut[n - 20..n]) > 0.05); + } + + #[test] + fn priming_with_the_audio_before_matches_an_uncut_stream() { + // The cut at 1470 input frames: an output frame lands exactly on it (× 160 / 147), and the + // sine isn't at a zero crossing there. + let input = sine(6_000, 2, 44_100.0, 2_500.0, 0); + let uncut = convert(44_100, 48_000, 2, &input, &[]); + let cut_at = 1_470; + let mut r = StreamResampler::new(44_100, 48_000, 2); + r.prime(&input[..cut_at * 2]); + r.push(&input[cut_at * 2..]); + r.finish(&[]); + let mut out = Vec::new(); + drain(&mut r, &mut out); + let skip = cut_at * 48_000 / 44_100 * 2; + assert_eq!(out.len(), uncut.len() - skip); + assert!(max_err(&out, &uncut[skip..]) < 1e-6); + // Unprimed, the first frames see silence before them: audibly off (a click). + let fresh = convert(44_100, 48_000, 2, &input[cut_at * 2..], &[]); + let fresh_err = max_err(&fresh[..20], &uncut[skip..skip + 20]); + assert!(fresh_err > 0.01, "{fresh_err}"); + } + + #[test] + fn long_streams_keep_memory_bounded() { + let mut r = StreamResampler::new(48_000, 44_100, 2); + let block = sine(1_024, 2, 48_000.0, 100.0, 0); + let mut out = vec![0f32; 4_096]; + for _ in 0..2_000 { + r.push(&block); + while r.pull(&mut out) == out.len() {} + } + assert!( + r.buf.len() < (COMPACT_FRAMES as usize + 2_048) * 2, + "{}", + r.buf.len() + ); + } +} diff --git a/docs/automix.md b/docs/automix.md index 9d0c3fb2..aa7b798e 100644 --- a/docs/automix.md +++ b/docs/automix.md @@ -378,8 +378,9 @@ The planner is deterministic: same inputs → same plan on every device. `store_analysis_record`, which replans the follow-up. One at a time, each track tried once per session. `on_analysis_produced` uploads the record (see *Sharing records*). -- **Android** (`ExoBackend.kt`, `MixProcessor.kt`, `PcmDecoder.kt`, - `LoudnessRecorder.kt`; Rust side `crates/hocket-android/src/dsp.rs`): +- **Android** (`ExoBackend.kt`, `MixProcessor.kt`, `OutputRate.kt`, + `PcmDecoder.kt`, `LoudnessRecorder.kt`; Rust side + `crates/hocket-android/src/dsp.rs`, `resampler.rs`): - **DSP**: a Media3 `BaseAudioProcessor` in the main player's `DefaultAudioSink` (PCM path only: 16-bit or float in, the same out; no float output, no offload) runs hocket-core's `DspChain` per buffer @@ -400,12 +401,37 @@ The planner is deterministic: same inputs → same plan on every device. renderer's `handleDiscontinuity` at every stream change forces a timing resync); a contiguous flush of the item already playing is the same stream going on and must not end a mix. - When B is at another sample rate the sink can't join gaplessly: it - drains A, flushes once still naming A (the new configuration takes over - after that flush), then flushes for B with no end of stream in between. - So every flush between an end of stream and the next frame belongs to - the join; without that, every mix between a 44.1 kHz and a 48 kHz track - was dropped and A cut off at its clip point. + When B is at another channel count the sink can't keep its + `AudioTrack`: it drains A, flushes once still naming A (the new + configuration takes over after that flush), then flushes for B with no + end of stream in between. So every flush between an end of stream and + the next frame belongs to the join. + - **Output rate** (`OutputRate`): the sink rebuilds its `AudioTrack` + whenever its output format changes, and a rebuild plays the old track + out to silence before the new one starts: a gap at every join between + tracks at different rates, in the middle of A's audio when the join is + a mix. So the processor keeps one output rate: the first stream's after + a reset when it is 44.1 or 48 kHz, else the device's + (`PROPERTY_OUTPUT_SAMPLE_RATE`). A library at one rate is never + resampled. Other streams go through a `StreamResampler` (JNI; a + Kaiser-windowed sinc with one row of taps per output phase, ~80 dB + stopband), after the mixer, so the mixer's sample-exact item accounting + is untouched. Output frame `j` is input frame `j × in / out` exactly, + and only the kernel's lookahead (~1.5 ms) is held back. + - A gapless join at the same format goes on through the same resampler: + no seam inside continuous audio. What it held back comes out after + the flush, which moves B's clock by ~1.5 ms. At the end of the + playlist (`MixClockSink.playToEndOfStream` tells the processor) they + come out at the end of stream. + - When the next stream is at another format (the sink configures it + while A plays), A's end of stream lets out A's last frames, with the + armed tail's `after` (A from the tail start through A's chain, at A's + format) as lookahead when the mix follows. B's resampler is primed + with the tail's `lead` (A just before the tail start, at B's format; + its frames line up with the tail's, both from one resampler run). So + the switch from A to the tail is as seamless as at one rate. + - The tail render decodes A from 60 ms before the tail start for that + (and so the tail's own first frames have filter history). - **Clipping**: an item with `mix_out_ms` gets `ClippingConfiguration(endPositionMs)` (on `load` only when its `next` has `mix_in`); the processor also drops frames past the clip end @@ -422,9 +448,9 @@ The planner is deterministic: same inputs → same plan on every device. stream reader as a `MediaDataSource`, `file:` from disk, else HTTP with the source's headers — probes B's rate and channels from its container, and calls `automix_prepare_tail`: a `DspChain` like A's (A's gain, the - EQ), channel map, **rubato** resampling to B's rate (the desktop - resampler, moved to `audio::resample` behind the feature `resample`, - which `hocket-android` enables; pure Rust), then `render_tail`. + EQ), channel map, resampling to B's rate (the `StreamResampler`, primed + with the 60 ms before the tail start), then `render_tail`; plus the + `lead` and `after` above. - **Mixing**: at B's first frame after a contiguous join, if the armed pair matches and the tail is at B's format, the processor outputs the tail's first `incoming_at_ms` at once, then B (its first