Android: keep one output rate so joins across sample rates stay gapless

android/playback/src/main/java/app/hocket/playback/ExoBackend.kt

@@ -1,6 +1,7 @@
 package app.hocket.playback
 
 import android.content.Context
+import android.media.AudioManager
 import android.net.Uri
 import android.os.Handler
 import android.os.Looper
@@ -164,6 +165,8 @@ class ExoBackend(
         private const val HANDOVER_POLL_MS = 40L
         /** A's tail is decoded and rendered once A is this close to its tail start. */
         private const val PREPARE_AHEAD_MS = 60_000L
+        /** A decoded before its tail start for the tail render (more than its resampler seam needs). */
+        private const val TAIL_CONTEXT_MS = 60L
         /**
          * A clipped item whose tail isn't ready this long before its clip point is unclipped: the
          * sink and the decoder run up to about a second ahead of what is heard.
@@ -242,14 +245,19 @@ class ExoBackend(
             PlaybackLog.i("AutoMix: ${armed.from} -> ${armed.to} plays unmixed: $reason")
         }
     }
-    private val processor: MixProcessor? = dsp?.let { MixProcessor(it, mixControl, mixListener) }
+    private val processor: MixProcessor? = dsp?.let { MixProcessor(it, mixControl, mixListener, deviceRate()) }
+
+    /** The device's output rate (what [OutputRate] uses when the first item isn't at 44.1 or 48 kHz). */
+    private fun deviceRate(): Int =
+        (context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager)?.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE)?.toIntOrNull() ?: 48_000
 
     private fun renderersFactory() = object : DefaultRenderersFactory(context) {
         override fun buildAudioSink(context: Context, enableFloatOutput: Boolean, enableAudioOutputPlaybackParams: Boolean): AudioSink? {
             val p = processor ?: return super.buildAudioSink(context, enableFloatOutput, enableAudioOutputPlaybackParams)
             // PCM path only: no float output, no offload (both are off by default), so every buffer
             // reaches the processor.
-            return MixClockSink(DefaultAudioSink.Builder(context).setAudioProcessors(arrayOf(p)).build()).also { clockSink = it }
+            val sink = DefaultAudioSink.Builder(context).setAudioProcessors(arrayOf(p)).build()
+            return MixClockSink(sink, onPlayToEnd = p::playToEnd).also { clockSink = it }
         }
     }
 
@@ -1084,15 +1092,21 @@ class ExoBackend(
         mixControl.armed = null
     }
 
-    /** Decode A's tail, then DSP, resample to B's format and render it in Rust. Null on failure (logged). */
+    /**
+     * Decode A's tail (from [TAIL_CONTEXT_MS] before it: filter history, and A's audio either side of
+     * the tail start for [OutputRate]), then DSP, resample to B's format and render it in Rust. Null
+     * on failure (logged).
+     */
     private fun renderTail(a: CoreMediaSource, b: CoreMediaSource, armed: MixControl.Armed, settings: String?, cancelled: () -> Boolean): MixControl.Tail? = try {
         val started = System.nanoTime()
         val decoder = PcmDecoder(streams)
-        val pcm = decoder.decode(a.url, a.headers, armed.tailStartMs, armed.tailSourceMs + 50, cancelled)
+        val from = (armed.tailStartMs - TAIL_CONTEXT_MS).coerceAtLeast(0)
+        val context = armed.tailStartMs - from
+        val pcm = decoder.decode(a.url, a.headers, from, context + armed.tailSourceMs + 50, cancelled)
         val (rate, channels) = decoder.probe(b.url, b.headers)
-        val samples = ffi.prepareTail(armed.planJson, pcm, channels, rate, settings, a.gainDb)
+        val r = ffi.prepareTail(armed.planJson, pcm, context.toInt(), channels, rate, settings, a.gainDb)
         PlaybackLog.i("AutoMix: tail ${a.key} -> ${b.key} ready (${pcm.durationMs} ms of A at ${pcm.rate} Hz -> ${armed.tailLenMs} ms at $rate Hz) in ${(System.nanoTime() - started) / 1_000_000} ms")
-        MixControl.Tail(samples, rate, channels)
+        MixControl.Tail(r.tail, rate, channels, r.lead, r.after, pcm.rate, pcm.channels)
     } catch (e: InterruptedException) {
         null
     } catch (e: Throwable) {

android/playback/src/main/java/app/hocket/playback/LoudnessRecorder.kt

@@ -108,12 +108,21 @@ internal class LoudnessRecorder(
     }
 }
 
+/**
+ * A rendered tail ([tail], at B's format) and A's audio either side of its start: [lead] just before
+ * it at B's format, [after] from it at A's format (`MixControl.Tail`).
+ */
+class RenderedTail(val tail: FloatArray, val lead: FloatArray, val after: FloatArray)
+
 /** The one-shot UniFFI calls of AutoMix, behind a seam (no native library in JVM tests). */
 interface AutomixFfi {
     fun usable(record: AnalysisRecord, durationMs: Long): Boolean
     fun loudnessRecord(head: PcmDecoder.Pcm, tail: PcmDecoder.Pcm, tailStartMs: Long, durationMs: Long, transcoded: Boolean): AnalysisRecord?
-    /** The rendered tail as interleaved floats at [outRate] × [outChannels]. */
-    fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): FloatArray
+    /**
+     * The rendered tail from [pcm] (A from [contextMs] before the tail start), as interleaved floats
+     * at [outRate] × [outChannels], with A's audio either side of the tail start.
+     */
+    fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): RenderedTail
 
     object Native : AutomixFfi {
         override fun usable(record: AnalysisRecord, durationMs: Long): Boolean =
@@ -128,10 +137,14 @@ interface AutomixFfi {
             return HocketJson.json.decodeFromString(AnalysisRecord.serializer(), json)
         }
 
-        override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): FloatArray {
-            val bytes = app.hocket.core.ffi.automixPrepareTail(
-                planJson, pcm.leBytes(), pcm.channels.toUInt(), pcm.rate.toUInt(), outChannels.toUInt(), outRate.toUInt(), audioSettingsJson, gainDb,
+        override fun prepareTail(planJson: String, pcm: PcmDecoder.Pcm, contextMs: Int, outChannels: Int, outRate: Int, audioSettingsJson: String?, gainDb: Double): RenderedTail {
+            val r = app.hocket.core.ffi.automixPrepareTail(
+                planJson, pcm.leBytes(), contextMs.toUInt(), pcm.channels.toUInt(), pcm.rate.toUInt(), outChannels.toUInt(), outRate.toUInt(), audioSettingsJson, gainDb,
             )
+            return RenderedTail(floats(r.tail), floats(r.lead), floats(r.after))
+        }
+
+        private fun floats(bytes: ByteArray): FloatArray {
             val floats = java.nio.ByteBuffer.wrap(bytes).order(java.nio.ByteOrder.LITTLE_ENDIAN).asFloatBuffer()
             return FloatArray(floats.remaining()).also { floats.get(it) }
         }

android/playback/src/main/java/app/hocket/playback/MixClockSink.kt

@@ -23,6 +23,8 @@ import kotlin.math.abs
 class MixClockSink(
     sink: AudioSink,
     private val nanoTime: () -> Long = System::nanoTime,
+    /** The renderer plays out the end of the whole playlist (the next end of stream is the last). */
+    private val onPlayToEnd: () -> Unit = {},
 ) : ForwardingAudioSink(sink) {
     private val requested = AtomicLong()
     private var correctionUs = 0L
@@ -75,6 +77,11 @@ class MixClockSink(
         return raw - correctionUs
     }
 
+    override fun playToEndOfStream() {
+        onPlayToEnd()
+        super.playToEndOfStream()
+    }
+
     override fun flush() {
         clear()
         super.flush()

android/playback/src/main/java/app/hocket/playback/MixProcessor.kt

@@ -18,8 +18,20 @@ class MixControl {
     /** A playlist item by media id (the queue key). */
     data class Item(val gainDb: Double, val clipEndMs: Long?)
 
-    /** A rendered tail: interleaved floats at B's rate and channel count. */
-    class Tail(val samples: FloatArray, val rate: Int, val channels: Int) {
+    /**
+     * A rendered tail: interleaved floats at B's rate and channel count. [lead] is A just before the
+     * tail (B's format) and [after] A from the tail's start (A's format, [afterRate] ×
+     * [afterChannels]): the [OutputRate] seam when A and B are converted differently.
+     */
+    class Tail(
+        val samples: FloatArray,
+        val rate: Int,
+        val channels: Int,
+        val lead: FloatArray = FloatArray(0),
+        val after: FloatArray = FloatArray(0),
+        val afterRate: Int = 0,
+        val afterChannels: Int = 0,
+    ) {
         val frames: Int get() = samples.size / channels.coerceAtLeast(1)
     }
 
@@ -110,6 +122,12 @@ class TransitionMixer(
 
     val mixing: Boolean get() = mix != null
 
+    /** The running mix's tail. */
+    val tail: MixControl.Tail? get() = mix?.tail
+
+    /** The last [startItem] continued the stream before it (a gapless join, or the same item going on). */
+    val joined: Boolean get() = joining
+
     fun configure(rate: Int, channels: Int) {
         if (chain == 0L) {
             chain = dsp.chainNew(rate, channels)
@@ -126,11 +144,12 @@ class TransitionMixer(
      * item's end without a seek (a gapless join), else a seek or a fresh start.
      */
     fun startItem(mediaId: String?, positionUs: Long, contiguous: Boolean) {
-        // Until a frame comes, a flush at the item's start still belongs to the join. When B's format
-        // differs from A's (another sample rate) the sink can't join gaplessly: it drains A, flushes
-        // once still naming A (the new configuration takes over after that flush), then flushes for
-        // B without another end of stream. Taken as a seek, that second flush would leave B unmixed.
-        // A seek while the sink waits for A to play out lands elsewhere and stays a seek.
+        // Until a frame comes, a flush at the item's start still belongs to the join. When the sink
+        // can't keep its `AudioTrack` for B (another channel count; the rate is always the same)
+        // it drains A, flushes once still naming A (the new configuration takes over after that
+        // flush), then flushes for B without another end of stream. Taken as a seek, that second
+        // flush would leave B unmixed. A seek while the sink waits for A to play out lands
+        // elsewhere and stays a seek.
         val joined = contiguous || (joining && itemFrames == 0L && positionUs < JOIN_START_US)
         joining = joined
         if (joined && mediaId != null && mediaId == item) {
@@ -298,22 +317,38 @@ class TransitionMixer(
 }
 
 /**
- * The ExoPlayer side of [TransitionMixer] (`docs/automix.md` → Android): installed in the main
- * player's `DefaultAudioSink`, on the PCM path only (no offload, 16-bit or float in, the same out).
+ * The ExoPlayer side of [TransitionMixer] and [OutputRate] (`docs/automix.md` → Android): installed
+ * in the main player's `DefaultAudioSink`, on the PCM path only (no offload, 16-bit or float in, the
+ * same out).
  *
  * Item boundaries come from the sink itself: at a gapless join it drains the processors
  * (`queueEndOfStream`) and flushes them with the new stream's [AudioProcessor.StreamMetadata],
  * whose timeline and period name the playlist item; a seek flushes without an end of stream first.
  * So the first frame after such a flush is the item's first frame, sample-exact, without counting
  * frames against durations. The sink then drains and flushes once more for the same item (a timing
  * resync after every stream change); [TransitionMixer.startItem] lets that through as the same stream.
- * At a format change (B at another sample rate) the sink drains A and flushes twice before B's first
- * frame, the first time still naming A and the second without an end of stream before it;
- * [TransitionMixer.startItem] takes every flush before the next frame as part of the join.
+ *
+ * Every item leaves at one rate ([OutputRate]), so a join at another sample rate is a gapless join
+ * like any other: the sink keeps its `AudioTrack`. (A change of channel count still rebuilds it:
+ * the sink drains A and flushes twice before B's first frame, the first time still naming A;
+ * [TransitionMixer.startItem] takes every flush before the next frame as part of the join.)
  */
-class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) : BaseAudioProcessor() {
+class MixProcessor(
+    dsp: DspEngine,
+    private val control: MixControl,
+    listener: MixListener,
+    /** The output rate when the first item isn't at 44.1 or 48 kHz: the device's. */
+    fallbackRate: Int,
+) : BaseAudioProcessor() {
     val mixer = TransitionMixer(dsp, control, listener)
+    val rate = OutputRate(dsp, fallbackRate)
     private var endedBeforeFlush = false
+    /** The input format the sink configured last: the next stream's, once it differs. */
+    private var configured: AudioProcessor.AudioFormat? = null
+    /** The stream ends at this end of stream: its resampler's last frames are still to come out. */
+    private var finishPending = false
+    /** The sink plays out the end of the playlist: the next end of stream is the last. */
+    private var lastEnd = false
     private val period = Timeline.Period()
     private val window = Timeline.Window()
 
@@ -322,31 +357,60 @@ class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) :
             PlaybackLog.w("DSP bypassed for PCM encoding ${inputAudioFormat.encoding}")
             return AudioProcessor.AudioFormat.NOT_SET
         }
-        return inputAudioFormat
+        configured = inputAudioFormat
+        val out = rate.latch(inputAudioFormat.sampleRate)
+        return AudioProcessor.AudioFormat(out, inputAudioFormat.channelCount, inputAudioFormat.encoding)
+    }
+
+    /** The sink is asked to play out the end of the playlist (on the playback thread, before its drain). */
+    fun playToEnd() {
+        lastEnd = true
     }
 
     override fun onQueueEndOfStream() {
         endedBeforeFlush = true
+        // The next stream is at another format (it was configured while this one played), or there
+        // is none: this resampler's held-back frames are this stream's last, and must come out now.
+        val next = configured
+        if (rate.resampling && (lastEnd || (next != null && rate.endsBefore(next.sampleRate, next.channelCount)))) finishPending = true
     }
 
+    override fun isEnded(): Boolean = super.isEnded() && !finishPending
+
     override fun onFlush(streamMetadata: AudioProcessor.StreamMetadata) {
+        finishPending = false
+        lastEnd = false
         mixer.configure(inputAudioFormat.sampleRate, inputAudioFormat.channelCount)
         mixer.startItem(mediaIdOf(streamMetadata), streamMetadata.positionOffsetUs, endedBeforeFlush)
+        val lead = mixer.tail?.takeIf { mixer.joined }?.lead
+        rate.start(inputAudioFormat.sampleRate, inputAudioFormat.channelCount, mixer.joined, lead)
         endedBeforeFlush = false
     }
 
     override fun onReset() {
         mixer.release()
+        rate.release()
+        configured = null
         endedBeforeFlush = false
+        finishPending = false
+        lastEnd = false
     }
 
     override fun queueInput(inputBuffer: ByteBuffer) {
         val format = inputAudioFormat
         val float = format.encoding == C.ENCODING_PCM_FLOAT
+        if (finishPending) {
+            // After the end of stream the pipeline asks for more with an empty buffer: the last frames.
+            finishPending = false
+            val n = finish()
+            emit(rate.output, n, format.channelCount, float)
+            return
+        }
         val frames = inputBuffer.remaining() / format.bytesPerFrame
         if (frames == 0) return
-        if (mixer.passThrough(frames)) {
-            // Nothing to do (no mix, nothing clipped, a passthrough chain): the input as is.
+        val pass = mixer.passThrough(frames)
+        if (pass && !rate.resampling) {
+            // Nothing to do (no mix, nothing clipped, a passthrough chain, the output rate): the input as is.
             replaceOutputBuffer(inputBuffer.remaining()).put(inputBuffer).flip()
             return
         }
@@ -359,14 +423,36 @@ class MixProcessor(dsp: DspEngine, control: MixControl, listener: MixListener) :
             for (i in 0 until samples) work.putFloat(i * 4, inputBuffer.getShort(start + i * 2) / 32768f)
         }
         inputBuffer.position(inputBuffer.limit())
-        val outFrames = mixer.process(frames)
-        val outSamples = outFrames * format.channelCount
-        val out = replaceOutputBuffer(outFrames * format.bytesPerFrame)
-        val src = mixer.output
+        var src = work
+        var n = frames
+        if (!pass) {
+            n = mixer.process(frames)
+            src = mixer.output
+        }
+        if (rate.resampling) {
+            n = rate.process(src, n)
+            src = rate.output
+        }
+        emit(src, n, format.channelCount, float)
+    }
+
+    /** The stream's last frames, with A's continuation as lookahead when the armed mix goes on from here. */
+    private fun finish(): Int {
+        val armed = control.armed
+        val tail = armed?.tail
+        val after = tail?.takeIf {
+            armed.from == mixer.item && it.afterRate == rate.inRate && it.afterChannels == rate.channels
+        }?.after
+        return rate.finish(after)
+    }
+
+    private fun emit(src: ByteBuffer, frames: Int, channels: Int, float: Boolean) {
+        val samples = frames * channels
+        val out = replaceOutputBuffer(samples * if (float) 4 else 2)
         if (float) {
-            for (i in 0 until outSamples) out.putFloat(src.getFloat(i * 4))
+            for (i in 0 until samples) out.putFloat(src.getFloat(i * 4))
         } else {
-            for (i in 0 until outSamples) out.putShort(toPcm16(src.getFloat(i * 4)))
+            for (i in 0 until samples) out.putShort(toPcm16(src.getFloat(i * 4)))
         }
         out.flip()
     }

android/playback/src/main/java/app/hocket/playback/NativeDsp.kt

@@ -25,6 +25,22 @@ interface DspEngine {
     fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int)
     /** The chain's soft limiter (−1 dB knee), for the sum of a tail and B. */
     fun limit(buffer: ByteBuffer, samples: Int)
+
+    /**
+     * A `StreamResampler` (`crates/hocket-android/src/resampler.rs`): one stream from [inRate] to
+     * [outRate], output frame `j` at input frame `j × inRate / outRate` exactly.
+     */
+    fun resamplerNew(inRate: Int, outRate: Int, channels: Int): Long
+    fun resamplerFree(handle: Long)
+    /** Frames before the stream's start, as history only. Before any input. */
+    fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int)
+    /** The stream ends: frames that would have followed, as lookahead only (0 samples: silence). */
+    fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int)
+    /**
+     * Take [inSamples] from [input], then write what is ready into [output] from sample [outOffset]
+     * up to [outCapacity]; returns the samples written. Call again with no input while it fills it.
+     */
+    fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int): Int
 }
 
 /**
@@ -56,4 +72,9 @@ object NativeDsp : DspEngine {
     external override fun shaperFree(handle: Long)
     external override fun shaperProcess(handle: Long, buffer: ByteBuffer, samples: Int)
     external override fun limit(buffer: ByteBuffer, samples: Int)
+    external override fun resamplerNew(inRate: Int, outRate: Int, channels: Int): Long
+    external override fun resamplerFree(handle: Long)
+    external override fun resamplerPrime(handle: Long, buffer: ByteBuffer, samples: Int)
+    external override fun resamplerFinish(handle: Long, buffer: ByteBuffer, samples: Int)
+    external override fun resamplerProcess(handle: Long, input: ByteBuffer, inSamples: Int, output: ByteBuffer, outOffset: Int, outCapacity: Int): Int
 }