diff --git a/data/src/commonMain/kotlin/com/maxrave/data/repository/StreamRepositoryImpl.kt b/data/src/commonMain/kotlin/com/maxrave/data/repository/StreamRepositoryImpl.kt index 083226568..bbd798aca 100644 --- a/data/src/commonMain/kotlin/com/maxrave/data/repository/StreamRepositoryImpl.kt +++ b/data/src/commonMain/kotlin/com/maxrave/data/repository/StreamRepositoryImpl.kt @@ -169,11 +169,9 @@ internal class StreamRepositoryImpl( ?: formatList.find { !it.isAudio && it.url.isNullOrEmpty().not() } val audioTwinItag = ITAG.highQualityTwinOf(itag) val audioFormat = - formatList.find { it.itag == itag } ?: if (audioTwinItag != null) { - formatList.find { it.itag == audioTwinItag } - } else { - formatList.find { it.isAudio && it.url.isNullOrEmpty().not() } - } + formatList.find { it.itag == itag } + ?: audioTwinItag?.let { twin -> formatList.find { it.itag == twin } } + ?: formatList.find { it.isAudio && it.url.isNullOrEmpty().not() } var format = if (isVideo) { videoFormat @@ -410,4 +408,4 @@ internal class StreamRepositoryImpl( } } } -} \ No newline at end of file +} diff --git a/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/BraveNewPipeUtils.kt b/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/BraveNewPipeUtils.kt index 1164f1c65..650d24a8e 100644 --- a/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/BraveNewPipeUtils.kt +++ b/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/BraveNewPipeUtils.kt @@ -2,6 +2,8 @@ package com.maxrave.kotlinytmusicscraper.extractor import com.maxrave.common.ITAG import com.maxrave.kotlinytmusicscraper.models.YouTubeClient +import com.maxrave.logger.Logger +import dev.maxrave.pipepipe.extractor.services.youtube.YoutubeParsingHelper import okhttp3.OkHttpClient import okhttp3.RequestBody.Companion.toRequestBody import org.schabi.newpipe.extractor.downloader.Downloader @@ -10,13 +12,10 @@ import org.schabi.newpipe.extractor.downloader.Response import org.schabi.newpipe.extractor.exceptions.ReCaptchaException import java.io.IOException import java.net.Proxy +import java.util.concurrent.TimeUnit -internal fun List>.hasRequiredItags(): Boolean { - val itags = this.mapTo(HashSet()) { it.first } - val hasAudio = ITAG.AUDIO.any { it in itags } - val hasVideo = ITAG.VIDEO.any { it in itags } - return hasAudio && hasVideo -} +internal fun List>.hasPlayableAudio(): Boolean = + any { it.first in ITAG.AUDIO && it.second.isNotBlank() } private val streamHealthCheckClient: OkHttpClient by lazy { OkHttpClient @@ -25,33 +24,106 @@ private val streamHealthCheckClient: OkHttpClient by lazy { } /** - * Pick one random URL whose itag belongs to [ITAG.AUDIO] or [ITAG.VIDEO] and HEAD it. Returns true - * only when the response code is in the 200..299 range. We only health-check those URLs because - * they are the ones the player will actually use; lower-quality extras can stay unverified. + * Probe ordinary audio first. A rejected rendition does not rule out the other audio URLs. + * After recovering with another rendition, return that audio alone so format selection cannot + * choose an unverified fallback. Inconclusive network checks still allow actual playback. */ -internal fun List>.headCheckRandomStream(): Boolean { - val required = ITAG.AUDIO + ITAG.VIDEO - val candidate = this.filter { it.first in required }.randomOrNull() ?: return false - return runCatching { - val request = - okhttp3.Request - .Builder() - .head() - .url(candidate.second) - .build() - streamHealthCheckClient.newCall(request).execute().use { response -> - response.code in 200..299 +internal fun List>.playableStreams( + videoId: String, + tier: String, +): List> { + val audioPriority = listOf(ITAG.AUDIO_OPUS_MEDIUM, ITAG.AUDIO_OPUS_LOW, ITAG.AUDIO_OPUS_HIGH, ITAG.AUDIO_AAC_HIGH) + val candidates = + filter { it.first in ITAG.AUDIO && it.second.isNotBlank() } + .sortedBy { audioPriority.indexOf(it.first) } + .distinctBy { it.second } + var hadRejection = false + for (candidate in candidates) { + val accepted = try { + val request = + okhttp3.Request + .Builder() + .get() + .header("Range", "bytes=0-0") + .url(candidate.second) + .build() + val clientName = request.url.queryParameter("c") ?: "unknown" + streamHealthCheckClient.newCall(request).execute().use { response -> + val rejected = response.code in 400..499 && response.code !in setOf(405, 408, 429) + if (response.code !in 200..299) { + Logger.w( + "Extractor", + "PipePipe[$tier] audio probe for $videoId itag=${candidate.first} client=$clientName: HTTP ${response.code}, rejected=$rejected", + ) + } + if (response.code in 200..299) { + response.body?.source()?.readByte() + Logger.i("Extractor", "PipePipe[$tier] audio probe passed for $videoId itag=${candidate.first} client=$clientName") + } + !rejected + } + } catch (e: IOException) { + Logger.w( + "Extractor", + "PipePipe[$tier] audio probe inconclusive for $videoId itag=${candidate.first}: ${e.javaClass.simpleName}; trying playback", + ) + true + } + if (accepted) { + if (!hadRejection) return this + Logger.i("Extractor", "PipePipe[$tier] recovered audio for $videoId with itag=${candidate.first}") + val audioItags = ITAG.AUDIO + setOf(140, 249) + return filter { it.first !in audioItags || it.second == candidate.second } } - }.getOrDefault(false) + hadRejection = true + } + return emptyList() } class BraveNewPipeDownloaderImpl( proxy: Proxy?, ) : Downloader() { + @Volatile + private var loginCookie: String? = null + + fun logIn(cookie: String?) { + loginCookie = cookie?.takeIf { it.isNotBlank() } + } + private val client = OkHttpClient .Builder() .proxy(proxy) + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .addNetworkInterceptor { chain -> + val request = chain.request() + val builder = request.newBuilder() + val cookie = loginCookie + val trustedHost = request.url.isHttps && request.url.host in setOf("www.youtube.com", "music.youtube.com") + if (trustedHost && cookie != null) { + // Brave's default downloader only sends consent cookies. Keep the fallback + // signed in too, with SAPISIDHASH bound to the actual YouTube origin. + builder.header("Cookie", cookie) + if (request.url.encodedPath.startsWith("/youtubei/v1/")) { + val origin = "https://${request.url.host}" + runCatching { YoutubeParsingHelper.getAuthorizationHeader(cookie, origin) } + .onSuccess { + builder.header("Authorization", it) + builder.header("Origin", origin) + builder.header("X-Origin", origin) + }.onFailure { + Logger.w("Extractor", "BravePipe login header unavailable: ${it.javaClass.simpleName}") + } + } + } else if (!trustedHost) { + // Also enforce this on redirects; credentials must never reach media, + // decoder, dislike or other third-party hosts. + builder.removeHeader("Cookie") + builder.removeHeader("Authorization") + } + chain.proceed(builder.build()) + } .build() @Throws(IOException::class, ReCaptchaException::class) diff --git a/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/Extractor.android.kt b/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/Extractor.android.kt index d7eddb8f1..8aac4aee9 100644 --- a/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/Extractor.android.kt +++ b/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/Extractor.android.kt @@ -27,12 +27,17 @@ actual class Extractor { actual fun init() { NewPipe.init(newPipeDownloader) + // PipePipe defaults to five seconds, which is too short for slower VPN routes. + ServiceList.YouTube.setLoadingTimeout(30) BraveNewPipe.init(braveNewPipeDownloader) YoutubeApiDecoder.setLocalDecoder(faradayDecoder) } actual fun logIn(cookie: String?) { ServiceList.YouTube.tokens = cookie ?: "" + braveNewPipeDownloader.logIn(cookie) + // The visionos player request is anonymous; TV requests carry the saved login session. + NewPipe.setYoutubePlayerClient(if (cookie.isNullOrEmpty()) "visionos" else "tv_samsung") } actual fun newPipePlayer(videoId: String): List> { @@ -110,19 +115,24 @@ actual class Extractor { tier: String, ): List>? { try { - val streamInfo = - StreamInfo.getInfo(ServiceList.YouTube, "https://music.youtube.com/watch?v=$videoId") - val streamsList = streamInfo.audioStreams + streamInfo.videoStreams + streamInfo.videoOnlyStreams + // The app already has metadata from WEB_REMIX. StreamInfo requires a title before + // extracting streams, so a TV response without a title can discard playable audio. + val streamExtractor = ServiceList.YouTube.getStreamExtractor("https://music.youtube.com/watch?v=$videoId") + streamExtractor.fetchPage() + val streamsList = + streamExtractor.audioStreams + + runCatching { streamExtractor.videoStreams }.getOrDefault(emptyList()) + + runCatching { streamExtractor.videoOnlyStreams }.getOrDefault(emptyList()) val pipeResult = streamsList.mapNotNull { (it.itagItem?.id ?: return@mapNotNull null) to it.content } - if (!pipeResult.hasRequiredItags()) { - Logger.d(TAG, "PipePipe[$tier] missing required itags for $videoId (got=${pipeResult.map { it.first }})") + if (!pipeResult.hasPlayableAudio()) { + Logger.w(TAG, "PipePipe[$tier] missing playable audio itags for $videoId (got=${pipeResult.map { it.first }})") return null } - if (!pipeResult.headCheckRandomStream()) { - Logger.d(TAG, "PipePipe[$tier] stream URL HEAD check failed (non 2xx) for $videoId") + val playableResult = pipeResult.playableStreams(videoId, tier) + if (playableResult.isEmpty()) { // A rejected URL is the one symptom of a stale player table: the signature is // well-formed and still wrong, so nothing threw on the way here. Only the on-device // table can go stale, so only that tier is worth invalidating. @@ -131,8 +141,8 @@ actual class Extractor { } val label = if (tier == LOCAL_TIER) faradayDecoder.lastOutcomeLabel else REMOTE_TIER ExtractSource.record(videoId, "PipePipe · $label") - Logger.d(TAG, "extract source=PipePipe[$tier] itags=${pipeResult.map { it.first }} for $videoId") - return pipeResult + Logger.i(TAG, "extract source=PipePipe[$tier] itags=${playableResult.map { it.first }} for $videoId") + return playableResult } catch (e: Throwable) { Logger.w(TAG, "PipePipe[$tier] extractor failed for $videoId: ${e.message}") return null @@ -332,4 +342,4 @@ actual class Extractor { return (DownloadProgress.failed("Error")) } } -} \ No newline at end of file +} diff --git a/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/NewPipeUtils.kt b/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/NewPipeUtils.kt index aedf10b89..d12eb5e32 100644 --- a/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/NewPipeUtils.kt +++ b/service/kotlinYtmusicScraper/src/androidMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/NewPipeUtils.kt @@ -17,6 +17,7 @@ import okhttp3.OkHttpClient import okhttp3.RequestBody.Companion.toRequestBody import java.io.IOException import java.net.Proxy +import java.util.concurrent.TimeUnit class NewPipeDownloaderImpl( proxy: Proxy?, @@ -25,6 +26,8 @@ class NewPipeDownloaderImpl( OkHttpClient .Builder() .proxy(proxy) + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) .build() @Throws(IOException::class, ReCaptchaException::class) @@ -36,7 +39,7 @@ class NewPipeDownloaderImpl( throw ReCaptchaException("reCaptcha Challenge requested", request.url()) } - return response.toNewPipeResponse() + return response.toNewPipeResponse(request) } @Throws(IOException::class, ReCaptchaException::class) @@ -68,7 +71,7 @@ class NewPipeDownloaderImpl( ) return } - callback?.onSuccess(response.toNewPipeResponse()) + callback?.onSuccess(response.toNewPipeResponse(request)) } catch (e: Exception) { callback?.onError(e) } finally { @@ -80,8 +83,13 @@ class NewPipeDownloaderImpl( return cancellable } - private fun okhttp3.Response.toNewPipeResponse(): Response { - val rawBytes = body?.bytes() ?: ByteArray(0) + private fun okhttp3.Response.toNewPipeResponse(extractorRequest: Request): Response { + val rawBytes = + filterSupplementaryPlayerResponse( + extractorRequest.url(), + extractorRequest.dataToSend(), + body?.bytes() ?: ByteArray(0), + ) return Response( code, message, diff --git a/service/kotlinYtmusicScraper/src/commonMain/kotlin/com/maxrave/kotlinytmusicscraper/YouTube.kt b/service/kotlinYtmusicScraper/src/commonMain/kotlin/com/maxrave/kotlinytmusicscraper/YouTube.kt index b462286f6..7d145afdd 100644 --- a/service/kotlinYtmusicScraper/src/commonMain/kotlin/com/maxrave/kotlinytmusicscraper/YouTube.kt +++ b/service/kotlinYtmusicScraper/src/commonMain/kotlin/com/maxrave/kotlinytmusicscraper/YouTube.kt @@ -1376,7 +1376,6 @@ class YouTube { tempRes: PlayerResponse, preferredAudioLanguage: String? = null, ): PlayerResponse? { - val listUrlSig = mutableListOf() var decodedSigResponse: PlayerResponse? var sigResponse: PlayerResponse? Logger.d(TAG, "YouTube TempRes ${tempRes.playabilityStatus}") @@ -1450,32 +1449,21 @@ class YouTube { }, ), ) - listUrlSig.addAll( - ( - decodedSigResponse - .streamingData - ?.adaptiveFormats - ?.mapNotNull { it.url } - ?.toMutableList() ?: mutableListOf() - ).apply { - decodedSigResponse - .streamingData - ?.formats - ?.mapNotNull { it.url } - ?.let { addAll(it) } - }, - ) - listUrlSig.forEach { - Logger.d(TAG, "YouTube NewPipe URL $it") - } - val randomUrl = listUrlSig.randomOrNull() ?: return null - if (listUrlSig.isNotEmpty() && !is403Url(randomUrl)) { - Logger.d(TAG, "YouTube NewPipe Found URL $randomUrl") - return decodedSigResponse - } else { - Logger.d(TAG, "YouTube NewPipe No URL Found") + // PipePipe already checks an audio URL before returning streams. A second random + // HEAD here could discard valid audio because an unrelated video URL is rejected. + val hasAudio = + decodedSigResponse.streamingData?.adaptiveFormats?.any { + it.itag in ITAG.AUDIO && !it.url.isNullOrBlank() + } == true + val hasManifest = + decodedSigResponse.streamingData?.formats?.any { + it.url?.let(::isManifestUrl) == true + } == true + if (!hasAudio && !hasManifest) { + Logger.w(TAG, "NewPipe returned no mapped audio or manifest for $videoId") return null } + return decodedSigResponse } fun isManifestUrl(url: String): Boolean = url.contains(".m3u8") || url.contains(".mpd") || url.contains("manifest") diff --git a/service/kotlinYtmusicScraper/src/commonMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/SupplementaryPlayerResponse.kt b/service/kotlinYtmusicScraper/src/commonMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/SupplementaryPlayerResponse.kt new file mode 100644 index 000000000..a9393486a --- /dev/null +++ b/service/kotlinYtmusicScraper/src/commonMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/SupplementaryPlayerResponse.kt @@ -0,0 +1,52 @@ +package com.maxrave.kotlinytmusicscraper.extractor + +import com.maxrave.common.ITAG +import com.maxrave.logger.Logger +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull + +/** + * PipePipe scans WEB_REMIX before its configured player and keeps the first similar stream. + * Limit the supplementary response to the Premium audio it was intended to supply, so its + * ordinary web streams cannot replace the TV streams with URLs that require a web PoToken. + */ +internal fun filterSupplementaryPlayerResponse( + requestUrl: String, + requestBody: ByteArray?, + responseBody: ByteArray, +): ByteArray { + if (!requestUrl.startsWith("https://music.youtube.com/youtubei/v1/player") || requestBody == null) { + return responseBody + } + return try { + val request = Json.parseToJsonElement(requestBody.decodeToString()) as? JsonObject ?: return responseBody + val context = request["context"] as? JsonObject + val client = context?.get("client") as? JsonObject + if ((client?.get("clientName") as? JsonPrimitive)?.contentOrNull != "WEB_REMIX") return responseBody + val response = Json.parseToJsonElement(responseBody.decodeToString()) as? JsonObject ?: return responseBody + val streamingData = response["streamingData"] as? JsonObject ?: return responseBody + val premiumItags = setOf(ITAG.AUDIO_AAC_HIGH, ITAG.AUDIO_OPUS_HIGH) + var removed = 0 + val filtered = streamingData.toMutableMap() + for (key in listOf("adaptiveFormats", "formats")) { + val formats = streamingData[key] as? JsonArray ?: continue + val premium = formats.filter { format -> + val itag = ((format as? JsonObject)?.get("itag") as? JsonPrimitive)?.intOrNull + itag != null && itag in premiumItags + } + removed += formats.size - premium.size + filtered[key] = JsonArray(premium) + } + if (removed == 0) return responseBody + val videoId = (request["videoId"] as? JsonPrimitive)?.contentOrNull + Logger.i("Extractor", "WEB_REMIX supplement for $videoId: removed $removed ordinary formats; keeping Premium audio") + JsonObject(response + ("streamingData" to JsonObject(filtered))).toString().encodeToByteArray() + } catch (e: IllegalArgumentException) { + Logger.w("Extractor", "Could not filter WEB_REMIX supplement: ${e::class.simpleName}") + responseBody + } +} diff --git a/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/BraveNewPipeUtils.kt b/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/BraveNewPipeUtils.kt index 7694410fb..a444f8157 100644 --- a/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/BraveNewPipeUtils.kt +++ b/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/BraveNewPipeUtils.kt @@ -2,6 +2,8 @@ package com.maxrave.kotlinytmusicscraper.extractor import com.maxrave.common.ITAG import com.maxrave.kotlinytmusicscraper.models.YouTubeClient +import com.maxrave.logger.Logger +import dev.maxrave.pipepipe.extractor.services.youtube.YoutubeParsingHelper import okhttp3.OkHttpClient import okhttp3.RequestBody.Companion.toRequestBody import org.schabi.newpipe.extractor.downloader.Downloader @@ -10,13 +12,10 @@ import org.schabi.newpipe.extractor.downloader.Response import org.schabi.newpipe.extractor.exceptions.ReCaptchaException import java.io.IOException import java.net.Proxy +import java.util.concurrent.TimeUnit -internal fun List>.hasRequiredItags(): Boolean { - val itags = this.mapTo(HashSet()) { it.first } - val hasAudio = ITAG.AUDIO.any { it in itags } - val hasVideo = ITAG.VIDEO.any { it in itags } - return hasAudio && hasVideo -} +internal fun List>.hasPlayableAudio(): Boolean = + any { it.first in ITAG.AUDIO && it.second.isNotBlank() } private val streamHealthCheckClient: OkHttpClient by lazy { OkHttpClient @@ -25,33 +24,106 @@ private val streamHealthCheckClient: OkHttpClient by lazy { } /** - * Pick one random URL whose itag belongs to [ITAG.AUDIO] or [ITAG.VIDEO] and HEAD it. Returns true - * only when the response code is in the 200..299 range. We only health-check those URLs because - * they are the ones the player will actually use; lower-quality extras can stay unverified. + * Probe ordinary audio first. A rejected rendition does not rule out the other audio URLs. + * After recovering with another rendition, return that audio alone so format selection cannot + * choose an unverified fallback. Inconclusive network checks still allow actual playback. */ -internal fun List>.headCheckRandomStream(): Boolean { - val required = ITAG.AUDIO + ITAG.VIDEO - val candidate = this.filter { it.first in required }.randomOrNull() ?: return false - return runCatching { - val request = - okhttp3.Request - .Builder() - .head() - .url(candidate.second) - .build() - streamHealthCheckClient.newCall(request).execute().use { response -> - response.code in 200..299 +internal fun List>.playableStreams( + videoId: String, + tier: String, +): List> { + val audioPriority = listOf(ITAG.AUDIO_OPUS_MEDIUM, ITAG.AUDIO_OPUS_LOW, ITAG.AUDIO_OPUS_HIGH, ITAG.AUDIO_AAC_HIGH) + val candidates = + filter { it.first in ITAG.AUDIO && it.second.isNotBlank() } + .sortedBy { audioPriority.indexOf(it.first) } + .distinctBy { it.second } + var hadRejection = false + for (candidate in candidates) { + val accepted = try { + val request = + okhttp3.Request + .Builder() + .get() + .header("Range", "bytes=0-0") + .url(candidate.second) + .build() + val clientName = request.url.queryParameter("c") ?: "unknown" + streamHealthCheckClient.newCall(request).execute().use { response -> + val rejected = response.code in 400..499 && response.code !in setOf(405, 408, 429) + if (response.code !in 200..299) { + Logger.w( + "Extractor", + "PipePipe[$tier] audio probe for $videoId itag=${candidate.first} client=$clientName: HTTP ${response.code}, rejected=$rejected", + ) + } + if (response.code in 200..299) { + response.body?.source()?.readByte() + Logger.i("Extractor", "PipePipe[$tier] audio probe passed for $videoId itag=${candidate.first} client=$clientName") + } + !rejected + } + } catch (e: IOException) { + Logger.w( + "Extractor", + "PipePipe[$tier] audio probe inconclusive for $videoId itag=${candidate.first}: ${e.javaClass.simpleName}; trying playback", + ) + true + } + if (accepted) { + if (!hadRejection) return this + Logger.i("Extractor", "PipePipe[$tier] recovered audio for $videoId with itag=${candidate.first}") + val audioItags = ITAG.AUDIO + setOf(140, 249) + return filter { it.first !in audioItags || it.second == candidate.second } } - }.getOrDefault(false) + hadRejection = true + } + return emptyList() } class BraveNewPipeDownloaderImpl( proxy: Proxy?, ) : Downloader() { + @Volatile + private var loginCookie: String? = null + + fun logIn(cookie: String?) { + loginCookie = cookie?.takeIf { it.isNotBlank() } + } + private val client = OkHttpClient .Builder() .proxy(proxy) + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .addNetworkInterceptor { chain -> + val request = chain.request() + val builder = request.newBuilder() + val cookie = loginCookie + val trustedHost = request.url.isHttps && request.url.host in setOf("www.youtube.com", "music.youtube.com") + if (trustedHost && cookie != null) { + // Brave's default downloader only sends consent cookies. Keep the fallback + // signed in too, with SAPISIDHASH bound to the actual YouTube origin. + builder.header("Cookie", cookie) + if (request.url.encodedPath.startsWith("/youtubei/v1/")) { + val origin = "https://${request.url.host}" + runCatching { YoutubeParsingHelper.getAuthorizationHeader(cookie, origin) } + .onSuccess { + builder.header("Authorization", it) + builder.header("Origin", origin) + builder.header("X-Origin", origin) + }.onFailure { + Logger.w("Extractor", "BravePipe login header unavailable: ${it.javaClass.simpleName}") + } + } + } else if (!trustedHost) { + // Also enforce this on redirects; credentials must never reach media, + // decoder, dislike or other third-party hosts. + builder.removeHeader("Cookie") + builder.removeHeader("Authorization") + } + chain.proceed(builder.build()) + } .build() @Throws(IOException::class, ReCaptchaException::class) diff --git a/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/Extractor.jvm.kt b/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/Extractor.jvm.kt index 34ae04b61..a090ee580 100644 --- a/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/Extractor.jvm.kt +++ b/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/Extractor.jvm.kt @@ -22,12 +22,17 @@ actual class Extractor { actual fun init() { NewPipe.init(newPipeDownloader) + // PipePipe defaults to five seconds, which is too short for slower VPN routes. + ServiceList.YouTube.setLoadingTimeout(30) BraveNewPipe.init(braveNewPipeDownloader) YoutubeApiDecoder.setLocalDecoder(faradayDecoder) } actual fun logIn(cookie: String?) { ServiceList.YouTube.tokens = cookie ?: "" + braveNewPipeDownloader.logIn(cookie) + // The visionos player request is anonymous; TV requests carry the saved login session. + NewPipe.setYoutubePlayerClient(if (cookie.isNullOrEmpty()) "visionos" else "tv_samsung") } actual fun newPipePlayer(videoId: String): List> { @@ -105,23 +110,27 @@ actual class Extractor { tier: String, ): List>? { try { - val streamInfo = - StreamInfo.getInfo(ServiceList.YouTube, "https://music.youtube.com/watch?v=$videoId") - val streamsList = streamInfo.audioStreams + streamInfo.videoStreams + streamInfo.videoOnlyStreams + // Metadata is already available in the app; a missing TV title must not stop audio extraction. + val streamExtractor = ServiceList.YouTube.getStreamExtractor("https://music.youtube.com/watch?v=$videoId") + streamExtractor.fetchPage() + val streamsList = + streamExtractor.audioStreams + + runCatching { streamExtractor.videoStreams }.getOrDefault(emptyList()) + + runCatching { streamExtractor.videoOnlyStreams }.getOrDefault(emptyList()) val temp = streamsList .mapNotNull { (it.itagItem?.id ?: return@mapNotNull null) to it.content }.toMutableList() - val manifest = streamInfo.dashMpdUrl.takeIf { !it.isNullOrEmpty() } ?: streamInfo.hlsUrl + val manifest = runCatching { streamExtractor.dashMpdUrl.takeIf { it.isNotEmpty() } ?: streamExtractor.hlsUrl }.getOrNull() if (!manifest.isNullOrEmpty()) temp.add(96 to manifest) val pipeResult = temp.toList() - if (!pipeResult.hasRequiredItags()) { - Logger.d(TAG, "PipePipe[$tier] missing required itags for $videoId (got=${pipeResult.map { it.first }})") + if (!pipeResult.hasPlayableAudio()) { + Logger.w(TAG, "PipePipe[$tier] missing playable audio itags for $videoId (got=${pipeResult.map { it.first }})") return null } - if (!pipeResult.headCheckRandomStream()) { - Logger.d(TAG, "PipePipe[$tier] stream URL HEAD check failed (non 2xx) for $videoId") + val playableResult = pipeResult.playableStreams(videoId, tier) + if (playableResult.isEmpty()) { // A rejected URL is the one symptom of a stale player table: the signature is // well-formed and still wrong, so nothing threw on the way here. Only the on-device // table can go stale, so only that tier is worth invalidating. @@ -130,8 +139,8 @@ actual class Extractor { } val label = if (tier == LOCAL_TIER) faradayDecoder.lastOutcomeLabel else REMOTE_TIER ExtractSource.record(videoId, "PipePipe · $label") - Logger.d(TAG, "extract source=PipePipe[$tier] itags=${pipeResult.map { it.first }} for $videoId") - return pipeResult + Logger.i(TAG, "extract source=PipePipe[$tier] itags=${playableResult.map { it.first }} for $videoId") + return playableResult } catch (e: Throwable) { Logger.w(TAG, "PipePipe[$tier] extractor failed for $videoId: ${e.message}") return null @@ -163,4 +172,4 @@ actual class Extractor { filePath: String, track: SongItem, ): DownloadProgress = DownloadProgress.AUDIO_DONE -} \ No newline at end of file +} diff --git a/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/NewPipeUtils.kt b/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/NewPipeUtils.kt index aedf10b89..d12eb5e32 100644 --- a/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/NewPipeUtils.kt +++ b/service/kotlinYtmusicScraper/src/jvmMain/kotlin/com/maxrave/kotlinytmusicscraper/extractor/NewPipeUtils.kt @@ -17,6 +17,7 @@ import okhttp3.OkHttpClient import okhttp3.RequestBody.Companion.toRequestBody import java.io.IOException import java.net.Proxy +import java.util.concurrent.TimeUnit class NewPipeDownloaderImpl( proxy: Proxy?, @@ -25,6 +26,8 @@ class NewPipeDownloaderImpl( OkHttpClient .Builder() .proxy(proxy) + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) .build() @Throws(IOException::class, ReCaptchaException::class) @@ -36,7 +39,7 @@ class NewPipeDownloaderImpl( throw ReCaptchaException("reCaptcha Challenge requested", request.url()) } - return response.toNewPipeResponse() + return response.toNewPipeResponse(request) } @Throws(IOException::class, ReCaptchaException::class) @@ -68,7 +71,7 @@ class NewPipeDownloaderImpl( ) return } - callback?.onSuccess(response.toNewPipeResponse()) + callback?.onSuccess(response.toNewPipeResponse(request)) } catch (e: Exception) { callback?.onError(e) } finally { @@ -80,8 +83,13 @@ class NewPipeDownloaderImpl( return cancellable } - private fun okhttp3.Response.toNewPipeResponse(): Response { - val rawBytes = body?.bytes() ?: ByteArray(0) + private fun okhttp3.Response.toNewPipeResponse(extractorRequest: Request): Response { + val rawBytes = + filterSupplementaryPlayerResponse( + extractorRequest.url(), + extractorRequest.dataToSend(), + body?.bytes() ?: ByteArray(0), + ) return Response( code, message,