Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -169,11 +169,9 @@ internal class StreamRepositoryImpl(
?: formatList.find { !it.isAudio && it.url.isNullOrEmpty().not() }
val audioTwinItag = ITAG.highQualityTwinOf(itag)
val audioFormat =
formatList.find { it.itag == itag } ?: if (audioTwinItag != null) {
formatList.find { it.itag == audioTwinItag }
} else {
formatList.find { it.isAudio && it.url.isNullOrEmpty().not() }
}
formatList.find { it.itag == itag }
?: audioTwinItag?.let { twin -> formatList.find { it.itag == twin } }
?: formatList.find { it.isAudio && it.url.isNullOrEmpty().not() }
var format =
if (isVideo) {
videoFormat
Expand Down Expand Up @@ -410,4 +408,4 @@ internal class StreamRepositoryImpl(
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package com.maxrave.kotlinytmusicscraper.extractor

import com.maxrave.common.ITAG
import com.maxrave.kotlinytmusicscraper.models.YouTubeClient
import com.maxrave.logger.Logger
import dev.maxrave.pipepipe.extractor.services.youtube.YoutubeParsingHelper
import okhttp3.OkHttpClient
import okhttp3.RequestBody.Companion.toRequestBody
import org.schabi.newpipe.extractor.downloader.Downloader
Expand All @@ -10,13 +12,10 @@ import org.schabi.newpipe.extractor.downloader.Response
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException
import java.io.IOException
import java.net.Proxy
import java.util.concurrent.TimeUnit

internal fun List<Pair<Int, String>>.hasRequiredItags(): Boolean {
val itags = this.mapTo(HashSet()) { it.first }
val hasAudio = ITAG.AUDIO.any { it in itags }
val hasVideo = ITAG.VIDEO.any { it in itags }
return hasAudio && hasVideo
}
internal fun List<Pair<Int, String>>.hasPlayableAudio(): Boolean =
any { it.first in ITAG.AUDIO && it.second.isNotBlank() }

private val streamHealthCheckClient: OkHttpClient by lazy {
OkHttpClient
Expand All @@ -25,33 +24,106 @@ private val streamHealthCheckClient: OkHttpClient by lazy {
}

/**
* Pick one random URL whose itag belongs to [ITAG.AUDIO] or [ITAG.VIDEO] and HEAD it. Returns true
* only when the response code is in the 200..299 range. We only health-check those URLs because
* they are the ones the player will actually use; lower-quality extras can stay unverified.
* Probe ordinary audio first. A rejected rendition does not rule out the other audio URLs.
* After recovering with another rendition, return that audio alone so format selection cannot
* choose an unverified fallback. Inconclusive network checks still allow actual playback.
*/
internal fun List<Pair<Int, String>>.headCheckRandomStream(): Boolean {
val required = ITAG.AUDIO + ITAG.VIDEO
val candidate = this.filter { it.first in required }.randomOrNull() ?: return false
return runCatching {
val request =
okhttp3.Request
.Builder()
.head()
.url(candidate.second)
.build()
streamHealthCheckClient.newCall(request).execute().use { response ->
response.code in 200..299
internal fun List<Pair<Int, String>>.playableStreams(
videoId: String,
tier: String,
): List<Pair<Int, String>> {
val audioPriority = listOf(ITAG.AUDIO_OPUS_MEDIUM, ITAG.AUDIO_OPUS_LOW, ITAG.AUDIO_OPUS_HIGH, ITAG.AUDIO_AAC_HIGH)
val candidates =
filter { it.first in ITAG.AUDIO && it.second.isNotBlank() }
.sortedBy { audioPriority.indexOf(it.first) }
.distinctBy { it.second }
var hadRejection = false
for (candidate in candidates) {
val accepted = try {
val request =
okhttp3.Request
.Builder()
.get()
.header("Range", "bytes=0-0")
.url(candidate.second)
.build()
val clientName = request.url.queryParameter("c") ?: "unknown"
streamHealthCheckClient.newCall(request).execute().use { response ->
val rejected = response.code in 400..499 && response.code !in setOf(405, 408, 429)
if (response.code !in 200..299) {
Logger.w(
"Extractor",
"PipePipe[$tier] audio probe for $videoId itag=${candidate.first} client=$clientName: HTTP ${response.code}, rejected=$rejected",
)
}
if (response.code in 200..299) {
response.body?.source()?.readByte()
Logger.i("Extractor", "PipePipe[$tier] audio probe passed for $videoId itag=${candidate.first} client=$clientName")
}
!rejected
}
} catch (e: IOException) {
Logger.w(
"Extractor",
"PipePipe[$tier] audio probe inconclusive for $videoId itag=${candidate.first}: ${e.javaClass.simpleName}; trying playback",
)
true
}
if (accepted) {
if (!hadRejection) return this
Logger.i("Extractor", "PipePipe[$tier] recovered audio for $videoId with itag=${candidate.first}")
val audioItags = ITAG.AUDIO + setOf(140, 249)
return filter { it.first !in audioItags || it.second == candidate.second }
}
}.getOrDefault(false)
hadRejection = true
}
return emptyList()
}

class BraveNewPipeDownloaderImpl(
proxy: Proxy?,
) : Downloader() {
@Volatile
private var loginCookie: String? = null

fun logIn(cookie: String?) {
loginCookie = cookie?.takeIf { it.isNotBlank() }
}

private val client =
OkHttpClient
.Builder()
.proxy(proxy)
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
.addNetworkInterceptor { chain ->
val request = chain.request()
val builder = request.newBuilder()
val cookie = loginCookie
val trustedHost = request.url.isHttps && request.url.host in setOf("www.youtube.com", "music.youtube.com")
if (trustedHost && cookie != null) {
// Brave's default downloader only sends consent cookies. Keep the fallback
// signed in too, with SAPISIDHASH bound to the actual YouTube origin.
builder.header("Cookie", cookie)
if (request.url.encodedPath.startsWith("/youtubei/v1/")) {
val origin = "https://${request.url.host}"
runCatching { YoutubeParsingHelper.getAuthorizationHeader(cookie, origin) }
.onSuccess {
builder.header("Authorization", it)
builder.header("Origin", origin)
builder.header("X-Origin", origin)
}.onFailure {
Logger.w("Extractor", "BravePipe login header unavailable: ${it.javaClass.simpleName}")
}
}
} else if (!trustedHost) {
// Also enforce this on redirects; credentials must never reach media,
// decoder, dislike or other third-party hosts.
builder.removeHeader("Cookie")
builder.removeHeader("Authorization")
}
chain.proceed(builder.build())
}
.build()

@Throws(IOException::class, ReCaptchaException::class)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,17 @@ actual class Extractor {

actual fun init() {
NewPipe.init(newPipeDownloader)
// PipePipe defaults to five seconds, which is too short for slower VPN routes.
ServiceList.YouTube.setLoadingTimeout(30)
BraveNewPipe.init(braveNewPipeDownloader)
YoutubeApiDecoder.setLocalDecoder(faradayDecoder)
}

actual fun logIn(cookie: String?) {
ServiceList.YouTube.tokens = cookie ?: ""
braveNewPipeDownloader.logIn(cookie)
// The visionos player request is anonymous; TV requests carry the saved login session.
NewPipe.setYoutubePlayerClient(if (cookie.isNullOrEmpty()) "visionos" else "tv_samsung")
}

actual fun newPipePlayer(videoId: String): List<Pair<Int, String>> {
Expand Down Expand Up @@ -110,19 +115,24 @@ actual class Extractor {
tier: String,
): List<Pair<Int, String>>? {
try {
val streamInfo =
StreamInfo.getInfo(ServiceList.YouTube, "https://music.youtube.com/watch?v=$videoId")
val streamsList = streamInfo.audioStreams + streamInfo.videoStreams + streamInfo.videoOnlyStreams
// The app already has metadata from WEB_REMIX. StreamInfo requires a title before
// extracting streams, so a TV response without a title can discard playable audio.
val streamExtractor = ServiceList.YouTube.getStreamExtractor("https://music.youtube.com/watch?v=$videoId")
streamExtractor.fetchPage()
val streamsList =
streamExtractor.audioStreams +
runCatching { streamExtractor.videoStreams }.getOrDefault(emptyList()) +
runCatching { streamExtractor.videoOnlyStreams }.getOrDefault(emptyList())
val pipeResult =
streamsList.mapNotNull {
(it.itagItem?.id ?: return@mapNotNull null) to it.content
}
if (!pipeResult.hasRequiredItags()) {
Logger.d(TAG, "PipePipe[$tier] missing required itags for $videoId (got=${pipeResult.map { it.first }})")
if (!pipeResult.hasPlayableAudio()) {
Logger.w(TAG, "PipePipe[$tier] missing playable audio itags for $videoId (got=${pipeResult.map { it.first }})")
return null
}
if (!pipeResult.headCheckRandomStream()) {
Logger.d(TAG, "PipePipe[$tier] stream URL HEAD check failed (non 2xx) for $videoId")
val playableResult = pipeResult.playableStreams(videoId, tier)
if (playableResult.isEmpty()) {
// A rejected URL is the one symptom of a stale player table: the signature is
// well-formed and still wrong, so nothing threw on the way here. Only the on-device
// table can go stale, so only that tier is worth invalidating.
Expand All @@ -131,8 +141,8 @@ actual class Extractor {
}
val label = if (tier == LOCAL_TIER) faradayDecoder.lastOutcomeLabel else REMOTE_TIER
ExtractSource.record(videoId, "PipePipe · $label")
Logger.d(TAG, "extract source=PipePipe[$tier] itags=${pipeResult.map { it.first }} for $videoId")
return pipeResult
Logger.i(TAG, "extract source=PipePipe[$tier] itags=${playableResult.map { it.first }} for $videoId")
return playableResult
} catch (e: Throwable) {
Logger.w(TAG, "PipePipe[$tier] extractor failed for $videoId: ${e.message}")
return null
Expand Down Expand Up @@ -332,4 +342,4 @@ actual class Extractor {
return (DownloadProgress.failed("Error"))
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import okhttp3.OkHttpClient
import okhttp3.RequestBody.Companion.toRequestBody
import java.io.IOException
import java.net.Proxy
import java.util.concurrent.TimeUnit

class NewPipeDownloaderImpl(
proxy: Proxy?,
Expand All @@ -25,6 +26,8 @@ class NewPipeDownloaderImpl(
OkHttpClient
.Builder()
.proxy(proxy)
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
.build()

@Throws(IOException::class, ReCaptchaException::class)
Expand All @@ -36,7 +39,7 @@ class NewPipeDownloaderImpl(
throw ReCaptchaException("reCaptcha Challenge requested", request.url())
}

return response.toNewPipeResponse()
return response.toNewPipeResponse(request)
}

@Throws(IOException::class, ReCaptchaException::class)
Expand Down Expand Up @@ -68,7 +71,7 @@ class NewPipeDownloaderImpl(
)
return
}
callback?.onSuccess(response.toNewPipeResponse())
callback?.onSuccess(response.toNewPipeResponse(request))
} catch (e: Exception) {
callback?.onError(e)
} finally {
Expand All @@ -80,8 +83,13 @@ class NewPipeDownloaderImpl(
return cancellable
}

private fun okhttp3.Response.toNewPipeResponse(): Response {
val rawBytes = body?.bytes() ?: ByteArray(0)
private fun okhttp3.Response.toNewPipeResponse(extractorRequest: Request): Response {
val rawBytes =
filterSupplementaryPlayerResponse(
extractorRequest.url(),
extractorRequest.dataToSend(),
body?.bytes() ?: ByteArray(0),
)
return Response(
code,
message,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1376,7 +1376,6 @@ class YouTube {
tempRes: PlayerResponse,
preferredAudioLanguage: String? = null,
): PlayerResponse? {
val listUrlSig = mutableListOf<String>()
var decodedSigResponse: PlayerResponse?
var sigResponse: PlayerResponse?
Logger.d(TAG, "YouTube TempRes ${tempRes.playabilityStatus}")
Expand Down Expand Up @@ -1450,32 +1449,21 @@ class YouTube {
},
),
)
listUrlSig.addAll(
(
decodedSigResponse
.streamingData
?.adaptiveFormats
?.mapNotNull { it.url }
?.toMutableList() ?: mutableListOf()
).apply {
decodedSigResponse
.streamingData
?.formats
?.mapNotNull { it.url }
?.let { addAll(it) }
},
)
listUrlSig.forEach {
Logger.d(TAG, "YouTube NewPipe URL $it")
}
val randomUrl = listUrlSig.randomOrNull() ?: return null
if (listUrlSig.isNotEmpty() && !is403Url(randomUrl)) {
Logger.d(TAG, "YouTube NewPipe Found URL $randomUrl")
return decodedSigResponse
} else {
Logger.d(TAG, "YouTube NewPipe No URL Found")
// PipePipe already checks an audio URL before returning streams. A second random
// HEAD here could discard valid audio because an unrelated video URL is rejected.
val hasAudio =
decodedSigResponse.streamingData?.adaptiveFormats?.any {
it.itag in ITAG.AUDIO && !it.url.isNullOrBlank()
} == true
val hasManifest =
decodedSigResponse.streamingData?.formats?.any {
it.url?.let(::isManifestUrl) == true
} == true
if (!hasAudio && !hasManifest) {
Logger.w(TAG, "NewPipe returned no mapped audio or manifest for $videoId")
return null
}
return decodedSigResponse
}

fun isManifestUrl(url: String): Boolean = url.contains(".m3u8") || url.contains(".mpd") || url.contains("manifest")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package com.maxrave.kotlinytmusicscraper.extractor

import com.maxrave.common.ITAG
import com.maxrave.logger.Logger
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.intOrNull

/**
* PipePipe scans WEB_REMIX before its configured player and keeps the first similar stream.
* Limit the supplementary response to the Premium audio it was intended to supply, so its
* ordinary web streams cannot replace the TV streams with URLs that require a web PoToken.
*/
internal fun filterSupplementaryPlayerResponse(
requestUrl: String,
requestBody: ByteArray?,
responseBody: ByteArray,
): ByteArray {
if (!requestUrl.startsWith("https://music.youtube.com/youtubei/v1/player") || requestBody == null) {
return responseBody
}
return try {
val request = Json.parseToJsonElement(requestBody.decodeToString()) as? JsonObject ?: return responseBody
val context = request["context"] as? JsonObject
val client = context?.get("client") as? JsonObject
if ((client?.get("clientName") as? JsonPrimitive)?.contentOrNull != "WEB_REMIX") return responseBody
val response = Json.parseToJsonElement(responseBody.decodeToString()) as? JsonObject ?: return responseBody
val streamingData = response["streamingData"] as? JsonObject ?: return responseBody
val premiumItags = setOf(ITAG.AUDIO_AAC_HIGH, ITAG.AUDIO_OPUS_HIGH)
var removed = 0
val filtered = streamingData.toMutableMap()
for (key in listOf("adaptiveFormats", "formats")) {
val formats = streamingData[key] as? JsonArray ?: continue
val premium = formats.filter { format ->
val itag = ((format as? JsonObject)?.get("itag") as? JsonPrimitive)?.intOrNull
itag != null && itag in premiumItags
}
removed += formats.size - premium.size
filtered[key] = JsonArray(premium)
}
if (removed == 0) return responseBody
val videoId = (request["videoId"] as? JsonPrimitive)?.contentOrNull
Logger.i("Extractor", "WEB_REMIX supplement for $videoId: removed $removed ordinary formats; keeping Premium audio")
JsonObject(response + ("streamingData" to JsonObject(filtered))).toString().encodeToByteArray()
} catch (e: IllegalArgumentException) {
Logger.w("Extractor", "Could not filter WEB_REMIX supplement: ${e::class.simpleName}")
responseBody
}
}
Loading