diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java index 9cb5bc86a..a8af9b3da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java @@ -9,6 +9,7 @@ public final class ChannelTabs { public static final String PLAYLISTS = "playlists"; public static final String PODCASTS = "podcasts"; public static final String ALBUMS = "albums"; + public static final String SEARCH = "search"; private ChannelTabs() { } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 102268f4b..316f65bb2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -2186,20 +2186,33 @@ public ChannelResponseData(final JsonObject responseJson, final String channelId } public static ChannelResponseData getChannelResponse(final String channelId, - final String params, - final Localization loc, - final ContentCountry country) + final String params, + final Localization loc, + final ContentCountry country) + throws ExtractionException, IOException { + return getChannelResponse(channelId, params, null, loc, country); + } + + public static ChannelResponseData getChannelResponse(final String channelId, + final String params, + @Nullable final String query, + final Localization loc, + final ContentCountry country) throws ExtractionException, IOException { String id = channelId; JsonObject ajaxJson = null; int level = 0; while (level < 3) { - final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( + final JsonBuilder bodyBuilder = prepareDesktopJsonBuilder( loc, country) .value("browseId", id) - .value("params", params) // Equal to videos - .done()) + .value("params", params); // Equal to videos + if (!isNullOrEmpty(query)) { + bodyBuilder.value("query", query); + } + + final byte[] body = JsonWriter.string(bodyBuilder.done()) .getBytes(UTF_8); final JsonObject jsonResponse = getJsonPostResponse("browse", body, loc); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index 8551eb028..fbe66be13 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -72,6 +72,8 @@ private String getChannelTabsParameters() throws ParsingException { return "EglwbGF5bGlzdHPyBgQKAkIA"; case ChannelTabs.PODCASTS: return "Eghwb2RjYXN0c_IGBQoDugEA"; + case ChannelTabs.SEARCH: + return "EgZzZWFyY2jyBgQKAloA"; default: throw new ParsingException("Unsupported channel tab: " + name); } @@ -86,7 +88,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException final String params = getChannelTabsParameters(); final ChannelResponseData data = getChannelResponse(channelIdFromId, - params, getExtractorLocalization(), getExtractorContentCountry()); + params, getSearchQuery(), getExtractorLocalization(), getExtractorContentCountry()); jsonResponse = data.responseJson; channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse); @@ -100,8 +102,11 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException @Override public String getUrl() throws ParsingException { try { - return YoutubeChannelTabLinkHandlerFactory.getInstance().getUrl("channel/" + getId(), + final String url = YoutubeChannelTabLinkHandlerFactory.getInstance().getUrl( + "channel/" + getId(), Collections.singletonList(new FilterItem(-1, getTab())), null); + return YoutubeChannelTabLinkHandlerFactory.appendSearchQueryIfNeeded( + url, getSearchQuery()); } catch (final ParsingException e) { return super.getUrl(); } @@ -199,14 +204,17 @@ private JsonObject getTabData() throws ParsingException { JsonObject foundTab = null; for (final Object tab : tabs) { - if (((JsonObject) tab).has("tabRenderer")) { - final String tabUrl = ((JsonObject) tab).getObject("tabRenderer").getObject("endpoint") - .getObject("commandMetadata").getObject("webCommandMetadata") - .getString("url"); - if (tabUrl != null && normalizeTabUrl(tabUrl).endsWith(urlSuffix)) { - foundTab = ((JsonObject) tab).getObject("tabRenderer"); - break; - } + final JsonObject tabRenderer = getTabRenderer((JsonObject) tab); + if (tabRenderer == null) { + continue; + } + + final String tabUrl = tabRenderer.getObject("endpoint") + .getObject("commandMetadata").getObject("webCommandMetadata") + .getString("url"); + if (tabUrl != null && normalizeTabUrl(tabUrl).endsWith(urlSuffix)) { + foundTab = tabRenderer; + break; } } @@ -296,6 +304,8 @@ public String getUploaderUrl() { if (item.has("gridVideoRenderer")) { commitVideo.accept(item.getObject("gridVideoRenderer")); + } else if (item.has("videoRenderer")) { + commitVideo.accept(item.getObject("videoRenderer")); } else if (item.has("richItemRenderer")) { final JsonObject richItem = item.getObject("richItemRenderer").getObject("content"); @@ -336,6 +346,9 @@ public String getUploaderName() { } else if (item.has("gridChannelRenderer")) { collector.commit(new YoutubeChannelInfoItemExtractor( item.getObject("gridChannelRenderer"))); + } else if (item.has("channelRenderer")) { + collector.commit(new YoutubeChannelInfoItemExtractor( + item.getObject("channelRenderer"))); } else if (item.has("shelfRenderer")) { return collectItem(collector, item.getObject("shelfRenderer") .getObject("content"), channelIds); @@ -357,6 +370,26 @@ public String getUploaderName() { return null; } + @Nullable + private String getSearchQuery() throws ParsingException { + if (!ChannelTabs.SEARCH.equals(getTab())) { + return null; + } + + return YoutubeChannelTabLinkHandlerFactory.getSearchQueryFromUrl(getOriginalUrl()); + } + + @Nullable + private static JsonObject getTabRenderer(@Nonnull final JsonObject tab) { + if (tab.has("tabRenderer")) { + return tab.getObject("tabRenderer"); + } + if (tab.has("expandableTabRenderer")) { + return tab.getObject("expandableTabRenderer"); + } + return null; + } + private void commitLockupItemIfSupported(@Nonnull final MultiInfoItemsCollector collector, @Nonnull final JsonObject lockupViewModel, @Nonnull final List channelIds) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java index ac9ee16a5..7b5198c72 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java @@ -2,18 +2,25 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.search.filter.Filter; import org.schabi.newpipe.extractor.search.filter.FilterItem; -import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters; +import org.schabi.newpipe.extractor.utils.Utils; +import java.net.URL; +import java.net.URLEncoder; +import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory { private static final YoutubeChannelTabLinkHandlerFactory INSTANCE = new YoutubeChannelTabLinkHandlerFactory(); - private final YoutubeFilters searchFilters = new YoutubeFilters(); private YoutubeChannelTabLinkHandlerFactory() { } @@ -36,15 +43,38 @@ public static String getUrlSuffix(final String tab) throws ParsingException { return "/shorts"; case ChannelTabs.CHANNELS: return "/channels"; + case ChannelTabs.SEARCH: + return "/search"; } throw new ParsingException("tab " + tab + " not supported"); } @Override - public String getUrl(final String id,@Nonnull final List selectedContentFilter, + public String getUrl(final String id, @Nonnull final List selectedContentFilter, final List selectedSortFilter) throws ParsingException { - return "https://www.youtube.com/" + id + getUrlSuffix(selectedContentFilter.get(0).getName()); + return "https://www.youtube.com/" + id + + getUrlSuffix(selectedContentFilter.get(0).getName()); + } + + @Override + public ListLinkHandler fromUrl(final String url, final String baseUrl) throws ParsingException { + if (url == null) { + throw new IllegalArgumentException("url may not be null"); + } + if (!acceptUrl(url)) { + throw new ParsingException("URL not accepted: " + url); + } + + final String id = getId(url); + final String tab = getTabFromUrl(url); + final List contentFilter = Collections.singletonList( + new FilterItem(Filter.ITEM_IDENTIFIER_UNKNOWN, tab)); + String cleanUrl = getUrl(id, contentFilter, null, baseUrl); + if (ChannelTabs.SEARCH.equals(tab)) { + cleanUrl = appendSearchQueryIfNeeded(cleanUrl, getSearchQueryFromUrl(url)); + } + return new ListLinkHandler(url, cleanUrl, id, contentFilter, null); } @Override @@ -62,5 +92,68 @@ public boolean onAcceptUrl(final String url) throws ParsingException { return true; } + public static String getSearchQueryFromUrl(final String url) throws ParsingException { + try { + final URL urlObj = Utils.stringToURL(url); + final String query = firstNonEmptyQueryValue(urlObj, "query", "search_query", "q"); + return query == null ? "" : query; + } catch (final Exception e) { + throw new ParsingException("Could not parse channel search query", e); + } + } + + public static String appendSearchQueryIfNeeded(final String url, + final String query) throws ParsingException { + if (isNullOrEmpty(query)) { + return url; + } + + try { + return url + "?query=" + URLEncoder.encode(query, UTF_8).replace("+", "%20"); + } catch (final Exception e) { + throw new ParsingException("Could not encode channel search query", e); + } + } + + private String getTabFromUrl(final String url) throws ParsingException { + try { + final URL urlObj = Utils.stringToURL(url); + final String[] pathSegments = urlObj.getPath().split("/"); + for (int i = pathSegments.length - 1; i >= 0; i--) { + switch (pathSegments[i]) { + case "videos": + return ChannelTabs.VIDEOS; + case "playlists": + return ChannelTabs.PLAYLISTS; + case "podcasts": + return ChannelTabs.PODCASTS; + case "streams": + return ChannelTabs.LIVESTREAMS; + case "shorts": + return ChannelTabs.SHORTS; + case "channels": + return ChannelTabs.CHANNELS; + case "search": + return ChannelTabs.SEARCH; + } + } + } catch (final Exception e) { + throw new ParsingException("Could not parse channel tab URL: " + e.getMessage(), e); + } + + return ChannelTabs.VIDEOS; + } + + private static String firstNonEmptyQueryValue(final URL url, + final String... names) { + for (final String name : names) { + final String value = Utils.getQueryValue(url, name); + if (!isNullOrEmpty(value)) { + return value; + } + } + return null; + } + }