Skip to content

feat(youtube): support channel search extraction #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Priveetee wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(youtube): support channel search extraction #65

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ public final class ChannelTabs {
public static final String PLAYLISTS = "playlists";
public static final String PODCASTS = "podcasts";
public static final String ALBUMS = "albums";
public static final String SEARCH = "search";

private ChannelTabs() {
}
Expand Down
25 changes: 19 additions & 6 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2186,20 +2186,33 @@ public ChannelResponseData(final JsonObject responseJson, final String channelId
}

public static ChannelResponseData getChannelResponse(final String channelId,
final String params,
final Localization loc,
final ContentCountry country)
final String params,
final Localization loc,
final ContentCountry country)
throws ExtractionException, IOException {
return getChannelResponse(channelId, params, null, loc, country);
}

public static ChannelResponseData getChannelResponse(final String channelId,
final String params,
@Nullable final String query,
final Localization loc,
final ContentCountry country)
throws ExtractionException, IOException {
String id = channelId;
JsonObject ajaxJson = null;

int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
final JsonBuilder<JsonObject> bodyBuilder = prepareDesktopJsonBuilder(
loc, country)
.value("browseId", id)
.value("params", params) // Equal to videos
.done())
.value("params", params); // Equal to videos
if (!isNullOrEmpty(query)) {
bodyBuilder.value("query", query);
}

final byte[] body = JsonWriter.string(bodyBuilder.done())
.getBytes(UTF_8);

final JsonObject jsonResponse = getJsonPostResponse("browse", body, loc);
Expand Down
53 changes: 43 additions & 10 deletions .../org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ private String getChannelTabsParameters() throws ParsingException {
return "EglwbGF5bGlzdHPyBgQKAkIA";
case ChannelTabs.PODCASTS:
return "Eghwb2RjYXN0c_IGBQoDugEA";
case ChannelTabs.SEARCH:
return "EgZzZWFyY2jyBgQKAloA";
default:
throw new ParsingException("Unsupported channel tab: " + name);
}
Expand All @@ -86,7 +88,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
final String params = getChannelTabsParameters();

final ChannelResponseData data = getChannelResponse(channelIdFromId,
params, getExtractorLocalization(), getExtractorContentCountry());
params, getSearchQuery(), getExtractorLocalization(), getExtractorContentCountry());

jsonResponse = data.responseJson;
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
Expand All @@ -100,8 +102,11 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
@Override
public String getUrl() throws ParsingException {
try {
return YoutubeChannelTabLinkHandlerFactory.getInstance().getUrl("channel/" + getId(),
final String url = YoutubeChannelTabLinkHandlerFactory.getInstance().getUrl(
"channel/" + getId(),
Collections.singletonList(new FilterItem(-1, getTab())), null);
return YoutubeChannelTabLinkHandlerFactory.appendSearchQueryIfNeeded(
url, getSearchQuery());
} catch (final ParsingException e) {
return super.getUrl();
}
Expand Down Expand Up @@ -199,14 +204,17 @@ private JsonObject getTabData() throws ParsingException {

JsonObject foundTab = null;
for (final Object tab : tabs) {
if (((JsonObject) tab).has("tabRenderer")) {
final String tabUrl = ((JsonObject) tab).getObject("tabRenderer").getObject("endpoint")
.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("url");
if (tabUrl != null && normalizeTabUrl(tabUrl).endsWith(urlSuffix)) {
foundTab = ((JsonObject) tab).getObject("tabRenderer");
break;
}
final JsonObject tabRenderer = getTabRenderer((JsonObject) tab);
if (tabRenderer == null) {
continue;
}

final String tabUrl = tabRenderer.getObject("endpoint")
.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("url");
if (tabUrl != null && normalizeTabUrl(tabUrl).endsWith(urlSuffix)) {
foundTab = tabRenderer;
break;
}
}

Expand Down Expand Up @@ -296,6 +304,8 @@ public String getUploaderUrl() {

if (item.has("gridVideoRenderer")) {
commitVideo.accept(item.getObject("gridVideoRenderer"));
} else if (item.has("videoRenderer")) {
commitVideo.accept(item.getObject("videoRenderer"));
} else if (item.has("richItemRenderer")) {
final JsonObject richItem = item.getObject("richItemRenderer").getObject("content");

Expand Down Expand Up @@ -336,6 +346,9 @@ public String getUploaderName() {
} else if (item.has("gridChannelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(
item.getObject("gridChannelRenderer")));
} else if (item.has("channelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(
item.getObject("channelRenderer")));
} else if (item.has("shelfRenderer")) {
return collectItem(collector, item.getObject("shelfRenderer")
.getObject("content"), channelIds);
Expand All @@ -357,6 +370,26 @@ public String getUploaderName() {
return null;
}

@Nullable
private String getSearchQuery() throws ParsingException {
if (!ChannelTabs.SEARCH.equals(getTab())) {
return null;
}

return YoutubeChannelTabLinkHandlerFactory.getSearchQueryFromUrl(getOriginalUrl());
}

@Nullable
private static JsonObject getTabRenderer(@Nonnull final JsonObject tab) {
if (tab.has("tabRenderer")) {
return tab.getObject("tabRenderer");
}
if (tab.has("expandableTabRenderer")) {
return tab.getObject("expandableTabRenderer");
}
return null;
}

private void commitLockupItemIfSupported(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject lockupViewModel,
@Nonnull final List<String> channelIds) {
Expand Down
101 changes: 97 additions & 4 deletions ...i/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,25 @@

import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.search.filter.Filter;
import org.schabi.newpipe.extractor.search.filter.FilterItem;
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
import org.schabi.newpipe.extractor.utils.Utils;

import java.net.URL;
import java.net.URLEncoder;
import java.util.Collections;
import java.util.List;

import javax.annotation.Nonnull;

import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;

public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeChannelTabLinkHandlerFactory INSTANCE =
new YoutubeChannelTabLinkHandlerFactory();
private final YoutubeFilters searchFilters = new YoutubeFilters();

private YoutubeChannelTabLinkHandlerFactory() {
}
Expand All @@ -36,15 +43,38 @@ public static String getUrlSuffix(final String tab) throws ParsingException {
return "/shorts";
case ChannelTabs.CHANNELS:
return "/channels";
case ChannelTabs.SEARCH:
return "/search";
}
throw new ParsingException("tab " + tab + " not supported");
}

@Override
public String getUrl(final String id,@Nonnull final List<FilterItem> selectedContentFilter,
public String getUrl(final String id, @Nonnull final List<FilterItem> selectedContentFilter,
final List<FilterItem> selectedSortFilter)
throws ParsingException {
return "https://www.youtube.com/" + id + getUrlSuffix(selectedContentFilter.get(0).getName());
return "https://www.youtube.com/" + id
+ getUrlSuffix(selectedContentFilter.get(0).getName());
}

@Override
public ListLinkHandler fromUrl(final String url, final String baseUrl) throws ParsingException {
if (url == null) {
throw new IllegalArgumentException("url may not be null");
}
if (!acceptUrl(url)) {
throw new ParsingException("URL not accepted: " + url);
}

final String id = getId(url);
final String tab = getTabFromUrl(url);
final List<FilterItem> contentFilter = Collections.singletonList(
new FilterItem(Filter.ITEM_IDENTIFIER_UNKNOWN, tab));
String cleanUrl = getUrl(id, contentFilter, null, baseUrl);
if (ChannelTabs.SEARCH.equals(tab)) {
cleanUrl = appendSearchQueryIfNeeded(cleanUrl, getSearchQueryFromUrl(url));
}
return new ListLinkHandler(url, cleanUrl, id, contentFilter, null);
}

@Override
Expand All @@ -62,5 +92,68 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
return true;
}

public static String getSearchQueryFromUrl(final String url) throws ParsingException {
try {
final URL urlObj = Utils.stringToURL(url);
final String query = firstNonEmptyQueryValue(urlObj, "query", "search_query", "q");
return query == null ? "" : query;
} catch (final Exception e) {
throw new ParsingException("Could not parse channel search query", e);
}
}

public static String appendSearchQueryIfNeeded(final String url,
final String query) throws ParsingException {
if (isNullOrEmpty(query)) {
return url;
}

try {
return url + "?query=" + URLEncoder.encode(query, UTF_8).replace("+", "%20");
} catch (final Exception e) {
throw new ParsingException("Could not encode channel search query", e);
}
}

private String getTabFromUrl(final String url) throws ParsingException {
try {
final URL urlObj = Utils.stringToURL(url);
final String[] pathSegments = urlObj.getPath().split("/");
for (int i = pathSegments.length - 1; i >= 0; i--) {
switch (pathSegments[i]) {
case "videos":
return ChannelTabs.VIDEOS;
case "playlists":
return ChannelTabs.PLAYLISTS;
case "podcasts":
return ChannelTabs.PODCASTS;
case "streams":
return ChannelTabs.LIVESTREAMS;
case "shorts":
return ChannelTabs.SHORTS;
case "channels":
return ChannelTabs.CHANNELS;
case "search":
return ChannelTabs.SEARCH;
}
}
} catch (final Exception e) {
throw new ParsingException("Could not parse channel tab URL: " + e.getMessage(), e);
}

return ChannelTabs.VIDEOS;
}

private static String firstNonEmptyQueryValue(final URL url,
final String... names) {
for (final String name : names) {
final String value = Utils.getQueryValue(url, name);
if (!isNullOrEmpty(value)) {
return value;
}
}
return null;
}


}