@@ -503,10 +503,12 @@ def web_crawl_md(
503503 self ,
504504 * ,
505505 url : str ,
506+ exclude_selectors : SequenceNotStr [str ] | Omit = omit ,
506507 follow_subdomains : bool | Omit = omit ,
507508 include_frames : bool | Omit = omit ,
508509 include_images : bool | Omit = omit ,
509510 include_links : bool | Omit = omit ,
511+ include_selectors : SequenceNotStr [str ] | Omit = omit ,
510512 max_age_ms : int | Omit = omit ,
511513 max_depth : int | Omit = omit ,
512514 max_pages : int | Omit = omit ,
@@ -531,6 +533,10 @@ def web_crawl_md(
531533 Args:
532534 url: The starting URL for the crawl (must include http:// or https:// protocol)
533535
536+ exclude_selectors: CSS selectors to remove before each crawled page is converted to Markdown.
537+ Applied after includeSelectors. Exclusion takes precedence: an element matching
538+ both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]".
539+
534540 follow_subdomains: When true, follow links on subdomains of the starting URL's domain (e.g.
535541 docs.example.com when starting from example.com). www and apex are always
536542 treated as equivalent.
@@ -542,6 +548,11 @@ def web_crawl_md(
542548
543549 include_links: Preserve hyperlinks in the Markdown output
544550
551+ include_selectors: CSS selectors. When provided, only matching HTML subtrees (and their
552+ descendants) are kept before each crawled page is converted to Markdown. When
553+ omitted, the entire document is kept. Examples: "article.main", "#content",
554+ "[role=main]".
555+
545556 max_age_ms: Return a cached result if a prior scrape for the same parameters exists and is
546557 younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
547558 omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
@@ -585,10 +596,12 @@ def web_crawl_md(
585596 body = maybe_transform (
586597 {
587598 "url" : url ,
599+ "exclude_selectors" : exclude_selectors ,
588600 "follow_subdomains" : follow_subdomains ,
589601 "include_frames" : include_frames ,
590602 "include_images" : include_images ,
591603 "include_links" : include_links ,
604+ "include_selectors" : include_selectors ,
592605 "max_age_ms" : max_age_ms ,
593606 "max_depth" : max_depth ,
594607 "max_pages" : max_pages ,
@@ -612,8 +625,10 @@ def web_scrape_html(
612625 self ,
613626 * ,
614627 url : str ,
628+ exclude_selectors : SequenceNotStr [str ] | Omit = omit ,
615629 headers : Dict [str , str ] | Omit = omit ,
616630 include_frames : bool | Omit = omit ,
631+ include_selectors : SequenceNotStr [str ] | Omit = omit ,
617632 max_age_ms : int | Omit = omit ,
618633 pdf : web_web_scrape_html_params .Pdf | Omit = omit ,
619634 timeout_ms : int | Omit = omit ,
@@ -631,12 +646,20 @@ def web_scrape_html(
631646 Args:
632647 url: Full URL to scrape (must include http:// or https:// protocol)
633648
649+ exclude_selectors: CSS selectors to remove from the result. Applied after includeSelectors.
650+ Exclusion takes precedence: an element matching both is removed. Examples:
651+ "nav", "footer", ".ad-banner", "[aria-hidden=true]".
652+
634653 headers: Optional outbound HTTP headers forwarded only to the target URL, sent as
635654 deep-object query params such as headers[X-Custom]=value. When provided, caching
636655 is bypassed: the result is neither read from nor written to cache.
637656
638657 include_frames: When true, iframes are rendered inline into the returned HTML.
639658
659+ include_selectors: CSS selectors. When provided, only matching subtrees (and their descendants) are
660+ kept and everything else is dropped. When omitted, the entire document is kept.
661+ Examples: "article.main", "#content", "[role=main]".
662+
640663 max_age_ms: Return a cached result if a prior scrape for the same parameters exists and is
641664 younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
642665 omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
@@ -670,8 +693,10 @@ def web_scrape_html(
670693 query = maybe_transform (
671694 {
672695 "url" : url ,
696+ "exclude_selectors" : exclude_selectors ,
673697 "headers" : headers ,
674698 "include_frames" : include_frames ,
699+ "include_selectors" : include_selectors ,
675700 "max_age_ms" : max_age_ms ,
676701 "pdf" : pdf ,
677702 "timeout_ms" : timeout_ms ,
@@ -759,10 +784,12 @@ def web_scrape_md(
759784 self ,
760785 * ,
761786 url : str ,
787+ exclude_selectors : SequenceNotStr [str ] | Omit = omit ,
762788 headers : Dict [str , str ] | Omit = omit ,
763789 include_frames : bool | Omit = omit ,
764790 include_images : bool | Omit = omit ,
765791 include_links : bool | Omit = omit ,
792+ include_selectors : SequenceNotStr [str ] | Omit = omit ,
766793 max_age_ms : int | Omit = omit ,
767794 pdf : web_web_scrape_md_params .Pdf | Omit = omit ,
768795 shorten_base64_images : bool | Omit = omit ,
@@ -783,6 +810,10 @@ def web_scrape_md(
783810 url: Full URL to scrape into LLM usable Markdown (must include http:// or https://
784811 protocol)
785812
813+ exclude_selectors: CSS selectors to remove before conversion to Markdown. Applied after
814+ includeSelectors. Exclusion takes precedence: an element matching both is
815+ removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]".
816+
786817 headers: Optional outbound HTTP headers forwarded only to the target URL, sent as
787818 deep-object query params such as headers[X-Custom]=value. When provided, caching
788819 is bypassed: the result is neither read from nor written to cache.
@@ -793,6 +824,10 @@ def web_scrape_md(
793824
794825 include_links: Preserve hyperlinks in Markdown output
795826
827+ include_selectors: CSS selectors. When provided, only matching HTML subtrees (and their
828+ descendants) are kept before conversion to Markdown. When omitted, the entire
829+ document is kept. Examples: "article.main", "#content", "[role=main]".
830+
796831 max_age_ms: Return a cached result if a prior scrape for the same parameters exists and is
797832 younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
798833 omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
@@ -830,10 +865,12 @@ def web_scrape_md(
830865 query = maybe_transform (
831866 {
832867 "url" : url ,
868+ "exclude_selectors" : exclude_selectors ,
833869 "headers" : headers ,
834870 "include_frames" : include_frames ,
835871 "include_images" : include_images ,
836872 "include_links" : include_links ,
873+ "include_selectors" : include_selectors ,
837874 "max_age_ms" : max_age_ms ,
838875 "pdf" : pdf ,
839876 "shorten_base64_images" : shorten_base64_images ,
@@ -1369,10 +1406,12 @@ async def web_crawl_md(
13691406 self ,
13701407 * ,
13711408 url : str ,
1409+ exclude_selectors : SequenceNotStr [str ] | Omit = omit ,
13721410 follow_subdomains : bool | Omit = omit ,
13731411 include_frames : bool | Omit = omit ,
13741412 include_images : bool | Omit = omit ,
13751413 include_links : bool | Omit = omit ,
1414+ include_selectors : SequenceNotStr [str ] | Omit = omit ,
13761415 max_age_ms : int | Omit = omit ,
13771416 max_depth : int | Omit = omit ,
13781417 max_pages : int | Omit = omit ,
@@ -1397,6 +1436,10 @@ async def web_crawl_md(
13971436 Args:
13981437 url: The starting URL for the crawl (must include http:// or https:// protocol)
13991438
1439+ exclude_selectors: CSS selectors to remove before each crawled page is converted to Markdown.
1440+ Applied after includeSelectors. Exclusion takes precedence: an element matching
1441+ both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]".
1442+
14001443 follow_subdomains: When true, follow links on subdomains of the starting URL's domain (e.g.
14011444 docs.example.com when starting from example.com). www and apex are always
14021445 treated as equivalent.
@@ -1408,6 +1451,11 @@ async def web_crawl_md(
14081451
14091452 include_links: Preserve hyperlinks in the Markdown output
14101453
1454+ include_selectors: CSS selectors. When provided, only matching HTML subtrees (and their
1455+ descendants) are kept before each crawled page is converted to Markdown. When
1456+ omitted, the entire document is kept. Examples: "article.main", "#content",
1457+ "[role=main]".
1458+
14111459 max_age_ms: Return a cached result if a prior scrape for the same parameters exists and is
14121460 younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
14131461 omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
@@ -1451,10 +1499,12 @@ async def web_crawl_md(
14511499 body = await async_maybe_transform (
14521500 {
14531501 "url" : url ,
1502+ "exclude_selectors" : exclude_selectors ,
14541503 "follow_subdomains" : follow_subdomains ,
14551504 "include_frames" : include_frames ,
14561505 "include_images" : include_images ,
14571506 "include_links" : include_links ,
1507+ "include_selectors" : include_selectors ,
14581508 "max_age_ms" : max_age_ms ,
14591509 "max_depth" : max_depth ,
14601510 "max_pages" : max_pages ,
@@ -1478,8 +1528,10 @@ async def web_scrape_html(
14781528 self ,
14791529 * ,
14801530 url : str ,
1531+ exclude_selectors : SequenceNotStr [str ] | Omit = omit ,
14811532 headers : Dict [str , str ] | Omit = omit ,
14821533 include_frames : bool | Omit = omit ,
1534+ include_selectors : SequenceNotStr [str ] | Omit = omit ,
14831535 max_age_ms : int | Omit = omit ,
14841536 pdf : web_web_scrape_html_params .Pdf | Omit = omit ,
14851537 timeout_ms : int | Omit = omit ,
@@ -1497,12 +1549,20 @@ async def web_scrape_html(
14971549 Args:
14981550 url: Full URL to scrape (must include http:// or https:// protocol)
14991551
1552+ exclude_selectors: CSS selectors to remove from the result. Applied after includeSelectors.
1553+ Exclusion takes precedence: an element matching both is removed. Examples:
1554+ "nav", "footer", ".ad-banner", "[aria-hidden=true]".
1555+
15001556 headers: Optional outbound HTTP headers forwarded only to the target URL, sent as
15011557 deep-object query params such as headers[X-Custom]=value. When provided, caching
15021558 is bypassed: the result is neither read from nor written to cache.
15031559
15041560 include_frames: When true, iframes are rendered inline into the returned HTML.
15051561
1562+ include_selectors: CSS selectors. When provided, only matching subtrees (and their descendants) are
1563+ kept and everything else is dropped. When omitted, the entire document is kept.
1564+ Examples: "article.main", "#content", "[role=main]".
1565+
15061566 max_age_ms: Return a cached result if a prior scrape for the same parameters exists and is
15071567 younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
15081568 omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
@@ -1536,8 +1596,10 @@ async def web_scrape_html(
15361596 query = await async_maybe_transform (
15371597 {
15381598 "url" : url ,
1599+ "exclude_selectors" : exclude_selectors ,
15391600 "headers" : headers ,
15401601 "include_frames" : include_frames ,
1602+ "include_selectors" : include_selectors ,
15411603 "max_age_ms" : max_age_ms ,
15421604 "pdf" : pdf ,
15431605 "timeout_ms" : timeout_ms ,
@@ -1625,10 +1687,12 @@ async def web_scrape_md(
16251687 self ,
16261688 * ,
16271689 url : str ,
1690+ exclude_selectors : SequenceNotStr [str ] | Omit = omit ,
16281691 headers : Dict [str , str ] | Omit = omit ,
16291692 include_frames : bool | Omit = omit ,
16301693 include_images : bool | Omit = omit ,
16311694 include_links : bool | Omit = omit ,
1695+ include_selectors : SequenceNotStr [str ] | Omit = omit ,
16321696 max_age_ms : int | Omit = omit ,
16331697 pdf : web_web_scrape_md_params .Pdf | Omit = omit ,
16341698 shorten_base64_images : bool | Omit = omit ,
@@ -1649,6 +1713,10 @@ async def web_scrape_md(
16491713 url: Full URL to scrape into LLM usable Markdown (must include http:// or https://
16501714 protocol)
16511715
1716+ exclude_selectors: CSS selectors to remove before conversion to Markdown. Applied after
1717+ includeSelectors. Exclusion takes precedence: an element matching both is
1718+ removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]".
1719+
16521720 headers: Optional outbound HTTP headers forwarded only to the target URL, sent as
16531721 deep-object query params such as headers[X-Custom]=value. When provided, caching
16541722 is bypassed: the result is neither read from nor written to cache.
@@ -1659,6 +1727,10 @@ async def web_scrape_md(
16591727
16601728 include_links: Preserve hyperlinks in Markdown output
16611729
1730+ include_selectors: CSS selectors. When provided, only matching HTML subtrees (and their
1731+ descendants) are kept before conversion to Markdown. When omitted, the entire
1732+ document is kept. Examples: "article.main", "#content", "[role=main]".
1733+
16621734 max_age_ms: Return a cached result if a prior scrape for the same parameters exists and is
16631735 younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
16641736 omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
@@ -1696,10 +1768,12 @@ async def web_scrape_md(
16961768 query = await async_maybe_transform (
16971769 {
16981770 "url" : url ,
1771+ "exclude_selectors" : exclude_selectors ,
16991772 "headers" : headers ,
17001773 "include_frames" : include_frames ,
17011774 "include_images" : include_images ,
17021775 "include_links" : include_links ,
1776+ "include_selectors" : include_selectors ,
17031777 "max_age_ms" : max_age_ms ,
17041778 "pdf" : pdf ,
17051779 "shorten_base64_images" : shorten_base64_images ,
0 commit comments