diff --git a/src/skillspector/javascript_tokens.py b/src/skillspector/javascript_tokens.py
index 8e0eced04..adec39b95 100644
--- a/src/skillspector/javascript_tokens.py
+++ b/src/skillspector/javascript_tokens.py
@@ -14,6 +14,7 @@
from bisect import bisect_left
from collections.abc import Callable, Iterator
from dataclasses import dataclass
+from html.parser import HTMLParser
from typing import Any
import esprima # type: ignore[import-untyped]
@@ -24,6 +25,7 @@
_MAX_TOKENS = 100_000
_MAX_NESTING = 128
_LINE_ENDS = "\r\n\u2028\u2029"
+_BrowserBinding = tuple[str, int, str]
_IDENTIFIER = re.compile(r"[A-Za-z_$][A-Za-z0-9_$]*")
_TEMPLATE_VALUE = re.compile(
r"\s*[A-Za-z_$][A-Za-z0-9_$]*(?:\s*\.\s*[A-Za-z_$][A-Za-z0-9_$]*)*\s*\Z"
@@ -387,6 +389,1089 @@ def assertion_template_shell_text(content: str, check_runtime: Callable[[], None
return "".join(output)
+def _nested_template_code(
+ content: str, check_runtime: Callable[[], None]
+) -> tuple[str, list[tuple[int, int, tuple[int, ...]]], set[int]] | None:
+ """Expose template expressions to the existing lexer, retaining exact offsets."""
+ if len(content) > _MAX_SOURCE_CHARS:
+ return None
+ output = list(content)
+ templates: list[tuple[int, int, tuple[int, ...]]] = []
+ expression_openers: set[int] = set()
+ next_check = 0
+
+ def check(position: int) -> None:
+ nonlocal next_check
+ if position >= next_check:
+ check_runtime()
+ next_check = position + 4096
+
+ def code(cursor: int, depth: int, *, interpolation: bool = False) -> int | None:
+ if depth >= _MAX_NESTING:
+ return None
+ braces = 0
+ while cursor < len(content):
+ check(cursor)
+ character = content[cursor]
+ if character in "'\"":
+ quote = character
+ cursor += 1
+ while cursor < len(content) and content[cursor] != quote:
+ check(cursor)
+ if content[cursor] in _LINE_ENDS:
+ return None
+ cursor += 2 if content[cursor] == "\\" else 1
+ if cursor >= len(content):
+ return None
+ elif content.startswith("//", cursor):
+ while cursor < len(content) and content[cursor] not in _LINE_ENDS:
+ check(cursor)
+ cursor += 1
+ continue
+ elif content.startswith("/*", cursor):
+ cursor += 2
+ while cursor < len(content) and not content.startswith("*/", cursor):
+ check(cursor)
+ cursor += 1
+ if cursor == len(content):
+ return None
+ cursor += 2
+ continue
+ elif character == "/":
+ # Regexp/division ambiguity cannot establish nested ownership.
+ return None
+ elif character == "`":
+ end = template(cursor, depth + 1)
+ if end is None:
+ return None
+ cursor = end
+ continue
+ elif character == "{":
+ braces += 1
+ if braces + depth >= _MAX_NESTING:
+ return None
+ elif character == "}":
+ if not braces and interpolation:
+ return cursor
+ braces -= 1
+ if braces < 0:
+ return None
+ cursor += 1
+ return None if interpolation or braces else cursor
+
+ def template(start: int, depth: int) -> int | None:
+ if depth >= _MAX_NESTING or len(templates) >= _MAX_TOKENS:
+ return None
+ delimiters = [start]
+ expression_openers.add(start)
+ output[start] = "("
+ cursor = start + 1
+ while cursor < len(content):
+ check(cursor)
+ if content[cursor] == "`":
+ output[cursor] = ")"
+ delimiters.append(cursor)
+ templates.append((start, cursor + 1, tuple(delimiters)))
+ return cursor + 1
+ if content.startswith("${", cursor):
+ end = code(cursor + 2, depth + 1, interpolation=True)
+ if end is None:
+ return None
+ expression_openers.add(cursor)
+ output[cursor], output[cursor + 1], output[end] = "(", " ", ")"
+ delimiters.extend((cursor, cursor + 1, end))
+ cursor = end + 1
+ continue
+ if content[cursor] == "\\":
+ # Decoding an escape needs a separate source/value proof.
+ return None
+ if content[cursor] not in _LINE_ENDS:
+ output[cursor] = " "
+ cursor += 1
+ return None
+
+ if code(0, 0) is None:
+ return None
+ check_runtime()
+ return "".join(output), templates, expression_openers
+
+
+def browser_escaped_list_shell_text(content: str, check_runtime: Callable[[], None]) -> str:
+ """Own only JSON-backed list values passed through a local escaping formatter.
+
+ This proves explicit source consumers, not that a browser program is inert.
+ As with assertion messages, ordinary built-ins are assumed; arbitrary runtime
+ getters/monkeypatching are outside this proof. Explicit aliases, mutation,
+ shadowing, unknown consumers, and executable expressions never gain ownership.
+ Only backticks change. Raw findings and interpolation bytes remain untouched.
+ """
+ if "JSON.parse" not in content or "replaceAll" not in content or "innerHTML" not in content:
+ return content
+ nested = _nested_template_code(content, check_runtime)
+ if nested is None:
+ return content
+ code, templates, expression_openers = nested
+ parsed = _tokens(code, check_runtime)
+ if parsed is None:
+ return content
+ tokens, pairs = parsed
+ values = [code[token.start : token.end] for token in tokens]
+ positions = [token.start for token in tokens]
+ reverse = {end: start for start, end in pairs.items()}
+ template_spans = {
+ bisect_left(positions, start): (bisect_left(positions, end), delimiters)
+ for start, end, delimiters in templates
+ }
+ for start in template_spans:
+ if start and (
+ tokens[start - 1].kind == "identifier"
+ and values[start - 1] != "return"
+ or values[start - 1] in {")", "]"}
+ ):
+ # A tag receives a strings array before its interpolation values.
+ # Projecting it as an ordinary call would shift parameter sources.
+ return content
+ fields: dict[int, int] = {}
+ for start, (_, delimiters) in template_spans.items():
+ for position in delimiters[1:-1:3]:
+ fields[bisect_left(positions, position)] = start
+ budget = _MAX_TOKENS * 4
+
+ def step(amount: int = 1) -> bool:
+ nonlocal budget
+ previous = budget
+ budget -= amount
+ if budget // 256 != previous // 256:
+ check_runtime()
+ return budget > 0
+
+ def parts(start: int, end: int) -> list[tuple[int, int]]:
+ result = []
+ begin = cursor = start
+ while cursor < end and step():
+ if values[cursor] == ",":
+ result.append((begin, cursor))
+ begin = cursor + 1
+ cursor = pairs.get(cursor, cursor) + 1
+ if begin < end:
+ result.append((begin, end))
+ return result
+
+ def arguments(opening: int) -> list[tuple[int, int]]:
+ return parts(opening + 1, pairs[opening])
+
+ enclosing: list[int | None] = []
+ blocks: list[tuple[int, int]] = []
+ stack: list[int] = []
+ block_stack: list[int] = []
+ for index, value in enumerate(values):
+ if not step():
+ return content
+ while stack and pairs[stack[-1]] == index:
+ opening = stack.pop()
+ if values[opening] == "{":
+ block_stack.pop()
+ enclosing.append(stack[-1] if stack else None)
+ blocks.append(
+ (block_stack[-1], pairs[block_stack[-1]]) if block_stack else (0, len(tokens))
+ )
+ if index in pairs:
+ stack.append(index)
+ if value == "{":
+ block_stack.append(index)
+
+ functions: dict[str, tuple[int, int, int, list[str]]] = {}
+ constants: dict[str, list[tuple[int, int, int, int, int]]] = {}
+ arrows: dict[int, tuple[int, int, int, list[str]]] = {}
+ native_methods = {
+ "map",
+ "filter",
+ "find",
+ "some",
+ "flatMap",
+ "forEach",
+ "join",
+ "replaceAll",
+ "trim",
+ "toLowerCase",
+ "includes",
+ "getElementById",
+ "querySelectorAll",
+ "addEventListener",
+ }
+ protected = {"String", "JSON", "document", "Object"}
+ forbidden = {"constructor", "prototype", "__proto__", "globalThis", "window", "self", "with"}
+ for index, value in enumerate(values):
+ if not step():
+ return content
+ if value in forbidden:
+ return content
+ if value in protected:
+ expected = {"String": ["("], "JSON": ["."], "document": ["."], "Object": ["."]}[value]
+ if values[index + 1 : index + 1 + len(expected)] != expected or (
+ index and values[index - 1] in {"function", ".", "?.", "const", "let", "var"}
+ ):
+ return content
+ if value == "JSON" and values[index + 2 : index + 4] not in (
+ ["parse", "("],
+ ["stringify", "("],
+ ):
+ return content
+ if value == "document" and values[index + 2 : index + 4] != ["getElementById", "("]:
+ return content
+ if value == "Object" and values[index + 2 : index + 4] != ["entries", "("]:
+ return content
+ if value in native_methods and index and values[index - 1] in {".", "?."}:
+ if values[index + 1 : index + 2] != ["("]:
+ return content
+ if values[index + 1 : index + 2] == [":"] and (
+ value in native_methods
+ or tokens[index].kind == "string"
+ and value[1:-1] in native_methods
+ ):
+ return content
+ if value == "]" and values[index + 1 : index + 2] in ([":"], ["="]):
+ return content
+ if value == "function":
+ opening = index + 2
+ closing = pairs.get(opening)
+ body = -1 if closing is None else closing + 1
+ end = pairs.get(body)
+ if (
+ tokens[index].depth
+ or end is None
+ or values[body] != "{"
+ or tokens[index + 1].kind != "identifier"
+ or values[index + 1] in functions
+ or len(functions) >= _MAX_NESTING
+ ):
+ return content
+ params = arguments(opening)
+ names = []
+ for begin, stop in params:
+ if (
+ tokens[begin].kind != "identifier"
+ or values[begin] in protected
+ or values[begin] in names
+ ):
+ return content
+ if stop != begin + 1 and not (
+ stop == begin + 3
+ and values[begin + 1] == "="
+ and tokens[begin + 2].kind == "string"
+ ):
+ return content
+ names.append(values[begin])
+ functions[values[index + 1]] = index + 1, body, end, names
+ if value == "const":
+ if (
+ index + 2 >= len(tokens)
+ or tokens[index + 1].kind != "identifier"
+ or values[index + 2] != "="
+ ):
+ return content
+ stop = index + 3
+ while stop < len(tokens) and values[stop] != ";" and step():
+ stop = pairs.get(stop, stop) + 1
+ if stop == len(tokens):
+ return content
+ constants.setdefault(values[index + 1], []).append(
+ (index + 1, index + 3, stop, *blocks[index])
+ )
+ if value == "=>":
+ if index and values[index - 1] == ")":
+ opening = reverse[index - 1]
+ begin, stop = opening + 1, index - 1
+ call = enclosing[opening]
+ elif index and tokens[index - 1].kind == "identifier":
+ begin, stop = index - 1, index
+ call = enclosing[begin]
+ else:
+ return content
+ if begin < stop and values[begin] in {"{", "["} and pairs.get(begin) == stop - 1:
+ begin, stop = begin + 1, stop - 1
+ names = []
+ for lower, upper in parts(begin, stop):
+ if (
+ upper != lower + 1
+ or tokens[lower].kind != "identifier"
+ or values[lower] in protected
+ or values[lower] in names
+ ):
+ return content
+ names.append(values[lower])
+ body = index + 1
+ end = pairs.get(body, body) + 1
+ if body not in pairs:
+ while end < len(tokens) and values[end] not in {",", ";", ")", "]", "}"} and step():
+ end = pairs.get(end, end) + 1
+ if call is not None:
+ arrows[index] = call, body, end, names
+
+ arrows_by_call: dict[int, list[tuple[int, int, int, list[str]]]] = {}
+ arrows_by_body: dict[tuple[int, int], tuple[int, int, int, list[str]]] = {}
+ arrows_by_name: dict[str, list[tuple[int, int, int, list[str]]]] = {}
+ for row in arrows.values():
+ arrows_by_call.setdefault(row[0], []).append(row)
+ arrows_by_body[row[1:3]] = row
+ for name in row[3]:
+ arrows_by_name.setdefault(name, []).append(row)
+
+ def literal(index: int) -> str | None:
+ value = values[index]
+ return value[1:-1] if tokens[index].kind == "string" and "\\" not in value else None
+
+ # Verify the actual text escaping operation, with no aliases or extra body.
+ escapes: set[str] = set()
+ for name, (_, body, end, params) in functions.items():
+ if len(params) != 1:
+ continue
+ sequence = values[body + 1 : end]
+ prefix = ["return", "String", "(", params[0], "??"]
+ cursor = body + 1
+ if sequence[:5] != prefix or literal(cursor + 5) != "" or values[cursor + 6] != ")":
+ continue
+ cursor += 7
+ for before, after in [("&", "&"), ("<", "<"), (">", ">"), ('"', """)]:
+ if (
+ values[cursor : cursor + 3] != [".", "replaceAll", "("]
+ or literal(cursor + 3) != before
+ or values[cursor + 4 : cursor + 5] != [","]
+ or literal(cursor + 5) != after
+ or values[cursor + 6 : cursor + 7] != [")"]
+ ):
+ break
+ cursor += 7
+ else:
+ if values[cursor:end] == [";"]:
+ escapes.add(name)
+
+ def source(start: int, end: int) -> str:
+ return content[tokens[start].start : tokens[end - 1].end]
+
+ def template_fields(start: int) -> list[tuple[int, int]]:
+ delimiters = template_spans[start][1]
+ return [
+ (bisect_left(positions, p) + 1, pairs[bisect_left(positions, p)])
+ for p in delimiters[1:-1:3]
+ ]
+
+ def text_slot(template: int, field: int, *, inert: bool = False) -> bool:
+ """The selected HTML fragment occurs in ordinary literal markup text."""
+ marker = "SKILLSPECTOR_SELECTED_FRAGMENT"
+ if marker in content:
+ return False
+ _, delimiters = template_spans[template]
+ if not step(1 + (delimiters[-1] - delimiters[0]) // 32):
+ return False
+ chunks = []
+ cursor = delimiters[0] + 1
+ for dollar, _, close in zip(
+ delimiters[1:-1:3], delimiters[2:-1:3], delimiters[3:-1:3], strict=True
+ ):
+ chunks.append(content[cursor:dollar])
+ chunks.append(marker if bisect_left(positions, dollar) == field else "VALUE")
+ cursor = close + 1
+ chunks.append(content[cursor : delimiters[-1]])
+
+ class Slot(HTMLParser):
+ valid = True
+ found = False
+ context = ""
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ if inert and (
+ tag not in {"div", "span", "ul", "li", "p", "strong", "em", "b", "code"}
+ or any(name not in {"class", "id", "title"} for name, _ in attrs)
+ ):
+ self.valid = False
+ if tag in {"script", "style", "iframe", "object", "embed", "textarea", "title"}:
+ self.context = tag
+ if any(value is not None and marker in value for _, value in attrs):
+ self.valid = False
+
+ def handle_endtag(self, tag: str) -> None:
+ if tag == self.context:
+ self.context = ""
+
+ def handle_data(self, data: str) -> None:
+ if marker in data:
+ self.found = True
+ self.valid = self.valid and not self.context
+
+ parser = Slot(convert_charrefs=False)
+ check_runtime()
+ parser.feed("".join(chunks))
+ parser.close()
+ check_runtime()
+ return parser.valid and parser.found
+
+ # The formatter has one escaped empty branch and one map/escape/join branch.
+ formatters: set[str] = set()
+ for name, (_, body, end, params) in functions.items():
+ if len(params) != 2:
+ continue
+ items, empty = params
+ for escape in escapes:
+ if not step(end - body + 1):
+ return content
+ compact = re.sub(r"\s+", "", source(body + 1, end))
+ head = f"if(!{items}||!{items}.length){{return"
+ if not compact.startswith(head):
+ continue
+ returns = [index for index in range(body + 1, end) if values[index] == "return"]
+ if len(returns) != 2:
+ continue
+ first, second = (index + 1 for index in returns)
+ if first not in template_spans or second not in template_spans:
+ continue
+ first_end, _ = template_spans[first]
+ second_end, _ = template_spans[second]
+ if values[first_end:second] != [";", "}", "return"] or values[second_end:end] != [";"]:
+ continue
+ a, b = template_fields(first), template_fields(second)
+ if len(a) != 1 or len(b) != 1:
+ continue
+ if values[a[0][0] : a[0][1]] != [escape, "(", empty, ")"] or not text_slot(
+ first, a[0][0] - 1, inert=True
+ ):
+ continue
+ begin, stop = b[0]
+ if values[begin : begin + 4] != [items, ".", "map", "("]:
+ continue
+ call = begin + 3
+ args = arguments(call)
+ if len(args) != 1:
+ continue
+ matching = arrows_by_call.get(call, [])
+ if len(matching) != 1:
+ continue
+ _, inner, inner_end, names = matching[0]
+ if (
+ len(names) != 1
+ or inner not in template_spans
+ or template_spans[inner][0] != inner_end
+ ):
+ continue
+ item = names[0]
+ field = template_fields(inner)
+ if len(field) != 1 or not text_slot(inner, field[0][0] - 1, inert=True):
+ continue
+ expression = values[field[0][0] : field[0][1]]
+ simple = [escape, "(", item, ")"]
+ fallback = [
+ escape,
+ "(",
+ "typeof",
+ item,
+ "===",
+ "'string'",
+ "?",
+ item,
+ ":",
+ "JSON",
+ ".",
+ "stringify",
+ "(",
+ item,
+ ")",
+ ")",
+ ]
+ if expression != simple and expression != fallback:
+ continue
+ close = pairs[call] + 1
+ if (
+ values[close : close + 3] != [".", "join", "("]
+ or literal(close + 3) != ""
+ or values[close + 4 : stop] != [")"]
+ ):
+ continue
+ if text_slot(second, begin - 1, inert=True):
+ formatters.add(name)
+ if not formatters:
+ return content
+
+ # A closed callable vocabulary rules out explicit escapes of JSON values
+ # into arbitrary code. Native method spelling alone never proves a selected
+ # receiver: ordinary() below separately traces that receiver back to JSON.
+ callbacks: set[int] = set()
+ function_parameter_openers = {declaration + 1 for declaration, _, _, _ in functions.values()}
+ defaults = {
+ a + 1
+ for opening in function_parameter_openers
+ for a, b in arguments(opening)
+ if b == a + 3 and values[a + 1] == "="
+ }
+ declaration_indices = {row[0] for rows in constants.values() for row in rows}
+ mutated: set[str] = set()
+ member_writes: list[int] = []
+ for index, value in enumerate(values):
+ if not step():
+ return content
+ if value == "=" and index - 1 not in declaration_indices and index not in defaults:
+ if index < 3 or values[index - 2] != ".":
+ return content
+ member_writes.append(index)
+ if tokens[index - 3].kind == "identifier":
+ mutated.add(values[index - 3])
+ if (
+ value.endswith("=") and value not in {"=", "==", "===", "!=", "!==", "<=", ">=", "=>"}
+ ) or value in {"++", "--", "delete", "new", "class", "import", "export", "let", "var"}:
+ return content
+ if value != "(" or tokens[index].start in expression_openers:
+ continue
+ closing = pairs[index]
+ if index in function_parameter_openers or values[closing + 1 : closing + 2] == ["=>"]:
+ continue
+ previous = values[index - 1] if index else ""
+ if previous == "if":
+ continue
+ if index and tokens[index - 1].kind == "identifier":
+ if index > 1 and values[index - 2] == ".":
+ receiver = values[index - 3] if index > 2 else ""
+ if not (
+ previous in native_methods
+ or receiver == "JSON"
+ and previous in {"parse", "stringify"}
+ or receiver == "Object"
+ and previous == "entries"
+ ):
+ return content
+ args = arguments(index)
+ if previous == "replaceAll" and (
+ len(args) != 2 or any(b != a + 1 or literal(a) is None for a, b in args)
+ ):
+ return content
+ if previous in {
+ "map",
+ "filter",
+ "find",
+ "some",
+ "flatMap",
+ "forEach",
+ "addEventListener",
+ }:
+ slot = 1 if previous == "addEventListener" else 0
+ if len(args) <= slot:
+ return content
+ a, b = args[slot]
+ if b == a + 1 and values[a] in functions:
+ callbacks.add(a)
+ elif index not in arrows_by_call:
+ return content
+ elif previous not in functions and previous != "String":
+ return content
+ elif previous in {")", "]", "?."}:
+ return content
+ for index, value in enumerate(values):
+ if not step():
+ return content
+ if value in functions and index != functions[value][0]:
+ if index and values[index - 1] in {".", "?."}:
+ return content
+ if values[index + 1 : index + 2] != ["("] and index not in callbacks:
+ return content
+
+ selected_names = escapes | formatters
+ for index, value in enumerate(values):
+ if not step():
+ return content
+ if value in selected_names:
+ if index == functions[value][0]:
+ continue
+ if value in escapes and index in callbacks:
+ continue
+ if (
+ values[index + 1 : index + 2] != ["("]
+ or index
+ and values[index - 1] in {".", "?.", "function", "new"}
+ ):
+ return content
+ if (
+ tokens[index].kind == "string"
+ and literal(index) in selected_names | protected | native_methods
+ and index >= 2
+ and values[index - 1] == "["
+ and values[index + 1 : index + 2] == ["]"]
+ and (tokens[index - 2].kind == "identifier" or values[index - 2] in {")", "]", "}"})
+ ):
+ # Protect computed member references, not text arguments such as a
+ # DOM element whose ID happens to be "filter" or "map".
+ return content
+
+ def receiver_start(end: int, depth: int = 0) -> int | None:
+ if depth >= _MAX_NESTING or end <= 0 or not step():
+ return None
+ last = end - 1
+ if last in reverse:
+ opening = reverse[last]
+ if opening and (
+ tokens[opening - 1].kind == "identifier" or values[opening - 1] in {")", "]"}
+ ):
+ return receiver_start(opening, depth + 1)
+ return opening
+ if tokens[last].kind not in {"identifier", "string", "number"}:
+ return None
+ if last >= 2 and values[last - 1] in {".", "?."}:
+ return receiver_start(last - 1, depth + 1)
+ return last
+
+ # Resolve the nearest explicit binding, including callback/function
+ # parameters. A same-spelled outer DOM or JSON constant is not that value.
+ binding_rows: dict[_BrowserBinding, tuple[int, int, int, int]] = {}
+ bindings_by_name: dict[str, list[_BrowserBinding]] = {}
+ for name, rows in constants.items():
+ for declaration, a, b, lower, upper in rows:
+ key = ("const", declaration, name)
+ binding_rows[key] = a, b, lower, upper
+ bindings_by_name.setdefault(name, []).append(key)
+ for declaration, body, stop, params in functions.values():
+ for slot, parameter in enumerate(params):
+ key = ("function", declaration, parameter)
+ binding_rows[key] = slot, declaration, body + 1, stop
+ bindings_by_name.setdefault(parameter, []).append(key)
+ for call, body, stop, params in arrows.values():
+ for slot, parameter in enumerate(params):
+ key = ("arrow", body, parameter)
+ binding_rows[key] = slot, call, body, stop
+ bindings_by_name.setdefault(parameter, []).append(key)
+
+ def binding(name: str, index: int) -> _BrowserBinding | None:
+ owner = None
+ width = len(tokens) + 1
+ for key in bindings_by_name.get(name, []):
+ if not step():
+ return None
+ _, _, lower, upper = binding_rows[key]
+ if lower <= index < upper:
+ size = upper - lower
+ if size == width:
+ return None
+ if size < width:
+ owner, width = key, size
+ return owner
+
+ calls_by_function: dict[int, list[int]] = {}
+ for index, value in enumerate(values):
+ if not step():
+ return content
+ if value in functions and index != functions[value][0]:
+ declaration = functions[value][0]
+ if binding(value, index) is not None:
+ return content
+ calls_by_function.setdefault(declaration, []).append(index)
+
+ resolving: set[tuple[str, int, str]] = set()
+ dependencies: set[_BrowserBinding] = set()
+
+ def ordinary(start: int, end: int, env: frozenset[str] = frozenset(), depth: int = 0) -> bool:
+ if start >= end or depth >= 32 or not step():
+ return False
+ if start in template_spans and template_spans[start][0] == end:
+ return all(ordinary(a, b, env, depth + 1) for a, b in template_fields(start))
+ if values[start] == "(" and pairs.get(start) == end - 1:
+ return ordinary(start + 1, end - 1, env, depth + 1)
+ cursor = start
+ while cursor < end:
+ if not step():
+ return False
+ if values[cursor] in {"===", "!==", "==", "!=", "&&", "||", "??"}:
+ return ordinary(start, cursor, env, depth + 1) and ordinary(
+ cursor + 1, end, env, depth + 1
+ )
+ cursor = pairs.get(cursor, cursor) + 1
+ if values[start] in {"!", "typeof"}:
+ return ordinary(start + 1, end, env, depth + 1)
+ cursor = start + 1
+ value = values[start]
+ if value in {"{", "["} and start in pairs:
+ closing = pairs[start]
+ if closing >= end:
+ return False
+ for a, b in parts(start + 1, closing):
+ if values[start] == "{" and a + 1 < b and values[a + 1] == ":":
+ if tokens[a].kind != "identifier" or values[a] in forbidden | native_methods:
+ return False
+ a += 2
+ elif value == "[" and values[a] == "...":
+ a += 1
+ if not ordinary(a, b, env, depth + 1):
+ return False
+ cursor = closing + 1
+ elif value == "Object" and values[start + 1 : start + 4] == [".", "entries", "("]:
+ opening = start + 3
+ args = arguments(opening)
+ if len(args) != 1 or not ordinary(*args[0], env, depth + 1):
+ return False
+ cursor = pairs[opening] + 1
+ elif value == "JSON" and values[start + 1 : start + 4] == [".", "parse", "("]:
+ opening = start + 3
+ args = arguments(opening)
+ if len(args) != 1:
+ return False
+ a, b = args[0]
+ if (
+ values[a : a + 4] != ["document", ".", "getElementById", "("]
+ or literal(a + 4) is None
+ or values[a + 5 : b] != [")", ".", "textContent"]
+ ):
+ return False
+ cursor = pairs[opening] + 1
+ elif start + 1 < end and values[start + 1] == "(" and value in functions:
+ declaration, body, stop, params = functions[value]
+ opening = start + 1
+ args = arguments(opening)
+ key = "call", declaration, value
+ if (
+ key in resolving
+ or len(args) != len(params)
+ or values[body + 1 : body + 2] != ["return"]
+ or values[stop - 1 : stop] != [";"]
+ ):
+ return False
+ if not all(ordinary(a, b, env, depth + 1) for a, b in args):
+ return False
+ resolving.add(key)
+ valid = ordinary(body + 2, stop - 1, frozenset(params), depth + 1)
+ resolving.remove(key)
+ if not valid:
+ return False
+ cursor = pairs[opening] + 1
+ elif (
+ tokens[start].kind in {"string", "number"}
+ or value in {"true", "false", "null"}
+ or value in env
+ ):
+ pass
+ elif tokens[start].kind == "identifier":
+ if value in mutated:
+ return False
+ key = binding(value, start)
+ if key is None or key in resolving:
+ return False
+ dependencies.add(key)
+ a, b, _, _ = binding_rows[key]
+ resolving.add(key)
+ if key[0] == "const":
+ valid = ordinary(a, b, frozenset(), depth + 1)
+ elif key[0] == "arrow":
+ call = b
+ begin = receiver_start(call - 2)
+ valid = (
+ begin is not None
+ and values[call - 1] in {"map", "filter", "find", "some", "flatMap", "forEach"}
+ and ordinary(begin, call - 2, frozenset(), depth + 1)
+ )
+ else:
+ # Every explicit invocation must provide an ordinary argument.
+ # A recursive unchanged forwarding edge adds no new source.
+ valid = True
+ sources = 0
+ for call in calls_by_function.get(key[1], []):
+ if not step():
+ valid = False
+ break
+ if call in callbacks:
+ opening = enclosing[call]
+ begin = None if opening is None else receiver_start(opening - 2)
+ good = (
+ a == 0
+ and opening is not None
+ and begin is not None
+ and values[opening - 1]
+ in {"map", "filter", "find", "some", "flatMap", "forEach"}
+ and ordinary(begin, opening - 2, frozenset(), depth + 1)
+ )
+ else:
+ args = arguments(call + 1)
+ if a >= len(args):
+ params = arguments(key[1] + 1)
+ lower, upper = params[a]
+ good = upper == lower + 3 and literal(lower + 2) is not None
+ else:
+ lower, upper = args[a]
+ if upper == lower + 1 and binding(values[lower], lower) == key:
+ continue
+ good = ordinary(lower, upper, frozenset(), depth + 1)
+ sources += 1
+ if not good:
+ valid = False
+ break
+ valid = valid and sources > 0
+ resolving.remove(key)
+ if not valid:
+ return False
+ else:
+ return False
+ while cursor < end and step():
+ if (
+ values[cursor] == "["
+ and pairs.get(cursor) == cursor + 2
+ and tokens[cursor + 1].kind == "number"
+ ):
+ cursor += 3
+ continue
+ if values[cursor] not in {".", "?."} or cursor + 1 >= end:
+ return False
+ member = values[cursor + 1]
+ if tokens[cursor + 1].kind != "identifier" or member in forbidden:
+ return False
+ cursor += 2
+ if cursor < end and values[cursor] == "(":
+ if member not in {
+ "map",
+ "filter",
+ "find",
+ "some",
+ "flatMap",
+ "join",
+ "trim",
+ "toLowerCase",
+ }:
+ return False
+ args = arguments(cursor)
+ if member in {"trim", "toLowerCase"}:
+ if args:
+ return False
+ elif member == "join":
+ if len(args) > 1 or any(b != a + 1 or literal(a) is None for a, b in args):
+ return False
+ else:
+ matches = arrows_by_call.get(cursor, [])
+ if len(args) != 1:
+ return False
+ a, b = args[0]
+ if member in {"filter", "find", "some"} and b == a + 1 and a in callbacks:
+ # These native operations retain existing members;
+ # dependency closure separately audits the predicate.
+ cursor = pairs[cursor] + 1
+ continue
+ if len(matches) != 1:
+ return False
+ _, body, stop, names = matches[0]
+ if member in {"filter", "find", "some"}:
+ # A simple predicate can select only existing JSON
+ # members. Its values need not become part of output.
+ if not step(stop - body + 1) or any(
+ values[i] not in {".", "?.", "!", "===", "!==", "==", "!=", "&&", "||"}
+ and tokens[i].kind not in {"identifier", "string", "number"}
+ for i in range(body, stop)
+ ):
+ return False
+ elif not ordinary(body, stop, env | frozenset(names), depth + 1):
+ return False
+ cursor = pairs[cursor] + 1
+ return cursor == end
+
+ def display(start: int, end: int) -> bool:
+ escaped = False
+ for _ in range(_MAX_NESTING):
+ if not step():
+ return False
+ if values[end : end + 3] == [".", "join", "("]:
+ args = arguments(end + 2)
+ if len(args) != 1 or args[0][1] != args[0][0] + 1 or literal(args[0][0]) != "":
+ return False
+ end = pairs[end + 2] + 1
+ continue
+ parent = enclosing[start]
+ if parent in fields and start == parent + 1 and pairs[parent] == end:
+ template = fields[parent]
+ if not text_slot(template, parent):
+ return False
+ start, end = template, template_spans[template][0]
+ continue
+ arrow = arrows_by_body.get((start, end))
+ if arrow is not None:
+ call = arrow[0]
+ begin = receiver_start(call - 2)
+ if values[call - 1] != "map" or begin is None or not ordinary(begin, call - 2):
+ return False
+ start, end = begin, pairs[call] + 1
+ continue
+ if parent is not None and parent and values[parent - 1] in formatters:
+ args = arguments(parent)
+ if (
+ not args
+ or args[0] != (start, end)
+ or len(args) > 2
+ or any(b != a + 1 or literal(a) is None for a, b in args[1:])
+ ):
+ return False
+ escaped = True
+ start, end = parent - 1, pairs[parent] + 1
+ continue
+ if (
+ start >= 4
+ and values[start - 3 : start] == [".", "innerHTML", "="]
+ and values[end : end + 1] == [";"]
+ ):
+ return escaped and dom(start - 4, start - 3) == "element"
+ return False
+ return False
+
+ def dom(start: int, end: int, depth: int = 0) -> str | None:
+ if depth >= 32 or start >= end or not step():
+ return None
+ cursor = start + 1
+ kind = None
+ if values[start : start + 4] == ["document", ".", "getElementById", "("]:
+ if literal(start + 4) is None or values[start + 5 : start + 6] != [")"]:
+ return None
+ cursor, kind = start + 6, "element"
+ elif tokens[start].kind == "identifier":
+ key = binding(values[start], start)
+ if key is None:
+ return None
+ dependencies.add(key)
+ a, b, _, _ = binding_rows[key]
+ if key[0] == "const":
+ kind = dom(a, b, depth + 1)
+ elif key[0] == "arrow" and a == 0 and values[b - 1] == "forEach":
+ begin = receiver_start(b - 2)
+ if begin is not None and dom(begin, b - 2, depth + 1) == "list":
+ kind = "element"
+ if kind is None:
+ return None
+ if cursor < end and values[cursor : cursor + 3] == [".", "querySelectorAll", "("]:
+ args = arguments(cursor + 2)
+ if (
+ kind != "element"
+ or len(args) != 1
+ or args[0][1] != args[0][0] + 1
+ or literal(args[0][0]) is None
+ ):
+ return None
+ cursor, kind = pairs[cursor + 2] + 1, "list"
+ return kind if cursor == end else None
+
+ member_values: list[tuple[_BrowserBinding, int, int]] = []
+ for index in member_writes:
+ begin = receiver_start(index - 2)
+ if begin is None:
+ return content
+ if (
+ values[index - 1] in {"innerHTML", "textContent", "value"}
+ and dom(begin, index - 2) == "element"
+ ):
+ continue
+ if begin != index - 3:
+ return content
+ owner = binding(values[begin], begin)
+ if owner is None or owner[0] != "const":
+ return content
+ a, b, _, _ = binding_rows[owner]
+ if values[a] != "{" or pairs.get(a) != b - 1:
+ return content
+ stop = index + 1
+ while stop < len(tokens) and values[stop] != ";" and step():
+ stop = pairs.get(stop, stop) + 1
+ if stop == len(tokens) or budget <= 0:
+ return content
+ member_values.append((owner, index + 1, stop))
+
+ for index in callbacks:
+ if values[index] not in escapes:
+ continue
+ call = enclosing[index]
+ if call is None:
+ return content
+ begin = receiver_start(call - 2)
+ if begin is None or values[call - 1] != "map" or not ordinary(begin, call - 2):
+ return content
+ output = list(content)
+ changed = False
+ for start, (end, delimiters) in template_spans.items():
+ if not step():
+ return content
+ expressions = template_fields(start)
+ if not expressions or any(
+ _TEMPLATE_VALUE.fullmatch(source(a, b)) is None for a, b in expressions
+ ):
+ continue
+ if not display(start, end) or not all(ordinary(a, b) for a, b in expressions):
+ continue
+ output[delimiters[0]] = output[delimiters[-1]] = " "
+ changed = True
+
+ # Follow explicit aliases and helper returns that carry a proven value.
+ # Unknown native-looking receivers must not acquire it through a callback.
+ def dependency(index: int) -> bool:
+ if (
+ tokens[index].kind != "identifier"
+ or index
+ and values[index - 1] in {".", "?."}
+ or values[index + 1 : index + 2] == [":"]
+ ):
+ return False
+ if values[index] == "document":
+ # A second literal lookup can reacquire the selected element even
+ # without referring to its local constant binding.
+ return True
+ key = binding(values[index], index)
+ if key is None and values[index] in functions:
+ key = ("call", functions[values[index]][0], values[index])
+ return key in dependencies
+
+ def carries(start: int, end: int) -> bool:
+ return step(end - start + 1) and any(dependency(i) for i in range(start, end))
+
+ for _ in range(_MAX_NESTING):
+ before = len(dependencies)
+ for name, rows in constants.items():
+ for declaration, a, b, _, _ in rows:
+ if carries(a, b):
+ dependencies.add(("const", declaration, name))
+ for owner, a, b in member_values:
+ if carries(a, b):
+ dependencies.add(owner)
+ for name, (declaration, a, b, params) in functions.items():
+ if carries(a, b):
+ dependencies.add(("call", declaration, name))
+ for call in calls_by_function.get(declaration, []):
+ if not step():
+ return content
+ if call in callbacks:
+ opening = enclosing[call]
+ begin = None if opening is None else receiver_start(opening - 2)
+ if begin is not None and carries(begin, opening - 2):
+ dependencies.update(
+ ("function", declaration, parameter) for parameter in params
+ )
+ else:
+ for parameter, (lower, upper) in zip(params, arguments(call + 1), strict=False):
+ if carries(lower, upper):
+ dependencies.add(("function", declaration, parameter))
+ for call, body, _, params in arrows.values():
+ begin = receiver_start(call - 2) if call >= 3 and values[call - 2] == "." else None
+ if begin is not None and carries(begin, call - 2):
+ dependencies.update(("arrow", body, parameter) for parameter in params)
+ if budget <= 0:
+ return content
+ if len(dependencies) == before:
+ break
+ else:
+ return content
+ for opening in pairs:
+ if values[opening] != "(" or opening < 3 or values[opening - 2] != ".":
+ continue
+ member = values[opening - 1]
+ if member not in native_methods:
+ continue
+ closing = pairs[opening]
+ if not step(closing - opening + 1):
+ return content
+ if not any(dependency(i) for i in range(opening + 1, closing)):
+ continue
+ begin = receiver_start(opening - 2)
+ if begin is None or not (
+ ordinary(begin, opening - 2) or dom(begin, opening - 2) is not None
+ ):
+ return content
+ check_runtime()
+ return "".join(output) if changed and budget > 0 else content
+
+
JavaScriptTemplateSpans = tuple[tuple[int, ...], tuple[int, ...]]
_SHELL_EXECUTION_CALLS = frozenset(
diff --git a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py
index 8daa81895..da0d01819 100644
--- a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py
+++ b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py
@@ -46,7 +46,7 @@
from skillspector.logging_config import get_logger
from skillspector.models import AnalyzerFinding, Finding, Location, Severity
from skillspector.python_ast import ParsedPythonFile, parse_python_source
-from skillspector.python_tokens import PythonLiteralSpans
+from skillspector.python_tokens import PythonLiteralSpans, python_browser_template_shell_text
from skillspector.python_tokens import python_literal_spans as _python_literal_spans
from skillspector.python_tokens import python_string_body as _python_string_body
from skillspector.security_reconstruction import validated_json_string_spans
@@ -92,6 +92,7 @@
_PRINTF_STATIC_CHARS = 256
_PRINTF_STATIC_ARGUMENTS = 32
_PRINTF_STATIC_WORD_RE = re.compile(r"[-A-Za-z0-9_./*?%]{0,64}")
+_PRINTF_QUOTED_WORD_RE = re.compile(r"[-A-Za-z0-9_./*?% \t\r\n]{0,64}")
# A bounded ``printf`` format operand may also carry POSIX backslash escapes.
_PRINTF_FORMAT_WORD_RE = re.compile(r"[-A-Za-z0-9_./*?%\\]{0,64}")
# The old deterministic alphabet plus the control/high-byte values needed by
@@ -621,6 +622,8 @@ class _ShellDelimiterFrame:
inherited_quote_closed: bool = False
pending_case_clauses: int = 0
open_case_clauses: int = 0
+ pending_heredocs: tuple[tuple[str, bool, bool], ...] = ()
+ arithmetic: bool = False
def _is_shell_command_word_start(content: str, start: int) -> bool:
@@ -715,6 +718,84 @@ def _skip_ansi_c_quote_tail(content: str, start: int, limit: int) -> int | None:
return None
+def _static_heredoc_header(
+ content: str, start: int, limit: int
+) -> tuple[tuple[str, bool, bool], int] | None:
+ """Read one bounded literal heredoc delimiter, without evaluating it."""
+ cursor = start + 2
+ strip_tabs = content[cursor : cursor + 1] == "-"
+ cursor += strip_tabs
+ whitespace_limit = min(limit, cursor + _SHELL_COMMAND_WORD_CHARS)
+ while cursor < whitespace_limit and content[cursor] in " \t":
+ cursor += 1
+ if cursor < limit and content[cursor] in " \t":
+ return None
+ quote = content[cursor : cursor + 1]
+ quoted = quote in {"'", '"'}
+ cursor += quoted
+ name = re.match(
+ r"[A-Za-z_][A-Za-z0-9_]*",
+ content[cursor : min(limit, cursor + _SHELL_COMMAND_WORD_CHARS + 1)],
+ )
+ if name is None or len(name[0]) > _SHELL_COMMAND_WORD_CHARS:
+ return None
+ cursor += len(name[0])
+ if quoted:
+ if cursor >= limit or content[cursor] != quote:
+ return None
+ cursor += 1
+ if cursor < limit and not (content[cursor].isspace() or content[cursor] in ";|&()<>"):
+ return None
+ return (name[0], strip_tabs, quoted), cursor
+
+
+def _skip_static_heredoc_bodies(
+ content: str,
+ start: int,
+ limit: int,
+ pending: tuple[tuple[str, bool, bool], ...],
+ check_runtime: Callable[[], None] | None,
+) -> int | None:
+ """Skip ordered bodies only after their declaring command's newline.
+
+ This proves outer lexical state only. The raw scanner still inspects every
+ body, including substitutions in unquoted heredocs. Delimiter comparison
+ joins unquoted continuations while retaining at most one delimiter's width.
+ """
+ cursor = start
+ for delimiter, strip_tabs, quoted in pending:
+ logical_prefix = ""
+ while cursor < limit:
+ if check_runtime is not None:
+ check_runtime()
+ newline = content.find("\n", cursor, limit)
+ line_end = limit if newline < 0 else newline
+ if line_end > cursor and content[line_end - 1] == "\r":
+ line_end -= 1
+ line = content[cursor:line_end]
+ continued = (
+ not quoted and newline >= 0 and (len(line) - len(line.rstrip("\\"))) % 2 == 1
+ )
+ if continued:
+ line = line[:-1]
+ if strip_tabs and not logical_prefix:
+ line = line.lstrip("\t")
+ # Once overlong, a logical line cannot become the delimiter. Keep
+ # a capped prefix instead of accumulating arbitrarily long bodies.
+ logical_prefix += line[: max(0, len(delimiter) + 1 - len(logical_prefix))]
+ if not continued and logical_prefix == delimiter:
+ cursor = limit if newline < 0 else newline + 1
+ break
+ if newline < 0:
+ return None
+ cursor = newline + 1
+ if not continued:
+ logical_prefix = ""
+ else:
+ return None
+ return cursor
+
+
def _skip_shell_delimited_expansion(
content: str,
start: int,
@@ -734,6 +815,7 @@ def _skip_shell_delimited_expansion(
root_kind,
start,
inherited_double_quote=root_inherited_double_quote,
+ arithmetic=root_kind == "command" and content.startswith("$((", start),
)
]
cursor = start + opener_width
@@ -782,6 +864,10 @@ def push(kind: str, frame_start: int | None, width: int) -> None:
kind,
frame_start,
inherited_double_quote=inherited_double_quote,
+ arithmetic=(
+ (kind == "paren" and (parent.arithmetic or content.startswith("((", cursor)))
+ or (frame_start is not None and content.startswith("$((", frame_start))
+ ),
)
)
cursor += width
@@ -837,6 +923,32 @@ def at_shell_keyword(keyword: str) -> bool:
cursor += 1
continue
+ if frame.kind in {"command", "paren"} and not frame.arithmetic:
+ if character == "\n" and frame.pending_heredocs:
+ body_end = _skip_static_heredoc_bodies(
+ content, cursor + 1, limit, frame.pending_heredocs, check_runtime
+ )
+ if body_end is None:
+ cache_failure()
+ return None
+ frame.pending_heredocs = ()
+ frame.word_started = False
+ cursor = body_end
+ continue
+ if content.startswith("<<<", cursor):
+ cursor += 3
+ frame.word_started = False
+ continue
+ if content.startswith("<<", cursor):
+ header = _static_heredoc_header(content, cursor, limit)
+ if header is None or len(frame.pending_heredocs) == _SHELL_COMMAND_STRING_ARGUMENTS:
+ cache_failure()
+ return None
+ spec, cursor = header
+ frame.pending_heredocs += (spec,)
+ frame.word_started = True
+ continue
+
# Ordinary words cannot change the delimiter stack. Consume them in
# one bounded match rather than testing each character for every shell
# opener, quote, and keyword. Nested static-evaluator windows otherwise
@@ -938,6 +1050,11 @@ def at_shell_keyword(keyword: str) -> bool:
continue
if character == "(":
+ if frame.pending_heredocs:
+ # Plain subshell grammar can share pending heredocs with the
+ # enclosing command. Do not assign that body to a new frame.
+ cache_failure()
+ return None
push("paren", None, 1)
continue
if character == ")":
@@ -948,6 +1065,9 @@ def at_shell_keyword(keyword: str) -> bool:
frame.word_started = False
cursor += 1
continue
+ if frame.pending_heredocs:
+ cache_failure()
+ return None
endpoint = close_frame(cursor + 1)
cursor += 1
if endpoint is not None:
@@ -1190,9 +1310,11 @@ def _next_shell_invocation_word(
"""Read one direct shell word without traversing nested substitutions."""
cursor = start
limit = len(content)
+ next_runtime_check = ((cursor + 4095) // 4096) * 4096
while cursor < limit:
- if cursor % 4096 == 0:
+ if cursor >= next_runtime_check:
check_runtime()
+ next_runtime_check = cursor + 4096
if content[cursor].isspace():
cursor += 1
continue
@@ -1212,8 +1334,9 @@ def _next_shell_invocation_word(
word_started = False
unquoted_characters = 0
while cursor < limit:
- if cursor % 4096 == 0:
+ if cursor >= next_runtime_check:
check_runtime()
+ next_runtime_check = cursor + 4096
character = content[cursor]
if quote is not None:
if character == quote:
@@ -1474,31 +1597,66 @@ def _bounded_printf_word(result: str) -> str | None:
return result
+def _has_literal_printf_arguments(inner: str) -> bool:
+ """Prove literal arguments without treating shell expansions as text."""
+ quote: str | None = None
+ cursor = 0
+ while cursor < len(inner):
+ character = inner[cursor]
+ if character == "\\" and quote != "'":
+ if cursor + 1 >= len(inner):
+ return False
+ cursor += 3 if inner.startswith("\\\r\n", cursor) else 2
+ continue
+ if quote is not None:
+ if character == quote:
+ quote = None
+ elif quote == '"' and character in "$`":
+ return False
+ elif character in "'\"":
+ quote = character
+ elif character in "$`;|&<>()\r\n*?[]{}~#":
+ return False
+ cursor += 1
+ if quote is not None:
+ return False
+ command, _, limited = _next_shell_invocation_word(inner, 0, lambda: None)
+ return command == "printf" and not limited
+
+
def _static_printf_substitution(
content: str,
start: int,
end: int,
*,
backtick: bool = False,
+ quoted_result: bool = False,
) -> str | None:
- """Evaluate a tiny, bounded subset of shell ``printf`` deterministically."""
+ """Evaluate bounded literal ``printf`` output, preserving its quoting."""
inner_start = start + (1 if backtick else 2)
inner_end = end - 1
inner = content[inner_start:inner_end]
+ if len(inner) > _PRINTF_STATIC_CHARS:
+ return None
+ literal_result = quoted_result and not backtick and _has_literal_printf_arguments(inner)
+ if quoted_result and not backtick and not literal_result:
+ command, _, _ = _next_shell_invocation_word(inner, 0, lambda: None)
+ if command == "printf":
+ return None
recognized, arguments = _printf_invocation_arguments(inner)
- if (
- not recognized
- or len(inner) > _PRINTF_STATIC_CHARS
- or re.search(r"[;|&<>\r\n]", inner) is not None
- ):
+ if not recognized or (not literal_result and re.search(r"[;|&<>\r\n]", inner) is not None):
return None
if arguments[:1] == ["--"]:
arguments = arguments[1:]
+ elif literal_result and arguments and arguments[0].startswith("-"):
+ # In particular, ``-v`` assigns a variable instead of producing output.
+ return None
if not arguments or len(arguments) > _PRINTF_STATIC_ARGUMENTS + 1:
return None
format_word, *values = arguments
+ value_pattern = _PRINTF_QUOTED_WORD_RE if literal_result else _PRINTF_STATIC_WORD_RE
if _PRINTF_FORMAT_WORD_RE.fullmatch(format_word) is None or any(
- _PRINTF_STATIC_WORD_RE.fullmatch(value) is None for value in values
+ value_pattern.fullmatch(value) is None for value in values
):
return None
@@ -1538,17 +1696,27 @@ def flush_literal() -> bool:
return None
conversions = sum(segment is None for segment in segments)
+
+ def bounded_result(result: str) -> str | None:
+ if literal_result:
+ # Quoted command substitution keeps one word and strips trailing
+ # LF bytes only. Its internal whitespace never becomes shell syntax.
+ if len(result) > _PRINTF_STATIC_CHARS or "\x00" in result:
+ return None
+ return result.rstrip("\n")
+ return _bounded_printf_word(result)
+
if conversions == 0:
- return _bounded_printf_word("".join(segments))
+ return bounded_result("".join(segments))
remaining = iter(values)
output: list[str] = []
cycles = max(1, (len(values) + conversions - 1) // conversions)
for _ in range(cycles):
for segment in segments:
output.append(next(remaining, "") if segment is None else segment)
- if sum(len(piece) for piece in output) > 64:
+ if sum(len(piece) for piece in output) > (_PRINTF_STATIC_CHARS if literal_result else 64):
return None
- return _bounded_printf_word("".join(output))
+ return bounded_result("".join(output))
def _may_have_destructive_outer_operands(content: str) -> bool:
@@ -1788,6 +1956,7 @@ def _parse_shell_command_word(
check_runtime: Callable[[], None] | None = None,
owned_word_positions: set[int] | None = None,
enclosing_delimiter: str | None = None,
+ quoted_printf_allowed: bool = True,
) -> _ShellCommandWord | None:
runtime_check = check_runtime or (lambda: None)
# The shell recognizes ``NAME=value`` before expansion and never runs the
@@ -1856,6 +2025,7 @@ def reconstructs_command(
content,
cursor,
substitution_end,
+ quoted_result=quoted_printf_allowed,
)
if static_value is None:
output.append("$()")
@@ -2115,6 +2285,57 @@ def _destructive_command_words(content: str) -> Iterator[tuple[int, int]]:
yield start, parsed.end
+class _ShellOwnershipBudgetExceededError(Exception):
+ """Stop optional quote proof without swallowing the caller's cancellation."""
+
+
+def _literal_shell_command_spans(
+ content: str, begin: int, end: int, check_runtime: Callable[[], None]
+) -> dict[int, int] | None:
+ """Locate exact single-quoted shell -c bodies once per lexical scope."""
+ source = content[begin:end]
+ if "'" not in source:
+ return {}
+ spans: list[tuple[int, int]] = []
+ # Invocation readers report periodic progress across escaped sequences.
+ # Bound repeated delimiter scans proportionally to this scope, so a valid
+ # literal command before deeply nested unrelated input cannot make this
+ # optional ownership pass quadratic. Exhaustion publishes no proof.
+ remaining_checks = 4 + 4 * ((len(source) + 4095) // 4096)
+
+ def check_proof_runtime() -> None:
+ nonlocal remaining_checks
+ check_runtime()
+ remaining_checks -= 1
+ if remaining_checks < 0:
+ raise _ShellOwnershipBudgetExceededError
+
+ clause_starts = _shell_clause_starts(source, check_runtime)
+ next_clause_start = next(clause_starts, None)
+ clause_start = 0
+ last_selected_start = -1
+ try:
+ for option in re.finditer(r"(-[A-Za-z]+)\s+(?:\\\r?\n[ \t]*)*'", source):
+ if "c" not in option[1]:
+ continue
+ check_runtime()
+ # Select the nearest preceding clause for full validation of its
+ # wrapper and literal body; ambiguous context retains no proof.
+ # Both iterators move forward once; unrelated clauses need no parse.
+ while next_clause_start is not None and next_clause_start <= option.start():
+ clause_start = next_clause_start
+ next_clause_start = next(clause_starts, None)
+ if clause_start == last_selected_start:
+ continue
+ last_selected_start = clause_start
+ _command_string_from_clause(
+ source, clause_start, check_proof_runtime, literal_command_spans=spans
+ )
+ except _ShellOwnershipBudgetExceededError:
+ return None
+ return {begin + start: begin + stop for start, stop in spans}
+
+
def _has_destructive_command_string(content: str) -> bool:
"""Return whether an executable string contains a proven destructive command."""
for command_start, body_start in _destructive_command_words(content):
@@ -2125,16 +2346,21 @@ def _has_destructive_command_string(content: str) -> bool:
def _shell_quoted_candidate_limits(
- content: str, check_runtime: Callable[[], None]
+ content: str,
+ check_runtime: Callable[[], None],
+ *,
+ quote_closers: set[int] | None = None,
) -> dict[int, int]:
"""Prove single-line double-quote bounds by walking shell source in order.
This is only an ownership refinement, never an execution or finding parser.
Comments and single quotes cannot open double-quoted spans. Substitution
- bodies have independent lexical state. Unsupported heredocs/backticks or
+ bodies have independent lexical state. Literal heredoc bodies cannot own
+ quotes in the surrounding command. Unsupported delimiters/backticks or
unresolved syntax discard ALL ownership, retaining the conservative scan.
"""
limits: dict[int, int] = {}
+ closers: set[int] = set()
parameter_cache: dict[int, _ParameterExpansionEnd] = {}
substitution_cache: dict[int, int | None] = {}
@@ -2147,16 +2373,44 @@ def walk(begin: int, end: int, depth: int = 0) -> bool:
quote_start = begin
ansi_c = False
candidates: list[int] = []
+ pending_heredocs: tuple[tuple[str, bool, bool], ...] = ()
+ command_spans = _literal_shell_command_spans(content, begin, end, check_runtime)
+ if command_spans is None:
+ return False
while cursor < end:
if (cursor - begin) % 1024 == 0:
check_runtime()
character = content[cursor]
+ if character == "\n" and pending_heredocs:
+ if quote is not None:
+ return False
+ body_end = _skip_static_heredoc_bodies(
+ content, cursor + 1, end, pending_heredocs, check_runtime
+ )
+ if body_end is None:
+ return False
+ pending_heredocs = ()
+ word_started = False
+ cursor = body_end
+ continue
+ if quote is not None and character == quote:
+ closers.add(cursor)
+ if (
+ not ansi_c
+ and "\n" not in content[quote_start:cursor]
+ and "\r" not in content[quote_start:cursor]
+ ):
+ for position in candidates:
+ limits[position] = cursor
+ quote = None
+ cursor += 1
+ continue
if quote == "'":
if character == "\\" and ansi_c:
cursor += 2
continue
- if character == "'":
- quote = None
+ if not ansi_c and _SHELL_COMMAND_WORD_START_RE.match(content, cursor):
+ candidates.append(cursor)
cursor += 1
continue
if character == "\\":
@@ -2166,17 +2420,23 @@ def walk(begin: int, end: int, depth: int = 0) -> bool:
word_started = True
cursor += 2
continue
- if quote == '"' and character == '"':
- if (
- "\n" not in content[quote_start:cursor]
- and "\r" not in content[quote_start:cursor]
- ):
- for position in candidates:
- limits[position] = cursor
- quote = None
- cursor += 1
+ if quote is None and content.startswith("<<<", cursor):
+ # A here-string takes an ordinary shell word; unlike a
+ # heredoc it does not change the ownership of later lines.
+ cursor += 3
+ word_started = False
continue
- if character == "`" or (quote is None and content.startswith("<<", cursor)):
+ if quote is None and content.startswith("<<", cursor):
+ header = _static_heredoc_header(content, cursor, end)
+ if header is None or len(pending_heredocs) == _SHELL_COMMAND_STRING_ARGUMENTS:
+ return False
+ spec, cursor = header
+ pending_heredocs += (spec,)
+ word_started = True
+ continue
+ if quote is None and character == "(" and pending_heredocs:
+ return False
+ if character == "`":
return False
if quote is None and character == "#" and not word_started:
while cursor < end and content[cursor] not in "\r\n":
@@ -2195,6 +2455,14 @@ def walk(begin: int, end: int, depth: int = 0) -> bool:
cursor += 2
continue
if quote is None and character in "'\"":
+ command_end = command_spans.get(cursor + 1) if character == "'" else None
+ if command_end is not None:
+ if not walk(cursor + 1, command_end, depth + 1):
+ return False
+ closers.add(command_end)
+ cursor = command_end + 1
+ word_started = True
+ continue
quote = character
ansi_c = False
quote_start = cursor
@@ -2218,6 +2486,22 @@ def walk(begin: int, end: int, depth: int = 0) -> bool:
word_started = True
continue
if content.startswith("${", cursor):
+ if quote == '"':
+ # A quoted simple variable used as a trim pattern owns its
+ # inner quotes. Do not apply this to the ksh-sensitive $"
+ # cases handled by the general parameter parser below.
+ pattern = re.match(
+ r"\$\{[A-Za-z_][A-Za-z0-9_]*(?:#{1,2}|%{1,2})"
+ r'"\$(?:[A-Za-z_][A-Za-z0-9_]*|\{[A-Za-z_][A-Za-z0-9_]*\})"'
+ r"""[^\\'"$`{}\r\n]*\}""",
+ content[cursor : min(end, cursor + _SHELL_COMMAND_WORD_CHARS + 1)],
+ )
+ if pattern is not None and len(pattern[0]) <= _SHELL_COMMAND_WORD_CHARS:
+ pattern_end = cursor + len(pattern[0])
+ closers.add(content.rfind('"', cursor, pattern_end))
+ cursor = pattern_end
+ word_started = True
+ continue
inherited_closed = [False]
parameter_end = _skip_parameter_expansion(
content,
@@ -2241,9 +2525,17 @@ def walk(begin: int, end: int, depth: int = 0) -> bool:
if quote is None:
word_started = not (character.isspace() or character in ";|&()<>")
cursor += 1
- return quote is None and cursor <= end
+ return quote is None and not pending_heredocs and cursor <= end
- return limits if walk(0, len(content)) else {}
+ if not walk(0, len(content)):
+ return {}
+ if quote_closers is not None:
+ quote_closers.update(
+ position
+ for position in closers
+ if content[position + 1 : position + 2] not in ("$", "`")
+ )
+ return limits
def _quoted_candidate_is_complete(
@@ -2251,6 +2543,7 @@ def _quoted_candidate_is_complete(
start: int,
quote_limits: dict[int, int] | None,
check_runtime: Callable[[], None],
+ quoted_printf_allowed: bool = True,
) -> bool:
"""Refine a failure only inside a lexically proven shell quote.
@@ -2262,12 +2555,16 @@ def _quoted_candidate_is_complete(
if end is None:
return False
fragment = content[start:end]
- parsed = _parse_shell_command_word(fragment, 0, check_runtime=check_runtime)
+ parsed = _parse_shell_command_word(
+ fragment, 0, check_runtime=check_runtime, quoted_printf_allowed=quoted_printf_allowed
+ )
if parsed is None or parsed.limited:
return False
return not (
parsed.dynamic
- and _has_runtime_command_operand_exhaustion(fragment, 0, parsed.end, check_runtime)
+ and _has_runtime_command_operand_exhaustion(
+ fragment, 0, parsed.end, check_runtime, quoted_printf_allowed=quoted_printf_allowed
+ )
)
@@ -2415,6 +2712,8 @@ def _has_shell_command_word_exhaustion(
quote_limits: dict[int, int] | None = None,
file_type: str = "shell",
_command_string_depth: int = 0,
+ _quoted_printf_allowed: bool = True,
+ _startup_exposure: list[bool] | None = None,
) -> bool:
"""Find candidate command words whose deterministic parse hit a safety bound.
@@ -2423,6 +2722,9 @@ def _has_shell_command_word_exhaustion(
``perl_eval_blocks`` recognizes Perl's exception-trapping ``eval BLOCK``,
which reparses no string.
"""
+ quoted_printf_allowed = _quoted_printf_allowed and not _shell_redefines_printf(
+ content, check_runtime
+ )
parsed_through = 0
# Completed words own closing quotes, never executable expansion starts.
# Inner commands remain independent candidates; never suppress their bodies.
@@ -2433,9 +2735,20 @@ def _has_shell_command_word_exhaustion(
may_have_destructive_outer_operands = _may_have_destructive_outer_operands(content)
json_openers = sorted(structural_quote_openers or ())
complete_comments: set[tuple[int, int]] = set()
+ # Short text without shell-expansion openers cannot trigger these shell-word
+ # reconstruction or span limits. Keep its prior lazy ownership paths instead
+ # of parsing an ordinary Python module for every candidate letter.
+ python_word_source = (
+ python_source
+ if python_source is not None
+ and (len(content) > _SHELL_COMMAND_WORD_CHARS or "$" in content or "`" in content)
+ else None
+ )
for candidate in _SHELL_COMMAND_WORD_START_RE.finditer(content):
check_runtime()
start = candidate.start()
+ if python_word_source is not None and python_word_source.is_string_delimiter(start):
+ continue
if start in owned_word_positions:
continue
if javascript_source is not None and javascript_source.owns(start):
@@ -2450,6 +2763,9 @@ def _has_shell_command_word_exhaustion(
not json_string_start
and not assignment_quote
and not _is_shell_command_word_start(content, start)
+ and not (
+ python_word_source is not None and python_word_source.starts_string_body(start)
+ )
):
continue
if _has_quoted_assignment_prefix(content, start):
@@ -2479,6 +2795,7 @@ def _has_shell_command_word_exhaustion(
enclosing_delimiter=(
_assignment_value_wrapper(content, start) if assignment_quote else None
),
+ quoted_printf_allowed=quoted_printf_allowed,
)
if parsed is None:
substitution_start = (
@@ -2503,6 +2820,27 @@ def _has_shell_command_word_exhaustion(
check_runtime=check_runtime,
):
return True
+ if content[start] == "`":
+ substitution_end = _skip_backtick_substitution(
+ content, start, len(content), check_runtime, end_cache=backtick_end_cache
+ )
+ if (
+ substitution_end is not None
+ and _is_printf_substitution(
+ content,
+ start,
+ substitution_end,
+ backtick=True,
+ check_runtime=check_runtime,
+ )
+ and not (
+ file_type in _HASH_COMMENT_FILE_TYPES
+ and _is_comment_candidate(
+ content, start, python_source, file_type=file_type
+ )
+ )
+ ):
+ return True
# An unclosed expansion or quote can consume the rest of the
# artifact. Once that unresolved span exceeds the command-word
# budget, treating it as clean would turn malformed, deeply nested
@@ -2524,9 +2862,19 @@ def _has_shell_command_word_exhaustion(
# expressions cannot continue a shell word into later code.
unresolved_end = min(unresolved_end, javascript_source.word_end(start))
if unresolved_end - start > _SHELL_COMMAND_WORD_CHARS:
- if not _quoted_candidate_is_complete(content, start, quote_limits, check_runtime):
+ if not _quoted_candidate_is_complete(
+ content, start, quote_limits, check_runtime, quoted_printf_allowed
+ ):
return True
continue
+ # A mis-paired quote can produce a completed word that crosses several
+ # Python tokens. It cannot own quotes or skip candidates beyond its
+ # proven host region, even when that lexical parse itself succeeded.
+ owned_end = (
+ min(parsed.end, python_word_source.word_end(start))
+ if python_word_source is not None
+ else parsed.end
+ )
# Comments end at a newline, and a quote after ``=`` in prose or host
# source may be mis-paired. Such a word cannot own a later line.
confined_word = "\n" in content[start : parsed.end] and (
@@ -2540,7 +2888,8 @@ def _has_shell_command_word_exhaustion(
owned_word_positions.update(
position
for position in candidate_word_positions
- if content[position + 1 : position + 2] not in ("$", "`")
+ if (python_word_source is None or position < owned_end)
+ and content[position + 1 : position + 2] not in ("$", "`")
)
# Only executable nested substitutions retain independent command
# positions. A plain dynamic data argument still owns its inner bytes;
@@ -2559,7 +2908,7 @@ def _has_shell_command_word_exhaustion(
or not any(marker in raw_word for marker in ("$(", "`"))
or simple_backtick_parameter
) and not (assignment_quote and confined_word):
- parsed_through = max(parsed_through, parsed.end)
+ parsed_through = max(parsed_through, owned_end)
if assignment_quote:
# An assignment value never names the command, and main never
# parsed this position, so it only claims ownership.
@@ -2574,7 +2923,9 @@ def _has_shell_command_word_exhaustion(
# spans and runtime-selected operands keep their existing
# conservative treatment.
continue
- if not _quoted_candidate_is_complete(content, start, quote_limits, check_runtime):
+ if not _quoted_candidate_is_complete(
+ content, start, quote_limits, check_runtime, quoted_printf_allowed
+ ):
return True
if (
parsed.dynamic
@@ -2584,11 +2935,14 @@ def _has_shell_command_word_exhaustion(
start,
parsed.end,
check_runtime,
+ quoted_printf_allowed=quoted_printf_allowed,
)
):
comment = python_source.comment(start) if python_source is not None else None
if comment is None:
- if _quoted_candidate_is_complete(content, start, quote_limits, check_runtime):
+ if _quoted_candidate_is_complete(
+ content, start, quote_limits, check_runtime, quoted_printf_allowed
+ ):
continue
# Host code after a Python string's closing quote is not shell
# text, so a command in the string takes operands only from
@@ -2600,7 +2954,9 @@ def _has_shell_command_word_exhaustion(
if python_source is not None
else None
)
- if value is None or _has_runtime_command_operand_exhaustion(*value, check_runtime):
+ if value is None or _has_runtime_command_operand_exhaustion(
+ *value, check_runtime, quoted_printf_allowed=quoted_printf_allowed
+ ):
return True
continue
# A Python comment is prose that no shell receives, so a command
@@ -2612,23 +2968,33 @@ def _has_shell_command_word_exhaustion(
content[comment[0] : comment[1]],
check_runtime,
_command_string_depth=_command_string_depth,
+ _quoted_printf_allowed=quoted_printf_allowed,
):
return True
complete_comments.add(comment)
- for command_string in _shell_command_strings(
+ for command_string, startup_context in _shell_command_strings(
content,
check_runtime,
perl_eval_blocks=perl_eval_blocks,
python_source=python_source,
+ startup_exposure=_startup_exposure,
):
if command_string is None or _command_string_depth >= 8:
return True
if _has_destructive_command_string(command_string):
return True
+ command_closers: set[int] = set()
+ command_limits = _shell_quoted_candidate_limits(
+ command_string, check_runtime, quote_closers=command_closers
+ )
if _has_shell_command_word_exhaustion(
command_string,
check_runtime,
+ structural_quote_closers=command_closers,
+ quote_limits=command_limits,
_command_string_depth=_command_string_depth + 1,
+ _quoted_printf_allowed=quoted_printf_allowed,
+ _startup_exposure=startup_context,
):
return True
return False
@@ -2639,6 +3005,8 @@ def _has_runtime_command_operand_exhaustion(
start: int,
word_end: int,
check_runtime: Callable[[], None],
+ *,
+ quoted_printf_allowed: bool = True,
) -> bool:
"""Return whether a runtime-selected command's operands are undecidable."""
tokens, command_end, exhausted = _bounded_shell_tokens(
@@ -2646,6 +3014,7 @@ def _has_runtime_command_operand_exhaustion(
start,
word_end,
check_runtime=check_runtime,
+ quoted_printf_allowed=quoted_printf_allowed,
)
if exhausted and command_end - start < _ROOT_GLOB_COMMAND_CHARS:
# Earlier prose (for example "row-first") cannot supply this
@@ -2670,16 +3039,32 @@ def _shell_command_strings(
*,
perl_eval_blocks: bool = False,
python_source: _PythonSourceOwnership | None = None,
-) -> Iterator[str | None]:
+ startup_exposure: list[bool] | None = None,
+) -> Iterator[tuple[str | None, list[bool]]]:
"""Yield bounded strings reparsed by ``eval`` or a shell ``-c`` wrapper."""
- for clause_start in _shell_clause_starts(content, check_runtime):
+ nested_substitutions = startup_exposure is not None and ("$(" in content or "`" in content)
+ if startup_exposure is None:
+ startup_exposure = [False]
+ startup_work = [4 * len(content) + _ROOT_GLOB_COMMAND_CHARS]
+ for clause_start in _nonempty_shell_clause_starts(content, check_runtime):
check_runtime()
+ if not startup_exposure[0]:
+ startup_exposure[0] = _shell_clause_sets_startup(
+ content, clause_start, check_runtime, startup_work
+ )
+ starts_shell = [False]
recognized, command_string = _command_string_from_clause(
content,
clause_start,
check_runtime,
perl_eval_blocks=perl_eval_blocks,
+ startup_exposure=startup_exposure,
+ starts_shell=starts_shell,
)
+ if nested_substitutions and startup_exposure[0]:
+ # Quoted substitutions can launch shells outside this clause walk.
+ yield None, startup_exposure
+ return
if not recognized:
continue
if python_source is not None:
@@ -2691,7 +3076,274 @@ def _shell_command_strings(
# ``signal.alarm(timeout)`` names no ``timeout`` command.
# Shell text can only be reparsed from a string literal.
continue
- yield command_string
+ # eval shares this shell's settings; a child shell has its own scope.
+ # Keep exposure monotonic across eval, including settings in its body.
+ yield command_string, [False] if starts_shell[0] else startup_exposure
+
+
+def _shell_clause_sets_startup(
+ content: str,
+ start: int,
+ check_runtime: Callable[[], None],
+ remaining_work: list[int],
+) -> bool:
+ """Find explicit startup settings in bounded command-prefix syntax."""
+ source = content[start : start + _ROOT_GLOB_COMMAND_CHARS + 1]
+ cursor = 0
+ mode: str | None = None
+ setter_mode: str | None = None
+ skip_operand = False
+ read_options = True
+ read_array = False
+ printf_options = True
+ setting_namerefs = False
+
+ def startup_target(word: str) -> bool:
+ # A setter can modify an already exported variable even without a
+ # preceding export in this source. Dynamic destinations remain unknown.
+ return bool(re.fullmatch(r"(?:BASH_ENV|ENV)(?:\[.*\])?", word, re.DOTALL)) or any(
+ marker in word
+ for marker in (_DYNAMIC_SHELL_WORD_SENTINEL, _RUNTIME_SHELL_PARAMETER_SENTINEL)
+ )
+
+ for _ in range(_SHELL_COMMAND_STRING_ARGUMENTS):
+ check_runtime()
+ while cursor < len(source):
+ if source.startswith("\\\r\n", cursor):
+ cursor += 3
+ elif source.startswith("\\\n", cursor):
+ cursor += 2
+ elif source[cursor] in " \t":
+ cursor += 1
+ else:
+ break
+ if cursor >= len(source) or source[cursor] in "\r\n;|&(){}#":
+ return cursor > _ROOT_GLOB_COMMAND_CHARS or skip_operand
+ word_start = cursor
+ word, cursor, limited = _next_shell_invocation_word(source, cursor, check_runtime)
+ remaining_work[0] -= cursor - word_start
+ if remaining_work[0] < 0 or limited or cursor > _ROOT_GLOB_COMMAND_CHARS:
+ # Proof work is linear in the containing source, even when clause
+ # starts overlap malformed quoted words or nested expansions.
+ return True
+ if word is None:
+ return False
+ if skip_operand:
+ skip_operand = False
+ continue
+ if mode == "setter-target":
+ if startup_target(word):
+ return True
+ mode = setter_mode
+ continue
+ if mode == "printf":
+ if any(
+ marker in word
+ for marker in (_DYNAMIC_SHELL_WORD_SENTINEL, _RUNTIME_SHELL_PARAMETER_SENTINEL)
+ ):
+ return True
+ if printf_options and word == "--":
+ printf_options = False
+ continue
+ if printf_options and word == "-v":
+ setter_mode = "printf"
+ mode = "setter-target"
+ continue
+ if printf_options and word.startswith("-v"):
+ if startup_target(word[2:]):
+ return True
+ continue
+ # Bash's %n writes to a variable named by its argument, even without
+ # -v. Escaped percent pairs cannot introduce a conversion.
+ return re.search(r"%[-+ #0'.*0-9]*[hjlLtz]*n", word.replace("%%", "")) is not None
+ if mode == "read":
+ if read_options and word == "--":
+ read_options = False
+ continue
+ if read_options and word.startswith("-") and word != "-":
+ for index, option in enumerate(word[1:], 2):
+ if option in "Eers":
+ continue
+ if option not in "adinNptu":
+ return True # Unsupported option arity cannot prove the target.
+ attached = word[index:]
+ if option == "a":
+ read_array = True
+ if attached:
+ if startup_target(attached):
+ return True
+ else:
+ setter_mode = "read"
+ mode = "setter-target"
+ else:
+ skip_operand = not attached
+ break
+ continue
+ read_options = False
+ if not read_array and startup_target(word):
+ return True
+ continue
+ assignment = re.fullmatch(r"(BASH_ENV|ENV)(\+?=)(.*)", word, re.DOTALL)
+ if assignment is not None and (
+ mode in {"env", "setting"} or _SHELL_ASSIGNMENT_PREFIX_RE.match(source, word_start)
+ ):
+ if assignment[2] == "+=" or assignment[3]:
+ return True
+ continue
+ if mode == "setting":
+ destination = word.partition("=")[0].removesuffix("+")
+ if startup_target(destination):
+ return True # Exporting an unknown inherited value is unresolved.
+ if setting_namerefs and re.fullmatch(r"-[A-Za-z]*n[A-Za-z]*", word):
+ # A name reference can redirect a later assignment to a startup
+ # variable without spelling that variable at the write site.
+ return True
+ if word.startswith(("-", "+")) or re.fullmatch(
+ r"[A-Za-z_][A-Za-z0-9_]*(?:=.*)?", word, re.DOTALL
+ ):
+ continue
+ return False
+ if mode == "env":
+ if re.match(r"[A-Za-z_][A-Za-z0-9_]*=", word) is not None:
+ continue
+ if word in {"-i", "--ignore-environment", "--"} or re.fullmatch(
+ r"(?:--unset|--chdir)=.+", word
+ ):
+ continue
+ if word in {"-u", "--unset", "-C", "--chdir"}:
+ skip_operand = True
+ continue
+ if word.startswith("-"):
+ return True
+ mode = None
+ elif mode in {"command", "builtin"}:
+ if word == "--" or (mode == "command" and word == "-p"):
+ continue
+ if word.startswith("-"):
+ return False
+ mode = None
+ command = word.casefold().rsplit("/", 1)[-1]
+ if command in {"export", "declare", "typeset", "readonly", "local"}:
+ mode = "setting"
+ setting_namerefs = command in {"declare", "typeset", "local"}
+ elif command in {"env", "command", "builtin", "read", "printf"}:
+ mode = command
+ elif command in _SHELL_CLAUSE_PREFIX_WORDS or _SHELL_ASSIGNMENT_PREFIX_RE.match(
+ source, word_start
+ ):
+ continue
+ else:
+ return False
+ return True
+
+
+def _shell_redefines_printf(content: str, check_runtime: Callable[[], None]) -> bool:
+ """Do not widen builtin evaluation across an explicit local replacement."""
+ work = 4 * len(content) + _ROOT_GLOB_COMMAND_CHARS
+ for start in _nonempty_shell_clause_starts(content, check_runtime):
+ check_runtime()
+ source = content[start : start + _ROOT_GLOB_COMMAND_CHARS + 1]
+ cursor = 0
+ wrapper: str | None = None
+ for _ in range(8):
+ word_start = cursor
+ while word_start < len(source):
+ if source.startswith("\\\r\n", word_start):
+ word_start += 3
+ elif source.startswith("\\\n", word_start):
+ word_start += 2
+ elif source[word_start] in " \t":
+ word_start += 1
+ else:
+ break
+ word, end, limited = _next_shell_invocation_word(source, cursor, check_runtime)
+ work -= end - cursor
+ cursor = end
+ if work < 0 or limited or cursor > _ROOT_GLOB_COMMAND_CHARS:
+ return True
+ if _SHELL_ASSIGNMENT_PREFIX_RE.match(source, word_start):
+ continue
+ if wrapper is not None:
+ if word == "--" or (wrapper == "command" and word == "-p"):
+ continue
+ if word is None or word.startswith("-"):
+ break
+ wrapper = None
+ if word in {"command", "builtin"}:
+ wrapper = word
+ continue
+ if word in _SHELL_CLAUSE_PREFIX_WORDS or word in {"if", "while", "until"}:
+ continue
+ if word in {"eval", "source", ".", "enable", "trap"}:
+ # Imported/evaluated code, handlers and builtin configuration
+ # can replace printf in this shell.
+ # Retain the earlier narrow decoder across those unknown effects.
+ return True
+ if word == "function":
+ name, end, limited = _next_shell_invocation_word(source, cursor, check_runtime)
+ work -= end - cursor
+ if limited or work < 0 or end > _ROOT_GLOB_COMMAND_CHARS or name == "printf":
+ return True
+ break
+ if word == "alias":
+ for _ in range(_SHELL_COMMAND_STRING_ARGUMENTS):
+ while cursor < len(source) and source[cursor] in " \t":
+ cursor += 1
+ if cursor >= len(source) or source[cursor] in "\r\n;|&()#":
+ break
+ alias, end, limited = _next_shell_invocation_word(source, cursor, check_runtime)
+ work -= end - cursor
+ cursor = end
+ if limited or work < 0 or cursor > _ROOT_GLOB_COMMAND_CHARS:
+ return True
+ if alias is None:
+ break
+ if alias.startswith("printf=") or any(
+ marker in alias
+ for marker in (
+ _DYNAMIC_SHELL_WORD_SENTINEL,
+ _RUNTIME_SHELL_PARAMETER_SENTINEL,
+ )
+ ):
+ return True
+ else:
+ return True
+ break
+ if word != "printf":
+ break
+ # Bash accepts line continuations around the empty parameter list.
+ for expected in "()":
+ while cursor < len(source):
+ if source.startswith("\\\r\n", cursor):
+ cursor += 3
+ elif source.startswith("\\\n", cursor):
+ cursor += 2
+ elif source[cursor] in " \t":
+ cursor += 1
+ else:
+ break
+ if cursor > _ROOT_GLOB_COMMAND_CHARS:
+ return True
+ if cursor == len(source) or source[cursor] != expected:
+ break
+ cursor += 1
+ else:
+ return True
+ break
+ else:
+ return True
+ return False
+
+
+def _nonempty_shell_clause_starts(content: str, check_runtime: Callable[[], None]) -> Iterator[int]:
+ """Avoid reparsing the same trailing command across empty physical lines."""
+ for start in _shell_clause_starts(content, check_runtime):
+ while start < len(content) and content[start] in " \t":
+ if start % 4096 == 0:
+ check_runtime()
+ start += 1
+ if start < len(content) and content[start] not in "\r\n;|&(){}#":
+ yield start
def _shell_clause_starts(
@@ -2799,14 +3451,34 @@ def _command_string_from_clause(
check_runtime: Callable[[], None],
*,
perl_eval_blocks: bool = False,
+ literal_command_spans: list[tuple[int, int]] | None = None,
+ startup_exposure: list[bool] | None = None,
+ starts_shell: list[bool] | None = None,
) -> tuple[bool, str | None]:
"""Resolve a bounded wrapper chain to an ``eval`` or shell command string."""
cursor = start
pending: str | None = None
wrapper_seen = False
+ last_word_start = start
+
+ def observe_assignment(word: str) -> None:
+ if startup_exposure is not None and re.fullmatch(r"(?:BASH_ENV|ENV)=.+", word, re.DOTALL):
+ startup_exposure[0] = True
def next_word() -> tuple[str | None, bool]:
- nonlocal cursor
+ nonlocal cursor, last_word_start
+ last_word_start = cursor
+ while literal_command_spans is not None and last_word_start < len(content):
+ if last_word_start % 4096 == 0:
+ check_runtime()
+ if content[last_word_start].isspace():
+ last_word_start += 1
+ elif content.startswith("\\\r\n", last_word_start):
+ last_word_start += 3
+ elif content.startswith("\\\n", last_word_start):
+ last_word_start += 2
+ else:
+ break
word, cursor, limited = _next_shell_invocation_word(content, cursor, check_runtime)
return word, limited
@@ -2822,8 +3494,20 @@ def resolved_command_string(word: str | None, limited: bool) -> str | None:
return None
return word
+ def commandless_wrapper() -> tuple[bool, None]:
+ # env/command/builtin can end without invoking another command. A
+ # redirection or opening parenthesis may still hide a following one.
+ ended = (
+ cursor >= len(content)
+ or content[cursor] == "#"
+ or _ends_wrapped_clause(content, cursor)
+ )
+ return not ended, None
+
for _ in range(32):
while cursor < len(content) and content[cursor].isspace():
+ if cursor % 4096 == 0:
+ check_runtime()
cursor += 1
redirection = _SHELL_REDIRECTION_PREFIX_RE.match(content, cursor)
if redirection is not None:
@@ -2840,6 +3524,7 @@ def resolved_command_string(word: str | None, limited: bool) -> str | None:
if word is None:
return False, None
if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", word, re.DOTALL) is not None:
+ observe_assignment(word)
continue
command = word.casefold().rsplit("/", 1)[-1]
if command in _SHELL_CLAUSE_PREFIX_WORDS:
@@ -2866,15 +3551,91 @@ def resolved_command_string(word: str | None, limited: bool) -> str | None:
return False, None
if option.startswith("-") and not option.startswith("--"):
if "c" in option[1:]:
+ if starts_shell is not None:
+ starts_shell[0] = True
+ if startup_exposure is not None and startup_exposure[0]:
+ # A startup file can replace the child's builtins.
+ return True, None
command_string, limited = next_word()
- return True, resolved_command_string(command_string, limited)
+ resolved = resolved_command_string(command_string, limited)
+ if (
+ literal_command_spans is not None
+ and resolved is not None
+ and content[last_word_start : last_word_start + 1] == "'"
+ and content[cursor - 1 : cursor] == "'"
+ and cursor - last_word_start == len(resolved) + 2
+ and content[last_word_start + 1 : cursor - 1] == resolved
+ ):
+ # One literal has an exact source-to-body mapping.
+ # Concatenation, expansion and ANSI-C escapes cannot
+ # lend their decoded offsets to the original text.
+ literal_command_spans.append((last_word_start + 1, cursor - 1))
+ return True, resolved
continue
if option.startswith("+") or option.startswith("--"):
continue
return False, None
return True, None
- if command in {"env", "command", "nohup"}:
+ if command == "env":
+ wrapper_seen = True
+ for _ in range(_SHELL_COMMAND_STRING_ARGUMENTS):
+ option, limited = next_word()
+ if limited:
+ return True, None
+ if option is None:
+ return commandless_wrapper()
+ if option == "--":
+ pending, limited = next_word()
+ if limited:
+ return True, None
+ if pending is None:
+ return commandless_wrapper()
+ break
+ if re.match(r"[A-Za-z_][A-Za-z0-9_]*=", option) is not None:
+ observe_assignment(option)
+ continue
+ if option in {"-i", "--ignore-environment"} or re.fullmatch(
+ r"(?:--unset|--chdir)=.+", option
+ ):
+ continue
+ if option in {"-u", "--unset", "-C", "--chdir"}:
+ operand, limited = next_word()
+ if limited or operand is None:
+ return True, None
+ continue
+ if option.startswith("-"):
+ return True, None
+ pending = option
+ break
+ else:
+ return True, None
+ continue
+ if command in {"command", "builtin"}:
+ wrapper_seen = True
+ for _ in range(_SHELL_COMMAND_STRING_ARGUMENTS):
+ option, limited = next_word()
+ if limited:
+ return True, None
+ if option is None:
+ return commandless_wrapper()
+ if option == "--":
+ pending, limited = next_word()
+ if limited:
+ return True, None
+ if pending is None:
+ return commandless_wrapper()
+ break
+ if command == "command" and option == "-p":
+ continue
+ if option.startswith("-"):
+ return False, None
+ pending = option
+ break
+ else:
+ return True, None
+ continue
+ if command == "nohup":
wrapper_seen = True
continue
if command in {"sudo", "nice", "xargs"}:
@@ -3139,6 +3900,23 @@ def comment(self, position: int) -> tuple[int, int] | None:
return None
return owner
+ def starts_string_body(self, position: int) -> bool:
+ """Return whether a candidate starts the value of a proven string token."""
+ owner = self._owner(position)
+ return (
+ owner is not None
+ and self._content[owner[0]] != "#"
+ and position == _python_string_body(self._content, *owner)[0]
+ )
+
+ def is_string_delimiter(self, position: int) -> bool:
+ """Return whether Python owns a string prefix or opening/closing quote."""
+ owner = self._owner(position)
+ if owner is None or self._content[owner[0]] == "#":
+ return False
+ start, end = _python_string_body(self._content, *owner)
+ return not start <= position < end
+
def string_value(self, start: int, end: int) -> tuple[str, int, int] | None:
"""Return the proven string value that holds a shell word, and its span.
@@ -3329,6 +4107,7 @@ def _bounded_shell_tokens(
body_start: int,
*,
check_runtime: Callable[[], None] | None = None,
+ quoted_printf_allowed: bool = True,
) -> tuple[tuple[_ShellToken, ...], int, bool]:
"""Return argument words from one security-view-bounded shell command."""
runtime_check = check_runtime or (lambda: None)
@@ -3468,6 +4247,7 @@ def flush(*, complete: bool = True) -> None:
content,
cursor,
substitution_end,
+ quoted_result=quoted_printf_allowed,
)
parse_limited = parse_limited or (
static_value is None
@@ -5128,6 +5908,7 @@ def _tm1_candidates(
*,
direct_shell_only: bool = False,
shell_content: str | None = None,
+ check_runtime: Callable[[], None],
) -> Iterator[tuple[int, int, str, float, bool, int | None, int | None, str | None]]:
def direct_matches(
pattern_index: int,
@@ -5235,6 +6016,7 @@ def destructive_candidates(
seen_commands: set[tuple[int, int]] = set()
covered_until = 0
for command_start, body_start in _destructive_command_words(candidate_content):
+ check_runtime()
if command_start < covered_until:
continue
command_key = (command_start, body_start)
@@ -5246,12 +6028,15 @@ def destructive_candidates(
# character-by-character shell parse for every explanatory ``rm`` noun.
if _is_root_glob_documentation(candidate_content, command_start, body_start):
continue
- tokens, command_end, _ = _bounded_shell_tokens(
+ tokens, command_end, exhausted = _bounded_shell_tokens(
candidate_content,
command_start,
body_start,
+ check_runtime=check_runtime,
)
- covered_until = max(covered_until, command_end)
+ # An unresolved quote cannot own and hide later executable candidates.
+ if not exhausted:
+ covered_until = max(covered_until, command_end)
command = candidate_content[command_start:command_end]
if _has_destructive_root_glob(tokens) or _has_destructive_root_path(tokens):
yield command_start, command_end, command, 0.9, False, None, None, None
@@ -5414,6 +6199,7 @@ def _markdown_shell_text(
*,
complete_context: bool = True,
word_apostrophes: bool = False,
+ shell_fence_ranges: list[tuple[int, int]] | None = None,
hardware_apostrophe_ambiguity: list[bool] | None = None,
) -> str:
"""Mask Markdown delimiters while retaining code and exact source offsets.
@@ -5423,6 +6209,8 @@ def _markdown_shell_text(
inside their bodies. Pair equal-length runs in linear time.
``word_apostrophes`` selects the prose reading, which also masks the
possessives and contractions of plain prose lines.
+ ``shell_fence_ranges`` receives closed, explicitly labelled shell bodies
+ whose quote ownership can be checked independently of surrounding prose.
"""
output = list(content)
prose_apostrophes: set[int] = set()
@@ -5467,6 +6255,7 @@ def mask_inline_delimiters() -> None:
fence_literal_ownership = True
fence_has_literal_apostrophe = False
fence_has_nonliteral_apostrophe = False
+ shell_fence_start: int | None = None
quoted_block = False
html_end: str | None = None
paragraph_open = False
@@ -5560,6 +6349,9 @@ def mask_inline_delimiters() -> None:
begin, end = closing_fence.span(1)
body_start = offset + len(stripped) - len(fence_body)
output[body_start + begin : body_start + end] = " " * (end - begin)
+ if shell_fence_start is not None and shell_fence_ranges is not None:
+ shell_fence_ranges.append((shell_fence_start, offset))
+ shell_fence_start = None
fence = None
fence_literal_apostrophes.clear()
fence_literal_ownership = True
@@ -5659,6 +6451,14 @@ def mask_inline_delimiters() -> None:
fence_has_literal_apostrophe = False
fence_has_nonliteral_apostrophe = False
output[offset + prefix + begin : offset + prefix + end] = " " * (end - begin)
+ if (
+ complete_context
+ and shell_fence_ranges is not None
+ and not has_list_marker
+ and not paragraph_in_list
+ and stripped[prefix + end :].strip() in {"bash", "sh", "shell"}
+ ):
+ shell_fence_start = offset + len(line)
else:
minimum_indent = (
column if has_list_marker else paragraph_list_indent if paragraph_in_list else 0
@@ -5839,7 +6639,9 @@ def _has_bounded_parse_exhaustion(
# -replace values have their own narrowly proven expression context.
structural_quote_closers = None
structural_quote_openers = None
+ shell_fence_ranges: list[tuple[int, int]] = []
json_strings: list[tuple[int, int]] = []
+ json_operand_spans: list[tuple[int, int]] = []
python_source = None
javascript_source = None
if file_type == "perl" and complete_context:
@@ -5853,17 +6655,24 @@ def _has_bounded_parse_exhaustion(
# Interpolated templates in files with shell calls retain conservative
# bounds because this analyzer does not prove their data flow.
javascript_source = _JavaScriptTemplateOwnership(content, check_runtime)
+ if file_type in {"json", "markdown"} and complete_context:
+ json_strings = validated_json_string_spans(content, check_runtime)
+ structural_quote_closers = {end - 1 for _, end in json_strings}
+ structural_quote_openers = {start for start, _ in json_strings}
+ # Raw substitution delimiters must survive Markdown's inline projection.
+ json_operand_spans = [
+ (start, end)
+ for start, end in json_strings
+ if "$(" not in content[start:end] and "`" not in content[start:end]
+ ]
if file_type == "markdown":
- if complete_context:
- json_strings = validated_json_string_spans(content, check_runtime)
- structural_quote_closers = {end - 1 for _, end in json_strings}
- structural_quote_openers = {start for start, _ in json_strings}
hardware_apostrophe_ambiguity = [False]
content = _markdown_shell_text(
content,
check_runtime,
complete_context=complete_context,
word_apostrophes=word_apostrophes,
+ shell_fence_ranges=shell_fence_ranges,
hardware_apostrophe_ambiguity=hardware_apostrophe_ambiguity,
)
if hardware_apostrophe_ambiguity[0]:
@@ -5876,6 +6685,30 @@ def _has_bounded_parse_exhaustion(
if file_type == "javascript" and complete_context
else content
)
+ if file_type == "python" and complete_context:
+ shell_word_content = python_browser_template_shell_text(content, check_runtime)
+ quote_limits = None
+ if file_type == "shell" and complete_context:
+ # A closing delimiter in proven shell source cannot open a new
+ # command word. Nested substitutions retain their own lexical walk,
+ # so their executable candidates remain independently inspected.
+ structural_quote_closers = set()
+ quote_limits = _shell_quoted_candidate_limits(
+ content, check_runtime, quote_closers=structural_quote_closers
+ )
+ elif file_type == "markdown" and complete_context:
+ quote_limits = {}
+ for start, end in shell_fence_ranges:
+ check_runtime()
+ closers: set[int] = set()
+ limits = _shell_quoted_candidate_limits(
+ content[start:end], check_runtime, quote_closers=closers
+ )
+ quote_limits.update(
+ {start + position: start + limit for position, limit in limits.items()}
+ )
+ assert structural_quote_closers is not None
+ structural_quote_closers.update(start + position for position in closers)
# Perl ``eval BLOCK`` is recognized from its clause-initial ``eval`` and the
# adjacent brace alone, so unlike quote ownership it needs no full context.
if _has_shell_command_word_exhaustion(
@@ -5886,14 +6719,11 @@ def _has_bounded_parse_exhaustion(
python_source=python_source,
javascript_source=javascript_source,
perl_eval_blocks=file_type == "perl",
- quote_limits=(
- _shell_quoted_candidate_limits(content, check_runtime)
- if file_type == "shell" and complete_context
- else None
- ),
+ quote_limits=quote_limits,
file_type=file_type,
):
return True
+ json_starts = [start for start, _ in json_operand_spans]
covered_until = 0
for command_start, body_start in _destructive_command_words(content):
check_runtime()
@@ -5910,6 +6740,28 @@ def _has_bounded_parse_exhaustion(
body_start,
check_runtime=check_runtime,
)
+ if exhausted:
+ word = content[command_start:body_start]
+ boundary = quote_limits.get(command_start) if quote_limits is not None else None
+ index = bisect_right(json_starts, command_start) - 1
+ if boundary is None and index >= 0:
+ begin, end = json_operand_spans[index]
+ if begin < command_start < body_start < end:
+ boundary = end - 1
+ if (
+ boundary is not None
+ and not any(character in word for character in "$`'\"\\")
+ and word.casefold().rsplit("/", 1)[-1] in _DESTRUCTIVE_COMMAND_BASENAMES
+ ):
+ # A proven enclosing closer terminates operands; reconstructed
+ # command words retain their original uncertainty and bounds.
+ tokens, command_end, exhausted = _bounded_shell_tokens(
+ content[command_start:boundary],
+ 0,
+ body_start - command_start,
+ check_runtime=check_runtime,
+ )
+ command_end += command_start
covered_until = max(covered_until, command_end)
if exhausted or _has_unsupported_brace_expansion(tokens):
return True
@@ -6085,6 +6937,7 @@ def ctx(start: int) -> str:
content,
direct_shell_only=_direct_shell_only,
shell_content=shell_content,
+ check_runtime=runtime_check,
):
runtime_check()
variable_match = variable_matches.get((match_start, match_end))
diff --git a/src/skillspector/python_tokens.py b/src/skillspector/python_tokens.py
index a8beee090..0f17ad4f4 100644
--- a/src/skillspector/python_tokens.py
+++ b/src/skillspector/python_tokens.py
@@ -20,9 +20,11 @@
from bisect import bisect_right
from collections import OrderedDict
from collections.abc import Callable
+from html.parser import HTMLParser
from threading import Lock
-from skillspector.python_ast import MAX_PYTHON_AST_SOURCE_CHARS
+from skillspector.javascript_tokens import browser_escaped_list_shell_text
+from skillspector.python_ast import MAX_PYTHON_AST_SOURCE_CHARS, parse_python_source
# The tokenizer and the compiler disagree about a carriage return that does
# not start CRLF, so such source never acquires token ownership.
@@ -75,6 +77,9 @@
OrderedDict()
)
+_HTML_SCRIPT_OPEN_RE = re.compile(r"", re.IGNORECASE)
+
def python_literal_spans(
content: str,
@@ -182,6 +187,681 @@ def offset(position: tuple[int, int]) -> int:
return tuple(starts), tuple(ends)
+def python_fstring_literal_spans(
+ content: str,
+ start: int,
+ end: int,
+ check_runtime: Callable[[], None],
+) -> PythonLiteralSpans | None:
+ """Return exact constant-text runs of one proven outer f-string token.
+
+ Replacement expressions and their format specifications never gain literal
+ ownership. Doubled braces keep both original source bytes in their run.
+ """
+ owners = python_literal_spans(content, check_runtime)
+ if owners is None:
+ return None
+ owner_index = bisect_right(owners[0], start) - 1
+ if owner_index < 0 or (owners[0][owner_index], owners[1][owner_index]) != (start, end):
+ return None
+ source = content[start:end]
+ line_starts = [0, *(match.end() for match in re.finditer("\n", source))]
+ starts: list[int] = []
+ ends: list[int] = []
+ template_depth = field_depth = 0
+ complete = False
+
+ def offset(position: tuple[int, int]) -> int:
+ return start + line_starts[position[0] - 1] + position[1]
+
+ try:
+ for index, token in enumerate(tokenize.generate_tokens(io.StringIO(source).readline)):
+ if index % 256 == 0:
+ check_runtime()
+ if token.type in _PYTHON_TEMPLATE_STRING_STARTS:
+ if not template_depth and (
+ token.type != tokenize.FSTRING_START or offset(token.start) != start
+ ):
+ return None
+ template_depth += 1
+ elif token.type in _PYTHON_TEMPLATE_STRING_ENDS:
+ template_depth -= 1
+ if template_depth == 0:
+ complete = offset(token.end) == end and field_depth == 0
+ elif template_depth == 1 and token.type == tokenize.OP:
+ if token.string == "{":
+ field_depth += 1
+ elif token.string == "}":
+ field_depth -= 1
+ elif template_depth == 1 and not field_depth and token.type == tokenize.FSTRING_MIDDLE:
+ literal_start, literal_end = offset(token.start), offset(token.end)
+ if (
+ token.string.endswith(("{", "}"))
+ and content[literal_end : literal_end + 1] == token.string[-1]
+ ):
+ literal_end += 1
+ raw = content[literal_start:literal_end]
+ if raw.replace("{{", "{").replace("}}", "}") != token.string:
+ return None
+ if ends and ends[-1] == literal_start:
+ ends[-1] = literal_end
+ else:
+ starts.append(literal_start)
+ ends.append(literal_end)
+ except (tokenize.TokenError, SyntaxError, ValueError, IndexError):
+ return None
+ if not complete or template_depth or field_depth:
+ return None
+ return tuple(starts), tuple(ends)
+
+
+def _browser_html_fields(
+ literal: ast.JoinedStr,
+ field_kind: Callable[[ast.AST], str | None],
+ check_runtime: Callable[[], None],
+) -> bool:
+ """Accept one ordinary HTML document with inert, escaped Python fields."""
+ marker = "__skillspector_python_html_field_"
+ fields: dict[str, str | None] = {}
+ chunks: list[str] = []
+ for index, value in enumerate(literal.values):
+ check_runtime()
+ if isinstance(value, ast.Constant) and isinstance(value.value, str):
+ if marker in value.value:
+ return False
+ chunks.append(value.value)
+ elif (
+ isinstance(value, ast.FormattedValue)
+ and value.conversion == -1
+ and value.format_spec is None
+ ):
+ key = f"{marker}{index}__"
+ fields[key] = field_kind(value.value)
+ chunks.append(key)
+ else:
+ return False
+ if any(kind is None for kind in fields.values()):
+ return False
+
+ class Document(HTMLParser):
+ valid = True
+ context = ""
+ executable_scripts = 0
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ check_runtime()
+ if tag not in {
+ "html",
+ "head",
+ "title",
+ "meta",
+ "style",
+ "body",
+ "div",
+ "span",
+ "aside",
+ "main",
+ "section",
+ "input",
+ "button",
+ "strong",
+ "ul",
+ "li",
+ "h1",
+ "h2",
+ "h3",
+ "p",
+ "script",
+ }:
+ self.valid = False
+ names = [name for name, _ in attrs]
+ if len(names) != len(set(names)):
+ self.valid = False
+ for name, value in attrs:
+ if name not in {
+ "id",
+ "class",
+ "title",
+ "lang",
+ "charset",
+ "name",
+ "content",
+ "type",
+ "placeholder",
+ } and not name.startswith("data-"):
+ self.valid = False
+ if value is not None and marker in value:
+ self.valid = False
+ attributes = dict(attrs)
+ if tag == "meta" and not (
+ set(attributes) == {"charset"}
+ or attributes.get("name") == "viewport"
+ and set(attributes) == {"name", "content"}
+ ):
+ self.valid = False
+ if tag == "script":
+ if not attrs:
+ self.executable_scripts += 1
+ self.context = "script"
+ elif attributes.get("type") == "application/json" and set(attributes) <= {
+ "id",
+ "type",
+ }:
+ self.context = "json"
+ else:
+ self.valid = False
+ elif tag in {"style", "title"}:
+ self.context = tag
+
+ def handle_endtag(self, tag: str) -> None:
+ if tag in {"script", "style", "title"}:
+ self.context = ""
+
+ def handle_data(self, data: str) -> None:
+ check_runtime()
+ if self.context == "style" and re.search(
+ r"(?:@import|url\s*\(|expression\s*\(|\\)", data, re.IGNORECASE
+ ):
+ self.valid = False
+ if marker not in data:
+ return
+ if self.context == "json":
+ if fields.pop(data.strip(), None) != "json":
+ self.valid = False
+ elif self.context in {"script", "style"}:
+ self.valid = False
+ else:
+ for key in re.findall(marker + r"[0-9]+__", data):
+ if fields.pop(key, None) != "text":
+ self.valid = False
+
+ def handle_comment(self, data: str) -> None:
+ if marker in data:
+ self.valid = False
+
+ parser = Document(convert_charrefs=False)
+ parser.feed("".join(chunks))
+ parser.close()
+ return parser.valid and parser.executable_scripts == 1 and not fields
+
+
+def _python_browser_strings(
+ content: str, check_runtime: Callable[[], None]
+) -> set[tuple[int, int]]:
+ """Prove explicit pathlib consumers and escaped HTML field boundaries.
+
+ Only direct write_text arguments and direct returns from unaliased local
+ functions used exclusively by those writes qualify. Unknown consumers,
+ rebinding, mutation and unsupported scope forms retain the shell reading.
+ Ordinary pathlib/JSON/text protocols are assumed, as with Node assertions;
+ explicit local protocol overrides fall back. This is not a proof against
+ arbitrary runtime getters or monkeypatching outside the inspected source.
+ """
+ with _PYTHON_OWNERSHIP_PARSE_LOCK, warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ parsed = parse_python_source(content, "")
+ tree = parsed.tree
+ if tree is None:
+ return set()
+ parents: dict[ast.AST, ast.AST] = {}
+ scopes: dict[ast.AST, ast.AST] = {}
+ bindings: dict[tuple[ast.AST, str], list[ast.AST]] = {}
+ returns: dict[ast.AST, list[ast.Return]] = {}
+ references: dict[str, list[ast.Name]] = {}
+ nodes: list[ast.AST] = []
+ pending: list[tuple[ast.AST, ast.AST]] = [(tree, tree)]
+ while pending:
+ node, scope = pending.pop()
+ if len(nodes) >= 100_000:
+ return set()
+ if len(nodes) % 256 == 0:
+ check_runtime()
+ nodes.append(node)
+ scopes[node] = scope
+ if isinstance(node, ast.Return):
+ returns.setdefault(scope, []).append(node)
+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load):
+ references.setdefault(node.id, []).append(node)
+ names: list[str] = []
+ if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
+ names = [node.id]
+ elif isinstance(node, ast.arg):
+ names = [node.arg]
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ names = [node.name]
+ if node.name in {
+ "replace",
+ "__str__",
+ "__format__",
+ "__fspath__",
+ "__html__",
+ "__getattribute__",
+ "__getattr__",
+ "__getitem__",
+ }:
+ return set()
+ elif isinstance(node, (ast.Import, ast.ImportFrom)):
+ names = [alias.asname or alias.name.split(".")[0] for alias in node.names]
+ elif isinstance(node, ast.ExceptHandler) and node.name:
+ names = [node.name]
+ elif isinstance(node, (ast.Global, ast.Nonlocal, ast.NamedExpr, ast.Match)):
+ return set()
+ if (
+ "*" in names
+ or isinstance(node, ast.Attribute)
+ and isinstance(node.ctx, (ast.Store, ast.Del))
+ and node.attr
+ in {
+ "Path",
+ "write_text",
+ "resolve",
+ "cwd",
+ "home",
+ "escape",
+ "dumps",
+ "replace",
+ "__code__",
+ "__defaults__",
+ "__kwdefaults__",
+ }
+ ):
+ return set()
+ if (
+ isinstance(node, ast.Name)
+ and isinstance(node.ctx, ast.Load)
+ and node.id in {"eval", "exec", "setattr", "delattr", "globals", "locals", "vars"}
+ or isinstance(node, ast.Attribute)
+ and node.attr in {"setattr", "delattr", "__dict__", "__setattr__"}
+ ):
+ return set()
+ for name in names:
+ if name in {
+ "replace",
+ "__str__",
+ "__format__",
+ "__fspath__",
+ "__html__",
+ "__getattribute__",
+ "__getattr__",
+ "__getitem__",
+ }:
+ return set()
+ bindings.setdefault((scope, name), []).append(node)
+ child_scope = (
+ node
+ if isinstance(
+ node,
+ (
+ ast.FunctionDef,
+ ast.AsyncFunctionDef,
+ ast.ClassDef,
+ ast.Lambda,
+ ast.ListComp,
+ ast.SetComp,
+ ast.DictComp,
+ ast.GeneratorExp,
+ ),
+ )
+ else scope
+ )
+ for child in ast.iter_child_nodes(node):
+ parents[child] = node
+ pending.append((child, child_scope))
+
+ budget = 100_000
+
+ def binding(name: ast.Name) -> ast.AST | None:
+ nonlocal budget
+ budget -= 1
+ if budget <= 0:
+ return None
+ if budget % 256 == 0:
+ check_runtime()
+ scope = scopes[name]
+ first = True
+ while True:
+ # Methods do not close over class-local bindings. Function closures
+ # and comprehension bindings do shadow a module import.
+ candidates = (
+ []
+ if not first and isinstance(scope, ast.ClassDef)
+ else bindings.get((scope, name.id), [])
+ )
+ if candidates:
+ return candidates[0] if len(candidates) == 1 else None
+ if scope is tree:
+ return None
+ scope = scopes[scope]
+ first = False
+ budget -= 1
+ if budget <= 0:
+ return None
+
+ def imported(node: ast.AST, module: str, name: str | None = None) -> bool:
+ if not isinstance(node, ast.Name):
+ return False
+ owner = binding(node)
+ if isinstance(owner, ast.Import) and name is None:
+ return any(
+ alias.name == module and alias.asname is None and node.id == module
+ for alias in owner.names
+ )
+ if isinstance(owner, ast.ImportFrom) and owner.module == module and owner.level == 0:
+ return any(
+ alias.name == name and alias.asname is None and node.id == name
+ for alias in owner.names
+ )
+ return False
+
+ def assigned(node: ast.Name) -> ast.AST | None:
+ owner = binding(node)
+ parent = parents.get(owner) if owner is not None else None
+ if isinstance(parent, ast.Assign) and parent.targets == [owner]:
+ return parent.value
+ if isinstance(parent, ast.AnnAssign) and parent.target is owner:
+ return parent.value
+ return None
+
+ def annotation(node: ast.AST) -> bool:
+ parent = parents.get(node)
+ return bool(
+ isinstance(parent, ast.arg)
+ and parent.annotation is node
+ or isinstance(parent, (ast.FunctionDef, ast.AsyncFunctionDef))
+ and parent.returns is node
+ or isinstance(parent, ast.AnnAssign)
+ and parent.annotation is node
+ )
+
+ # Audit every explicit use of the trusted imports, including aliases that
+ # could let another call replace a constructor or escaping implementation.
+ for name in ("Path", "json", "escape"):
+ for reference in references.get(name, []):
+ check_runtime()
+ if annotation(reference):
+ continue
+ module, member = {
+ "Path": ("pathlib", "Path"),
+ "json": ("json", None),
+ "escape": ("html", "escape"),
+ }[name]
+ if not imported(reference, module, member):
+ return set()
+ parent = parents.get(reference)
+ if (
+ name in {"Path", "escape"}
+ and isinstance(parent, ast.Call)
+ and parent.func is reference
+ ):
+ continue
+ if isinstance(parent, ast.Attribute) and isinstance(parent.ctx, ast.Load):
+ allowed = {"Path": {"cwd", "home"}, "json": {"loads", "dumps"}, "escape": set()}[
+ name
+ ]
+ call = parents.get(parent)
+ if parent.attr in allowed and isinstance(call, ast.Call) and call.func is parent:
+ continue
+ return set()
+
+ path_bindings: set[ast.AST] = set()
+ path_functions: set[ast.FunctionDef] = set()
+
+ def path(node: ast.AST, depth: int = 0) -> bool:
+ if depth >= 16 or budget <= 0:
+ return False
+ if isinstance(node, ast.Name):
+ value = assigned(node)
+ owner = binding(node)
+ if owner is not None:
+ path_bindings.add(owner)
+ return value is not None and path(value, depth + 1)
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div):
+ # An arbitrary RHS can run __rtruediv__ and replace the receiver.
+ native_text = isinstance(node.right, ast.JoinedStr) or (
+ isinstance(node.right, ast.Constant) and isinstance(node.right.value, str)
+ )
+ return native_text and path(node.left, depth + 1)
+ if not isinstance(node, ast.Call):
+ return False
+ if imported(node.func, "pathlib", "Path"):
+ return True
+ if isinstance(node.func, ast.Attribute):
+ if node.func.attr in {"cwd", "home"} and imported(node.func.value, "pathlib", "Path"):
+ return not node.args and not node.keywords
+ if node.func.attr == "resolve":
+ return path(node.func.value, depth + 1)
+ if isinstance(node.func, ast.Name):
+ function = binding(node.func)
+ if (
+ isinstance(function, ast.FunctionDef)
+ and scopes[function] is tree
+ and not function.decorator_list
+ ):
+ path_functions.add(function)
+ values = returns.get(function, [])
+ return bool(values) and all(
+ part.value is not None and path(part.value, depth + 1) for part in values
+ )
+ return False
+
+ def written(node: ast.AST) -> bool:
+ scope = scopes[node]
+ if scope is not tree and (
+ not isinstance(scope, ast.FunctionDef) or scopes[scope] is not tree
+ ):
+ return False
+ parent = parents.get(node)
+ return bool(
+ isinstance(parent, ast.Call)
+ and len(parent.args) == 1
+ and parent.args[0] is node
+ and len(parent.keywords) == 1
+ and parent.keywords[0].arg == "encoding"
+ and isinstance(parent.keywords[0].value, ast.Constant)
+ and parent.keywords[0].value.value in {"utf-8", "utf8", "UTF-8", "UTF8"}
+ and isinstance(parent.func, ast.Attribute)
+ and parent.func.attr == "write_text"
+ and path(parent.func.value)
+ )
+
+ def path_use(node: ast.AST, depth: int = 0) -> bool:
+ """A selected Path receiver cannot escape through a bound method/property."""
+ if depth >= 16:
+ return False
+ parent = parents.get(node)
+ if isinstance(parent, ast.FormattedValue):
+ return True
+ if not isinstance(parent, ast.Attribute) or not isinstance(parent.ctx, ast.Load):
+ return False
+ if parent.attr == "parent":
+ return path_use(parent, depth + 1)
+ call = parents.get(parent)
+ if not isinstance(call, ast.Call) or call.func is not parent:
+ return False
+ if parent.attr in {"write_text", "mkdir", "as_uri"}:
+ # These return native scalar/None values, not the selected receiver.
+ return True
+ if parent.attr == "resolve" and not call.args and not call.keywords:
+ return path_use(call, depth + 1)
+ return False
+
+ def field_kind(node: ast.AST, depth: int = 0) -> str | None:
+ if depth >= 16 or budget <= 0:
+ return None
+ if isinstance(node, ast.Name):
+ value = assigned(node)
+ return None if value is None else field_kind(value, depth + 1)
+ if not isinstance(node, ast.Call):
+ return None
+ if (
+ imported(node.func, "html", "escape")
+ and len(node.args) == 1
+ and not node.keywords
+ and isinstance(node.args[0], (ast.Constant, ast.Name, ast.Subscript))
+ ):
+ return "text"
+ if (
+ isinstance(node.func, ast.Attribute)
+ and node.func.attr == "replace"
+ and len(node.args) == 2
+ and not node.keywords
+ ):
+ before, after = node.args
+ value = node.func.value
+ if (
+ isinstance(before, ast.Constant)
+ and before.value == ""
+ and isinstance(after, ast.Constant)
+ and after.value == "<\\/"
+ and isinstance(value, ast.Call)
+ and isinstance(value.func, ast.Attribute)
+ and value.func.attr == "dumps"
+ and imported(value.func.value, "json")
+ and len(value.args) == 1
+ and not isinstance(value.args[0], ast.Starred)
+ and all(
+ keyword.arg == "separators"
+ and isinstance(keyword.value, ast.Tuple)
+ and len(keyword.value.elts) == 2
+ and all(
+ isinstance(item, ast.Constant) and isinstance(item.value, str)
+ for item in keyword.value.elts
+ )
+ for keyword in value.keywords
+ )
+ ):
+ return "json"
+ return None
+
+ result: set[tuple[int, int]] = set()
+ for node in nodes:
+ if (
+ not isinstance(node, ast.JoinedStr)
+ or not node.values
+ or not isinstance(node.values[0], ast.Constant)
+ or not isinstance(node.values[0].value, str)
+ or not node.values[0].value.lstrip().lower().startswith("")
+ ):
+ continue
+ check_runtime()
+ consumed = written(node)
+ parent = parents.get(node)
+ function = scopes[node]
+ if (
+ isinstance(parent, ast.Return)
+ and isinstance(function, ast.FunctionDef)
+ and scopes[function] is tree
+ and not function.decorator_list
+ and returns.get(function) == [parent]
+ ):
+ uses = references.get(function.name, [])
+ consumed = bool(uses) and all(
+ binding(part) is function
+ and isinstance(parents[part], ast.Call)
+ and parents[part].func is part
+ and written(parents[part])
+ for part in uses
+ )
+ if consumed and _browser_html_fields(node, field_kind, check_runtime):
+ start_column = parsed.character_column(node.lineno, node.col_offset)
+ end_column = parsed.character_column(node.end_lineno, node.end_col_offset)
+ if start_column is not None and end_column is not None:
+ result.add(
+ (
+ parsed.line_character_starts[node.lineno - 1] + start_column,
+ parsed.line_character_starts[node.end_lineno - 1] + end_column,
+ )
+ )
+ for reference_nodes in references.values():
+ for reference in reference_nodes:
+ check_runtime()
+ owner = binding(reference)
+ if budget <= 0:
+ return set()
+ parent = parents.get(reference)
+ if owner in path_functions:
+ if not isinstance(parent, ast.Call) or parent.func is not reference:
+ return set()
+ if owner in path_bindings:
+ if not path_use(reference):
+ return set()
+ return result if budget > 0 else set()
+
+
+def python_browser_template_shell_text(content: str, check_runtime: Callable[[], None]) -> str:
+ """Project display delimiters in constant browser scripts of HTML f-strings.
+
+ Neither Python replacement fields nor escape decoding may establish a
+ script boundary. Original lengths and every non-delimiter byte survive.
+ """
+ if "`" not in content or "" not in content.lower():
+ return content
+ owners = python_literal_spans(content, check_runtime)
+ if owners is None:
+ return content
+ browser_strings = _python_browser_strings(content, check_runtime)
+ if not browser_strings:
+ return content
+ output: list[str] | None = None
+ for start, end in zip(*owners, strict=True):
+ check_runtime()
+ if (start, end) not in browser_strings:
+ continue
+ if content[start] == "#":
+ continue
+ body_start, body_end = python_string_body(content, start, end)
+ if "f" not in content[start:body_start].lower():
+ continue
+ body = content[body_start:body_end]
+ if not body.lstrip().lower().startswith(
+ ""
+ ) or not body.rstrip().lower().endswith("