diff --git a/src/skillspector/javascript_tokens.py b/src/skillspector/javascript_tokens.py index 8e0eced04..adec39b95 100644 --- a/src/skillspector/javascript_tokens.py +++ b/src/skillspector/javascript_tokens.py @@ -14,6 +14,7 @@ from bisect import bisect_left from collections.abc import Callable, Iterator from dataclasses import dataclass +from html.parser import HTMLParser from typing import Any import esprima # type: ignore[import-untyped] @@ -24,6 +25,7 @@ _MAX_TOKENS = 100_000 _MAX_NESTING = 128 _LINE_ENDS = "\r\n\u2028\u2029" +_BrowserBinding = tuple[str, int, str] _IDENTIFIER = re.compile(r"[A-Za-z_$][A-Za-z0-9_$]*") _TEMPLATE_VALUE = re.compile( r"\s*[A-Za-z_$][A-Za-z0-9_$]*(?:\s*\.\s*[A-Za-z_$][A-Za-z0-9_$]*)*\s*\Z" @@ -387,6 +389,1089 @@ def assertion_template_shell_text(content: str, check_runtime: Callable[[], None return "".join(output) +def _nested_template_code( + content: str, check_runtime: Callable[[], None] +) -> tuple[str, list[tuple[int, int, tuple[int, ...]]], set[int]] | None: + """Expose template expressions to the existing lexer, retaining exact offsets.""" + if len(content) > _MAX_SOURCE_CHARS: + return None + output = list(content) + templates: list[tuple[int, int, tuple[int, ...]]] = [] + expression_openers: set[int] = set() + next_check = 0 + + def check(position: int) -> None: + nonlocal next_check + if position >= next_check: + check_runtime() + next_check = position + 4096 + + def code(cursor: int, depth: int, *, interpolation: bool = False) -> int | None: + if depth >= _MAX_NESTING: + return None + braces = 0 + while cursor < len(content): + check(cursor) + character = content[cursor] + if character in "'\"": + quote = character + cursor += 1 + while cursor < len(content) and content[cursor] != quote: + check(cursor) + if content[cursor] in _LINE_ENDS: + return None + cursor += 2 if content[cursor] == "\\" else 1 + if cursor >= len(content): + return None + elif content.startswith("//", cursor): + while cursor < len(content) and content[cursor] not in _LINE_ENDS: + check(cursor) + cursor += 1 + continue + elif content.startswith("/*", cursor): + cursor += 2 + while cursor < len(content) and not content.startswith("*/", cursor): + check(cursor) + cursor += 1 + if cursor == len(content): + return None + cursor += 2 + continue + elif character == "/": + # Regexp/division ambiguity cannot establish nested ownership. + return None + elif character == "`": + end = template(cursor, depth + 1) + if end is None: + return None + cursor = end + continue + elif character == "{": + braces += 1 + if braces + depth >= _MAX_NESTING: + return None + elif character == "}": + if not braces and interpolation: + return cursor + braces -= 1 + if braces < 0: + return None + cursor += 1 + return None if interpolation or braces else cursor + + def template(start: int, depth: int) -> int | None: + if depth >= _MAX_NESTING or len(templates) >= _MAX_TOKENS: + return None + delimiters = [start] + expression_openers.add(start) + output[start] = "(" + cursor = start + 1 + while cursor < len(content): + check(cursor) + if content[cursor] == "`": + output[cursor] = ")" + delimiters.append(cursor) + templates.append((start, cursor + 1, tuple(delimiters))) + return cursor + 1 + if content.startswith("${", cursor): + end = code(cursor + 2, depth + 1, interpolation=True) + if end is None: + return None + expression_openers.add(cursor) + output[cursor], output[cursor + 1], output[end] = "(", " ", ")" + delimiters.extend((cursor, cursor + 1, end)) + cursor = end + 1 + continue + if content[cursor] == "\\": + # Decoding an escape needs a separate source/value proof. + return None + if content[cursor] not in _LINE_ENDS: + output[cursor] = " " + cursor += 1 + return None + + if code(0, 0) is None: + return None + check_runtime() + return "".join(output), templates, expression_openers + + +def browser_escaped_list_shell_text(content: str, check_runtime: Callable[[], None]) -> str: + """Own only JSON-backed list values passed through a local escaping formatter. + + This proves explicit source consumers, not that a browser program is inert. + As with assertion messages, ordinary built-ins are assumed; arbitrary runtime + getters/monkeypatching are outside this proof. Explicit aliases, mutation, + shadowing, unknown consumers, and executable expressions never gain ownership. + Only backticks change. Raw findings and interpolation bytes remain untouched. + """ + if "JSON.parse" not in content or "replaceAll" not in content or "innerHTML" not in content: + return content + nested = _nested_template_code(content, check_runtime) + if nested is None: + return content + code, templates, expression_openers = nested + parsed = _tokens(code, check_runtime) + if parsed is None: + return content + tokens, pairs = parsed + values = [code[token.start : token.end] for token in tokens] + positions = [token.start for token in tokens] + reverse = {end: start for start, end in pairs.items()} + template_spans = { + bisect_left(positions, start): (bisect_left(positions, end), delimiters) + for start, end, delimiters in templates + } + for start in template_spans: + if start and ( + tokens[start - 1].kind == "identifier" + and values[start - 1] != "return" + or values[start - 1] in {")", "]"} + ): + # A tag receives a strings array before its interpolation values. + # Projecting it as an ordinary call would shift parameter sources. + return content + fields: dict[int, int] = {} + for start, (_, delimiters) in template_spans.items(): + for position in delimiters[1:-1:3]: + fields[bisect_left(positions, position)] = start + budget = _MAX_TOKENS * 4 + + def step(amount: int = 1) -> bool: + nonlocal budget + previous = budget + budget -= amount + if budget // 256 != previous // 256: + check_runtime() + return budget > 0 + + def parts(start: int, end: int) -> list[tuple[int, int]]: + result = [] + begin = cursor = start + while cursor < end and step(): + if values[cursor] == ",": + result.append((begin, cursor)) + begin = cursor + 1 + cursor = pairs.get(cursor, cursor) + 1 + if begin < end: + result.append((begin, end)) + return result + + def arguments(opening: int) -> list[tuple[int, int]]: + return parts(opening + 1, pairs[opening]) + + enclosing: list[int | None] = [] + blocks: list[tuple[int, int]] = [] + stack: list[int] = [] + block_stack: list[int] = [] + for index, value in enumerate(values): + if not step(): + return content + while stack and pairs[stack[-1]] == index: + opening = stack.pop() + if values[opening] == "{": + block_stack.pop() + enclosing.append(stack[-1] if stack else None) + blocks.append( + (block_stack[-1], pairs[block_stack[-1]]) if block_stack else (0, len(tokens)) + ) + if index in pairs: + stack.append(index) + if value == "{": + block_stack.append(index) + + functions: dict[str, tuple[int, int, int, list[str]]] = {} + constants: dict[str, list[tuple[int, int, int, int, int]]] = {} + arrows: dict[int, tuple[int, int, int, list[str]]] = {} + native_methods = { + "map", + "filter", + "find", + "some", + "flatMap", + "forEach", + "join", + "replaceAll", + "trim", + "toLowerCase", + "includes", + "getElementById", + "querySelectorAll", + "addEventListener", + } + protected = {"String", "JSON", "document", "Object"} + forbidden = {"constructor", "prototype", "__proto__", "globalThis", "window", "self", "with"} + for index, value in enumerate(values): + if not step(): + return content + if value in forbidden: + return content + if value in protected: + expected = {"String": ["("], "JSON": ["."], "document": ["."], "Object": ["."]}[value] + if values[index + 1 : index + 1 + len(expected)] != expected or ( + index and values[index - 1] in {"function", ".", "?.", "const", "let", "var"} + ): + return content + if value == "JSON" and values[index + 2 : index + 4] not in ( + ["parse", "("], + ["stringify", "("], + ): + return content + if value == "document" and values[index + 2 : index + 4] != ["getElementById", "("]: + return content + if value == "Object" and values[index + 2 : index + 4] != ["entries", "("]: + return content + if value in native_methods and index and values[index - 1] in {".", "?."}: + if values[index + 1 : index + 2] != ["("]: + return content + if values[index + 1 : index + 2] == [":"] and ( + value in native_methods + or tokens[index].kind == "string" + and value[1:-1] in native_methods + ): + return content + if value == "]" and values[index + 1 : index + 2] in ([":"], ["="]): + return content + if value == "function": + opening = index + 2 + closing = pairs.get(opening) + body = -1 if closing is None else closing + 1 + end = pairs.get(body) + if ( + tokens[index].depth + or end is None + or values[body] != "{" + or tokens[index + 1].kind != "identifier" + or values[index + 1] in functions + or len(functions) >= _MAX_NESTING + ): + return content + params = arguments(opening) + names = [] + for begin, stop in params: + if ( + tokens[begin].kind != "identifier" + or values[begin] in protected + or values[begin] in names + ): + return content + if stop != begin + 1 and not ( + stop == begin + 3 + and values[begin + 1] == "=" + and tokens[begin + 2].kind == "string" + ): + return content + names.append(values[begin]) + functions[values[index + 1]] = index + 1, body, end, names + if value == "const": + if ( + index + 2 >= len(tokens) + or tokens[index + 1].kind != "identifier" + or values[index + 2] != "=" + ): + return content + stop = index + 3 + while stop < len(tokens) and values[stop] != ";" and step(): + stop = pairs.get(stop, stop) + 1 + if stop == len(tokens): + return content + constants.setdefault(values[index + 1], []).append( + (index + 1, index + 3, stop, *blocks[index]) + ) + if value == "=>": + if index and values[index - 1] == ")": + opening = reverse[index - 1] + begin, stop = opening + 1, index - 1 + call = enclosing[opening] + elif index and tokens[index - 1].kind == "identifier": + begin, stop = index - 1, index + call = enclosing[begin] + else: + return content + if begin < stop and values[begin] in {"{", "["} and pairs.get(begin) == stop - 1: + begin, stop = begin + 1, stop - 1 + names = [] + for lower, upper in parts(begin, stop): + if ( + upper != lower + 1 + or tokens[lower].kind != "identifier" + or values[lower] in protected + or values[lower] in names + ): + return content + names.append(values[lower]) + body = index + 1 + end = pairs.get(body, body) + 1 + if body not in pairs: + while end < len(tokens) and values[end] not in {",", ";", ")", "]", "}"} and step(): + end = pairs.get(end, end) + 1 + if call is not None: + arrows[index] = call, body, end, names + + arrows_by_call: dict[int, list[tuple[int, int, int, list[str]]]] = {} + arrows_by_body: dict[tuple[int, int], tuple[int, int, int, list[str]]] = {} + arrows_by_name: dict[str, list[tuple[int, int, int, list[str]]]] = {} + for row in arrows.values(): + arrows_by_call.setdefault(row[0], []).append(row) + arrows_by_body[row[1:3]] = row + for name in row[3]: + arrows_by_name.setdefault(name, []).append(row) + + def literal(index: int) -> str | None: + value = values[index] + return value[1:-1] if tokens[index].kind == "string" and "\\" not in value else None + + # Verify the actual text escaping operation, with no aliases or extra body. + escapes: set[str] = set() + for name, (_, body, end, params) in functions.items(): + if len(params) != 1: + continue + sequence = values[body + 1 : end] + prefix = ["return", "String", "(", params[0], "??"] + cursor = body + 1 + if sequence[:5] != prefix or literal(cursor + 5) != "" or values[cursor + 6] != ")": + continue + cursor += 7 + for before, after in [("&", "&"), ("<", "<"), (">", ">"), ('"', """)]: + if ( + values[cursor : cursor + 3] != [".", "replaceAll", "("] + or literal(cursor + 3) != before + or values[cursor + 4 : cursor + 5] != [","] + or literal(cursor + 5) != after + or values[cursor + 6 : cursor + 7] != [")"] + ): + break + cursor += 7 + else: + if values[cursor:end] == [";"]: + escapes.add(name) + + def source(start: int, end: int) -> str: + return content[tokens[start].start : tokens[end - 1].end] + + def template_fields(start: int) -> list[tuple[int, int]]: + delimiters = template_spans[start][1] + return [ + (bisect_left(positions, p) + 1, pairs[bisect_left(positions, p)]) + for p in delimiters[1:-1:3] + ] + + def text_slot(template: int, field: int, *, inert: bool = False) -> bool: + """The selected HTML fragment occurs in ordinary literal markup text.""" + marker = "SKILLSPECTOR_SELECTED_FRAGMENT" + if marker in content: + return False + _, delimiters = template_spans[template] + if not step(1 + (delimiters[-1] - delimiters[0]) // 32): + return False + chunks = [] + cursor = delimiters[0] + 1 + for dollar, _, close in zip( + delimiters[1:-1:3], delimiters[2:-1:3], delimiters[3:-1:3], strict=True + ): + chunks.append(content[cursor:dollar]) + chunks.append(marker if bisect_left(positions, dollar) == field else "VALUE") + cursor = close + 1 + chunks.append(content[cursor : delimiters[-1]]) + + class Slot(HTMLParser): + valid = True + found = False + context = "" + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if inert and ( + tag not in {"div", "span", "ul", "li", "p", "strong", "em", "b", "code"} + or any(name not in {"class", "id", "title"} for name, _ in attrs) + ): + self.valid = False + if tag in {"script", "style", "iframe", "object", "embed", "textarea", "title"}: + self.context = tag + if any(value is not None and marker in value for _, value in attrs): + self.valid = False + + def handle_endtag(self, tag: str) -> None: + if tag == self.context: + self.context = "" + + def handle_data(self, data: str) -> None: + if marker in data: + self.found = True + self.valid = self.valid and not self.context + + parser = Slot(convert_charrefs=False) + check_runtime() + parser.feed("".join(chunks)) + parser.close() + check_runtime() + return parser.valid and parser.found + + # The formatter has one escaped empty branch and one map/escape/join branch. + formatters: set[str] = set() + for name, (_, body, end, params) in functions.items(): + if len(params) != 2: + continue + items, empty = params + for escape in escapes: + if not step(end - body + 1): + return content + compact = re.sub(r"\s+", "", source(body + 1, end)) + head = f"if(!{items}||!{items}.length){{return" + if not compact.startswith(head): + continue + returns = [index for index in range(body + 1, end) if values[index] == "return"] + if len(returns) != 2: + continue + first, second = (index + 1 for index in returns) + if first not in template_spans or second not in template_spans: + continue + first_end, _ = template_spans[first] + second_end, _ = template_spans[second] + if values[first_end:second] != [";", "}", "return"] or values[second_end:end] != [";"]: + continue + a, b = template_fields(first), template_fields(second) + if len(a) != 1 or len(b) != 1: + continue + if values[a[0][0] : a[0][1]] != [escape, "(", empty, ")"] or not text_slot( + first, a[0][0] - 1, inert=True + ): + continue + begin, stop = b[0] + if values[begin : begin + 4] != [items, ".", "map", "("]: + continue + call = begin + 3 + args = arguments(call) + if len(args) != 1: + continue + matching = arrows_by_call.get(call, []) + if len(matching) != 1: + continue + _, inner, inner_end, names = matching[0] + if ( + len(names) != 1 + or inner not in template_spans + or template_spans[inner][0] != inner_end + ): + continue + item = names[0] + field = template_fields(inner) + if len(field) != 1 or not text_slot(inner, field[0][0] - 1, inert=True): + continue + expression = values[field[0][0] : field[0][1]] + simple = [escape, "(", item, ")"] + fallback = [ + escape, + "(", + "typeof", + item, + "===", + "'string'", + "?", + item, + ":", + "JSON", + ".", + "stringify", + "(", + item, + ")", + ")", + ] + if expression != simple and expression != fallback: + continue + close = pairs[call] + 1 + if ( + values[close : close + 3] != [".", "join", "("] + or literal(close + 3) != "" + or values[close + 4 : stop] != [")"] + ): + continue + if text_slot(second, begin - 1, inert=True): + formatters.add(name) + if not formatters: + return content + + # A closed callable vocabulary rules out explicit escapes of JSON values + # into arbitrary code. Native method spelling alone never proves a selected + # receiver: ordinary() below separately traces that receiver back to JSON. + callbacks: set[int] = set() + function_parameter_openers = {declaration + 1 for declaration, _, _, _ in functions.values()} + defaults = { + a + 1 + for opening in function_parameter_openers + for a, b in arguments(opening) + if b == a + 3 and values[a + 1] == "=" + } + declaration_indices = {row[0] for rows in constants.values() for row in rows} + mutated: set[str] = set() + member_writes: list[int] = [] + for index, value in enumerate(values): + if not step(): + return content + if value == "=" and index - 1 not in declaration_indices and index not in defaults: + if index < 3 or values[index - 2] != ".": + return content + member_writes.append(index) + if tokens[index - 3].kind == "identifier": + mutated.add(values[index - 3]) + if ( + value.endswith("=") and value not in {"=", "==", "===", "!=", "!==", "<=", ">=", "=>"} + ) or value in {"++", "--", "delete", "new", "class", "import", "export", "let", "var"}: + return content + if value != "(" or tokens[index].start in expression_openers: + continue + closing = pairs[index] + if index in function_parameter_openers or values[closing + 1 : closing + 2] == ["=>"]: + continue + previous = values[index - 1] if index else "" + if previous == "if": + continue + if index and tokens[index - 1].kind == "identifier": + if index > 1 and values[index - 2] == ".": + receiver = values[index - 3] if index > 2 else "" + if not ( + previous in native_methods + or receiver == "JSON" + and previous in {"parse", "stringify"} + or receiver == "Object" + and previous == "entries" + ): + return content + args = arguments(index) + if previous == "replaceAll" and ( + len(args) != 2 or any(b != a + 1 or literal(a) is None for a, b in args) + ): + return content + if previous in { + "map", + "filter", + "find", + "some", + "flatMap", + "forEach", + "addEventListener", + }: + slot = 1 if previous == "addEventListener" else 0 + if len(args) <= slot: + return content + a, b = args[slot] + if b == a + 1 and values[a] in functions: + callbacks.add(a) + elif index not in arrows_by_call: + return content + elif previous not in functions and previous != "String": + return content + elif previous in {")", "]", "?."}: + return content + for index, value in enumerate(values): + if not step(): + return content + if value in functions and index != functions[value][0]: + if index and values[index - 1] in {".", "?."}: + return content + if values[index + 1 : index + 2] != ["("] and index not in callbacks: + return content + + selected_names = escapes | formatters + for index, value in enumerate(values): + if not step(): + return content + if value in selected_names: + if index == functions[value][0]: + continue + if value in escapes and index in callbacks: + continue + if ( + values[index + 1 : index + 2] != ["("] + or index + and values[index - 1] in {".", "?.", "function", "new"} + ): + return content + if ( + tokens[index].kind == "string" + and literal(index) in selected_names | protected | native_methods + and index >= 2 + and values[index - 1] == "[" + and values[index + 1 : index + 2] == ["]"] + and (tokens[index - 2].kind == "identifier" or values[index - 2] in {")", "]", "}"}) + ): + # Protect computed member references, not text arguments such as a + # DOM element whose ID happens to be "filter" or "map". + return content + + def receiver_start(end: int, depth: int = 0) -> int | None: + if depth >= _MAX_NESTING or end <= 0 or not step(): + return None + last = end - 1 + if last in reverse: + opening = reverse[last] + if opening and ( + tokens[opening - 1].kind == "identifier" or values[opening - 1] in {")", "]"} + ): + return receiver_start(opening, depth + 1) + return opening + if tokens[last].kind not in {"identifier", "string", "number"}: + return None + if last >= 2 and values[last - 1] in {".", "?."}: + return receiver_start(last - 1, depth + 1) + return last + + # Resolve the nearest explicit binding, including callback/function + # parameters. A same-spelled outer DOM or JSON constant is not that value. + binding_rows: dict[_BrowserBinding, tuple[int, int, int, int]] = {} + bindings_by_name: dict[str, list[_BrowserBinding]] = {} + for name, rows in constants.items(): + for declaration, a, b, lower, upper in rows: + key = ("const", declaration, name) + binding_rows[key] = a, b, lower, upper + bindings_by_name.setdefault(name, []).append(key) + for declaration, body, stop, params in functions.values(): + for slot, parameter in enumerate(params): + key = ("function", declaration, parameter) + binding_rows[key] = slot, declaration, body + 1, stop + bindings_by_name.setdefault(parameter, []).append(key) + for call, body, stop, params in arrows.values(): + for slot, parameter in enumerate(params): + key = ("arrow", body, parameter) + binding_rows[key] = slot, call, body, stop + bindings_by_name.setdefault(parameter, []).append(key) + + def binding(name: str, index: int) -> _BrowserBinding | None: + owner = None + width = len(tokens) + 1 + for key in bindings_by_name.get(name, []): + if not step(): + return None + _, _, lower, upper = binding_rows[key] + if lower <= index < upper: + size = upper - lower + if size == width: + return None + if size < width: + owner, width = key, size + return owner + + calls_by_function: dict[int, list[int]] = {} + for index, value in enumerate(values): + if not step(): + return content + if value in functions and index != functions[value][0]: + declaration = functions[value][0] + if binding(value, index) is not None: + return content + calls_by_function.setdefault(declaration, []).append(index) + + resolving: set[tuple[str, int, str]] = set() + dependencies: set[_BrowserBinding] = set() + + def ordinary(start: int, end: int, env: frozenset[str] = frozenset(), depth: int = 0) -> bool: + if start >= end or depth >= 32 or not step(): + return False + if start in template_spans and template_spans[start][0] == end: + return all(ordinary(a, b, env, depth + 1) for a, b in template_fields(start)) + if values[start] == "(" and pairs.get(start) == end - 1: + return ordinary(start + 1, end - 1, env, depth + 1) + cursor = start + while cursor < end: + if not step(): + return False + if values[cursor] in {"===", "!==", "==", "!=", "&&", "||", "??"}: + return ordinary(start, cursor, env, depth + 1) and ordinary( + cursor + 1, end, env, depth + 1 + ) + cursor = pairs.get(cursor, cursor) + 1 + if values[start] in {"!", "typeof"}: + return ordinary(start + 1, end, env, depth + 1) + cursor = start + 1 + value = values[start] + if value in {"{", "["} and start in pairs: + closing = pairs[start] + if closing >= end: + return False + for a, b in parts(start + 1, closing): + if values[start] == "{" and a + 1 < b and values[a + 1] == ":": + if tokens[a].kind != "identifier" or values[a] in forbidden | native_methods: + return False + a += 2 + elif value == "[" and values[a] == "...": + a += 1 + if not ordinary(a, b, env, depth + 1): + return False + cursor = closing + 1 + elif value == "Object" and values[start + 1 : start + 4] == [".", "entries", "("]: + opening = start + 3 + args = arguments(opening) + if len(args) != 1 or not ordinary(*args[0], env, depth + 1): + return False + cursor = pairs[opening] + 1 + elif value == "JSON" and values[start + 1 : start + 4] == [".", "parse", "("]: + opening = start + 3 + args = arguments(opening) + if len(args) != 1: + return False + a, b = args[0] + if ( + values[a : a + 4] != ["document", ".", "getElementById", "("] + or literal(a + 4) is None + or values[a + 5 : b] != [")", ".", "textContent"] + ): + return False + cursor = pairs[opening] + 1 + elif start + 1 < end and values[start + 1] == "(" and value in functions: + declaration, body, stop, params = functions[value] + opening = start + 1 + args = arguments(opening) + key = "call", declaration, value + if ( + key in resolving + or len(args) != len(params) + or values[body + 1 : body + 2] != ["return"] + or values[stop - 1 : stop] != [";"] + ): + return False + if not all(ordinary(a, b, env, depth + 1) for a, b in args): + return False + resolving.add(key) + valid = ordinary(body + 2, stop - 1, frozenset(params), depth + 1) + resolving.remove(key) + if not valid: + return False + cursor = pairs[opening] + 1 + elif ( + tokens[start].kind in {"string", "number"} + or value in {"true", "false", "null"} + or value in env + ): + pass + elif tokens[start].kind == "identifier": + if value in mutated: + return False + key = binding(value, start) + if key is None or key in resolving: + return False + dependencies.add(key) + a, b, _, _ = binding_rows[key] + resolving.add(key) + if key[0] == "const": + valid = ordinary(a, b, frozenset(), depth + 1) + elif key[0] == "arrow": + call = b + begin = receiver_start(call - 2) + valid = ( + begin is not None + and values[call - 1] in {"map", "filter", "find", "some", "flatMap", "forEach"} + and ordinary(begin, call - 2, frozenset(), depth + 1) + ) + else: + # Every explicit invocation must provide an ordinary argument. + # A recursive unchanged forwarding edge adds no new source. + valid = True + sources = 0 + for call in calls_by_function.get(key[1], []): + if not step(): + valid = False + break + if call in callbacks: + opening = enclosing[call] + begin = None if opening is None else receiver_start(opening - 2) + good = ( + a == 0 + and opening is not None + and begin is not None + and values[opening - 1] + in {"map", "filter", "find", "some", "flatMap", "forEach"} + and ordinary(begin, opening - 2, frozenset(), depth + 1) + ) + else: + args = arguments(call + 1) + if a >= len(args): + params = arguments(key[1] + 1) + lower, upper = params[a] + good = upper == lower + 3 and literal(lower + 2) is not None + else: + lower, upper = args[a] + if upper == lower + 1 and binding(values[lower], lower) == key: + continue + good = ordinary(lower, upper, frozenset(), depth + 1) + sources += 1 + if not good: + valid = False + break + valid = valid and sources > 0 + resolving.remove(key) + if not valid: + return False + else: + return False + while cursor < end and step(): + if ( + values[cursor] == "[" + and pairs.get(cursor) == cursor + 2 + and tokens[cursor + 1].kind == "number" + ): + cursor += 3 + continue + if values[cursor] not in {".", "?."} or cursor + 1 >= end: + return False + member = values[cursor + 1] + if tokens[cursor + 1].kind != "identifier" or member in forbidden: + return False + cursor += 2 + if cursor < end and values[cursor] == "(": + if member not in { + "map", + "filter", + "find", + "some", + "flatMap", + "join", + "trim", + "toLowerCase", + }: + return False + args = arguments(cursor) + if member in {"trim", "toLowerCase"}: + if args: + return False + elif member == "join": + if len(args) > 1 or any(b != a + 1 or literal(a) is None for a, b in args): + return False + else: + matches = arrows_by_call.get(cursor, []) + if len(args) != 1: + return False + a, b = args[0] + if member in {"filter", "find", "some"} and b == a + 1 and a in callbacks: + # These native operations retain existing members; + # dependency closure separately audits the predicate. + cursor = pairs[cursor] + 1 + continue + if len(matches) != 1: + return False + _, body, stop, names = matches[0] + if member in {"filter", "find", "some"}: + # A simple predicate can select only existing JSON + # members. Its values need not become part of output. + if not step(stop - body + 1) or any( + values[i] not in {".", "?.", "!", "===", "!==", "==", "!=", "&&", "||"} + and tokens[i].kind not in {"identifier", "string", "number"} + for i in range(body, stop) + ): + return False + elif not ordinary(body, stop, env | frozenset(names), depth + 1): + return False + cursor = pairs[cursor] + 1 + return cursor == end + + def display(start: int, end: int) -> bool: + escaped = False + for _ in range(_MAX_NESTING): + if not step(): + return False + if values[end : end + 3] == [".", "join", "("]: + args = arguments(end + 2) + if len(args) != 1 or args[0][1] != args[0][0] + 1 or literal(args[0][0]) != "": + return False + end = pairs[end + 2] + 1 + continue + parent = enclosing[start] + if parent in fields and start == parent + 1 and pairs[parent] == end: + template = fields[parent] + if not text_slot(template, parent): + return False + start, end = template, template_spans[template][0] + continue + arrow = arrows_by_body.get((start, end)) + if arrow is not None: + call = arrow[0] + begin = receiver_start(call - 2) + if values[call - 1] != "map" or begin is None or not ordinary(begin, call - 2): + return False + start, end = begin, pairs[call] + 1 + continue + if parent is not None and parent and values[parent - 1] in formatters: + args = arguments(parent) + if ( + not args + or args[0] != (start, end) + or len(args) > 2 + or any(b != a + 1 or literal(a) is None for a, b in args[1:]) + ): + return False + escaped = True + start, end = parent - 1, pairs[parent] + 1 + continue + if ( + start >= 4 + and values[start - 3 : start] == [".", "innerHTML", "="] + and values[end : end + 1] == [";"] + ): + return escaped and dom(start - 4, start - 3) == "element" + return False + return False + + def dom(start: int, end: int, depth: int = 0) -> str | None: + if depth >= 32 or start >= end or not step(): + return None + cursor = start + 1 + kind = None + if values[start : start + 4] == ["document", ".", "getElementById", "("]: + if literal(start + 4) is None or values[start + 5 : start + 6] != [")"]: + return None + cursor, kind = start + 6, "element" + elif tokens[start].kind == "identifier": + key = binding(values[start], start) + if key is None: + return None + dependencies.add(key) + a, b, _, _ = binding_rows[key] + if key[0] == "const": + kind = dom(a, b, depth + 1) + elif key[0] == "arrow" and a == 0 and values[b - 1] == "forEach": + begin = receiver_start(b - 2) + if begin is not None and dom(begin, b - 2, depth + 1) == "list": + kind = "element" + if kind is None: + return None + if cursor < end and values[cursor : cursor + 3] == [".", "querySelectorAll", "("]: + args = arguments(cursor + 2) + if ( + kind != "element" + or len(args) != 1 + or args[0][1] != args[0][0] + 1 + or literal(args[0][0]) is None + ): + return None + cursor, kind = pairs[cursor + 2] + 1, "list" + return kind if cursor == end else None + + member_values: list[tuple[_BrowserBinding, int, int]] = [] + for index in member_writes: + begin = receiver_start(index - 2) + if begin is None: + return content + if ( + values[index - 1] in {"innerHTML", "textContent", "value"} + and dom(begin, index - 2) == "element" + ): + continue + if begin != index - 3: + return content + owner = binding(values[begin], begin) + if owner is None or owner[0] != "const": + return content + a, b, _, _ = binding_rows[owner] + if values[a] != "{" or pairs.get(a) != b - 1: + return content + stop = index + 1 + while stop < len(tokens) and values[stop] != ";" and step(): + stop = pairs.get(stop, stop) + 1 + if stop == len(tokens) or budget <= 0: + return content + member_values.append((owner, index + 1, stop)) + + for index in callbacks: + if values[index] not in escapes: + continue + call = enclosing[index] + if call is None: + return content + begin = receiver_start(call - 2) + if begin is None or values[call - 1] != "map" or not ordinary(begin, call - 2): + return content + output = list(content) + changed = False + for start, (end, delimiters) in template_spans.items(): + if not step(): + return content + expressions = template_fields(start) + if not expressions or any( + _TEMPLATE_VALUE.fullmatch(source(a, b)) is None for a, b in expressions + ): + continue + if not display(start, end) or not all(ordinary(a, b) for a, b in expressions): + continue + output[delimiters[0]] = output[delimiters[-1]] = " " + changed = True + + # Follow explicit aliases and helper returns that carry a proven value. + # Unknown native-looking receivers must not acquire it through a callback. + def dependency(index: int) -> bool: + if ( + tokens[index].kind != "identifier" + or index + and values[index - 1] in {".", "?."} + or values[index + 1 : index + 2] == [":"] + ): + return False + if values[index] == "document": + # A second literal lookup can reacquire the selected element even + # without referring to its local constant binding. + return True + key = binding(values[index], index) + if key is None and values[index] in functions: + key = ("call", functions[values[index]][0], values[index]) + return key in dependencies + + def carries(start: int, end: int) -> bool: + return step(end - start + 1) and any(dependency(i) for i in range(start, end)) + + for _ in range(_MAX_NESTING): + before = len(dependencies) + for name, rows in constants.items(): + for declaration, a, b, _, _ in rows: + if carries(a, b): + dependencies.add(("const", declaration, name)) + for owner, a, b in member_values: + if carries(a, b): + dependencies.add(owner) + for name, (declaration, a, b, params) in functions.items(): + if carries(a, b): + dependencies.add(("call", declaration, name)) + for call in calls_by_function.get(declaration, []): + if not step(): + return content + if call in callbacks: + opening = enclosing[call] + begin = None if opening is None else receiver_start(opening - 2) + if begin is not None and carries(begin, opening - 2): + dependencies.update( + ("function", declaration, parameter) for parameter in params + ) + else: + for parameter, (lower, upper) in zip(params, arguments(call + 1), strict=False): + if carries(lower, upper): + dependencies.add(("function", declaration, parameter)) + for call, body, _, params in arrows.values(): + begin = receiver_start(call - 2) if call >= 3 and values[call - 2] == "." else None + if begin is not None and carries(begin, call - 2): + dependencies.update(("arrow", body, parameter) for parameter in params) + if budget <= 0: + return content + if len(dependencies) == before: + break + else: + return content + for opening in pairs: + if values[opening] != "(" or opening < 3 or values[opening - 2] != ".": + continue + member = values[opening - 1] + if member not in native_methods: + continue + closing = pairs[opening] + if not step(closing - opening + 1): + return content + if not any(dependency(i) for i in range(opening + 1, closing)): + continue + begin = receiver_start(opening - 2) + if begin is None or not ( + ordinary(begin, opening - 2) or dom(begin, opening - 2) is not None + ): + return content + check_runtime() + return "".join(output) if changed and budget > 0 else content + + JavaScriptTemplateSpans = tuple[tuple[int, ...], tuple[int, ...]] _SHELL_EXECUTION_CALLS = frozenset( diff --git a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py index 8daa81895..da0d01819 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py +++ b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py @@ -46,7 +46,7 @@ from skillspector.logging_config import get_logger from skillspector.models import AnalyzerFinding, Finding, Location, Severity from skillspector.python_ast import ParsedPythonFile, parse_python_source -from skillspector.python_tokens import PythonLiteralSpans +from skillspector.python_tokens import PythonLiteralSpans, python_browser_template_shell_text from skillspector.python_tokens import python_literal_spans as _python_literal_spans from skillspector.python_tokens import python_string_body as _python_string_body from skillspector.security_reconstruction import validated_json_string_spans @@ -92,6 +92,7 @@ _PRINTF_STATIC_CHARS = 256 _PRINTF_STATIC_ARGUMENTS = 32 _PRINTF_STATIC_WORD_RE = re.compile(r"[-A-Za-z0-9_./*?%]{0,64}") +_PRINTF_QUOTED_WORD_RE = re.compile(r"[-A-Za-z0-9_./*?% \t\r\n]{0,64}") # A bounded ``printf`` format operand may also carry POSIX backslash escapes. _PRINTF_FORMAT_WORD_RE = re.compile(r"[-A-Za-z0-9_./*?%\\]{0,64}") # The old deterministic alphabet plus the control/high-byte values needed by @@ -621,6 +622,8 @@ class _ShellDelimiterFrame: inherited_quote_closed: bool = False pending_case_clauses: int = 0 open_case_clauses: int = 0 + pending_heredocs: tuple[tuple[str, bool, bool], ...] = () + arithmetic: bool = False def _is_shell_command_word_start(content: str, start: int) -> bool: @@ -715,6 +718,84 @@ def _skip_ansi_c_quote_tail(content: str, start: int, limit: int) -> int | None: return None +def _static_heredoc_header( + content: str, start: int, limit: int +) -> tuple[tuple[str, bool, bool], int] | None: + """Read one bounded literal heredoc delimiter, without evaluating it.""" + cursor = start + 2 + strip_tabs = content[cursor : cursor + 1] == "-" + cursor += strip_tabs + whitespace_limit = min(limit, cursor + _SHELL_COMMAND_WORD_CHARS) + while cursor < whitespace_limit and content[cursor] in " \t": + cursor += 1 + if cursor < limit and content[cursor] in " \t": + return None + quote = content[cursor : cursor + 1] + quoted = quote in {"'", '"'} + cursor += quoted + name = re.match( + r"[A-Za-z_][A-Za-z0-9_]*", + content[cursor : min(limit, cursor + _SHELL_COMMAND_WORD_CHARS + 1)], + ) + if name is None or len(name[0]) > _SHELL_COMMAND_WORD_CHARS: + return None + cursor += len(name[0]) + if quoted: + if cursor >= limit or content[cursor] != quote: + return None + cursor += 1 + if cursor < limit and not (content[cursor].isspace() or content[cursor] in ";|&()<>"): + return None + return (name[0], strip_tabs, quoted), cursor + + +def _skip_static_heredoc_bodies( + content: str, + start: int, + limit: int, + pending: tuple[tuple[str, bool, bool], ...], + check_runtime: Callable[[], None] | None, +) -> int | None: + """Skip ordered bodies only after their declaring command's newline. + + This proves outer lexical state only. The raw scanner still inspects every + body, including substitutions in unquoted heredocs. Delimiter comparison + joins unquoted continuations while retaining at most one delimiter's width. + """ + cursor = start + for delimiter, strip_tabs, quoted in pending: + logical_prefix = "" + while cursor < limit: + if check_runtime is not None: + check_runtime() + newline = content.find("\n", cursor, limit) + line_end = limit if newline < 0 else newline + if line_end > cursor and content[line_end - 1] == "\r": + line_end -= 1 + line = content[cursor:line_end] + continued = ( + not quoted and newline >= 0 and (len(line) - len(line.rstrip("\\"))) % 2 == 1 + ) + if continued: + line = line[:-1] + if strip_tabs and not logical_prefix: + line = line.lstrip("\t") + # Once overlong, a logical line cannot become the delimiter. Keep + # a capped prefix instead of accumulating arbitrarily long bodies. + logical_prefix += line[: max(0, len(delimiter) + 1 - len(logical_prefix))] + if not continued and logical_prefix == delimiter: + cursor = limit if newline < 0 else newline + 1 + break + if newline < 0: + return None + cursor = newline + 1 + if not continued: + logical_prefix = "" + else: + return None + return cursor + + def _skip_shell_delimited_expansion( content: str, start: int, @@ -734,6 +815,7 @@ def _skip_shell_delimited_expansion( root_kind, start, inherited_double_quote=root_inherited_double_quote, + arithmetic=root_kind == "command" and content.startswith("$((", start), ) ] cursor = start + opener_width @@ -782,6 +864,10 @@ def push(kind: str, frame_start: int | None, width: int) -> None: kind, frame_start, inherited_double_quote=inherited_double_quote, + arithmetic=( + (kind == "paren" and (parent.arithmetic or content.startswith("((", cursor))) + or (frame_start is not None and content.startswith("$((", frame_start)) + ), ) ) cursor += width @@ -837,6 +923,32 @@ def at_shell_keyword(keyword: str) -> bool: cursor += 1 continue + if frame.kind in {"command", "paren"} and not frame.arithmetic: + if character == "\n" and frame.pending_heredocs: + body_end = _skip_static_heredoc_bodies( + content, cursor + 1, limit, frame.pending_heredocs, check_runtime + ) + if body_end is None: + cache_failure() + return None + frame.pending_heredocs = () + frame.word_started = False + cursor = body_end + continue + if content.startswith("<<<", cursor): + cursor += 3 + frame.word_started = False + continue + if content.startswith("<<", cursor): + header = _static_heredoc_header(content, cursor, limit) + if header is None or len(frame.pending_heredocs) == _SHELL_COMMAND_STRING_ARGUMENTS: + cache_failure() + return None + spec, cursor = header + frame.pending_heredocs += (spec,) + frame.word_started = True + continue + # Ordinary words cannot change the delimiter stack. Consume them in # one bounded match rather than testing each character for every shell # opener, quote, and keyword. Nested static-evaluator windows otherwise @@ -938,6 +1050,11 @@ def at_shell_keyword(keyword: str) -> bool: continue if character == "(": + if frame.pending_heredocs: + # Plain subshell grammar can share pending heredocs with the + # enclosing command. Do not assign that body to a new frame. + cache_failure() + return None push("paren", None, 1) continue if character == ")": @@ -948,6 +1065,9 @@ def at_shell_keyword(keyword: str) -> bool: frame.word_started = False cursor += 1 continue + if frame.pending_heredocs: + cache_failure() + return None endpoint = close_frame(cursor + 1) cursor += 1 if endpoint is not None: @@ -1190,9 +1310,11 @@ def _next_shell_invocation_word( """Read one direct shell word without traversing nested substitutions.""" cursor = start limit = len(content) + next_runtime_check = ((cursor + 4095) // 4096) * 4096 while cursor < limit: - if cursor % 4096 == 0: + if cursor >= next_runtime_check: check_runtime() + next_runtime_check = cursor + 4096 if content[cursor].isspace(): cursor += 1 continue @@ -1212,8 +1334,9 @@ def _next_shell_invocation_word( word_started = False unquoted_characters = 0 while cursor < limit: - if cursor % 4096 == 0: + if cursor >= next_runtime_check: check_runtime() + next_runtime_check = cursor + 4096 character = content[cursor] if quote is not None: if character == quote: @@ -1474,31 +1597,66 @@ def _bounded_printf_word(result: str) -> str | None: return result +def _has_literal_printf_arguments(inner: str) -> bool: + """Prove literal arguments without treating shell expansions as text.""" + quote: str | None = None + cursor = 0 + while cursor < len(inner): + character = inner[cursor] + if character == "\\" and quote != "'": + if cursor + 1 >= len(inner): + return False + cursor += 3 if inner.startswith("\\\r\n", cursor) else 2 + continue + if quote is not None: + if character == quote: + quote = None + elif quote == '"' and character in "$`": + return False + elif character in "'\"": + quote = character + elif character in "$`;|&<>()\r\n*?[]{}~#": + return False + cursor += 1 + if quote is not None: + return False + command, _, limited = _next_shell_invocation_word(inner, 0, lambda: None) + return command == "printf" and not limited + + def _static_printf_substitution( content: str, start: int, end: int, *, backtick: bool = False, + quoted_result: bool = False, ) -> str | None: - """Evaluate a tiny, bounded subset of shell ``printf`` deterministically.""" + """Evaluate bounded literal ``printf`` output, preserving its quoting.""" inner_start = start + (1 if backtick else 2) inner_end = end - 1 inner = content[inner_start:inner_end] + if len(inner) > _PRINTF_STATIC_CHARS: + return None + literal_result = quoted_result and not backtick and _has_literal_printf_arguments(inner) + if quoted_result and not backtick and not literal_result: + command, _, _ = _next_shell_invocation_word(inner, 0, lambda: None) + if command == "printf": + return None recognized, arguments = _printf_invocation_arguments(inner) - if ( - not recognized - or len(inner) > _PRINTF_STATIC_CHARS - or re.search(r"[;|&<>\r\n]", inner) is not None - ): + if not recognized or (not literal_result and re.search(r"[;|&<>\r\n]", inner) is not None): return None if arguments[:1] == ["--"]: arguments = arguments[1:] + elif literal_result and arguments and arguments[0].startswith("-"): + # In particular, ``-v`` assigns a variable instead of producing output. + return None if not arguments or len(arguments) > _PRINTF_STATIC_ARGUMENTS + 1: return None format_word, *values = arguments + value_pattern = _PRINTF_QUOTED_WORD_RE if literal_result else _PRINTF_STATIC_WORD_RE if _PRINTF_FORMAT_WORD_RE.fullmatch(format_word) is None or any( - _PRINTF_STATIC_WORD_RE.fullmatch(value) is None for value in values + value_pattern.fullmatch(value) is None for value in values ): return None @@ -1538,17 +1696,27 @@ def flush_literal() -> bool: return None conversions = sum(segment is None for segment in segments) + + def bounded_result(result: str) -> str | None: + if literal_result: + # Quoted command substitution keeps one word and strips trailing + # LF bytes only. Its internal whitespace never becomes shell syntax. + if len(result) > _PRINTF_STATIC_CHARS or "\x00" in result: + return None + return result.rstrip("\n") + return _bounded_printf_word(result) + if conversions == 0: - return _bounded_printf_word("".join(segments)) + return bounded_result("".join(segments)) remaining = iter(values) output: list[str] = [] cycles = max(1, (len(values) + conversions - 1) // conversions) for _ in range(cycles): for segment in segments: output.append(next(remaining, "") if segment is None else segment) - if sum(len(piece) for piece in output) > 64: + if sum(len(piece) for piece in output) > (_PRINTF_STATIC_CHARS if literal_result else 64): return None - return _bounded_printf_word("".join(output)) + return bounded_result("".join(output)) def _may_have_destructive_outer_operands(content: str) -> bool: @@ -1788,6 +1956,7 @@ def _parse_shell_command_word( check_runtime: Callable[[], None] | None = None, owned_word_positions: set[int] | None = None, enclosing_delimiter: str | None = None, + quoted_printf_allowed: bool = True, ) -> _ShellCommandWord | None: runtime_check = check_runtime or (lambda: None) # The shell recognizes ``NAME=value`` before expansion and never runs the @@ -1856,6 +2025,7 @@ def reconstructs_command( content, cursor, substitution_end, + quoted_result=quoted_printf_allowed, ) if static_value is None: output.append("$()") @@ -2115,6 +2285,57 @@ def _destructive_command_words(content: str) -> Iterator[tuple[int, int]]: yield start, parsed.end +class _ShellOwnershipBudgetExceededError(Exception): + """Stop optional quote proof without swallowing the caller's cancellation.""" + + +def _literal_shell_command_spans( + content: str, begin: int, end: int, check_runtime: Callable[[], None] +) -> dict[int, int] | None: + """Locate exact single-quoted shell -c bodies once per lexical scope.""" + source = content[begin:end] + if "'" not in source: + return {} + spans: list[tuple[int, int]] = [] + # Invocation readers report periodic progress across escaped sequences. + # Bound repeated delimiter scans proportionally to this scope, so a valid + # literal command before deeply nested unrelated input cannot make this + # optional ownership pass quadratic. Exhaustion publishes no proof. + remaining_checks = 4 + 4 * ((len(source) + 4095) // 4096) + + def check_proof_runtime() -> None: + nonlocal remaining_checks + check_runtime() + remaining_checks -= 1 + if remaining_checks < 0: + raise _ShellOwnershipBudgetExceededError + + clause_starts = _shell_clause_starts(source, check_runtime) + next_clause_start = next(clause_starts, None) + clause_start = 0 + last_selected_start = -1 + try: + for option in re.finditer(r"(-[A-Za-z]+)\s+(?:\\\r?\n[ \t]*)*'", source): + if "c" not in option[1]: + continue + check_runtime() + # Select the nearest preceding clause for full validation of its + # wrapper and literal body; ambiguous context retains no proof. + # Both iterators move forward once; unrelated clauses need no parse. + while next_clause_start is not None and next_clause_start <= option.start(): + clause_start = next_clause_start + next_clause_start = next(clause_starts, None) + if clause_start == last_selected_start: + continue + last_selected_start = clause_start + _command_string_from_clause( + source, clause_start, check_proof_runtime, literal_command_spans=spans + ) + except _ShellOwnershipBudgetExceededError: + return None + return {begin + start: begin + stop for start, stop in spans} + + def _has_destructive_command_string(content: str) -> bool: """Return whether an executable string contains a proven destructive command.""" for command_start, body_start in _destructive_command_words(content): @@ -2125,16 +2346,21 @@ def _has_destructive_command_string(content: str) -> bool: def _shell_quoted_candidate_limits( - content: str, check_runtime: Callable[[], None] + content: str, + check_runtime: Callable[[], None], + *, + quote_closers: set[int] | None = None, ) -> dict[int, int]: """Prove single-line double-quote bounds by walking shell source in order. This is only an ownership refinement, never an execution or finding parser. Comments and single quotes cannot open double-quoted spans. Substitution - bodies have independent lexical state. Unsupported heredocs/backticks or + bodies have independent lexical state. Literal heredoc bodies cannot own + quotes in the surrounding command. Unsupported delimiters/backticks or unresolved syntax discard ALL ownership, retaining the conservative scan. """ limits: dict[int, int] = {} + closers: set[int] = set() parameter_cache: dict[int, _ParameterExpansionEnd] = {} substitution_cache: dict[int, int | None] = {} @@ -2147,16 +2373,44 @@ def walk(begin: int, end: int, depth: int = 0) -> bool: quote_start = begin ansi_c = False candidates: list[int] = [] + pending_heredocs: tuple[tuple[str, bool, bool], ...] = () + command_spans = _literal_shell_command_spans(content, begin, end, check_runtime) + if command_spans is None: + return False while cursor < end: if (cursor - begin) % 1024 == 0: check_runtime() character = content[cursor] + if character == "\n" and pending_heredocs: + if quote is not None: + return False + body_end = _skip_static_heredoc_bodies( + content, cursor + 1, end, pending_heredocs, check_runtime + ) + if body_end is None: + return False + pending_heredocs = () + word_started = False + cursor = body_end + continue + if quote is not None and character == quote: + closers.add(cursor) + if ( + not ansi_c + and "\n" not in content[quote_start:cursor] + and "\r" not in content[quote_start:cursor] + ): + for position in candidates: + limits[position] = cursor + quote = None + cursor += 1 + continue if quote == "'": if character == "\\" and ansi_c: cursor += 2 continue - if character == "'": - quote = None + if not ansi_c and _SHELL_COMMAND_WORD_START_RE.match(content, cursor): + candidates.append(cursor) cursor += 1 continue if character == "\\": @@ -2166,17 +2420,23 @@ def walk(begin: int, end: int, depth: int = 0) -> bool: word_started = True cursor += 2 continue - if quote == '"' and character == '"': - if ( - "\n" not in content[quote_start:cursor] - and "\r" not in content[quote_start:cursor] - ): - for position in candidates: - limits[position] = cursor - quote = None - cursor += 1 + if quote is None and content.startswith("<<<", cursor): + # A here-string takes an ordinary shell word; unlike a + # heredoc it does not change the ownership of later lines. + cursor += 3 + word_started = False continue - if character == "`" or (quote is None and content.startswith("<<", cursor)): + if quote is None and content.startswith("<<", cursor): + header = _static_heredoc_header(content, cursor, end) + if header is None or len(pending_heredocs) == _SHELL_COMMAND_STRING_ARGUMENTS: + return False + spec, cursor = header + pending_heredocs += (spec,) + word_started = True + continue + if quote is None and character == "(" and pending_heredocs: + return False + if character == "`": return False if quote is None and character == "#" and not word_started: while cursor < end and content[cursor] not in "\r\n": @@ -2195,6 +2455,14 @@ def walk(begin: int, end: int, depth: int = 0) -> bool: cursor += 2 continue if quote is None and character in "'\"": + command_end = command_spans.get(cursor + 1) if character == "'" else None + if command_end is not None: + if not walk(cursor + 1, command_end, depth + 1): + return False + closers.add(command_end) + cursor = command_end + 1 + word_started = True + continue quote = character ansi_c = False quote_start = cursor @@ -2218,6 +2486,22 @@ def walk(begin: int, end: int, depth: int = 0) -> bool: word_started = True continue if content.startswith("${", cursor): + if quote == '"': + # A quoted simple variable used as a trim pattern owns its + # inner quotes. Do not apply this to the ksh-sensitive $" + # cases handled by the general parameter parser below. + pattern = re.match( + r"\$\{[A-Za-z_][A-Za-z0-9_]*(?:#{1,2}|%{1,2})" + r'"\$(?:[A-Za-z_][A-Za-z0-9_]*|\{[A-Za-z_][A-Za-z0-9_]*\})"' + r"""[^\\'"$`{}\r\n]*\}""", + content[cursor : min(end, cursor + _SHELL_COMMAND_WORD_CHARS + 1)], + ) + if pattern is not None and len(pattern[0]) <= _SHELL_COMMAND_WORD_CHARS: + pattern_end = cursor + len(pattern[0]) + closers.add(content.rfind('"', cursor, pattern_end)) + cursor = pattern_end + word_started = True + continue inherited_closed = [False] parameter_end = _skip_parameter_expansion( content, @@ -2241,9 +2525,17 @@ def walk(begin: int, end: int, depth: int = 0) -> bool: if quote is None: word_started = not (character.isspace() or character in ";|&()<>") cursor += 1 - return quote is None and cursor <= end + return quote is None and not pending_heredocs and cursor <= end - return limits if walk(0, len(content)) else {} + if not walk(0, len(content)): + return {} + if quote_closers is not None: + quote_closers.update( + position + for position in closers + if content[position + 1 : position + 2] not in ("$", "`") + ) + return limits def _quoted_candidate_is_complete( @@ -2251,6 +2543,7 @@ def _quoted_candidate_is_complete( start: int, quote_limits: dict[int, int] | None, check_runtime: Callable[[], None], + quoted_printf_allowed: bool = True, ) -> bool: """Refine a failure only inside a lexically proven shell quote. @@ -2262,12 +2555,16 @@ def _quoted_candidate_is_complete( if end is None: return False fragment = content[start:end] - parsed = _parse_shell_command_word(fragment, 0, check_runtime=check_runtime) + parsed = _parse_shell_command_word( + fragment, 0, check_runtime=check_runtime, quoted_printf_allowed=quoted_printf_allowed + ) if parsed is None or parsed.limited: return False return not ( parsed.dynamic - and _has_runtime_command_operand_exhaustion(fragment, 0, parsed.end, check_runtime) + and _has_runtime_command_operand_exhaustion( + fragment, 0, parsed.end, check_runtime, quoted_printf_allowed=quoted_printf_allowed + ) ) @@ -2415,6 +2712,8 @@ def _has_shell_command_word_exhaustion( quote_limits: dict[int, int] | None = None, file_type: str = "shell", _command_string_depth: int = 0, + _quoted_printf_allowed: bool = True, + _startup_exposure: list[bool] | None = None, ) -> bool: """Find candidate command words whose deterministic parse hit a safety bound. @@ -2423,6 +2722,9 @@ def _has_shell_command_word_exhaustion( ``perl_eval_blocks`` recognizes Perl's exception-trapping ``eval BLOCK``, which reparses no string. """ + quoted_printf_allowed = _quoted_printf_allowed and not _shell_redefines_printf( + content, check_runtime + ) parsed_through = 0 # Completed words own closing quotes, never executable expansion starts. # Inner commands remain independent candidates; never suppress their bodies. @@ -2433,9 +2735,20 @@ def _has_shell_command_word_exhaustion( may_have_destructive_outer_operands = _may_have_destructive_outer_operands(content) json_openers = sorted(structural_quote_openers or ()) complete_comments: set[tuple[int, int]] = set() + # Short text without shell-expansion openers cannot trigger these shell-word + # reconstruction or span limits. Keep its prior lazy ownership paths instead + # of parsing an ordinary Python module for every candidate letter. + python_word_source = ( + python_source + if python_source is not None + and (len(content) > _SHELL_COMMAND_WORD_CHARS or "$" in content or "`" in content) + else None + ) for candidate in _SHELL_COMMAND_WORD_START_RE.finditer(content): check_runtime() start = candidate.start() + if python_word_source is not None and python_word_source.is_string_delimiter(start): + continue if start in owned_word_positions: continue if javascript_source is not None and javascript_source.owns(start): @@ -2450,6 +2763,9 @@ def _has_shell_command_word_exhaustion( not json_string_start and not assignment_quote and not _is_shell_command_word_start(content, start) + and not ( + python_word_source is not None and python_word_source.starts_string_body(start) + ) ): continue if _has_quoted_assignment_prefix(content, start): @@ -2479,6 +2795,7 @@ def _has_shell_command_word_exhaustion( enclosing_delimiter=( _assignment_value_wrapper(content, start) if assignment_quote else None ), + quoted_printf_allowed=quoted_printf_allowed, ) if parsed is None: substitution_start = ( @@ -2503,6 +2820,27 @@ def _has_shell_command_word_exhaustion( check_runtime=check_runtime, ): return True + if content[start] == "`": + substitution_end = _skip_backtick_substitution( + content, start, len(content), check_runtime, end_cache=backtick_end_cache + ) + if ( + substitution_end is not None + and _is_printf_substitution( + content, + start, + substitution_end, + backtick=True, + check_runtime=check_runtime, + ) + and not ( + file_type in _HASH_COMMENT_FILE_TYPES + and _is_comment_candidate( + content, start, python_source, file_type=file_type + ) + ) + ): + return True # An unclosed expansion or quote can consume the rest of the # artifact. Once that unresolved span exceeds the command-word # budget, treating it as clean would turn malformed, deeply nested @@ -2524,9 +2862,19 @@ def _has_shell_command_word_exhaustion( # expressions cannot continue a shell word into later code. unresolved_end = min(unresolved_end, javascript_source.word_end(start)) if unresolved_end - start > _SHELL_COMMAND_WORD_CHARS: - if not _quoted_candidate_is_complete(content, start, quote_limits, check_runtime): + if not _quoted_candidate_is_complete( + content, start, quote_limits, check_runtime, quoted_printf_allowed + ): return True continue + # A mis-paired quote can produce a completed word that crosses several + # Python tokens. It cannot own quotes or skip candidates beyond its + # proven host region, even when that lexical parse itself succeeded. + owned_end = ( + min(parsed.end, python_word_source.word_end(start)) + if python_word_source is not None + else parsed.end + ) # Comments end at a newline, and a quote after ``=`` in prose or host # source may be mis-paired. Such a word cannot own a later line. confined_word = "\n" in content[start : parsed.end] and ( @@ -2540,7 +2888,8 @@ def _has_shell_command_word_exhaustion( owned_word_positions.update( position for position in candidate_word_positions - if content[position + 1 : position + 2] not in ("$", "`") + if (python_word_source is None or position < owned_end) + and content[position + 1 : position + 2] not in ("$", "`") ) # Only executable nested substitutions retain independent command # positions. A plain dynamic data argument still owns its inner bytes; @@ -2559,7 +2908,7 @@ def _has_shell_command_word_exhaustion( or not any(marker in raw_word for marker in ("$(", "`")) or simple_backtick_parameter ) and not (assignment_quote and confined_word): - parsed_through = max(parsed_through, parsed.end) + parsed_through = max(parsed_through, owned_end) if assignment_quote: # An assignment value never names the command, and main never # parsed this position, so it only claims ownership. @@ -2574,7 +2923,9 @@ def _has_shell_command_word_exhaustion( # spans and runtime-selected operands keep their existing # conservative treatment. continue - if not _quoted_candidate_is_complete(content, start, quote_limits, check_runtime): + if not _quoted_candidate_is_complete( + content, start, quote_limits, check_runtime, quoted_printf_allowed + ): return True if ( parsed.dynamic @@ -2584,11 +2935,14 @@ def _has_shell_command_word_exhaustion( start, parsed.end, check_runtime, + quoted_printf_allowed=quoted_printf_allowed, ) ): comment = python_source.comment(start) if python_source is not None else None if comment is None: - if _quoted_candidate_is_complete(content, start, quote_limits, check_runtime): + if _quoted_candidate_is_complete( + content, start, quote_limits, check_runtime, quoted_printf_allowed + ): continue # Host code after a Python string's closing quote is not shell # text, so a command in the string takes operands only from @@ -2600,7 +2954,9 @@ def _has_shell_command_word_exhaustion( if python_source is not None else None ) - if value is None or _has_runtime_command_operand_exhaustion(*value, check_runtime): + if value is None or _has_runtime_command_operand_exhaustion( + *value, check_runtime, quoted_printf_allowed=quoted_printf_allowed + ): return True continue # A Python comment is prose that no shell receives, so a command @@ -2612,23 +2968,33 @@ def _has_shell_command_word_exhaustion( content[comment[0] : comment[1]], check_runtime, _command_string_depth=_command_string_depth, + _quoted_printf_allowed=quoted_printf_allowed, ): return True complete_comments.add(comment) - for command_string in _shell_command_strings( + for command_string, startup_context in _shell_command_strings( content, check_runtime, perl_eval_blocks=perl_eval_blocks, python_source=python_source, + startup_exposure=_startup_exposure, ): if command_string is None or _command_string_depth >= 8: return True if _has_destructive_command_string(command_string): return True + command_closers: set[int] = set() + command_limits = _shell_quoted_candidate_limits( + command_string, check_runtime, quote_closers=command_closers + ) if _has_shell_command_word_exhaustion( command_string, check_runtime, + structural_quote_closers=command_closers, + quote_limits=command_limits, _command_string_depth=_command_string_depth + 1, + _quoted_printf_allowed=quoted_printf_allowed, + _startup_exposure=startup_context, ): return True return False @@ -2639,6 +3005,8 @@ def _has_runtime_command_operand_exhaustion( start: int, word_end: int, check_runtime: Callable[[], None], + *, + quoted_printf_allowed: bool = True, ) -> bool: """Return whether a runtime-selected command's operands are undecidable.""" tokens, command_end, exhausted = _bounded_shell_tokens( @@ -2646,6 +3014,7 @@ def _has_runtime_command_operand_exhaustion( start, word_end, check_runtime=check_runtime, + quoted_printf_allowed=quoted_printf_allowed, ) if exhausted and command_end - start < _ROOT_GLOB_COMMAND_CHARS: # Earlier prose (for example "row-first") cannot supply this @@ -2670,16 +3039,32 @@ def _shell_command_strings( *, perl_eval_blocks: bool = False, python_source: _PythonSourceOwnership | None = None, -) -> Iterator[str | None]: + startup_exposure: list[bool] | None = None, +) -> Iterator[tuple[str | None, list[bool]]]: """Yield bounded strings reparsed by ``eval`` or a shell ``-c`` wrapper.""" - for clause_start in _shell_clause_starts(content, check_runtime): + nested_substitutions = startup_exposure is not None and ("$(" in content or "`" in content) + if startup_exposure is None: + startup_exposure = [False] + startup_work = [4 * len(content) + _ROOT_GLOB_COMMAND_CHARS] + for clause_start in _nonempty_shell_clause_starts(content, check_runtime): check_runtime() + if not startup_exposure[0]: + startup_exposure[0] = _shell_clause_sets_startup( + content, clause_start, check_runtime, startup_work + ) + starts_shell = [False] recognized, command_string = _command_string_from_clause( content, clause_start, check_runtime, perl_eval_blocks=perl_eval_blocks, + startup_exposure=startup_exposure, + starts_shell=starts_shell, ) + if nested_substitutions and startup_exposure[0]: + # Quoted substitutions can launch shells outside this clause walk. + yield None, startup_exposure + return if not recognized: continue if python_source is not None: @@ -2691,7 +3076,274 @@ def _shell_command_strings( # ``signal.alarm(timeout)`` names no ``timeout`` command. # Shell text can only be reparsed from a string literal. continue - yield command_string + # eval shares this shell's settings; a child shell has its own scope. + # Keep exposure monotonic across eval, including settings in its body. + yield command_string, [False] if starts_shell[0] else startup_exposure + + +def _shell_clause_sets_startup( + content: str, + start: int, + check_runtime: Callable[[], None], + remaining_work: list[int], +) -> bool: + """Find explicit startup settings in bounded command-prefix syntax.""" + source = content[start : start + _ROOT_GLOB_COMMAND_CHARS + 1] + cursor = 0 + mode: str | None = None + setter_mode: str | None = None + skip_operand = False + read_options = True + read_array = False + printf_options = True + setting_namerefs = False + + def startup_target(word: str) -> bool: + # A setter can modify an already exported variable even without a + # preceding export in this source. Dynamic destinations remain unknown. + return bool(re.fullmatch(r"(?:BASH_ENV|ENV)(?:\[.*\])?", word, re.DOTALL)) or any( + marker in word + for marker in (_DYNAMIC_SHELL_WORD_SENTINEL, _RUNTIME_SHELL_PARAMETER_SENTINEL) + ) + + for _ in range(_SHELL_COMMAND_STRING_ARGUMENTS): + check_runtime() + while cursor < len(source): + if source.startswith("\\\r\n", cursor): + cursor += 3 + elif source.startswith("\\\n", cursor): + cursor += 2 + elif source[cursor] in " \t": + cursor += 1 + else: + break + if cursor >= len(source) or source[cursor] in "\r\n;|&(){}#": + return cursor > _ROOT_GLOB_COMMAND_CHARS or skip_operand + word_start = cursor + word, cursor, limited = _next_shell_invocation_word(source, cursor, check_runtime) + remaining_work[0] -= cursor - word_start + if remaining_work[0] < 0 or limited or cursor > _ROOT_GLOB_COMMAND_CHARS: + # Proof work is linear in the containing source, even when clause + # starts overlap malformed quoted words or nested expansions. + return True + if word is None: + return False + if skip_operand: + skip_operand = False + continue + if mode == "setter-target": + if startup_target(word): + return True + mode = setter_mode + continue + if mode == "printf": + if any( + marker in word + for marker in (_DYNAMIC_SHELL_WORD_SENTINEL, _RUNTIME_SHELL_PARAMETER_SENTINEL) + ): + return True + if printf_options and word == "--": + printf_options = False + continue + if printf_options and word == "-v": + setter_mode = "printf" + mode = "setter-target" + continue + if printf_options and word.startswith("-v"): + if startup_target(word[2:]): + return True + continue + # Bash's %n writes to a variable named by its argument, even without + # -v. Escaped percent pairs cannot introduce a conversion. + return re.search(r"%[-+ #0'.*0-9]*[hjlLtz]*n", word.replace("%%", "")) is not None + if mode == "read": + if read_options and word == "--": + read_options = False + continue + if read_options and word.startswith("-") and word != "-": + for index, option in enumerate(word[1:], 2): + if option in "Eers": + continue + if option not in "adinNptu": + return True # Unsupported option arity cannot prove the target. + attached = word[index:] + if option == "a": + read_array = True + if attached: + if startup_target(attached): + return True + else: + setter_mode = "read" + mode = "setter-target" + else: + skip_operand = not attached + break + continue + read_options = False + if not read_array and startup_target(word): + return True + continue + assignment = re.fullmatch(r"(BASH_ENV|ENV)(\+?=)(.*)", word, re.DOTALL) + if assignment is not None and ( + mode in {"env", "setting"} or _SHELL_ASSIGNMENT_PREFIX_RE.match(source, word_start) + ): + if assignment[2] == "+=" or assignment[3]: + return True + continue + if mode == "setting": + destination = word.partition("=")[0].removesuffix("+") + if startup_target(destination): + return True # Exporting an unknown inherited value is unresolved. + if setting_namerefs and re.fullmatch(r"-[A-Za-z]*n[A-Za-z]*", word): + # A name reference can redirect a later assignment to a startup + # variable without spelling that variable at the write site. + return True + if word.startswith(("-", "+")) or re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_]*(?:=.*)?", word, re.DOTALL + ): + continue + return False + if mode == "env": + if re.match(r"[A-Za-z_][A-Za-z0-9_]*=", word) is not None: + continue + if word in {"-i", "--ignore-environment", "--"} or re.fullmatch( + r"(?:--unset|--chdir)=.+", word + ): + continue + if word in {"-u", "--unset", "-C", "--chdir"}: + skip_operand = True + continue + if word.startswith("-"): + return True + mode = None + elif mode in {"command", "builtin"}: + if word == "--" or (mode == "command" and word == "-p"): + continue + if word.startswith("-"): + return False + mode = None + command = word.casefold().rsplit("/", 1)[-1] + if command in {"export", "declare", "typeset", "readonly", "local"}: + mode = "setting" + setting_namerefs = command in {"declare", "typeset", "local"} + elif command in {"env", "command", "builtin", "read", "printf"}: + mode = command + elif command in _SHELL_CLAUSE_PREFIX_WORDS or _SHELL_ASSIGNMENT_PREFIX_RE.match( + source, word_start + ): + continue + else: + return False + return True + + +def _shell_redefines_printf(content: str, check_runtime: Callable[[], None]) -> bool: + """Do not widen builtin evaluation across an explicit local replacement.""" + work = 4 * len(content) + _ROOT_GLOB_COMMAND_CHARS + for start in _nonempty_shell_clause_starts(content, check_runtime): + check_runtime() + source = content[start : start + _ROOT_GLOB_COMMAND_CHARS + 1] + cursor = 0 + wrapper: str | None = None + for _ in range(8): + word_start = cursor + while word_start < len(source): + if source.startswith("\\\r\n", word_start): + word_start += 3 + elif source.startswith("\\\n", word_start): + word_start += 2 + elif source[word_start] in " \t": + word_start += 1 + else: + break + word, end, limited = _next_shell_invocation_word(source, cursor, check_runtime) + work -= end - cursor + cursor = end + if work < 0 or limited or cursor > _ROOT_GLOB_COMMAND_CHARS: + return True + if _SHELL_ASSIGNMENT_PREFIX_RE.match(source, word_start): + continue + if wrapper is not None: + if word == "--" or (wrapper == "command" and word == "-p"): + continue + if word is None or word.startswith("-"): + break + wrapper = None + if word in {"command", "builtin"}: + wrapper = word + continue + if word in _SHELL_CLAUSE_PREFIX_WORDS or word in {"if", "while", "until"}: + continue + if word in {"eval", "source", ".", "enable", "trap"}: + # Imported/evaluated code, handlers and builtin configuration + # can replace printf in this shell. + # Retain the earlier narrow decoder across those unknown effects. + return True + if word == "function": + name, end, limited = _next_shell_invocation_word(source, cursor, check_runtime) + work -= end - cursor + if limited or work < 0 or end > _ROOT_GLOB_COMMAND_CHARS or name == "printf": + return True + break + if word == "alias": + for _ in range(_SHELL_COMMAND_STRING_ARGUMENTS): + while cursor < len(source) and source[cursor] in " \t": + cursor += 1 + if cursor >= len(source) or source[cursor] in "\r\n;|&()#": + break + alias, end, limited = _next_shell_invocation_word(source, cursor, check_runtime) + work -= end - cursor + cursor = end + if limited or work < 0 or cursor > _ROOT_GLOB_COMMAND_CHARS: + return True + if alias is None: + break + if alias.startswith("printf=") or any( + marker in alias + for marker in ( + _DYNAMIC_SHELL_WORD_SENTINEL, + _RUNTIME_SHELL_PARAMETER_SENTINEL, + ) + ): + return True + else: + return True + break + if word != "printf": + break + # Bash accepts line continuations around the empty parameter list. + for expected in "()": + while cursor < len(source): + if source.startswith("\\\r\n", cursor): + cursor += 3 + elif source.startswith("\\\n", cursor): + cursor += 2 + elif source[cursor] in " \t": + cursor += 1 + else: + break + if cursor > _ROOT_GLOB_COMMAND_CHARS: + return True + if cursor == len(source) or source[cursor] != expected: + break + cursor += 1 + else: + return True + break + else: + return True + return False + + +def _nonempty_shell_clause_starts(content: str, check_runtime: Callable[[], None]) -> Iterator[int]: + """Avoid reparsing the same trailing command across empty physical lines.""" + for start in _shell_clause_starts(content, check_runtime): + while start < len(content) and content[start] in " \t": + if start % 4096 == 0: + check_runtime() + start += 1 + if start < len(content) and content[start] not in "\r\n;|&(){}#": + yield start def _shell_clause_starts( @@ -2799,14 +3451,34 @@ def _command_string_from_clause( check_runtime: Callable[[], None], *, perl_eval_blocks: bool = False, + literal_command_spans: list[tuple[int, int]] | None = None, + startup_exposure: list[bool] | None = None, + starts_shell: list[bool] | None = None, ) -> tuple[bool, str | None]: """Resolve a bounded wrapper chain to an ``eval`` or shell command string.""" cursor = start pending: str | None = None wrapper_seen = False + last_word_start = start + + def observe_assignment(word: str) -> None: + if startup_exposure is not None and re.fullmatch(r"(?:BASH_ENV|ENV)=.+", word, re.DOTALL): + startup_exposure[0] = True def next_word() -> tuple[str | None, bool]: - nonlocal cursor + nonlocal cursor, last_word_start + last_word_start = cursor + while literal_command_spans is not None and last_word_start < len(content): + if last_word_start % 4096 == 0: + check_runtime() + if content[last_word_start].isspace(): + last_word_start += 1 + elif content.startswith("\\\r\n", last_word_start): + last_word_start += 3 + elif content.startswith("\\\n", last_word_start): + last_word_start += 2 + else: + break word, cursor, limited = _next_shell_invocation_word(content, cursor, check_runtime) return word, limited @@ -2822,8 +3494,20 @@ def resolved_command_string(word: str | None, limited: bool) -> str | None: return None return word + def commandless_wrapper() -> tuple[bool, None]: + # env/command/builtin can end without invoking another command. A + # redirection or opening parenthesis may still hide a following one. + ended = ( + cursor >= len(content) + or content[cursor] == "#" + or _ends_wrapped_clause(content, cursor) + ) + return not ended, None + for _ in range(32): while cursor < len(content) and content[cursor].isspace(): + if cursor % 4096 == 0: + check_runtime() cursor += 1 redirection = _SHELL_REDIRECTION_PREFIX_RE.match(content, cursor) if redirection is not None: @@ -2840,6 +3524,7 @@ def resolved_command_string(word: str | None, limited: bool) -> str | None: if word is None: return False, None if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", word, re.DOTALL) is not None: + observe_assignment(word) continue command = word.casefold().rsplit("/", 1)[-1] if command in _SHELL_CLAUSE_PREFIX_WORDS: @@ -2866,15 +3551,91 @@ def resolved_command_string(word: str | None, limited: bool) -> str | None: return False, None if option.startswith("-") and not option.startswith("--"): if "c" in option[1:]: + if starts_shell is not None: + starts_shell[0] = True + if startup_exposure is not None and startup_exposure[0]: + # A startup file can replace the child's builtins. + return True, None command_string, limited = next_word() - return True, resolved_command_string(command_string, limited) + resolved = resolved_command_string(command_string, limited) + if ( + literal_command_spans is not None + and resolved is not None + and content[last_word_start : last_word_start + 1] == "'" + and content[cursor - 1 : cursor] == "'" + and cursor - last_word_start == len(resolved) + 2 + and content[last_word_start + 1 : cursor - 1] == resolved + ): + # One literal has an exact source-to-body mapping. + # Concatenation, expansion and ANSI-C escapes cannot + # lend their decoded offsets to the original text. + literal_command_spans.append((last_word_start + 1, cursor - 1)) + return True, resolved continue if option.startswith("+") or option.startswith("--"): continue return False, None return True, None - if command in {"env", "command", "nohup"}: + if command == "env": + wrapper_seen = True + for _ in range(_SHELL_COMMAND_STRING_ARGUMENTS): + option, limited = next_word() + if limited: + return True, None + if option is None: + return commandless_wrapper() + if option == "--": + pending, limited = next_word() + if limited: + return True, None + if pending is None: + return commandless_wrapper() + break + if re.match(r"[A-Za-z_][A-Za-z0-9_]*=", option) is not None: + observe_assignment(option) + continue + if option in {"-i", "--ignore-environment"} or re.fullmatch( + r"(?:--unset|--chdir)=.+", option + ): + continue + if option in {"-u", "--unset", "-C", "--chdir"}: + operand, limited = next_word() + if limited or operand is None: + return True, None + continue + if option.startswith("-"): + return True, None + pending = option + break + else: + return True, None + continue + if command in {"command", "builtin"}: + wrapper_seen = True + for _ in range(_SHELL_COMMAND_STRING_ARGUMENTS): + option, limited = next_word() + if limited: + return True, None + if option is None: + return commandless_wrapper() + if option == "--": + pending, limited = next_word() + if limited: + return True, None + if pending is None: + return commandless_wrapper() + break + if command == "command" and option == "-p": + continue + if option.startswith("-"): + return False, None + pending = option + break + else: + return True, None + continue + if command == "nohup": wrapper_seen = True continue if command in {"sudo", "nice", "xargs"}: @@ -3139,6 +3900,23 @@ def comment(self, position: int) -> tuple[int, int] | None: return None return owner + def starts_string_body(self, position: int) -> bool: + """Return whether a candidate starts the value of a proven string token.""" + owner = self._owner(position) + return ( + owner is not None + and self._content[owner[0]] != "#" + and position == _python_string_body(self._content, *owner)[0] + ) + + def is_string_delimiter(self, position: int) -> bool: + """Return whether Python owns a string prefix or opening/closing quote.""" + owner = self._owner(position) + if owner is None or self._content[owner[0]] == "#": + return False + start, end = _python_string_body(self._content, *owner) + return not start <= position < end + def string_value(self, start: int, end: int) -> tuple[str, int, int] | None: """Return the proven string value that holds a shell word, and its span. @@ -3329,6 +4107,7 @@ def _bounded_shell_tokens( body_start: int, *, check_runtime: Callable[[], None] | None = None, + quoted_printf_allowed: bool = True, ) -> tuple[tuple[_ShellToken, ...], int, bool]: """Return argument words from one security-view-bounded shell command.""" runtime_check = check_runtime or (lambda: None) @@ -3468,6 +4247,7 @@ def flush(*, complete: bool = True) -> None: content, cursor, substitution_end, + quoted_result=quoted_printf_allowed, ) parse_limited = parse_limited or ( static_value is None @@ -5128,6 +5908,7 @@ def _tm1_candidates( *, direct_shell_only: bool = False, shell_content: str | None = None, + check_runtime: Callable[[], None], ) -> Iterator[tuple[int, int, str, float, bool, int | None, int | None, str | None]]: def direct_matches( pattern_index: int, @@ -5235,6 +6016,7 @@ def destructive_candidates( seen_commands: set[tuple[int, int]] = set() covered_until = 0 for command_start, body_start in _destructive_command_words(candidate_content): + check_runtime() if command_start < covered_until: continue command_key = (command_start, body_start) @@ -5246,12 +6028,15 @@ def destructive_candidates( # character-by-character shell parse for every explanatory ``rm`` noun. if _is_root_glob_documentation(candidate_content, command_start, body_start): continue - tokens, command_end, _ = _bounded_shell_tokens( + tokens, command_end, exhausted = _bounded_shell_tokens( candidate_content, command_start, body_start, + check_runtime=check_runtime, ) - covered_until = max(covered_until, command_end) + # An unresolved quote cannot own and hide later executable candidates. + if not exhausted: + covered_until = max(covered_until, command_end) command = candidate_content[command_start:command_end] if _has_destructive_root_glob(tokens) or _has_destructive_root_path(tokens): yield command_start, command_end, command, 0.9, False, None, None, None @@ -5414,6 +6199,7 @@ def _markdown_shell_text( *, complete_context: bool = True, word_apostrophes: bool = False, + shell_fence_ranges: list[tuple[int, int]] | None = None, hardware_apostrophe_ambiguity: list[bool] | None = None, ) -> str: """Mask Markdown delimiters while retaining code and exact source offsets. @@ -5423,6 +6209,8 @@ def _markdown_shell_text( inside their bodies. Pair equal-length runs in linear time. ``word_apostrophes`` selects the prose reading, which also masks the possessives and contractions of plain prose lines. + ``shell_fence_ranges`` receives closed, explicitly labelled shell bodies + whose quote ownership can be checked independently of surrounding prose. """ output = list(content) prose_apostrophes: set[int] = set() @@ -5467,6 +6255,7 @@ def mask_inline_delimiters() -> None: fence_literal_ownership = True fence_has_literal_apostrophe = False fence_has_nonliteral_apostrophe = False + shell_fence_start: int | None = None quoted_block = False html_end: str | None = None paragraph_open = False @@ -5560,6 +6349,9 @@ def mask_inline_delimiters() -> None: begin, end = closing_fence.span(1) body_start = offset + len(stripped) - len(fence_body) output[body_start + begin : body_start + end] = " " * (end - begin) + if shell_fence_start is not None and shell_fence_ranges is not None: + shell_fence_ranges.append((shell_fence_start, offset)) + shell_fence_start = None fence = None fence_literal_apostrophes.clear() fence_literal_ownership = True @@ -5659,6 +6451,14 @@ def mask_inline_delimiters() -> None: fence_has_literal_apostrophe = False fence_has_nonliteral_apostrophe = False output[offset + prefix + begin : offset + prefix + end] = " " * (end - begin) + if ( + complete_context + and shell_fence_ranges is not None + and not has_list_marker + and not paragraph_in_list + and stripped[prefix + end :].strip() in {"bash", "sh", "shell"} + ): + shell_fence_start = offset + len(line) else: minimum_indent = ( column if has_list_marker else paragraph_list_indent if paragraph_in_list else 0 @@ -5839,7 +6639,9 @@ def _has_bounded_parse_exhaustion( # -replace values have their own narrowly proven expression context. structural_quote_closers = None structural_quote_openers = None + shell_fence_ranges: list[tuple[int, int]] = [] json_strings: list[tuple[int, int]] = [] + json_operand_spans: list[tuple[int, int]] = [] python_source = None javascript_source = None if file_type == "perl" and complete_context: @@ -5853,17 +6655,24 @@ def _has_bounded_parse_exhaustion( # Interpolated templates in files with shell calls retain conservative # bounds because this analyzer does not prove their data flow. javascript_source = _JavaScriptTemplateOwnership(content, check_runtime) + if file_type in {"json", "markdown"} and complete_context: + json_strings = validated_json_string_spans(content, check_runtime) + structural_quote_closers = {end - 1 for _, end in json_strings} + structural_quote_openers = {start for start, _ in json_strings} + # Raw substitution delimiters must survive Markdown's inline projection. + json_operand_spans = [ + (start, end) + for start, end in json_strings + if "$(" not in content[start:end] and "`" not in content[start:end] + ] if file_type == "markdown": - if complete_context: - json_strings = validated_json_string_spans(content, check_runtime) - structural_quote_closers = {end - 1 for _, end in json_strings} - structural_quote_openers = {start for start, _ in json_strings} hardware_apostrophe_ambiguity = [False] content = _markdown_shell_text( content, check_runtime, complete_context=complete_context, word_apostrophes=word_apostrophes, + shell_fence_ranges=shell_fence_ranges, hardware_apostrophe_ambiguity=hardware_apostrophe_ambiguity, ) if hardware_apostrophe_ambiguity[0]: @@ -5876,6 +6685,30 @@ def _has_bounded_parse_exhaustion( if file_type == "javascript" and complete_context else content ) + if file_type == "python" and complete_context: + shell_word_content = python_browser_template_shell_text(content, check_runtime) + quote_limits = None + if file_type == "shell" and complete_context: + # A closing delimiter in proven shell source cannot open a new + # command word. Nested substitutions retain their own lexical walk, + # so their executable candidates remain independently inspected. + structural_quote_closers = set() + quote_limits = _shell_quoted_candidate_limits( + content, check_runtime, quote_closers=structural_quote_closers + ) + elif file_type == "markdown" and complete_context: + quote_limits = {} + for start, end in shell_fence_ranges: + check_runtime() + closers: set[int] = set() + limits = _shell_quoted_candidate_limits( + content[start:end], check_runtime, quote_closers=closers + ) + quote_limits.update( + {start + position: start + limit for position, limit in limits.items()} + ) + assert structural_quote_closers is not None + structural_quote_closers.update(start + position for position in closers) # Perl ``eval BLOCK`` is recognized from its clause-initial ``eval`` and the # adjacent brace alone, so unlike quote ownership it needs no full context. if _has_shell_command_word_exhaustion( @@ -5886,14 +6719,11 @@ def _has_bounded_parse_exhaustion( python_source=python_source, javascript_source=javascript_source, perl_eval_blocks=file_type == "perl", - quote_limits=( - _shell_quoted_candidate_limits(content, check_runtime) - if file_type == "shell" and complete_context - else None - ), + quote_limits=quote_limits, file_type=file_type, ): return True + json_starts = [start for start, _ in json_operand_spans] covered_until = 0 for command_start, body_start in _destructive_command_words(content): check_runtime() @@ -5910,6 +6740,28 @@ def _has_bounded_parse_exhaustion( body_start, check_runtime=check_runtime, ) + if exhausted: + word = content[command_start:body_start] + boundary = quote_limits.get(command_start) if quote_limits is not None else None + index = bisect_right(json_starts, command_start) - 1 + if boundary is None and index >= 0: + begin, end = json_operand_spans[index] + if begin < command_start < body_start < end: + boundary = end - 1 + if ( + boundary is not None + and not any(character in word for character in "$`'\"\\") + and word.casefold().rsplit("/", 1)[-1] in _DESTRUCTIVE_COMMAND_BASENAMES + ): + # A proven enclosing closer terminates operands; reconstructed + # command words retain their original uncertainty and bounds. + tokens, command_end, exhausted = _bounded_shell_tokens( + content[command_start:boundary], + 0, + body_start - command_start, + check_runtime=check_runtime, + ) + command_end += command_start covered_until = max(covered_until, command_end) if exhausted or _has_unsupported_brace_expansion(tokens): return True @@ -6085,6 +6937,7 @@ def ctx(start: int) -> str: content, direct_shell_only=_direct_shell_only, shell_content=shell_content, + check_runtime=runtime_check, ): runtime_check() variable_match = variable_matches.get((match_start, match_end)) diff --git a/src/skillspector/python_tokens.py b/src/skillspector/python_tokens.py index a8beee090..0f17ad4f4 100644 --- a/src/skillspector/python_tokens.py +++ b/src/skillspector/python_tokens.py @@ -20,9 +20,11 @@ from bisect import bisect_right from collections import OrderedDict from collections.abc import Callable +from html.parser import HTMLParser from threading import Lock -from skillspector.python_ast import MAX_PYTHON_AST_SOURCE_CHARS +from skillspector.javascript_tokens import browser_escaped_list_shell_text +from skillspector.python_ast import MAX_PYTHON_AST_SOURCE_CHARS, parse_python_source # The tokenizer and the compiler disagree about a carriage return that does # not start CRLF, so such source never acquires token ownership. @@ -75,6 +77,9 @@ OrderedDict() ) +_HTML_SCRIPT_OPEN_RE = re.compile(r"", re.IGNORECASE) +_HTML_SCRIPT_CLOSE_RE = re.compile(r"", re.IGNORECASE) + def python_literal_spans( content: str, @@ -182,6 +187,681 @@ def offset(position: tuple[int, int]) -> int: return tuple(starts), tuple(ends) +def python_fstring_literal_spans( + content: str, + start: int, + end: int, + check_runtime: Callable[[], None], +) -> PythonLiteralSpans | None: + """Return exact constant-text runs of one proven outer f-string token. + + Replacement expressions and their format specifications never gain literal + ownership. Doubled braces keep both original source bytes in their run. + """ + owners = python_literal_spans(content, check_runtime) + if owners is None: + return None + owner_index = bisect_right(owners[0], start) - 1 + if owner_index < 0 or (owners[0][owner_index], owners[1][owner_index]) != (start, end): + return None + source = content[start:end] + line_starts = [0, *(match.end() for match in re.finditer("\n", source))] + starts: list[int] = [] + ends: list[int] = [] + template_depth = field_depth = 0 + complete = False + + def offset(position: tuple[int, int]) -> int: + return start + line_starts[position[0] - 1] + position[1] + + try: + for index, token in enumerate(tokenize.generate_tokens(io.StringIO(source).readline)): + if index % 256 == 0: + check_runtime() + if token.type in _PYTHON_TEMPLATE_STRING_STARTS: + if not template_depth and ( + token.type != tokenize.FSTRING_START or offset(token.start) != start + ): + return None + template_depth += 1 + elif token.type in _PYTHON_TEMPLATE_STRING_ENDS: + template_depth -= 1 + if template_depth == 0: + complete = offset(token.end) == end and field_depth == 0 + elif template_depth == 1 and token.type == tokenize.OP: + if token.string == "{": + field_depth += 1 + elif token.string == "}": + field_depth -= 1 + elif template_depth == 1 and not field_depth and token.type == tokenize.FSTRING_MIDDLE: + literal_start, literal_end = offset(token.start), offset(token.end) + if ( + token.string.endswith(("{", "}")) + and content[literal_end : literal_end + 1] == token.string[-1] + ): + literal_end += 1 + raw = content[literal_start:literal_end] + if raw.replace("{{", "{").replace("}}", "}") != token.string: + return None + if ends and ends[-1] == literal_start: + ends[-1] = literal_end + else: + starts.append(literal_start) + ends.append(literal_end) + except (tokenize.TokenError, SyntaxError, ValueError, IndexError): + return None + if not complete or template_depth or field_depth: + return None + return tuple(starts), tuple(ends) + + +def _browser_html_fields( + literal: ast.JoinedStr, + field_kind: Callable[[ast.AST], str | None], + check_runtime: Callable[[], None], +) -> bool: + """Accept one ordinary HTML document with inert, escaped Python fields.""" + marker = "__skillspector_python_html_field_" + fields: dict[str, str | None] = {} + chunks: list[str] = [] + for index, value in enumerate(literal.values): + check_runtime() + if isinstance(value, ast.Constant) and isinstance(value.value, str): + if marker in value.value: + return False + chunks.append(value.value) + elif ( + isinstance(value, ast.FormattedValue) + and value.conversion == -1 + and value.format_spec is None + ): + key = f"{marker}{index}__" + fields[key] = field_kind(value.value) + chunks.append(key) + else: + return False + if any(kind is None for kind in fields.values()): + return False + + class Document(HTMLParser): + valid = True + context = "" + executable_scripts = 0 + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + check_runtime() + if tag not in { + "html", + "head", + "title", + "meta", + "style", + "body", + "div", + "span", + "aside", + "main", + "section", + "input", + "button", + "strong", + "ul", + "li", + "h1", + "h2", + "h3", + "p", + "script", + }: + self.valid = False + names = [name for name, _ in attrs] + if len(names) != len(set(names)): + self.valid = False + for name, value in attrs: + if name not in { + "id", + "class", + "title", + "lang", + "charset", + "name", + "content", + "type", + "placeholder", + } and not name.startswith("data-"): + self.valid = False + if value is not None and marker in value: + self.valid = False + attributes = dict(attrs) + if tag == "meta" and not ( + set(attributes) == {"charset"} + or attributes.get("name") == "viewport" + and set(attributes) == {"name", "content"} + ): + self.valid = False + if tag == "script": + if not attrs: + self.executable_scripts += 1 + self.context = "script" + elif attributes.get("type") == "application/json" and set(attributes) <= { + "id", + "type", + }: + self.context = "json" + else: + self.valid = False + elif tag in {"style", "title"}: + self.context = tag + + def handle_endtag(self, tag: str) -> None: + if tag in {"script", "style", "title"}: + self.context = "" + + def handle_data(self, data: str) -> None: + check_runtime() + if self.context == "style" and re.search( + r"(?:@import|url\s*\(|expression\s*\(|\\)", data, re.IGNORECASE + ): + self.valid = False + if marker not in data: + return + if self.context == "json": + if fields.pop(data.strip(), None) != "json": + self.valid = False + elif self.context in {"script", "style"}: + self.valid = False + else: + for key in re.findall(marker + r"[0-9]+__", data): + if fields.pop(key, None) != "text": + self.valid = False + + def handle_comment(self, data: str) -> None: + if marker in data: + self.valid = False + + parser = Document(convert_charrefs=False) + parser.feed("".join(chunks)) + parser.close() + return parser.valid and parser.executable_scripts == 1 and not fields + + +def _python_browser_strings( + content: str, check_runtime: Callable[[], None] +) -> set[tuple[int, int]]: + """Prove explicit pathlib consumers and escaped HTML field boundaries. + + Only direct write_text arguments and direct returns from unaliased local + functions used exclusively by those writes qualify. Unknown consumers, + rebinding, mutation and unsupported scope forms retain the shell reading. + Ordinary pathlib/JSON/text protocols are assumed, as with Node assertions; + explicit local protocol overrides fall back. This is not a proof against + arbitrary runtime getters or monkeypatching outside the inspected source. + """ + with _PYTHON_OWNERSHIP_PARSE_LOCK, warnings.catch_warnings(): + warnings.simplefilter("ignore") + parsed = parse_python_source(content, "") + tree = parsed.tree + if tree is None: + return set() + parents: dict[ast.AST, ast.AST] = {} + scopes: dict[ast.AST, ast.AST] = {} + bindings: dict[tuple[ast.AST, str], list[ast.AST]] = {} + returns: dict[ast.AST, list[ast.Return]] = {} + references: dict[str, list[ast.Name]] = {} + nodes: list[ast.AST] = [] + pending: list[tuple[ast.AST, ast.AST]] = [(tree, tree)] + while pending: + node, scope = pending.pop() + if len(nodes) >= 100_000: + return set() + if len(nodes) % 256 == 0: + check_runtime() + nodes.append(node) + scopes[node] = scope + if isinstance(node, ast.Return): + returns.setdefault(scope, []).append(node) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + references.setdefault(node.id, []).append(node) + names: list[str] = [] + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + names = [node.id] + elif isinstance(node, ast.arg): + names = [node.arg] + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names = [node.name] + if node.name in { + "replace", + "__str__", + "__format__", + "__fspath__", + "__html__", + "__getattribute__", + "__getattr__", + "__getitem__", + }: + return set() + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names = [alias.asname or alias.name.split(".")[0] for alias in node.names] + elif isinstance(node, ast.ExceptHandler) and node.name: + names = [node.name] + elif isinstance(node, (ast.Global, ast.Nonlocal, ast.NamedExpr, ast.Match)): + return set() + if ( + "*" in names + or isinstance(node, ast.Attribute) + and isinstance(node.ctx, (ast.Store, ast.Del)) + and node.attr + in { + "Path", + "write_text", + "resolve", + "cwd", + "home", + "escape", + "dumps", + "replace", + "__code__", + "__defaults__", + "__kwdefaults__", + } + ): + return set() + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + and node.id in {"eval", "exec", "setattr", "delattr", "globals", "locals", "vars"} + or isinstance(node, ast.Attribute) + and node.attr in {"setattr", "delattr", "__dict__", "__setattr__"} + ): + return set() + for name in names: + if name in { + "replace", + "__str__", + "__format__", + "__fspath__", + "__html__", + "__getattribute__", + "__getattr__", + "__getitem__", + }: + return set() + bindings.setdefault((scope, name), []).append(node) + child_scope = ( + node + if isinstance( + node, + ( + ast.FunctionDef, + ast.AsyncFunctionDef, + ast.ClassDef, + ast.Lambda, + ast.ListComp, + ast.SetComp, + ast.DictComp, + ast.GeneratorExp, + ), + ) + else scope + ) + for child in ast.iter_child_nodes(node): + parents[child] = node + pending.append((child, child_scope)) + + budget = 100_000 + + def binding(name: ast.Name) -> ast.AST | None: + nonlocal budget + budget -= 1 + if budget <= 0: + return None + if budget % 256 == 0: + check_runtime() + scope = scopes[name] + first = True + while True: + # Methods do not close over class-local bindings. Function closures + # and comprehension bindings do shadow a module import. + candidates = ( + [] + if not first and isinstance(scope, ast.ClassDef) + else bindings.get((scope, name.id), []) + ) + if candidates: + return candidates[0] if len(candidates) == 1 else None + if scope is tree: + return None + scope = scopes[scope] + first = False + budget -= 1 + if budget <= 0: + return None + + def imported(node: ast.AST, module: str, name: str | None = None) -> bool: + if not isinstance(node, ast.Name): + return False + owner = binding(node) + if isinstance(owner, ast.Import) and name is None: + return any( + alias.name == module and alias.asname is None and node.id == module + for alias in owner.names + ) + if isinstance(owner, ast.ImportFrom) and owner.module == module and owner.level == 0: + return any( + alias.name == name and alias.asname is None and node.id == name + for alias in owner.names + ) + return False + + def assigned(node: ast.Name) -> ast.AST | None: + owner = binding(node) + parent = parents.get(owner) if owner is not None else None + if isinstance(parent, ast.Assign) and parent.targets == [owner]: + return parent.value + if isinstance(parent, ast.AnnAssign) and parent.target is owner: + return parent.value + return None + + def annotation(node: ast.AST) -> bool: + parent = parents.get(node) + return bool( + isinstance(parent, ast.arg) + and parent.annotation is node + or isinstance(parent, (ast.FunctionDef, ast.AsyncFunctionDef)) + and parent.returns is node + or isinstance(parent, ast.AnnAssign) + and parent.annotation is node + ) + + # Audit every explicit use of the trusted imports, including aliases that + # could let another call replace a constructor or escaping implementation. + for name in ("Path", "json", "escape"): + for reference in references.get(name, []): + check_runtime() + if annotation(reference): + continue + module, member = { + "Path": ("pathlib", "Path"), + "json": ("json", None), + "escape": ("html", "escape"), + }[name] + if not imported(reference, module, member): + return set() + parent = parents.get(reference) + if ( + name in {"Path", "escape"} + and isinstance(parent, ast.Call) + and parent.func is reference + ): + continue + if isinstance(parent, ast.Attribute) and isinstance(parent.ctx, ast.Load): + allowed = {"Path": {"cwd", "home"}, "json": {"loads", "dumps"}, "escape": set()}[ + name + ] + call = parents.get(parent) + if parent.attr in allowed and isinstance(call, ast.Call) and call.func is parent: + continue + return set() + + path_bindings: set[ast.AST] = set() + path_functions: set[ast.FunctionDef] = set() + + def path(node: ast.AST, depth: int = 0) -> bool: + if depth >= 16 or budget <= 0: + return False + if isinstance(node, ast.Name): + value = assigned(node) + owner = binding(node) + if owner is not None: + path_bindings.add(owner) + return value is not None and path(value, depth + 1) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + # An arbitrary RHS can run __rtruediv__ and replace the receiver. + native_text = isinstance(node.right, ast.JoinedStr) or ( + isinstance(node.right, ast.Constant) and isinstance(node.right.value, str) + ) + return native_text and path(node.left, depth + 1) + if not isinstance(node, ast.Call): + return False + if imported(node.func, "pathlib", "Path"): + return True + if isinstance(node.func, ast.Attribute): + if node.func.attr in {"cwd", "home"} and imported(node.func.value, "pathlib", "Path"): + return not node.args and not node.keywords + if node.func.attr == "resolve": + return path(node.func.value, depth + 1) + if isinstance(node.func, ast.Name): + function = binding(node.func) + if ( + isinstance(function, ast.FunctionDef) + and scopes[function] is tree + and not function.decorator_list + ): + path_functions.add(function) + values = returns.get(function, []) + return bool(values) and all( + part.value is not None and path(part.value, depth + 1) for part in values + ) + return False + + def written(node: ast.AST) -> bool: + scope = scopes[node] + if scope is not tree and ( + not isinstance(scope, ast.FunctionDef) or scopes[scope] is not tree + ): + return False + parent = parents.get(node) + return bool( + isinstance(parent, ast.Call) + and len(parent.args) == 1 + and parent.args[0] is node + and len(parent.keywords) == 1 + and parent.keywords[0].arg == "encoding" + and isinstance(parent.keywords[0].value, ast.Constant) + and parent.keywords[0].value.value in {"utf-8", "utf8", "UTF-8", "UTF8"} + and isinstance(parent.func, ast.Attribute) + and parent.func.attr == "write_text" + and path(parent.func.value) + ) + + def path_use(node: ast.AST, depth: int = 0) -> bool: + """A selected Path receiver cannot escape through a bound method/property.""" + if depth >= 16: + return False + parent = parents.get(node) + if isinstance(parent, ast.FormattedValue): + return True + if not isinstance(parent, ast.Attribute) or not isinstance(parent.ctx, ast.Load): + return False + if parent.attr == "parent": + return path_use(parent, depth + 1) + call = parents.get(parent) + if not isinstance(call, ast.Call) or call.func is not parent: + return False + if parent.attr in {"write_text", "mkdir", "as_uri"}: + # These return native scalar/None values, not the selected receiver. + return True + if parent.attr == "resolve" and not call.args and not call.keywords: + return path_use(call, depth + 1) + return False + + def field_kind(node: ast.AST, depth: int = 0) -> str | None: + if depth >= 16 or budget <= 0: + return None + if isinstance(node, ast.Name): + value = assigned(node) + return None if value is None else field_kind(value, depth + 1) + if not isinstance(node, ast.Call): + return None + if ( + imported(node.func, "html", "escape") + and len(node.args) == 1 + and not node.keywords + and isinstance(node.args[0], (ast.Constant, ast.Name, ast.Subscript)) + ): + return "text" + if ( + isinstance(node.func, ast.Attribute) + and node.func.attr == "replace" + and len(node.args) == 2 + and not node.keywords + ): + before, after = node.args + value = node.func.value + if ( + isinstance(before, ast.Constant) + and before.value == "") + ): + continue + check_runtime() + consumed = written(node) + parent = parents.get(node) + function = scopes[node] + if ( + isinstance(parent, ast.Return) + and isinstance(function, ast.FunctionDef) + and scopes[function] is tree + and not function.decorator_list + and returns.get(function) == [parent] + ): + uses = references.get(function.name, []) + consumed = bool(uses) and all( + binding(part) is function + and isinstance(parents[part], ast.Call) + and parents[part].func is part + and written(parents[part]) + for part in uses + ) + if consumed and _browser_html_fields(node, field_kind, check_runtime): + start_column = parsed.character_column(node.lineno, node.col_offset) + end_column = parsed.character_column(node.end_lineno, node.end_col_offset) + if start_column is not None and end_column is not None: + result.add( + ( + parsed.line_character_starts[node.lineno - 1] + start_column, + parsed.line_character_starts[node.end_lineno - 1] + end_column, + ) + ) + for reference_nodes in references.values(): + for reference in reference_nodes: + check_runtime() + owner = binding(reference) + if budget <= 0: + return set() + parent = parents.get(reference) + if owner in path_functions: + if not isinstance(parent, ast.Call) or parent.func is not reference: + return set() + if owner in path_bindings: + if not path_use(reference): + return set() + return result if budget > 0 else set() + + +def python_browser_template_shell_text(content: str, check_runtime: Callable[[], None]) -> str: + """Project display delimiters in constant browser scripts of HTML f-strings. + + Neither Python replacement fields nor escape decoding may establish a + script boundary. Original lengths and every non-delimiter byte survive. + """ + if "`" not in content or "" not in content.lower(): + return content + owners = python_literal_spans(content, check_runtime) + if owners is None: + return content + browser_strings = _python_browser_strings(content, check_runtime) + if not browser_strings: + return content + output: list[str] | None = None + for start, end in zip(*owners, strict=True): + check_runtime() + if (start, end) not in browser_strings: + continue + if content[start] == "#": + continue + body_start, body_end = python_string_body(content, start, end) + if "f" not in content[start:body_start].lower(): + continue + body = content[body_start:body_end] + if not body.lstrip().lower().startswith( + "" + ) or not body.rstrip().lower().endswith(""): + continue + runs = python_fstring_literal_spans(content, start, end, check_runtime) + if runs is None: + continue + for literal_start, literal_end in zip(*runs, strict=True): + search_start = literal_start + while ( + script := _HTML_SCRIPT_OPEN_RE.search(content, search_start, literal_end) + ) is not None: + check_runtime() + closing = _HTML_SCRIPT_CLOSE_RE.search(content, script.end(), literal_end) + if closing is None: + break + script_start, script_end = script.end(), closing.start() + search_start = closing.end() + raw = content[script_start:script_end] + if "\\" in raw: + continue + decoded: list[str] = [] + source_spans: list[tuple[int, int]] = [] + cursor = script_start + while cursor < script_end: + if len(decoded) % 4096 == 0: + check_runtime() + width = 2 if content[cursor : cursor + 2] in {"{{", "}}"} else 1 + decoded.append(content[cursor]) + source_spans.append((cursor, cursor + width)) + cursor += width + source = "".join(decoded) + projected = browser_escaped_list_shell_text(source, check_runtime) + if projected == source: + continue + if output is None: + output = list(content) + for position, (original, replacement) in enumerate( + zip(source, projected, strict=True) + ): + if position % 4096 == 0: + check_runtime() + if original != replacement: + lower, upper = source_spans[position] + output[lower:upper] = " " * (upper - lower) + check_runtime() + return content if output is None else "".join(output) + + class PythonStringClosers: """Lazily proven closing delimiters of one complete module's string literals. diff --git a/src/skillspector/security_reconstruction.py b/src/skillspector/security_reconstruction.py index 4751627c9..3c439cc2d 100644 --- a/src/skillspector/security_reconstruction.py +++ b/src/skillspector/security_reconstruction.py @@ -8,6 +8,7 @@ import json import re from array import array +from bisect import bisect_right from collections import Counter from collections.abc import Callable, Iterator from dataclasses import dataclass @@ -1246,9 +1247,14 @@ def _bounded_sentence_end( maximum_end: int, *, end_is_truncated: bool, + context_start: int | None = None, ) -> tuple[int, bool]: - for boundary_start, _ in _sentence_boundaries(text, start, min(len(text), maximum_end + 1)): - if boundary_start <= maximum_end: + # A directive can be inside a quoted string; retain that quote's state. + scan_start = start if context_start is None else context_start + for boundary_start, _ in _sentence_boundaries( + text, scan_start, min(len(text), maximum_end + 1) + ): + if start <= boundary_start <= maximum_end: return boundary_start, False exhausted = maximum_end < len(text) or end_is_truncated return maximum_end, exhausted @@ -1286,11 +1292,23 @@ def _is_fence_line(text: str, cursor: int) -> bool: ) -def _sentence_boundaries(text: str, start: int, end: int) -> Iterator[tuple[int, int]]: +def _sentence_boundaries( + text: str, + start: int, + end: int, + *, + quote_spans: tuple[list[int], list[int]] | None = None, + check_runtime: Callable[[], None] | None = None, +) -> Iterator[tuple[int, int]]: """Yield prose boundaries while ignoring punctuation inside balanced quotes.""" quote_close: str | None = None + quote_start = start cursor = start + next_runtime_check = start while cursor < end: + if check_runtime is not None and cursor >= next_runtime_check: + check_runtime() + next_runtime_check = cursor + 1024 character = text[cursor] if quote_close is None and character == "`" and _is_fence_line(text, cursor): # A Markdown code fence line delimits a block; it does not open a @@ -1304,6 +1322,9 @@ def _sentence_boundaries(text: str, start: int, end: int) -> Iterator[tuple[int, cursor += 2 continue if character == quote_close: + if quote_spans is not None: + quote_spans[0].append(quote_start) + quote_spans[1].append(cursor + 1) quote_close = None cursor += 1 continue @@ -1314,6 +1335,7 @@ def _sentence_boundaries(text: str, start: int, end: int) -> Iterator[tuple[int, and text[cursor - 1].isalnum() and text[cursor + 1].isalnum() ): + quote_start = cursor quote_close = _QUOTE_OPEN_TO_CLOSE[character] elif character == "\n": yield cursor, cursor + 1 @@ -1832,16 +1854,29 @@ def _classify_directive( directive: _Directive, *, end_is_truncated: bool, + context_start: int | None = None, ) -> _DirectiveClassification: if directive.exhausted: return _DirectiveClassification(None, False, True) + baseline = None + if context_start is not None: + baseline = _classify_directive(view, directive, end_is_truncated=end_is_truncated) + if baseline.active or not baseline.limited: + return baseline + + clause_start = _previous_sentence_boundary( + view.text, + directive.start, + MAX_MARKER_LOOKAHEAD_CHARS, + ) scope_cap = min(len(view.text), directive.end + MAX_MARKER_SCOPE_CHARS) scope_end, _ = _bounded_sentence_end( view.text, directive.end, scope_cap, end_is_truncated=end_is_truncated, + context_start=context_start, ) lookahead_cap = min(len(view.text), directive.end + MAX_MARKER_LOOKAHEAD_CHARS) lookahead_end, lookahead_exhausted = _bounded_sentence_end( @@ -1849,13 +1884,9 @@ def _classify_directive( directive.end, lookahead_cap, end_is_truncated=end_is_truncated, + context_start=context_start, ) first_sentence_end = lookahead_end - clause_start = _previous_sentence_boundary( - view.text, - directive.start, - MAX_MARKER_LOOKAHEAD_CHARS, - ) directive_clause = view.text[clause_start:first_sentence_end] if first_sentence_end < lookahead_cap and ( _FORWARD_PAYLOAD_REFERENCE_RE.search(directive_clause) is not None @@ -1873,6 +1904,23 @@ def _classify_directive( end_is_truncated=end_is_truncated, ) lookahead_exhausted = lookahead_exhausted or continuation_exhausted + if ( + baseline is not None + and context_start is not None + and any( + pattern.search(view.text, min(clause_start, context_start), lookahead_end) is not None + for pattern in ( + _UNAMBIGUOUS_ACTION_RE, + _CONTEXTUAL_ACTION_RE, + _PASTE_ACTION_RE, + ) + ) + ): + # Even an unbound action can acquire its payload through adjacent + # literals. Wrapper proof may repair an inert method-like header, but + # it cannot clear a prior limit for a potentially active instruction. + # Ambiguous or negated action cues also retain that conservative result. + return baseline payloads = _quoted_payloads(view.text, directive, scope_end) if not payloads: payloads = _anaphoric_payloads(view.text, directive, lookahead_end) @@ -1992,6 +2040,7 @@ def build_declared_marker_views( limited = False projection_blocked = False candidates: list[_ProjectionCandidate] = [] + quote_spans: tuple[list[int], list[int]] | None = None for directive in _directives( view.text, check_runtime, @@ -2014,10 +2063,50 @@ def build_declared_marker_views( if _verb_is_negated(view.text, directive.start, clause_start): continue + context_start = None + marker_quote = directive.end - len(directive.marker) - 2 + if ( + not source_end_is_truncated + and owned_source_start in (None, 0) + and view.source_offset(0) == 0 + and directive.marker + and directive.unsupported + and not directive.encoded + and not directive.is_tag + and directive.start <= marker_quote + and _QUOTE_OPEN_TO_CLOSE.get(view.text[marker_quote]) == view.text[directive.end - 1] + and view.text[marker_quote + 1 : directive.end - 1] == directive.marker + ): + # A bounded lookbehind can start inside an unrelated literal. + # Only a balanced, different-quote wrapper scanned from the view's + # origin can supply context; a marker opened by that wrapper's + # own closer keeps the conservative forward-only reading. + if quote_spans is None: + quote_spans = ([], []) + for _ in _sentence_boundaries( + view.text, + 0, + len(view.text), + quote_spans=quote_spans, + check_runtime=check_runtime, + ): + pass + starts, ends = quote_spans + index = bisect_right(starts, directive.start) - 1 + if ( + index >= 0 + and starts[index] < directive.start + and directive.end < ends[index] + and directive.end - starts[index] <= MAX_MARKER_LOOKAHEAD_CHARS + and view.text[starts[index]] != view.text[marker_quote] + ): + context_start = starts[index] + classification = _classify_directive( view, directive, end_is_truncated=source_end_is_truncated, + context_start=context_start, ) limited = limited or classification.limited if classification.active and classification.limited: diff --git a/tests/nodes/analyzers/test_command_wrapper_without_command.py b/tests/nodes/analyzers/test_command_wrapper_without_command.py index 44528263e..6de8d460b 100644 --- a/tests/nodes/analyzers/test_command_wrapper_without_command.py +++ b/tests/nodes/analyzers/test_command_wrapper_without_command.py @@ -14,6 +14,55 @@ _PADDING = "\n# ordinary padding\n" * 400 +@pytest.mark.parametrize( + "source", + [ + "env", + "env -i", + "env --", + "env -u OTHER", + "env -C /tmp", + "env NAME=value", + "command", + "command -p", + "command --", + "builtin", + "builtin --", + ], +) +@pytest.mark.parametrize("ending", ["", "\n", "; echo ready\n", " | cat\n", " # explanation\n"]) +def test_optional_command_wrappers_without_executable_are_complete( + source: str, ending: str +) -> None: + assert not _exhausted(source + ending, "shell") + + +@pytest.mark.parametrize("key", ["command", "builtin", "env"]) +def test_json_key_naming_optional_command_wrapper_is_complete(key: str) -> None: + source = '# Output\n\n```json\n{"' + key + '": "echo ready"}\n```\n' + + assert not _exhausted(source, "markdown") + + +@pytest.mark.parametrize( + "source", + [ + "env -u", + "env -C", + "env -S", + "env --unknown", + 'env >log bash -c "$CMD"', + 'command >log bash -c "$CMD"', + 'builtin >log eval "$CMD"', + 'env -- >log bash -c "$CMD"', + 'command -- >log bash -c "$CMD"', + 'builtin -- >log eval "$CMD"', + ], +) +def test_commandless_wrapper_refinement_keeps_unresolved_operands_partial(source: str) -> None: + assert _exhausted(source, "shell") + + def _exhausted(content: str, file_type: str, *, complete_context: bool = True) -> bool: return tm_module.has_bounded_parse_exhaustion( content, lambda: None, file_type=file_type, complete_context=complete_context diff --git a/tests/nodes/analyzers/test_eval_command_strings.py b/tests/nodes/analyzers/test_eval_command_strings.py index 9305ed30f..4c5c17d6f 100644 --- a/tests/nodes/analyzers/test_eval_command_strings.py +++ b/tests/nodes/analyzers/test_eval_command_strings.py @@ -3,6 +3,8 @@ """Eval operands retain their joined command semantics and parsing bounds.""" +import shlex + import pytest from skillspector.inspection_ledger import LedgerOutcome, LedgerReason @@ -10,6 +12,52 @@ from skillspector.nodes.analyzers import static_runner +@pytest.mark.parametrize("setting", ["ENV=prod", "export BASH_ENV=setup.sh"]) +@pytest.mark.parametrize("wrapper", ["eval", "builtin eval", "command -p eval"]) +@pytest.mark.parametrize("body", ["echo ready", "eval 'echo ready'"]) +def test_literal_eval_does_not_load_shell_startup_files( + setting: str, wrapper: str, body: str +) -> None: + content = f"{setting}; {wrapper} {shlex.quote(body)}" + + assert not tm_module.has_bounded_parse_exhaustion(content, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "content", + [ + "BASH_ENV=setup.sh; eval 'bash -c \"echo ready\"'", + "ENV=prod; builtin eval 'sh -c \"echo ready\"'", + "BASH_ENV=setup.sh; eval \"eval 'bash -c true'\"", + "eval 'export BASH_ENV=setup.sh'; bash -c 'echo ready'", + "builtin eval 'ENV=prod'; eval 'sh -c \"echo ready\"'", + "eval \"eval 'export BASH_ENV=setup.sh'\"; bash -c true", + "ENV=prod; eval 'ENV=; bash -c true'", + "BASH_ENV=setup.sh; eval 'echo \"$(bash -c true)\"'", + "BASH_ENV=setup.sh; eval 'echo \"`bash -c true`\"'", + "eval 'export BASH_ENV=setup.sh; echo \"$(bash -c true)\"'", + "eval 'export BASH_ENV=setup.sh; echo \"`bash -c true`\"'", + 'ENV=prod; eval "$SCRIPT"', + ], +) +def test_eval_preserves_startup_exposure_for_actual_child_shells(content: str) -> None: + assert tm_module.has_bounded_parse_exhaustion(content, lambda: None, file_type="shell") + + +def test_eval_startup_settings_inside_child_do_not_escape_to_parent() -> None: + content = "bash -c 'ENV=prod; eval \"echo ready\"'; bash -c 'echo ready'" + + assert not tm_module.has_bounded_parse_exhaustion(content, lambda: None, file_type="shell") + + +def test_literal_eval_with_startup_setting_keeps_destructive_finding() -> None: + content = "ENV=prod; eval 'rm -rf /'" + + findings = tm_module.analyze(content, "script.sh", "shell") + + assert any(finding.rule_id == "TM1" for finding in findings) + + @pytest.mark.parametrize( "content", [ diff --git a/tests/nodes/analyzers/test_host_language_shell_ownership.py b/tests/nodes/analyzers/test_host_language_shell_ownership.py index 51f3018d4..903b8a099 100644 --- a/tests/nodes/analyzers/test_host_language_shell_ownership.py +++ b/tests/nodes/analyzers/test_host_language_shell_ownership.py @@ -185,14 +185,6 @@ def test_destructive_command_inside_perl_eval_block_keeps_tm1() -> None: + " ]\n", id="f-string-inline-code-list", ), - pytest.param( - _OPTION_TEXT + "def message(tool):\n return (\n" - ' f"{tool} was not found at "\n' - ' f"$TOOL_ROOT, $UPSTREAM_ROOT/{tool}, "\n' - ' f"or $HOME/.cache/upstreams/{tool} "\n' - ' "or elsewhere"\n )\n', - id="concatenated-f-strings-naming-parameters", - ), pytest.param( _OPTION_TEXT + 'def run():\n """Run `$TOOL` once, then report."""\n', id="docstring-runtime-parameter", @@ -209,6 +201,66 @@ def test_python_literals_and_comments_own_their_bytes(source: str) -> None: assert _shell(content) is True +def test_concatenated_fstrings_keep_runtime_operand_ambiguity() -> None: + content = ( + _OPTION_TEXT + "def message(tool):\n return (\n" + ' f"{tool} was not found at "\n' + ' f"$TOOL_ROOT, $UPSTREAM_ROOT/{tool}, "\n' + ' f"or $HOME/.cache/upstreams/{tool} "\n' + ' "or elsewhere"\n )\n' + _TAIL + ) + # The body-start runtime command is now inspected. An empty upstream + # parameter can leave a root path, while a replacement field supplies flags. + # Concatenation must retain those possible operands even in report strings. + assert _python(content) is True + + +@pytest.mark.parametrize("target,partial", [("/", True), ("build", False)]) +@pytest.mark.parametrize("newline", ["\n", "\r\n"], ids=["lf", "crlf"]) +@pytest.mark.parametrize("leading", ["", "\n"], ids=["body-start", "next-line"]) +def test_completed_python_word_cannot_claim_a_later_string( + target: str, partial: bool, newline: str, leading: str +) -> None: + # A quote in the replacement expression can lexically pair with the next + # literal's quote. That apparent completed word must not hide a later + # runtime-selected command, even when the literal starts with it. + content = ( + "message = f\"{' '.join(parts)} ready\"\n" + f'REPORT = r"""{leading}$TOOL -rf {target}\nlabel(\'value\')\n"""\n' + ).replace("\n", newline) + + assert _python(content) is partial + + +@pytest.mark.parametrize( + "javascript", + [ + "execSync(`${{obj.cmd}}`);", + "run(`${{obj.cmd}}`);", + 'execSync(`${{obj.cmd}}` + " -rf /");', + 'const invoke = execSync; invoke(`${{obj.cmd}}` + " -rf /");', + ], +) +def test_python_constant_text_keeps_embedded_executable_javascript_partial( + javascript: str, +) -> None: + assert _python('REPORT = f"""' + javascript + '"""\n') is True + + +@pytest.mark.parametrize("prefix", ["r", "f", "rf"]) +@pytest.mark.parametrize("quote", ['"', '"""']) +@pytest.mark.parametrize("target,dangerous", [("/", True), ("build", False)]) +def test_short_prefixed_python_commands_keep_raw_findings( + prefix: str, quote: str, target: str, dangerous: bool +) -> None: + content = f"COMMAND = {prefix}{quote}rm -rf {target}{quote}\n" + + result = _ledger("scripts/command.py", content) + + assert any(finding.rule_id == "TM1" for finding in result["findings"]) is dangerous + assert result["inspection_ledger"][0]["outcome"] is LedgerOutcome.COMPLETED + + @pytest.mark.parametrize("newline", ["\r\n"], ids=["crlf"]) def test_python_ownership_preserves_crlf_offsets(newline: str) -> None: content = ('"""Module `doc."""\nFENCE = "\\n```\\n" # it\'s fine\n' + _TAIL).replace( @@ -877,6 +929,383 @@ def test_oversized_javascript_does_not_gain_template_ownership() -> None: assert assertion_template_shell_text(source, lambda: None) == source +# --- Escaped browser list values in Python HTML ---------------------------- + +_BROWSER_ASSIGNMENT = """panel.innerHTML = `
${listMarkup(rows.map((item) => `${item.name} :: ${item.value}`))}
`;""" +_BROWSER_SCRIPT = ( + """ +const data = JSON.parse(document.getElementById('data').textContent); +const panel = document.getElementById('panel'); +const rows = data.items; +function escapeText(value) { + return String(value ?? '').replaceAll('&', '&').replaceAll('<', '<') + .replaceAll('>', '>').replaceAll('"', '"'); +} +function listMarkup(items, emptyText='None') { + if (!items || !items.length) { + return `
${escapeText(emptyText)}
`; + } + return ``; +} +""" + + _BROWSER_ASSIGNMENT +) + + +def _browser_source(script: str = _BROWSER_SCRIPT, *, consumer: str | None = None) -> str: + encoded = script.replace("{", "{{").replace("}", "}}") + return ( + "from pathlib import Path\nfrom html import escape\nimport json\n" + "def render(data):\n" + " payload = json.dumps(data).replace('

{label}

" + ''''\n" + + (consumer or "Path('page.html').write_text(render(data), encoding='utf-8')") + + "\n" + ) + + +def test_escaped_browser_list_projects_only_selected_backticks() -> None: + source = _browser_source() + projected = python_tokens.python_browser_template_shell_text(source, lambda: None) + + assert _python(source) is False + assert ( + _ledger("scripts/page.py", source)["inspection_ledger"][0]["outcome"] + is LedgerOutcome.COMPLETED + ) + changes = [i for i, (a, b) in enumerate(zip(source, projected, strict=True)) if a != b] + assert len(changes) == 2 + assert all(source[i] == "`" and projected[i] == " " for i in changes) + assert projected.count("${{") == source.count("${{") + + +def test_escaped_browser_list_keeps_optional_property_reads() -> None: + source = _browser_source(_BROWSER_SCRIPT.replace("data.items", "data?.items")) + projected = python_tokens.python_browser_template_shell_text(source, lambda: None) + + assert _python(source) is False + changes = [i for i, (a, b) in enumerate(zip(source, projected, strict=True)) if a != b] + assert len(changes) == 2 + assert all(source[i] == "`" and projected[i] == " " for i in changes) + assert "data?.items" in projected + + +def test_escaped_browser_list_keeps_literal_security_findings() -> None: + script = _BROWSER_SCRIPT + '\nconst commandText = "rm -rf /";\n' + source = _browser_source(script) + result = _ledger("scripts/page.py", source) + + assert any(finding.rule_id == "TM1" for finding in result["findings"]) + # Delimiter ownership cannot complete a separate destructive shell candidate. + assert result["inspection_ledger"][0]["outcome"] is LedgerOutcome.PARTIAL + + +def test_escaped_browser_list_traces_local_parameters() -> None: + assignment = _BROWSER_ASSIGNMENT.replace("rows.map", "values.map") + script = _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "document.getElementById('filter').value = '';\nfunction renderList(values) {\n" + + assignment + + "\n}\nrenderList(rows);", + ) + source = _browser_source(script) + + assert _python(source) is False + projected = python_tokens.python_browser_template_shell_text(source, lambda: None) + assert sum(a != b for a, b in zip(source, projected, strict=True)) == 2 + + +def test_escaped_browser_list_traces_json_object_entries() -> None: + script = _BROWSER_SCRIPT.replace( + "const rows = data.items;", + "const rows = Object.entries(data.items).map(([name, value]) => ({name, value}));", + ) + source = _browser_source(script) + + assert _python(source) is False + projected = python_tokens.python_browser_template_shell_text(source, lambda: None) + assert sum(a != b for a, b in zip(source, projected, strict=True)) == 2 + + +@pytest.mark.parametrize("binding_kind", ["parameter", "local-const"]) +@pytest.mark.parametrize("receiver,partial", [("rows", False), ("external", True)]) +def test_browser_undefined_binding_requires_proven_data( + binding_kind: str, receiver: str, partial: bool +) -> None: + assignment = _BROWSER_ASSIGNMENT.replace("rows.map", "undefined.map") + if binding_kind == "parameter": + replacement = ( + f"function renderList(undefined) {{\n{assignment}\n}}\nrenderList({receiver});" + ) + else: + replacement = ( + f"function renderList() {{\nconst undefined = {receiver};\n" + f"{assignment}\n}}\nrenderList();" + ) + source = _browser_source(_BROWSER_SCRIPT.replace(_BROWSER_ASSIGNMENT, replacement)) + projected = python_tokens.python_browser_template_shell_text(source, lambda: None) + + assert _python(source) is partial + if partial: + assert projected == source + assert ( + _ledger("scripts/page.py", source)["inspection_ledger"][0]["reason_code"] + is LedgerReason.STATIC_PARSE_LIMIT + ) + else: + assert sum(a != b for a, b in zip(source, projected, strict=True)) == 2 + + +def test_escaped_browser_list_traces_native_filter_values() -> None: + script = _BROWSER_SCRIPT.replace( + "const rows = data.items;", + "function matches(row) {\n" + " const terms = [row.name, ...row.tags].join(' ').trim().toLowerCase();\n" + " return terms.includes(data.query);\n" + "}\nconst rows = data.items.filter(matches);", + ) + source = _browser_source(script) + + assert _python(source) is False + projected = python_tokens.python_browser_template_shell_text(source, lambda: None) + assert sum(a != b for a, b in zip(source, projected, strict=True)) == 2 + + +def test_tagged_browser_template_keeps_existing_completion(monkeypatch: pytest.MonkeyPatch) -> None: + script = _BROWSER_SCRIPT.replace( + "`${item.name} :: ${item.value}`", "tag`${item.name} :: ${item.value}`" + ) + source = _browser_source(script) + assert python_tokens.python_browser_template_shell_text(source, lambda: None) == source + observed = _python(source) + monkeypatch.setattr(tm, "python_browser_template_shell_text", lambda text, check: text) + assert _python(source) is observed + + +@pytest.mark.parametrize( + "script", + [ + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, "execSync(rows.map((item) => `${item.name}`));" + ), + _BROWSER_SCRIPT.replace(_BROWSER_ASSIGNMENT, "run(rows.map((item) => `${item.name}`));"), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, "const invoke = execSync; invoke(`${data.command}`);" + ), + _BROWSER_SCRIPT.replace("${item.name} ::", "${run(item.name)} ::"), + _BROWSER_SCRIPT.replace(_BROWSER_ASSIGNMENT, "panel.innerHTML = `${data.command}`;"), + _BROWSER_SCRIPT.replace(_BROWSER_ASSIGNMENT, "escapeText = run;\n" + _BROWSER_ASSIGNMENT), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, "function display(escapeText) {\n" + _BROWSER_ASSIGNMENT + "\n}" + ), + _BROWSER_SCRIPT.replace(_BROWSER_ASSIGNMENT, "mutate(data);\n" + _BROWSER_ASSIGNMENT), + *[ + _BROWSER_SCRIPT.replace(_BROWSER_ASSIGNMENT, call + "\n" + _BROWSER_ASSIGNMENT) + for call in ( + "external?.(rows);", + "external?.method(rows);", + "external.method?.(rows);", + "external['method']?.(rows);", + "external?.['method'](rows);", + "external?.['method']?.(rows);", + "(external.method)?.(rows);", + "external /* call */ ?. (rows);", + "external?.(panel);", + ) + ], + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, "external.map(() => data);\n" + _BROWSER_ASSIGNMENT + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "function leak(value) { external.map(() => value); }\nleak(data);\n" + + _BROWSER_ASSIGNMENT, + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "function invoke(strings, value) { external.map(() => value); }\ninvoke`${data}`;\n" + + _BROWSER_ASSIGNMENT, + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "rows.forEach((item) => external.map(() => item));\n" + _BROWSER_ASSIGNMENT, + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, "external.map(() => panel);\n" + _BROWSER_ASSIGNMENT + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "external.map(() => document.getElementById('panel'));\n" + _BROWSER_ASSIGNMENT, + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "const holder = {}; holder.value = data; external.map(() => holder.value);\n" + + _BROWSER_ASSIGNMENT, + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "const holder = {}; holder.value = panel; external.map(() => holder.value);\n" + + _BROWSER_ASSIGNMENT, + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "function renderInto(panel) {\n" + _BROWSER_ASSIGNMENT + "\n}\nrenderInto(external);", + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, "rows.forEach((panel) => {\n" + _BROWSER_ASSIGNMENT + "\n});" + ), + _BROWSER_SCRIPT.replace("const rows = data.items;", "const rows = {map: eval};"), + _BROWSER_SCRIPT.replace( + "const rows = data.items;", "const rows = Object.entries(external);" + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "const terms = external.toLowerCase(); terms.includes(data.query);\n" + + _BROWSER_ASSIGNMENT, + ), + _BROWSER_SCRIPT.replace( + _BROWSER_ASSIGNMENT, + "data.query.toLowerCase(data.query).includes(data.query);\n" + _BROWSER_ASSIGNMENT, + ), + _BROWSER_SCRIPT.replace( + "const rows = data.items;", + "const rows = {[\"map\"]: document.getElementById('p').onclick};", + ), + _BROWSER_SCRIPT.replace("
  • ${escapeText", '
  • ${escapeText'), + _BROWSER_SCRIPT.replace("${listMarkup(rows.map", "${execSync(listMarkup(rows.map").replace( + "))}", ")))}" + ), + ], + ids=[ + "exec", + "unknown-call", + "alias", + "executable-field", + "unescaped-output", + "reassign", + "shadow", + "data-escape", + "optional-call", + "optional-member-call", + "member-optional-call", + "computed-optional-call", + "optional-computed-member-call", + "optional-computed-optional-call", + "grouped-optional-call", + "commented-optional-call", + "optional-dom-escape", + "callback-escape", + "forwarded-data-escape", + "tagged-local-escape", + "forwarded-callback-escape", + "dom-receiver-escape", + "dom-lookup-escape", + "stored-data-escape", + "stored-dom-escape", + "dom-function-shadow", + "dom-callback-shadow", + "fake-map", + "unknown-object-entries", + "unknown-string-receiver", + "unexpected-string-argument", + "computed-map", + "formatter-handler", + "formatter-executed", + ], +) +def test_unproven_browser_consumers_remain_partial(script: str) -> None: + source = _browser_source(script) + + assert python_tokens.python_browser_template_shell_text(source, lambda: None) == source + assert _python(source) is True + assert ( + _ledger("scripts/page.py", source)["inspection_ledger"][0]["reason_code"] + is LedgerReason.STATIC_PARSE_LIMIT + ) + + +@pytest.mark.parametrize( + "consumer", + [ + "import subprocess\nsubprocess.run(render(data), shell=True)", + "unknown(render(data))", + "unknown(Path)\nPath('page.html').write_text(render(data))", + "unknown(json)\nPath('page.html').write_text(render(data))", + "p = Path('page.html')\nunknown(p)\np.write_text(render(data))", + "p = Path('page.html')\nunknown(p.write_text)\np.write_text(render(data))", + "p = Path('page.html')\nunknown(p.resolve)\np.write_text(render(data))", + "p = Path('page.html')\nunknown(p.parent)\np.write_text(render(data))", + "p = Path('page.html')\nunknown(p.resolve())\np.write_text(render(data))", + "Path.resolve = unknown\nPath('page.html').write_text(render(data))", + "(Path('page.html') / unknown).write_text(render(data))", + "def outer(Path):\n def inner():\n Path('page.html').write_text(render(data))", + "[Path('page.html').write_text(render(data)) for Path in commands]", + ], +) +def test_browser_html_requires_explicit_pathlib_consumers(consumer: str) -> None: + consumer = consumer.replace( + "write_text(render(data))", "write_text(render(data), encoding='utf-8')" + ) + source = _browser_source(consumer=consumer) + + assert python_tokens.python_browser_template_shell_text(source, lambda: None) == source + assert _python(source) is True + + +@pytest.mark.parametrize( + "arguments", + [ + "render(data)", + "render(data), encoding='utf-16'", + "render(data), encoding=codec", + "render(data), encoding='utf-8', errors='ignore'", + "render(data), 'utf-8', 'ignore'", + ], +) +def test_browser_html_requires_known_utf8_write(arguments: str) -> None: + source = _browser_source(consumer=f"Path('page.html').write_text({arguments})") + + assert python_tokens.python_browser_template_shell_text(source, lambda: None) == source + assert _python(source) is True + + +@pytest.mark.parametrize( + "before,after", + [ + ("label = escape(data['label'])", "label = data['label']"), + ("payload = json.dumps(data).replace('{label}

    ", ''), + ("function escapeText(value)", "function String(value)"), + ], +) +def test_browser_boundary_and_builtin_changes_do_not_gain_ownership( + before: str, after: str +) -> None: + source = _browser_source().replace(before, after) + + assert _python(source) is True + assert python_tokens.python_browser_template_shell_text(source, lambda: None) == source + + +def test_browser_ownership_preserves_fragment_bounds_and_cancellation() -> None: + source = _browser_source() + assert _python(source, complete_context=False) is True + assert _python(source.replace("", "")) is True + assert _python(_browser_source("(" * 150 + _BROWSER_SCRIPT + ")" * 150)) is True + oversized = _browser_source("/*" + "x" * 1_100_000 + "*/" + _BROWSER_SCRIPT) + assert python_tokens.python_browser_template_shell_text(oversized, lambda: None) == oversized + + def cancel() -> None: + raise TimeoutError("browser ownership deadline") + + with pytest.raises(TimeoutError, match="browser ownership deadline"): + python_tokens.python_browser_template_shell_text(source, cancel) + + # --- Whole-scan accounting ------------------------------------------------- diff --git a/tests/nodes/analyzers/test_perl_print_ownership.py b/tests/nodes/analyzers/test_perl_print_ownership.py index d214f4c31..2aced45f8 100644 --- a/tests/nodes/analyzers/test_perl_print_ownership.py +++ b/tests/nodes/analyzers/test_perl_print_ownership.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Perl host-string boundaries must not invent shell parse failures.""" +"""Perl host-string boundaries must not invent shell parse failures. + +Generic text supplies the unowned control: complete shell source now proves +its own quote boundaries independently of Perl string ownership. +""" import sys import time @@ -102,7 +106,7 @@ def test_crlf_inside_print_literal_does_not_acquire_ownership() -> None: assert tm._perl_literal_print_shell_text(content, lambda: None) == content assert tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="perl") == ( - tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="shell") + tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="text") ) @@ -135,7 +139,7 @@ def test_unsupported_or_ambiguous_source_keeps_conservative_parse_status(content def test_unproven_literal_delimiters_do_not_acquire_ownership(content: str) -> None: assert tm._perl_literal_print_shell_text(content, lambda: None) == content assert tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="perl") == ( - tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="shell") + tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="text") ) @@ -171,7 +175,7 @@ def test_ambiguous_perl_quote_tokens_cannot_hide_executable_shell( content = f'{prefix}\nqx{{\n# {comment_quote}\nprint "Use rm to remove a project\\n";\n}};\n' assert tm._perl_literal_print_shell_text(content, lambda: None) == content - assert tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="shell") is True + assert tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="text") is True result = _scan(content) assert any( row["outcome"] is LedgerOutcome.PARTIAL @@ -201,7 +205,7 @@ def test_quote_like_perl_variables_cannot_hide_executable_shell( ) assert tm._perl_literal_print_shell_text(content, lambda: None) == content - assert tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="shell") is True + assert tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="text") is True result = _scan(content) assert any( row["outcome"] is LedgerOutcome.PARTIAL @@ -306,7 +310,7 @@ def test_print_literal_does_not_bypass_shell_parser_bounds(newline: str) -> None ) -@pytest.mark.parametrize("file_type,complete_context", [("shell", True), ("perl", False)]) +@pytest.mark.parametrize("file_type,complete_context", [("text", True), ("perl", False)]) def test_only_complete_perl_source_can_grant_host_quote_ownership( file_type: str, complete_context: bool ) -> None: diff --git a/tests/nodes/analyzers/test_prose_shell_context.py b/tests/nodes/analyzers/test_prose_shell_context.py index 1fe4c6227..d2bc62fb9 100644 --- a/tests/nodes/analyzers/test_prose_shell_context.py +++ b/tests/nodes/analyzers/test_prose_shell_context.py @@ -64,6 +64,99 @@ ] +_NESTED_SHELL_QUOTES = 'echo "entry: $item => $(readlink -f "$item")"\n' + + +@pytest.mark.parametrize("fence", ["```", "~~~"]) +@pytest.mark.parametrize("language", ["bash", "sh", "shell"]) +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_closed_shell_fence_owns_nested_quotes(fence: str, language: str, newline: str) -> None: + content = ( + f"## Reader\n\n{fence}{language}\n" + _NESTED_SHELL_QUOTES + f"{fence}\n\n" + _OBSERVATIONS + ).replace("\n", newline) + + result = static_runner.run_static_patterns_with_ledger( + {"components": ["guide.md"], "file_cache": {"guide.md": content}}, [tm] + ) + + assert result["findings"] == [] + assert result["inspection_ledger"][0]["outcome"] is LedgerOutcome.COMPLETED + + +@pytest.mark.parametrize( + "opening,closing", + [ + ("```", "```"), + ("```text", "```"), + ("```python", "```"), + ("```bash title=example", "```"), + ("```bash", ""), + ("```bash", "~~~"), + ("````bash", "```"), + ("\\```bash", "```"), + ("```bash", "\\```"), + ("- ```bash", " ```"), + ("> ```bash", "> ```"), + ], +) +def test_unproven_fences_do_not_grant_shell_quote_ownership(opening: str, closing: str) -> None: + content = opening + "\n" + _NESTED_SHELL_QUOTES + closing + "\n" + _OBSERVATIONS + ranges: list[tuple[int, int]] = [] + + projected = tm._markdown_shell_text(content, lambda: None, shell_fence_ranges=ranges) + + assert len(projected) == len(content) + assert ranges == [] + + +def test_fence_ownership_requires_complete_context_and_preserves_offsets() -> None: + prefix = "## Reader\n\n ```bash\n" + content = prefix + _NESTED_SHELL_QUOTES + " ````\n" + _OBSERVATIONS + ranges: list[tuple[int, int]] = [] + projected = tm._markdown_shell_text(content, lambda: None, shell_fence_ranges=ranges) + + assert ranges == [(len(prefix), len(prefix) + len(_NESTED_SHELL_QUOTES))] + start, end = ranges[0] + assert projected[start:end] == content[start:end] == _NESTED_SHELL_QUOTES + assert len(projected) == len(content) + + ranges.clear() + tm._markdown_shell_text( + content, lambda: None, complete_context=False, shell_fence_ranges=ranges + ) + assert ranges == [] + + +@pytest.mark.parametrize("position", ["before", "inside", "after"]) +@pytest.mark.parametrize("command", ["$CMD -rf /", 'echo "$($CMD -rf /)"', "rm -rf /"]) +def test_shell_fence_quote_ownership_preserves_destructive_checks( + position: str, command: str +) -> None: + inside = _NESTED_SHELL_QUOTES + (command + "\n" if position == "inside" else "") + content = "```bash\n" + inside + "```\n" + _OBSERVATIONS + if position == "before": + content = command + "\n" + content + elif position == "after": + content += command + "\n" + + result = static_runner.run_static_patterns_with_ledger( + {"components": ["guide.md"], "file_cache": {"guide.md": content}}, [tm] + ) + + if command == "rm -rf /": + assert any(finding.rule_id == "TM1" for finding in result["findings"]) + else: + event = result["inspection_ledger"][0] + assert event["outcome"] is LedgerOutcome.PARTIAL + assert event["reason_code"] is LedgerReason.STATIC_PARSE_LIMIT + + +def test_closed_fence_with_unclosed_shell_quote_stays_partial() -> None: + content = '```bash\necho "unfinished\n```\n' + _OBSERVATIONS + + assert tm.has_bounded_parse_exhaustion(content, lambda: None, file_type="markdown") + + @pytest.mark.parametrize("content", _CASES) def test_benign_guide_has_complete_analyzer_coverage(content: str) -> None: result = static_runner.run_static_patterns_with_ledger( diff --git a/tests/nodes/analyzers/test_python_string_marker_ownership.py b/tests/nodes/analyzers/test_python_string_marker_ownership.py index eb691d054..da61dc579 100644 --- a/tests/nodes/analyzers/test_python_string_marker_ownership.py +++ b/tests/nodes/analyzers/test_python_string_marker_ownership.py @@ -182,6 +182,66 @@ def test_python_helper_with_removal_verb_at_string_end_scans_complete(content: s assert result["inspection_ledger"][0]["outcome"] is LedgerOutcome.COMPLETED +@pytest.mark.parametrize("quote", ["'", '"']) +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_bounded_lookbehind_cannot_invent_marker_quote_context(quote: str, newline: str) -> None: + # The bounded lookbehind starts inside the first long literal. Its period + # looks like a sentence boundary, and the closer then looks like an opener. + content = ( + "LABEL = " + + quote + + "x" * (MAX_MARKER_LOOKAHEAD_CHARS + 8) + + ". summary" + + quote + + "\nresult = value.strip() or f" + + quote + + "{name}" + + quote + + "\n" + + _UNQUOTED_TAIL + ).replace("\n", newline) + closers = PythonStringClosers(content, lambda: None) + + for closer_test in (None, closers.closes_string): + projection = build_declared_marker_views( + SecurityTextView("raw", content), + validated_string_closer=closer_test, + ) + assert projection.views == () + assert projection.limited is False + + result = _ledger("scripts/check.py", content, pi_module) + assert result["inspection_ledger"][0]["outcome"] is LedgerOutcome.COMPLETED + + +@pytest.mark.parametrize("concatenation", [" ", " + "]) +@pytest.mark.parametrize("marker", ["xyz", "x y"], ids=["supported", "unsupported"]) +@pytest.mark.parametrize("action", ["execute", "call"]) +@pytest.mark.parametrize("reverse", [False, True], ids=["forward-action", "reverse-action"]) +def test_wrapper_context_keeps_concatenated_marker_payload_partial( + concatenation: str, marker: str, action: str, reverse: bool +) -> None: + payload = f"r{marker}m -r{marker}f *" + instruction = ( + f"\"{action.capitalize()} \"{concatenation}\"'{payload}' after remove '{marker}'.\"" + if reverse + else f"\"Remove '{marker}' and {action} \"{concatenation}\"'{payload}'.\"" + ) + content = ( + 'LABEL = "' + "x" * (MAX_MARKER_LOOKAHEAD_CHARS + 8) + '. summary"\n' + 'result = value.strip() or f"{name}"\n' + "note = " + instruction + "\n" + _UNQUOTED_TAIL + ) + + result = _ledger("scripts/check.py", content) + + assert any(finding.rule_id == "TM1" for finding in result["findings"]) or any( + event["reason_code"] is LedgerReason.OBFUSCATED_INSTRUCTION_TEXT + for event in result["inspection_ledger"] + if event["outcome"] is LedgerOutcome.PARTIAL + ) + + def test_string_ownership_maps_into_later_static_windows() -> None: prefix = "".join(f"value_{index} = {index}\n" for index in range(24_000)) content = prefix + _ASSERT_FLAG + _LITERAL_TAIL diff --git a/tests/nodes/analyzers/test_security_reconstruction.py b/tests/nodes/analyzers/test_security_reconstruction.py index 1c9e2e9a6..048fd402d 100644 --- a/tests/nodes/analyzers/test_security_reconstruction.py +++ b/tests/nodes/analyzers/test_security_reconstruction.py @@ -933,6 +933,112 @@ def test_quoted_verb_with_filler_still_fails_closed() -> None: assert projection.limited is True +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +@pytest.mark.parametrize( + "head", + [ + "python3 -c 'print(value.strip(\"/\"))'\n", + "python3 -c \"print(value.strip('/'))\"\n", + "python3 -c 'values = []\nfor value in values:\n print(value.strip(\"/\"))'\n", + ], + ids=["single-quoted-argument", "double-quoted-argument", "multiline-argument"], +) +def test_marker_scope_preserves_enclosing_quote_state(head: str, newline: str) -> None: + content = (head + "echo ready\n" * (MAX_MARKER_LOOKAHEAD_CHARS // 10 + 1)).replace( + "\n", newline + ) + + projection = build_declared_marker_views(SecurityTextView("raw", content)) + + assert projection.views == () + assert projection.limited is False + + +@pytest.mark.parametrize( + "bounds", + [ + {"owned_source_start": 1}, + {"source_end_is_truncated": True}, + ], + ids=["ambiguous-prefix", "truncated-source"], +) +def test_marker_wrapper_context_requires_complete_source(bounds: dict) -> None: + content = "python3 -c 'print(value.strip(\"/\"))'\n" + "echo ready\n" * ( + MAX_MARKER_LOOKAHEAD_CHARS // 10 + 1 + ) + + projection = build_declared_marker_views(SecurityTextView("raw", content), **bounds) + + assert projection.views == () + assert projection.limited is True + + +def test_unclosed_marker_wrapper_does_not_supply_quote_context() -> None: + content = 'python3 -c \'print(value.strip("/"))"\n' + "echo ready\n" * ( + MAX_MARKER_LOOKAHEAD_CHARS // 10 + 1 + ) + + projection = build_declared_marker_views(SecurityTextView("raw", content)) + + assert projection.views == () + assert projection.limited is True + + +@pytest.mark.parametrize("wrapper", ["{}", 'note = "{}"', "note = '''{}'''"]) +@pytest.mark.parametrize("marker", ["xyz", "x y"]) +def test_enclosing_quote_state_preserves_active_marker_checks(wrapper: str, marker: str) -> None: + payload = f"r{marker}m -r{marker}f *" + content = wrapper.format(f"Remove '{marker}' and execute '{payload}'.") + "\n" + + projection = build_declared_marker_views(SecurityTextView("raw", content)) + + if marker == "xyz": + assert any(view.text == "rm -rf *" for view in projection.views) + assert projection.limited is False + else: + assert projection.views == () + assert projection.limited is True + + +def test_embedded_calls_do_not_hide_a_following_marker_instruction() -> None: + content = ( + "python3 -c 'print(first.strip(\"/\"))'\n" + "python3 -c 'print(second.strip(\"/\"))'\n" + + "echo ready\n" * (MAX_MARKER_LOOKAHEAD_CHARS // 10 + 1) + + "Remove 'xyz' and execute 'rxyzm -rxyzf *'.\n" + ) + + projection = build_declared_marker_views(SecurityTextView("raw", content)) + + assert [view.text for view in projection.views] == ["rm -rf *"] + assert projection.limited is False + + +@pytest.mark.parametrize("truncated", [False, True]) +def test_enclosing_quote_does_not_relax_marker_lookahead(truncated: bool) -> None: + content = "note = \"Remove 'xyz' and " + "a" * (MAX_MARKER_LOOKAHEAD_CHARS + 8) + if not truncated: + content += " execute 'rxyzm -rxyzf *'.\"\n" + + projection = build_declared_marker_views( + SecurityTextView("raw", content), source_end_is_truncated=truncated + ) + + assert projection.views == () + assert projection.limited is True + + +def test_truncated_enclosing_quote_stays_partial() -> None: + content = "note = \"Remove 'xyz' from a spelling example" + + projection = build_declared_marker_views( + SecurityTextView("raw", content), source_end_is_truncated=True + ) + + assert projection.views == () + assert projection.limited is True + + def test_benign_directives_do_not_consume_active_cap() -> None: directives = [f"Remove 'm{index}' from this spelling example." for index in range(9)] diff --git a/tests/nodes/analyzers/test_shell_quote_ownership.py b/tests/nodes/analyzers/test_shell_quote_ownership.py index daf92a42b..b109cc128 100644 --- a/tests/nodes/analyzers/test_shell_quote_ownership.py +++ b/tests/nodes/analyzers/test_shell_quote_ownership.py @@ -9,7 +9,100 @@ from typer.testing import CliRunner from skillspector.cli import app +from skillspector.inspection_ledger import LedgerOutcome, LedgerReason from skillspector.nodes.analyzers import static_patterns_tool_misuse as tm +from skillspector.nodes.analyzers import static_runner + + +@pytest.mark.parametrize("word", ["rm", "del", "erase", "/bin/rm"]) +@pytest.mark.parametrize( + ("path", "template"), + [ + ("example.json", '{{"x":"a {word} b"}}'), + ("example.json", '["a {word} b"]'), + ("example.sh", 'echo "a {word} b"'), + ("example.sh", "echo 'a {word} b'"), + ("example.md", '```json\n{{"x":"a {word} b"}}\n```'), + ("example.md", '```bash\necho "a {word} b"\n```'), + ], +) +def test_literal_removal_words_in_complete_quotes_finish_inspection( + path: str, template: str, word: str +) -> None: + result = static_runner.run_static_patterns_with_ledger( + {"components": [path], "file_cache": {path: template.format(word=word)}}, [tm] + ) + + assert result["inspection_ledger"][0]["outcome"] is LedgerOutcome.COMPLETED + assert not result["findings"] + + +@pytest.mark.parametrize( + ("path", "source"), + [ + ("example.json", '{"x":"a rm -rf /"}'), + ("example.sh", 'echo "a rm -rf /"'), + ("example.sh", "echo 'a rm -rf /'"), + ("example.sh", 'echo "a del b"; rm -rf /'), + ("example.sh", 'echo "a del b"; $(printf %s r m) -rf /'), + ("example.json", '{"x":"a del b", "y":"$CMD -rf /"}'), + ("example.md", '```json\n{"x":"a rm -rf /"}\n```'), + ("example.md", '```bash\necho "a del b"; rm -rf /\n```'), + ], +) +def test_proven_operand_boundaries_keep_destructive_findings(path: str, source: str) -> None: + result = static_runner.run_static_patterns_with_ledger( + {"components": [path], "file_cache": {path: source}}, [tm] + ) + + assert result["inspection_ledger"][0]["outcome"] is LedgerOutcome.COMPLETED + assert any(finding.rule_id == "TM1" for finding in result["findings"]) + + +@pytest.mark.parametrize( + ("path", "source"), + [ + ("example.json", '{"x":"a del b"'), + ("example.sh", 'echo "a del b\n' + "x" * (tm._SHELL_COMMAND_WORD_CHARS + 1)), + ("example.sh", 'echo "a del b"; $CMD -rf /'), + ("example.sh", 'echo "x: $(printf %s r m) -rf /"'), + ("example.sh", 'sh -c "a del b; $CMD -rf /"'), + ("example.json", '{"x":"x: $(printf %s r m) -rf /"}'), + ("example.sh", 'echo "a rm ' + "x" * (tm._ROOT_GLOB_COMMAND_CHARS + 1) + '"'), + ], +) +def test_quote_ownership_keeps_unresolved_commands_partial(path: str, source: str) -> None: + result = static_runner.run_static_patterns_with_ledger( + {"components": [path], "file_cache": {path: source}}, [tm] + ) + + row = result["inspection_ledger"][0] + assert row["outcome"] is LedgerOutcome.PARTIAL + assert row["reason_code"] is LedgerReason.STATIC_PARSE_LIMIT + + +@pytest.mark.parametrize( + ("file_type", "source"), + [("json", '{"x":"a del b"}'), ("shell", 'echo "a del b"')], +) +def test_quote_operand_boundary_requires_complete_context(file_type: str, source: str) -> None: + assert tm.has_bounded_parse_exhaustion( + source, lambda: None, file_type=file_type, complete_context=False + ) + + +def test_unresolved_operand_candidates_honor_the_analysis_deadline() -> None: + source = 'echo "prefix ' + "del harmless " * 1000 + checks = 0 + + def check_runtime() -> None: + nonlocal checks + checks += 1 + if checks == 10: + raise TimeoutError("operand scan deadline") + + with pytest.raises(TimeoutError, match="operand scan deadline"): + tm.analyze(source, "example.sh", "shell", check_runtime=check_runtime) @pytest.mark.parametrize("padding", [0, 5000, 9000]) @@ -27,6 +120,428 @@ def test_closed_data_quotes_do_not_consume_later_padding(source: str, padding: i assert not tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_nested_path_quote_after_here_string_does_not_consume_later_commands(newline: str) -> None: + source = """#!/bin/bash +mapfile -t matches < <(awk 'NF {print $1}' <<<"$listing") +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +root="$work/bundle" +cp "$(dirname "${SOURCE[0]}")/helper.sh" \\ + "$root/helper.sh" +chmod 700 "$root/helper.sh" +tool="$(resolve_one "$SPEC")" +archive="$work/archive.tbz2" +fetch "$tool" > "$archive" +tar -xf "$archive" -C "$root" +files=("$tool") +for file in first.py second.py; do + item="$(resolve_one "$BASE/$file")" + fetch "$item" > "$root/$file" +done +[[ ${#files[@]} -eq 1 ]] || { + echo "Expected one nonempty tool in the downloaded archive" >&2 + exit 1 +} +chmod 700 "${files[0]}" +printf 'Created %s\\n' "$root" +""".replace("\n", newline) + + assert not tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize("operator", ["<<<", "<<< "]) +def test_here_string_keeps_following_quote_bounds(operator: str) -> None: + source = f'cat {operator}"$VALUE"\necho "waiting $(get_count) ${{DELAY}}s"\n' + source += "# " + "." * 6000 + "\n" + + assert not tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "command", + [ + "$CMD -rf /", + '"$CMD" -rf /', + 'echo "x: $($CMD -rf /)"', + 'echo "x: $(echo "a b"; $CMD -rf /)"', + 'sh -c "$CMD -rf /"', + ], +) +def test_here_string_and_nested_quote_closers_do_not_hide_runtime_commands(command: str) -> None: + source = 'cat <<<"$VALUE"\ncp "$(dirname "$SOURCE")/file" "$DEST"\n' + source += command + "\n# " + "." * 6000 + "\n" + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +@pytest.mark.parametrize("header", ["< None: + source = f"cat {header}\nordinary help text\nEND\n" + source += """state="$(inspect -f '{{value}}' "$NAME")" || return +[[ "$state" == true\\|true\\|* && + " ${state#*|*|} " == *' /dev '* && + " ${state#*|*|} " == *' /tools '* ]] || { + echo "Required mounts are missing" >&2 + return 1 +} +output="$(detect_devices)" || return +printf '%s\\n' "$output" +trap 'rm -rf "$CACHE"' EXIT +""" + + assert not tm.has_bounded_parse_exhaustion( + source.replace("\n", newline), lambda: None, file_type="shell" + ) + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_multiple_heredocs_are_consumed_in_declaration_order(newline: str) -> None: + source = "cat <<'FIRST' <<-SECOND\nfirst body\nFIRST\n\tsecond body\n\tSECOND\n" + source += 'echo "waiting $(get_count) ${DELAY}s"\n# ' + "." * 6000 + "\n" + + assert not tm.has_bounded_parse_exhaustion( + source.replace("\n", newline), lambda: None, file_type="shell" + ) + + +@pytest.mark.parametrize("indent", [" ", " \t", "\t "]) +def test_tab_stripping_heredoc_does_not_strip_spaces(indent: str) -> None: + source = f"$(cat <<-END\nbody\n{indent}END\n)" + end_cache: dict[int, int | None] = {} + + assert tm._skip_command_substitution(source, 0, len(source), end_cache=end_cache) is None + assert end_cache == {0: None} + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +@pytest.mark.parametrize( + ("body", "strip_tabs", "quoted"), + [ + ("ordinary \\\ncontinued\nEND\n", False, False), + ("E\\\nN\\\nD\n", False, False), + ("body\\\\\nEND\n", False, False), + ("E\\\\\\\nND\nEND\n", False, False), + ("\tE\\\nND\n", True, False), + ("E\\\n\tND\nEND\n", True, False), + ("E\\\nND\nEND\n", False, True), + ], +) +def test_heredoc_continuations_compare_logical_delimiter_lines( + body: str, strip_tabs: bool, quoted: bool, newline: str +) -> None: + body = body.replace("\n", newline) + source = body + "$CMD -rf /\n" + + assert tm._skip_static_heredoc_bodies( + source, 0, len(source), (("END", strip_tabs, quoted),), lambda: None + ) == len(body) + + +@pytest.mark.parametrize("body", ["END", "E\\\nND", "\tE\\\nND"]) +def test_heredoc_delimiter_can_end_at_eof(body: str) -> None: + assert tm._skip_static_heredoc_bodies( + body, 0, len(body), (("END", True, False),), lambda: None + ) == len(body) + + +def test_large_continued_heredoc_checks_runtime_and_keeps_raw_commands() -> None: + body = ("x" * 5000 + "\\\n") * 40 + "END\n$CMD -rf /\nEND\n" + checks = 0 + + def check_runtime() -> None: + nonlocal checks + checks += 1 + + assert tm._skip_static_heredoc_bodies( + body, 0, len(body), (("END", False, False),), check_runtime + ) == len(body) + assert checks >= 40 + assert tm.has_bounded_parse_exhaustion("cat < None: + source = "$(cat <<'END'\n) and an unmatched quote: \"\nEND\nprintf ok\n)" + source = source.replace("\n", newline) + + assert tm._skip_command_substitution(source, 0, len(source), lambda: None) == len(source) + + +@pytest.mark.parametrize("delimiter", ["END", "'END'", '"END"']) +@pytest.mark.parametrize("command", ["$CMD -rf /", 'echo "$($CMD -rf /)"']) +def test_heredoc_ownership_keeps_runtime_payloads_partial(delimiter: str, command: str) -> None: + source = f"cat <<{delimiter}\n{command}\nEND\n" + source += 'echo "waiting $(get_count) ${DELAY}s"\n' + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize("delimiter", ["END", "'END'"]) +def test_heredoc_ownership_keeps_literal_destructive_findings(delimiter: str) -> None: + source = f"cat <<{delimiter}\nrm -rf /\nEND\necho ready\n" + + assert any(finding.rule_id == "TM1" for finding in tm.analyze(source, "run.sh", "shell")) + + +@pytest.mark.parametrize("header", ["<<'END-MARK'", "<<\\END", "<<$END", "<<'END", "<<"]) +def test_unsupported_heredoc_delimiter_never_proves_a_substitution_end(header: str) -> None: + source = f"$(cat {header}\n)\nEND\n)" + end_cache: dict[int, int | None] = {} + + assert tm._skip_command_substitution(source, 0, len(source), end_cache=end_cache) is None + assert end_cache == {0: None} + + +@pytest.mark.parametrize("separator", [" ", "\t"]) +@pytest.mark.parametrize("strip_tabs", [False, True]) +def test_heredoc_header_whitespace_accepts_exact_span_boundary( + separator: str, strip_tabs: bool +) -> None: + prefix = "cat " + source = prefix + ("<<-" if strip_tabs else "<<") + source += separator * tm._SHELL_COMMAND_WORD_CHARS + "'END'" + + assert tm._static_heredoc_header(source, len(prefix), len(source)) == ( + ("END", strip_tabs, True), + len(source), + ) + + +@pytest.mark.parametrize("separator", [" ", "\t"]) +def test_heredoc_header_whitespace_over_span_boundary_fails_closed(separator: str) -> None: + source = "$(cat <<" + separator * (tm._SHELL_COMMAND_WORD_CHARS + 1) + "END\n)\nEND\n)" + end_cache: dict[int, int | None] = {} + + assert tm._static_heredoc_header(source, source.index("<<"), len(source)) is None + assert tm._skip_command_substitution(source, 0, len(source), end_cache=end_cache) is None + assert end_cache == {0: None} + + +@pytest.mark.parametrize("suffix", ["", "\nEND\n)"]) +def test_pending_heredoc_cannot_close_before_its_body(suffix: str) -> None: + source = "$(cat < None: + source = "$(cat < None: + source = '$(cat < None: + for source in ("$((1 << count))", "$(echo $((1 << count)))", "$( ((1 << count)) )"): + assert tm._skip_command_substitution(source, 0, len(source), lambda: None) == len(source) + + +@pytest.mark.parametrize("operator", ["#", "##", "%", "%%"]) +@pytest.mark.parametrize("variable", ["$ROOT", "${ROOT}"]) +def test_simple_quoted_trim_pattern_preserves_outer_quote_ownership( + operator: str, variable: str +) -> None: + source = 'printf "%s" "${path' + operator + '"' + variable + '"/}"\n' + source += """state="$(inspect -f '{{value}}' "$NAME")" || return +[[ "$state" == true\\|true\\|* && " ${state#*|*|} " == *' /dev '* ]] || return +trap 'rm -rf "$CACHE"' EXIT +""" + + assert not tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + assert tm.has_bounded_parse_exhaustion(source + "$CMD -rf /\n", lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "pattern", + ['${path#"$($CMD -rf /)"}', '${path#"$ROOT"$"suffix"}', '${path#"${ROOT:-other}"}'], +) +def test_complex_trim_patterns_cannot_claim_simple_quote_ownership(pattern: str) -> None: + source = 'printf "%s" "' + pattern + '"\n' + closers: set[int] = set() + + assert tm._shell_quoted_candidate_limits(source, lambda: None, quote_closers=closers) == {} + assert closers == set() + assert tm.has_bounded_parse_exhaustion(source + "$CMD -rf /\n", lambda: None, file_type="shell") + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_literal_shell_command_has_its_own_quote_ownership(newline: str) -> None: + source = """MODE=test bash -c ' +if [[ "$1 $2" == "docker exec" && + "$*" != *" /bin/bash -c "* ]]; then + return 127 +fi +if [[ "$1 $2" == "docker exec" && "$*" == *" ldd "* ]]; then + printf "%s\\n" "library found" +fi +command "$@" +' +trap 'rm -rf "$CACHE"' EXIT +""" + + assert not tm.has_bounded_parse_exhaustion( + source.replace("\n", newline), lambda: None, file_type="shell" + ) + + +@pytest.mark.parametrize("command", ["$CMD -rf /", 'echo "$($CMD -rf /)"', 'sh -c "$CODE"']) +def test_literal_shell_command_ownership_keeps_unknown_commands_partial(command: str) -> None: + source = f"bash -c '\necho ready\n{command}\n'\n" + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize("argument", ['"$SCRIPT"', "'echo '\"ready\"", "$'echo ready'"]) +def test_transformed_shell_command_cannot_claim_original_literal_offsets(argument: str) -> None: + source = f"bash -c {argument}\n" + + assert tm._literal_shell_command_spans(source, 0, len(source), lambda: None) == {} + + +@pytest.mark.parametrize("shape", ["parameter", "alternating", "comment", "whitespace"]) +def test_literal_command_before_hostile_nesting_keeps_proof_work_bounded( + shape: str, monkeypatch: pytest.MonkeyPatch +) -> None: + repetitions = 2000 + if shape == "parameter": + tail = '$(env "X=${x:- ' * repetitions + "A" + '}" echo printf)' * repetitions + " -rf *" + elif shape == "alternating": + tail = ( + "$(env X=" + + "${x:-$(echo " * repetitions + + "z" + + ")}" * repetitions + + " echo printf) -rf /" + ) + elif shape == "comment": + tail = "$(env X=x $(echo " * repetitions + "# no newline" + ")" * repetitions + else: + tail = "\n" * 40000 + "$CMD -rf /\n" + source = "bash -c 'echo ready'\n" + tail + scanned = 0 + checks = 0 + original = tm._skip_shell_delimited_expansion + + def counted(content: str, start: int, limit: int, *args, **kwargs): + nonlocal scanned + result = original(content, start, limit, *args, **kwargs) + scanned += (limit if result is None else result) - start + return result + + def check_runtime() -> None: + nonlocal checks + checks += 1 + + monkeypatch.setattr(tm, "_skip_shell_delimited_expansion", counted) + closers: set[int] = set() + tm._shell_quoted_candidate_limits(source, check_runtime, quote_closers=closers) + assert scanned <= 8 * len(source) + assert checks > 0 + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "wrapper", ["bash", "env MODE=test bash", "sudo -n bash", "timeout 5 bash"] +) +def test_literal_command_proof_selects_only_relevant_clauses( + wrapper: str, monkeypatch: pytest.MonkeyPatch +) -> None: + prefix = "echo ready\n" * 1000 + source = prefix + wrapper + " -c 'echo ready'\n" + selected: list[int] = [] + original = tm._command_string_from_clause + + def recorded(content, start, check_runtime, **kwargs): + selected.append(start) + return original(content, start, check_runtime, **kwargs) + + monkeypatch.setattr(tm, "_command_string_from_clause", recorded) + spans = tm._literal_shell_command_spans(source, 0, len(source), lambda: None) + + assert selected == [len(prefix)] + assert spans is not None and len(spans) == 1 + assert source[next(iter(spans)) : next(iter(spans.values()))] == "echo ready" + + +@pytest.mark.parametrize( + "source", + [ + "# bash -c 'echo ready'\n", + "echo \"bash -c 'echo ready'\"\n", + "echo \"$(bash -c 'echo ready')\"\n", + "echo -" + "c" * 100000 + " /tmp/value'\n", + ], +) +def test_literal_command_option_lookalikes_cannot_claim_a_body(source: str) -> None: + assert tm._literal_shell_command_spans(source, 0, len(source), lambda: None) == {} + + +def test_selected_wrapper_budget_discards_previously_collected_literal_proof() -> None: + source = "bash -c 'echo ready'\n" + source += "eval " + "'part' " * 24 + '"-c \'echo ready\'" "$CMD -rf /"\n' + closers: set[int] = set() + + assert tm._literal_shell_command_spans(source, 0, len(source), lambda: None) is None + assert tm._shell_quoted_candidate_limits(source, lambda: None, quote_closers=closers) == {} + assert closers == set() + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize("escaped", ["\\\n", "\\\r\n", "\\x"]) +def test_invocation_runtime_checks_survive_escaped_offset_jumps(escaped: str) -> None: + source = " " + escaped * 12000 + "value" + checks = 0 + + def cancelled() -> None: + nonlocal checks + checks += 1 + if checks == 3: + raise TimeoutError("escaped word deadline") + + with pytest.raises(TimeoutError, match="escaped word deadline"): + tm._next_shell_invocation_word(source, 1, cancelled) + + +def test_literal_command_proof_propagates_real_runtime_exceptions() -> None: + source = "bash -c 'echo ready'\n" + + def cancelled() -> None: + raise TimeoutError("caller deadline") + + with pytest.raises(TimeoutError, match="caller deadline"): + tm._literal_shell_command_spans(source, 0, len(source), cancelled) + + +@pytest.mark.parametrize( + "tail", + [ + 'cat < None: + closers: set[int] = set() + source = 'cat <<<"$VALUE"\ncp "$(dirname "$SOURCE")/file" "$DEST"\n' + tail + + assert tm._shell_quoted_candidate_limits(source, lambda: None, quote_closers=closers) == {} + assert closers == set() + + @pytest.mark.parametrize( "source", [ @@ -72,7 +587,7 @@ def test_fragment_cannot_prove_initial_quote_state() -> None: @pytest.mark.parametrize( "source", [ - 'echo "prefix $(date) suffix"\ncat < None: + source = " \t\n" * 10000 + "bash -c 'echo ready'\n" + checks = 0 + + def check() -> None: + nonlocal checks + checks += 1 + assert checks < 1000 + + assert not tm.has_bounded_parse_exhaustion(source, check, file_type="shell") + + +@pytest.mark.parametrize( + "startup", + ["BASH_ENV=setup.sh ", "export BASH_ENV=setup.sh; ", "env BASH_ENV=setup.sh "], +) +def test_shell_startup_code_keeps_literal_comparison_partial(startup: str) -> None: + source = startup + 'bash -c \'test x = "$(printf "%s%s%s" r m " -rf /")"\'' + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_quoted_printf_preserves_multiline_literal_value(newline: str) -> None: + substitution = ( + '$(printf "%s\\n" "first named action" "second named action" \\\n' + ' "third named action" "fourth named action" "last named action")' + ).replace("\n", newline) + expected = "\n".join( + [ + "first named action", + "second named action", + "third named action", + "fourth named action", + "last named action", + ] + ) + + assert ( + tm._static_printf_substitution(substitution, 0, len(substitution), quoted_result=True) + == expected + ) + assert tm._static_printf_substitution(substitution, 0, len(substitution)) is None + assert not tm.has_bounded_parse_exhaustion( + f'test "$observed" = "{substitution}"\n', lambda: None, file_type="shell" + ) + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + ('printf "%s\\n\\n" "a b"', "a b"), + ('printf "%s\\r\\n" "a b"', "a b\r"), + ('printf "%s" "a\nb"', "a\nb"), + ('printf "%s" ""', ""), + ('printf "%%s\\n"', "%s"), + ('printf -- "-%s" "a b"', "-a b"), + ('printf "%s%s" "a " b c', "a bc"), + ], +) +def test_quoted_printf_has_exact_command_substitution_bytes(body: str, expected: str) -> None: + substitution = f"$({body})" + + assert ( + tm._static_printf_substitution(substitution, 0, len(substitution), quoted_result=True) + == expected + ) + + +@pytest.mark.parametrize( + "body", + [ + 'printf "%s" "$value"', + 'printf "%s" "$(get_value)"', + 'printf "%s" "`get_value`"', + 'printf "%s" *', + 'printf "%s" {a,b}', + 'printf "%s" ~', + 'printf "%s" a; echo b', + 'printf "%s" a\necho b', + 'printf "%s" a # omitted', + 'printf "%q" "a b"', + 'printf "%b" "a b"', + 'printf "%d" 10', + 'printf "%s\\c" "a b"', + 'printf "%s\\0" "a b"', + 'printf -v result "%s" "a b"', + 'printf "%s" "unterminated', + 'printf "%s" trailing\\', + 'printf "%s" ' + '"x" ' * 33, + 'printf "%s" "' + "x" * 65 + '"', + 'printf "%s" "a b"' + " " * 256, + ], +) +def test_quoted_printf_keeps_unknown_or_over_limit_values_unresolved(body: str) -> None: + substitution = f"$({body})" + + assert ( + tm._static_printf_substitution(substitution, 0, len(substitution), quoted_result=True) + is None + ) + + +@pytest.mark.parametrize("extra", ["", "x"]) +def test_quoted_printf_output_limit_is_exact(extra: str) -> None: + substitution = '$(printf "' + "a" * 62 + '%s" "xx' + extra + '" xx xx xx)' + result = tm._static_printf_substitution(substitution, 0, len(substitution), quoted_result=True) + + assert (result is not None) is (not extra) + if result is not None: + assert len(result) == tm._PRINTF_STATIC_CHARS + + +@pytest.mark.parametrize( + "source", + [ + '"$(printf "%s\\n" rm)" -rf /', + 'r"$(printf "%s\\n" m)" -rf /', + 'rm "$(printf "%s" -rf)" "$(printf "%s\\n" /)"', + ], +) +def test_quoted_printf_reconstruction_preserves_destructive_findings(source: str) -> None: + assert any( + finding.rule_id == "TM1" and finding.severity.value == "HIGH" + for finding in tm.analyze(source, "run.sh", "shell") + ) + + +@pytest.mark.parametrize( + "source", + [ + '"$(printf "%s" "rm -rf /")"', + '"$(printf "%s\\n" rm)"suffix -rf /', + ], +) +def test_quoted_printf_whitespace_never_becomes_separate_shell_words(source: str) -> None: + parsed = tm._parse_shell_command_word(source, 0) + + assert parsed is not None and not parsed.dynamic and not parsed.limited + assert parsed.text in {"rm -rf /", "rmsuffix"} + + +@pytest.mark.parametrize( + "source", + [ + '$(printf "%s\\n" "rm -rf /")', + '"$(printf "%s" "$command")" -rf /', + 'bash -c "$(printf "%s" "rm -rf /")"', + 'eval "$(printf "%s" "rm -rf /")"', + 'test x = "$(printf "%s" "a b")"; $CMD -rf /', + ], +) +def test_quoted_printf_keeps_other_execution_paths_partial(source: str) -> None: + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "setting", + [ + "BASH_ENV=setup.sh ", + "ENV=setup.sh ", + "export BASH_ENV=setup.sh; ", + 'export "BASH_ENV=setup.sh"; ', + "export BAS\\H_ENV=setup.sh; ", + "declare -x BASH_ENV=setup.sh; ", + "typeset -x ENV=setup.sh; ", + "readonly BASH_ENV=setup.sh; ", + "export BASH_ENV; ", + "env BASH_ENV=setup.sh ", + 'env "ENV=setup.sh" ', + "/usr/bin/env BASH_ENV=setup.sh ", + "command -p env BASH_ENV=setup.sh ", + "env -C /tmp BASH_ENV=setup.sh ", + "env --chdir /tmp ENV=setup.sh ", + "env -u OTHER BASH_ENV=setup.sh ", + "env --unset OTHER ENV=setup.sh ", + "env --unset=OTHER --chdir=/tmp BASH_ENV=setup.sh ", + "builtin export BASH_ENV=setup.sh; ", + "nohup env BASH_ENV=setup.sh ", + "/usr/bin/nohup env ENV=setup.sh ", + "sudo -n env BASH_ENV=setup.sh ", + "timeout 10 env BASH_ENV=setup.sh ", + "nice -n 5 env ENV=setup.sh ", + "BASH_ENV+=; ", + "BASH_ENV= ENV=setup.sh ", + "BASH_ENV=setup.sh; BASH_ENV=; ", + "command export BASH_ENV=setup.sh; ", + ], +) +def test_explicit_shell_startup_is_unknown_independently_of_printf(setting: str) -> None: + assert tm.has_bounded_parse_exhaustion( + setting + "bash -c 'echo ready'", lambda: None, file_type="shell" + ) + + +@pytest.mark.parametrize( + "prefix", + [ + "BASH_ENV= ", + "export BASH_ENV=; ", + "env ENV= ", + "env OTHER=example ", + "/usr/bin/env -C /tmp OTHER=example ", + "command -p env --unset OTHER OTHER=example ", + 'echo "BASH_ENV=setup.sh"; ', + "# export BASH_ENV=setup.sh\n", + "printf '%s' 'ENV=setup.sh'; ", + "env echo BASH_ENV=setup.sh; ", + ], +) +def test_inert_startup_text_or_initial_empty_settings_do_not_add_partial(prefix: str) -> None: + assert not tm.has_bounded_parse_exhaustion( + prefix + "bash -c 'echo ready'", lambda: None, file_type="shell" + ) + + +def test_literal_child_shell_keeps_its_own_startup_settings() -> None: + source = "bash -c 'export BASH_ENV=setup.sh; bash -c \"echo ready\"'" + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "definition", + [ + "printf() { echo rm; }; ", + "printf ( ) { echo rm; }; ", + "printf \\\n( \\\n) { echo rm; }; ", + "printf \\\r\n( \\\r\n) { echo rm; }; ", + "function printf { echo rm; }; ", + "function 'printf'() { echo rm; }; ", + "p\\rintf() { echo rm; }; ", + "helper() { echo ready; }; printf() { echo rm; }; ", + "printf" + " " * 8200 + "() { echo rm; }; ", + "shopt -s expand_aliases; alias printf='echo rm'; ", + "shopt -s expand_aliases; builtin alias printf='echo rm';\n", + "shopt -s expand_aliases; command -p alias printf='echo rm';\n", + "shopt -s expand_aliases; builtin -- alias printf='echo rm';\n", + "shopt -s expand_aliases\nX=y alias printf='echo rm'\n", + "alias other=echo 'printf=echo rm'; ", + 'alias "$replacement"; ', + ], +) +def test_explicit_printf_replacement_keeps_wider_quoted_result_partial(definition: str) -> None: + source = definition + '"$(printf "%s" "a b")" -rf /' + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "prefix", + [ + "helper() { echo ready; }; ", + "function helper { echo ready; }; ", + "echo 'printf() { echo rm; }'; ", + "# printf() { echo rm; }\n", + "alias other=echo; ", + "echo 'alias printf=echo'; ", + ], +) +def test_unrelated_or_inert_function_text_keeps_quoted_printf_literal(prefix: str) -> None: + source = prefix + 'test x = "$(printf "%s" "a b")"' + + assert not tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "setter", + [ + "set -a; read BASH_ENV", + "read -r first ENV", + "IFS= read -r BASH_ENV", + "builtin read -rp prompt ENV", + "command read -r -- BASH_ENV", + "read -a BASH_ENV", + "read -raENV", + "read -d ':' ENV", + "read -N1 BASH_ENV", + 'read -r "$target"', + "printf -v BASH_ENV '%s' /tmp/startup", + "builtin printf -vENV '%s' /tmp/startup", + "command -p printf -v ENV '%s' /tmp/startup", + 'printf -v "$target" %s /tmp/startup', + 'printf "$option" BASH_ENV %s /tmp/startup', + "printf -v harmless -v ENV '%s' /tmp/startup", + "printf -vharmless -vBASH_ENV '%s' /tmp/startup", + "read -a harmless -a BASH_ENV", + "read -aharmless -raENV", + "printf '%n' BASH_ENV", + "printf -- '%10n' ENV", + "printf -v harmless '%%%n' BASH_ENV", + "printf '%ln' BASH_ENV", + "printf '%hhn' ENV", + "declare -n hook=BASH_ENV; hook=setup.sh", + "typeset -nx hook=ENV; hook=setup.sh", + ], +) +def test_startup_variable_setters_keep_child_shell_review_partial(setter: str) -> None: + source = setter + "; bash -c 'printf %s ready'\n" + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize("name", ["BASH_ENV", "ENV", '"$name"']) +@pytest.mark.parametrize( + "declaration", + ["local", "local -x", "builtin local -x", "local +n -x", "declare -x", "export"], +) +def test_local_startup_setting_keeps_function_child_partial(name: str, declaration: str) -> None: + source = ( + f"run() {{ {declaration} {name}=setup.sh; " + 'bash -c \'echo "$(printf "%s\\n" "two words")"\'; }; run\n' + ) + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "declaration", + [ + "local message=BASH_ENV", + "local -x message=ENV", + 'export message="$value"', + "export -n message", + ], +) +def test_local_startup_name_in_value_does_not_taint_child(declaration: str) -> None: + source = f"run() {{ {declaration}; bash -c 'printf %s ready'; }}; run\n" + + assert not tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize("name", ["BASH_ENV", "ENV"]) +def test_local_reference_to_startup_variable_keeps_child_partial(name: str) -> None: + source = f"run() {{ local -n hook={name}; hook=setup.sh; bash -c 'printf %s ready'; }}; run\n" + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "prefix", + [ + "source setup.sh", + ". setup.sh", + "builtin source setup.sh", + "command . setup.sh", + "eval 'printf() { echo changed; }'", + "builtin eval 'printf() { echo changed; }'", + "enable -n printf", + "trap 'printf() { echo changed; }' DEBUG", + ], +) +def test_imported_or_evaluated_code_cannot_prove_quoted_printf(prefix: str) -> None: + source = prefix + '; "$(printf "%s\\n" "two words")" -rf /\n' + + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "prefix", + [ + "echo source setup.sh", + "echo eval", + "echo enable trap", + "# source setup.sh\ntrue", + "helper() { echo ready; }", + ], +) +def test_inert_source_mentions_do_not_disable_quoted_printf(prefix: str) -> None: + source = prefix + '; echo "$(printf "%s\\n" "two words")"\n' + + assert not tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") + + +@pytest.mark.parametrize( + "prefix", + [ + "read -r value", + "read -p BASH_ENV value", + "read -d ENV value", + "read -r -i BASH_ENV value", + "read -rpBASH_ENV value", + "read -a values", + "printf '%s' BASH_ENV", + "printf -- '-v' ENV", + "printf -v result '%s' BASH_ENV", + "printf -v result -v other '%s' ENV", + "printf -vresult -vother '%s' BASH_ENV", + "read -a first -a second", + "read -afirst -rasecond", + "read -a safe BASH_ENV", + "read -asafe ENV", + "printf '%%n' BASH_ENV", + "printf '%%%%n' ENV", + "echo 'read BASH_ENV'", + "# printf -v ENV value\n", + ], +) +def test_startup_names_in_non_setter_operands_do_not_taint_child(prefix: str) -> None: + source = prefix + "; bash -c 'printf %s ready'\n" + + assert not tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") diff --git a/tests/nodes/analyzers/test_static_parse_limit_literals.py b/tests/nodes/analyzers/test_static_parse_limit_literals.py index 72cac972a..57efe3236 100644 --- a/tests/nodes/analyzers/test_static_parse_limit_literals.py +++ b/tests/nodes/analyzers/test_static_parse_limit_literals.py @@ -42,6 +42,13 @@ pytest.param("$(printf '%%s')", "%s", id="escaped-percent"), ] +_QUOTED_LITERAL_SUBSTITUTIONS = [ + pytest.param("$(printf '%s%s' 'rm -rf' ' /')", "rm -rf /", id="assembled-destructive"), + pytest.param("$(printf '\\040rm')", " rm", id="escape-emits-separator"), + pytest.param("$(printf '\\047rm')", "'rm", id="escape-emits-quote"), + pytest.param("$(printf '\\133\\135')", "[]", id="decoded-brackets"), +] + @pytest.mark.parametrize("substitution,expected", _LITERAL_SUBSTITUTIONS) def test_bounded_printf_is_resolved_statically(substitution: str, expected: str) -> None: @@ -96,15 +103,11 @@ def test_rustup_machine_probe_reaches_completed_ledger_outcome() -> None: "substitution", [ pytest.param("$(printf 'rm -rf /')", id="literal-destructive"), - pytest.param("$(printf '%s%s' 'rm -rf' ' /')", id="assembled-destructive"), pytest.param("$(printf '\\162\\155 -rf /')", id="octal-prefixed-destructive"), - pytest.param("$(printf '\\040rm')", id="escape-emits-separator"), - pytest.param("$(printf '\\047rm')", id="escape-emits-quote"), pytest.param("$(printf '\\400')", id="octal-overflow-nul"), pytest.param("$(printf '\\440')", id="octal-overflow-space"), pytest.param("$(printf '\\447')", id="octal-truncates-to-quote"), pytest.param("$(printf '\\504')", id="octal-truncates-to-dollar"), - pytest.param("$(printf '\\133\\135')", id="decoded-brackets"), pytest.param("$(printf '\\qrm')", id="unknown-escape"), ], ) @@ -113,6 +116,52 @@ def test_destructive_or_undecidable_printf_stays_partial(substitution: str) -> N assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") is True +@pytest.mark.parametrize("substitution,expected", _QUOTED_LITERAL_SUBSTITUTIONS) +def test_quoted_printf_output_preserves_exact_bytes_and_next_argument( + substitution: str, expected: str +) -> None: + source = f'echo "{substitution}" SENTINEL\n' + parsed = tm._parse_shell_command_word(source, len("echo ")) + assert parsed is not None + assert parsed.text == expected + assert parsed.end == source.index(" SENTINEL") + assert not parsed.dynamic and not parsed.limited + following = tm._parse_shell_command_word(source, parsed.end + 1) + assert following is not None + assert following.text == "SENTINEL" + assert source[following.end :] == "\n" + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") is False + + +@pytest.mark.parametrize("substitution,_expected", _QUOTED_LITERAL_SUBSTITUTIONS) +@pytest.mark.parametrize("consumer", ["unquoted", "eval", "bash-c"]) +def test_quoted_literal_support_does_not_complete_unmodeled_execution( + substitution: str, _expected: str, consumer: str +) -> None: + if consumer == "unquoted": + source = f"{substitution} -rf /\n" + elif consumer == "eval": + source = f'eval "{substitution}"\n' + else: + source = f'bash -c "{substitution}"\n' + assert tm.has_bounded_parse_exhaustion(source, lambda: None, file_type="shell") is True + + +@pytest.mark.parametrize( + "source", + [ + '"$(printf "%s\\n" rm)" -rf /', + 'r"$(printf "%s\\n" m)" -rf /', + 'rm "$(printf "%s" -rf)" "$(printf "%s\\n" /)"', + ], +) +def test_quoted_printf_destructive_heads_and_operands_keep_high_finding(source: str) -> None: + findings = static_runner.run_static_patterns( + {"components": ["run.sh"], "file_cache": {"run.sh": source}}, [tm] + ) + assert any(finding.rule_id == "TM1" and finding.severity == "HIGH" for finding in findings) + + @pytest.mark.parametrize( "substitution", [