diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a1b1a34..d3e3c31d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -196,6 +196,9 @@ All notable changes to SkillOpt are documented here. This project adheres to of a sequence of characters, in scoring and in the gold answers recorded on rollout rows. The correct answer was scored as wrong and a one-letter answer as correct (thanks @Shizoqua). +- Keep the DocVQA answer when a stray `` or `` follows the + final answer, instead of scoring the wrong text and giving a correct answer + ANLS 0 (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/envs/docvqa/evaluator.py b/skillopt/envs/docvqa/evaluator.py index 85c09efa..e39c8fb8 100644 --- a/skillopt/envs/docvqa/evaluator.py +++ b/skillopt/envs/docvqa/evaluator.py @@ -2,11 +2,16 @@ import ast import json +import re from collections.abc import Iterable from typing import Any DEFAULT_ANLS_THRESHOLD = 0.5 +# A complete ... pair whose body contains no other "", +# so nested input keeps the innermost answer and stray tags elsewhere are ignored. +_ANSWER_RE = re.compile(r"((?:(?!).)*?)", re.DOTALL | re.IGNORECASE) + def _normalize_text(value: Any) -> str: if value is None: @@ -91,11 +96,9 @@ def _extract_answer_strings(raw: Any) -> list[str]: def extract_answer(text: str) -> str: - lower = text.lower() - start = lower.rfind("") - end = lower.rfind("") - if start != -1 and end != -1 and end > start: - return text[start + len(""):end].strip() + matches = _ANSWER_RE.findall(text) + if matches: + return matches[-1].strip() lines = [line.strip() for line in text.splitlines() if line.strip()] return lines[-1] if lines else text.strip() diff --git a/tests/test_docvqa_evaluator.py b/tests/test_docvqa_evaluator.py index a68519f0..b878854d 100644 --- a/tests/test_docvqa_evaluator.py +++ b/tests/test_docvqa_evaluator.py @@ -98,9 +98,25 @@ def test_extract_answer_strings_accepts_dataset_shapes(raw, expected): [ ("reasoning\nFoo", "Foo"), ("x then y", "y"), + ("xy", "y"), ("first line\n\nlast line ", "last line"), ("", ""), ], ) def test_extract_answer(text, expected): assert extract_answer(text) == expected + + +@pytest.mark.parametrize( + "text", + [ + "42 (formatted with the tag as asked)", + "The total is 42\nNote: only one block.", + "42\n", + "42 (wrapped in as asked)", + "42\n", + ], +) +def test_stray_answer_tag_after_the_last_complete_answer_is_ignored(text): + assert extract_answer(text) == "42" + assert evaluate(text, ["42"])["anls"] == 1.0