diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9a1b1a34..d3e3c31d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -196,6 +196,9 @@ All notable changes to SkillOpt are documented here. This project adheres to
of a sequence of characters, in scoring and in the gold answers recorded on
rollout rows. The correct answer was scored as wrong and a one-letter answer
as correct (thanks @Shizoqua).
+- Keep the DocVQA answer when a stray `` or `` follows the
+ final answer, instead of scoring the wrong text and giving a correct answer
+ ANLS 0 (thanks @Shizoqua).
### Security
- Prevent managed-identity credentials from being sent to non-Azure or
diff --git a/skillopt/envs/docvqa/evaluator.py b/skillopt/envs/docvqa/evaluator.py
index 85c09efa..e39c8fb8 100644
--- a/skillopt/envs/docvqa/evaluator.py
+++ b/skillopt/envs/docvqa/evaluator.py
@@ -2,11 +2,16 @@
import ast
import json
+import re
from collections.abc import Iterable
from typing import Any
DEFAULT_ANLS_THRESHOLD = 0.5
+# A complete ... pair whose body contains no other "",
+# so nested input keeps the innermost answer and stray tags elsewhere are ignored.
+_ANSWER_RE = re.compile(r"((?:(?!).)*?)", re.DOTALL | re.IGNORECASE)
+
def _normalize_text(value: Any) -> str:
if value is None:
@@ -91,11 +96,9 @@ def _extract_answer_strings(raw: Any) -> list[str]:
def extract_answer(text: str) -> str:
- lower = text.lower()
- start = lower.rfind("")
- end = lower.rfind("")
- if start != -1 and end != -1 and end > start:
- return text[start + len(""):end].strip()
+ matches = _ANSWER_RE.findall(text)
+ if matches:
+ return matches[-1].strip()
lines = [line.strip() for line in text.splitlines() if line.strip()]
return lines[-1] if lines else text.strip()
diff --git a/tests/test_docvqa_evaluator.py b/tests/test_docvqa_evaluator.py
index a68519f0..b878854d 100644
--- a/tests/test_docvqa_evaluator.py
+++ b/tests/test_docvqa_evaluator.py
@@ -98,9 +98,25 @@ def test_extract_answer_strings_accepts_dataset_shapes(raw, expected):
[
("reasoning\nFoo", "Foo"),
("x then y", "y"),
+ ("xy", "y"),
("first line\n\nlast line ", "last line"),
("", ""),
],
)
def test_extract_answer(text, expected):
assert extract_answer(text) == expected
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ "42 (formatted with the tag as asked)",
+ "The total is 42\nNote: only one block.",
+ "42\n",
+ "42 (wrapped in as asked)",
+ "42\n",
+ ],
+)
+def test_stray_answer_tag_after_the_last_complete_answer_is_ignored(text):
+ assert extract_answer(text) == "42"
+ assert evaluate(text, ["42"])["anls"] == 1.0