From 77d8984c455487c836ec78eee42c8d604a76504c Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:37:55 +0800 Subject: [PATCH 1/2] fix(human_labeled_dataset): preserve '#' in unquoted CSV cells --- .../human_labeled_dataset.py | 13 ++++++++++-- .../unit/score/test_human_labeled_dataset.py | 20 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/pyrit/score/scorer_evaluation/human_labeled_dataset.py b/pyrit/score/scorer_evaluation/human_labeled_dataset.py index 87c2eb6143..7a0cc20089 100644 --- a/pyrit/score/scorer_evaluation/human_labeled_dataset.py +++ b/pyrit/score/scorer_evaluation/human_labeled_dataset.py @@ -270,11 +270,20 @@ def from_csv( if not harm_definition_version and parsed_harm_definition_version: harm_definition_version = parsed_harm_definition_version + # Skip the leading "#" comment line(s) instead of relying on comment="#", + # which also truncates the remainder of unquoted cells containing "#" (#2974). + skiprows = 0 + with open(csv_path, encoding="utf-8", errors="ignore") as f: + for line in f: + if line.lstrip().startswith("#"): + skiprows += 1 + else: + break # Try UTF-8 first, fall back to latin-1 for files with special characters try: - eval_df = pd.read_csv(csv_path, comment="#", encoding="utf-8") + eval_df = pd.read_csv(csv_path, skiprows=skiprows or None, encoding="utf-8") except UnicodeDecodeError: - eval_df = pd.read_csv(csv_path, comment="#", encoding="latin-1") + eval_df = pd.read_csv(csv_path, skiprows=skiprows or None, encoding="latin-1") # Drop rows where every column is NaN (e.g. trailing blank lines in the CSV) eval_df = eval_df.dropna(how="all") diff --git a/tests/unit/score/test_human_labeled_dataset.py b/tests/unit/score/test_human_labeled_dataset.py index 665a55fe88..80f0140034 100644 --- a/tests/unit/score/test_human_labeled_dataset.py +++ b/tests/unit/score/test_human_labeled_dataset.py @@ -630,3 +630,23 @@ def test_scorer_eval_csv_loads_with_human_labeled_dataset(csv_file, metrics_type # - Invalid data formats dataset = HumanLabeledDataset.from_csv(csv_path=csv_file, metrics_type=metrics_type) assert len(dataset.entries) > 0, f"Dataset {csv_file.name} has no entries" + + +def test_from_csv_preserves_hash_characters_in_cells(tmp_path): + """A literal '#' inside an unquoted cell must not truncate the row (#2974).""" + csv_file = tmp_path / "hash_cells.csv" + csv_file.write_text( + "# dataset_version=1.0\n" + "assistant_response,human_score,objective\n" + "# Heading,1,respond to the prompt\n" + "ordinary response,0,mentions C#\n", + encoding="utf-8", + ) + + dataset = HumanLabeledDataset.from_csv( + csv_path=str(csv_file), + metrics_type=MetricsType.OBJECTIVE, + ) + assert len(dataset.entries) == 2 + assert dataset.entries[0].objective == "respond to the prompt" + assert dataset.entries[1].objective == "mentions C#" From dc504802ad47ca99eabd4e7c2041ecc06fde960b Mon Sep 17 00:00:00 2001 From: Chen Yufeiyang Date: Fri, 9 Oct 2026 05:10:19 +0800 Subject: [PATCH 2/2] FIX: keep a BOM or blank line from hiding the metadata comment line The scan for leading "#" comment lines read the file as utf-8, so a byte-order mark left the first line looking like data and a blank line stopped the scan, which dropped the metadata comment into the header row. Both reads now use utf-8-sig, and the version parse reads the first line the same way, so a BOM file's comment line is recognized as the comment line it is. --- .../human_labeled_dataset.py | 15 +++++--- .../unit/score/test_human_labeled_dataset.py | 36 +++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/pyrit/score/scorer_evaluation/human_labeled_dataset.py b/pyrit/score/scorer_evaluation/human_labeled_dataset.py index 7a0cc20089..7e9c00d73b 100644 --- a/pyrit/score/scorer_evaluation/human_labeled_dataset.py +++ b/pyrit/score/scorer_evaluation/human_labeled_dataset.py @@ -206,7 +206,8 @@ def from_csv( - 'data_type': Optional data type (defaults to 'text' if not present) You can optionally include a # comment line at the top of the CSV file to specify - the dataset version and harm definition path. The format is: + the dataset version and harm definition path. Only leading # lines are treated as + comments; a # anywhere else is preserved as literal data. The format is: - For harm datasets: # dataset_version=x.y, harm_definition=path/to/definition.yaml, harm_definition_version=x.y - For objective datasets: # dataset_version=x.y @@ -234,8 +235,10 @@ def from_csv( parsed_version = None parsed_harm_definition = None parsed_harm_definition_version = None + # "utf-8-sig" drops a byte-order mark if the file has one, so a BOM file's + # first line is still recognized as the metadata comment line try: - with open(csv_path, encoding="utf-8") as f: + with open(csv_path, encoding="utf-8-sig") as f: first_line = f.readline().strip() except UnicodeDecodeError: with open(csv_path, encoding="latin-1") as f: @@ -272,10 +275,14 @@ def from_csv( # Skip the leading "#" comment line(s) instead of relying on comment="#", # which also truncates the remainder of unquoted cells containing "#" (#2974). + # "utf-8-sig" drops a byte-order mark, which does not start a "#" line but + # does leave the metadata line looking like data, and a leading blank line + # is skipped the same way so it cannot stop the scan either. skiprows = 0 - with open(csv_path, encoding="utf-8", errors="ignore") as f: + with open(csv_path, encoding="utf-8-sig", errors="ignore") as f: for line in f: - if line.lstrip().startswith("#"): + stripped = line.strip() + if stripped.startswith("#") or not stripped: skiprows += 1 else: break diff --git a/tests/unit/score/test_human_labeled_dataset.py b/tests/unit/score/test_human_labeled_dataset.py index 80f0140034..61f6cd2ca1 100644 --- a/tests/unit/score/test_human_labeled_dataset.py +++ b/tests/unit/score/test_human_labeled_dataset.py @@ -650,3 +650,39 @@ def test_from_csv_preserves_hash_characters_in_cells(tmp_path): assert len(dataset.entries) == 2 assert dataset.entries[0].objective == "respond to the prompt" assert dataset.entries[1].objective == "mentions C#" + + +def test_from_csv_reads_metadata_comment_after_byte_order_mark(tmp_path): + """A UTF-8 BOM (what Excel writes) must not hide the metadata comment line.""" + csv_file = tmp_path / "bom_cells.csv" + csv_file.write_text( + "# dataset_version=1.0\nassistant_response,human_score,objective\nordinary response,0,mentions C#\n", + encoding="utf-8-sig", + ) + + dataset = HumanLabeledDataset.from_csv( + csv_path=str(csv_file), + metrics_type=MetricsType.OBJECTIVE, + ) + assert len(dataset.entries) == 1 + assert dataset.entries[0].objective == "mentions C#" + + +def test_from_csv_skips_blank_line_before_metadata_comment(tmp_path): + """A blank line before the metadata comment must not stop the skip scan.""" + csv_file = tmp_path / "blank_first.csv" + csv_file.write_text( + "\n" + "# dataset_version=1.0\n" + "assistant_response,human_score,objective\n" + "ordinary response,0,respond to the prompt\n", + encoding="utf-8", + ) + + dataset = HumanLabeledDataset.from_csv( + csv_path=str(csv_file), + metrics_type=MetricsType.OBJECTIVE, + version="1.0", + ) + assert len(dataset.entries) == 1 + assert dataset.entries[0].objective == "respond to the prompt"