From 731db058580a9b596e7b004282c5cf1cf587bfbf Mon Sep 17 00:00:00 2001 From: findalexli Date: Tue, 3 Nov 2026 21:59:04 -0800 Subject: [PATCH] juliet_vulnerability_analyzer: accept the release as a CWE-416 source --- .../juliet_vulnerability_analyzer/NOTES.md | 12 +++++++++- .../environment/workspace/scoring/score.py | 23 +++++++++++++++++-- .../scoring/score.py | 23 +++++++++++++++++-- .../juliet_vulnerability_analyzer/task.toml | 2 +- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md index b69a00a..cc03080 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md +++ b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md @@ -11,6 +11,16 @@ raw score. ## Changes +### 1.2.0 + +The README says `source_stmt` identifies the "source/allocation/release witness", but the CWE-416 +labels use the allocation, so a finding that starts at the release kept its detection credit and +lost its trace credit. For CWE-416 only, `trace_valid` now also accepts a `free` or `delete` of a +value the sink uses, in the sink's function, between the labelled allocation and the sink. Each of +the 120 hidden labels has exactly one such statement. Nothing that was accepted is now rejected, so +scores only rise, and only for submissions that use the release as source. Regression summary +describes 1.0.0. Issue: https://github.com/ByteDance-Seed/EdgeBench/issues/50 + ### 1.1.0 `infer` is no longer a banned word. The check also matched the English word, so one comment using @@ -23,7 +33,7 @@ and only for submissions containing the word. Regression summary describes 1.0.0 - Workspace (`juliet-static-analyzer-public-v3`: starter analyzer, public facts and labels, scorer, tools) and the hidden evaluator (`juliet-static-analyzer-evaluator-v3`, with the hidden facts and labels): the task authors' repository (not public), identical by sha256 and mode to the - published v1 images, except `scoring/score.py` in both, changed in 1.1.0. + published v1 images, except `scoring/score.py` in both, changed in 1.1.0 and 1.2.0. - `verify_task.sh`: the version shipped in the published v1 images, which dropped the Chinese README (also absent here). - `judge/eval_hidden.py` (v1's inline eval program, one statement per line), `judge/run.sh` and diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py b/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py index 6832785..d0a5297 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py +++ b/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py @@ -114,11 +114,29 @@ def stmt_ids_by_case(cases:list[dict[str,Any]]) -> dict[str,set[int]]: out[str(c.get('case_id'))]=ids return out -def trace_valid(pred:dict[str,Any], label:dict[str,Any], stmt_ids:set[int]) -> tuple[bool,str]: +def release_witnesses(label:dict[str,Any], stmts:list[dict[str,Any]]) -> set[int]: + # CWE-416: the README allows a release witness as source_stmt. Accept a free/delete of a value + # the sink uses, in the sink's function, between the labelled allocation and the sink. + if str(label.get('cwe'))!='CWE-416': return set() + by_id={s.get('stmt_id'):s for s in stmts} + sink=int(label.get('sink_stmt')); src=int(label.get('source_stmt',sink)); s_sink=by_id.get(sink) or {} + used=set(s_sink.get('uses') or []) + out=set() + for s in stmts: + sid=s.get('stmt_id') + if not isinstance(sid,int) or not src tuple[bool,str]: expected=numeric_list(label.get('trace',[])) source=int(label.get('source_stmt', expected[0] if expected else label.get('sink_stmt'))) sink=int(label.get('sink_stmt')) got=pred.get('trace',[]) + if int(pred.get('source_stmt',-1)) in release: + source=int(pred.get('source_stmt')); expected=[source,sink] if int(pred.get('source_stmt',-1)) != source: return False,'wrong_source' if not got: @@ -150,12 +168,13 @@ def evaluate(cases:list[dict[str,Any]], labels:list[dict[str,Any]], preds:list[d adv_pk={k for k in pk if k in adv_lk} adv_f1=f1(len(adv_lk & adv_pk),len(adv_pk-adv_lk),len(adv_lk-adv_pk)) ids=stmt_ids_by_case(cases) + stmts_by_case={str(c.get('case_id')):c.get('stmts',[]) for c in cases} trace_req=0; trace_ok=0; trace_invalid=0; reasons={} for k in tp: expected=numeric_list(label_by[k].get('trace',[])) if expected: trace_req+=1 - good, reason=trace_valid(pred_by[k], label_by[k], ids.get(k[0],set())) + good, reason=trace_valid(pred_by[k], label_by[k], ids.get(k[0],set()), release_witnesses(label_by[k], stmts_by_case.get(k[0],[]))) if good: trace_ok+=1 else: diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py b/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py index 6832785..d0a5297 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py +++ b/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py @@ -114,11 +114,29 @@ def stmt_ids_by_case(cases:list[dict[str,Any]]) -> dict[str,set[int]]: out[str(c.get('case_id'))]=ids return out -def trace_valid(pred:dict[str,Any], label:dict[str,Any], stmt_ids:set[int]) -> tuple[bool,str]: +def release_witnesses(label:dict[str,Any], stmts:list[dict[str,Any]]) -> set[int]: + # CWE-416: the README allows a release witness as source_stmt. Accept a free/delete of a value + # the sink uses, in the sink's function, between the labelled allocation and the sink. + if str(label.get('cwe'))!='CWE-416': return set() + by_id={s.get('stmt_id'):s for s in stmts} + sink=int(label.get('sink_stmt')); src=int(label.get('source_stmt',sink)); s_sink=by_id.get(sink) or {} + used=set(s_sink.get('uses') or []) + out=set() + for s in stmts: + sid=s.get('stmt_id') + if not isinstance(sid,int) or not src tuple[bool,str]: expected=numeric_list(label.get('trace',[])) source=int(label.get('source_stmt', expected[0] if expected else label.get('sink_stmt'))) sink=int(label.get('sink_stmt')) got=pred.get('trace',[]) + if int(pred.get('source_stmt',-1)) in release: + source=int(pred.get('source_stmt')); expected=[source,sink] if int(pred.get('source_stmt',-1)) != source: return False,'wrong_source' if not got: @@ -150,12 +168,13 @@ def evaluate(cases:list[dict[str,Any]], labels:list[dict[str,Any]], preds:list[d adv_pk={k for k in pk if k in adv_lk} adv_f1=f1(len(adv_lk & adv_pk),len(adv_pk-adv_lk),len(adv_lk-adv_pk)) ids=stmt_ids_by_case(cases) + stmts_by_case={str(c.get('case_id')):c.get('stmts',[]) for c in cases} trace_req=0; trace_ok=0; trace_invalid=0; reasons={} for k in tp: expected=numeric_list(label_by[k].get('trace',[])) if expected: trace_req+=1 - good, reason=trace_valid(pred_by[k], label_by[k], ids.get(k[0],set())) + good, reason=trace_valid(pred_by[k], label_by[k], ids.get(k[0],set()), release_witnesses(label_by[k], stmts_by_case.get(k[0],[]))) if good: trace_ok+=1 else: diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/task.toml b/edgebench/tasks/juliet_vulnerability_analyzer/task.toml index 567b56b..0d5a15d 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/task.toml +++ b/edgebench/tasks/juliet_vulnerability_analyzer/task.toml @@ -4,7 +4,7 @@ name = "Juliet Vulnerability Analyzer" category = "Systems & Software Engineering" language = "Python" description = "Implement a deterministic static analyzer that processes structured program facts to detect vulnerabilities across six CWE categories (stack/heap overflow, integer overflow, null dereference, use-after-free, command injection)." -version = "1.1.0" +version = "1.2.0" status = "migrated" [environment]