juliet_vulnerability_analyzer: accept the release as a CWE-416 source

edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md

@@ -11,6 +11,16 @@ raw score.
 
 ## Changes
 
+### 1.2.0
+
+The README says `source_stmt` identifies the "source/allocation/release witness", but the CWE-416
+labels use the allocation, so a finding that starts at the release kept its detection credit and
+lost its trace credit. For CWE-416 only, `trace_valid` now also accepts a `free` or `delete` of a
+value the sink uses, in the sink's function, between the labelled allocation and the sink. Each of
+the 120 hidden labels has exactly one such statement. Nothing that was accepted is now rejected, so
+scores only rise, and only for submissions that use the release as source. Regression summary
+describes 1.0.0. Issue: https://github.com/ByteDance-Seed/EdgeBench/issues/50
+
 ### 1.1.0
 
 `infer` is no longer a banned word. The check also matched the English word, so one comment using
@@ -23,7 +33,7 @@ and only for submissions containing the word. Regression summary describes 1.0.0
 - Workspace (`juliet-static-analyzer-public-v3`: starter analyzer, public facts and labels,
   scorer, tools) and the hidden evaluator (`juliet-static-analyzer-evaluator-v3`, with the hidden
   facts and labels): the task authors' repository (not public), identical by sha256 and mode to the
-  published v1 images, except `scoring/score.py` in both, changed in 1.1.0.
+  published v1 images, except `scoring/score.py` in both, changed in 1.1.0 and 1.2.0.
 - `verify_task.sh`: the version shipped in the published v1 images, which dropped the Chinese
   README (also absent here).
 - `judge/eval_hidden.py` (v1's inline eval program, one statement per line), `judge/run.sh` and

edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py

@@ -114,11 +114,29 @@ def stmt_ids_by_case(cases:list[dict[str,Any]]) -> dict[str,set[int]]:
         out[str(c.get('case_id'))]=ids
     return out
 
-def trace_valid(pred:dict[str,Any], label:dict[str,Any], stmt_ids:set[int]) -> tuple[bool,str]:
+def release_witnesses(label:dict[str,Any], stmts:list[dict[str,Any]]) -> set[int]:
+    # CWE-416: the README allows a release witness as source_stmt. Accept a free/delete of a value
+    # the sink uses, in the sink's function, between the labelled allocation and the sink.
+    if str(label.get('cwe'))!='CWE-416': return set()
+    by_id={s.get('stmt_id'):s for s in stmts}
+    sink=int(label.get('sink_stmt')); src=int(label.get('source_stmt',sink)); s_sink=by_id.get(sink) or {}
+    used=set(s_sink.get('uses') or [])
+    out=set()
+    for s in stmts:
+        sid=s.get('stmt_id')
+        if not isinstance(sid,int) or not src<sid<sink or s.get('function')!=s_sink.get('function'): continue
+        call=s.get('call') or {}
+        if call.get('callee')=='free' or (s.get('tokens') or [''])[0]=='delete':
+            if used & set(s.get('uses') or []): out.add(sid)
+    return out
+
+def trace_valid(pred:dict[str,Any], label:dict[str,Any], stmt_ids:set[int], release:set[int]=frozenset()) -> tuple[bool,str]:
     expected=numeric_list(label.get('trace',[]))
     source=int(label.get('source_stmt', expected[0] if expected else label.get('sink_stmt')))
     sink=int(label.get('sink_stmt'))
     got=pred.get('trace',[])
+    if int(pred.get('source_stmt',-1)) in release:
+        source=int(pred.get('source_stmt')); expected=[source,sink]
     if int(pred.get('source_stmt',-1)) != source:
         return False,'wrong_source'
     if not got:
@@ -150,12 +168,13 @@ def evaluate(cases:list[dict[str,Any]], labels:list[dict[str,Any]], preds:list[d
     adv_pk={k for k in pk if k in adv_lk}
     adv_f1=f1(len(adv_lk & adv_pk),len(adv_pk-adv_lk),len(adv_lk-adv_pk))
     ids=stmt_ids_by_case(cases)
+    stmts_by_case={str(c.get('case_id')):c.get('stmts',[]) for c in cases}
     trace_req=0; trace_ok=0; trace_invalid=0; reasons={}
     for k in tp:
         expected=numeric_list(label_by[k].get('trace',[]))
         if expected:
             trace_req+=1
-            good, reason=trace_valid(pred_by[k], label_by[k], ids.get(k[0],set()))
+            good, reason=trace_valid(pred_by[k], label_by[k], ids.get(k[0],set()), release_witnesses(label_by[k], stmts_by_case.get(k[0],[])))
             if good:
                 trace_ok+=1
             else:

edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py

@@ -114,11 +114,29 @@ def stmt_ids_by_case(cases:list[dict[str,Any]]) -> dict[str,set[int]]:
         out[str(c.get('case_id'))]=ids
     return out
 
-def trace_valid(pred:dict[str,Any], label:dict[str,Any], stmt_ids:set[int]) -> tuple[bool,str]:
+def release_witnesses(label:dict[str,Any], stmts:list[dict[str,Any]]) -> set[int]:
+    # CWE-416: the README allows a release witness as source_stmt. Accept a free/delete of a value
+    # the sink uses, in the sink's function, between the labelled allocation and the sink.
+    if str(label.get('cwe'))!='CWE-416': return set()
+    by_id={s.get('stmt_id'):s for s in stmts}
+    sink=int(label.get('sink_stmt')); src=int(label.get('source_stmt',sink)); s_sink=by_id.get(sink) or {}
+    used=set(s_sink.get('uses') or [])
+    out=set()
+    for s in stmts:
+        sid=s.get('stmt_id')
+        if not isinstance(sid,int) or not src<sid<sink or s.get('function')!=s_sink.get('function'): continue
+        call=s.get('call') or {}
+        if call.get('callee')=='free' or (s.get('tokens') or [''])[0]=='delete':
+            if used & set(s.get('uses') or []): out.add(sid)
+    return out
+
+def trace_valid(pred:dict[str,Any], label:dict[str,Any], stmt_ids:set[int], release:set[int]=frozenset()) -> tuple[bool,str]:
     expected=numeric_list(label.get('trace',[]))
     source=int(label.get('source_stmt', expected[0] if expected else label.get('sink_stmt')))
     sink=int(label.get('sink_stmt'))
     got=pred.get('trace',[])
+    if int(pred.get('source_stmt',-1)) in release:
+        source=int(pred.get('source_stmt')); expected=[source,sink]
     if int(pred.get('source_stmt',-1)) != source:
         return False,'wrong_source'
     if not got:
@@ -150,12 +168,13 @@ def evaluate(cases:list[dict[str,Any]], labels:list[dict[str,Any]], preds:list[d
     adv_pk={k for k in pk if k in adv_lk}
     adv_f1=f1(len(adv_lk & adv_pk),len(adv_pk-adv_lk),len(adv_lk-adv_pk))
     ids=stmt_ids_by_case(cases)
+    stmts_by_case={str(c.get('case_id')):c.get('stmts',[]) for c in cases}
     trace_req=0; trace_ok=0; trace_invalid=0; reasons={}
     for k in tp:
         expected=numeric_list(label_by[k].get('trace',[]))
         if expected:
             trace_req+=1
-            good, reason=trace_valid(pred_by[k], label_by[k], ids.get(k[0],set()))
+            good, reason=trace_valid(pred_by[k], label_by[k], ids.get(k[0],set()), release_witnesses(label_by[k], stmts_by_case.get(k[0],[])))
             if good:
                 trace_ok+=1
             else:

edgebench/tasks/juliet_vulnerability_analyzer/task.toml

@@ -4,7 +4,7 @@ name = "Juliet Vulnerability Analyzer"
 category = "Systems & Software Engineering"
 language = "Python"
 description = "Implement a deterministic static analyzer that processes structured program facts to detect vulnerabilities across six CWE categories (stack/heap overflow, integer overflow, null dereference, use-after-free, command injection)."
-version = "1.1.0"
+version = "1.2.0"
 status = "migrated"
 
 [environment]