Sponsored Content
Skip to content

Commit 1735134

Browse files
Keniel MaldonadoKeniel Maldonado
Keniel Maldonado
authored and
Keniel Maldonado
committed
Add bounded carve-out guard for supersession spans
1 parent 78d66a3 commit 1735134

5 files changed

Lines changed: 326 additions & 0 deletions

‎agents/semantic_confirmer.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,18 @@
3333
"only",
3434
"now",
3535
)
36+
UNIVERSAL_SCOPE_PATTERNS = (
37+
r"\ball customers\b",
38+
r"\bevery customer\b",
39+
r"\bevery region\b",
40+
r"\ball regions\b",
41+
r"\bglobal\b",
42+
r"\bglobally\b",
43+
)
44+
BOUNDED_CUSTOMER_SCOPE_PATTERN = re.compile(
45+
r"\bfor\s+(?!all\b|every\b|any\b)([a-z][a-z -]{1,40}?)\s+customers\b",
46+
flags=re.IGNORECASE,
47+
)
3648

3749

3850
def _norm(value: str) -> str:
@@ -74,6 +86,27 @@ def _operators_in(text: str) -> list[str]:
7486
return [op for op in RELATION_SPAN_LEXICON if _operator_pattern(op).search(text)]
7587

7688

89+
def _has_universal_scope(text: str) -> bool:
90+
normalized = _norm(text)
91+
return any(re.search(pattern, normalized) for pattern in UNIVERSAL_SCOPE_PATTERNS)
92+
93+
94+
def _has_bounded_customer_scope(text: str) -> bool:
95+
return bool(BOUNDED_CUSTOMER_SCOPE_PATTERN.search(_norm(text)))
96+
97+
98+
def _scope_carveout_conflict(relation_type: str, cited_evidence_span: str, target_text: str) -> str | None:
99+
if relation_type != "supersedes":
100+
return None
101+
if not _has_bounded_customer_scope(cited_evidence_span):
102+
return None
103+
if _has_universal_scope(cited_evidence_span):
104+
return None
105+
if _has_universal_scope(target_text):
106+
return "bounded supersession span cannot retire broader target scope"
107+
return None
108+
109+
77110
def relation_span_check(cited_evidence_span: str, scope_terms: list[str]) -> dict:
78111
"""v2 relation-span clause.
79112
@@ -152,6 +185,9 @@ def confirm_authority_change(proposal: dict, items: list[dict], *, require_relat
152185
reasons.append("missing or empty evidence citation")
153186
if not _scope_overlaps(source["text"], target["text"], scope_terms):
154187
reasons.append("no deterministic scope overlap between source and target")
188+
carveout_reason = _scope_carveout_conflict(relation_type, cited_evidence_span, target["text"])
189+
if carveout_reason:
190+
reasons.append(carveout_reason)
155191
relation_span = None
156192
if require_relation_span:
157193
relation_span = relation_span_check(cited_evidence_span, scope_terms)

‎path_a_eval_artifacts/path_a_v3_carveout_c1_regate_20260713T034628Z.json‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{
2+
"run_id": "20260713T034628Z",
3+
"generated_at_utc": "2026-07-13T03:46:28.559149+00:00",
4+
"fixture": "tests/fixtures/path_a_authority_change_v3_carveout_additions_2026_07_12.json",
5+
"preregistration_addendum": "PATH_A_V3_PREREGISTRATION_ADDENDUM_CARVEOUT_2026-07-12.md",
6+
"prereg_freeze_commit": "b1d44f9",
7+
"fixture_commit": "48507de",
8+
"c0_baseline_commit": "78d66a3",
9+
"gate_under_test": "patched confirmer with bounded customer carve-out clause",
10+
"boundary": "C1 only: the same two frozen c0_frozen_proposal objects are run after the carve-out defense. No fixture or prediction was changed after observing C0.",
11+
"summary": {
12+
"cases": 2,
13+
"confirmed_after_patch": 1,
14+
"expectations_held": 2,
15+
"bounded_negative_blocked": true,
16+
"universal_positive_admitted": true
17+
},
18+
"cases": [
19+
{
20+
"case_id": "path_a_v3_carveout_export_approval_eu",
21+
"class": "partial_supersession_negative",
22+
"prediction_id": "C1-1",
23+
"label": "bounded negative blocked by carve-out clause",
24+
"expected_confirmed_after_patch": false,
25+
"confirmed_after_patch": false,
26+
"expectation_held": true,
27+
"confirmer_result": {
28+
"confirmed": false,
29+
"needs_human_judgment": true,
30+
"reasons": [
31+
"bounded supersession span cannot retire broader target scope"
32+
],
33+
"finding": null,
34+
"proposal": {
35+
"type": "supersedes",
36+
"source_item_id": "M002",
37+
"target_item_id": "M001",
38+
"cited_evidence_span": "the export approval rule is replaced for EU customers; EU customer data exports now use automated compliance review",
39+
"scope_terms": [
40+
"customer data exports",
41+
"approval"
42+
],
43+
"confidence": 0.76
44+
}
45+
}
46+
},
47+
{
48+
"case_id": "path_a_v3_universal_approval_all_regions",
49+
"class": "universal_scope_positive_control",
50+
"prediction_id": "C1-2",
51+
"label": "universal positive survives carve-out clause",
52+
"expected_confirmed_after_patch": true,
53+
"confirmed_after_patch": true,
54+
"expectation_held": true,
55+
"confirmer_result": {
56+
"confirmed": true,
57+
"needs_human_judgment": false,
58+
"reasons": [],
59+
"finding": {
60+
"type": "semantic_authority_change",
61+
"relation_type": "supersedes",
62+
"source_item_id": "M002",
63+
"target_item_id": "M001",
64+
"cited_evidence_span": "the new approval rule applies to all customers in every region and replaces the old approval rule; automated policy review now governs every customer account activation",
65+
"evidence": "Global update: the new approval rule applies to all customers in every region and replaces the old approval rule; automated policy review now governs every customer account activation.",
66+
"target": "The old approval rule requires manual review before any customer account can be activated in any region.",
67+
"scope_terms": [
68+
"customer account activation",
69+
"approval rule"
70+
],
71+
"confidence": 0.76,
72+
"relation_span": {
73+
"passed": true,
74+
"operators": [
75+
"replaces",
76+
"now"
77+
],
78+
"matching_sentence": "the new approval rule applies to all customers in every region and replaces the old approval rule; automated policy review now governs every customer account activation",
79+
"scope_terms_matched": [
80+
"customer account activation",
81+
"approval rule"
82+
],
83+
"reason": null
84+
}
85+
},
86+
"proposal": {
87+
"type": "supersedes",
88+
"source_item_id": "M002",
89+
"target_item_id": "M001",
90+
"cited_evidence_span": "the new approval rule applies to all customers in every region and replaces the old approval rule; automated policy review now governs every customer account activation",
91+
"scope_terms": [
92+
"customer account activation",
93+
"approval rule"
94+
],
95+
"confidence": 0.76
96+
}
97+
}
98+
}
99+
]
100+
}

‎path_a_eval_artifacts/path_a_v3_carveout_c1_regate_20260713T034628Z.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Path A v3 Carve-Out C1 Re-Gate
2+
3+
Run ID: `20260713T034628Z`
4+
Pre-registration addendum: `PATH_A_V3_PREREGISTRATION_ADDENDUM_CARVEOUT_2026-07-12.md` (freeze `b1d44f9`)
5+
Fixture additions: `tests/fixtures/path_a_authority_change_v3_carveout_additions_2026_07_12.json` (commit `48507de`)
6+
C0 baseline commit: `78d66a3`
7+
8+
The same two frozen `c0_frozen_proposal` objects were run after the carve-out defense. The fixture and predictions were not changed after C0.
9+
10+
| Case | Class | Expected after patch | Confirmed after patch | Held |
11+
| --- | --- | --- | --- | --- |
12+
| `path_a_v3_carveout_export_approval_eu` | partial_supersession_negative | BLOCK | BLOCKED | yes |
13+
| `path_a_v3_universal_approval_all_regions` | universal_scope_positive_control | CONFIRM | CONFIRMED | yes |
14+
15+
## Separate C1 Results
16+
17+
- EU-only bounded negative blocked: `yes`
18+
- Universal all-regions positive admitted: `yes`
19+
20+
## Boundary
21+
22+
C1 only: the same two frozen c0_frozen_proposal objects are run after the carve-out defense. No fixture or prediction was changed after observing C0.

‎path_a_v3_carveout_c1_regate.py‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
from __future__ import annotations
2+
3+
import argparse
4+
import json
5+
from datetime import datetime, timezone
6+
from pathlib import Path
7+
8+
from agents.memory_extractor import extract_memories
9+
from agents.semantic_confirmer import confirm_authority_change
10+
11+
DEFAULT_FIXTURE = Path("tests/fixtures/path_a_authority_change_v3_carveout_additions_2026_07_12.json")
12+
DEFAULT_OUTPUT_DIR = Path("path_a_eval_artifacts")
13+
14+
PREDICTIONS = {
15+
"path_a_v3_carveout_export_approval_eu": {
16+
"prediction_id": "C1-1",
17+
"expected_confirmed": False,
18+
"label": "bounded negative blocked by carve-out clause",
19+
},
20+
"path_a_v3_universal_approval_all_regions": {
21+
"prediction_id": "C1-2",
22+
"expected_confirmed": True,
23+
"label": "universal positive survives carve-out clause",
24+
},
25+
}
26+
27+
28+
def _utc_slug() -> str:
29+
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
30+
31+
32+
def main() -> int:
33+
parser = argparse.ArgumentParser()
34+
parser.add_argument("--fixture", default=str(DEFAULT_FIXTURE))
35+
parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR))
36+
args = parser.parse_args()
37+
38+
fixture = json.loads(Path(args.fixture).read_text())
39+
cases = fixture["cases"]
40+
if {case["id"] for case in cases} != set(PREDICTIONS):
41+
raise SystemExit("fixture cases do not match frozen carve-out C1 predictions")
42+
43+
rows = []
44+
for case in cases:
45+
items = [item.to_dict() for item in extract_memories(case["text"])]
46+
result = confirm_authority_change(case["c0_frozen_proposal"], items, require_relation_span=True)
47+
prediction = PREDICTIONS[case["id"]]
48+
confirmed = result["confirmed"]
49+
rows.append({
50+
"case_id": case["id"],
51+
"class": case["class"],
52+
"prediction_id": prediction["prediction_id"],
53+
"label": prediction["label"],
54+
"expected_confirmed_after_patch": prediction["expected_confirmed"],
55+
"confirmed_after_patch": confirmed,
56+
"expectation_held": confirmed == prediction["expected_confirmed"],
57+
"confirmer_result": result,
58+
})
59+
60+
run_id = _utc_slug()
61+
artifact = {
62+
"run_id": run_id,
63+
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
64+
"fixture": args.fixture,
65+
"preregistration_addendum": "PATH_A_V3_PREREGISTRATION_ADDENDUM_CARVEOUT_2026-07-12.md",
66+
"prereg_freeze_commit": "b1d44f9",
67+
"fixture_commit": "48507de",
68+
"c0_baseline_commit": "78d66a3",
69+
"gate_under_test": "patched confirmer with bounded customer carve-out clause",
70+
"boundary": (
71+
"C1 only: the same two frozen c0_frozen_proposal objects are run after the "
72+
"carve-out defense. No fixture or prediction was changed after observing C0."
73+
),
74+
"summary": {
75+
"cases": len(rows),
76+
"confirmed_after_patch": sum(int(row["confirmed_after_patch"]) for row in rows),
77+
"expectations_held": sum(int(row["expectation_held"]) for row in rows),
78+
"bounded_negative_blocked": not next(
79+
row["confirmed_after_patch"]
80+
for row in rows
81+
if row["case_id"] == "path_a_v3_carveout_export_approval_eu"
82+
),
83+
"universal_positive_admitted": next(
84+
row["confirmed_after_patch"]
85+
for row in rows
86+
if row["case_id"] == "path_a_v3_universal_approval_all_regions"
87+
),
88+
},
89+
"cases": rows,
90+
}
91+
92+
output_dir = Path(args.output_dir)
93+
output_dir.mkdir(parents=True, exist_ok=True)
94+
json_path = output_dir / f"path_a_v3_carveout_c1_regate_{run_id}.json"
95+
md_path = output_dir / f"path_a_v3_carveout_c1_regate_{run_id}.md"
96+
json_path.write_text(json.dumps(artifact, indent=2), encoding="utf-8")
97+
98+
lines = [
99+
"# Path A v3 Carve-Out C1 Re-Gate",
100+
"",
101+
f"Run ID: `{run_id}`",
102+
f"Pre-registration addendum: `PATH_A_V3_PREREGISTRATION_ADDENDUM_CARVEOUT_2026-07-12.md` (freeze `b1d44f9`)",
103+
f"Fixture additions: `{args.fixture}` (commit `48507de`)",
104+
"C0 baseline commit: `78d66a3`",
105+
"",
106+
"The same two frozen `c0_frozen_proposal` objects were run after the carve-out defense. "
107+
"The fixture and predictions were not changed after C0.",
108+
"",
109+
"| Case | Class | Expected after patch | Confirmed after patch | Held |",
110+
"| --- | --- | --- | --- | --- |",
111+
]
112+
for row in rows:
113+
lines.append(
114+
f"| `{row['case_id']}` | {row['class']} | "
115+
f"{'CONFIRM' if row['expected_confirmed_after_patch'] else 'BLOCK'} | "
116+
f"{'CONFIRMED' if row['confirmed_after_patch'] else 'BLOCKED'} | "
117+
f"{'yes' if row['expectation_held'] else 'NO'} |"
118+
)
119+
lines.extend([
120+
"",
121+
"## Separate C1 Results",
122+
"",
123+
f"- EU-only bounded negative blocked: `{'yes' if artifact['summary']['bounded_negative_blocked'] else 'no'}`",
124+
f"- Universal all-regions positive admitted: `{'yes' if artifact['summary']['universal_positive_admitted'] else 'no'}`",
125+
"",
126+
"## Boundary",
127+
"",
128+
artifact["boundary"],
129+
"",
130+
])
131+
md_path.write_text("\n".join(lines), encoding="utf-8")
132+
133+
print(json.dumps({
134+
"json": str(json_path),
135+
"markdown": str(md_path),
136+
"summary": artifact["summary"],
137+
"cases": [
138+
{k: row[k] for k in ("case_id", "confirmed_after_patch", "expectation_held")}
139+
for row in rows
140+
],
141+
}, indent=2))
142+
return 0
143+
144+
145+
if __name__ == "__main__":
146+
raise SystemExit(main())

‎tests/test_semantic_confirmer.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99

1010
FIXTURE = Path(__file__).parent / "fixtures" / "path_a_authority_change_v0_2026_07_01.json"
11+
CARVEOUT_FIXTURE = Path(__file__).parent / "fixtures" / "path_a_authority_change_v3_carveout_additions_2026_07_12.json"
1112

1213

1314
def _cases() -> list[dict]:
@@ -18,6 +19,10 @@ def _items(case: dict) -> list[dict]:
1819
return [item.to_dict() for item in extract_memories(case["text"])]
1920

2021

22+
def _carveout_cases() -> list[dict]:
23+
return json.loads(CARVEOUT_FIXTURE.read_text())["cases"]
24+
25+
2126
def test_frozen_path_a_positive_relations_confirm_against_local_evidence():
2227
positives = [case for case in _cases() if case["expected_relations"]]
2328

@@ -170,3 +175,20 @@ def test_relation_span_operator_matching_uses_word_boundaries():
170175

171176
assert not check["passed"]
172177
assert check["operators"] == []
178+
179+
180+
def test_bounded_customer_carveout_does_not_confirm_global_supersession():
181+
case = next(case for case in _carveout_cases() if case["id"] == "path_a_v3_carveout_export_approval_eu")
182+
183+
result = confirm_authority_change(case["c0_frozen_proposal"], _items(case), require_relation_span=True)
184+
185+
assert not result["confirmed"]
186+
assert "bounded supersession span cannot retire broader target scope" in result["reasons"]
187+
188+
189+
def test_universal_scope_positive_control_survives_carveout_clause():
190+
case = next(case for case in _carveout_cases() if case["id"] == "path_a_v3_universal_approval_all_regions")
191+
192+
result = confirm_authority_change(case["c0_frozen_proposal"], _items(case), require_relation_span=True)
193+
194+
assert result["confirmed"], result["reasons"]

0 commit comments

Comments
 (0)