diff --git a/archive.py b/archive.py index ce3cb2a..50c8734 100644 --- a/archive.py +++ b/archive.py @@ -691,8 +691,14 @@ def git_coverage_preconditions(coverage, repo_root, asset_subtree, now=None): ctx = {} now = time.time() if now is None else now - # 1a. Repo present. + # 1a. git present + repo present. rc 127 from _git means the git binary is + # not on PATH — a common --globus compute-node gap (like hsi), reported + # distinctly so a disabled-exclusion run is diagnosable rather than being + # mistaken for a missing repo. rc, out, _ = _git(repo_root, "rev-parse", "--show-toplevel") + if rc == 127: + return (False, "git not available on PATH (compute node may need a " + "module load / PATH fix)", ctx) if rc != 0 or not out.strip(): return (False, f"git repo not found at {repo_root}", ctx) toplevel = os.path.normpath(out.strip()) @@ -805,8 +811,13 @@ def git_file_verdict(repo_root, full_path, ctx): require_pushed = ctx.get("require_pushed", True) remote_ref = ctx.get("remote_ref") - # L — tracked in the index. + # L — tracked in the index. git-missing (rc 127) -> ERROR, never a false + # UNTRACKED (which would look like "back up" for the right reason but the + # wrong cause — the operator must see that git itself was unavailable). rc, _, _ = _git(repo_root, "ls-files", "--error-unmatch", "--", rel) + if rc == 127: + ev["reason"] = "git unavailable" + return (COV_ERROR, ev) if rc != 0: return (COV_UNTRACKED, ev) @@ -874,6 +885,204 @@ def git_file_verdict(repo_root, full_path, ctx): return (COV_COVERED, ev) +def companion_guard(member_records): + """ + INV-C smoke alarm (EXCLUSION_SPEC.md §4). `member_records` is a list of + {arcname, tier, excluded(bool)}. Returns a list of violation dicts. A + directory that had matched files must still ship at least one member AND must + never have a DATA-classified member excluded. DATA is never a candidate so a + dropped-DATA violation signals a logic bug; the zero-survivor case catches a + directory whose entire matched set was excluded (a misclassification hint). + """ + import os + by_dir = {} + for r in member_records: + by_dir.setdefault(os.path.dirname(r["arcname"]), []).append(r) + violations = [] + for d, recs in sorted(by_dir.items()): + survivors = [r for r in recs if not r["excluded"]] + dropped_data = [r["arcname"] for r in recs + if r["excluded"] and r["tier"] == EXCL_DATA] + if dropped_data: + violations.append({"directory": d, "reason": "would drop DATA member(s)", + "members": dropped_data}) + elif not survivors: + violations.append({"directory": d, + "reason": "directory would ship zero members", + "members": [r["arcname"] for r in recs]}) + return violations + + +def evaluate_exclusions(matched_members, source_folder, spec, + exclude_optional=(), now=None): + """ + Decide which already-matched members to EXCLUDE, with exhaustive error + handling and a fully serializable diagnostic report. **NEVER raises** — any + failure (malformed spec, git unavailable, fetch failure, a per-file gate + error, or any unexpected exception) degrades to "exclude nothing, back up + everything" with a recorded reason. So exclusion can never fail an archive, + and — crucially for --globus, where this runs on a compute node — the report + is plain dicts/lists/strings that travel back to the launcher intact for the + startup banner + completion email. + + matched_members: iterable of (full_path, arcname) already matched by + file_pattern (the first filter). `spec` is the RAW dict + (recompiled here, so it is safe to ship across the Compute + boundary). Returns a report dict: + + status: "OFF" — no spec configured (feature not requested) + "DISABLED" — spec present but exclusion could not run safely + (disabled_reason set) -> everything backed up + "ON" — exclusion ran (some candidates may still be demoted) + disabled_reason, error, spec_sha256, coverage, + excluded[], excluded_arcnames[], demoted[], contested_data[], + companion_violations[], counts{}, excluded_bytes + + On a companion-guard violation, exclusion is suppressed for the whole run + (drop nothing) AND the violation is reported, so the wiring can raise a loud + [EXCLUSION-ABORT] while the data is already safe regardless. + """ + import os + + members = list(matched_members) + report = { + "status": "OFF", "disabled_reason": None, "error": None, + "spec_sha256": None, "coverage": {}, + "excluded": [], "excluded_arcnames": [], "demoted": [], + "contested_data": [], "companion_violations": [], + "counts": {}, "excluded_bytes": 0, + } + if not spec: + return report + + try: + try: + compiled = compile_exclude_spec(spec, exclude_optional) + except ValueError as e: + report["status"] = "DISABLED" + report["disabled_reason"] = f"invalid exclude spec: {e}" + return report + + report["spec_sha256"] = compiled["spec_sha256"] + coverage = compiled["coverage"] + repo_root = coverage.get("repo_root") + if not repo_root: + report["status"] = "DISABLED" + report["disabled_reason"] = "exclude spec coverage has no repo_root" + return report + + try: + ok, reason, ctx = git_coverage_preconditions( + coverage, repo_root, source_folder, now=now) + except Exception as e: + report["status"] = "DISABLED" + report["disabled_reason"] = f"coverage preconditions errored: {e}" + return report + if not ok: + report["status"] = "DISABLED" + report["disabled_reason"] = reason + return report + + report["status"] = "ON" + report["coverage"] = { + "repo_head": ctx.get("repo_head"), + "remote_ref": ctx.get("remote_ref"), + "require_pushed": ctx.get("require_pushed"), + "remote_durable_attestation": ctx.get("remote_durable_attestation"), + "fetch_ts": ctx.get("fetch_ts"), + } + + counts = {k: 0 for k in ( + "data_allowlist", "data_default", "exclude_excluded", + "exclude_demoted", "optional_shipped", "optional_dropped_excluded", + "optional_dropped_demoted")} + # Sub-tier rules, for flagging a DATA file that ALSO matched one (proof + # that precedence — not luck — kept it; EXCLUSION_SPEC.md §6). + sub_rules = list(compiled["exclude_identity"]) + \ + [(rid, rx, note) for (rid, rx, _d, note) in compiled["optional_identity"]] + + member_records = [] + for full_path, arcname in members: + cls = classify_tier(arcname, compiled) + rec = {"arcname": arcname, "tier": cls["tier"], "excluded": False} + member_records.append(rec) + + if cls["tier"] == EXCL_DATA: + if cls["data_kind"] == "DATA-allowlist": + counts["data_allowlist"] += 1 + hit = next((rid for (rid, rx, _n) in sub_rules + if rx.search(arcname)), None) + if hit: + report["contested_data"].append( + {"arcname": arcname, "rule_id": cls["rule_id"], + "also_matched": hit}) + else: + counts["data_default"] += 1 + continue + + if not cls["candidate"]: + counts["optional_shipped"] += 1 # OPTIONAL, default backup + continue + + # Candidate (EXCLUDE, or OPTIONAL opted-out): demand a COVERED proof. + try: + verdict, ev = git_file_verdict(repo_root, full_path, ctx) + except Exception as e: + verdict, ev = COV_ERROR, {"reason": f"{type(e).__name__}: {e}"} + + is_excl = cls["tier"] == EXCL_EXCLUDE + if verdict == COV_COVERED: + rec["excluded"] = True + report["excluded"].append( + {"arcname": arcname, "rule_id": cls["rule_id"], + "verdict": verdict, "blob_oid": ev.get("blob_oid"), + "remote_ref": ev.get("remote_ref")}) + counts["exclude_excluded" if is_excl + else "optional_dropped_excluded"] += 1 + try: + report["excluded_bytes"] += os.path.getsize(full_path) + except OSError: + pass + else: + report["demoted"].append( + {"arcname": arcname, "rule_id": cls["rule_id"], + "verdict": verdict, "reason": ev.get("reason", "")}) + counts["exclude_demoted" if is_excl + else "optional_dropped_demoted"] += 1 + + report["counts"] = counts + + # INV-C companion guard. On any violation: default-safe — suppress ALL + # exclusion this run (drop nothing) and report it loudly so the wiring + # can raise [EXCLUSION-ABORT]; the data is already safe either way. + violations = companion_guard(member_records) + if violations: + report["companion_violations"] = violations + report["status"] = "DISABLED" + report["disabled_reason"] = ( + "companion guard tripped (a directory would ship zero members or " + "drop a DATA member) — exclusion suppressed, everything backed " + "up; investigate classification") + report["excluded"] = [] + report["excluded_arcnames"] = [] + report["excluded_bytes"] = 0 + return report + + report["excluded_arcnames"] = [e["arcname"] for e in report["excluded"]] + return report + + except Exception as e: + # Absolute backstop: exclusion can NEVER fail an archive. Anything that + # slipped past the targeted handlers above lands here -> back up all. + report["status"] = "DISABLED" + report["error"] = f"{type(e).__name__}: {e}" + report["disabled_reason"] = f"unexpected exclusion error: {e}" + report["excluded"] = [] + report["excluded_arcnames"] = [] + report["excluded_bytes"] = 0 + return report + + def make_zip_files(source_folder, file_pattern, tmp_dir, project_name, compression="deflate", allow_empty_files=False): """ diff --git a/tests/test_exclusion_primitives.py b/tests/test_exclusion_primitives.py index 139555c..9d6db16 100644 --- a/tests/test_exclusion_primitives.py +++ b/tests/test_exclusion_primitives.py @@ -428,5 +428,211 @@ def test_preconditions_dirty_subtree_disables(self): self.assertIn("uncommitted", reason) +class TestEvaluateExclusionsErrors(unittest.TestCase): + """The never-raises orchestrator's failure modes (no repo needed). Every one + must degrade to 'back up everything' with a clear, serializable reason — the + contract that keeps exclusion from ever failing an archive (esp. --globus, + where this runs remotely and the reason must travel back).""" + + def _spec(self, **over): + s = json.loads(json.dumps(SPEC)) # deep copy + s["coverage"].update(over) + return s + + def test_off_when_no_spec(self): + r = archive.evaluate_exclusions([("/x/a.json", "a.json")], "/x", None) + self.assertEqual(r["status"], "OFF") + self.assertEqual(r["excluded_arcnames"], []) + + def test_disabled_on_malformed_spec(self): + bad = {"coverage": {"repo_root": "/x", "remote_durable_attestation": True}, + "exclude_identity": [{"id": "x", "kind": "regex", "pattern": r"\.json$"}]} + r = archive.evaluate_exclusions([("/x/a.json", "a.json")], "/x", bad) + self.assertEqual(r["status"], "DISABLED") + self.assertIn("invalid exclude spec", r["disabled_reason"]) + self.assertEqual(r["excluded_arcnames"], []) + + def test_disabled_without_repo_root(self): + spec = json.loads(json.dumps(SPEC)) + spec["coverage"].pop("repo_root", None) + r = archive.evaluate_exclusions([("/x/a.json", "a.json")], "/x", spec) + self.assertEqual(r["status"], "DISABLED") + self.assertIn("repo_root", r["disabled_reason"]) + + def test_disabled_when_repo_missing(self): + spec = self._spec(repo_root="/no/such/repo/here") + r = archive.evaluate_exclusions([("/no/such/repo/here/a.json", "a.json")], + "/no/such/repo/here", spec) + self.assertEqual(r["status"], "DISABLED") + + def test_git_missing_is_reported_distinctly(self): + # Simulate git absent on a --globus compute node: every _git call -> 127. + orig = archive._git + archive._git = lambda *a, **k: (127, "", "git: not found") + try: + spec = self._spec(repo_root="/tmp") + r = archive.evaluate_exclusions([("/tmp/a.json", "a.json")], "/tmp", spec) + finally: + archive._git = orig + self.assertEqual(r["status"], "DISABLED") + self.assertIn("git not available", r["disabled_reason"]) + self.assertEqual(r["excluded_arcnames"], []) + + def test_never_raises_on_unexpected_internal_error(self): + # Force an exception deep in the walk; the catch-all backstop must turn it + # into DISABLED + back-up-everything, never propagate. + orig = archive.classify_tier + + def boom(*a, **k): + raise RuntimeError("synthetic explosion") + archive.classify_tier = boom + try: + spec = self._spec(repo_root="/tmp") + # preconditions must pass to reach the walk, so stub them too. + orig_pc = archive.git_coverage_preconditions + archive.git_coverage_preconditions = lambda *a, **k: ( + True, "ok", {"require_pushed": True, "remote_ref": "origin/main", + "repo_head": "abc"}) + try: + r = archive.evaluate_exclusions( + [("/tmp/X_status.json", "X_status.json")], "/tmp", spec) + finally: + archive.git_coverage_preconditions = orig_pc + finally: + archive.classify_tier = orig + self.assertEqual(r["status"], "DISABLED") + self.assertIsNotNone(r["error"]) + self.assertIn("synthetic explosion", r["error"]) + self.assertEqual(r["excluded_arcnames"], []) + + +@unittest.skipUnless(_have_git(), "git not available") +class TestEvaluateExclusionsLive(unittest.TestCase): + """The orchestrator end-to-end against a real repo + bare remote.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.remote = os.path.join(self.tmp, "remote.git") + self.work = os.path.join(self.tmp, "work") + subprocess.run(["git", "init", "--bare", "-b", "main", self.remote], + capture_output=True, check=True) + subprocess.run(["git", "init", "-b", "main", self.work], + capture_output=True, check=True) + for k, v in [("user.email", "t@t"), ("user.name", "t"), + ("core.autocrlf", "false"), ("commit.gpgsign", "false")]: + _git(self.work, "config", k, v) + _git(self.work, "remote", "add", "origin", self.remote) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write(self, rel, content="x\n"): + p = os.path.join(self.work, rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as fh: + fh.write(content) + return p + + def _spec(self, **cov): + s = json.loads(json.dumps(SPEC)) + s["coverage"]["repo_root"] = self.work + s["coverage"].update(cov) + return s + + def _members(self, *rels): + return [(os.path.join(self.work, r), r) for r in rels] + + def test_excludes_covered_demotes_stale_keeps_data(self): + # All files committed (so the subtree is CLEAN and run-level + # preconditions pass), but one excludable file is committed AFTER the + # push -> not on the remote -> STALE -> demoted to backup. The realistic + # per-file demotion: a clean tree can still hold an unpushed blob. + self._write("E/E_status.json") # EXCLUDE, pushed -> covered + self._write("E/scan.tiff") # DATA, kept + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "c1") + _git(self.work, "push", "-u", "origin", "main") + self._write("E/E_parse-warnings.json") # EXCLUDE, committed but NOT pushed + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "c2") + + r = archive.evaluate_exclusions( + self._members("E/E_status.json", "E/E_parse-warnings.json", "E/scan.tiff"), + self.work, self._spec()) + + self.assertEqual(r["status"], "ON", r.get("disabled_reason")) + self.assertEqual(r["excluded_arcnames"], ["E/E_status.json"]) + demoted = {d["arcname"]: d["verdict"] for d in r["demoted"]} + self.assertEqual(demoted, {"E/E_parse-warnings.json": archive.COV_STALE}) + self.assertEqual(r["counts"]["data_allowlist"], 1) # scan.tiff (imagery) + self.assertGreater(r["excluded_bytes"], 0) + + def test_dirty_subtree_disables_whole_run(self): + # Run-level precondition: ANY uncommitted change in the asset subtree + # disables exclusion for the whole run (back up everything) — exclusion + # never operates over a tree it can't fully prove. + self._write("E/E_status.json") + self._write("E/scan.tiff") + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "c") + _git(self.work, "push", "-u", "origin", "main") + self._write("E/loose.txt", "uncommitted\n") # dirties the subtree + + r = archive.evaluate_exclusions( + self._members("E/E_status.json", "E/scan.tiff"), self.work, self._spec()) + self.assertEqual(r["status"], "DISABLED") + self.assertIn("uncommitted", r["disabled_reason"]) + self.assertEqual(r["excluded_arcnames"], []) + + def test_contested_data_is_logged(self): + # A DATA rule and an EXCLUDE rule both match special_status.json; DATA + # precedence wins, and the contest is logged as proof (not luck). + spec = self._spec() + spec["data_identity"].insert( + 0, {"id": "special", "kind": "regex", + "pattern": r"(^|/)special_[^/]*\.json$", "note": "special data"}) + self._write("D/special_status.json") # matches special (DATA) AND status + self._write("D/keep.tiff") # ensure dir has a survivor + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "c") + _git(self.work, "push", "-u", "origin", "main") + + r = archive.evaluate_exclusions( + self._members("D/special_status.json", "D/keep.tiff"), self.work, spec) + self.assertEqual(r["status"], "ON", r.get("disabled_reason")) + self.assertEqual(r["excluded_arcnames"], []) # DATA never excluded + self.assertEqual(len(r["contested_data"]), 1) + self.assertEqual(r["contested_data"][0]["arcname"], "D/special_status.json") + self.assertEqual(r["contested_data"][0]["rule_id"], "special") + + def test_companion_guard_suppresses_all_exclusion(self): + # A directory whose ENTIRE matched set is covered metadata would ship + # zero members -> companion guard suppresses exclusion run-wide. + self._write("M/M_status.json") + self._write("M/catalog.json") + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "c") + _git(self.work, "push", "-u", "origin", "main") + + r = archive.evaluate_exclusions( + self._members("M/M_status.json", "M/catalog.json"), self.work, self._spec()) + self.assertEqual(r["status"], "DISABLED") + self.assertTrue(r["companion_violations"]) + self.assertIn("zero members", r["companion_violations"][0]["reason"]) + self.assertEqual(r["excluded_arcnames"], []) # nothing dropped — safe + + def test_serializable_report(self): + # The report must JSON-serialize cleanly (it crosses the Compute boundary + # back to the launcher under --globus). + self._write("E/E_status.json") + self._write("E/keep.tiff") + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "c") + _git(self.work, "push", "-u", "origin", "main") + r = archive.evaluate_exclusions( + self._members("E/E_status.json", "E/keep.tiff"), self.work, self._spec()) + json.dumps(r) # must not raise + + if __name__ == "__main__": unittest.main()