diff --git a/CLAUDE.md b/CLAUDE.md index 8b39108..79b5908 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -287,8 +287,18 @@ missing `git` on a compute node all degrade to "back up everything" with a recor reason (banner + run-log `exclusion` block + completion email). The **companion guard** (INV-C) suppresses exclusion run-wide if a directory would ship zero members, and the run continues backing up everything with a loud `[EXCLUSION-ABORT]` alert. -Runs under `--globus` too (the spec PATH is read on the compute node). Reclaim must -re-apply the recorded excluded set + re-run the gate (INV-E) — pending. +Runs under `--globus` too (the spec PATH is read on the compute node). + +**Reclaim (INV-E)** re-applies the archive's recorded exclusion at reclaim time: +`reclaim.recheck_exclusions()` subtracts the recorded `excluded_arcnames` from the +live walk **and re-runs the git gate** against the current repo for each one. A +metadata-only change reads `SAFE` (no false DRIFT → the backup skips the re-ship — +this is what actually delivers the byte savings); an excluded file that is no longer +`COVERED` (edited/regenerated since the archive, so its current bytes aren't on tape) +reads `DRIFTED`, never a silent `--delete`. Logs without an `exclusion` block behave +exactly as before. Still pending: the standalone post-hoc re-verifier, and +resume-under-exclusion (the archive currently forces a fresh zip when an +`exclude_spec` is set). --- diff --git a/archive.py b/archive.py index 106130c..33f0b31 100644 --- a/archive.py +++ b/archive.py @@ -1011,11 +1011,20 @@ def evaluate_exclusions(matched_members, source_folder, spec, report["status"] = "ON" report["coverage"] = { + # repo_root + the gate config are recorded so the run log is + # SELF-CONTAINED for reclaim's re-gate (INV-E): reclaim re-runs the + # coverage gate against the current repo for each recorded exclusion, + # using exactly these values — no dependency on the exclude_spec file + # still being present/unchanged at reclaim time. + "repo_root": ctx.get("repo_root"), "repo_head": ctx.get("repo_head"), "remote_ref": ctx.get("remote_ref"), "require_pushed": ctx.get("require_pushed"), "remote_durable_attestation": ctx.get("remote_durable_attestation"), "fetch_ts": ctx.get("fetch_ts"), + "forbid_remote_under": (coverage or {}).get("forbid_remote_under", + ["/depot/"]), + "max_fetch_age": (coverage or {}).get("max_fetch_age", "24h"), } counts = {k: 0 for k in ( diff --git a/reclaim.py b/reclaim.py index 82c2324..c6c4088 100644 --- a/reclaim.py +++ b/reclaim.py @@ -59,6 +59,13 @@ import subprocess import sys +# archive.py ships alongside reclaim.py; reclaim re-uses its git coverage gate to +# re-prove recorded exclusions at reclaim time (INV-E). Insert the script's own +# dir so `import archive` works whether run as ./reclaim.py, python reclaim.py, +# or via a PATH-linked wrapper. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import archive + HSI = "/opt/hsi/bin/hsi" DEFAULT_LOG_DIR = "/depot/florasense/etc/logs_fortress/archive_log" # Durable record of each --delete run, written on Depot so it survives /tmp @@ -206,7 +213,7 @@ def hsi_exists(tar_path): TOP_LEVEL_ONLY_PATTERN = r"^[^/]+$" -def scan_source(folder, pattern=None, cutoff=None, limit=3): +def scan_source(folder, pattern=None, cutoff=None, limit=3, exclude_arcnames=None): """One metadata-only walk of the live source, optionally scoped by the archive's file_pattern (regex against the relative path, exactly like archive.py's walk). Returns (live_count, newer_count, newer_samples, @@ -219,10 +226,18 @@ def scan_source(folder, pattern=None, cutoff=None, limit=3): TOP_LEVEL_ONLY_PATTERN to a non-recursive scan — without that, verifying a root-files-only target walks its entire (possibly multi-TB) tree to match a handful of files. + + exclude_arcnames (INV-E): a set of relpaths that the archive EXCLUDED and + that reclaim has RE-PROVEN still git-COVERED — subtracted from the walk so + live count/bytes/newer line up with the (post-exclusion) archived set. A file + matching file_pattern but NOT in this set still counts (so newly-added or + no-longer-covered metadata surfaces as drift). None/empty = no subtraction + (fully back-compatible with pre-exclusion logs). """ if not os.path.isdir(folder): return None rx = re.compile(pattern) if pattern else None + excluded = set(exclude_arcnames or ()) cut = cutoff.timestamp() if cutoff is not None else None live = 0 newer = 0 @@ -241,6 +256,8 @@ def scan_source(folder, pattern=None, cutoff=None, limit=3): rel = os.path.relpath(p, folder) if rx is not None and not rx.search(rel): continue + if rel in excluded: + continue # re-proven-covered exclusion — not part of the archive live += 1 try: st = os.stat(p) @@ -334,6 +351,76 @@ def resolve_source_folder(log, config): return log.get("source_folder") or (config or {}).get("source_folder") +def recheck_exclusions(log, source_folder): + """ + INV-E: re-prove each exclusion the archive recorded is STILL git-COVERED + against the CURRENT repo. Returns: + + {applies: bool, + still_covered: set[relpath], # subtract these from the live walk + no_longer_covered: [(relpath, verdict)], # edited/regenerated -> DRIFTED + reason: str | None} + + applies=False when the log predates exclusion (no `exclusion` block) or its + status != "ON" — reclaim then behaves exactly as before (fully back-compat). + + FAIL-SAFE: if the exclusions cannot be re-proven now (no repo_root recorded, + git unavailable, preconditions fail), NOTHING is marked still-covered — every + recorded exclusion falls into no_longer_covered, so the target reads as + DRIFTED (never wrongly SAFE / never a --delete of bytes not on tape). Uses + the coverage config recorded in the log, so it needs no access to the + original exclude_spec file. + """ + excl = (log or {}).get("exclusion") or {} + result = {"applies": False, "still_covered": set(), + "no_longer_covered": [], "reason": None} + if excl.get("status") != "ON": + return result + result["applies"] = True + + recorded = list(excl.get("excluded_arcnames") + or [e.get("arcname") for e in excl.get("excluded", [])]) + recorded = [r for r in recorded if r] + if not recorded: + return result # exclusion ran but dropped nothing — nothing to subtract + + cov = excl.get("coverage") or {} + repo_root = cov.get("repo_root") + if not repo_root or not source_folder: + result["no_longer_covered"] = [(r, "NO-REPO-ROOT") for r in recorded] + result["reason"] = "no repo_root recorded — cannot re-prove exclusions" + return result + + coverage_cfg = { + "repo_root": repo_root, + "require_pushed": cov.get("require_pushed", True), + "remote_durable_attestation": cov.get("remote_durable_attestation", True), + "forbid_remote_under": cov.get("forbid_remote_under", ["/depot/"]), + "max_fetch_age": cov.get("max_fetch_age", "24h"), + } + try: + ok, reason, ctx = archive.git_coverage_preconditions( + coverage_cfg, repo_root, source_folder) + except Exception as e: + ok, reason, ctx = False, f"gate error: {type(e).__name__}: {e}", {} + if not ok: + result["no_longer_covered"] = [(r, "GATE-OFF") for r in recorded] + result["reason"] = f"cannot re-prove exclusions ({reason})" + return result + + for rel in recorded: + fp = os.path.join(source_folder, rel) + try: + verdict, _ev = archive.git_file_verdict(repo_root, fp, ctx) + except Exception as e: + verdict = "ERROR" + if verdict == archive.COV_COVERED: + result["still_covered"].add(rel) + else: + result["no_longer_covered"].append((rel, verdict)) + return result + + def verify(log, source_folder, holds, file_pattern=None, canonical_roots=(), allowlist=None, opt_in=False): """ @@ -361,6 +448,13 @@ def verify(log, source_folder, holds, file_pattern=None, canonical_roots=(), run_dt = run_datetime(log.get("run_timestamp")) reasons = [] + # INV-E: re-apply the archive's recorded exclusions to the live walk — but + # ONLY those we can RE-PROVE are still git-COVERED right now. A recorded + # exclusion that is no longer covered (edited/regenerated on disk since the + # archive, so its current bytes are NOT on tape) is deliberately left in the + # walk so it surfaces as drift below — never silently SAFE, never --delete'd. + excl_check = recheck_exclusions(log, source_folder) + held = any((h == project) or (source_folder and h in source_folder) for h in holds) # Fast structural signals @@ -370,8 +464,12 @@ def verify(log, source_folder, holds, file_pattern=None, canonical_roots=(), reasons.append(f"log status = {status!r}") # One pattern-scoped metadata walk gives count, newer-than-run, and bytes - # together (legacy verify did three separate unscoped walks). - scan = scan_source(source_folder, pattern, run_dt) if source_folder else None + # together (legacy verify did three separate unscoped walks). Under INV-E the + # re-proven-covered exclusions are subtracted so live lines up with the + # (post-exclusion) archived set. + scan = scan_source(source_folder, pattern, run_dt, + exclude_arcnames=excl_check["still_covered"] + ) if source_folder else None if source_folder is None: reasons.append("source_folder unknown (not in log or config)") @@ -405,6 +503,15 @@ def verify(log, source_folder, holds, file_pattern=None, canonical_roots=(), elif live is None: verdict = GONE reasons.append("source already removed from Depot") + elif excl_check.get("applies") and excl_check.get("no_longer_covered"): + # INV-E: a file the archive EXCLUDED is no longer git-COVERED — its + # current on-disk bytes were never written to tape. Block reclaim. + verdict = DRIFTED + gone = excl_check["no_longer_covered"] + sample = ", ".join(f"{a} ({v})" for a, v in gone[:3]) + reasons.append(f"{len(gone)} previously-excluded file(s) no longer " + f"git-COVERED (e.g. {sample}) — edited/regenerated since " + f"archive; current bytes are not on tape") elif newer_count > 0: verdict = DRIFTED reasons.append(f"{newer_count} file(s) newer than archive run " diff --git a/tests/test_exclusion_reclaim.py b/tests/test_exclusion_reclaim.py new file mode 100644 index 0000000..ece7b6c --- /dev/null +++ b/tests/test_exclusion_reclaim.py @@ -0,0 +1,214 @@ +""" +Tests for reclaim's INV-E behavior (docs/EXCLUSION_SPEC.md): reclaim must +subtract the archive's recorded exclusions from its live walk AND re-prove each +is still git-COVERED, so a metadata-only change reads SAFE (no false DRIFT, the +waste this feature kills) while an excluded file edited/regenerated since the +archive — whose current bytes are NOT on tape — reads DRIFTED (never a silent +--delete). + +Builds a real throwaway git repo (data + metadata committed + pushed to a local +bare remote), constructs archive logs as archive.py would write them, and drives +reclaim.verify() / reclaim.recheck_exclusions(). hsi is stubbed. All local. + +Run: python3 -m unittest discover -s tests +""" + +import datetime +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import archive +import reclaim + + +def _git(cwd, *args): + r = subprocess.run(["git", "-C", cwd, *args], capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError(f"git {' '.join(args)} failed: {r.stderr}") + return r + + +def _have_git(): + try: + subprocess.run(["git", "--version"], capture_output=True) + return True + except OSError: + return False + + +# A run stamp far enough back that live files (created "now") are NOT counted as +# newer-than-run unless we deliberately age them — isolates the exclusion logic +# from the newer-mtime check except where a test sets mtimes explicitly. +RUN_STAMP = "20260201_000000" # 2026-02-01 +RUN_EPOCH = datetime.datetime(2026, 2, 1).timestamp() +PAST_EPOCH = datetime.datetime(2026, 1, 1).timestamp() # before the run + + +@unittest.skipUnless(_have_git(), "git not available") +class TestReclaimExclusion(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.remote = os.path.join(self.tmp, "remote.git") + self.work = os.path.join(self.tmp, "work") # repo root + self.asset = os.path.join(self.work, "asset") # source_folder + subprocess.run(["git", "init", "--bare", "-b", "main", self.remote], + capture_output=True, check=True) + subprocess.run(["git", "init", "-b", "main", self.work], + capture_output=True, check=True) + for k, v in [("user.email", "t@t"), ("user.name", "t"), + ("core.autocrlf", "false"), ("commit.gpgsign", "false")]: + _git(self.work, "config", k, v) + _git(self.work, "remote", "add", "origin", self.remote) + self._write("asset/scan.tiff", b"\x89imagedata!") # DATA + self._write("asset/values.csv", b"a,b\n1,2\n") # DATA + self._write("asset/X_status.json", b'{"s":1}') # EXCLUDE + self._write("asset/catalog.json", b'{"c":1}') # EXCLUDE + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "c") + _git(self.work, "push", "-u", "origin", "main") + # Age the DATA files to before the run stamp so they never look "newer". + for rel in ("asset/scan.tiff", "asset/values.csv"): + os.utime(os.path.join(self.work, rel), (PAST_EPOCH, PAST_EPOCH)) + + self._orig_hsi = reclaim.hsi_exists + reclaim.hsi_exists = lambda tar: True # pretend the Fortress tar exists + + def tearDown(self): + reclaim.hsi_exists = self._orig_hsi + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write(self, rel, content): + p = os.path.join(self.work, rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "wb") as fh: + fh.write(content) + return p + + def _bytes(self, *rels): + return sum(os.path.getsize(os.path.join(self.asset, r)) for r in rels) + + def _coverage(self): + return {"repo_root": self.work, "require_pushed": True, + "remote_durable_attestation": True, + "forbid_remote_under": ["/depot/"], "max_fetch_age": "24h"} + + def _log(self, exclusion=None, archived=("scan.tiff", "values.csv")): + """An archive log as archive.py writes it. source_files carries only the + archived (post-exclusion) members.""" + return { + "project": "T", "run_timestamp": "T_" + RUN_STAMP, + "source_folder": self.asset, "fortress_tar": "/f/T.tar", + "source_files": {a: "md5" for a in archived}, + "source_bytes": self._bytes(*archived), + "file_pattern": ".*", "status": "success", + "exclusion": exclusion or {"status": "OFF"}, + } + + def _excl_on(self, excluded=("X_status.json", "catalog.json")): + return {"status": "ON", "spec_sha256": "deadbeef", + "excluded_arcnames": list(excluded), + "excluded": [{"arcname": a} for a in excluded], + "coverage": self._coverage()} + + # ── the money case: metadata regenerated (newer than run) still reads SAFE ── + def test_regenerated_metadata_still_covered_is_safe(self): + # Regenerate BOTH metadata files with new content, re-commit + push, so + # they are newer-than-run but still git-COVERED. + self._write("asset/X_status.json", b'{"s":2,"regenerated":true}') + self._write("asset/catalog.json", b'{"c":2,"regenerated":true}') + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "regen metadata") + _git(self.work, "push", "origin", "main") + + v = reclaim.verify(self._log(self._excl_on()), self.asset, holds=[]) + self.assertEqual(v["verdict"], reclaim.SAFE, + f"{v['verdict']}: {v['reasons']}") + self.assertEqual(v["live"], 2) # data only; metadata subtracted + self.assertEqual(v["archived"], 2) + + def test_without_inv_e_this_would_drift(self): + # Sanity contrast: the SAME live state with a log that has NO exclusion + # block (pre-feature) drifts, because the metadata is counted. + self._write("asset/X_status.json", b'{"s":2}') + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "regen") + _git(self.work, "push", "origin", "main") + # archived here = data only (2), but no exclusion block -> nothing + # subtracted -> live = 4 -> DRIFTED. + v = reclaim.verify(self._log(exclusion={"status": "OFF"}), self.asset, holds=[]) + self.assertEqual(v["verdict"], reclaim.DRIFTED) + + # ── an excluded file no longer covered => DRIFTED (never a silent delete) ── + def test_excluded_file_no_longer_pushed_is_drifted(self): + # Rewrite one excluded file and COMMIT it but do NOT push -> its current + # blob is not on the remote (STALE) -> its bytes are not on tape. + self._write("asset/X_status.json", b'{"s":99,"edited":true}') + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "edit status, do not push") + + v = reclaim.verify(self._log(self._excl_on()), self.asset, holds=[]) + self.assertEqual(v["verdict"], reclaim.DRIFTED) + self.assertTrue(any("no longer git-COVERED" in r for r in v["reasons"]), + v["reasons"]) + + def test_new_metadata_since_archive_is_drifted(self): + # A NEW metadata file (not in the recorded excluded set) must count as + # drift — exclusion never hides source growth. + self._write("asset/Y_status.json", b'{"s":1}') + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "new status") + _git(self.work, "push", "origin", "main") + + v = reclaim.verify(self._log(self._excl_on()), self.asset, holds=[]) + self.assertEqual(v["verdict"], reclaim.DRIFTED) + + def test_gate_unprovable_is_conservatively_drifted(self): + # If the exclusions cannot be re-proven now (dirty subtree fails + # preconditions), reclaim must NOT call the target SAFE. + self._write("asset/loose.txt", b"uncommitted") # dirties the subtree + v = reclaim.verify(self._log(self._excl_on()), self.asset, holds=[]) + self.assertEqual(v["verdict"], reclaim.DRIFTED) + + def test_no_exclusion_block_behaves_as_before(self): + # Pre-feature log: archived counts ALL files; live == archived -> SAFE, + # and recheck does not apply. Age the metadata too so nothing is newer + # than the run (setUp only ages the DATA files). + for rel in ("asset/X_status.json", "asset/catalog.json"): + os.utime(os.path.join(self.work, rel), (PAST_EPOCH, PAST_EPOCH)) + v = reclaim.verify( + self._log(exclusion={"status": "OFF"}, + archived=("scan.tiff", "values.csv", + "X_status.json", "catalog.json")), + self.asset, holds=[]) + self.assertEqual(v["verdict"], reclaim.SAFE, v["reasons"]) + + # ── recheck_exclusions unit behavior ── + def test_recheck_applies_false_without_block(self): + r = reclaim.recheck_exclusions(self._log(), self.asset) + self.assertFalse(r["applies"]) + self.assertEqual(r["still_covered"], set()) + + def test_recheck_all_covered(self): + r = reclaim.recheck_exclusions(self._log(self._excl_on()), self.asset) + self.assertTrue(r["applies"]) + self.assertEqual(r["still_covered"], {"X_status.json", "catalog.json"}) + self.assertEqual(r["no_longer_covered"], []) + + def test_recheck_flags_uncovered(self): + self._write("asset/catalog.json", b'{"c":777}') + _git(self.work, "add", "-A") + _git(self.work, "commit", "-m", "edit catalog, no push") + r = reclaim.recheck_exclusions(self._log(self._excl_on()), self.asset) + self.assertIn("X_status.json", r["still_covered"]) + gone = dict(r["no_longer_covered"]) + self.assertIn("catalog.json", gone) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_opt_in_allowlist.py b/tests/test_opt_in_allowlist.py index f5a5c30..46ab80b 100644 --- a/tests/test_opt_in_allowlist.py +++ b/tests/test_opt_in_allowlist.py @@ -41,7 +41,8 @@ def setUp(self): self._orig_scan = reclaim.scan_source reclaim.hsi_exists = lambda tar: True # scan_source -> (live, newer_count, newer_samples, live_bytes); archived=0 - reclaim.scan_source = lambda folder, pattern, run_dt: (0, 0, [], None) + # (accepts the INV-E exclude_arcnames kwarg; these logs have no exclusion) + reclaim.scan_source = lambda folder, pattern, run_dt, **kw: (0, 0, [], None) def tearDown(self): reclaim.hsi_exists = self._orig_hsi