feat(exclusion): reclaim re-applies + re-proves the excluded set (INV-E)

CLAUDE.md

@@ -287,8 +287,18 @@ missing `git` on a compute node all degrade to "back up everything" with a recor
 reason (banner + run-log `exclusion` block + completion email). The **companion
 guard** (INV-C) suppresses exclusion run-wide if a directory would ship zero members,
 and the run continues backing up everything with a loud `[EXCLUSION-ABORT]` alert.
-Runs under `--globus` too (the spec PATH is read on the compute node). Reclaim must
-re-apply the recorded excluded set + re-run the gate (INV-E) — pending.
+Runs under `--globus` too (the spec PATH is read on the compute node).
+
+**Reclaim (INV-E)** re-applies the archive's recorded exclusion at reclaim time:
+`reclaim.recheck_exclusions()` subtracts the recorded `excluded_arcnames` from the
+live walk **and re-runs the git gate** against the current repo for each one. A
+metadata-only change reads `SAFE` (no false DRIFT → the backup skips the re-ship —
+this is what actually delivers the byte savings); an excluded file that is no longer
+`COVERED` (edited/regenerated since the archive, so its current bytes aren't on tape)
+reads `DRIFTED`, never a silent `--delete`. Logs without an `exclusion` block behave
+exactly as before. Still pending: the standalone post-hoc re-verifier, and
+resume-under-exclusion (the archive currently forces a fresh zip when an
+`exclude_spec` is set).
 
 ---
 

archive.py

@@ -1011,11 +1011,20 @@ def evaluate_exclusions(matched_members, source_folder, spec,
 
         report["status"] = "ON"
         report["coverage"] = {
+            # repo_root + the gate config are recorded so the run log is
+            # SELF-CONTAINED for reclaim's re-gate (INV-E): reclaim re-runs the
+            # coverage gate against the current repo for each recorded exclusion,
+            # using exactly these values — no dependency on the exclude_spec file
+            # still being present/unchanged at reclaim time.
+            "repo_root": ctx.get("repo_root"),
             "repo_head": ctx.get("repo_head"),
             "remote_ref": ctx.get("remote_ref"),
             "require_pushed": ctx.get("require_pushed"),
             "remote_durable_attestation": ctx.get("remote_durable_attestation"),
             "fetch_ts": ctx.get("fetch_ts"),
+            "forbid_remote_under": (coverage or {}).get("forbid_remote_under",
+                                                        ["/depot/"]),
+            "max_fetch_age": (coverage or {}).get("max_fetch_age", "24h"),
         }
 
         counts = {k: 0 for k in (

reclaim.py

@@ -59,6 +59,13 @@
 import subprocess
 import sys
 
+# archive.py ships alongside reclaim.py; reclaim re-uses its git coverage gate to
+# re-prove recorded exclusions at reclaim time (INV-E). Insert the script's own
+# dir so `import archive` works whether run as ./reclaim.py, python reclaim.py,
+# or via a PATH-linked wrapper.
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import archive
+
 HSI = "/opt/hsi/bin/hsi"
 DEFAULT_LOG_DIR = "/depot/florasense/etc/logs_fortress/archive_log"
 # Durable record of each --delete run, written on Depot so it survives /tmp
@@ -206,7 +213,7 @@ def hsi_exists(tar_path):
 TOP_LEVEL_ONLY_PATTERN = r"^[^/]+$"
 
 
-def scan_source(folder, pattern=None, cutoff=None, limit=3):
+def scan_source(folder, pattern=None, cutoff=None, limit=3, exclude_arcnames=None):
     """One metadata-only walk of the live source, optionally scoped by the
     archive's file_pattern (regex against the relative path, exactly like
     archive.py's walk). Returns (live_count, newer_count, newer_samples,
@@ -219,10 +226,18 @@ def scan_source(folder, pattern=None, cutoff=None, limit=3):
     TOP_LEVEL_ONLY_PATTERN to a non-recursive scan — without that, verifying a
     root-files-only target walks its entire (possibly multi-TB) tree to match
     a handful of files.
+
+    exclude_arcnames (INV-E): a set of relpaths that the archive EXCLUDED and
+    that reclaim has RE-PROVEN still git-COVERED — subtracted from the walk so
+    live count/bytes/newer line up with the (post-exclusion) archived set. A file
+    matching file_pattern but NOT in this set still counts (so newly-added or
+    no-longer-covered metadata surfaces as drift). None/empty = no subtraction
+    (fully back-compatible with pre-exclusion logs).
     """
     if not os.path.isdir(folder):
         return None
     rx = re.compile(pattern) if pattern else None
+    excluded = set(exclude_arcnames or ())
     cut = cutoff.timestamp() if cutoff is not None else None
     live = 0
     newer = 0
@@ -241,6 +256,8 @@ def scan_source(folder, pattern=None, cutoff=None, limit=3):
             rel = os.path.relpath(p, folder)
             if rx is not None and not rx.search(rel):
                 continue
+            if rel in excluded:
+                continue   # re-proven-covered exclusion — not part of the archive
             live += 1
             try:
                 st = os.stat(p)
@@ -334,6 +351,76 @@ def resolve_source_folder(log, config):
     return log.get("source_folder") or (config or {}).get("source_folder")
 
 
+def recheck_exclusions(log, source_folder):
+    """
+    INV-E: re-prove each exclusion the archive recorded is STILL git-COVERED
+    against the CURRENT repo. Returns:
+
+      {applies: bool,
+       still_covered: set[relpath],           # subtract these from the live walk
+       no_longer_covered: [(relpath, verdict)],  # edited/regenerated -> DRIFTED
+       reason: str | None}
+
+    applies=False when the log predates exclusion (no `exclusion` block) or its
+    status != "ON" — reclaim then behaves exactly as before (fully back-compat).
+
+    FAIL-SAFE: if the exclusions cannot be re-proven now (no repo_root recorded,
+    git unavailable, preconditions fail), NOTHING is marked still-covered — every
+    recorded exclusion falls into no_longer_covered, so the target reads as
+    DRIFTED (never wrongly SAFE / never a --delete of bytes not on tape). Uses
+    the coverage config recorded in the log, so it needs no access to the
+    original exclude_spec file.
+    """
+    excl = (log or {}).get("exclusion") or {}
+    result = {"applies": False, "still_covered": set(),
+              "no_longer_covered": [], "reason": None}
+    if excl.get("status") != "ON":
+        return result
+    result["applies"] = True
+
+    recorded = list(excl.get("excluded_arcnames")
+                    or [e.get("arcname") for e in excl.get("excluded", [])])
+    recorded = [r for r in recorded if r]
+    if not recorded:
+        return result   # exclusion ran but dropped nothing — nothing to subtract
+
+    cov = excl.get("coverage") or {}
+    repo_root = cov.get("repo_root")
+    if not repo_root or not source_folder:
+        result["no_longer_covered"] = [(r, "NO-REPO-ROOT") for r in recorded]
+        result["reason"] = "no repo_root recorded — cannot re-prove exclusions"
+        return result
+
+    coverage_cfg = {
+        "repo_root": repo_root,
+        "require_pushed": cov.get("require_pushed", True),
+        "remote_durable_attestation": cov.get("remote_durable_attestation", True),
+        "forbid_remote_under": cov.get("forbid_remote_under", ["/depot/"]),
+        "max_fetch_age": cov.get("max_fetch_age", "24h"),
+    }
+    try:
+        ok, reason, ctx = archive.git_coverage_preconditions(
+            coverage_cfg, repo_root, source_folder)
+    except Exception as e:
+        ok, reason, ctx = False, f"gate error: {type(e).__name__}: {e}", {}
+    if not ok:
+        result["no_longer_covered"] = [(r, "GATE-OFF") for r in recorded]
+        result["reason"] = f"cannot re-prove exclusions ({reason})"
+        return result
+
+    for rel in recorded:
+        fp = os.path.join(source_folder, rel)
+        try:
+            verdict, _ev = archive.git_file_verdict(repo_root, fp, ctx)
+        except Exception as e:
+            verdict = "ERROR"
+        if verdict == archive.COV_COVERED:
+            result["still_covered"].add(rel)
+        else:
+            result["no_longer_covered"].append((rel, verdict))
+    return result
+
+
 def verify(log, source_folder, holds, file_pattern=None, canonical_roots=(),
            allowlist=None, opt_in=False):
     """
@@ -361,6 +448,13 @@ def verify(log, source_folder, holds, file_pattern=None, canonical_roots=(),
     run_dt = run_datetime(log.get("run_timestamp"))
     reasons = []
 
+    # INV-E: re-apply the archive's recorded exclusions to the live walk — but
+    # ONLY those we can RE-PROVE are still git-COVERED right now. A recorded
+    # exclusion that is no longer covered (edited/regenerated on disk since the
+    # archive, so its current bytes are NOT on tape) is deliberately left in the
+    # walk so it surfaces as drift below — never silently SAFE, never --delete'd.
+    excl_check = recheck_exclusions(log, source_folder)
+
     held = any((h == project) or (source_folder and h in source_folder) for h in holds)
 
     # Fast structural signals
@@ -370,8 +464,12 @@ def verify(log, source_folder, holds, file_pattern=None, canonical_roots=(),
         reasons.append(f"log status = {status!r}")
 
     # One pattern-scoped metadata walk gives count, newer-than-run, and bytes
-    # together (legacy verify did three separate unscoped walks).
-    scan = scan_source(source_folder, pattern, run_dt) if source_folder else None
+    # together (legacy verify did three separate unscoped walks). Under INV-E the
+    # re-proven-covered exclusions are subtracted so live lines up with the
+    # (post-exclusion) archived set.
+    scan = scan_source(source_folder, pattern, run_dt,
+                       exclude_arcnames=excl_check["still_covered"]
+                       ) if source_folder else None
     if source_folder is None:
         reasons.append("source_folder unknown (not in log or config)")
 
@@ -405,6 +503,15 @@ def verify(log, source_folder, holds, file_pattern=None, canonical_roots=(),
     elif live is None:
         verdict = GONE
         reasons.append("source already removed from Depot")
+    elif excl_check.get("applies") and excl_check.get("no_longer_covered"):
+        # INV-E: a file the archive EXCLUDED is no longer git-COVERED — its
+        # current on-disk bytes were never written to tape. Block reclaim.
+        verdict = DRIFTED
+        gone = excl_check["no_longer_covered"]
+        sample = ", ".join(f"{a} ({v})" for a, v in gone[:3])
+        reasons.append(f"{len(gone)} previously-excluded file(s) no longer "
+                       f"git-COVERED (e.g. {sample}) — edited/regenerated since "
+                       f"archive; current bytes are not on tape")
     elif newer_count > 0:
         verdict = DRIFTED
         reasons.append(f"{newer_count} file(s) newer than archive run "