From d866a6d867204a0a2b4d07250b5c3e2cf99d6805 Mon Sep 17 00:00:00 2001 From: "Doucette, Jarrod S" Date: Sun, 19 Jul 2026 13:26:01 -0400 Subject: [PATCH] fix(routing): route on basename length too, not just total arcname length Two production drops (repository_X1D_2_spectral-standoff, repository_X1J_1_visual-injury, 2026-07-18/19) hit indexed htar -L's silent member-omission again, caught by the ship_object stdout scan (no data lost) but not by choose_transport's HTAR_ARCNAME_LEN_LIMIT: both had short total relpaths (166, 124 chars) but a basename alone over 100 chars (102, 110). ustar's 100-byte "name" field holds only the final path component, so an overlong basename drops a member regardless of total path length or nesting depth. Reproduced live on Negishi with synthetic files at a short total path and varying basename length: 99-char basenames archived cleanly, 100-char basenames were dropped every time, same boundary whether the file sat at the top of the tree or several directories deep. Added HTAR_BASENAME_LEN_LIMIT (95, a 5-char margin under the confirmed 100-char cutoff) and route to htar_large when any member's basename is at/over it, alongside the existing size and arcname checks. Co-Authored-By: Claude Sonnet 5 --- archive.py | 61 ++++++++++++++++++++++++++++++------ tests/test_routing_wiring.py | 41 ++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/archive.py b/archive.py index f2429e9..e32de4d 100644 --- a/archive.py +++ b/archive.py @@ -847,6 +847,33 @@ def decode_blonde(blonde): # member gets anywhere near the drop threshold. HTAR_ARCNAME_LEN_LIMIT = 240 +# The 249-char ceiling above is a TOTAL relpath limit — it is not the whole story. +# ustar (the format `htar -L` writes) splits a path into a 100-byte "name" field +# and a 155-byte "prefix" field, joined on a '/'. A long path can still fit if the +# final component (the basename) is short enough to land entirely in the 100-byte +# name field with everything else absorbed into the prefix; but if the BASENAME +# ALONE is too long for that field, no split point exists and the member is +# dropped regardless of how short the total path is. Two 2026-07-18/19 production +# drops were exactly this, both caught by the ship_object stdout scan (defense in +# depth worked — no data lost) but neither would have been caught by +# HTAR_ARCNAME_LEN_LIMIT alone: +# - repository_X1D_2_spectral-standoff: total relpath 166 chars, basename alone +# 102 chars ("VIS_R_2602698_..._260422112752320.apf"). +# - repository_X1J_1_visual-injury: total relpath 124 chars, basename alone +# 110 chars ("X1J-PH-PNC-NT-..._20260616T123649-0400.JPEG"). +# Reproduced live on Negishi 2026-07-19 with synthetic files at a SHORT total path +# (so the total-length check couldn't be what's firing) and varying basename +# length: 99-char basenames archived cleanly, 100-char basenames were dropped +# every time ("WARNING: name too long for tar archive- file omitted"), and the +# same 99/100 boundary held whether the file sat at the top of the tree or three +# directories deep — confirming this is a basename-only constraint, independent +# of HTAR_ARCNAME_LEN_LIMIT and of nesting depth. `htar_large` has no such +# ceiling (see HTAR_ARCNAME_LEN_LIMIT's own note). Set 5 chars under the +# confirmed 100-char cutoff — a smaller margin than HTAR_ARCNAME_LEN_LIMIT's +# because the field itself is small enough that hugging the exact boundary is +# riskier per spare char. +HTAR_BASENAME_LEN_LIMIT = 95 + # Stall detection (2026-07-18 incident: repository_X1D_3_metabolomics_rawspectra, # Slurm job 40793013 on Negishi, ran 32h+ before manual cancellation). Its .out file # showed the destination `hsi -q mkdir -p ...` line and then nothing — the following @@ -1155,20 +1182,30 @@ def budget_warnings(objects, catalog, shard_target=SHARD_TARGET_DEFAULT, def choose_transport(arcnames, catalog): """ Pick the Fortress transport for an object (RFC §2.1). "htar_large" (plain tar, - no .tar.idx) is forced if EITHER: + no .tar.idx) is forced if ANY of: - some member's size >= HTAR_MEMBER_LIMIT (2^36, a huge solo) — htar's 64 GiB single-file limit; or - some member's arcname length >= HTAR_ARCNAME_LEN_LIMIT — indexed `htar -L` silently DROPS overlong relpaths (confirmed on Negishi; see the constant's comment), so any member near that ceiling routes the whole object away - from indexed htar rather than risk a partial, silently-incomplete tar. + from indexed htar rather than risk a partial, silently-incomplete tar; or + - some member's BASENAME (final '/'-separated component) length >= + HTAR_BASENAME_LEN_LIMIT — a distinct ustar constraint from the arcname + check above: the format's 100-byte "name" field holds only the final path + component, so a long basename drops a member regardless of how short the + total relpath is (see HTAR_BASENAME_LEN_LIMIT's comment for the two + production drops that motivated this and the live Negishi repro). Checking + arcname length alone does not catch this case. Otherwise "htar" (indexed .tar.idx, fast listing + random-access restore). Pure + deterministic. A member missing from the catalog (vanished mid-walk) is - treated as 0 bytes for the size check (its arcname length still counts). + treated as 0 bytes for the size check (its arcname/basename length still count). """ max_size = max((catalog.get(a, (0, 0))[1] for a in arcnames), default=0) max_name_len = max((len(a) for a in arcnames), default=0) - if max_size >= HTAR_MEMBER_LIMIT or max_name_len >= HTAR_ARCNAME_LEN_LIMIT: + max_basename_len = max((len(a.rsplit("/", 1)[-1]) for a in arcnames), default=0) + if (max_size >= HTAR_MEMBER_LIMIT + or max_name_len >= HTAR_ARCNAME_LEN_LIMIT + or max_basename_len >= HTAR_BASENAME_LEN_LIMIT): return "htar_large" return "htar" @@ -3429,10 +3466,11 @@ def ship_object(source_folder, arcnames, object_stem, catalog, fortress_base_dir Ship ONE routed object (Phase 2, RFC §2.5) — LOCAL only. Tars the object's members DIRECTLY from source (no zip), choosing the transport via choose_transport: real `htar` (indexed .idx, random-access restore) when every member is < HTAR_MEMBER_LIMIT - AND every arcname is < HTAR_ARCNAME_LEN_LIMIT, else `htar_large` (plain tar, no - index) for a >=64 GiB solo or an overlong-path member. Computes per-file MD5 from - source, round-trip verifies every member off tape, and writes the object's own - {stem}_{ts}.txt/.json run-log pair (invariant #4). + AND every arcname is < HTAR_ARCNAME_LEN_LIMIT AND every basename is < + HTAR_BASENAME_LEN_LIMIT, else `htar_large` (plain tar, no index) for a >=64 GiB + solo, an overlong-path member, or an overlong-basename member. Computes per-file + MD5 from source, round-trip verifies every member off tape, and writes the + object's own {stem}_{ts}.txt/.json run-log pair (invariant #4). Returns dict(fortress_tar, transport, idx, run_timestamp, n_files, source_bytes, tar_bytes, file_checksums, txt_path, json_path, create_warning). Raises RuntimeError @@ -3542,8 +3580,11 @@ def ship_object(source_folder, arcnames, object_stem, catalog, fortress_base_dir # - htar (indexed, -L listfile): "WARNING: name too long for tar # archive- file omitted" + " []" on the next line. Should be # unreachable now that choose_transport routes any object with an - # overlong arcname to htar_large first (HTAR_ARCNAME_LEN_LIMIT) — this - # is the defense-in-depth backstop for anything that slips past that. + # overlong arcname (HTAR_ARCNAME_LEN_LIMIT) OR overlong basename + # (HTAR_BASENAME_LEN_LIMIT — a short-total-path/long-final-component + # member the arcname check alone misses, see that constant's comment) + # to htar_large first — this is the defense-in-depth backstop for + # anything that slips past both of those. # - htar (indexed): "INFO: Not readable: " — a permission/IO # error on the source file, skipped rather than aborting the run. # - htar_large (plain `tar | hsi put`): "tar: : Cannot open: ..." diff --git a/tests/test_routing_wiring.py b/tests/test_routing_wiring.py index f9be4ea..e27a1e7 100644 --- a/tests/test_routing_wiring.py +++ b/tests/test_routing_wiring.py @@ -105,8 +105,17 @@ def test_long_arcname_forces_htar_large_regardless_of_size(self): self.assertEqual(archive.choose_transport([long_name], c), "htar_large") def test_short_arcname_at_limit_boundary(self): - under = "a" * (archive.HTAR_ARCNAME_LEN_LIMIT - 1) - at = "a" * archive.HTAR_ARCNAME_LEN_LIMIT + # Directory prefix carries the length, basename ("f.txt") stays well under + # HTAR_BASENAME_LEN_LIMIT — isolates the arcname-length check from the + # separate basename-length check below. + def path_of_len(n): + prefix_len = n - len("/f.txt") + return "a" * prefix_len + "/f.txt" + + under = path_of_len(archive.HTAR_ARCNAME_LEN_LIMIT - 1) + at = path_of_len(archive.HTAR_ARCNAME_LEN_LIMIT) + self.assertEqual(len(under), archive.HTAR_ARCNAME_LEN_LIMIT - 1) + self.assertEqual(len(at), archive.HTAR_ARCNAME_LEN_LIMIT) c = {under: (1, SMALL), at: (1, SMALL)} self.assertEqual(archive.choose_transport([under], c), "htar") self.assertEqual(archive.choose_transport([at], c), "htar_large") @@ -116,6 +125,34 @@ def test_long_arcname_among_short_ones_decides(self): c = {"short": (1, SMALL), long_name: (1, SMALL)} self.assertEqual(archive.choose_transport(["short", long_name], c), "htar_large") + def test_long_basename_short_total_path_forces_htar_large(self): + # Negishi 2026-07-19: repository_X1D_2_spectral-standoff (total relpath 166 + # chars, well under HTAR_ARCNAME_LEN_LIMIT) and repository_X1J_1_visual-injury + # (124 chars) both had members dropped by indexed htar -L — not because the + # total path was long, but because the BASENAME alone (102 and 110 chars + # respectively) exceeded ustar's 100-byte name-field. Reproduced live: a + # short total path with a >=100-char basename is dropped regardless of + # nesting depth. A short path + long basename must still route to + # htar_large — HTAR_ARCNAME_LEN_LIMIT alone must not decide. + long_basename = "b" * 105 + ".txt" + short_path = "d/" + long_basename + self.assertLess(len(short_path), archive.HTAR_ARCNAME_LEN_LIMIT) + self.assertGreaterEqual(len(long_basename), archive.HTAR_BASENAME_LEN_LIMIT) + c = {short_path: (1, SMALL)} + self.assertEqual(archive.choose_transport([short_path], c), "htar_large") + + def test_basename_at_limit_boundary(self): + under = "d/" + "b" * (archive.HTAR_BASENAME_LEN_LIMIT - 1) + at = "d/" + "b" * archive.HTAR_BASENAME_LEN_LIMIT + c = {under: (1, SMALL), at: (1, SMALL)} + self.assertEqual(archive.choose_transport([under], c), "htar") + self.assertEqual(archive.choose_transport([at], c), "htar_large") + + def test_long_basename_among_short_ones_decides(self): + long_basename = "d/" + "b" * 105 + ".txt" + c = {"short": (1, SMALL), long_basename: (1, SMALL)} + self.assertEqual(archive.choose_transport(["short", long_basename], c), "htar_large") + class TestObjectProjectName(unittest.TestCase): def test_solo_and_shard_stems(self):