fix(routing): route on basename length too, not just total arcname length

archive.py

@@ -847,6 +847,33 @@ def decode_blonde(blonde):
 # member gets anywhere near the drop threshold.
 HTAR_ARCNAME_LEN_LIMIT = 240
 
+# The 249-char ceiling above is a TOTAL relpath limit — it is not the whole story.
+# ustar (the format `htar -L` writes) splits a path into a 100-byte "name" field
+# and a 155-byte "prefix" field, joined on a '/'. A long path can still fit if the
+# final component (the basename) is short enough to land entirely in the 100-byte
+# name field with everything else absorbed into the prefix; but if the BASENAME
+# ALONE is too long for that field, no split point exists and the member is
+# dropped regardless of how short the total path is. Two 2026-07-18/19 production
+# drops were exactly this, both caught by the ship_object stdout scan (defense in
+# depth worked — no data lost) but neither would have been caught by
+# HTAR_ARCNAME_LEN_LIMIT alone:
+#   - repository_X1D_2_spectral-standoff: total relpath 166 chars, basename alone
+#     102 chars ("VIS_R_2602698_..._260422112752320.apf").
+#   - repository_X1J_1_visual-injury: total relpath 124 chars, basename alone
+#     110 chars ("X1J-PH-PNC-NT-..._20260616T123649-0400.JPEG").
+# Reproduced live on Negishi 2026-07-19 with synthetic files at a SHORT total path
+# (so the total-length check couldn't be what's firing) and varying basename
+# length: 99-char basenames archived cleanly, 100-char basenames were dropped
+# every time ("WARNING: name too long for tar archive- file omitted"), and the
+# same 99/100 boundary held whether the file sat at the top of the tree or three
+# directories deep — confirming this is a basename-only constraint, independent
+# of HTAR_ARCNAME_LEN_LIMIT and of nesting depth. `htar_large` has no such
+# ceiling (see HTAR_ARCNAME_LEN_LIMIT's own note). Set 5 chars under the
+# confirmed 100-char cutoff — a smaller margin than HTAR_ARCNAME_LEN_LIMIT's
+# because the field itself is small enough that hugging the exact boundary is
+# riskier per spare char.
+HTAR_BASENAME_LEN_LIMIT = 95
+
 # Stall detection (2026-07-18 incident: repository_X1D_3_metabolomics_rawspectra,
 # Slurm job 40793013 on Negishi, ran 32h+ before manual cancellation). Its .out file
 # showed the destination `hsi -q mkdir -p ...` line and then nothing — the following
@@ -1155,20 +1182,30 @@ def budget_warnings(objects, catalog, shard_target=SHARD_TARGET_DEFAULT,
 def choose_transport(arcnames, catalog):
     """
     Pick the Fortress transport for an object (RFC §2.1). "htar_large" (plain tar,
-    no .tar.idx) is forced if EITHER:
+    no .tar.idx) is forced if ANY of:
       - some member's size >= HTAR_MEMBER_LIMIT (2^36, a huge solo) — htar's 64 GiB
         single-file limit; or
       - some member's arcname length >= HTAR_ARCNAME_LEN_LIMIT — indexed `htar -L`
         silently DROPS overlong relpaths (confirmed on Negishi; see the constant's
         comment), so any member near that ceiling routes the whole object away
-        from indexed htar rather than risk a partial, silently-incomplete tar.
+        from indexed htar rather than risk a partial, silently-incomplete tar; or
+      - some member's BASENAME (final '/'-separated component) length >=
+        HTAR_BASENAME_LEN_LIMIT — a distinct ustar constraint from the arcname
+        check above: the format's 100-byte "name" field holds only the final path
+        component, so a long basename drops a member regardless of how short the
+        total relpath is (see HTAR_BASENAME_LEN_LIMIT's comment for the two
+        production drops that motivated this and the live Negishi repro). Checking
+        arcname length alone does not catch this case.
     Otherwise "htar" (indexed .tar.idx, fast listing + random-access restore).
     Pure + deterministic. A member missing from the catalog (vanished mid-walk) is
-    treated as 0 bytes for the size check (its arcname length still counts).
+    treated as 0 bytes for the size check (its arcname/basename length still count).
     """
     max_size = max((catalog.get(a, (0, 0))[1] for a in arcnames), default=0)
     max_name_len = max((len(a) for a in arcnames), default=0)
-    if max_size >= HTAR_MEMBER_LIMIT or max_name_len >= HTAR_ARCNAME_LEN_LIMIT:
+    max_basename_len = max((len(a.rsplit("/", 1)[-1]) for a in arcnames), default=0)
+    if (max_size >= HTAR_MEMBER_LIMIT
+            or max_name_len >= HTAR_ARCNAME_LEN_LIMIT
+            or max_basename_len >= HTAR_BASENAME_LEN_LIMIT):
         return "htar_large"
     return "htar"
 
@@ -3429,10 +3466,11 @@ def ship_object(source_folder, arcnames, object_stem, catalog, fortress_base_dir
     Ship ONE routed object (Phase 2, RFC §2.5) — LOCAL only. Tars the object's members
     DIRECTLY from source (no zip), choosing the transport via choose_transport: real
     `htar` (indexed .idx, random-access restore) when every member is < HTAR_MEMBER_LIMIT
-    AND every arcname is < HTAR_ARCNAME_LEN_LIMIT, else `htar_large` (plain tar, no
-    index) for a >=64 GiB solo or an overlong-path member. Computes per-file MD5 from
-    source, round-trip verifies every member off tape, and writes the object's own
-    {stem}_{ts}.txt/.json run-log pair (invariant #4).
+    AND every arcname is < HTAR_ARCNAME_LEN_LIMIT AND every basename is <
+    HTAR_BASENAME_LEN_LIMIT, else `htar_large` (plain tar, no index) for a >=64 GiB
+    solo, an overlong-path member, or an overlong-basename member. Computes per-file
+    MD5 from source, round-trip verifies every member off tape, and writes the
+    object's own {stem}_{ts}.txt/.json run-log pair (invariant #4).
 
     Returns dict(fortress_tar, transport, idx, run_timestamp, n_files, source_bytes,
     tar_bytes, file_checksums, txt_path, json_path, create_warning). Raises RuntimeError
@@ -3542,8 +3580,11 @@ def ship_object(source_folder, arcnames, object_stem, catalog, fortress_base_dir
         #   - htar (indexed, -L listfile): "WARNING: name too long for tar
         #     archive- file omitted" + "  [<relpath>]" on the next line. Should be
         #     unreachable now that choose_transport routes any object with an
-        #     overlong arcname to htar_large first (HTAR_ARCNAME_LEN_LIMIT) — this
-        #     is the defense-in-depth backstop for anything that slips past that.
+        #     overlong arcname (HTAR_ARCNAME_LEN_LIMIT) OR overlong basename
+        #     (HTAR_BASENAME_LEN_LIMIT — a short-total-path/long-final-component
+        #     member the arcname check alone misses, see that constant's comment)
+        #     to htar_large first — this is the defense-in-depth backstop for
+        #     anything that slips past both of those.
         #   - htar (indexed): "INFO: Not readable: <relpath>" — a permission/IO
         #     error on the source file, skipped rather than aborting the run.
         #   - htar_large (plain `tar | hsi put`): "tar: <relpath>: Cannot open: ..."

tests/test_routing_wiring.py

@@ -105,8 +105,17 @@ def test_long_arcname_forces_htar_large_regardless_of_size(self):
         self.assertEqual(archive.choose_transport([long_name], c), "htar_large")
 
     def test_short_arcname_at_limit_boundary(self):
-        under = "a" * (archive.HTAR_ARCNAME_LEN_LIMIT - 1)
-        at = "a" * archive.HTAR_ARCNAME_LEN_LIMIT
+        # Directory prefix carries the length, basename ("f.txt") stays well under
+        # HTAR_BASENAME_LEN_LIMIT — isolates the arcname-length check from the
+        # separate basename-length check below.
+        def path_of_len(n):
+            prefix_len = n - len("/f.txt")
+            return "a" * prefix_len + "/f.txt"
+
+        under = path_of_len(archive.HTAR_ARCNAME_LEN_LIMIT - 1)
+        at = path_of_len(archive.HTAR_ARCNAME_LEN_LIMIT)
+        self.assertEqual(len(under), archive.HTAR_ARCNAME_LEN_LIMIT - 1)
+        self.assertEqual(len(at), archive.HTAR_ARCNAME_LEN_LIMIT)
         c = {under: (1, SMALL), at: (1, SMALL)}
         self.assertEqual(archive.choose_transport([under], c), "htar")
         self.assertEqual(archive.choose_transport([at], c), "htar_large")
@@ -116,6 +125,34 @@ def test_long_arcname_among_short_ones_decides(self):
         c = {"short": (1, SMALL), long_name: (1, SMALL)}
         self.assertEqual(archive.choose_transport(["short", long_name], c), "htar_large")
 
+    def test_long_basename_short_total_path_forces_htar_large(self):
+        # Negishi 2026-07-19: repository_X1D_2_spectral-standoff (total relpath 166
+        # chars, well under HTAR_ARCNAME_LEN_LIMIT) and repository_X1J_1_visual-injury
+        # (124 chars) both had members dropped by indexed htar -L — not because the
+        # total path was long, but because the BASENAME alone (102 and 110 chars
+        # respectively) exceeded ustar's 100-byte name-field. Reproduced live: a
+        # short total path with a >=100-char basename is dropped regardless of
+        # nesting depth. A short path + long basename must still route to
+        # htar_large — HTAR_ARCNAME_LEN_LIMIT alone must not decide.
+        long_basename = "b" * 105 + ".txt"
+        short_path = "d/" + long_basename
+        self.assertLess(len(short_path), archive.HTAR_ARCNAME_LEN_LIMIT)
+        self.assertGreaterEqual(len(long_basename), archive.HTAR_BASENAME_LEN_LIMIT)
+        c = {short_path: (1, SMALL)}
+        self.assertEqual(archive.choose_transport([short_path], c), "htar_large")
+
+    def test_basename_at_limit_boundary(self):
+        under = "d/" + "b" * (archive.HTAR_BASENAME_LEN_LIMIT - 1)
+        at = "d/" + "b" * archive.HTAR_BASENAME_LEN_LIMIT
+        c = {under: (1, SMALL), at: (1, SMALL)}
+        self.assertEqual(archive.choose_transport([under], c), "htar")
+        self.assertEqual(archive.choose_transport([at], c), "htar_large")
+
+    def test_long_basename_among_short_ones_decides(self):
+        long_basename = "d/" + "b" * 105 + ".txt"
+        c = {"short": (1, SMALL), long_basename: (1, SMALL)}
+        self.assertEqual(archive.choose_transport(["short", long_basename], c), "htar_large")
+
 
 class TestObjectProjectName(unittest.TestCase):
     def test_solo_and_shard_stems(self):