diff --git a/CLAUDE.md b/CLAUDE.md index 3552994..d74034e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,15 +236,21 @@ Known issues below). The budget is **not one number**: a *create* streams per-file progress, so silence there really is a hang and it keeps `HTAR_STALL_SECONDS` (2h); the post-create *round-trip -verify retrieve* is silent **by design** while HPSS queues and stages the tape, so -`ship_object` bounds it with the config's **`retrieve_stall_seconds`** instead (default -in `stall_defaults.json` — data, not code; a per-asset config may override it; there is -deliberately no code-side fallback, so a caller that forgets to thread it down raises). -Conflating the two is a real, expensive bug, not a theoretical one: on 2026-08-03 a -single 6.62 GB `X1D_3_metabolomics_rawspectra` shard had its healthy retrieve killed at -exactly 2h twice per job (its stall log reads `activity resumed after 7200.1s silence`), -so verify never passed, no success record was written, and every scheduled run re-shipped -the whole shard — 48 identical tars, ~318 GB of duplicate tape, over 2.5 weeks. +verify retrieve* is silent **by design** while HPSS queues and stages the tape, so both +`ship_object` **and** `send_to_fortress` bound it with the config's +**`retrieve_stall_seconds`** instead (default in `stall_defaults.json` — data, not code; +a per-asset config may override it; there is deliberately no code-side fallback, so a +caller that forgets to thread it down raises). Conflating the two is a real, expensive +bug, not a theoretical one: on 2026-08-03 a single 6.62 GB `X1D_3_metabolomics_rawspectra` +shard (routed via `ship_object`) had its healthy retrieve killed at exactly 2h twice per +job (its stall log reads `activity resumed after 7200.1s silence`), so verify never +passed, no success record was written, and every scheduled run re-shipped the whole +shard — 48 identical tars, ~318 GB of duplicate tape, over 2.5 weeks. The identical bug +independently existed in `send_to_fortress`'s own inline `run_watched()` twin (whole-target +and Phase-3 leveled targets) until it was given the same fix: confirmed hit on +`repository_X0H_2_spectral-standoff` (job 41417829, 2026-07-31), a leveled-incremental +target whose round-trip retrieve was killed at exactly "no output for 2.0h" after a +successful create. Short, no-progress calls (`hsi mkdir`/`hsi ls`) just need `timeout=HSI_SHORT_CALL_TIMEOUT_SECONDS` (5 min) on a plain `subprocess.run()`. A new diff --git a/archive.py b/archive.py index 829786c..e4344d7 100644 --- a/archive.py +++ b/archive.py @@ -3034,6 +3034,7 @@ def send_to_fortress(zip_path, zip_checksum, file_checksums, members, source_bytes, source_folder, file_pattern, fortress_base_dir, emails, project_name, log_dir, cleanup_zip_on_success=False, exclusion=None, + retrieve_stall_seconds=None, extra_record_fields=None, quiet_success=False): """ On the RCAC compute system: transfer the zip to Fortress via htar_large, @@ -3047,6 +3048,21 @@ def send_to_fortress(zip_path, zip_checksum, file_checksums, members, All steps use full stdout+stderr logging since no binary streaming is involved. Log files are named after the zip (e.g. X0F_20260331_140000.txt/.json). + retrieve_stall_seconds is REQUIRED (no code-side default): the silence budget for + the two post-create round-trip verify retrieve calls only (hsi get, then the local + tar xvf), threaded down from the config (`retrieve_stall_seconds`, whose own + default lives in stall_defaults.json). Passing None raises rather than quietly + reinstating the create-side default this function's inline run_watched() otherwise + uses (7200s / 2h — kept in sync with HTAR_STALL_SECONDS by inspection, invariant + #1), which is far too tight for a large whole-target/leveled tar's tape stage: the + same class of bug that burned ~318 GB of duplicate tape on + repository_X1D_3_metabolomics_rawspectra via ship_object's twin of this gap (see + HTAR_STALL_SECONDS) — confirmed hit here too on + repository_X0H_2_spectral-standoff (job 41417829, 2026-07-31): create succeeded, + then the round-trip retrieve was killed at exactly "no output for 2.0h". The create + call keeps the hardcoded 7200s default: a create streams per-file progress, so + silence there really is a hang. + extra_record_fields: optional dict merged into run_record at construction (default None is behavior-neutral for the existing whole-target caller). Phase 3 (leveled incremental, RFC §3) tags leveled_object/badge/level/ @@ -3076,6 +3092,15 @@ def send_to_fortress(zip_path, zip_checksum, file_checksums, members, import smtplib from email.mime.text import MIMEText + if not isinstance(retrieve_stall_seconds, int) \ + or isinstance(retrieve_stall_seconds, bool) or retrieve_stall_seconds <= 0: + raise RuntimeError( + f"send_to_fortress: retrieve_stall_seconds must be a positive integer " + f"(seconds), got {retrieve_stall_seconds!r} — thread it down from the " + f"config (default in stall_defaults.json); there is deliberately no " + f"code-side fallback that could silently reinstate the create-side " + f"2h bound.") + # hsi lives at /opt/hsi/bin on Negishi but is not always on the compute # node PATH (only the login node sources /etc/profile.d snippets that add # it). htar_large invokes hsi, so we prepend explicitly to avoid the @@ -3529,6 +3554,7 @@ def worker(): f'>{hsi_get_out} 2>{hsi_get_err}') try: hsi_result = run_watched(hsi_get_cmd, [hsi_get_out, hsi_get_err], + stall_seconds=retrieve_stall_seconds, activity_log=os.path.join(log_dir, f"{zip_ts}.hsiget.stall.log")) except RuntimeError as e: write_log(f"hsi get stalled: {e}", level="ERROR") @@ -3590,6 +3616,7 @@ def worker(): tar_cmd = f'tar xvf {retrieved_tar} -C {retrieve_dir} >{tar_extract_out} 2>{tar_extract_err}' try: tar_result = run_watched(tar_cmd, [tar_extract_out, tar_extract_err], + stall_seconds=retrieve_stall_seconds, activity_log=os.path.join(log_dir, f"{zip_ts}.tarextract.stall.log")) except RuntimeError as e: write_log(f"tar extraction stalled: {e}", level="ERROR") @@ -4561,6 +4588,10 @@ def ship_level(level_obj, stem): zip_path, zip_checksum, file_checksums, zip_members, lvl_source_bytes, source_folder, file_pattern, fortress_base, emails, project_name, log_dir, cleanup_zip_on_success, exclusion, + # retrieve_stall_seconds is passed POSITIONALLY: `run` is the + # local/globus dispatcher (def run(fn, *fn_args)) and forwards + # positional args only. + retrieve_stall_seconds, {"leveled_object": True, "badge": badge_of(project_name, source_folder, file_pattern), "level": level_obj["level"], "blonde": level_obj["blonde"], @@ -4800,7 +4831,11 @@ def ship_level(level_obj, stem): send_to_fortress, zip_path, zip_checksum, file_checksums, members, source_bytes, source_folder, file_pattern, fortress_base, - emails, project_name, log_dir, cleanup_zip_on_success, exclusion + emails, project_name, log_dir, cleanup_zip_on_success, exclusion, + # retrieve_stall_seconds is passed POSITIONALLY: `run` is the + # local/globus dispatcher (def run(fn, *fn_args)) and forwards + # positional args only. + retrieve_stall_seconds, ) except Exception as e: msg = ( diff --git a/config.example.json b/config.example.json index 4a4ea1b..5c12a32 100644 --- a/config.example.json +++ b/config.example.json @@ -35,7 +35,7 @@ "comment_exclude_optional": "Optional (default []). Per-asset opt-out list of Tier-2 (OPTIONAL) rule ids from the exclude_spec (e.g. omics provenance/QC: aux_info, fastp, checksums). Honored only if this asset's file_pattern also ships the raw inputs those artifacts are regenerable from. Omit to back up all OPTIONAL artifacts (safe default).", "exclude_optional": [], - "comment_retrieve_stall_seconds": "Optional override (default: stall_defaults.json's retrieve_stall_seconds, currently 21600 = 6h). How long the post-create round-trip VERIFY RETRIEVE may go without producing any output before it is killed as a hung HPSS connection. Much larger than the create-side bound on purpose: a retrieve is silent while HPSS queues and stages the tape, and a too-tight bound kills a healthy retrieve — which, because verify never passes, makes the next run re-ship the whole object (the 2026-08 X1D shard loop; see stall_defaults.json). Raise it for a target whose tape-stage latency is legitimately longer.", + "comment_retrieve_stall_seconds": "Optional override (default: stall_defaults.json's retrieve_stall_seconds, currently 21600 = 6h). How long the post-create round-trip VERIFY RETRIEVE may go without producing any output before it is killed as a hung HPSS connection — applies whichever transport this target ships through (whole-target/leveled-incremental via send_to_fortress, or size-routed objects via ship_object; see stall_defaults.json). Much larger than the create-side bound on purpose: a retrieve is silent while HPSS queues and stages the tape, and a too-tight bound kills a healthy retrieve — which, because verify never passes, makes the next run re-ship the whole object (the 2026-08 X1D shard loop and the X0H whole-target loop; see stall_defaults.json). Raise it for a target whose tape-stage latency is legitimately longer (e.g. a much larger whole-target tar).", "retrieve_stall_seconds": 21600, "comment_size_routing": "Optional (default false). Phase-2 size-routing (docs/RFC_incremental_v2.md §2): instead of one whole-target tar, ship each file >= t_small as its own 'solo' object and bundle smaller files into content-addressed 'shards' (shard = hash(relpath) mod K). Unchanged objects skip, so a GROWING target ships only its new files (the append win). LOCAL mode only — refused under --globus. When true, log_dir MUST be shared+durable (a Depot logs dir), NOT home or scratch: the per-target manifest lives at {log_dir}/_vault/ and reclaim must find it. Omit for the default whole-target behavior.", diff --git a/stall_defaults.json b/stall_defaults.json index a22853e..a7b1762 100644 --- a/stall_defaults.json +++ b/stall_defaults.json @@ -1,6 +1,6 @@ { "comment": "Engine-wide stall-watch defaults. Read by archive.load_config() for EVERY config (per-asset configs may override a key by name). This file is REQUIRED: a missing/malformed/invalid file raises rather than falling back to a value hidden in code, so the operative timeout is always inspectable here.", - "comment_retrieve_stall_seconds": "How long ship_object's post-create round-trip verify retrieve (htar -xvf, or hsi get + tar xf) may go without producing ANY output before it is killed as a hung HPSS connection. Deliberately much larger than the create-side HTAR_STALL_SECONDS (2h): a create streams per-file progress continuously, so silence there really is a hang, whereas a retrieve of a large routed shard is silent BY DESIGN while HPSS queues and stages the tape. 2026-08-03 (repository_X1D_3_metabolomics_rawspectra, a single 6,624,961,024-byte shard): the retrieve went quiet, was killed at exactly the 2h bound on both attempts, and its own stall log shows activity resuming 0.1s past the kill line ('activity resumed after 7200.1s silence') — legitimate tape-stage latency, misread as a hang. Verify never passed, so no success record was written and the next scheduled run re-shipped the whole shard from scratch: 48 identical 6.62 GB tars (~318 GB of duplicate tape) over 2026-07-17..08-03. 6h gives ~3x margin over the one observed gap while staying well inside the 48h Slurm walltime even if both retrieve attempts burn their full budget.", + "comment_retrieve_stall_seconds": "How long a target's post-create round-trip verify retrieve may go without producing ANY output before it is killed as a hung HPSS connection. Shared by BOTH transport paths: ship_object's routed-object retrieve (htar -xvf, or hsi get + tar xf) and send_to_fortress's whole-target/leveled-incremental retrieve (hsi get + tar xvf) via its own self-contained inline run_watched() twin (invariant #1) — one config knob, same semantics, regardless of which path an asset ships through. Deliberately much larger than the create-side HTAR_STALL_SECONDS (2h): a create streams per-file progress continuously, so silence there really is a hang, whereas a retrieve is silent BY DESIGN while HPSS queues and stages the tape. 2026-08-03 (repository_X1D_3_metabolomics_rawspectra, a single 6,624,961,024-byte routed shard): the retrieve went quiet, was killed at exactly the 2h bound on both attempts, and its own stall log shows activity resuming 0.1s past the kill line ('activity resumed after 7200.1s silence') — legitimate tape-stage latency, misread as a hang. Verify never passed, so no success record was written and the next scheduled run re-shipped the whole shard from scratch: 48 identical 6.62 GB tars (~318 GB of duplicate tape) over 2026-07-17..08-03. The identical bug independently existed in send_to_fortress's retrieve (confirmed on repository_X0H_2_spectral-standoff, job 41417829, 2026-07-31 — a leveled-incremental, multi-TB whole-target tar, killed at exactly 'no output for 2.0h') until it was given this same fix. 6h gives ~3x margin over the one precisely-measured gap (X1D's 0.1s overrun) while staying well inside the 48h Slurm walltime even if both retrieve attempts burn their full budget; a per-asset config may raise this further for a target whose tape-stage latency is legitimately longer (e.g. a much larger whole-target tar like X0H's).", "retrieve_stall_seconds": 21600 } diff --git a/tests/test_send_to_fortress_retrieve_stall.py b/tests/test_send_to_fortress_retrieve_stall.py new file mode 100644 index 0000000..6881a4e --- /dev/null +++ b/tests/test_send_to_fortress_retrieve_stall.py @@ -0,0 +1,287 @@ +""" +Unit tests for send_to_fortress()'s CONFIGURABLE round-trip-verify retrieve stall +bound — the whole-target/leveled-incremental twin of what +tests/test_retrieve_stall_config.py already covers for ship_object (size-routing). + +send_to_fortress carries its OWN, fully self-contained inline stall-watch twin +(`run_watched`, invariant #1: no outer-scope references, imports inside — it must +survive being dill-serialized to a Globus Compute endpoint) rather than sharing +ship_object's module-level `_run_with_stall_watch`. PR #37 threaded a configurable +`retrieve_stall_seconds` into ship_object's retrieve calls but left this twin on its +own hardcoded 7200s (2h) default for BOTH the create and the two retrieve steps (hsi +get, then local tar xvf) — the exact class of bug PR #37 fixed, independently present +here. Confirmed hit on repository_X0H_2_spectral-standoff (job 41417829, 2026-07-31, +a leveled-incremental, multi-TB whole-target tar): create succeeded, then the +round-trip retrieve was killed at exactly "no output for 2.0h". + +No real hsi/htar_large/tar/tape: send_to_fortress's create and retrieve calls run +through its inline run_watched(), which drives a real subprocess.Popen (shell=True) — +so, mirroring test_ship_object_htar_rc.py's approach, this fakes the `htar_large`/ +`hsi`/`tar` executables themselves as tiny shell-script stand-ins on PATH rather than +mocking subprocess. Because run_watched is a closure local to send_to_fortress (not a +module attribute), it cannot be intercepted the way `archive._run_with_stall_watch` +is in test_retrieve_stall_config.py — instead, each behavioral assertion runs +send_to_fortress in a background thread with a bounded `join()`, so a regression that +silently falls back to the 7200s create-side default fails FAST (in a few seconds) +rather than actually hanging the test suite for two hours. +""" +import os +import shutil +import stat +import sys +import tempfile +import threading +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import archive # noqa: E402 + +# send_to_fortress's own inline run_watched() polls every min(poll_interval, stall_seconds) +# seconds (poll_interval defaults to 30, un-overridable from outside — it is not exposed +# as a config knob, only stall_seconds is). A fast fake command can therefore "miss" the +# very first poll (caught mid-fork/exec, before it has actually exited) and only be +# noticed a full poll_interval later — this is a real property of the create step's own +# polling loop, not something introduced by this fix. The retrieve-stall tests below must +# get through one create step before reaching the retrieve step under test, so their +# bounded waits need enough slack to absorb that up-to-~30s detection latency on top of +# the (tiny, deliberately small) configured retrieve bound itself. +_CREATE_POLL_SLACK_SECONDS = 40 + + +def _write_fake_bin(bin_dir, name, body): + path = os.path.join(bin_dir, name) + with open(path, "w") as fh: + fh.write("#!/bin/sh\n" + body) + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return path + + +class FakeFortressEnv: + """ + Installs fake `htar_large`, `hsi`, `tar` executables on PATH for the duration of + a `with` block, matching send_to_fortress's exact command shapes: + create: htar_large -cvf + retrieve A: hsi "get : " + retrieve B: tar xvf -C + plus the short `hsi "mkdir -p "` preflight call. + + create_body / get_body / tar_body are raw `sh` script bodies (not full + scripts) so each test can shape exactly one step's behavior (fast success, + or a silent hang) while leaving the others as simple, fast no-ops. + """ + + def __init__(self, create_body='echo "HTAR SUCCESSFUL"; exit 0', + get_body='local=$(echo "$1" | awk \'{print $2}\'); ' + 'printf "fake-tar-bytes" > "$local"; exit 0', + tar_body='exit 0'): + self.create_body = create_body + self.get_body = get_body + self.tar_body = tar_body + self._d = None + self._old_path = None + self.call_log_path = None + + def __enter__(self): + self._d = tempfile.mkdtemp() + bin_dir = os.path.join(self._d, "bin") + os.makedirs(bin_dir) + self.call_log_path = os.path.join(self._d, "calls.log") + + _write_fake_bin(bin_dir, "htar_large", f''' +printf 'htar_large %s\\n' "$*" >> "{self.call_log_path}" +{self.create_body} +''') + _write_fake_bin(bin_dir, "hsi", f''' +printf 'hsi %s\\n' "$*" >> "{self.call_log_path}" +case "$1" in + "mkdir "*) exit 0 ;; + "get "*) + {self.get_body} + ;; + *) exit 0 ;; +esac +''') + _write_fake_bin(bin_dir, "tar", f''' +printf 'tar %s\\n' "$*" >> "{self.call_log_path}" +case "$1" in + xvf) {self.tar_body} ;; + *) exit 0 ;; +esac +''') + + self._old_path = os.environ.get("PATH", "") + # send_to_fortress prepends /opt/hsi/bin ahead of PATH if it exists for real + # and isn't already present (its own defensive PATH-fixup for compute nodes) + # — pre-seed it (if missing) so that guard skips re-prepending, keeping + # bin_dir first regardless of what the host actually has installed. + base_path = self._old_path + real_hsi_bin = "/opt/hsi/bin" + if real_hsi_bin not in base_path.split(os.pathsep): + base_path = f"{base_path}{os.pathsep}{real_hsi_bin}" if base_path else real_hsi_bin + os.environ["PATH"] = bin_dir + os.pathsep + base_path + return self + + def __exit__(self, *exc): + os.environ["PATH"] = self._old_path + shutil.rmtree(self._d, ignore_errors=True) + + def calls(self): + if not os.path.isfile(self.call_log_path): + return [] + with open(self.call_log_path) as fh: + return [line.rstrip("\n") for line in fh] + + +class _RunInThread: + """ + Runs a callable on a background daemon thread so a test can bound how long it + waits without risking a real multi-hour hang if send_to_fortress doesn't honor a + small stall bound (i.e., silently falls back to the 7200s/2h create default). + """ + + def __init__(self, fn, *args, **kwargs): + self.result = {} + + def _target(): + try: + self.result["value"] = fn(*args, **kwargs) + except BaseException as e: # noqa: BLE001 - re-raised via .error + self.result["error"] = e + + self.thread = threading.Thread(target=_target, daemon=True) + + def start_and_join(self, timeout): + self.thread.start() + self.thread.join(timeout) + return not self.thread.is_alive() + + +class TestSendToFortressRetrieveStallConfig(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.d, ignore_errors=True) + self.source_folder = os.path.join(self.d, "src") + os.makedirs(self.source_folder) + with open(os.path.join(self.source_folder, "a.txt"), "wb") as fh: + fh.write(b"hello world") + self.tmp_dir = os.path.join(self.d, "tmp") + self.log_dir = os.path.join(self.d, "logs") + os.makedirs(self.tmp_dir) + + zip_path, zip_checksum, file_checksums, members, source_bytes, exclusion = \ + archive.make_zip_files(self.source_folder, ".*", self.tmp_dir, "testproj") + self.zip_path = zip_path + self.zip_checksum = zip_checksum + self.file_checksums = file_checksums + self.members = members + self.source_bytes = source_bytes + self.exclusion = exclusion + + def _send(self, retrieve_stall_seconds, **kwargs): + # smtplib.SMTP is mocked to fail instantly: send_to_fortress emails on every + # failure path, and a real (unmocked) attempt to an unreachable smtp.purdue.edu + # would add its own unpredictable delay on top of what these tests already + # bound carefully. send_email's own try/except swallows the induced error. + with mock.patch("smtplib.SMTP", side_effect=OSError("network disabled for test")): + return archive.send_to_fortress( + self.zip_path, self.zip_checksum, self.file_checksums, self.members, + self.source_bytes, self.source_folder, ".*", "/tape/base", + ["ops@example.com"], "testproj", self.log_dir, + False, self.exclusion, retrieve_stall_seconds, **kwargs) + + def test_hsi_get_retrieve_is_bounded_by_configured_value_not_2h_default(self): + # hsi get sleeps in total silence well past our tiny configured bound — + # correct code kills it quickly; buggy code (hardcoded 7200s) would still + # be running (and would eventually get a rc=0 that has produced no file). + env = FakeFortressEnv(get_body="sleep 8; exit 0") + with env: + runner = _RunInThread(self._send, 1) + finished = runner.start_and_join(timeout=_CREATE_POLL_SLACK_SECONDS) + calls = env.calls() + + self.assertTrue( + finished, + f"send_to_fortress did not honor the small retrieve_stall_seconds for " + f"hsi get — still running after {_CREATE_POLL_SLACK_SECONDS}s, looks " + f"like it fell back to the 2h create-side default") + self.assertIn("error", runner.result) + self.assertIsInstance(runner.result["error"], RuntimeError) + self.assertIn("stalled", str(runner.result["error"])) + # The call log's printf runs BEFORE the fake's `sleep 8`, so it is written + # even though the process is killed mid-sleep — confirms it really was the + # "get" invocation that stalled, not the create or anything else. + self.assertTrue(any(c.startswith('hsi get ') for c in calls), + f"expected a stalled hsi get call, got: {calls}") + + def test_tar_extract_retrieve_is_bounded_by_configured_value_not_2h_default(self): + # hsi get succeeds fast (writes the local "tar" immediately); the LOCAL tar + # xvf extract step is the one that hangs silently this time. + env = FakeFortressEnv(tar_body="sleep 8; exit 0") + with env: + runner = _RunInThread(self._send, 1) + finished = runner.start_and_join(timeout=_CREATE_POLL_SLACK_SECONDS) + calls = env.calls() + + self.assertTrue( + finished, + f"send_to_fortress did not honor the small retrieve_stall_seconds for " + f"the tar extract step — still running after {_CREATE_POLL_SLACK_SECONDS}s") + self.assertIn("error", runner.result) + self.assertIsInstance(runner.result["error"], RuntimeError) + self.assertIn("stalled", str(runner.result["error"])) + self.assertTrue(any(c.startswith('tar xvf ') for c in calls), + f"expected a stalled tar xvf call, got: {calls}") + + def test_create_keeps_its_own_default_unaffected_by_small_retrieve_value(self): + # htar_large stays silent for longer than our tiny configured retrieve + # bound before writing anything — if create wrongly used + # retrieve_stall_seconds instead of its own hardcoded default, it would be + # killed well before the 2s mark; the correct behavior is for it to still + # be running (blocked in the create step) at 1.3s. + env = FakeFortressEnv(create_body="sleep 2; echo HTAR_SUCCESSFUL; exit 0") + with env: + runner = _RunInThread(self._send, 1) # tiny bound — only matters if + # (incorrectly) applied to create + still_running = not runner.start_and_join(timeout=1.3) + self.assertTrue( + still_running, + "create was already finished (and likely killed) well before its " + "own silence window elapsed — looks like it used the small " + "retrieve_stall_seconds instead of its own hardcoded default") + # Wait for the background thread to fully finish WHILE the fake + # bins/PATH are still in place (run_watched's own poll_interval, not + # overridable, means detecting the create's completion can itself take + # up to ~30s — see _CREATE_POLL_SLACK_SECONDS). Joining outside the + # `with` block would let env.__exit__ rip out the fake PATH out from + # under a still-running thread, which could then race a later test's + # own FakeFortressEnv over the shared os.environ["PATH"]. + runner.thread.join(_CREATE_POLL_SLACK_SECONDS) + self.assertFalse( + runner.thread.is_alive(), + "background send_to_fortress call did not finish in time — " + "would otherwise leak past this test and race a later test's PATH") + + def test_absent_bound_raises_before_any_tape_work(self): + env = FakeFortressEnv() + with env: + with self.assertRaises(RuntimeError) as ctx: + self._send(None) + calls = env.calls() + self.assertIn("retrieve_stall_seconds", str(ctx.exception)) + self.assertEqual(calls, [], "must raise before any htar_large/hsi/tar call") + + def test_nonpositive_or_wrongly_typed_bound_raises(self): + for bad in (0, -1, "21600", 1.5, True): + env = FakeFortressEnv() + with env: + with self.assertRaises(RuntimeError, msg=f"accepted {bad!r}"): + self._send(bad) + self.assertEqual( + env.calls(), [], + f"must raise before any tape work for bad={bad!r}") + + +if __name__ == "__main__": + unittest.main()