From 179d65dc8469d75f04a43a2297a70f8f3c1b8add Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Mon, 13 Jul 2026 19:45:58 -0400 Subject: [PATCH 1/6] Add temporary GLANCE DB3 trace pilot --- .gitignore | 6 + DB3_UPLOAD_PIPELINE_REVIEW.md | 123 ++++ api/data_loader_pg.py | 747 ++++++++++++++++++++++ api/glance_db3/__init__.py | 31 + api/glance_db3/trace_import.py | 566 ++++++++++++++++ api/main.py | 6 +- api/routers/dataset_v2.py | 43 +- api/routers/upload.py | 331 +++++++++- api/scripts/add_equipment_runs.sql | 60 ++ api/tests/test_glance_db3_trace_import.py | 652 +++++++++++++++++++ geddes/k8s/05-postgres.yaml | 94 +++ geddes/k8s/postgres/schema_v2.sql | 44 ++ web/app/data/catalog/[id]/page.tsx | 88 ++- web/app/data/catalog/page.tsx | 26 +- web/app/data/upload/page.tsx | 107 +++- web/components/run-trace-explorer.tsx | 494 ++++++++++++++ web/lib/api-client.ts | 93 ++- 17 files changed, 3448 insertions(+), 63 deletions(-) create mode 100644 DB3_UPLOAD_PIPELINE_REVIEW.md create mode 100644 api/glance_db3/__init__.py create mode 100644 api/glance_db3/trace_import.py create mode 100644 api/tests/test_glance_db3_trace_import.py create mode 100644 web/components/run-trace-explorer.tsx diff --git a/.gitignore b/.gitignore index e8f91a9..94b4bdf 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ debug_*.py !api/tests/test_phase3_security.py !api/tests/test_e2e_admin_flow.py !api/tests/test_e2e_platform.py +!api/tests/test_glance_db3_trace_import.py # Development files temp.py @@ -90,6 +91,11 @@ process_etcher_data/OUTPUT_DATASET.csv *.db *.sqlite *.sqlite3 +*.db3 + +# Local source exports and conversion scratch data. GLANCE DB3 files can contain +# proprietary equipment traces and must never be committed. +/incoming/ # Cache directories .cache/ diff --git a/DB3_UPLOAD_PIPELINE_REVIEW.md b/DB3_UPLOAD_PIPELINE_REVIEW.md new file mode 100644 index 0000000..caed031 --- /dev/null +++ b/DB3_UPLOAD_PIPELINE_REVIEW.md @@ -0,0 +1,123 @@ +# Temporary GLANCE DB3 Test Import + +## Decision confirmed with Paul + +One catalog row represents one physical GLANCE machine run. It does **not** +represent one timestamped sample. + +Every sample row and every parameter value from that run is retained in a linked +time-series trace. The source absolute timestamp remains authoritative. Relative +seconds are calculated only when a user reads or plots one or more traces, with +each run independently starting at its earliest sample. + +No input/output summaries are invented in this branch. The physical catalog run +has empty scalar input/output objects until Paul and the team define scientifically +valid summary rules. + +## Test-only scope + +Uploading a `.db3` is a temporary way to test this data model and plotting flow. +It is opt-in and must not be treated as the long-term GLANCE integration. A future +production integration should read from the authoritative GLANCE service or an +approved export pipeline rather than depend on users moving SQLite files around. + +DB3 upload is disabled unless `DT_DB3_EQUIPMENT_TOOL_MAP` explicitly maps the +selected registered equipment ID to the GLANCE tool name embedded in the export: + +```json +{ + "": ["VLN-11304-CTC-PM1"] +} +``` + +This mapping is intentional. It prevents an export from one machine from being +silently attached to another machine as more GLANCE tools are tested. The default +DB3 limit is 64 MB and can be adjusted with `DT_DB3_UPLOAD_MAX_MB`. + +## Workflow + +1. The user selects an approved equipment registration and uploads one `.db3`. +2. The API streams the file to upload storage in bounded chunks. It does not read + the complete DB3 into one in-memory bytes object and does not preprocess inside + the async request handler. +3. The upload appears as `success`. The user selects a disposable test project and + clicks **Process**. +4. The existing synchronous background task reads SQLite in read-only mode and + extracts one object per physical run. A malformed run is reported and skipped + without discarding other valid physical runs. +5. One database transaction upserts the physical catalog parents and replaces all + linked samples. A retry is idempotent by source upload plus GLANCE run ID. +6. The run detail page retrieves a trace through + `GET /dataset/v2/runs/{run_id}/trace` and can compare up to six runs using an + absolute/source or dynamically calculated relative time axis. + +The old 52 generated input/output CSV records are gone. A single DB3 now requires +one Process action. + +## Persisted model + +`equipment_runs` stores one parent per physical run, including: + +- lot name for display; +- GLANCE run, tool, and recipe identifiers; +- tool, recipe, material, start/end, and status provenance; +- source filename, upload ID, size, and SHA-256; +- parameter metadata keyed by the stable GLANCE parameter ID (`p`). + +`equipment_run_trace_samples` stores every sample linked to its parent: + +- source sample-record ID; +- source GLANCE tool ID recorded on the sample; +- deterministic sample index; +- exact `sample_time_raw` text; +- parsed timezone-naive wall time for ordering and relative-time arithmetic; +- a JSON object containing every value present at that timestamp. + +The source export does not declare a timezone, so the API reports +`timestamp_timezone: null` and never labels the values UTC. Missing values remain +null and empty sample rows remain `{}` so plots show real gaps. + +Lot name is not a technical identity. Paul's sample contains repeated lot names, +including many physical runs named `FM-2'`. The durable source identity is the +data-upload UUID plus GLANCE `idruns`; lot name remains searchable display data. + +## Paul's sample expectation + +For `incoming/paul_db3_pipeline/MyDatabaseSample.db3`, the extractor asserts: + +- 27 rows in the source `runs` table; +- 26 physical runs with sample records, therefore 26 catalog parents; +- 11,697 linked sample rows; +- 11,696 samples with values and one retained empty-value sample; +- 242 source parameter definitions; +- 795,328 parameter values; +- one source tool: `VLN-11304-CTC-PM1`. + +These counts intentionally differ from the discarded CSV conversion, which +removed the empty sample and expanded samples into catalog rows. + +## Verification + +The focused extractor test builds a small SQLite fixture at runtime. It covers +duplicate lot names, fractional and irregular timestamps, missing values, an +empty-value sample, a no-sample run, source checksum/provenance, and per-run +failure isolation. + +```bash +api/.venv/bin/python -m pytest -q api/tests/test_glance_db3_trace_import.py +api/.venv/bin/python -m py_compile \ + api/glance_db3/trace_import.py \ + api/data_loader_pg.py \ + api/routers/upload.py \ + api/routers/dataset_v2.py + +cd web +npx tsc --noEmit +npm run lint +``` + +The local `incoming/` directory and all `*.db3` files are ignored because source +exports can contain proprietary equipment traces. No generated CSV/intermediate +directories are created. Upload storage is not currently a persistent Kubernetes +volume, so the database trace is the durable test result; the uploaded DB3 itself +must be treated as a temporary processing artifact. diff --git a/api/data_loader_pg.py b/api/data_loader_pg.py index d3682a3..a3b5617 100644 --- a/api/data_loader_pg.py +++ b/api/data_loader_pg.py @@ -164,6 +164,11 @@ def _shape_run_record(row) -> Dict[str, Any]: raw_payload = rec.pop("raw_payload_json", None) if raw_payload is not None: rec["raw_payload"] = raw_payload + # Canonical etcher rows do not use the generic GLANCE trace table. Keeping + # these keys in the shared response shape lets clients identify trace-backed + # generic runs without special-casing an absent property. + rec.setdefault("source_run_id", None) + rec.setdefault("trace_sample_count", 0) return rec @@ -413,6 +418,92 @@ def ensure_equipment_runs_pg() -> None: conn.close() +def ensure_equipment_run_trace_samples_pg() -> None: + """Create the per-sample trace table used by generic equipment runs. + + GLANCE timestamps currently arrive without a timezone. ``sample_time`` is + therefore deliberately ``TIMESTAMP WITHOUT TIME ZONE``: it supports sorting + and relative-time arithmetic without claiming an offset the source did not + provide. ``sample_time_raw`` preserves the exact source text (including any + future fractional seconds or explicit offset) for provenance and display. + """ + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_index INT NOT NULL, + source_sample_id BIGINT NOT NULL, + source_tool_id BIGINT, + sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + sample_time_raw TEXT NOT NULL, + values_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, sample_index), + UNIQUE (equipment_run_id, source_sample_id), + CHECK (jsonb_typeof(values_json) = 'object') + ) + """ + ) + cur.execute( + "ALTER TABLE equipment_run_trace_samples " + "ADD COLUMN IF NOT EXISTS source_tool_id BIGINT" + ) + cur.execute( + """ + CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time + ON equipment_run_trace_samples( + equipment_run_id, sample_time, sample_index + ) + """ + ) + cur.execute( + "ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY" + ) + cur.execute("GRANT SELECT ON equipment_run_trace_samples TO api_client") + cur.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_trace_samples' + AND policyname = 'equipment_run_trace_visibility_policy' + ) THEN + CREATE POLICY equipment_run_trace_visibility_policy + ON equipment_run_trace_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; + END + $$; + """ + ) + conn.commit() + finally: + conn.close() + + def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: """SELECT that shapes generic equipment_runs into the same record format used by the etcher /runs endpoints (so the catalog can render both).""" @@ -431,6 +522,11 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: r.upload_filename, r.source, r.project_id, + NULLIF(r.raw_payload_json #>> '{{glance,run_id}}', '') AS source_run_id, + NULLIF( + r.raw_payload_json #>> '{{glance,run_start_time}}', '' + ) AS source_run_start_time, + COALESCE(trace_counts.sample_count, 0)::int AS trace_sample_count, ''::text AS execution_request_id, p.name AS project_name, r.equipment_id AS equipment_id, @@ -441,6 +537,11 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: FROM equipment_runs r LEFT JOIN projects p ON p.id = r.project_id LEFT JOIN equipment_metadata em ON em.domain_id = r.equipment_id + LEFT JOIN LATERAL ( + SELECT COUNT(*)::int AS sample_count + FROM equipment_run_trace_samples trace_sample + WHERE trace_sample.equipment_run_id = r.id + ) trace_counts ON true """ @@ -455,6 +556,13 @@ def _shape_equipment_run_record(row) -> Dict[str, Any]: if hasattr(rec[ts_field], "isoformat") else str(rec[ts_field]) ) + source_run_start_time = rec.pop("source_run_start_time", None) + if rec.get("source") == "glance_db3" and source_run_start_time: + # equipment_runs.run_date is TIMESTAMPTZ, so inserting a naive GLANCE + # wall time makes Postgres attach the session timezone. Present the exact + # source text instead; raw trace metadata explicitly reports that its + # timezone is unknown rather than fabricating an offset for the catalog. + rec["run_date"] = str(source_run_start_time) # Expose the validated input set-points under the same "features" key the # etcher rows use, and surface measured outputs alongside. rec["features"] = rec.pop("inputs_json", {}) or {} @@ -464,6 +572,17 @@ def _shape_equipment_run_record(row) -> Dict[str, Any]: rec.setdefault("avg_etch_rate", None) rec.setdefault("range_etch_rate", None) rec.setdefault("range_nm", None) + source_run_id = rec.get("source_run_id") + if source_run_id not in (None, ""): + try: + rec["source_run_id"] = int(source_run_id) + except (TypeError, ValueError): + # GLANCE uses numeric ids today, but preserving an unexpected source + # value is safer than hiding provenance from a future importer. + rec["source_run_id"] = str(source_run_id) + else: + rec["source_run_id"] = None + rec["trace_sample_count"] = int(rec.get("trace_sample_count") or 0) rec["equipment_name"] = rec.get("equipment_name") or rec.get("equipment_id") or "" raw_payload = rec.pop("raw_payload_json", None) if raw_payload is not None: @@ -558,6 +677,420 @@ def sync_equipment_runs_pg( conn.close() +def _normalize_trace_parameter_key(value: Any) -> str: + """Return a stable JSON key for a GLANCE parameter identifier.""" + text = str(value or "").strip() + if not text: + return "" + match = re.fullmatch(r"[pP]?(\d+)", text) + if match: + return f"p{int(match.group(1))}" + return text + + +def _parse_trace_sample_time(value: Any, *, source_sample_id: int) -> tuple[datetime, str]: + """Parse a source timestamp without inventing a timezone. + + The exact input text is returned alongside a timezone-naive datetime for the + Postgres ``TIMESTAMP WITHOUT TIME ZONE`` column. If a future source includes + an explicit offset, normalize the arithmetic value to UTC before dropping + ``tzinfo`` while retaining the exact offset-bearing source text. Naive source + values remain naive; no equipment timezone is guessed. + """ + if isinstance(value, datetime): + raw = value.isoformat() + parsed = value + else: + raw = str(value or "").strip() + if not raw: + raise ValueError( + f"GLANCE sample {source_sample_id} is missing an absolute timestamp" + ) + candidate = raw[:-1] + "+00:00" if raw.endswith("Z") else raw + try: + parsed = datetime.fromisoformat(candidate) + except ValueError as exc: + raise ValueError( + f"GLANCE sample {source_sample_id} has an invalid timestamp {raw!r}" + ) from exc + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc) + return parsed.replace(tzinfo=None), raw + + +def sync_equipment_run_traces_pg( + *, + equipment_id: str, + project_id: str, + upload_id: str, + upload_filename: str, + runs: List[Dict[str, Any]], +) -> int: + """Atomically upsert physical GLANCE runs and their complete sample traces. + + One item in ``runs`` becomes one ``equipment_runs`` catalog record. Each item + carries ``source_run_id``, optional scalar ``inputs``/``outputs`` and ``raw`` + provenance, ``parameters`` metadata, and ``samples``. A sample accepts + ``source_sample_id`` (or ``sample_record_id``), ``sample_time_raw`` (or + ``sample_time``/``timestamp``), and a ``values`` mapping keyed by stable + parameter ids such as ``p130``. + + Parent rows and samples share one transaction. For DB3 imports ``row_index`` + stores the GLANCE source run id, making ``(upload_id, row_index)`` the stable + ``(upload, source run)`` identity. Reprocessing replaces traces in place, + preserves catalog run ids even when another source run is skipped or later + recovers, and removes stale rows from an earlier conversion. + """ + from psycopg2.extras import execute_values + + if not str(equipment_id or "").strip(): + raise ValueError("equipment_id is required for GLANCE trace ingestion") + if not str(project_id or "").strip(): + raise ValueError("project_id is required for GLANCE trace ingestion") + if not str(upload_id or "").strip(): + raise ValueError("upload_id is required for idempotent GLANCE trace ingestion") + if not runs: + raise ValueError("No physical GLANCE runs with trace samples were found") + + normalized_runs: list[dict[str, Any]] = [] + seen_source_run_ids: set[int] = set() + + for run in runs: + source_run_value = run.get("source_run_id", run.get("run_id")) + try: + source_run_id = int(source_run_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"Invalid or missing GLANCE source run id: {source_run_value!r}" + ) from exc + if source_run_id in seen_source_run_ids: + raise ValueError(f"Duplicate GLANCE source run id {source_run_id}") + seen_source_run_ids.add(source_run_id) + + raw_parameters = run.get("parameters", run.get("trace_parameters", [])) or [] + parameters: list[dict[str, Any]] = [] + parameter_by_key: dict[str, dict[str, Any]] = {} + for parameter in raw_parameters: + if not isinstance(parameter, dict): + raise ValueError( + f"GLANCE run {source_run_id} contains invalid parameter metadata" + ) + source_parameter_id = parameter.get( + "source_parameter_id", + parameter.get("source_id", parameter.get("idparameters")), + ) + key = _normalize_trace_parameter_key( + parameter.get("key", source_parameter_id) + ) + if not key: + raise ValueError( + f"GLANCE run {source_run_id} contains a parameter without a stable key" + ) + if key in parameter_by_key: + raise ValueError( + f"GLANCE run {source_run_id} contains duplicate parameter key {key!r}" + ) + metadata = { + "key": key, + "name": str( + parameter.get( + "name", + parameter.get("display_name", parameter.get("source_name", key)), + ) + or key + ), + "unit": str(parameter.get("unit") or ""), + } + if source_parameter_id not in (None, ""): + try: + metadata["source_parameter_id"] = int(source_parameter_id) + except (TypeError, ValueError): + metadata["source_parameter_id"] = str(source_parameter_id) + source_name = parameter.get("source_name") + if source_name not in (None, ""): + metadata["source_name"] = str(source_name) + parameter_by_key[key] = metadata + parameters.append(metadata) + + raw_samples = run.get("samples", run.get("trace_samples", [])) or [] + if not raw_samples: + raise ValueError(f"GLANCE run {source_run_id} has no trace samples") + samples: list[dict[str, Any]] = [] + seen_source_sample_ids: set[int] = set() + seen_sample_indices: set[int] = set() + for fallback_index, sample in enumerate(raw_samples): + if not isinstance(sample, dict): + raise ValueError( + f"GLANCE run {source_run_id} contains an invalid trace sample" + ) + sample_id_value = sample.get( + "source_sample_id", sample.get("sample_record_id") + ) + try: + source_sample_id = int(sample_id_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"GLANCE run {source_run_id} has an invalid sample id {sample_id_value!r}" + ) from exc + if source_sample_id in seen_source_sample_ids: + raise ValueError( + f"GLANCE run {source_run_id} repeats sample id {source_sample_id}" + ) + seen_source_sample_ids.add(source_sample_id) + + source_tool_value = sample.get( + "source_tool_id", sample.get("glance_tool_id") + ) + if source_tool_value in (None, ""): + source_tool_id = None + else: + try: + source_tool_id = int(source_tool_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"GLANCE sample {source_sample_id} has an invalid source tool id " + f"{source_tool_value!r}" + ) from exc + if not ( + -9_223_372_036_854_775_808 + <= source_tool_id + <= 9_223_372_036_854_775_807 + ): + raise ValueError( + f"GLANCE sample {source_sample_id} source tool id is outside " + "the PostgreSQL BIGINT range" + ) + + sample_index_value = sample.get("sample_index", fallback_index) + try: + sample_index = int(sample_index_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"GLANCE sample {source_sample_id} has invalid index {sample_index_value!r}" + ) from exc + if sample_index < 0 or sample_index in seen_sample_indices: + raise ValueError( + f"GLANCE run {source_run_id} has invalid or duplicate sample index {sample_index}" + ) + seen_sample_indices.add(sample_index) + + raw_time_value = sample.get( + "sample_time_raw", + sample.get("sample_time", sample.get("timestamp")), + ) + sample_time, sample_time_raw = _parse_trace_sample_time( + raw_time_value, + source_sample_id=source_sample_id, + ) + raw_values = sample.get("values", sample.get("values_json", {})) or {} + if not isinstance(raw_values, dict): + raise ValueError( + f"GLANCE sample {source_sample_id} values must be an object" + ) + values: dict[str, Any] = {} + for raw_key, raw_value in raw_values.items(): + key = _normalize_trace_parameter_key(raw_key) + if not key: + raise ValueError( + f"GLANCE sample {source_sample_id} contains a blank parameter key" + ) + if key in values: + raise ValueError( + f"GLANCE sample {source_sample_id} repeats parameter key {key!r}" + ) + values[key] = raw_value + if key not in parameter_by_key: + fallback_metadata = {"key": key, "name": key, "unit": ""} + parameter_by_key[key] = fallback_metadata + parameters.append(fallback_metadata) + samples.append( + { + "sample_index": sample_index, + "source_sample_id": source_sample_id, + "source_tool_id": source_tool_id, + "sample_time": sample_time, + "sample_time_raw": sample_time_raw, + "values": _json_safe_payload(values), + } + ) + + samples.sort(key=lambda sample: sample["sample_index"]) + raw_payload_value = run.get("raw", run.get("raw_payload", {})) or {} + if not isinstance(raw_payload_value, dict): + raise ValueError(f"GLANCE run {source_run_id} raw provenance must be an object") + raw_payload = _json_safe_payload(raw_payload_value) + glance_metadata = raw_payload.get("glance") + if not isinstance(glance_metadata, dict): + glance_metadata = {} + glance_metadata["run_id"] = source_run_id + source_run_start_time = run.get("run_start_time") + if source_run_start_time in (None, ""): + source_run_start_time = glance_metadata.get("run_start_time") + if source_run_start_time in (None, ""): + source_run_start_time = run.get("run_date") + if source_run_start_time not in (None, ""): + glance_metadata["run_start_time"] = ( + source_run_start_time.isoformat() + if hasattr(source_run_start_time, "isoformat") + else str(source_run_start_time) + ) + raw_payload["glance"] = glance_metadata + + trace_metadata = raw_payload.get("trace") + if not isinstance(trace_metadata, dict): + trace_metadata = {} + trace_metadata.update( + { + "sample_count": len(samples), + "parameters": parameters, + "timestamp_timezone": run.get("timestamp_timezone"), + "start_time": min(samples, key=lambda sample: sample["sample_time"])[ + "sample_time_raw" + ], + "end_time": max(samples, key=lambda sample: sample["sample_time"])[ + "sample_time_raw" + ], + } + ) + raw_payload["trace"] = trace_metadata + raw_payload["source_upload_id"] = str(upload_id) + raw_payload["source_db3_filename"] = str(upload_filename or "") + + normalized_runs.append( + { + "source_run_id": source_run_id, + "lotname": str(run.get("lotname", run.get("lot_name", "")) or ""), + "run_date": run.get("run_date", run.get("run_start_time")) or None, + "is_outlier": bool(run.get("is_outlier", False)), + "is_calibration_recipe": bool( + run.get("is_calibration_recipe", False) + ), + "outlier_type": str(run.get("outlier_type", "") or ""), + "inputs": _json_safe_payload(run.get("inputs", {}) or {}), + "outputs": _json_safe_payload(run.get("outputs", {}) or {}), + "raw": raw_payload, + "samples": samples, + } + ) + + # Stable ordering keeps each source run attached to the same upload row and + # therefore the same exposed catalog id on a retry. + normalized_runs.sort(key=lambda run: run["source_run_id"]) + + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", + (f"equipment_run_trace_sync|{upload_id}",), + ) + parent_values = [] + for run in normalized_runs: + parent_values.append( + ( + equipment_id, + project_id, + run["lotname"], + run["run_date"], + run["is_outlier"], + run["is_calibration_recipe"], + run["outlier_type"], + Json(run["inputs"]), + Json(run["outputs"]), + Json(run["raw"]), + upload_id, + run["source_run_id"], + upload_filename, + "glance_db3", + ) + ) + + returned_parents = execute_values( + cur, + """ + INSERT INTO equipment_runs ( + equipment_id, project_id, lotname, run_date, + is_outlier, is_calibration_recipe, outlier_type, + inputs_json, outputs_json, raw_payload_json, + upload_id, row_index, upload_filename, source + ) VALUES %s + ON CONFLICT (upload_id, row_index) WHERE upload_id IS NOT NULL + DO UPDATE SET + equipment_id = EXCLUDED.equipment_id, + project_id = EXCLUDED.project_id, + lotname = EXCLUDED.lotname, + run_date = EXCLUDED.run_date, + is_outlier = EXCLUDED.is_outlier, + is_calibration_recipe = EXCLUDED.is_calibration_recipe, + outlier_type = EXCLUDED.outlier_type, + inputs_json = EXCLUDED.inputs_json, + outputs_json = EXCLUDED.outputs_json, + raw_payload_json = EXCLUDED.raw_payload_json, + upload_filename = EXCLUDED.upload_filename, + source = EXCLUDED.source + RETURNING id, row_index + """, + parent_values, + fetch=True, + ) + parent_id_by_source_run = { + int(source_run_id): int(parent_id) + for parent_id, source_run_id in returned_parents + } + if len(parent_id_by_source_run) != len(normalized_runs): + raise RuntimeError("Could not resolve every GLANCE catalog run after upsert") + + parent_ids = list(parent_id_by_source_run.values()) + cur.execute( + "DELETE FROM equipment_run_trace_samples " + "WHERE equipment_run_id = ANY(%s)", + (parent_ids,), + ) + trace_values = [] + for run in normalized_runs: + parent_id = parent_id_by_source_run[run["source_run_id"]] + for sample in run["samples"]: + trace_values.append( + ( + parent_id, + sample["sample_index"], + sample["source_sample_id"], + sample["source_tool_id"], + sample["sample_time"], + sample["sample_time_raw"], + Json(sample["values"]), + ) + ) + execute_values( + cur, + """ + INSERT INTO equipment_run_trace_samples ( + equipment_run_id, sample_index, source_sample_id, source_tool_id, + sample_time, sample_time_raw, values_json + ) VALUES %s + """, + trace_values, + page_size=500, + ) + # Cascading deletion removes samples belonging to a stale physical + # run if a deterministic re-conversion yields fewer runs. + cur.execute( + """ + DELETE FROM equipment_runs + WHERE upload_id = %s + AND NOT (id = ANY(%s)) + """, + (upload_id, parent_ids), + ) + conn.commit() + return len(normalized_runs) + except Exception: + conn.rollback() + raise + finally: + conn.close() + + def merge_equipment_runs_pg( *, equipment_id: str, @@ -975,6 +1508,220 @@ def get_run_detail_pg( logger.error(f"Error fetching run {run_id} from Postgres: {e}") return None + +def get_equipment_run_trace_pg( + run_id: int, + *, + parameters: Optional[List[str]] = None, + nanohub_user_id: Optional[str] = None, + is_admin: bool = False, +) -> Optional[Dict[str, Any]]: + """Fetch a generic equipment run's complete trace under catalog RLS. + + Absolute source timestamp text is always returned. ``relative_seconds`` is + derived at query time from the earliest sample in the complete run, before + optional parameter selection, so selecting a sparse parameter can never move + the run's zero point. Every sample remains in the response; missing selected + values are represented as JSON null to preserve plot gaps. + """ + if run_id < EQUIPMENT_RUN_ID_OFFSET: + return None + + requested_keys: list[str] = [] + seen_requested: set[str] = set() + for requested in parameters or []: + # Accept both repeated query parameters and a comma-separated value for + # simple clients while exposing one canonical list internally. + for value in str(requested or "").split(","): + key = _normalize_trace_parameter_key(value) + if key and key not in seen_requested: + requested_keys.append(key) + seen_requested.add(key) + + conn = get_pg_superuser_connection() if is_admin else get_pg_connection(nanohub_user_id) + try: + internal_run_id = run_id - EQUIPMENT_RUN_ID_OFFSET + with conn.cursor(cursor_factory=RealDictCursor) as cur: + # Selecting the parent first applies equipment_runs RLS and avoids + # distinguishing a private run from one that does not exist. + cur.execute( + """ + SELECT + r.id, + r.lotname, + r.raw_payload_json, + r.source + FROM equipment_runs r + WHERE r.id = %s + LIMIT 1 + """, + (internal_run_id,), + ) + parent = cur.fetchone() + if not parent: + conn.commit() + return None + + cur.execute( + """ + SELECT + sample_index, + source_sample_id, + source_tool_id, + sample_time, + sample_time_raw, + values_json, + EXTRACT( + EPOCH FROM ( + sample_time - MIN(sample_time) OVER () + ) + )::double precision AS relative_seconds + FROM equipment_run_trace_samples + WHERE equipment_run_id = %s + ORDER BY sample_time, sample_index + """, + (internal_run_id,), + ) + sample_rows = [dict(row) for row in cur.fetchall()] + conn.commit() + + if not sample_rows: + return None + + raw_payload = parent.get("raw_payload_json") or {} + if not isinstance(raw_payload, dict): + raw_payload = {} + glance_metadata = raw_payload.get("glance") or {} + if not isinstance(glance_metadata, dict): + glance_metadata = {} + trace_metadata = raw_payload.get("trace") or {} + if not isinstance(trace_metadata, dict): + trace_metadata = {} + + parameter_records: list[dict[str, Any]] = [] + parameter_by_key: dict[str, dict[str, Any]] = {} + for raw_parameter in trace_metadata.get("parameters", []) or []: + if not isinstance(raw_parameter, dict): + continue + key = _normalize_trace_parameter_key( + raw_parameter.get( + "key", + raw_parameter.get( + "source_parameter_id", raw_parameter.get("source_id") + ), + ) + ) + if not key or key in parameter_by_key: + continue + parameter = { + "key": key, + "name": str( + raw_parameter.get( + "name", raw_parameter.get("display_name", key) + ) + or key + ), + "unit": str(raw_parameter.get("unit") or ""), + } + if raw_parameter.get("source_parameter_id") not in (None, ""): + parameter["source_parameter_id"] = raw_parameter[ + "source_parameter_id" + ] + if raw_parameter.get("source_name") not in (None, ""): + parameter["source_name"] = str(raw_parameter["source_name"]) + parameter_by_key[key] = parameter + parameter_records.append(parameter) + + # Be resilient to older trace rows whose parameter metadata was missing: + # stable JSON keys still make the values retrievable and self-consistent. + for sample_row in sample_rows: + values = sample_row.get("values_json") or {} + if not isinstance(values, dict): + values = {} + normalized_values: dict[str, Any] = {} + for raw_key, value in values.items(): + key = _normalize_trace_parameter_key(raw_key) + if not key: + continue + normalized_values[key] = value + if key not in parameter_by_key: + parameter = {"key": key, "name": key, "unit": ""} + parameter_by_key[key] = parameter + parameter_records.append(parameter) + sample_row["values_json"] = normalized_values + + if requested_keys: + unknown = [key for key in requested_keys if key not in parameter_by_key] + if unknown: + raise ValueError( + "Unknown trace parameter key(s): " + ", ".join(unknown) + ) + selected_keys = requested_keys + response_parameters = [parameter_by_key[key] for key in selected_keys] + else: + selected_keys = [parameter["key"] for parameter in parameter_records] + response_parameters = parameter_records + + samples: list[dict[str, Any]] = [] + for sample_row in sample_rows: + values = sample_row["values_json"] + raw_timestamp = str(sample_row.get("sample_time_raw") or "").strip() + if not raw_timestamp: + parsed_timestamp = sample_row.get("sample_time") + raw_timestamp = ( + parsed_timestamp.isoformat() + if hasattr(parsed_timestamp, "isoformat") + else str(parsed_timestamp or "") + ) + samples.append( + { + "sample_index": int(sample_row["sample_index"]), + "sample_record_id": int(sample_row["source_sample_id"]), + "source_tool_id": ( + int(sample_row["source_tool_id"]) + if sample_row.get("source_tool_id") is not None + else None + ), + "timestamp": raw_timestamp, + "relative_seconds": float(sample_row["relative_seconds"] or 0.0), + "values": {key: values.get(key) for key in selected_keys}, + } + ) + + source_run_value = glance_metadata.get("run_id") + try: + source_run_id: Any = int(source_run_value) + except (TypeError, ValueError): + source_run_id = source_run_value + + min_row = min(sample_rows, key=lambda row: row["sample_time"]) + max_row = max(sample_rows, key=lambda row: row["sample_time"]) + return { + "run_id": run_id, + "source_run_id": source_run_id, + "lot_name": str(parent.get("lotname") or ""), + "sample_count": len(samples), + "start_time": str(min_row.get("sample_time_raw") or ""), + "end_time": str(max_row.get("sample_time_raw") or ""), + "timestamp_timezone": trace_metadata.get("timestamp_timezone"), + "parameters": response_parameters, + "samples": samples, + } + except MissingRuntimeConfiguration as exc: + logger.error( + "Postgres runtime configuration missing while fetching run trace: %s", + exc, + ) + raise RuntimeError("Postgres credentials are not configured") from exc + except ValueError: + raise + except Exception as exc: + logger.error("Error fetching trace for run %s: %s", run_id, exc) + return None + finally: + conn.close() + + def get_projects_list_pg(nanohub_user_id: Optional[str] = None, is_admin: bool = False) -> List[Dict[str, Any]]: """ Fetch projects from Postgres. diff --git a/api/glance_db3/__init__.py b/api/glance_db3/__init__.py new file mode 100644 index 0000000..ba9b2e0 --- /dev/null +++ b/api/glance_db3/__init__.py @@ -0,0 +1,31 @@ +"""Read temporary GLANCE DB3 test uploads as physical runs with full traces.""" + +from .trace_import import ( + Db3TraceExtraction, + Db3TraceImportResult, + GlanceDb3Error, + GlanceDb3SchemaError, + GlanceParameter, + GlanceRunTrace, + GlanceSourceTool, + GlanceTraceSample, + RunTraceFailure, + extract_db3_trace, + extract_db3_run_traces, + parameter_key, +) + +__all__ = [ + "Db3TraceExtraction", + "Db3TraceImportResult", + "GlanceDb3Error", + "GlanceDb3SchemaError", + "GlanceParameter", + "GlanceRunTrace", + "GlanceSourceTool", + "GlanceTraceSample", + "RunTraceFailure", + "extract_db3_trace", + "extract_db3_run_traces", + "parameter_key", +] diff --git a/api/glance_db3/trace_import.py b/api/glance_db3/trace_import.py new file mode 100644 index 0000000..bdcbc7f --- /dev/null +++ b/api/glance_db3/trace_import.py @@ -0,0 +1,566 @@ +"""Read GLANCE SQLite exports as one complete time-series trace per run. + +This module deliberately does not conform GLANCE data to the platform's +row-oriented CSV upload templates. A GLANCE run is the catalog-level object; +its sample records remain an attached trace. Source timestamps and parameter +names are therefore retained exactly as SQLite returned them. +""" + +from __future__ import annotations + +import base64 +import hashlib +import math +import sqlite3 +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any + + +_REQUIRED_TABLES = frozenset( + {"runs", "samplerecord", "data", "parameters", "tools", "recipes"} +) + + +class GlanceDb3Error(ValueError): + """Base error for a DB3 file that cannot be extracted safely.""" + + +class GlanceDb3SchemaError(GlanceDb3Error): + """Raised when a file is SQLite but not the expected GLANCE schema.""" + + +def _optional_text(value: Any) -> str | None: + """Return SQLite text without normalizing timestamps or other metadata.""" + if value is None: + return None + return value if isinstance(value, str) else str(value) + + +def _required_int(value: Any, field_name: str) -> int: + if value is None or isinstance(value, bool): + raise GlanceDb3Error(f"{field_name} must be an integer, got {value!r}") + try: + numeric = Decimal(str(value)) + except (InvalidOperation, TypeError, ValueError) as exc: + raise GlanceDb3Error( + f"{field_name} must be an integer, got {value!r}" + ) from exc + if not numeric.is_finite() or numeric != numeric.to_integral_value(): + raise GlanceDb3Error(f"{field_name} must be an integer, got {value!r}") + return int(numeric) + + +def _optional_int(value: Any, field_name: str) -> int | None: + if value is None: + return None + return _required_int(value, field_name) + + +def parameter_key(parameter_id: int) -> str: + """Return the stable, source-ID-based key used in every ``values_json``.""" + return f"p{parameter_id}" + + +def _json_value(value: Any) -> Any: + """Convert uncommon SQLite values without making the result non-JSON-safe.""" + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float): + # Strict JSON has no NaN or infinities. Preserve their source meaning as + # text rather than silently converting them to null. + return value if math.isfinite(value) else str(value) + if isinstance(value, bytes): + return "base64:" + base64.b64encode(value).decode("ascii") + return str(value) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _connect_readonly(path: Path) -> sqlite3.Connection: + # Path.as_uri() safely escapes spaces and URI metacharacters in filenames. + connection = sqlite3.connect( + f"{path.resolve().as_uri()}?mode=ro&immutable=1", + uri=True, + ) + connection.row_factory = sqlite3.Row + return connection + + +@dataclass(frozen=True) +class GlanceParameter: + key: str + glance_parameter_id: int + raw_name: str | None + unit: str | None + + def to_dict(self) -> dict[str, Any]: + return { + "key": self.key, + "glance_parameter_id": self.glance_parameter_id, + "raw_name": self.raw_name, + "unit": self.unit, + } + + def to_sync_dict(self) -> dict[str, Any]: + return { + "key": self.key, + "source_parameter_id": self.glance_parameter_id, + # No display-name transformation is performed during extraction, so + # the source name is also the honest initial UI label. + "name": self.raw_name, + "source_name": self.raw_name, + "unit": self.unit, + } + + +@dataclass(frozen=True) +class GlanceSourceTool: + glance_tool_id: int + name: str | None + description: str | None + + def to_dict(self) -> dict[str, Any]: + return { + "glance_tool_id": self.glance_tool_id, + "name": self.name, + "description": self.description, + } + + +@dataclass(frozen=True) +class GlanceTraceSample: + glance_sample_record_id: int + glance_run_id: int + glance_tool_id: int | None + sample_time: str | None + values_json: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + return { + "glance_sample_record_id": self.glance_sample_record_id, + "glance_run_id": self.glance_run_id, + "glance_tool_id": self.glance_tool_id, + "sample_time": self.sample_time, + "values_json": dict(self.values_json), + } + + def to_sync_dict(self, sample_index: int) -> dict[str, Any]: + return { + "source_sample_id": self.glance_sample_record_id, + "sample_index": sample_index, + "source_tool_id": self.glance_tool_id, + "sample_time_raw": self.sample_time, + "values_json": dict(self.values_json), + } + + +@dataclass(frozen=True) +class GlanceRunTrace: + glance_run_id: int + glance_tool_id: int | None + tool_name: str | None + lot_name: str | None + material_name: str | None + glance_recipe_id: int | None + recipe_name: str | None + run_start_time: str | None + run_end_time: str | None + run_status: str | None + parameter_metadata: tuple[GlanceParameter, ...] + samples: tuple[GlanceTraceSample, ...] + + @property + def sample_count(self) -> int: + return len(self.samples) + + def provenance_dict(self) -> dict[str, Any]: + """Return compact run metadata suitable for ``raw_payload_json``.""" + return { + "glance_run_id": self.glance_run_id, + "glance_tool_id": self.glance_tool_id, + "tool_name": self.tool_name, + "lot_name": self.lot_name, + "material_name": self.material_name, + "glance_recipe_id": self.glance_recipe_id, + "recipe_name": self.recipe_name, + "run_start_time": self.run_start_time, + "run_end_time": self.run_end_time, + "run_status": self.run_status, + "sample_count": self.sample_count, + } + + def to_dict(self) -> dict[str, Any]: + result = self.provenance_dict() + result["parameter_metadata"] = [ + parameter.to_dict() for parameter in self.parameter_metadata + ] + result["samples"] = [sample.to_dict() for sample in self.samples] + return result + + def to_sync_dict(self, project_id: str | None = None) -> dict[str, Any]: + """Shape this run for the dedicated trace persistence backend.""" + return { + "project_id": project_id, + "source_run_id": self.glance_run_id, + "lotname": self.lot_name or "", + "run_start_time": self.run_start_time, + "run_date": self.run_start_time, + "inputs": {}, + "outputs": {}, + "raw": {"glance": self.provenance_dict()}, + "parameters": [ + parameter.to_sync_dict() for parameter in self.parameter_metadata + ], + "samples": [ + sample.to_sync_dict(sample_index) + for sample_index, sample in enumerate(self.samples) + ], + "source": "glance_db3", + } + + +@dataclass(frozen=True) +class RunTraceFailure: + glance_run_id: int | str | None + error: str + + def to_dict(self) -> dict[str, Any]: + return { + "glance_run_id": self.glance_run_id, + "error": self.error, + } + + +@dataclass(frozen=True) +class Db3TraceImportResult: + source_filename: str + source_size_bytes: int + source_sha256: str + source_tools: tuple[GlanceSourceTool, ...] + parameters: tuple[GlanceParameter, ...] + runs: tuple[GlanceRunTrace, ...] + failures: tuple[RunTraceFailure, ...] + + @property + def run_count(self) -> int: + return len(self.runs) + + @property + def sample_count(self) -> int: + return sum(run.sample_count for run in self.runs) + + def to_dict(self) -> dict[str, Any]: + return { + "source_filename": self.source_filename, + "source_size_bytes": self.source_size_bytes, + "source_sha256": self.source_sha256, + "run_count": self.run_count, + "sample_count": self.sample_count, + "source_tools": [tool.to_dict() for tool in self.source_tools], + "parameters": [parameter.to_dict() for parameter in self.parameters], + "runs": [run.to_dict() for run in self.runs], + "failures": [failure.to_dict() for failure in self.failures], + } + + +def _validate_schema(connection: sqlite3.Connection) -> None: + rows = connection.execute( + "SELECT name FROM sqlite_master WHERE type IN ('table', 'view')" + ).fetchall() + table_names = {str(row["name"]) for row in rows} + missing = sorted(_REQUIRED_TABLES - table_names) + if missing: + raise GlanceDb3SchemaError( + "DB3 file is missing required GLANCE tables: " + ", ".join(missing) + ) + + +def _load_parameters( + connection: sqlite3.Connection, +) -> tuple[tuple[GlanceParameter, ...], dict[int, GlanceParameter]]: + rows = connection.execute( + """ + SELECT idparameters, name, unit + FROM parameters + ORDER BY idparameters + """ + ).fetchall() + parameters: list[GlanceParameter] = [] + by_id: dict[int, GlanceParameter] = {} + for row in rows: + parameter_id = _required_int(row["idparameters"], "parameters.idparameters") + if parameter_id in by_id: + raise GlanceDb3SchemaError( + f"Duplicate parameters.idparameters value: {parameter_id}" + ) + parameter = GlanceParameter( + key=parameter_key(parameter_id), + glance_parameter_id=parameter_id, + raw_name=_optional_text(row["name"]), + unit=_optional_text(row["unit"]), + ) + parameters.append(parameter) + by_id[parameter_id] = parameter + return tuple(parameters), by_id + + +def _load_source_tools( + connection: sqlite3.Connection, +) -> tuple[GlanceSourceTool, ...]: + rows = connection.execute( + """ + SELECT idtools, name, description + FROM tools + ORDER BY idtools + """ + ).fetchall() + tools: list[GlanceSourceTool] = [] + seen_ids: set[int] = set() + for row in rows: + tool_id = _required_int(row["idtools"], "tools.idtools") + if tool_id in seen_ids: + raise GlanceDb3SchemaError(f"Duplicate tools.idtools value: {tool_id}") + seen_ids.add(tool_id) + tools.append( + GlanceSourceTool( + glance_tool_id=tool_id, + name=_optional_text(row["name"]), + description=_optional_text(row["description"]), + ) + ) + return tuple(tools) + + +def _run_failure_id(value: Any) -> int | str | None: + if value is None: + return None + try: + return _required_int(value, "runs.idruns") + except GlanceDb3Error: + return str(value) + + +def _extract_run( + connection: sqlite3.Connection, + run_row: sqlite3.Row, + parameter_by_id: dict[int, GlanceParameter], +) -> GlanceRunTrace: + raw_run_id = run_row["idruns"] + run_id = _required_int(raw_run_id, "runs.idruns") + run_tool_id = _optional_int(run_row["idtools"], "runs.idtools") + recipe_id = _optional_int(run_row["idrecipes"], "runs.idrecipes") + + rows = connection.execute( + """ + SELECT + s.idsamplerecord, + s.idruns, + s.idtools AS sample_tool_id, + s.time AS sample_time, + d.idsamplerecord AS data_sample_record_id, + d.idparameters, + d.value + FROM samplerecord s + LEFT JOIN data d ON d.idsamplerecord = s.idsamplerecord + WHERE s.idruns = ? + ORDER BY s.time, s.idsamplerecord, d.idparameters + """, + (raw_run_id,), + ).fetchall() + if not rows: + # The caller selects only runs with samples. Treat a disappearing run as + # a source consistency failure instead of creating an empty catalog run. + raise GlanceDb3Error(f"GLANCE run {run_id} has no sample rows") + + samples: list[GlanceTraceSample] = [] + used_parameter_ids: set[int] = set() + row_index = 0 + while row_index < len(rows): + first = rows[row_index] + sample_id = _required_int( + first["idsamplerecord"], "samplerecord.idsamplerecord" + ) + sample_run_id = _required_int(first["idruns"], "samplerecord.idruns") + if sample_run_id != run_id: + raise GlanceDb3Error( + f"Sample {sample_id} belongs to run {sample_run_id}, expected {run_id}" + ) + + values: dict[str, Any] = {} + while row_index < len(rows): + row = rows[row_index] + current_sample_id = _required_int( + row["idsamplerecord"], "samplerecord.idsamplerecord" + ) + if current_sample_id != sample_id: + break + if row["data_sample_record_id"] is not None: + parameter_id = _required_int( + row["idparameters"], "data.idparameters" + ) + key = parameter_key(parameter_id) + if key in values: + raise GlanceDb3Error( + f"Sample {sample_id} contains duplicate parameter {parameter_id}" + ) + values[key] = _json_value(row["value"]) + used_parameter_ids.add(parameter_id) + if parameter_id not in parameter_by_id: + parameter_by_id[parameter_id] = GlanceParameter( + key=key, + glance_parameter_id=parameter_id, + raw_name=None, + unit=None, + ) + row_index += 1 + + samples.append( + GlanceTraceSample( + glance_sample_record_id=sample_id, + glance_run_id=run_id, + glance_tool_id=_optional_int( + first["sample_tool_id"], "samplerecord.idtools" + ), + sample_time=_optional_text(first["sample_time"]), + values_json=values, + ) + ) + + used_parameters = tuple( + parameter_by_id[parameter_id] + for parameter_id in sorted(used_parameter_ids) + ) + return GlanceRunTrace( + glance_run_id=run_id, + glance_tool_id=run_tool_id, + tool_name=_optional_text(run_row["toolname"]), + lot_name=_optional_text(run_row["lotname"]), + material_name=_optional_text(run_row["materialname"]), + glance_recipe_id=recipe_id, + recipe_name=_optional_text(run_row["recipename"]), + run_start_time=_optional_text(run_row["starttime"]), + run_end_time=_optional_text(run_row["endtime"]), + run_status=_optional_text(run_row["status"]), + parameter_metadata=used_parameters, + samples=tuple(samples), + ) + + +def extract_db3_trace(db3_path: Path) -> Db3TraceImportResult: + """Extract one full, unaggregated trace for every GLANCE run with samples. + + Runs are isolated during extraction: malformed metadata or trace rows for one + run produce a ``RunTraceFailure`` while other valid runs are still returned. + File-level failures (not SQLite, missing schema, invalid parameter registry) + raise ``GlanceDb3Error`` because no run can be interpreted safely. + """ + path = Path(db3_path) + if not path.is_file(): + raise FileNotFoundError(f"DB3 file not found: {path}") + + source_size_bytes = path.stat().st_size + source_sha256 = _sha256(path) + try: + connection = _connect_readonly(path) + except sqlite3.Error as exc: + raise GlanceDb3Error(f"Could not open DB3 file: {exc}") from exc + + try: + try: + _validate_schema(connection) + parameters, parameter_by_id = _load_parameters(connection) + source_tools = _load_source_tools(connection) + run_rows = connection.execute( + """ + SELECT + r.idruns, + r.idtools, + r.lotname, + r.materialname, + r.idrecipes, + r.starttime, + r.endtime, + r.status, + t.name AS toolname, + rc.recipename + FROM runs r + LEFT JOIN tools t ON t.idtools = r.idtools + LEFT JOIN recipes rc ON rc.idrecipes = r.idrecipes + WHERE EXISTS ( + SELECT 1 + FROM samplerecord s + WHERE s.idruns = r.idruns + ) + ORDER BY r.idruns + """ + ).fetchall() + except sqlite3.Error as exc: + raise GlanceDb3SchemaError( + f"Could not read expected GLANCE schema: {exc}" + ) from exc + + traces: list[GlanceRunTrace] = [] + failures: list[RunTraceFailure] = [] + for run_row in run_rows: + try: + traces.append(_extract_run(connection, run_row, parameter_by_id)) + except (GlanceDb3Error, sqlite3.Error) as exc: + failures.append( + RunTraceFailure( + glance_run_id=_run_failure_id(run_row["idruns"]), + error=str(exc), + ) + ) + + # Include parameters discovered in data even if a malformed source did + # not declare them in the parameters table. Their raw metadata remains + # null, but their p values are not lost. + all_parameters = tuple( + parameter_by_id[parameter_id] + for parameter_id in sorted(parameter_by_id) + ) + return Db3TraceImportResult( + source_filename=path.name, + source_size_bytes=source_size_bytes, + source_sha256=source_sha256, + source_tools=source_tools, + parameters=all_parameters, + runs=tuple(traces), + failures=tuple(failures), + ) + finally: + connection.close() + + +def extract_db3_run_traces(db3_path: str | Path) -> Db3TraceImportResult: + """Backward-compatible alias for the canonical ``extract_db3_trace`` API.""" + return extract_db3_trace(Path(db3_path)) + + +# Keep the earlier development name import-compatible while callers migrate to +# the explicit ``Db3TraceImportResult`` name. +Db3TraceExtraction = Db3TraceImportResult + + +__all__ = [ + "Db3TraceExtraction", + "Db3TraceImportResult", + "GlanceDb3Error", + "GlanceDb3SchemaError", + "GlanceParameter", + "GlanceRunTrace", + "GlanceSourceTool", + "GlanceTraceSample", + "RunTraceFailure", + "extract_db3_trace", + "extract_db3_run_traces", + "parameter_key", +] diff --git a/api/main.py b/api/main.py index 318d629..fb0357a 100644 --- a/api/main.py +++ b/api/main.py @@ -44,6 +44,8 @@ "project_members", "etcher_runs", "etcher_run_files", + "equipment_runs", + "equipment_run_trace_samples", "samples", "experiment_samples", "run_samples", @@ -141,6 +143,7 @@ def _run_startup_migrations() -> None: """ from data_loader_pg import get_pg_superuser_connection from data_loader_pg import ensure_etcher_run_file_refs_pg + from data_loader_pg import ensure_equipment_run_trace_samples_pg from data_loader_pg import ensure_equipment_runs_pg from data_loader_pg import ensure_etcher_recipe_name_column_pg from metadata_pg import ( @@ -168,8 +171,9 @@ def _run_startup_migrations() -> None: lock_conn.commit() ensure_etcher_run_file_refs_pg() - ensure_equipment_runs_pg() ensure_etcher_recipe_name_column_pg() + ensure_equipment_runs_pg() + ensure_equipment_run_trace_samples_pg() ensure_experiment_proposal_generation_columns_pg() ensure_experiment_type_reuse_columns_pg() ensure_experiment_type_versioning_columns_pg() diff --git a/api/routers/dataset_v2.py b/api/routers/dataset_v2.py index 53f0768..fe1901e 100644 --- a/api/routers/dataset_v2.py +++ b/api/routers/dataset_v2.py @@ -8,7 +8,7 @@ from datetime import datetime, timezone from typing import Literal, Optional -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query from fastapi import Request from fastapi.responses import JSONResponse from pydantic import BaseModel @@ -18,6 +18,7 @@ add_project_member_pg, create_project_pg, delete_project_pg, + get_equipment_run_trace_pg, get_projects_list_pg, get_run_detail_pg, get_runs_list_pg, @@ -756,7 +757,7 @@ def run_detail( platform_user: PlatformUser = Depends(get_platform_user), ): """ - Return a single etcher_runs row by ``idruns``, joined with project context. + Return one visible canonical or generic catalog run with project context. RLS applies for non-admins; admins bypass via the superuser connection. Returns 404 when the run does not exist OR the caller can't see it — we @@ -778,6 +779,44 @@ def run_detail( return record +@router.get("/runs/{run_id}/trace") +def run_trace( + run_id: int, + parameters: Optional[list[str]] = Query(default=None), + platform_user: PlatformUser = Depends(get_platform_user), +): + """Return the complete timestamped trace linked to a generic equipment run. + + ``parameters`` may be repeated (``?parameters=p130¶meters=p132``) to + reduce each sample's values object. Absolute source timestamps are always + returned, and relative seconds are computed dynamically from the earliest + sample in the complete run. Missing values remain null so plots preserve + gaps instead of connecting across them. + """ + is_admin = platform_user.role == "admin" + logger.info( + "[dataset/v2/runs/%s/trace] Fetching trace for user=%s role=%s parameters=%s", + run_id, + platform_user.id, + platform_user.role, + parameters or [], + ) + try: + record = get_equipment_run_trace_pg( + run_id=run_id, + parameters=parameters, + nanohub_user_id=platform_user.id, + is_admin=is_admin, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if record is None: + # As with run detail, do not reveal whether the run is absent, private, + # canonical (non-trace), or simply has no linked trace. + raise HTTPException(status_code=404, detail=f"Trace for run {run_id} not found") + return record + + @router.post("/runs/sync") async def sync_runs( request: Request, diff --git a/api/routers/upload.py b/api/routers/upload.py index f9f938a..4c8ccaa 100644 --- a/api/routers/upload.py +++ b/api/routers/upload.py @@ -1,9 +1,10 @@ """ -Data upload router — stores CSV/Excel files with lightweight structural checks. +Data upload router — stores CSV/Excel files and the temporary GLANCE DB3 test path. """ import csv -import io import hashlib +import io +import json import logging import math import os @@ -16,12 +17,14 @@ from fastapi.responses import StreamingResponse from psycopg2.extras import Json from pydantic import BaseModel +from starlette.concurrency import run_in_threadpool from domain_configs import derive_expected_columns, load_domain_config from data_loader_pg import ( get_pg_connection, get_projects_list_pg, merge_equipment_runs_pg, + sync_equipment_run_traces_pg, sync_equipment_runs_pg, sync_runs_pg, ) @@ -32,6 +35,7 @@ list_uploads_pg, ) from security import PlatformUser, get_platform_user +from glance_db3.trace_import import extract_db3_trace logger = logging.getLogger(__name__) @@ -111,6 +115,86 @@ def resolve_uploads_dir() -> Path: # Storage directory for uploaded files UPLOADS_DIR = resolve_uploads_dir() +CSV_EXCEL_UPLOAD_LIMIT_BYTES = 50 * 1024 * 1024 +DB3_UPLOAD_CHUNK_BYTES = 1024 * 1024 + + +def _db3_upload_limit_bytes() -> int: + raw = os.getenv("DT_DB3_UPLOAD_MAX_MB", "64").strip() + try: + max_mb = int(raw) + except ValueError: + max_mb = 64 + return max(max_mb, 1) * 1024 * 1024 + + +def _db3_allowed_source_tools(equipment_id: str) -> set[str]: + """Return the explicitly configured GLANCE source tools for equipment. + + DB3 ingestion is intentionally a temporary, opt-in test path. Requiring an + equipment-to-tool map prevents a source export from being silently attached + to the wrong registered machine as more GLANCE tools are introduced. + """ + raw = os.getenv("DT_DB3_EQUIPMENT_TOOL_MAP", "").strip() + if not raw: + raise ValueError( + "DB3 uploads are disabled until DT_DB3_EQUIPMENT_TOOL_MAP maps this " + "equipment ID to its GLANCE source tool name." + ) + try: + configured = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("DT_DB3_EQUIPMENT_TOOL_MAP is not valid JSON.") from exc + if not isinstance(configured, dict): + raise ValueError("DT_DB3_EQUIPMENT_TOOL_MAP must be a JSON object.") + + values = configured.get(equipment_id) + if isinstance(values, str): + values = [values] + if not isinstance(values, list): + raise ValueError( + f"DB3 uploads are not enabled for equipment '{equipment_id}'." + ) + allowed = {str(value).strip() for value in values if str(value).strip()} + if not allowed: + raise ValueError( + f"DB3 uploads are not enabled for equipment '{equipment_id}'." + ) + return allowed + + +def _copy_upload_with_limit(source: Any, destination: Path, max_bytes: int) -> int: + """Stream an uploaded file to disk without buffering the whole DB3.""" + total = 0 + try: + source.seek(0) + with destination.open("wb") as target: + while True: + chunk = source.read(DB3_UPLOAD_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise ValueError("too_large") + target.write(chunk) + except Exception: + destination.unlink(missing_ok=True) + raise + return total + + +def _validate_db3_timestamp(value: Any, field_name: str) -> str: + """Validate an ISO source timestamp while returning its exact text.""" + raw = str(value or "").strip() + if not raw: + raise ValueError(f"{field_name} is missing") + candidate = raw[:-1] + "+00:00" if raw.endswith("Z") else raw + try: + datetime.fromisoformat(candidate) + except ValueError as exc: + raise ValueError(f"{field_name} is invalid: {raw!r}") from exc + return raw + QUDT_UNIT_LABEL_BY_CODE = { "CentiM3-PER-MIN": "sccm", "MilliTORR": "mTorr", @@ -836,21 +920,99 @@ async def upload_file( # Check extension ext = Path(file.filename).suffix.lower() - if ext not in (".csv", ".xlsx", ".xls"): + if ext not in (".csv", ".xlsx", ".xls", ".db3"): raise HTTPException( status_code=400, - detail=f"Unsupported file type '{ext}'. Use CSV, XLSX, or XLS." + detail=f"Unsupported file type '{ext}'. Use CSV, XLSX, XLS, or DB3." + ) + + equipment_upload_dir = UPLOADS_DIR / equipment_id + equipment_upload_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + original_name = Path(file.filename).name + safe_name = f"{timestamp}_{original_name}" + dest = equipment_upload_dir / safe_name + + if ext == ".db3": + try: + _db3_allowed_source_tools(equipment_id) + except ValueError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + + max_size = _db3_upload_limit_bytes() + try: + size_bytes = await run_in_threadpool( + _copy_upload_with_limit, + file.file, + dest, + max_size, + ) + except ValueError as exc: + if str(exc) == "too_large": + max_mb = max_size // (1024 * 1024) + raise HTTPException( + status_code=400, + detail=f"File exceeds {max_mb} MB limit", + ) from exc + raise + + if size_bytes == 0: + dest.unlink(missing_ok=True) + raise HTTPException(status_code=400, detail="File is empty") + + upload_record = create_upload_record_pg( + equipment_id=equipment_id, + filename=original_name, + stored_as=safe_name, + storage_path=str(dest), + size_bytes=size_bytes, + row_count=0, + columns=[], + errors=[], + status="success", + user=user, + kind="all", + ) + + logger.info( + "Stored DB3 upload %s (%s bytes) for equipment %s; catalog import is pending", + original_name, + size_bytes, + equipment_id, ) - # Read file contents + return { + "status": "success", + "upload_id": upload_record["upload_id"], + "equipment_id": equipment_id, + "kind": "all", + "filename": original_name, + "stored_as": safe_name, + "size_bytes": size_bytes, + "row_count": 0, + "columns": [], + "errors": None, + "message": ( + "DB3 stored for test import. Choose a project and click Process; " + "each physical GLANCE run will become one catalog run with its " + "full timestamped trace attached." + ), + "processing_status": "stored_only", + } + + # CSV/Excel validation still needs the in-memory content, but these uploads + # retain the existing, much smaller 50 MB limit. Large DB3 files use the + # streaming branch above and never become one giant Python bytes object. contents = await file.read() size_bytes = len(contents) - if size_bytes == 0: raise HTTPException(status_code=400, detail="File is empty") + if size_bytes > CSV_EXCEL_UPLOAD_LIMIT_BYTES: + max_mb = CSV_EXCEL_UPLOAD_LIMIT_BYTES // (1024 * 1024) + raise HTTPException(status_code=400, detail=f"File exceeds {max_mb} MB limit") - if size_bytes > 50 * 1024 * 1024: # 50 MB limit - raise HTTPException(status_code=400, detail="File exceeds 50 MB limit") + with dest.open("wb") as stored_file: + stored_file.write(contents) # Parse CSV to validate basic structure. # @@ -949,16 +1111,6 @@ async def upload_file( row_count = -1 columns = [] - # Store the file - equipment_upload_dir = UPLOADS_DIR / equipment_id - equipment_upload_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - safe_name = f"{timestamp}_{file.filename}" - dest = equipment_upload_dir / safe_name - - with open(dest, "wb") as f: - f.write(contents) - # Only hard errors drive failure; warnings are surfaced but stored # alongside so the upload still reaches a processable "success" state. status = "error" if errors else "success" @@ -1505,8 +1657,9 @@ def _execute_upload_processing( can poll for progress instead of blocking on a long ingestion. The canonical etcher routes through sync_runs_pg() (the same sink as Azure). - Split input/output uploads are merged on the lot + run-date key; every other - generic upload is ingested into the equipment_runs table. + Split input/output uploads are merged on the lot + run-date key. A DB3 is + extracted here (in Starlette's background threadpool) into one generic + equipment_runs parent per physical GLANCE run plus linked trace samples. """ equipment_id = upload_record["equipment_id"] kind = _normalize_kind(upload_record.get("kind")) @@ -1520,6 +1673,142 @@ def _fail(errors: list[str], row_count: int) -> None: row_count=row_count, ) + if Path(upload_record["filename"]).suffix.lower() == ".db3": + try: + allowed_tools = _db3_allowed_source_tools(equipment_id) + extraction = extract_db3_trace(Path(upload_record["storage_path"])) + + observed_tools = { + str(run.tool_name).strip() + for run in extraction.runs + if str(run.tool_name or "").strip() + } + run_rows: list[dict[str, Any]] = [] + import_warnings = [ + f"Skipped GLANCE run {failure.glance_run_id}: {failure.error}" + for failure in extraction.failures + ] + for run in extraction.runs: + source_tool_name = str(run.tool_name or "").strip() + if not source_tool_name: + import_warnings.append( + f"Skipped GLANCE run {run.glance_run_id}: source tool name is missing" + ) + continue + if source_tool_name not in allowed_tools: + import_warnings.append( + f"Skipped GLANCE run {run.glance_run_id}: source tool " + f"'{source_tool_name}' is not mapped to equipment '{equipment_id}'" + ) + continue + try: + if not 0 <= run.glance_run_id <= 2_147_483_647: + raise ValueError("source run ID is outside the supported range") + first_sample_time = next( + ( + sample.sample_time + for sample in run.samples + if sample.sample_time + ), + None, + ) + run_date = _validate_db3_timestamp( + run.run_start_time or first_sample_time, + "run start timestamp", + ) + for sample in run.samples: + if not ( + 0 + <= sample.glance_sample_record_id + <= 9_223_372_036_854_775_807 + ): + raise ValueError( + "one or more source sample IDs are outside the supported range" + ) + _validate_db3_timestamp( + sample.sample_time, + f"sample {sample.glance_sample_record_id} timestamp", + ) + parameter_metadata = [ + { + "key": parameter.key, + "source_parameter_id": parameter.glance_parameter_id, + "name": parameter.raw_name or parameter.key, + "source_name": parameter.raw_name, + "unit": parameter.unit or "", + } + for parameter in run.parameter_metadata + ] + raw_payload = { + "glance": run.provenance_dict(), + "source_file": { + "type": "glance_db3", + "filename": extraction.source_filename, + "size_bytes": extraction.source_size_bytes, + "sha256": extraction.source_sha256, + }, + } + run_rows.append( + { + "lotname": run.lot_name or f"GLANCE run {run.glance_run_id}", + "run_date": run_date, + "inputs": {}, + "outputs": {}, + "raw": raw_payload, + "source": "glance_db3", + "source_run_id": run.glance_run_id, + "parameters": parameter_metadata, + "timestamp_timezone": None, + "samples": [ + { + "source_sample_id": sample.glance_sample_record_id, + "source_tool_id": sample.glance_tool_id, + "sample_time_raw": sample.sample_time, + "values": sample.values_json, + } + for sample in run.samples + ], + } + ) + except Exception as exc: # isolate one malformed physical run + import_warnings.append( + f"Skipped GLANCE run {run.glance_run_id}: {exc}" + ) + + if not run_rows: + _fail( + (import_warnings or ["No physical runs with trace samples were found."]), + 0, + ) + return + + inserted = sync_equipment_run_traces_pg( + equipment_id=equipment_id, + project_id=project_id, + upload_id=upload_id, + upload_filename=upload_record["filename"], + runs=run_rows, + ) + except Exception as exc: # noqa: BLE001 - persist an actionable DB3 error + logger.exception("DB3 upload %s import failed", upload_id) + _fail([f"DB3 import failed: {exc}"], 0) + return + + logger.info( + "DB3 upload %s processed: %s physical run(s), %s trace sample(s), source tools=%s", + upload_id, + inserted, + sum(len(row["samples"]) for row in run_rows), + sorted(observed_tools), + ) + _update_upload_record( + upload_id=upload_id, + status="processed", + errors=(prior_warnings + import_warnings)[:50], + row_count=len(run_rows), + ) + return + try: if is_canonical_etcher: rows, errors = _csv_upload_to_sync_rows( @@ -1587,7 +1876,7 @@ def process_upload( user: PlatformUser = Depends(get_platform_user), ): """ - Schedule a stored CSV upload for ingestion into canonical run records. + Schedule a stored CSV or test DB3 upload for ingestion into catalog runs. Validation and ingestion can take minutes for large files, so this endpoint performs only the fast authorization/consistency checks synchronously, marks diff --git a/api/scripts/add_equipment_runs.sql b/api/scripts/add_equipment_runs.sql index b39e5ef..62b287e 100644 --- a/api/scripts/add_equipment_runs.sql +++ b/api/scripts/add_equipment_runs.sql @@ -66,3 +66,63 @@ BEGIN END IF; END $$; + +-- Full time-series traces are stored separately so one equipment_runs row still +-- represents one physical machine run. GLANCE timestamps are currently naive; +-- keep the parsed wall time timezone-free and preserve the exact source text. +CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_index INT NOT NULL, + source_sample_id BIGINT NOT NULL, + source_tool_id BIGINT, + sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + sample_time_raw TEXT NOT NULL, + values_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, sample_index), + UNIQUE (equipment_run_id, source_sample_id), + CHECK (jsonb_typeof(values_json) = 'object') +); + +ALTER TABLE equipment_run_trace_samples +ADD COLUMN IF NOT EXISTS source_tool_id BIGINT; + +CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time + ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); + +ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; + +GRANT SELECT ON equipment_run_trace_samples TO api_client; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_trace_samples' + AND policyname = 'equipment_run_trace_visibility_policy' + ) THEN + CREATE POLICY equipment_run_trace_visibility_policy + ON equipment_run_trace_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; +END +$$; diff --git a/api/tests/test_glance_db3_trace_import.py b/api/tests/test_glance_db3_trace_import.py new file mode 100644 index 0000000..778f640 --- /dev/null +++ b/api/tests/test_glance_db3_trace_import.py @@ -0,0 +1,652 @@ +import hashlib +import io +import json +import os +import sqlite3 +import sys +import unittest +from datetime import datetime +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +from glance_db3.trace_import import Db3TraceImportResult, extract_db3_trace +from data_loader_pg import ( + EQUIPMENT_RUN_ID_OFFSET, + _equipment_runs_base_select, + get_equipment_run_trace_pg, + _normalize_trace_parameter_key, + _parse_trace_sample_time, + sync_equipment_run_traces_pg, +) +from routers import upload as upload_router + + +def _write_fixture( + path: Path, + *, + include_bad_run: bool = False, + include_bad_timestamp_run: bool = False, +) -> None: + connection = sqlite3.connect(path) + try: + connection.executescript( + """ + CREATE TABLE runs ( + idruns NUMBRIC PRIMARY KEY, + idtools NUMBRIC, + lotname TEXT, + idrecipes NUMBRIC, + starttime TEXT, + endtime TEXT, + status TEXT, + materialname TEXT + ); + CREATE TABLE samplerecord ( + idsamplerecord NUMBRIC PRIMARY KEY, + idruns NUMBRIC, + idtools NUMBRIC, + time TEXT + ); + CREATE TABLE data ( + idsamplerecord NUMBRIC, + idparameters NUMBRIC, + value NUMBRIC, + PRIMARY KEY (idsamplerecord, idparameters) + ); + CREATE TABLE parameters ( + idparameters NUMBRIC PRIMARY KEY, + name TEXT, + unit TEXT + ); + CREATE TABLE tools ( + idtools NUMBRIC PRIMARY KEY, + name TEXT, + description TEXT + ); + CREATE TABLE recipes ( + idrecipes NUMBRIC PRIMARY KEY, + idtools NUMBRIC, + recipename TEXT, + recipefile TEXT, + hash TEXT, + timestamp TEXT + ); + """ + ) + connection.execute( + "INSERT INTO tools (idtools, name) VALUES (?, ?)", + (5, "VLN-11304-CTC-PM1"), + ) + connection.execute( + "INSERT INTO recipes (idrecipes, idtools, recipename) VALUES (?, ?, ?)", + (9, 5, "O2 clean-Engineering"), + ) + connection.executemany( + "INSERT INTO parameters (idparameters, name, unit) VALUES (?, ?, ?)", + [ + (1, "G Gas 1 Flow", "sccm"), + (2, "P Chamber Pressure", "mTorr"), + (7, "Unused Parameter", "V"), + ], + ) + connection.executemany( + """ + INSERT INTO runs ( + idruns, idtools, lotname, idrecipes, starttime, endtime, + status, materialname + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + ( + 101, + 5, + "LOT-REUSED", + 9, + "2026-07-01 10:00:00.000001", + "2026-07-01 10:00:10.999999", + "processed", + "Silicon", + ), + ( + 102, + 5, + "LOT-REUSED", + 9, + "2026-07-02 11:00:00.100000", + "2026-07-02 11:00:01.200000", + "processed", + "Silicon", + ), + ( + 103, + 5, + "NO-SAMPLES", + 9, + "2026-07-03 12:00:00", + "2026-07-03 12:01:00", + "aborted", + "Silicon", + ), + ( + 104, + 5, + "EMPTY-TRACE", + 9, + "2026-07-03 14:00:00.000010", + "2026-07-03 14:00:01.000020", + "processed", + "Silicon", + ), + ], + ) + connection.executemany( + """ + INSERT INTO samplerecord (idsamplerecord, idruns, idtools, time) + VALUES (?, ?, ?, ?) + """, + [ + # IDs are deliberately out of timestamp order. + (1001, 101, 5, "2026-07-01 10:00:02.875000"), + (1002, 101, 5, "2026-07-01 10:00:00.125000"), + # This intentionally has no data rows and must still survive. + (1003, 101, 5, "2026-07-01 10:00:09.000001"), + (2001, 102, 5, "2026-07-02 11:00:00.333333"), + # A run with samples but no values is still a physical run. + (4001, 104, 5, "2026-07-03 14:00:00.500015"), + ], + ) + connection.executemany( + "INSERT INTO data (idsamplerecord, idparameters, value) VALUES (?, ?, ?)", + [ + (1001, 1, 1.25), + # Explicit SQL null is distinct from an absent p2 key. + (1001, 2, None), + (1002, 2, 44.5), + (2001, 1, 2.5), + ], + ) + if include_bad_run: + connection.execute( + """ + INSERT INTO runs ( + idruns, idtools, lotname, idrecipes, starttime, endtime, + status, materialname + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + 300.5, + 5, + "MALFORMED", + 9, + "2026-07-04 13:00:00", + "2026-07-04 13:01:00", + "processed", + "Silicon", + ), + ) + connection.execute( + """ + INSERT INTO samplerecord (idsamplerecord, idruns, idtools, time) + VALUES (?, ?, ?, ?) + """, + (3001, 300.5, 5, "2026-07-04 13:00:00.555555"), + ) + if include_bad_timestamp_run: + connection.execute( + """ + INSERT INTO runs ( + idruns, idtools, lotname, idrecipes, starttime, endtime, + status, materialname + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + 105, + 5, + "BAD-TIMESTAMP", + 9, + "not-a-timestamp", + "2026-07-05 13:01:00", + "processed", + "Silicon", + ), + ) + connection.execute( + """ + INSERT INTO samplerecord (idsamplerecord, idruns, idtools, time) + VALUES (?, ?, ?, ?) + """, + (5001, 105, 5, "also-not-a-timestamp"), + ) + connection.commit() + finally: + connection.close() + + +class GlanceDb3TraceImportTests(unittest.TestCase): + def test_generic_catalog_select_renders_glance_json_paths(self) -> None: + query = _equipment_runs_base_select(include_raw_payload=True) + self.assertIn("'{glance,run_id}'", query) + self.assertIn("'{glance,run_start_time}'", query) + + def test_extracts_one_complete_trace_per_run_without_aggregation(self) -> None: + with TemporaryDirectory() as temporary_directory: + db3_path = Path(temporary_directory) / "fixture with spaces.db3" + _write_fixture(db3_path) + expected_sha256 = hashlib.sha256(db3_path.read_bytes()).hexdigest() + + result = extract_db3_trace(db3_path) + + self.assertIsInstance(result, Db3TraceImportResult) + self.assertEqual(result.source_filename, db3_path.name) + self.assertEqual(result.source_sha256, expected_sha256) + self.assertEqual( + hashlib.sha256(db3_path.read_bytes()).hexdigest(), + expected_sha256, + ) + self.assertEqual(result.source_size_bytes, db3_path.stat().st_size) + self.assertEqual(result.run_count, 3) + self.assertEqual(result.sample_count, 5) + self.assertEqual(result.failures, ()) + self.assertEqual(len(result.source_tools), 1) + self.assertEqual(result.source_tools[0].glance_tool_id, 5) + self.assertEqual(result.source_tools[0].name, "VLN-11304-CTC-PM1") + + first, second, empty_trace = result.runs + self.assertEqual( + (first.glance_run_id, second.glance_run_id, empty_trace.glance_run_id), + (101, 102, 104), + ) + # Lot names are display metadata and must not merge physical runs. + self.assertEqual(first.lot_name, "LOT-REUSED") + self.assertEqual(second.lot_name, "LOT-REUSED") + self.assertEqual(first.glance_tool_id, 5) + self.assertEqual(first.tool_name, "VLN-11304-CTC-PM1") + self.assertEqual(first.glance_recipe_id, 9) + self.assertEqual(first.recipe_name, "O2 clean-Engineering") + self.assertEqual(first.material_name, "Silicon") + self.assertEqual(first.run_status, "processed") + self.assertEqual(first.run_start_time, "2026-07-01 10:00:00.000001") + self.assertEqual(first.run_end_time, "2026-07-01 10:00:10.999999") + + self.assertEqual( + [sample.sample_time for sample in first.samples], + [ + "2026-07-01 10:00:00.125000", + "2026-07-01 10:00:02.875000", + "2026-07-01 10:00:09.000001", + ], + ) + self.assertEqual( + [sample.glance_sample_record_id for sample in first.samples], + [1002, 1001, 1003], + ) + self.assertEqual(first.samples[0].values_json, {"p2": 44.5}) + self.assertNotIn("p1", first.samples[0].values_json) + self.assertEqual(first.samples[1].values_json, {"p1": 1.25, "p2": None}) + self.assertEqual(first.samples[2].values_json, {}) + self.assertEqual(empty_trace.sample_count, 1) + self.assertEqual(empty_trace.samples[0].values_json, {}) + self.assertEqual(empty_trace.parameter_metadata, ()) + + metadata = {item.key: item for item in first.parameter_metadata} + self.assertEqual(metadata["p1"].raw_name, "G Gas 1 Flow") + self.assertEqual(metadata["p1"].unit, "sccm") + self.assertEqual(metadata["p2"].raw_name, "P Chamber Pressure") + self.assertEqual(metadata["p2"].unit, "mTorr") + self.assertNotIn("p7", metadata) + self.assertEqual( + [parameter.key for parameter in result.parameters], + ["p1", "p2", "p7"], + ) + + serialized = result.to_dict() + json.dumps(serialized, allow_nan=False) + self.assertNotIn("relative_sample_time", json.dumps(serialized)) + self.assertNotIn("run_date", serialized["runs"][0]["samples"][0]) + sync_row = first.to_sync_dict(project_id="project-test") + self.assertEqual(sync_row["project_id"], "project-test") + self.assertEqual(sync_row["lotname"], "LOT-REUSED") + self.assertEqual(sync_row["run_date"], first.run_start_time) + self.assertEqual(sync_row["source"], "glance_db3") + self.assertEqual(sync_row["source_run_id"], 101) + self.assertEqual(sync_row["parameters"][0]["key"], "p1") + self.assertEqual(sync_row["parameters"][0]["source_parameter_id"], 1) + self.assertEqual(sync_row["parameters"][0]["name"], "G Gas 1 Flow") + self.assertEqual(sync_row["samples"][0]["source_sample_id"], 1002) + self.assertEqual(sync_row["samples"][0]["source_tool_id"], 5) + self.assertEqual(sync_row["samples"][0]["sample_index"], 0) + self.assertEqual(sync_row["samples"][2]["values_json"], {}) + + def test_a_malformed_run_does_not_discard_valid_run_traces(self) -> None: + with TemporaryDirectory() as temporary_directory: + db3_path = Path(temporary_directory) / "partial.db3" + _write_fixture(db3_path, include_bad_run=True) + + result = extract_db3_trace(db3_path) + + self.assertEqual( + [run.glance_run_id for run in result.runs], + [101, 102, 104], + ) + self.assertEqual(len(result.failures), 1) + self.assertEqual(result.failures[0].glance_run_id, "300.5") + self.assertIn("runs.idruns must be an integer", result.failures[0].error) + + def test_background_processing_syncs_physical_runs_and_keeps_partial_success( + self, + ) -> None: + with TemporaryDirectory() as temporary_directory: + db3_path = Path(temporary_directory) / "partial.db3" + _write_fixture( + db3_path, + include_bad_run=True, + include_bad_timestamp_run=True, + ) + sync_calls: list[dict] = [] + upload_updates: list[dict] = [] + + def capture_sync(**kwargs): + sync_calls.append(kwargs) + return len(kwargs["runs"]) + + equipment_id = "test-glance-equipment" + tool_map = json.dumps({equipment_id: ["VLN-11304-CTC-PM1"]}) + with ( + patch.dict( + os.environ, + {"DT_DB3_EQUIPMENT_TOOL_MAP": tool_map}, + clear=False, + ), + patch.object( + upload_router, + "sync_equipment_run_traces_pg", + side_effect=capture_sync, + ), + patch.object( + upload_router, + "_update_upload_record", + side_effect=lambda **kwargs: upload_updates.append(kwargs), + ), + ): + upload_router._execute_upload_processing( + upload_id="00000000-0000-0000-0000-000000000001", + upload_record={ + "equipment_id": equipment_id, + "filename": db3_path.name, + "storage_path": str(db3_path), + "kind": "all", + }, + project_id="project-test", + prior_warnings=[], + ) + + self.assertEqual(len(sync_calls), 1) + synced_runs = sync_calls[0]["runs"] + self.assertEqual( + [run["source_run_id"] for run in synced_runs], + [101, 102, 104], + ) + self.assertEqual( + [run["lotname"] for run in synced_runs[:2]], + ["LOT-REUSED", "LOT-REUSED"], + ) + self.assertEqual(synced_runs[0]["parameters"][0]["name"], "G Gas 1 Flow") + self.assertEqual(synced_runs[-1]["samples"][0]["values"], {}) + self.assertEqual(synced_runs[-1]["samples"][0]["source_tool_id"], 5) + self.assertEqual( + synced_runs[0]["raw"]["source_file"]["sha256"], + hashlib.sha256(db3_path.read_bytes()).hexdigest(), + ) + self.assertEqual(upload_updates[-1]["status"], "processed") + self.assertEqual(upload_updates[-1]["row_count"], 3) + self.assertTrue( + any("Skipped GLANCE run 300.5" in warning for warning in upload_updates[-1]["errors"]) + ) + self.assertTrue( + any( + "Skipped GLANCE run 105" in warning + and "timestamp" in warning + for warning in upload_updates[-1]["errors"] + ) + ) + + def test_db3_copy_is_chunked_and_removes_an_oversized_partial_file(self) -> None: + with TemporaryDirectory() as temporary_directory: + destination = Path(temporary_directory) / "stored.db3" + payload = b"db3-test-payload" + copied = upload_router._copy_upload_with_limit( + io.BytesIO(payload), + destination, + len(payload), + ) + self.assertEqual(copied, len(payload)) + self.assertEqual(destination.read_bytes(), payload) + + with self.assertRaisesRegex(ValueError, "too_large"): + upload_router._copy_upload_with_limit( + io.BytesIO(payload), + destination, + len(payload) - 1, + ) + self.assertFalse(destination.exists()) + + def test_trace_sync_uses_source_run_identity_and_normalizes_offset_math(self) -> None: + class FakeCursor: + def __init__(self): + self.executions: list[tuple[str, object]] = [] + + def execute(self, query, params=None): + self.executions.append((query, params)) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class FakeConnection: + def __init__(self): + self.cursor_instance = FakeCursor() + self.committed = False + self.rolled_back = False + self.closed = False + + def cursor(self, cursor_factory=None): + return self.cursor_instance + + def commit(self): + self.committed = True + + def rollback(self): + self.rolled_back = True + + def close(self): + self.closed = True + + connection = FakeConnection() + value_batches: list[tuple[str, list[tuple]]] = [] + + def capture_execute_values(cursor, query, values, **kwargs): + rows = list(values) + value_batches.append((query, rows)) + if "INSERT INTO equipment_runs" in query: + return [ + (7_000 + index, row[11]) + for index, row in enumerate(rows) + ] + return None + + runs = [ + { + "source_run_id": source_run_id, + "lotname": "LOT-REUSED", + "run_start_time": f"2026-07-01 10:00:{source_run_id}.000001", + "run_date": f"2026-07-01 10:00:{source_run_id}.000001", + "inputs": {}, + "outputs": {}, + "raw": {"glance": {"run_start_time": "source-exact"}}, + "parameters": [ + { + "key": "p1", + "source_parameter_id": 1, + "name": "Gas flow", + "unit": "sccm", + } + ], + "samples": [ + { + "source_sample_id": source_run_id * 100, + "source_tool_id": 5, + "sample_time_raw": ( + f"2026-07-01 10:00:{source_run_id}.500001" + ), + "values": {}, + } + ], + } + for source_run_id in (22, 11) + ] + + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=connection, + ), + patch( + "psycopg2.extras.execute_values", + side_effect=capture_execute_values, + ), + ): + inserted = sync_equipment_run_traces_pg( + equipment_id="equipment-test", + project_id="project-test", + upload_id="00000000-0000-0000-0000-000000000002", + upload_filename="source.db3", + runs=runs, + ) + + self.assertEqual(inserted, 2) + parent_rows = value_batches[0][1] + self.assertEqual([row[11] for row in parent_rows], [11, 22]) + trace_rows = value_batches[1][1] + self.assertEqual([row[0] for row in trace_rows], [7_000, 7_001]) + self.assertEqual([row[2] for row in trace_rows], [1_100, 2_200]) + self.assertEqual([row[3] for row in trace_rows], [5, 5]) + self.assertEqual(trace_rows[0][6].adapted, {}) + self.assertTrue(connection.committed) + self.assertFalse(connection.rolled_back) + self.assertTrue(connection.closed) + + self.assertEqual(_normalize_trace_parameter_key("P001"), "p1") + parsed, raw = _parse_trace_sample_time( + "2026-07-01T10:00:00.123456-05:00", + source_sample_id=1, + ) + self.assertEqual(raw, "2026-07-01T10:00:00.123456-05:00") + self.assertEqual(parsed.isoformat(), "2026-07-01T15:00:00.123456") + + def test_trace_read_keeps_empty_samples_as_parameter_gaps(self) -> None: + parent = { + "id": 7, + "lotname": "LOT-TRACE", + "source": "glance_db3", + "raw_payload_json": { + "glance": {"run_id": 101}, + "trace": { + "timestamp_timezone": None, + "parameters": [ + { + "key": "p1", + "name": "Gas flow", + "unit": "sccm", + "source_parameter_id": 1, + } + ], + }, + }, + } + sample_rows = [ + { + "sample_index": 0, + "source_sample_id": 1001, + "source_tool_id": 5, + "sample_time": datetime(2026, 7, 1, 10, 0, 0, 125000), + "sample_time_raw": "2026-07-01 10:00:00.125000", + "values_json": {"p1": 1.25}, + "relative_seconds": 0.0, + }, + { + "sample_index": 1, + "source_sample_id": 1002, + "source_tool_id": 5, + "sample_time": datetime(2026, 7, 1, 10, 0, 2, 875000), + "sample_time_raw": "2026-07-01 10:00:02.875000", + "values_json": {}, + "relative_seconds": 2.75, + }, + ] + + class ReadCursor: + def __init__(self): + self.mode = "" + + def execute(self, query, params=None): + self.mode = "samples" if "trace_samples" in query else "parent" + + def fetchone(self): + return parent if self.mode == "parent" else None + + def fetchall(self): + return sample_rows if self.mode == "samples" else [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class ReadConnection: + def __init__(self): + self.closed = False + + def cursor(self, cursor_factory=None): + return ReadCursor() + + def commit(self): + pass + + def close(self): + self.closed = True + + connection = ReadConnection() + with patch("data_loader_pg.get_pg_connection", return_value=connection): + result = get_equipment_run_trace_pg( + EQUIPMENT_RUN_ID_OFFSET + 7, + parameters=["p1"], + nanohub_user_id="researcher-test", + ) + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(result["source_run_id"], 101) + self.assertEqual(result["sample_count"], 2) + self.assertEqual(result["samples"][0]["source_tool_id"], 5) + self.assertEqual(result["samples"][0]["values"], {"p1": 1.25}) + self.assertEqual(result["samples"][1]["values"], {"p1": None}) + self.assertEqual(result["samples"][1]["relative_seconds"], 2.75) + self.assertEqual( + result["samples"][1]["timestamp"], + "2026-07-01 10:00:02.875000", + ) + self.assertIsNone(result["timestamp_timezone"]) + self.assertTrue(connection.closed) + + +if __name__ == "__main__": + unittest.main() diff --git a/geddes/k8s/05-postgres.yaml b/geddes/k8s/05-postgres.yaml index 1481a62..9d8c28e 100644 --- a/geddes/k8s/05-postgres.yaml +++ b/geddes/k8s/05-postgres.yaml @@ -118,6 +118,100 @@ data: ) ); + -- Generic catalog rows: one row represents one physical equipment run. + CREATE TABLE IF NOT EXISTS equipment_runs ( + id BIGSERIAL PRIMARY KEY, + equipment_id VARCHAR(255) NOT NULL, + project_id VARCHAR(50) REFERENCES projects(id) ON DELETE SET NULL, + lotname VARCHAR(255), + run_date TIMESTAMP WITH TIME ZONE, + is_outlier BOOLEAN DEFAULT false, + is_calibration_recipe BOOLEAN DEFAULT false, + outlier_type TEXT DEFAULT '', + inputs_json JSONB NOT NULL DEFAULT '{}'::jsonb, + outputs_json JSONB NOT NULL DEFAULT '{}'::jsonb, + raw_payload_json JSONB NOT NULL DEFAULT '{}'::jsonb, + upload_id UUID, + row_index INT, + upload_filename VARCHAR(255) DEFAULT '', + source VARCHAR(100) DEFAULT 'data_upload', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + ); + + CREATE INDEX IF NOT EXISTS idx_equipment_runs_equipment_id + ON equipment_runs(equipment_id); + CREATE INDEX IF NOT EXISTS idx_equipment_runs_project_id + ON equipment_runs(project_id); + CREATE INDEX IF NOT EXISTS idx_equipment_runs_upload_id + ON equipment_runs(upload_id); + CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row + ON equipment_runs(upload_id, row_index) + WHERE upload_id IS NOT NULL; + + ALTER TABLE equipment_runs ENABLE ROW LEVEL SECURITY; + GRANT SELECT ON equipment_runs TO api_client; + + CREATE POLICY equipment_run_visibility_policy ON equipment_runs + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM projects p + WHERE p.id = equipment_runs.project_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + + -- Complete GLANCE traces linked to those physical run rows. + CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_index INT NOT NULL, + source_sample_id BIGINT NOT NULL, + source_tool_id BIGINT, + sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + sample_time_raw TEXT NOT NULL, + values_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, sample_index), + UNIQUE (equipment_run_id, source_sample_id), + CHECK (jsonb_typeof(values_json) = 'object') + ); + + CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time + ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); + + ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; + GRANT SELECT ON equipment_run_trace_samples TO api_client; + + CREATE POLICY equipment_run_trace_visibility_policy + ON equipment_run_trace_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + CREATE TABLE IF NOT EXISTS users ( id VARCHAR(255) PRIMARY KEY, name VARCHAR(255) NOT NULL, diff --git a/geddes/k8s/postgres/schema_v2.sql b/geddes/k8s/postgres/schema_v2.sql index f7a9517..6d5f69e 100644 --- a/geddes/k8s/postgres/schema_v2.sql +++ b/geddes/k8s/postgres/schema_v2.sql @@ -145,6 +145,50 @@ CREATE POLICY equipment_run_visibility_policy ON equipment_runs ) ); +-- Complete parameter traces linked to one physical generic equipment run. +-- sample_time intentionally has no timezone because current GLANCE DB3 exports +-- do not declare one; sample_time_raw preserves the exact source timestamp. +CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_index INT NOT NULL, + source_sample_id BIGINT NOT NULL, + source_tool_id BIGINT, + sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + sample_time_raw TEXT NOT NULL, + values_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, sample_index), + UNIQUE (equipment_run_id, source_sample_id), + CHECK (jsonb_typeof(values_json) = 'object') +); + +CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time +ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); + +ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; +GRANT SELECT ON equipment_run_trace_samples TO api_client; + +CREATE POLICY equipment_run_trace_visibility_policy + ON equipment_run_trace_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + CREATE TABLE IF NOT EXISTS users ( id VARCHAR(255) PRIMARY KEY, -- NanoHUB Subject ID or Email name VARCHAR(255) NOT NULL, diff --git a/web/app/data/catalog/[id]/page.tsx b/web/app/data/catalog/[id]/page.tsx index 95bdb31..a5eac0a 100644 --- a/web/app/data/catalog/[id]/page.tsx +++ b/web/app/data/catalog/[id]/page.tsx @@ -4,6 +4,7 @@ import { use, useEffect, useState } from "react"; import Link from "next/link"; import { cn } from "@/lib/utils"; import { getV2Run, type V2Run } from "@/lib/api-client"; +import { RunTraceExplorer } from "@/components/run-trace-explorer"; import { ArrowLeft, Cpu, @@ -40,6 +41,13 @@ function formatNumber(value: number | null | undefined, digits = 2): string { function formatTimestamp(value: string | null | undefined): string { if (!value) return "—"; + const sourceText = value.trim(); + const hasExplicitTimezone = /(?:z|[+-]\d{2}:?\d{2})$/i.test(sourceText); + if (!hasExplicitTimezone) { + // GLANCE currently exports naive wall times. Keep that exact clock value + // visible instead of letting the browser silently attach its own zone. + return `${sourceText.replace("T", " ")} (source time; timezone not recorded)`; + } const d = new Date(value); if (Number.isNaN(d.getTime())) return value; return d.toLocaleString("en-US", { @@ -107,10 +115,12 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string ); } + const isGlanceTrace = run.source === "glance_db3"; const badges: { label: string; className: string }[] = []; + if (isGlanceTrace) badges.push({ label: "Imported GLANCE Trace", className: "text-violet-500 bg-violet-500/10" }); if (run.is_calibration) badges.push({ label: "Calibration", className: "text-blue-500 bg-blue-500/10" }); if (run.is_outlier) badges.push({ label: "Outlier", className: "text-amber-500 bg-amber-500/10" }); - if (badges.length === 0) badges.push({ label: "Clean Validated", className: "text-emerald-500 bg-emerald-500/10" }); + if (!isGlanceTrace && badges.length === 0) badges.push({ label: "Clean Validated", className: "text-emerald-500 bg-emerald-500/10" }); const accessKey = (run.project_access || "private") as keyof typeof ACCESS_STYLE; const access = ACCESS_STYLE[accessKey] || ACCESS_STYLE.private; @@ -121,6 +131,8 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string const genericOutputs = run.outputs && Object.keys(run.outputs).length > 0 ? run.outputs : null; const isEtcher = run.equipment_id === "etcher" || run.avg_etch_rate !== null; + const glanceProvenance = run.raw_payload?.glance; + const sourceFile = run.raw_payload?.source_file; return (
@@ -325,6 +337,10 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string
+ {(run.trace_sample_count ?? 0) > 0 && ( + + )} + {/* Provenance */}

@@ -332,9 +348,53 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string

-
Source run id (idruns)
+
Catalog run ID
{run.run_id}
+ {run.source_run_id !== null && run.source_run_id !== undefined && ( +
+
Source GLANCE run ID
+
{run.source_run_id}
+
+ )} + {(glanceProvenance?.tool_name || glanceProvenance?.glance_tool_id != null) && ( +
+
Source GLANCE tool
+
+ {glanceProvenance.tool_name || "—"} + {glanceProvenance.glance_tool_id != null && ( + + ID {glanceProvenance.glance_tool_id} + + )} +
+
+ )} + {(glanceProvenance?.recipe_name || glanceProvenance?.glance_recipe_id != null) && ( +
+
Source recipe
+
+ {glanceProvenance.recipe_name || "—"} + {glanceProvenance.glance_recipe_id != null && ( + + ID {glanceProvenance.glance_recipe_id} + + )} +
+
+ )} + {glanceProvenance?.material_name && ( +
+
Source material
+
{glanceProvenance.material_name}
+
+ )} + {glanceProvenance?.run_status && ( +
+
Source run status
+
{glanceProvenance.run_status}
+
+ )}
Execution request ID
{run.execution_request_id || "—"}
@@ -351,6 +411,30 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string
Equipment id
{run.equipment_id || "—"}
+ {glanceProvenance?.run_end_time && ( +
+
Source run end
+
+ {formatTimestamp(glanceProvenance.run_end_time)} +
+
+ )} + {sourceFile?.filename && ( +
+
Source DB3
+
+ {sourceFile.filename} +
+
+ )} + {sourceFile?.sha256 && ( +
+
Source SHA-256
+
+ {sourceFile.sha256} +
+
+ )}
{(run.file_refs || []).length > 0 && (
diff --git a/web/app/data/catalog/page.tsx b/web/app/data/catalog/page.tsx index 5e077c5..e07b024 100644 --- a/web/app/data/catalog/page.tsx +++ b/web/app/data/catalog/page.tsx @@ -14,6 +14,10 @@ import { Loader2, } from "lucide-react"; +const isGlanceTrace = (run: V2Run) => run.source === "glance_db3"; +const isCleanValidated = (run: V2Run) => + !isGlanceTrace(run) && !run.is_outlier && !run.is_calibration; + export default function DataCatalogPage() { const [runs, setRuns] = useState([]); const [loading, setLoading] = useState(true); @@ -83,9 +87,10 @@ export default function DataCatalogPage() { const matchesSearch = r.lot_name?.toLowerCase().includes(q); const matchesOutlier = filterOutliers === "all" ? true : - filterOutliers === "clean" ? (!r.is_outlier && !r.is_calibration) : + filterOutliers === "clean" ? isCleanValidated(r) : filterOutliers === "outlier" ? r.is_outlier : - r.is_calibration; + filterOutliers === "calibration" ? r.is_calibration : + isGlanceTrace(r); return matchesSearch && matchesOutlier; }); @@ -129,7 +134,7 @@ export default function DataCatalogPage() {

Total Runs Mapped

-

{runs.filter(r => !r.is_outlier && !r.is_calibration).length}

+

{runs.filter(isCleanValidated).length}

Clean Production Runs

@@ -155,7 +160,7 @@ export default function DataCatalogPage() { />
- {(["all", "clean", "outlier", "calibration"] as const).map((level) => ( + {(["all", "clean", "outlier", "calibration", "glance"] as const).map((level) => ( ))}
@@ -177,16 +182,21 @@ export default function DataCatalogPage() { {filtered.map((r) => { // Show every flag as its own badge so a row that is both a // calibration and an outlier renders both pills (rather than - // hiding one behind a priority order). Clean rows get a single - // "Clean Validated" pill. + // hiding one behind a priority order). A GLANCE trace has not + // undergone scientific validation, so it gets a neutral import + // badge and is kept out of the clean/validated classification. + const glanceTrace = isGlanceTrace(r); const badges: { label: string; className: string }[] = []; + if (glanceTrace) { + badges.push({ label: "Imported GLANCE Trace", className: "text-violet-500 bg-violet-500/10" }); + } if (r.is_calibration) { badges.push({ label: "Calibration", className: "text-blue-500 bg-blue-500/10" }); } if (r.is_outlier) { badges.push({ label: "Outlier", className: "text-amber-500 bg-amber-500/10" }); } - if (badges.length === 0) { + if (!glanceTrace && badges.length === 0) { badges.push({ label: "Clean Validated", className: "text-emerald-500 bg-emerald-500/10" }); } diff --git a/web/app/data/upload/page.tsx b/web/app/data/upload/page.tsx index fc3a70c..b6ec1f4 100644 --- a/web/app/data/upload/page.tsx +++ b/web/app/data/upload/page.tsx @@ -10,6 +10,7 @@ import { uploadFile, type EquipmentSimple, type PersistedUpload, + type UploadResult, type UploadKind, type V2Project, } from "@/lib/api-client"; @@ -41,6 +42,10 @@ interface UploadedFile { timestamp: string; } +function isDb3File(filename: string): boolean { + return filename.toLowerCase().endsWith(".db3"); +} + function mapPersistedUpload(upload: PersistedUpload): UploadedFile { return { id: upload.id, @@ -62,16 +67,60 @@ function mapPersistedUpload(upload: PersistedUpload): UploadedFile { errors: upload.errors.length > 0 ? upload.errors : undefined, message: upload.status === "processed" - ? "Processed into the data catalog." + ? isDb3File(upload.filename) + ? `Imported ${upload.row_count.toLocaleString()} physical GLANCE runs with linked traces.` + : "Processed into the data catalog." : upload.status === "success" - ? "Stored upload record. Choose a project and process it to add rows to the catalog." - : "Stored upload record with structural validation issues.", + ? isDb3File(upload.filename) + ? "DB3 stored for test import. Choose a project and process its physical runs and traces." + : "Stored upload record. Choose a project and process it to add rows to the catalog." + : upload.status === "stored" + ? "Stored for provenance; no catalog processing is available for this record." + : upload.status === "processing" + ? isDb3File(upload.filename) + ? "Importing physical GLANCE runs and their timestamped traces..." + : "Processing rows into the data catalog..." + : upload.status === "failed" || upload.status === "error" + ? "Processing failed. Review the errors below." + : "Upload status is unavailable.", timestamp: upload.uploaded_at, }; } +function mapUploadResult( + upload: UploadResult, + fallbackTimestamp: string, + fallbackType = "text/csv", +): UploadedFile { + return { + id: upload.upload_id, + name: upload.filename, + size: upload.size_bytes, + type: fallbackType, + status: + upload.status === "processed" + ? "processed" + : upload.status === "processing" + ? "processing" + : upload.status === "failed" + ? "failed" + : upload.status === "stored" + ? "stored" + : upload.status === "success" + ? "success" + : "error", + equipmentId: upload.equipment_id, + kind: upload.kind ?? "all", + rows: upload.row_count > 0 ? upload.row_count : undefined, + columns: upload.columns, + errors: upload.errors || undefined, + message: upload.message, + timestamp: fallbackTimestamp, + }; +} + function shouldRestoreUpload(upload: PersistedUpload): boolean { - return upload.status !== "error" && upload.status !== "failed"; + return Boolean(upload.id); } export default function DataUploadPage() { @@ -176,6 +225,7 @@ export default function DataUploadPage() { for (const rawFile of Array.from(fileList)) { const ts = new Date().toISOString(); + const fileKind: UploadKind = isDb3File(rawFile.name) ? "all" : effectiveKind; // Add file in "uploading" state setFiles((prev) => [ @@ -185,7 +235,7 @@ export default function DataUploadPage() { type: rawFile.type || "text/csv", status: "uploading", equipmentId: selectedEquipment, - kind: effectiveKind, + kind: fileKind, timestamp: ts, }, ...prev, @@ -195,33 +245,33 @@ export default function DataUploadPage() { setTimeout(() => { setFiles((prev) => prev.map((f) => - f.name === rawFile.name && f.timestamp === ts ? { ...f, status: "validating" } : f + f.name === rawFile.name && + f.timestamp === ts && + f.status === "uploading" + ? { ...f, status: "validating" } + : f ) ); }, 400); // Actually send to backend - const result = await uploadFile(rawFile, selectedEquipment, effectiveKind); + const uploadResponse = await uploadFile(rawFile, selectedEquipment, fileKind); + const result = uploadResponse.data; setFiles((prev) => prev.map((f) => { if (f.name !== rawFile.name || f.timestamp !== ts) return f; if (!result) { - return { ...f, status: "error", errors: ["API unavailable — is the FastAPI server running?"] }; + return { + ...f, + status: "error", + errors: [uploadResponse.error || "Upload failed."], + }; } - return { - ...f, - id: result.upload_id, - status: result.status === "success" ? "success" : "error", - equipmentId: result.equipment_id, - kind: result.kind ?? effectiveKind, - rows: result.row_count > 0 ? result.row_count : undefined, - columns: result.columns, - errors: result.errors || undefined, - message: result.message, - }; + return mapUploadResult(result, ts, rawFile.type || "text/csv"); }) ); + } }; @@ -442,7 +492,7 @@ export default function DataUploadPage() { ref={fileInputRef} type="file" multiple - accept=".csv,.xlsx,.xls" + accept=".csv,.xlsx,.xls,.db3" onChange={handleFileSelect} className="hidden" /> @@ -452,7 +502,7 @@ export default function DataUploadPage() {

Drop files here or click to browse

- Supports CSV, XLSX, XLS — max 50 MB per file + CSV/Excel max 50 MB; temporary DB3 limit is server-configured (64 MB by default)

@@ -461,6 +511,9 @@ export default function DataUploadPage() { Excel + + DB3 +
@@ -528,13 +581,17 @@ export default function DataUploadPage() { {file.status === "validating" && (

Checking required columns and row structure...

)} - {(file.status === "success" || file.status === "processed") && ( + {(file.status === "success" || file.status === "processed" || file.status === "stored") && (

{file.message ?? `Stored successfully${file.rows ? ` after checking ${file.rows} data rows` : ""}.`}

)} {file.status === "processing" && ( -

Processing rows into the data catalog...

+

+ {isDb3File(file.name) + ? "Importing physical GLANCE runs and their timestamped traces..." + : "Processing rows into the data catalog..."} +

)} {(file.status === "error" || file.status === "failed") && file.errors && (
@@ -545,7 +602,7 @@ export default function DataUploadPage() { ))}
)} - {(file.status === "success" || file.status === "processed") && file.errors && ( + {(file.status === "success" || file.status === "processed" || file.status === "stored") && file.errors && (
{file.errors.map((warning, j) => (

@@ -601,7 +658,7 @@ export default function DataUploadPage() {

3. Processing

- Processing runs in the background and the status updates here automatically — no need to reload. Input and output files are merged by lot name and run date. Excel files are stored for provenance but are not ingested automatically. + Processing runs in the background and the status updates here automatically — no need to reload. Input and output CSV files are merged by lot name and run date. Excel files are stored for provenance but are not ingested automatically. The temporary DB3 test path creates one catalog row per physical GLANCE run and stores every timestamped sample as its linked trace.

diff --git a/web/components/run-trace-explorer.tsx b/web/components/run-trace-explorer.tsx new file mode 100644 index 0000000..4d43cdf --- /dev/null +++ b/web/components/run-trace-explorer.tsx @@ -0,0 +1,494 @@ +"use client"; + +import dynamic from "next/dynamic"; +import { useEffect, useMemo, useState } from "react"; +import { Activity, AlertCircle, Clock3, Loader2 } from "lucide-react"; + +import { ErrorBoundary } from "@/components/ErrorBoundary"; +import { + getV2Runs, + getV2RunTrace, + type V2Run, + type V2RunTrace, + type V2RunTraceParameter, +} from "@/lib/api-client"; +import { cn } from "@/lib/utils"; + +const Plot = dynamic(() => import("react-plotly.js"), { ssr: false }); + +const MAX_SELECTED_RUNS = 6; +const TRACE_COLORS = [ + "#8b5cf6", + "#10b981", + "#f59e0b", + "#3b82f6", + "#ef4444", + "#ec4899", + "#14b8a6", + "#f97316", +]; + +type AxisMode = "relative" | "absolute"; + +interface RunTraceExplorerProps { + run: V2Run; +} + +interface ComparisonResult { + requestKey: string; + traces: V2RunTrace[]; + warning: string | null; +} + +function sortParameters( + parameters: V2RunTraceParameter[], +): V2RunTraceParameter[] { + return [...parameters].sort((left, right) => { + const byName = left.name.localeCompare(right.name, undefined, { + numeric: true, + sensitivity: "base", + }); + return byName || left.key.localeCompare(right.key, undefined, { numeric: true }); + }); +} + +function parameterLabel(parameter: V2RunTraceParameter): string { + const unit = parameter.unit?.trim() ? ` (${parameter.unit.trim()})` : ""; + return `${parameter.name}${unit} · ${parameter.key}`; +} + +function traceLabel(trace: V2RunTrace): string { + const lot = trace.lot_name || `Catalog run ${trace.run_id}`; + return trace.source_run_id === null || trace.source_run_id === undefined + ? lot + : `${lot} · GLANCE ${trace.source_run_id}`; +} + +function finiteValue(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function colorForRun(runId: number): string { + return TRACE_COLORS[Math.abs(runId) % TRACE_COLORS.length]; +} + +export function RunTraceExplorer({ run }: RunTraceExplorerProps) { + const [currentTrace, setCurrentTrace] = useState(null); + const [peerRuns, setPeerRuns] = useState([]); + const [selectedRunIds, setSelectedRunIds] = useState([run.run_id]); + const [selectedParameter, setSelectedParameter] = useState(""); + const [axisMode, setAxisMode] = useState("relative"); + const [comparisonResult, setComparisonResult] = + useState(null); + const [initialLoading, setInitialLoading] = useState(true); + const [traceError, setTraceError] = useState(null); + + const selectedPeerIds = useMemo( + () => selectedRunIds.filter((runId) => runId !== run.run_id), + [run.run_id, selectedRunIds], + ); + const comparisonRequestKey = `${selectedParameter}:${selectedPeerIds.join(",")}`; + + useEffect(() => { + let cancelled = false; + + async function loadInitialTrace() { + const [trace, visibleRuns] = await Promise.all([ + getV2RunTrace(run.run_id), + run.project_id + ? getV2Runs({ + project_id: run.project_id, + include_outliers: true, + limit: 200, + }) + : Promise.resolve([]), + ]); + + if (cancelled) return; + + if (!trace) { + setTraceError("The complete time-series trace could not be loaded."); + setInitialLoading(false); + return; + } + + const parameters = sortParameters(trace.parameters); + setCurrentTrace(trace); + setSelectedParameter(parameters[0]?.key ?? ""); + setPeerRuns( + (visibleRuns ?? []) + .filter( + (candidate) => + candidate.run_id !== run.run_id && + candidate.project_id === run.project_id && + candidate.equipment_id === run.equipment_id && + (candidate.trace_sample_count ?? 0) > 0, + ) + .sort((left, right) => + (right.run_date ?? "").localeCompare(left.run_date ?? ""), + ), + ); + setInitialLoading(false); + } + + void loadInitialTrace(); + return () => { + cancelled = true; + }; + }, [run.equipment_id, run.project_id, run.run_id]); + + useEffect(() => { + let cancelled = false; + + if (!selectedParameter || selectedPeerIds.length === 0) return; + + async function loadPeerTraces() { + const results = await Promise.all( + selectedPeerIds.map(async (runId) => ({ + runId, + trace: await getV2RunTrace(runId, [selectedParameter]), + })), + ); + + if (cancelled) return; + + const available = results + .map((result) => result.trace) + .filter((trace): trace is V2RunTrace => trace !== null); + const missing = results.filter((result) => result.trace === null); + + setComparisonResult({ + requestKey: comparisonRequestKey, + traces: available, + warning: + missing.length > 0 + ? `${missing.length} selected run${missing.length === 1 ? "" : "s"} could not be loaded.` + : null, + }); + } + + void loadPeerTraces(); + return () => { + cancelled = true; + }; + }, [comparisonRequestKey, selectedParameter, selectedPeerIds]); + + const comparisonIsCurrent = comparisonResult?.requestKey === comparisonRequestKey; + const comparisonWarning = + comparisonResult?.requestKey === comparisonRequestKey + ? comparisonResult.warning + : null; + const comparisonLoading = + Boolean(selectedParameter) && + selectedPeerIds.length > 0 && + !comparisonIsCurrent; + + const parameterOptions = useMemo( + () => sortParameters(currentTrace?.parameters ?? []), + [currentTrace], + ); + const parameter = parameterOptions.find( + (candidate) => candidate.key === selectedParameter, + ); + const displayedTraces = useMemo( + () => { + if (!currentTrace) return []; + const matchingPeerTraces = + comparisonResult?.requestKey === comparisonRequestKey + ? comparisonResult.traces + : []; + return [currentTrace, ...matchingPeerTraces]; + }, + [comparisonRequestKey, comparisonResult, currentTrace], + ); + + const plotData = useMemo(() => { + if (!selectedParameter) return []; + + return displayedTraces.map((trace) => ({ + x: trace.samples.map((sample) => + axisMode === "relative" ? sample.relative_seconds : sample.timestamp, + ), + y: trace.samples.map((sample) => + finiteValue(sample.values[selectedParameter]), + ), + text: trace.samples.map( + (sample) => + `Absolute: ${sample.timestamp}
` + + `Relative: ${sample.relative_seconds.toFixed(3)} s
` + + `Sample record: ${sample.sample_record_id ?? "—"}`, + ), + type: "scattergl" as const, + mode: "lines" as const, + name: traceLabel(trace), + legendgroup: String(trace.run_id), + connectgaps: false, + line: { color: colorForRun(trace.run_id), width: 1.6 }, + hovertemplate: + "%{fullData.name}
Value: %{y}
%{text}", + })); + }, [axisMode, displayedTraces, selectedParameter]); + + const runsWithoutValues = useMemo(() => { + if (!selectedParameter) return []; + return displayedTraces.filter( + (trace) => + !trace.samples.some( + (sample) => finiteValue(sample.values[selectedParameter]) !== null, + ), + ); + }, [displayedTraces, selectedParameter]); + + function togglePeerRun(runId: number) { + setSelectedRunIds((current) => { + if (current.includes(runId)) { + return current.filter((selectedId) => selectedId !== runId); + } + if (current.length >= MAX_SELECTED_RUNS) return current; + return [...current, runId]; + }); + } + + if (initialLoading) { + return ( +
+
+ Loading complete run trace… +
+
+ ); + } + + if (traceError || !currentTrace) { + return ( +
+
+ +
+

Time-series trace unavailable

+

+ {traceError ?? "No trace samples were returned for this run."} +

+
+
+
+ ); + } + + return ( +
+
+
+
+ +

+ Complete parameter trace +

+
+

+ Every recorded sample is retained. Relative time is calculated from each + run's first absolute sample only when displayed. +

+
+
+ {currentTrace.sample_count.toLocaleString()} samples +
+
+ +
+ + +
+ + Time axis + +
+ {(["relative", "absolute"] as const).map((mode) => ( + + ))} +
+
+
+ +
+
+

+ Compare runs ({selectedRunIds.length}/{MAX_SELECTED_RUNS}) +

+ {selectedRunIds.length >= MAX_SELECTED_RUNS && ( +

Comparison limit reached

+ )} +
+
+ + {peerRuns.map((peer) => { + const selected = selectedRunIds.includes(peer.run_id); + const disabled = !selected && selectedRunIds.length >= MAX_SELECTED_RUNS; + return ( + + ); + })} + {peerRuns.length === 0 && ( +

+ No other trace-backed runs from this equipment are available in this + project. +

+ )} +
+
+ +
+ {comparisonLoading && ( +

+ Loading selected run + traces… +

+ )} + {comparisonWarning && ( +

+ {comparisonWarning} +

+ )} + {runsWithoutValues.length > 0 && ( +

+ {`${runsWithoutValues.map(traceLabel).join(", ")} ${ + runsWithoutValues.length === 1 ? "has" : "have" + } no numeric values for this parameter.`} +

+ )} +
+ + {parameter ? ( + + + + ) : ( +
+ No trace parameters were reported for this run. +
+ )} + +
+ +

+ {axisMode === "relative" + ? "Relative seconds are derived at read time from each run's first absolute sample; they are not stored as source data." + : currentTrace.timestamp_timezone + ? `Source timestamps are shown in ${currentTrace.timestamp_timezone}.` + : "GLANCE did not record a source timezone. Source clock values are shown as recorded."} +

+
+
+ ); +} + +export default RunTraceExplorer; diff --git a/web/lib/api-client.ts b/web/lib/api-client.ts index 62b58e3..75c8e4f 100644 --- a/web/lib/api-client.ts +++ b/web/lib/api-client.ts @@ -150,6 +150,8 @@ export interface V2RunFileRef { export interface V2Run { run_id: number; + source_run_id?: number | string | null; + trace_sample_count?: number; execution_request_id: string; lot_name: string; run_date: string | null; @@ -169,6 +171,31 @@ export interface V2Run { pi_name: string | null; project_access: V2Project["access"] | null; file_refs: V2RunFileRef[]; + source?: string; + upload_filename?: string; + raw_payload?: { + glance?: { + glance_run_id?: number | string | null; + glance_tool_id?: number | string | null; + tool_name?: string | null; + lot_name?: string | null; + material_name?: string | null; + glance_recipe_id?: number | string | null; + recipe_name?: string | null; + run_start_time?: string | null; + run_end_time?: string | null; + run_status?: string | null; + sample_count?: number | null; + run_id?: number | string | null; + }; + source_file?: { + type?: string | null; + filename?: string | null; + size_bytes?: number | null; + sha256?: string | null; + }; + [key: string]: unknown; + }; } export function getV2Projects() { @@ -474,6 +501,44 @@ export function getV2Run(runId: number | string) { return apiFetch(`/dataset/v2/runs/${encodeURIComponent(String(runId))}`); } +export interface V2RunTraceParameter { + key: string; + name: string; + unit: string | null; +} + +export interface V2RunTraceSample { + sample_record_id: number | string | null; + source_tool_id: number | string | null; + timestamp: string; + relative_seconds: number; + values: Record; +} + +export interface V2RunTrace { + run_id: number; + source_run_id: number | string | null; + lot_name: string; + sample_count: number; + start_time: string | null; + end_time: string | null; + timestamp_timezone: string | null; + parameters: V2RunTraceParameter[]; + samples: V2RunTraceSample[]; +} + +export function getV2RunTrace( + runId: number | string, + parameters: string[] = [], +) { + const query = new URLSearchParams(); + parameters.forEach((parameter) => query.append("parameters", parameter)); + const suffix = query.size > 0 ? `?${query.toString()}` : ""; + return apiFetch( + `/dataset/v2/runs/${encodeURIComponent(String(runId))}/trace${suffix}`, + ); +} + export interface ParityData { source: string; model_type?: string; @@ -1274,11 +1339,16 @@ export interface ProcessUploadResult { message: string; } +export interface UploadFileResponse { + data: UploadResult | null; + error: string | null; +} + export async function uploadFile( file: File, equipmentId: string, kind: UploadKind = "all", -): Promise { +): Promise { try { const formData = new FormData(); formData.append("file", file); @@ -1292,12 +1362,27 @@ export async function uploadFile( }); if (!response.ok) { - return null; + let error = `Upload failed (HTTP ${response.status}).`; + try { + const body = (await response.json()) as { detail?: unknown }; + if (typeof body.detail === "string" && body.detail.trim()) { + error = body.detail; + } + } catch { + // The response was not JSON; retain the status-based message. + } + return { data: null, error }; } - return (await response.json()) as UploadResult; + return { + data: (await response.json()) as UploadResult, + error: null, + }; } catch { - return null; + return { + data: null, + error: "API unavailable — is the FastAPI server running?", + }; } } From c7bb8bd585150d1efbeca2fd21737a9f382f9dd2 Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Mon, 13 Jul 2026 19:51:06 -0400 Subject: [PATCH 2/6] Stream large API uploads through Next --- web/app/api/dt/[...path]/route.ts | 14 +++++++++++--- web/next.config.ts | 6 ++++++ web/proxy.ts | 7 ++++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/web/app/api/dt/[...path]/route.ts b/web/app/api/dt/[...path]/route.ts index d18b329..e1f1312 100644 --- a/web/app/api/dt/[...path]/route.ts +++ b/web/app/api/dt/[...path]/route.ts @@ -41,6 +41,10 @@ const BLOCKED_RESPONSE_HEADERS = new Set([ export const runtime = "nodejs"; +type StreamingRequestInit = RequestInit & { + duplex?: "half"; +}; + function buildTargetUrl(path: string[], request: NextRequest): string { const sanitizedPath = path.map(encodeURIComponent).join("/"); const suffix = request.nextUrl.search || ""; @@ -70,7 +74,7 @@ async function resolveSession() { async function buildRequestInit( request: NextRequest, session: NonNullable>>, -): Promise { +): Promise { const headers = new Headers(); request.headers.forEach((value, key) => { @@ -86,14 +90,18 @@ async function buildRequestInit( headers.set("X-User-Role", session.user.role || "researcher"); headers.set("X-User-Org", session.user.organization || "Purdue University"); - const init: RequestInit = { + const init: StreamingRequestInit = { method: request.method, headers, cache: "no-store", }; if (request.method !== "GET" && request.method !== "HEAD") { - init.body = await request.arrayBuffer(); + // Keep large uploads as a stream all the way to FastAPI. Buffering here + // used Next's 10 MiB proxy clone limit and made otherwise-valid DB3 files + // fail before the API could enforce its own configured size limit. + init.body = request.body; + init.duplex = "half"; } return init; diff --git a/web/next.config.ts b/web/next.config.ts index 1327cdb..454bb34 100644 --- a/web/next.config.ts +++ b/web/next.config.ts @@ -3,6 +3,12 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { // Required for Docker / Geddes Kubernetes deployment output: "standalone", + experimental: { + // Preserve headroom for any authenticated proxied request that is not on + // the explicitly streamed upload route. The API retains the authoritative + // per-file limits. + proxyClientMaxBodySize: "70mb", + }, }; export default nextConfig; diff --git a/web/proxy.ts b/web/proxy.ts index 9e1bded..fbabe2c 100644 --- a/web/proxy.ts +++ b/web/proxy.ts @@ -92,9 +92,10 @@ export async function proxy(req: NextRequest) { export const config = { // Match every request except Next.js internals and the file extensions - // we already treat as public. Keeping the matcher tight avoids an - // unnecessary getToken() call on every static asset request. + // we already treat as public. The upload route is excluded from this outer + // proxy because Next clones matching request bodies before route handling. + // Its catch-all route still calls auth() and remains the trust boundary. matcher: [ - "/((?!_next/static|_next/image|_next/data|favicon.ico).*)", + "/((?!api/dt/data/upload(?:/|$)|_next/static|_next/image|_next/data|favicon.ico).*)", ], }; From be5289bcf5f17e1fb8930c96f3cf62bb87b71f9b Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Mon, 13 Jul 2026 20:21:02 -0400 Subject: [PATCH 3/6] Isolate temporary DB3 pilot deployment --- .dockerignore | 30 +++ .gitignore | 2 + DB3_UPLOAD_PIPELINE_REVIEW.md | 24 +- api/main.py | 4 + api/metadata_pg.py | 60 ++++- api/routers/upload.py | 216 ++++++++++++++++- api/scripts/init_db.sql | 1 + api/tests/test_glance_db3_trace_import.py | 159 ++++++++++++- api/tests/test_phase3_security.py | 108 +++++++++ geddes/k8s/05-postgres.yaml | 1 + geddes/k8s/pilots/db3/README.md | 145 ++++++++++++ geddes/k8s/pilots/db3/configmap.yaml | 19 ++ geddes/k8s/pilots/db3/deploy.sh | 276 ++++++++++++++++++++++ geddes/k8s/pilots/db3/ingress.yaml | 26 ++ geddes/k8s/pilots/db3/pvc.yaml | 15 ++ geddes/k8s/pilots/db3/rollback.sh | 254 ++++++++++++++++++++ geddes/k8s/postgres/schema_v2.sql | 1 + web/app/data/upload/page.tsx | 50 +++- web/lib/api-client.ts | 13 +- 19 files changed, 1380 insertions(+), 24 deletions(-) create mode 100644 .dockerignore create mode 100644 geddes/k8s/pilots/db3/README.md create mode 100644 geddes/k8s/pilots/db3/configmap.yaml create mode 100755 geddes/k8s/pilots/db3/deploy.sh create mode 100644 geddes/k8s/pilots/db3/ingress.yaml create mode 100644 geddes/k8s/pilots/db3/pvc.yaml create mode 100755 geddes/k8s/pilots/db3/rollback.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..72e2298 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +.git +.github +.agents +.codex + +**/__pycache__ +**/*.py[cod] +**/.pytest_cache +**/.venv +**/node_modules +**/.next + +**/.env +**/.env.* +**/local.settings.json +**/appsettings.json +**/*kubeconfig* + +incoming +**/*.db +**/*.db3 +**/*.sqlite +**/*.sqlite3 +nanohub/confidential +nanohub/config +nanohub/data +uploader/staging + +.DS_Store +*.log diff --git a/.gitignore b/.gitignore index 94b4bdf..e2bdb82 100644 --- a/.gitignore +++ b/.gitignore @@ -128,6 +128,8 @@ temp_azure/ *.ps1 *.sh !scripts/check_deployment_state.sh +!geddes/k8s/pilots/db3/deploy.sh +!geddes/k8s/pilots/db3/rollback.sh # VS Code settings .vscode/settings.json diff --git a/DB3_UPLOAD_PIPELINE_REVIEW.md b/DB3_UPLOAD_PIPELINE_REVIEW.md index caed031..c60657b 100644 --- a/DB3_UPLOAD_PIPELINE_REVIEW.md +++ b/DB3_UPLOAD_PIPELINE_REVIEW.md @@ -21,8 +21,12 @@ It is opt-in and must not be treated as the long-term GLANCE integration. A futu production integration should read from the authoritative GLANCE service or an approved export pipeline rather than depend on users moving SQLite files around. -DB3 upload is disabled unless `DT_DB3_EQUIPMENT_TOOL_MAP` explicitly maps the -selected registered equipment ID to the GLANCE tool name embedded in the export: +DB3 upload is default-off. It requires all three temporary pilot controls: + +- `DT_ENABLE_DB3_UPLOAD=true`; +- the authenticated nanoHUB ID in `DT_DB3_ALLOWED_USER_IDS`; +- `DT_DB3_EQUIPMENT_TOOL_MAP` explicitly mapping the selected registered + equipment ID to the GLANCE tool name embedded in the export. ```json { @@ -32,7 +36,9 @@ selected registered equipment ID to the GLANCE tool name embedded in the export: This mapping is intentional. It prevents an export from one machine from being silently attached to another machine as more GLANCE tools are tested. The default -DB3 limit is 64 MB and can be adjusted with `DT_DB3_UPLOAD_MAX_MB`. +DB3 limit is 64 MB and can be adjusted with `DT_DB3_UPLOAD_MAX_MB`. The UI reads +an authenticated runtime capability endpoint, so users outside the allowlist do +not see DB3 as an accepted upload type or see DB3 upload records. ## Workflow @@ -51,6 +57,11 @@ DB3 limit is 64 MB and can be adjusted with `DT_DB3_UPLOAD_MAX_MB`. `GET /dataset/v2/runs/{run_id}/trace` and can compare up to six runs using an absolute/source or dynamically calculated relative time axis. +Processing uses an in-process background task for this pilot. A timestamped +processing lease is refreshed by a heartbeat while work is active. If the +worker disappears, polling returns the abandoned DB3 to a retryable state after +the configured stale interval; trace synchronization remains idempotent. + The old 52 generated input/output CSV records are gone. A single DB3 now requires one Process action. @@ -118,6 +129,7 @@ npm run lint The local `incoming/` directory and all `*.db3` files are ignored because source exports can contain proprietary equipment traces. No generated CSV/intermediate -directories are created. Upload storage is not currently a persistent Kubernetes -volume, so the database trace is the durable test result; the uploaded DB3 itself -must be treated as a temporary processing artifact. +directories are created. The Geddes pilot stores raw uploads on its own encrypted +PVC, isolated from the normal deployment manifest. The removable deployment and +digest-pinned rollback procedure live in `geddes/k8s/pilots/db3/`; that package +must not be merged into the final production platform. diff --git a/api/main.py b/api/main.py index fb0357a..5668b9c 100644 --- a/api/main.py +++ b/api/main.py @@ -162,6 +162,8 @@ def _run_startup_migrations() -> None: ensure_sample_and_publication_tables_pg, ensure_equipment_access_table_pg, ensure_data_uploads_kind_column_pg, + ensure_data_uploads_processing_started_at_pg, + recover_stale_db3_uploads_pg, ) lock_conn = get_pg_superuser_connection() @@ -189,6 +191,8 @@ def _run_startup_migrations() -> None: ensure_sample_and_publication_tables_pg() ensure_equipment_access_table_pg() ensure_data_uploads_kind_column_pg() + ensure_data_uploads_processing_started_at_pg() + recover_stale_db3_uploads_pg() finally: try: with lock_conn.cursor() as cur: diff --git a/api/metadata_pg.py b/api/metadata_pg.py index 9ae481c..e0e04ba 100644 --- a/api/metadata_pg.py +++ b/api/metadata_pg.py @@ -1,5 +1,6 @@ import json import logging +import os import re import uuid from datetime import date, datetime @@ -5481,6 +5482,63 @@ def ensure_data_uploads_kind_column_pg() -> None: conn.close() +def ensure_data_uploads_processing_started_at_pg() -> None: + """Add a processing lease timestamp for restart-safe upload retries.""" + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + "ALTER TABLE data_uploads " + "ADD COLUMN IF NOT EXISTS processing_started_at TIMESTAMPTZ" + ) + conn.commit() + finally: + conn.close() + + +def recover_stale_db3_uploads_pg() -> None: + """Return abandoned pilot processing leases to a retryable state.""" + if os.getenv("DT_ENABLE_DB3_UPLOAD", "").strip().casefold() not in { + "1", + "true", + "yes", + "on", + }: + return + + raw_stale_minutes = os.getenv("DT_DB3_PROCESSING_STALE_MINUTES", "30").strip() + try: + stale_minutes = max(int(raw_stale_minutes), 1) + except ValueError: + stale_minutes = 30 + + conn = get_pg_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE data_uploads + SET status = 'success', + processing_started_at = NULL, + errors_json = COALESCE(errors_json, '[]'::jsonb) + || jsonb_build_array( + 'Previous DB3 processing did not finish; retry processing.' + ) + WHERE status = 'processing' + AND LOWER(filename) LIKE '%%.db3' + AND ( + processing_started_at IS NULL + OR processing_started_at < CURRENT_TIMESTAMP + - (%s * INTERVAL '1 minute') + ) + """, + (stale_minutes,), + ) + conn.commit() + finally: + conn.close() + + def create_upload_record_pg( *, equipment_id: str, @@ -5553,7 +5611,6 @@ def list_uploads_pg(user: PlatformUser) -> list[dict[str, Any]]: equipment_id, filename, stored_as, - storage_path, size_bytes, row_count, columns_json, @@ -5588,7 +5645,6 @@ def list_uploads_pg(user: PlatformUser) -> list[dict[str, Any]]: "equipment_id": record["equipment_id"], "filename": record["filename"], "stored_as": record["stored_as"], - "storage_path": record["storage_path"], "size_bytes": int(record["size_bytes"] or 0), "row_count": int(record["row_count"] or 0), "columns": _coerce_json(record.get("columns_json"), []), diff --git a/api/routers/upload.py b/api/routers/upload.py index 4c8ccaa..94831cd 100644 --- a/api/routers/upload.py +++ b/api/routers/upload.py @@ -8,6 +8,7 @@ import logging import math import os +import threading from datetime import datetime from decimal import Decimal, InvalidOperation from pathlib import Path @@ -33,6 +34,7 @@ create_upload_record_pg, get_equipment_pg, list_uploads_pg, + recover_stale_db3_uploads_pg, ) from security import PlatformUser, get_platform_user from glance_db3.trace_import import extract_db3_trace @@ -119,6 +121,41 @@ def resolve_uploads_dir() -> Path: DB3_UPLOAD_CHUNK_BYTES = 1024 * 1024 +def _db3_upload_enabled() -> bool: + return os.getenv("DT_ENABLE_DB3_UPLOAD", "").strip().casefold() in { + "1", + "true", + "yes", + "on", + } + + +def _db3_allowed_user_ids() -> set[str]: + raw = os.getenv("DT_DB3_ALLOWED_USER_IDS", "") + return { + user_id.strip().casefold() + for user_id in raw.split(",") + if user_id.strip() + } + + +def _db3_user_is_allowed(user: PlatformUser) -> bool: + return _db3_upload_enabled() and user.id.strip().casefold() in _db3_allowed_user_ids() + + +def _assert_db3_user_allowed(user: PlatformUser) -> None: + if not _db3_upload_enabled(): + raise HTTPException( + status_code=503, + detail="The temporary DB3 upload pilot is disabled.", + ) + if user.id.strip().casefold() not in _db3_allowed_user_ids(): + raise HTTPException( + status_code=403, + detail="DB3 uploads are limited to the temporary pilot participants.", + ) + + def _db3_upload_limit_bytes() -> int: raw = os.getenv("DT_DB3_UPLOAD_MAX_MB", "64").strip() try: @@ -135,6 +172,9 @@ def _db3_allowed_source_tools(equipment_id: str) -> set[str]: equipment-to-tool map prevents a source export from being silently attached to the wrong registered machine as more GLANCE tools are introduced. """ + if not _db3_upload_enabled(): + raise ValueError("The temporary DB3 upload pilot is disabled.") + raw = os.getenv("DT_DB3_EQUIPMENT_TOOL_MAP", "").strip() if not raw: raise ValueError( @@ -163,6 +203,50 @@ def _db3_allowed_source_tools(equipment_id: str) -> set[str]: return allowed +def _db3_mapped_equipment_ids() -> list[str]: + """Return valid configured equipment IDs without exposing source tool names.""" + if not _db3_upload_enabled(): + return [] + raw = os.getenv("DT_DB3_EQUIPMENT_TOOL_MAP", "").strip() + try: + configured = json.loads(raw) if raw else {} + except json.JSONDecodeError: + return [] + if not isinstance(configured, dict): + return [] + return sorted( + str(equipment_id) + for equipment_id, values in configured.items() + if str(equipment_id).strip() + and ( + (isinstance(values, str) and values.strip()) + or ( + isinstance(values, list) + and any(str(value).strip() for value in values) + ) + ) + ) + + +def _assert_db3_pilot_project(project: dict[str, Any]) -> None: + expected_project_id = os.getenv("DT_DB3_PILOT_PROJECT_ID", "").strip() + if not expected_project_id: + raise HTTPException( + status_code=503, + detail="The temporary DB3 pilot project is not configured.", + ) + if str(project.get("id") or "").strip() != expected_project_id: + raise HTTPException( + status_code=400, + detail="DB3 traces can only be processed into the private pilot project.", + ) + if str(project.get("access") or "").strip().casefold() != "private": + raise HTTPException( + status_code=503, + detail="The configured DB3 pilot project is not private.", + ) + + def _copy_upload_with_limit(source: Any, destination: Path, max_bytes: int) -> int: """Stream an uploaded file to disk without buffering the whole DB3.""" total = 0 @@ -434,7 +518,7 @@ def _update_upload_record( cur.execute( """ UPDATE data_uploads - SET status = %s, errors_json = %s + SET status = %s, errors_json = %s, processing_started_at = NULL WHERE id = %s """, (status, Json(errors), upload_id), @@ -443,7 +527,8 @@ def _update_upload_record( cur.execute( """ UPDATE data_uploads - SET status = %s, errors_json = %s, row_count = %s + SET status = %s, errors_json = %s, row_count = %s, + processing_started_at = NULL WHERE id = %s """, (status, Json(errors), row_count, upload_id), @@ -466,7 +551,8 @@ def _claim_upload_for_processing(upload_id: str) -> bool: cur.execute( """ UPDATE data_uploads - SET status = 'processing', errors_json = '[]'::jsonb + SET status = 'processing', errors_json = '[]'::jsonb, + processing_started_at = CURRENT_TIMESTAMP WHERE id = %s AND status <> 'processing' RETURNING id """, @@ -479,6 +565,32 @@ def _claim_upload_for_processing(upload_id: str) -> bool: conn.close() +def _touch_upload_processing_lease(upload_id: str) -> None: + conn = get_pg_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE data_uploads + SET processing_started_at = CURRENT_TIMESTAMP + WHERE id = %s AND status = 'processing' + """, + (upload_id,), + ) + conn.commit() + finally: + conn.close() + + +def _db3_processing_heartbeat_seconds() -> int: + raw = os.getenv("DT_DB3_PROCESSING_HEARTBEAT_SECONDS", "60").strip() + try: + seconds = int(raw) + except ValueError: + seconds = 60 + return min(max(seconds, 5), 300) + + def _load_upload_record(upload_id: str, user: PlatformUser) -> dict[str, Any]: conn = get_pg_connection() try: @@ -513,7 +625,12 @@ def _load_upload_record(upload_id: str, user: PlatformUser) -> dict[str, Any]: "errors": stored_issues, "kind": _normalize_kind(row[8]), } - if user.role != "admin": + if Path(record["filename"]).suffix.lower() == ".db3": + # Pilot participants deliberately share the DB3 queue so Paul, Rich, + # and the platform administrator can help one another. No same-org + # fallback applies to proprietary DB3 exports. + _assert_db3_user_allowed(user) + elif user.role != "admin": same_owner = record["owner_id"] == user.id same_org = record["owner_org"] and record["owner_org"] == user.organization if not same_owner and not same_org: @@ -926,6 +1043,9 @@ async def upload_file( detail=f"Unsupported file type '{ext}'. Use CSV, XLSX, XLS, or DB3." ) + if ext == ".db3": + _assert_db3_user_allowed(user) + equipment_upload_dir = UPLOADS_DIR / equipment_id equipment_upload_dir.mkdir(parents=True, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") @@ -1167,12 +1287,37 @@ async def upload_file( } +@router.get("/upload-capabilities") +def get_upload_capabilities( + user: PlatformUser = Depends(get_platform_user), +): + """Describe runtime-gated upload features for the authenticated caller.""" + mapped_equipment_ids = _db3_mapped_equipment_ids() + return { + "db3_upload": { + "enabled": _db3_user_is_allowed(user) and bool(mapped_equipment_ids), + "max_mb": _db3_upload_limit_bytes(), + "equipment_ids": mapped_equipment_ids if _db3_user_is_allowed(user) else [], + } + } + + @router.get("/uploads") def list_uploads( user: PlatformUser = Depends(get_platform_user), ): """List all previously uploaded files.""" - return list_uploads_pg(user) + db3_allowed = _db3_user_is_allowed(user) + if db3_allowed: + recover_stale_db3_uploads_pg() + uploads = list_uploads_pg(user) + if db3_allowed: + return uploads + return [ + record + for record in uploads + if Path(str(record.get("filename") or "")).suffix.lower() != ".db3" + ] def _effective_equipment_config(equipment: dict[str, Any]) -> dict[str, Any]: @@ -1650,6 +1795,52 @@ def _execute_upload_processing( upload_record: dict[str, Any], project_id: str, prior_warnings: list[str], +) -> None: + """Execute an upload with a live processing lease for DB3 imports.""" + is_db3 = Path(upload_record["filename"]).suffix.lower() == ".db3" + if not is_db3: + _execute_upload_processing_impl( + upload_id=upload_id, + upload_record=upload_record, + project_id=project_id, + prior_warnings=prior_warnings, + ) + return + + stop_heartbeat = threading.Event() + + def heartbeat() -> None: + interval = _db3_processing_heartbeat_seconds() + while not stop_heartbeat.wait(interval): + try: + _touch_upload_processing_lease(upload_id) + except Exception: # noqa: BLE001 - recovery remains available + logger.exception("DB3 upload %s lease heartbeat failed", upload_id) + + heartbeat_thread = threading.Thread( + target=heartbeat, + name=f"db3-lease-{upload_id}", + daemon=True, + ) + heartbeat_thread.start() + try: + _execute_upload_processing_impl( + upload_id=upload_id, + upload_record=upload_record, + project_id=project_id, + prior_warnings=prior_warnings, + ) + finally: + stop_heartbeat.set() + heartbeat_thread.join(timeout=1) + + +def _execute_upload_processing_impl( + *, + upload_id: str, + upload_record: dict[str, Any], + project_id: str, + prior_warnings: list[str], ) -> None: """Parse and ingest a stored upload, updating its status to ``processed`` or ``failed``. Runs in a background task (Starlette executes sync background @@ -1789,9 +1980,15 @@ def _fail(errors: list[str], row_count: int) -> None: upload_filename=upload_record["filename"], runs=run_rows, ) - except Exception as exc: # noqa: BLE001 - persist an actionable DB3 error + except Exception: # noqa: BLE001 - persist an actionable DB3 error logger.exception("DB3 upload %s import failed", upload_id) - _fail([f"DB3 import failed: {exc}"], 0) + _fail( + [ + "DB3 import failed. The source file could not be processed; " + "contact the pilot administrator for the server-side details." + ], + 0, + ) return logger.info( @@ -1889,8 +2086,13 @@ def process_upload( raise HTTPException(status_code=400, detail="project_id is required") project = _assert_project_visible(project_id, user) + if _db3_user_is_allowed(user): + recover_stale_db3_uploads_pg() upload_record = _load_upload_record(upload_id, user) equipment_id = upload_record["equipment_id"] + if Path(upload_record["filename"]).suffix.lower() == ".db3": + _assert_db3_user_allowed(user) + _assert_db3_pilot_project(project) # Non-blocking range warnings recorded at validation time should survive # processing so the catalog/upload UI can keep surfacing them. A "success" diff --git a/api/scripts/init_db.sql b/api/scripts/init_db.sql index 1c3862d..f2a1804 100644 --- a/api/scripts/init_db.sql +++ b/api/scripts/init_db.sql @@ -487,6 +487,7 @@ CREATE TABLE IF NOT EXISTS data_uploads ( kind VARCHAR(20) NOT NULL DEFAULT 'all', owner_id VARCHAR(255) REFERENCES users(id), owner_org VARCHAR(255) DEFAULT '', + processing_started_at TIMESTAMP WITH TIME ZONE, uploaded_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); diff --git a/api/tests/test_glance_db3_trace_import.py b/api/tests/test_glance_db3_trace_import.py index 778f640..c3d8a5f 100644 --- a/api/tests/test_glance_db3_trace_import.py +++ b/api/tests/test_glance_db3_trace_import.py @@ -10,6 +10,7 @@ from tempfile import TemporaryDirectory from unittest.mock import patch +from fastapi import HTTPException API_DIR = Path(__file__).resolve().parents[1] if str(API_DIR) not in sys.path: @@ -25,6 +26,7 @@ sync_equipment_run_traces_pg, ) from routers import upload as upload_router +from security import PlatformUser def _write_fixture( @@ -229,6 +231,158 @@ def _write_fixture( class GlanceDb3TraceImportTests(unittest.TestCase): + def test_db3_pilot_is_default_off_and_requires_an_explicit_user_allowlist( + self, + ) -> None: + allowed_user = PlatformUser( + id="will3042", + email="will3042@purdue.edu", + name="Paul Williams", + role="equipment_owner", + organization="Birck Nanotechnology Center", + ) + other_user = PlatformUser( + id="same-org-user", + email="same-org-user@purdue.edu", + name="Same Org User", + role="researcher", + organization="Birck Nanotechnology Center", + ) + + with patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "false", + "DT_DB3_ALLOWED_USER_IDS": "will3042,hosler0", + }, + clear=False, + ): + self.assertFalse(upload_router._db3_user_is_allowed(allowed_user)) + with self.assertRaises(HTTPException) as disabled: + upload_router._assert_db3_user_allowed(allowed_user) + self.assertEqual(disabled.exception.status_code, 503) + + with patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_ALLOWED_USER_IDS": "will3042,hosler0", + }, + clear=False, + ): + self.assertTrue(upload_router._db3_user_is_allowed(allowed_user)) + self.assertFalse(upload_router._db3_user_is_allowed(other_user)) + with self.assertRaises(HTTPException) as forbidden: + upload_router._assert_db3_user_allowed(other_user) + self.assertEqual(forbidden.exception.status_code, 403) + + def test_db3_capability_exposes_only_mapped_equipment_to_pilot_users( + self, + ) -> None: + pilot_user = PlatformUser( + id="hosler0", + email="hosler0@purdue.edu", + name="Richard Hosler", + role="equipment_owner", + organization="Birck Nanotechnology Center", + ) + other_user = PlatformUser( + id="researcher", + email="researcher@purdue.edu", + name="Researcher", + role="researcher", + organization="Birck Nanotechnology Center", + ) + equipment_id = "plasmatherm_versaline_icp_rie_etcher" + with patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_ALLOWED_USER_IDS": "will3042,hosler0", + "DT_DB3_EQUIPMENT_TOOL_MAP": json.dumps( + {equipment_id: ["VLN-11304-CTC-PM1"]} + ), + }, + clear=False, + ): + pilot_capabilities = upload_router.get_upload_capabilities(pilot_user) + other_capabilities = upload_router.get_upload_capabilities(other_user) + + self.assertTrue(pilot_capabilities["db3_upload"]["enabled"]) + self.assertEqual( + pilot_capabilities["db3_upload"]["equipment_ids"], + [equipment_id], + ) + self.assertFalse(other_capabilities["db3_upload"]["enabled"]) + self.assertEqual(other_capabilities["db3_upload"]["equipment_ids"], []) + + def test_db3_processing_requires_the_configured_private_project(self) -> None: + project_id = "glance_db3_pilot_deadbeef" + with patch.dict( + os.environ, + {"DT_DB3_PILOT_PROJECT_ID": project_id}, + clear=False, + ): + upload_router._assert_db3_pilot_project( + {"id": project_id, "access": "private"} + ) + + with self.assertRaises(HTTPException) as wrong_project: + upload_router._assert_db3_pilot_project( + {"id": "shared-demo", "access": "private"} + ) + self.assertEqual(wrong_project.exception.status_code, 400) + + with self.assertRaises(HTTPException) as open_project: + upload_router._assert_db3_pilot_project( + {"id": project_id, "access": "open"} + ) + self.assertEqual(open_project.exception.status_code, 503) + + def test_db3_processing_failure_does_not_expose_the_server_path(self) -> None: + updates: list[dict] = [] + equipment_id = "test-glance-equipment" + with ( + patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_EQUIPMENT_TOOL_MAP": json.dumps( + {equipment_id: ["VLN-11304-CTC-PM1"]} + ), + }, + clear=False, + ), + patch.object( + upload_router, + "extract_db3_trace", + side_effect=FileNotFoundError( + "/var/lib/dt-api/uploads/private/source.db3" + ), + ), + patch.object( + upload_router, + "_update_upload_record", + side_effect=lambda **kwargs: updates.append(kwargs), + ), + ): + upload_router._execute_upload_processing( + upload_id="00000000-0000-0000-0000-000000000099", + upload_record={ + "equipment_id": equipment_id, + "filename": "source.db3", + "storage_path": "/var/lib/dt-api/uploads/private/source.db3", + "kind": "all", + }, + project_id="glance_db3_pilot_deadbeef", + prior_warnings=[], + ) + + self.assertEqual(updates[-1]["status"], "failed") + rendered_errors = " ".join(updates[-1]["errors"]) + self.assertNotIn("/var/lib", rendered_errors) + self.assertIn("contact the pilot administrator", rendered_errors) + def test_generic_catalog_select_renders_glance_json_paths(self) -> None: query = _equipment_runs_base_select(include_raw_payload=True) self.assertIn("'{glance,run_id}'", query) @@ -360,7 +514,10 @@ def capture_sync(**kwargs): with ( patch.dict( os.environ, - {"DT_DB3_EQUIPMENT_TOOL_MAP": tool_map}, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_EQUIPMENT_TOOL_MAP": tool_map, + }, clear=False, ), patch.object( diff --git a/api/tests/test_phase3_security.py b/api/tests/test_phase3_security.py index 1de8b3e..6b8c855 100644 --- a/api/tests/test_phase3_security.py +++ b/api/tests/test_phase3_security.py @@ -766,6 +766,114 @@ def test_upload_records_equipment_association(self): ) self.assertEqual(create_upload_record_pg.call_args.kwargs["user"].id, "owner-1") + def test_upload_listing_never_exposes_server_storage_paths(self): + now = datetime.now(timezone.utc) + connection = FakeConnection( + [ + { + "contains": "FROM data_uploads", + "fetchall": [ + { + "id": "00000000-0000-0000-0000-000000000001", + "equipment_id": "etcher_01", + "filename": "runs.csv", + "stored_as": "stored-runs.csv", + "storage_path": "/private/server/path/stored-runs.csv", + "size_bytes": 12, + "row_count": 1, + "columns_json": ["LOTNAME"], + "errors_json": [], + "status": "success", + "kind": "all", + "owner_id": "owner-1", + "owner_org": "Birck", + "uploaded_at": now, + } + ], + } + ] + ) + user = metadata_pg.PlatformUser( + id="owner-1", + email="owner@example.com", + name="Equipment Owner", + role="equipment_owner", + organization="Birck", + ) + + with patch.object(metadata_pg, "get_pg_connection", return_value=connection): + records = metadata_pg.list_uploads_pg(user) + + self.assertEqual(len(records), 1) + self.assertNotIn("storage_path", records[0]) + self.assertTrue(connection.closed) + + def test_db3_upload_is_rejected_when_temporary_pilot_is_disabled(self): + with patch.dict( + os.environ, + {"DT_ENABLE_DB3_UPLOAD": "false"}, + clear=False, + ), patch.object( + upload, + "get_equipment_pg", + return_value={ + "domain_id": "etcher_01", + "owner_id": "owner-1", + "owner_org": "Birck", + "status": "approved", + "columns": [], + "config_json": {}, + }, + ), patch.object(upload, "create_upload_record_pg") as create_record: + response = self.client.post( + "/api/data/upload", + headers=OWNER_HEADERS, + data={"equipment_id": "etcher_01"}, + files={"file": ("source.db3", b"sqlite", "application/vnd.sqlite3")}, + ) + + self.assertEqual(response.status_code, 503) + self.assertEqual( + response.json()["detail"], + "The temporary DB3 upload pilot is disabled.", + ) + self.assertEqual(create_record.call_count, 0) + + def test_db3_upload_rejects_a_same_org_user_outside_the_allowlist(self): + with patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_ALLOWED_USER_IDS": "will3042,hosler0", + "DT_DB3_EQUIPMENT_TOOL_MAP": '{"etcher_01":["tool-1"]}', + }, + clear=False, + ), patch.object( + upload, + "get_equipment_pg", + return_value={ + "domain_id": "etcher_01", + "owner_id": "owner-1", + "owner_org": "Birck", + "status": "approved", + "columns": [], + "config_json": {}, + }, + ), patch.object(upload, "create_upload_record_pg") as create_record: + response = self.client.post( + "/api/data/upload", + headers=OWNER_HEADERS, + data={"equipment_id": "etcher_01"}, + files={"file": ("source.db3", b"sqlite", "application/vnd.sqlite3")}, + ) + + self.assertEqual(response.status_code, 403) + self.assertEqual( + response.json()["detail"], + "DB3 uploads are limited to the temporary pilot participants.", + ) + self.assertEqual(create_record.call_count, 0) + def test_upload_reports_missing_required_columns(self): with patch.object( upload, diff --git a/geddes/k8s/05-postgres.yaml b/geddes/k8s/05-postgres.yaml index 9d8c28e..91bfcc0 100644 --- a/geddes/k8s/05-postgres.yaml +++ b/geddes/k8s/05-postgres.yaml @@ -376,6 +376,7 @@ data: kind VARCHAR(20) NOT NULL DEFAULT 'all', owner_id VARCHAR(255) REFERENCES users(id), owner_org VARCHAR(255) DEFAULT '', + processing_started_at TIMESTAMP WITH TIME ZONE, uploaded_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); diff --git a/geddes/k8s/pilots/db3/README.md b/geddes/k8s/pilots/db3/README.md new file mode 100644 index 0000000..830b8bb --- /dev/null +++ b/geddes/k8s/pilots/db3/README.md @@ -0,0 +1,145 @@ +# Temporary Geddes DB3 pilot + +This package exists only for the branch-only Paul/Rich DB3 pilot on Geddes. It +must not be merged into or treated as the final production platform. The final +platform does not include DB3 upload support. Pilot images must use a distinct +`db3-pilot-*` tag and an immutable digest; never retag `v52`, `v53`, or a normal +release tag with pilot code. + +The package deliberately isolates the temporary operational changes: + +- DB3 is enabled only by `dt-db3-pilot-config` and limited to nanoHUB users + `ngholiza`, `will3042` (Paul), and `hosler0` (Rich). +- Only GLANCE tool `VLN-11304-CTC-PM1` may map to approved equipment + `plasmatherm_versaline_icp_rie_etcher`. +- The app accepts at most 64 MiB. A separate exact-path ingress permits 70 MiB + for multipart overhead only at `/api/dt/data/upload`. +- Raw files use a 5 GiB encrypted, `ReadWriteMany` pilot PVC whose storage class + deletes the backing volume when the claim is deleted. +- A DB3 processing lease is refreshed every 60 seconds while work is active. It + becomes retryable after 30 minutes without a heartbeat if an API restart + interrupts the in-process task. Trace writes are idempotent. +- Existing `dt-api-secret` and `dt-web-secret` references are preserved. No + secret value is stored in this directory. + +Use a dedicated private pilot project containing only Paul, Rich, and the pilot +administrator. The allowlist is an additional boundary, not a replacement for +normal project membership and equipment authorization. + +The live pilot is pinned to private project +`glance_db3_paul_rich_pilot_89f3f0b8` (`GLANCE DB3 Paul-Rich Pilot`), tied to +Paul's approved Versaline ICP RIE registration. Its members are `ngholiza`, +`will3042`, and `hosler0`. + +## Preflight and deploy + +Build and push API/web images from this branch with a unique tag, then resolve +each pushed tag to its registry digest. Pass the full immutable references; the +script rejects tag-only images. For example: + +```sh +export KUBECONFIG=/tmp/geddes-kubeconfig.yaml +export KUBE_CONTEXT=geddes +export NAMESPACE=nanohub + +API_IMAGE='geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-@sha256:' +WEB_IMAGE='geddes-registry.rcac.purdue.edu/sdx/dt-web:db3-pilot-@sha256:' + +./deploy.sh --api-image "$API_IMAGE" --web-image "$WEB_IMAGE" --dry-run +./deploy.sh --api-image "$API_IMAGE" --web-image "$WEB_IMAGE" +``` + +`--dry-run` performs server-side validation of the ConfigMap, PVC, exact-path +Ingress, and both deployment patches without changing the cluster. Deployment +applies storage/config first, rolls out and validates the API before the web +image, then exposes the large-body ingress only after both are ready. +The script rejects non-pilot tags, unexpected repositories, and deployments +that have moved beyond the recorded v52/v53 baseline. A ConfigMap checksum in +the API pod template guarantees configuration changes trigger a rollout. + +The package is intentionally fixed to the manifest namespace `nanohub`; the +scripts reject any other `NAMESPACE` value instead of silently deploying a +cross-namespace mix of resources. `KUBE_CONTEXT` is explicit and configurable. + +After deployment, verify the normal site and upload a real DB3 through the +browser-facing `/api/dt/data/upload` path. Confirm the created runs and trace +samples are visible only inside the private pilot project. Record the upload +UUIDs and project ID; they are the safest cleanup keys. + +## Rollback + +The rollback is pinned to the exact pre-pilot images observed on Geddes: + +- API v52: `sha256:b99e0ed5fca6ec908c41dd96b9351f400677f2a494be3bbe964284096f3443c6` +- Web v53: `sha256:8bfda58813952bbfca463dd6c901de9a73f4ef0b0f2f2d74d72fc626b164bd75` + +Validate, then roll back the route and workloads: + +```sh +./rollback.sh --dry-run +./rollback.sh +``` + +The default rollback immediately removes the large-body ingress, restores the +two pre-pilot image digests, removes the ConfigMap reference and PVC mount from +`dt-api`, waits for the rollout, and deletes the pilot ConfigMap. It intentionally +retains the PVC so rollback cannot erase Paul/Rich uploads by accident. + +Do not delete the PVC while a DB3 request is processing. Once the rollback has +finished, needed raw files have been copied elsewhere, and the upload UUIDs have +been recorded, complete resource removal with an explicit acknowledgement: + +```sh +CONFIRM_NO_DB3_UPLOAD_PROCESSING=yes ./rollback.sh --delete-pvc +``` + +The second command is cleanup-only: it refuses to run while a pilot image, +ConfigMap reference, or volume mount remains, and it never changes deployment +images. This prevents a delayed PVC cleanup from downgrading a later platform +release. The encrypted storage class uses a `Delete` reclaim policy, so PVC +deletion erases the backing volume rather than archiving it. + +## Processed database cleanup + +The raw DB3 file on the PVC and the processed PostgreSQL rows are independent. +Deleting the PVC does not remove database rows. Cleanup must be based on the +recorded pilot upload UUIDs, reviewed before execution, and run by an authorized +database administrator. A safe transaction shape is: + +```sql +BEGIN; + +CREATE TEMP TABLE db3_pilot_upload_ids ( + id uuid PRIMARY KEY +) ON COMMIT DROP; + +-- Replace these placeholders only with UUIDs recorded from this pilot. +INSERT INTO db3_pilot_upload_ids (id) VALUES + (''); + +-- Inspect the exact scope before either DELETE. +SELECT id, equipment_id, project_id, upload_id, upload_filename +FROM equipment_runs +WHERE upload_id IN (SELECT id FROM db3_pilot_upload_ids); + +SELECT id, equipment_id, filename, owner_id, uploaded_at +FROM data_uploads +WHERE id IN (SELECT id FROM db3_pilot_upload_ids); + +-- Deleting parent runs cascades to equipment_run_trace_samples. +DELETE FROM equipment_runs +WHERE upload_id IN (SELECT id FROM db3_pilot_upload_ids); + +DELETE FROM data_uploads +WHERE id IN (SELECT id FROM db3_pilot_upload_ids); + +COMMIT; +``` + +Remove the dedicated private pilot project only after checking it contains no +non-pilot work. Project deletion alone is not DB3 cleanup because the +`equipment_runs.project_id` foreign key uses `ON DELETE SET NULL`. + +The additive `equipment_run_trace_samples` table and its index/RLS policy may +remain after rollback; they are inert without pilot code and data. Do not drop +the table as part of the operational rollback. diff --git a/geddes/k8s/pilots/db3/configmap.yaml b/geddes/k8s/pilots/db3/configmap.yaml new file mode 100644 index 0000000..4ae3522 --- /dev/null +++ b/geddes/k8s/pilots/db3/configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: dt-db3-pilot-config + namespace: nanohub + labels: + app.kubernetes.io/part-of: dt-db3-pilot + app.kubernetes.io/managed-by: dt-db3-pilot-scripts +data: + # This path is deliberately disabled by default in application code. These + # values opt in only the temporary Geddes pilot deployment. + DT_ENABLE_DB3_UPLOAD: "true" + DT_DB3_ALLOWED_USER_IDS: "ngholiza,will3042,hosler0" + DT_DB3_EQUIPMENT_TOOL_MAP: '{"plasmatherm_versaline_icp_rie_etcher":["VLN-11304-CTC-PM1"]}' + DT_DB3_PILOT_PROJECT_ID: "glance_db3_paul_rich_pilot_89f3f0b8" + DT_DB3_UPLOAD_MAX_MB: "64" + DT_DB3_PROCESSING_STALE_MINUTES: "30" + DT_DB3_PROCESSING_HEARTBEAT_SECONDS: "60" + DT_UPLOADS_DIR: "/var/lib/dt-api/uploads" diff --git a/geddes/k8s/pilots/db3/deploy.sh b/geddes/k8s/pilots/db3/deploy.sh new file mode 100755 index 0000000..4dc77d0 --- /dev/null +++ b/geddes/k8s/pilots/db3/deploy.sh @@ -0,0 +1,276 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +KUBE_CONTEXT="${KUBE_CONTEXT:-geddes}" +NAMESPACE="${NAMESPACE:-nanohub}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" + +API_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-api" +WEB_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-web" +BASELINE_API_TAG="$API_REPOSITORY:v52" +BASELINE_WEB_TAG="$WEB_REPOSITORY:v53" +BASELINE_API_DIGEST="$API_REPOSITORY@sha256:b99e0ed5fca6ec908c41dd96b9351f400677f2a494be3bbe964284096f3443c6" +BASELINE_WEB_DIGEST="$WEB_REPOSITORY@sha256:8bfda58813952bbfca463dd6c901de9a73f4ef0b0f2f2d74d72fc626b164bd75" + +API_IMAGE="" +WEB_IMAGE="" +DRY_RUN=false + +usage() { + cat <<'USAGE' +Usage: + deploy.sh --api-image IMAGE@sha256:DIGEST --web-image IMAGE@sha256:DIGEST [--dry-run] + +Environment: + KUBECONFIG kubeconfig file used by kubectl (recommended: set explicitly) + KUBE_CONTEXT target context (default: geddes) + NAMESPACE target namespace (default: nanohub) + ROLLOUT_TIMEOUT kubectl rollout timeout (default: 300s) + +Only digest-pinned images are accepted. A branch-only tag may precede the +digest, for example dt-api:db3-pilot-abc123@sha256:... +USAGE +} + +while (($#)); do + case "$1" in + --api-image) + [[ $# -ge 2 ]] || { usage >&2; exit 2; } + API_IMAGE="$2" + shift 2 + ;; + --web-image) + [[ $# -ge 2 ]] || { usage >&2; exit 2; } + WEB_IMAGE="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +require_command() { + command -v "$1" >/dev/null 2>&1 || { + echo "Required command not found: $1" >&2 + exit 1 + } +} + +validate_image() { + local label="$1" + local image="$2" + local repository="$3" + if [[ ! "$image" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "$label must be pinned to a sha256 digest: $image" >&2 + exit 2 + fi + local tagged_image="${image%@sha256:*}" + local tag="${tagged_image#"$repository:"}" + if [[ "$tagged_image" == "$image" \ + || "$tagged_image" != "$repository:"* \ + || ! "$tag" =~ ^db3-pilot-[a-zA-Z0-9._-]+$ ]]; then + echo "$label must use $repository:db3-pilot-@sha256:" >&2 + exit 2 + fi +} + +validate_current_image() { + local label="$1" + local image="$2" + local baseline_tag="$3" + local baseline_digest="$4" + local repository="$5" + if [[ "$image" == "$baseline_tag" || "$image" == "$baseline_digest" ]]; then + return + fi + if [[ "$image" == "$repository":db3-pilot-* ]]; then + return + fi + echo "Refusing pilot deploy: $label is not on the recorded baseline or an existing DB3 pilot image: $image" >&2 + exit 1 +} + +require_command kubectl +require_command jq +require_command shasum + +[[ -n "$API_IMAGE" && -n "$WEB_IMAGE" ]] || { + usage >&2 + exit 2 +} +validate_image "--api-image" "$API_IMAGE" "$API_REPOSITORY" +validate_image "--web-image" "$WEB_IMAGE" "$WEB_REPOSITORY" + +config_json="$(kubectl create --dry-run=client -o json \ + -f "$SCRIPT_DIR/configmap.yaml")" +pilot_project_id="$(jq -r '.data.DT_DB3_PILOT_PROJECT_ID // ""' \ + <<<"$config_json")" +if [[ -z "$pilot_project_id" \ + || "$pilot_project_id" == REPLACE_WITH_* ]]; then + echo "DT_DB3_PILOT_PROJECT_ID must name the existing private pilot project" >&2 + exit 2 +fi +config_checksum="$(shasum -a 256 "$SCRIPT_DIR/configmap.yaml" | awk '{print $1}')" + +KUBECTL=(kubectl --context "$KUBE_CONTEXT" --namespace "$NAMESPACE") + +if [[ "$NAMESPACE" != "nanohub" ]]; then + echo "These manifests are explicitly scoped to namespace nanohub; got: $NAMESPACE" >&2 + exit 2 +fi + +if ! kubectl config get-contexts "$KUBE_CONTEXT" -o name 2>/dev/null \ + | grep -Fxq "$KUBE_CONTEXT"; then + echo "kubectl context does not exist: $KUBE_CONTEXT" >&2 + exit 1 +fi + +echo "Target: context=$KUBE_CONTEXT namespace=$NAMESPACE" +echo "API image: $API_IMAGE" +echo "Web image: $WEB_IMAGE" + +api_json="$("${KUBECTL[@]}" get deployment dt-api -o json)" +web_json="$("${KUBECTL[@]}" get deployment dt-web -o json)" + +current_api_image="$(jq -r '.spec.template.spec.containers[] | select(.name == "dt-api") | .image' <<<"$api_json")" +current_web_image="$(jq -r '.spec.template.spec.containers[] | select(.name == "dt-web") | .image' <<<"$web_json")" +validate_current_image "dt-api" "$current_api_image" "$BASELINE_API_TAG" \ + "$BASELINE_API_DIGEST" "$API_REPOSITORY" +validate_current_image "dt-web" "$current_web_image" "$BASELINE_WEB_TAG" \ + "$BASELINE_WEB_DIGEST" "$WEB_REPOSITORY" + +jq -e '.spec.template.spec.containers + | any(.name == "dt-api")' >/dev/null <<<"$api_json" || { + echo "deployment/dt-api has no container named dt-api" >&2 + exit 1 +} +jq -e '.spec.template.spec.containers + | any(.name == "dt-web")' >/dev/null <<<"$web_json" || { + echo "deployment/dt-web has no container named dt-web" >&2 + exit 1 +} +jq -e '.spec.template.spec.containers[] + | select(.name == "dt-api") + | (.envFrom // []) + | any(.secretRef.name == "dt-api-secret")' >/dev/null <<<"$api_json" || { + echo "Refusing to patch: dt-api-secret is not present in dt-api envFrom" >&2 + exit 1 +} + +"${KUBECTL[@]}" get secret dt-api-secret >/dev/null +kubectl --context "$KUBE_CONTEXT" get storageclass \ + ceph-multinode-encrypted >/dev/null + +api_env_from="$(jq -c ' + [.spec.template.spec.containers[] + | select(.name == "dt-api") + | (.envFrom // [])][0] + | if any(.configMapRef.name == "dt-db3-pilot-config") + then . + else . + [{"configMapRef":{"name":"dt-db3-pilot-config"}}] + end' <<<"$api_json")" +api_mounts="$(jq -c ' + [.spec.template.spec.containers[] + | select(.name == "dt-api") + | (.volumeMounts // [])][0] + | if any(.name == "dt-db3-pilot-uploads") + then . + else . + [{ + "name":"dt-db3-pilot-uploads", + "mountPath":"/var/lib/dt-api/uploads" + }] + end' <<<"$api_json")" +api_volumes="$(jq -c ' + (.spec.template.spec.volumes // []) + | if any(.name == "dt-db3-pilot-uploads") + then . + else . + [{ + "name":"dt-db3-pilot-uploads", + "persistentVolumeClaim":{"claimName":"dt-db3-pilot-uploads"} + }] + end' <<<"$api_json")" + +api_patch="$(jq -nc \ + --arg image "$API_IMAGE" \ + --arg config_checksum "$config_checksum" \ + --argjson env_from "$api_env_from" \ + --argjson mounts "$api_mounts" \ + --argjson volumes "$api_volumes" ' + {"spec":{"template":{ + "metadata":{"annotations":{ + "dt.purdue.edu/db3-pilot-config-sha256":$config_checksum + }}, + "spec":{ + "containers":[{ + "name":"dt-api", + "image":$image, + "envFrom":$env_from, + "volumeMounts":$mounts + }], + "volumes":$volumes + }}}}' )" +web_patch="$(jq -nc --arg image "$WEB_IMAGE" ' + {"spec":{"template":{"spec":{"containers":[{ + "name":"dt-web", + "image":$image + }]}}}}' )" + +echo "Validating pilot resources and deployment patches on the API server..." +"${KUBECTL[@]}" apply --dry-run=server -o name \ + -f "$SCRIPT_DIR/configmap.yaml" \ + -f "$SCRIPT_DIR/pvc.yaml" \ + -f "$SCRIPT_DIR/ingress.yaml" +"${KUBECTL[@]}" patch deployment dt-api --type=strategic \ + --patch "$api_patch" --dry-run=server -o name +"${KUBECTL[@]}" patch deployment dt-web --type=strategic \ + --patch "$web_patch" --dry-run=server -o name + +if [[ "$DRY_RUN" == true ]]; then + echo "Dry run complete; no resources were changed." + exit 0 +fi + +# The upload route is exposed only after both pilot workloads become ready. +"${KUBECTL[@]}" apply \ + -f "$SCRIPT_DIR/configmap.yaml" \ + -f "$SCRIPT_DIR/pvc.yaml" +"${KUBECTL[@]}" patch deployment dt-api --type=strategic \ + --patch "$api_patch" +"${KUBECTL[@]}" rollout status deployment/dt-api \ + --timeout="$ROLLOUT_TIMEOUT" + +# API startup installs/validates the additive trace schema and capability +# before the pilot UI can direct a user into the new workflow. +"${KUBECTL[@]}" patch deployment dt-web --type=strategic \ + --patch "$web_patch" +"${KUBECTL[@]}" rollout status deployment/dt-web \ + --timeout="$ROLLOUT_TIMEOUT" +"${KUBECTL[@]}" apply -f "$SCRIPT_DIR/ingress.yaml" + +actual_api_image="$("${KUBECTL[@]}" get deployment dt-api \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="dt-api")].image}')" +actual_web_image="$("${KUBECTL[@]}" get deployment dt-web \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="dt-web")].image}')" +[[ "$actual_api_image" == "$API_IMAGE" ]] || { + echo "dt-api image verification failed: $actual_api_image" >&2 + exit 1 +} +[[ "$actual_web_image" == "$WEB_IMAGE" ]] || { + echo "dt-web image verification failed: $actual_web_image" >&2 + exit 1 +} + +echo "DB3 pilot deployed. The 70 MiB ingress exception is limited to:" +echo " https://dt-nanohub.geddes.rcac.purdue.edu/api/dt/data/upload" diff --git a/geddes/k8s/pilots/db3/ingress.yaml b/geddes/k8s/pilots/db3/ingress.yaml new file mode 100644 index 0000000..c4546fb --- /dev/null +++ b/geddes/k8s/pilots/db3/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: dt-db3-pilot-upload + namespace: nanohub + labels: + app.kubernetes.io/part-of: dt-db3-pilot + app.kubernetes.io/managed-by: dt-db3-pilot-scripts + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "70m" + nginx.ingress.kubernetes.io/proxy-request-buffering: "off" + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" +spec: + rules: + - host: dt-nanohub.geddes.rcac.purdue.edu + http: + paths: + - path: /api/dt/data/upload + pathType: Exact + backend: + service: + name: dt-web + port: + number: 3000 diff --git a/geddes/k8s/pilots/db3/pvc.yaml b/geddes/k8s/pilots/db3/pvc.yaml new file mode 100644 index 0000000..85c9055 --- /dev/null +++ b/geddes/k8s/pilots/db3/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: dt-db3-pilot-uploads + namespace: nanohub + labels: + app.kubernetes.io/part-of: dt-db3-pilot + app.kubernetes.io/managed-by: dt-db3-pilot-scripts +spec: + accessModes: + - ReadWriteMany + storageClassName: ceph-multinode-encrypted + resources: + requests: + storage: 5Gi diff --git a/geddes/k8s/pilots/db3/rollback.sh b/geddes/k8s/pilots/db3/rollback.sh new file mode 100755 index 0000000..1aba1c2 --- /dev/null +++ b/geddes/k8s/pilots/db3/rollback.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +set -euo pipefail + +KUBE_CONTEXT="${KUBE_CONTEXT:-geddes}" +NAMESPACE="${NAMESPACE:-nanohub}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" + +# These are the exact images running before the pilot. Keep them digest-pinned +# even though v52/v53 are retained here as human-readable provenance. +ROLLBACK_API_IMAGE="geddes-registry.rcac.purdue.edu/sdx/dt-api@sha256:b99e0ed5fca6ec908c41dd96b9351f400677f2a494be3bbe964284096f3443c6" +ROLLBACK_WEB_IMAGE="geddes-registry.rcac.purdue.edu/sdx/dt-web@sha256:8bfda58813952bbfca463dd6c901de9a73f4ef0b0f2f2d74d72fc626b164bd75" +BASELINE_API_TAG="geddes-registry.rcac.purdue.edu/sdx/dt-api:v52" +BASELINE_WEB_TAG="geddes-registry.rcac.purdue.edu/sdx/dt-web:v53" +API_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-api" +WEB_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-web" + +DRY_RUN=false +DELETE_PVC=false + +usage() { + cat <<'USAGE' +Usage: + rollback.sh [--dry-run] [--delete-pvc] + +Environment: + KUBECONFIG kubeconfig file used by kubectl (recommended: set explicitly) + KUBE_CONTEXT target context (default: geddes) + NAMESPACE target namespace (default: nanohub) + ROLLOUT_TIMEOUT kubectl rollout timeout (default: 300s) + +The default rollback disables the route, removes the pilot configuration and +mount from dt-api, and restores the pre-pilot v52/v53 image digests. It retains +the PVC to protect uploaded files. After that rollback, erase the PVC in a +separate cleanup-only run with --delete-pvc and +CONFIRM_NO_DB3_UPLOAD_PROCESSING=yes after recording/backing up needed data. +USAGE +} + +while (($#)); do + case "$1" in + --dry-run) + DRY_RUN=true + shift + ;; + --delete-pvc) + DELETE_PVC=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +require_command() { + command -v "$1" >/dev/null 2>&1 || { + echo "Required command not found: $1" >&2 + exit 1 + } +} + +require_command kubectl +require_command jq + +if [[ "$DELETE_PVC" == true \ + && "${CONFIRM_NO_DB3_UPLOAD_PROCESSING:-}" != "yes" ]]; then + echo "PVC deletion requires CONFIRM_NO_DB3_UPLOAD_PROCESSING=yes" >&2 + exit 2 +fi + +KUBECTL=(kubectl --context "$KUBE_CONTEXT" --namespace "$NAMESPACE") + +if [[ "$NAMESPACE" != "nanohub" ]]; then + echo "These manifests are explicitly scoped to namespace nanohub; got: $NAMESPACE" >&2 + exit 2 +fi + +if ! kubectl config get-contexts "$KUBE_CONTEXT" -o name 2>/dev/null \ + | grep -Fxq "$KUBE_CONTEXT"; then + echo "kubectl context does not exist: $KUBE_CONTEXT" >&2 + exit 1 +fi + +echo "Target: context=$KUBE_CONTEXT namespace=$NAMESPACE" +echo "Rollback API image (pre-pilot v52): $ROLLBACK_API_IMAGE" +echo "Rollback web image (pre-pilot v53): $ROLLBACK_WEB_IMAGE" + +api_json="$("${KUBECTL[@]}" get deployment dt-api -o json)" +web_json="$("${KUBECTL[@]}" get deployment dt-web -o json)" + +current_api_image="$(jq -r '.spec.template.spec.containers[] | select(.name == "dt-api") | .image' <<<"$api_json")" +current_web_image="$(jq -r '.spec.template.spec.containers[] | select(.name == "dt-web") | .image' <<<"$web_json")" + +if [[ "$DELETE_PVC" == true ]]; then + if [[ "$current_api_image" == "$API_REPOSITORY":db3-pilot-* \ + || "$current_web_image" == "$WEB_REPOSITORY":db3-pilot-* ]]; then + echo "Refusing PVC cleanup while a pilot image is still deployed; run the normal rollback first" >&2 + exit 1 + fi + if jq -e ' + (.spec.template.spec.volumes // []) + | any(.name == "dt-db3-pilot-uploads")' >/dev/null <<<"$api_json"; then + echo "Refusing PVC cleanup: dt-api still references the pilot volume" >&2 + exit 1 + fi + if jq -e ' + .spec.template.spec.containers[] | select(.name == "dt-api") + | ((.envFrom // []) | any(.configMapRef.name == "dt-db3-pilot-config")) + or ((.volumeMounts // []) | any(.name == "dt-db3-pilot-uploads"))' \ + >/dev/null <<<"$api_json"; then + echo "Refusing PVC cleanup: dt-api still has pilot configuration or mounts" >&2 + exit 1 + fi + if [[ "$DRY_RUN" == true ]]; then + echo "Dry run complete; PVC cleanup preconditions pass and no resources were changed." + exit 0 + fi + "${KUBECTL[@]}" delete ingress dt-db3-pilot-upload --ignore-not-found + "${KUBECTL[@]}" delete configmap dt-db3-pilot-config --ignore-not-found + "${KUBECTL[@]}" delete pvc dt-db3-pilot-uploads --ignore-not-found + echo "Pilot PVC deleted; the encrypted storage class has Delete reclaim policy." + exit 0 +fi + +validate_rollback_source() { + local label="$1" + local image="$2" + local baseline_tag="$3" + local baseline_digest="$4" + local repository="$5" + if [[ "$image" == "$baseline_tag" || "$image" == "$baseline_digest" \ + || "$image" == "$repository":db3-pilot-* ]]; then + return + fi + echo "Refusing rollback: $label is a later or unrelated release, not the recorded baseline/pilot: $image" >&2 + exit 1 +} + +validate_rollback_source "dt-api" "$current_api_image" "$BASELINE_API_TAG" \ + "$ROLLBACK_API_IMAGE" "$API_REPOSITORY" +validate_rollback_source "dt-web" "$current_web_image" "$BASELINE_WEB_TAG" \ + "$ROLLBACK_WEB_IMAGE" "$WEB_REPOSITORY" + +api_index="$(jq -r ' + [.spec.template.spec.containers[].name] | index("dt-api")' <<<"$api_json")" +web_index="$(jq -r ' + [.spec.template.spec.containers[].name] | index("dt-web")' <<<"$web_json")" +[[ "$api_index" != "null" ]] || { + echo "deployment/dt-api has no container named dt-api" >&2 + exit 1 +} +[[ "$web_index" != "null" ]] || { + echo "deployment/dt-web has no container named dt-web" >&2 + exit 1 +} + +api_env_from="$(jq -c --argjson i "$api_index" ' + (.spec.template.spec.containers[$i].envFrom // []) + | map(select(.configMapRef.name != "dt-db3-pilot-config"))' \ + <<<"$api_json")" +api_mounts="$(jq -c --argjson i "$api_index" ' + (.spec.template.spec.containers[$i].volumeMounts // []) + | map(select(.name != "dt-db3-pilot-uploads"))' <<<"$api_json")" +api_volumes="$(jq -c ' + (.spec.template.spec.volumes // []) + | map(select(.name != "dt-db3-pilot-uploads"))' <<<"$api_json")" +api_has_env_from="$(jq -c --argjson i "$api_index" ' + .spec.template.spec.containers[$i] | has("envFrom")' <<<"$api_json")" +api_has_mounts="$(jq -c --argjson i "$api_index" ' + .spec.template.spec.containers[$i] | has("volumeMounts")' <<<"$api_json")" +api_has_volumes="$(jq -c ' + .spec.template.spec | has("volumes")' <<<"$api_json")" +api_has_checksum="$(jq -c ' + (.spec.template.metadata.annotations // {}) + | has("dt.purdue.edu/db3-pilot-config-sha256")' <<<"$api_json")" + +# JSON Patch replaces the complete arrays after filtering only the named pilot +# entries. This preserves every unrelated live envFrom, mount, and volume. +api_patch="$(jq -nc \ + --argjson i "$api_index" \ + --arg image "$ROLLBACK_API_IMAGE" \ + --argjson env_from "$api_env_from" \ + --argjson mounts "$api_mounts" \ + --argjson volumes "$api_volumes" \ + --argjson has_env_from "$api_has_env_from" \ + --argjson has_mounts "$api_has_mounts" \ + --argjson has_volumes "$api_has_volumes" \ + --argjson has_checksum "$api_has_checksum" ' + [ + {"op":"replace", + "path":("/spec/template/spec/containers/" + ($i|tostring) + "/image"), + "value":$image} + ] + + (if $has_env_from then [( + {"op":(if $env_from == [] then "remove" else "replace" end), + "path":("/spec/template/spec/containers/" + ($i|tostring) + "/envFrom")} + + (if $env_from == [] then {} else {"value":$env_from} end) + )] else [] end) + + (if $has_mounts then [( + {"op":(if $mounts == [] then "remove" else "replace" end), + "path":("/spec/template/spec/containers/" + ($i|tostring) + "/volumeMounts")} + + (if $mounts == [] then {} else {"value":$mounts} end) + )] else [] end) + + (if $has_volumes then [( + {"op":(if $volumes == [] then "remove" else "replace" end), + "path":"/spec/template/spec/volumes"} + + (if $volumes == [] then {} else {"value":$volumes} end) + )] else [] end) + + (if $has_checksum then [{ + "op":"remove", + "path":"/spec/template/metadata/annotations/dt.purdue.edu~1db3-pilot-config-sha256" + }] else [] end)' )" +web_patch="$(jq -nc \ + --argjson i "$web_index" \ + --arg image "$ROLLBACK_WEB_IMAGE" ' + [{ + "op":"replace", + "path":("/spec/template/spec/containers/" + ($i|tostring) + "/image"), + "value":$image + }]' )" + +echo "Validating rollback patches on the API server..." +"${KUBECTL[@]}" patch deployment dt-api --type=json \ + --patch "$api_patch" --dry-run=server -o name +"${KUBECTL[@]}" patch deployment dt-web --type=json \ + --patch "$web_patch" --dry-run=server -o name + +if [[ "$DRY_RUN" == true ]]; then + echo "Dry run complete; no resources were changed." + exit 0 +fi + +# Stop admitting new large uploads before changing either workload. +"${KUBECTL[@]}" delete ingress dt-db3-pilot-upload --ignore-not-found +"${KUBECTL[@]}" patch deployment dt-api --type=json --patch "$api_patch" +"${KUBECTL[@]}" patch deployment dt-web --type=json --patch "$web_patch" + +"${KUBECTL[@]}" rollout status deployment/dt-api \ + --timeout="$ROLLOUT_TIMEOUT" +"${KUBECTL[@]}" rollout status deployment/dt-web \ + --timeout="$ROLLOUT_TIMEOUT" + +"${KUBECTL[@]}" delete configmap dt-db3-pilot-config --ignore-not-found + +echo "Pilot workload rollback complete. PVC retained for data safety:" +echo " persistentvolumeclaim/dt-db3-pilot-uploads" +echo "After DB/file retention decisions, erase it with:" +echo " CONFIRM_NO_DB3_UPLOAD_PROCESSING=yes $0 --delete-pvc" diff --git a/geddes/k8s/postgres/schema_v2.sql b/geddes/k8s/postgres/schema_v2.sql index 6d5f69e..0653384 100644 --- a/geddes/k8s/postgres/schema_v2.sql +++ b/geddes/k8s/postgres/schema_v2.sql @@ -356,6 +356,7 @@ CREATE TABLE IF NOT EXISTS data_uploads ( kind VARCHAR(20) NOT NULL DEFAULT 'all', owner_id VARCHAR(255) REFERENCES users(id), owner_org VARCHAR(255) DEFAULT '', + processing_started_at TIMESTAMP WITH TIME ZONE, uploaded_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); diff --git a/web/app/data/upload/page.tsx b/web/app/data/upload/page.tsx index b6ec1f4..139a2a2 100644 --- a/web/app/data/upload/page.tsx +++ b/web/app/data/upload/page.tsx @@ -4,6 +4,7 @@ import { useState, useRef, useEffect } from "react"; import { cn } from "@/lib/utils"; import { getEquipmentListSimple, + getUploadCapabilities, getUploads, getV2Projects, processUpload, @@ -12,6 +13,7 @@ import { type PersistedUpload, type UploadResult, type UploadKind, + type UploadCapabilities, type V2Project, } from "@/lib/api-client"; import { @@ -133,24 +135,34 @@ export default function DataUploadPage() { const [selectedProject, setSelectedProject] = useState(""); const [processingUploadId, setProcessingUploadId] = useState(null); const [uploadKind, setUploadKind] = useState("input"); + const [uploadCapabilities, setUploadCapabilities] = useState({ + db3_upload: { enabled: false, max_mb: 64, equipment_ids: [] }, + }); const fileInputRef = useRef(null); // The canonical etcher is ingested as one combined file; every other // registered tool uses paired input/output templates. const isEtcher = selectedEquipment === "etcher"; const effectiveKind: UploadKind = isEtcher ? "all" : uploadKind; + const db3EnabledForSelectedEquipment = + uploadCapabilities.db3_upload.enabled && + uploadCapabilities.db3_upload.equipment_ids.includes(selectedEquipment); useEffect(() => { const fetchEquipment = async () => { - const [equipmentData, uploadData, projectData] = await Promise.all([ + const [equipmentData, uploadData, projectData, capabilities] = await Promise.all([ getEquipmentListSimple(), getUploads(), getV2Projects(), + getUploadCapabilities(), ]); setEquipment(equipmentData ?? []); setFiles((uploadData ?? []).filter(shouldRestoreUpload).map(mapPersistedUpload)); setProjects(projectData ?? []); + if (capabilities) { + setUploadCapabilities(capabilities); + } setEquipmentLoading(false); if (equipmentData && equipmentData.length > 0) { @@ -225,6 +237,24 @@ export default function DataUploadPage() { for (const rawFile of Array.from(fileList)) { const ts = new Date().toISOString(); + if (isDb3File(rawFile.name) && !db3EnabledForSelectedEquipment) { + setFiles((prev) => [ + { + name: rawFile.name, + size: rawFile.size, + type: rawFile.type || "application/vnd.sqlite3", + status: "error", + equipmentId: selectedEquipment, + kind: "all", + timestamp: ts, + errors: [ + "DB3 import is not enabled for this user and selected equipment.", + ], + }, + ...prev, + ]); + continue; + } const fileKind: UploadKind = isDb3File(rawFile.name) ? "all" : effectiveKind; // Add file in "uploading" state @@ -354,6 +384,7 @@ export default function DataUploadPage() {

Upload Data

Store CSV or Excel files against registered equipment and run lightweight structural validation. + {uploadCapabilities.db3_upload.enabled && " A temporary DB3 pilot is enabled for approved participants."}

@@ -492,7 +523,7 @@ export default function DataUploadPage() { ref={fileInputRef} type="file" multiple - accept=".csv,.xlsx,.xls,.db3" + accept={db3EnabledForSelectedEquipment ? ".csv,.xlsx,.xls,.db3" : ".csv,.xlsx,.xls"} onChange={handleFileSelect} className="hidden" /> @@ -502,7 +533,9 @@ export default function DataUploadPage() {

Drop files here or click to browse

- CSV/Excel max 50 MB; temporary DB3 limit is server-configured (64 MB by default) + {db3EnabledForSelectedEquipment + ? `CSV/Excel max 50 MB; temporary DB3 pilot max ${uploadCapabilities.db3_upload.max_mb} MB` + : "CSV/Excel max 50 MB"}

@@ -511,9 +544,11 @@ export default function DataUploadPage() { Excel - - DB3 - + {db3EnabledForSelectedEquipment && ( + + DB3 pilot + + )}
@@ -658,7 +693,8 @@ export default function DataUploadPage() {

3. Processing

- Processing runs in the background and the status updates here automatically — no need to reload. Input and output CSV files are merged by lot name and run date. Excel files are stored for provenance but are not ingested automatically. The temporary DB3 test path creates one catalog row per physical GLANCE run and stores every timestamped sample as its linked trace. + Processing runs in the background and the status updates here automatically — no need to reload. Input and output CSV files are merged by lot name and run date. Excel files are stored for provenance but are not ingested automatically. + {uploadCapabilities.db3_upload.enabled && " The temporary DB3 pilot creates one catalog row per physical GLANCE run and stores every timestamped sample as its linked trace."}

diff --git a/web/lib/api-client.ts b/web/lib/api-client.ts index 75c8e4f..4ce3b18 100644 --- a/web/lib/api-client.ts +++ b/web/lib/api-client.ts @@ -1316,7 +1316,6 @@ export interface PersistedUpload { equipment_id: string; filename: string; stored_as: string; - storage_path: string; size_bytes: number; row_count: number; columns: string[]; @@ -1328,6 +1327,14 @@ export interface PersistedUpload { uploaded_at: string; } +export interface UploadCapabilities { + db3_upload: { + enabled: boolean; + max_mb: number; + equipment_ids: string[]; + }; +} + export interface ProcessUploadResult { status: "processing" | "processed" | "failed" | string; upload_id: string; @@ -1390,6 +1397,10 @@ export function getUploads() { return apiFetch("/data/uploads"); } +export function getUploadCapabilities() { + return apiFetch("/data/upload-capabilities"); +} + export function processUpload(uploadId: string, projectId: string) { return apiJson( `/data/uploads/${encodeURIComponent(uploadId)}/process`, From 5f231696b8b1cea456fd8b5810026b46b8accc7d Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Mon, 13 Jul 2026 20:30:21 -0400 Subject: [PATCH 4/6] Report DB3 upload limit in megabytes --- api/routers/upload.py | 2 +- api/tests/test_glance_db3_trace_import.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/routers/upload.py b/api/routers/upload.py index 94831cd..c2e2e00 100644 --- a/api/routers/upload.py +++ b/api/routers/upload.py @@ -1296,7 +1296,7 @@ def get_upload_capabilities( return { "db3_upload": { "enabled": _db3_user_is_allowed(user) and bool(mapped_equipment_ids), - "max_mb": _db3_upload_limit_bytes(), + "max_mb": _db3_upload_limit_bytes() // (1024 * 1024), "equipment_ids": mapped_equipment_ids if _db3_user_is_allowed(user) else [], } } diff --git a/api/tests/test_glance_db3_trace_import.py b/api/tests/test_glance_db3_trace_import.py index c3d8a5f..a7ea722 100644 --- a/api/tests/test_glance_db3_trace_import.py +++ b/api/tests/test_glance_db3_trace_import.py @@ -309,6 +309,7 @@ def test_db3_capability_exposes_only_mapped_equipment_to_pilot_users( other_capabilities = upload_router.get_upload_capabilities(other_user) self.assertTrue(pilot_capabilities["db3_upload"]["enabled"]) + self.assertEqual(pilot_capabilities["db3_upload"]["max_mb"], 64) self.assertEqual( pilot_capabilities["db3_upload"]["equipment_ids"], [equipment_id], From 4ec5d7e9f76985950789210840e334b642edecc3 Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Mon, 13 Jul 2026 20:38:54 -0400 Subject: [PATCH 5/6] Record live DB3 pilot validation --- geddes/k8s/pilots/db3/LIVE_TEST_2026-07-13.md | 105 ++++++++++++++++++ geddes/k8s/pilots/db3/README.md | 3 + 2 files changed, 108 insertions(+) create mode 100644 geddes/k8s/pilots/db3/LIVE_TEST_2026-07-13.md diff --git a/geddes/k8s/pilots/db3/LIVE_TEST_2026-07-13.md b/geddes/k8s/pilots/db3/LIVE_TEST_2026-07-13.md new file mode 100644 index 0000000..813e14f --- /dev/null +++ b/geddes/k8s/pilots/db3/LIVE_TEST_2026-07-13.md @@ -0,0 +1,105 @@ +# Geddes DB3 pilot live validation — 2026-07-13 + +This is an operational record for the temporary, branch-only Paul/Rich pilot. +It is not a production-platform specification and must not be merged into the +final platform. The retained file and rows below are intentionally still live +for Paul and Rich; do not delete them until their testing is complete. + +## Deployed pilot + +- Branch: `feat/db3-upload-pipeline` +- Namespace: `nanohub` +- API: `geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-5f23169@sha256:71113340698a18721c91ec31a042f3b4834ea01ff9a6b6038f7ab868383a6acc` +- Web: `geddes-registry.rcac.purdue.edu/sdx/dt-web:db3-pilot-be5289b@sha256:e2917e858ab0da5ba399676358527769f30db99e2d8e13480a7dc4129860e01d` +- Private project: `glance_db3_paul_rich_pilot_89f3f0b8` +- Equipment: `plasmatherm_versaline_icp_rie_etcher` +- Allowed source tool: `VLN-11304-CTC-PM1` +- Members/pilot allowlist: `ngholiza`, `will3042`, `hosler0` +- Storage: encrypted 5 GiB RWX PVC `dt-db3-pilot-uploads` +- Upload ingress exception: exact path `/api/dt/data/upload`, 70 MiB; the + application limit remains 64 MiB. + +Both deployments were 1/1 Ready with zero restarts after import. The public +health endpoint returned HTTP 200. Server-side dry runs of both the exact live +deploy and the pinned rollback passed without changing resources. + +## Retained validation upload + +- Upload UUID: `014864df-70a2-4137-a245-80210a54494b` +- Source filename: `MyDatabaseSample.db3` +- Stored filename: `20260714_003147_169913_MyDatabaseSample.db3` +- Size: 40,203,264 bytes +- SHA-256: `8c9472ce37d63c5bbcae129476f0b7f164db191ad286ec36054dcc6402a4b706` +- Status: `processed`, row count 26, zero errors + +The real file traversed the public authenticated Auth.js/Next.js/Ingress/API +path. Its source and PVC copies have the same byte count and SHA-256. The upload +API response does not expose its server-side storage path. + +## Imported data + +Direct PostgreSQL aggregates for the upload UUID: + +- 26 physical `equipment_runs`, all attached to the private pilot project +- 11,697 linked `equipment_run_trace_samples` +- 795,328 retained parameter values +- one retained empty-value sample (`{}`) +- one source tool, `VLN-11304-CTC-PM1` (`source_tool_id=3`) +- 68 distinct parameter keys/source IDs used by the retained traces + +The source DB3 registry contains 242 definitions; the 68 definitions actually +used by these traces are persisted with each run. No unused parameter registry +rows are presented as trace columns. + +Catalog and trace API verification used run `5000000022` (source run `11204`): +the detail and full trace returned HTTP 200 with 35 samples and 68 parameters. +A one-parameter trace returned all 35 timestamps with one value slot per sample, +and relative time covered 0.0 through 36.0 seconds. + +## Authorization and retry checks + +- Paul (`will3042`): private project visible; 26 runs and all 11,697 samples + visible; DB3 capability enabled; retained DB3 upload visible. +- Rich (`hosler0`): the same project, run, trace, capability, and upload access. +- Active nonmember (`denphi`): project absent; zero pilot runs/samples; DB3 + capability disabled; DB3 upload absent; direct detail and trace requests for + run `5000000022` returned HTTP 404. + +RLS is enabled on both parent and trace tables with project-membership/open +project SELECT policies. The parent upload/run uniqueness constraint and trace +source-sample uniqueness constraint, JSON-object check, and trace-time index are +present. + +Two simultaneous retries of the processed upload returned one HTTP 200 and one +HTTP 409. The accepted retry completed with the same 26 runs, 11,697 samples, +and 795,328 values, demonstrating the live claim guard and idempotent trace +replacement. + +## Code/build checks + +- Focused DB3 Python suite: 11 passed. +- DB3-named phase-3 security checks: 2 passed. +- TypeScript type check: passed. +- Next.js production build, locally and in the image build: passed. +- Deployment and rollback scripts: `bash -n` and Geddes server-side dry runs + passed. + +The broader phase-3 security file currently reports 21 passed and five failures +in existing experiment/project-creation tests outside the DB3 paths. One reaches +an unmocked local PostgreSQL connection; the others assert older validation or +module-patching behavior. The changed DB3 tests pass. ESLint likewise retains +seven errors and three warnings in unrelated pre-existing pages; changed files +do not add lint findings. + +## Rollback and cleanup identity + +The ordinary rollback restores these exact pre-pilot images while retaining the +PVC: + +- API v52: `sha256:b99e0ed5fca6ec908c41dd96b9351f400677f2a494be3bbe964284096f3443c6` +- Web v53: `sha256:8bfda58813952bbfca463dd6c901de9a73f4ef0b0f2f2d74d72fc626b164bd75` + +Use upload UUID `014864df-70a2-4137-a245-80210a54494b` for reviewed database +cleanup. The raw PVC and PostgreSQL rows are independent; removing one does not +remove the other. Follow the guarded steps in `README.md` only after Paul and +Rich confirm the retained pilot data is no longer needed. diff --git a/geddes/k8s/pilots/db3/README.md b/geddes/k8s/pilots/db3/README.md index 830b8bb..a2c3f53 100644 --- a/geddes/k8s/pilots/db3/README.md +++ b/geddes/k8s/pilots/db3/README.md @@ -31,6 +31,9 @@ The live pilot is pinned to private project Paul's approved Versaline ICP RIE registration. Its members are `ngholiza`, `will3042`, and `hosler0`. +The retained real-file validation and cleanup identifiers are recorded in +[`LIVE_TEST_2026-07-13.md`](LIVE_TEST_2026-07-13.md). + ## Preflight and deploy Build and push API/web images from this branch with a unique tag, then resolve From 344fb62da00fb10e66b0801d61e8fa5e95c56765 Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Tue, 14 Jul 2026 10:59:02 -0400 Subject: [PATCH 6/6] Move DT platform namespace and fix retraining --- .gitignore | 1 + api/FAIR_ARCHIVAL_WALKTHROUGH.md | 2 +- api/data_loader_pg.py | 2 +- api/tests/E2E_README.md | 8 +-- api/tests/test_e2e_admin_flow.py | 4 +- api/tests/test_retrain.py | 62 +++++++++++++++++++ api/training/retrain.py | 39 +++++++++++- geddes/README.md | 4 +- geddes/k8s/01-secrets.yaml.example | 6 +- geddes/k8s/02-api.yaml | 13 +++- geddes/k8s/03-web.yaml | 11 +++- geddes/k8s/04-ingress.yaml | 4 +- geddes/k8s/05-postgres.yaml | 18 ++++-- .../07-postgrest-public-ingress.example.yaml | 2 +- geddes/k8s/07-postgrest-public-ingress.yaml | 2 +- geddes/k8s/08-retrain-cron.yaml | 9 ++- geddes/k8s/pilots/db3/README.md | 4 +- geddes/k8s/pilots/db3/configmap.yaml | 2 +- geddes/k8s/pilots/db3/deploy.sh | 8 +-- geddes/k8s/pilots/db3/ingress.yaml | 2 +- geddes/k8s/pilots/db3/pvc.yaml | 2 +- geddes/k8s/pilots/db3/rollback.sh | 8 +-- 22 files changed, 170 insertions(+), 43 deletions(-) create mode 100644 api/tests/test_retrain.py diff --git a/.gitignore b/.gitignore index e2bdb82..df29bd7 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ debug_*.py !api/tests/test_e2e_admin_flow.py !api/tests/test_e2e_platform.py !api/tests/test_glance_db3_trace_import.py +!api/tests/test_retrain.py # Development files temp.py diff --git a/api/FAIR_ARCHIVAL_WALKTHROUGH.md b/api/FAIR_ARCHIVAL_WALKTHROUGH.md index c310b6f..687e0d9 100644 --- a/api/FAIR_ARCHIVAL_WALKTHROUGH.md +++ b/api/FAIR_ARCHIVAL_WALKTHROUGH.md @@ -55,7 +55,7 @@ Comprehensive rewrite with researcher-friendly guide: setup, usage, architecture 1. **Run the migration**: ```bash - kubectl exec -n nanohub dt-db-v2-0 -- psql -U postgres -d digital_twin -f /tmp/add_fair_columns.sql + kubectl exec -n ncn-digitaltwins-zchen dt-db-v2-0 -- psql -U postgres -d digital_twin -f /tmp/add_fair_columns.sql ``` 2. **Add PURR env vars** to the API deployment: diff --git a/api/data_loader_pg.py b/api/data_loader_pg.py index a3b5617..44c6faf 100644 --- a/api/data_loader_pg.py +++ b/api/data_loader_pg.py @@ -51,7 +51,7 @@ def is_mock_db_enabled() -> bool: def _default_pg_host() -> str: if os.getenv("KUBERNETES_SERVICE_HOST"): - return "dt-db-v2-srv.nanohub.svc.cluster.local" + return "dt-db-v2-srv.ncn-digitaltwins-zchen.svc.cluster.local" return "localhost" diff --git a/api/tests/E2E_README.md b/api/tests/E2E_README.md index 13008f8..c65c9ca 100644 --- a/api/tests/E2E_README.md +++ b/api/tests/E2E_README.md @@ -70,9 +70,9 @@ reachable. ### 1. Pull the shared secrets out of Kubernetes ```bash -kubectl get secret -n nanohub dt-api-secret -o jsonpath='{.data.DT_SYSTEM_TOKEN}' | base64 -d -kubectl get secret -n nanohub dt-api-secret -o jsonpath='{.data.INGESTION_TOKEN}' | base64 -d -kubectl get secret -n nanohub dt-postgres-secret -o jsonpath='{.data.superuser-password}' | base64 -d +kubectl get secret -n ncn-digitaltwins-zchen dt-api-secret -o jsonpath='{.data.DT_SYSTEM_TOKEN}' | base64 -d +kubectl get secret -n ncn-digitaltwins-zchen dt-api-secret -o jsonpath='{.data.INGESTION_TOKEN}' | base64 -d +kubectl get secret -n ncn-digitaltwins-zchen dt-postgres-secret -o jsonpath='{.data.superuser-password}' | base64 -d ``` (Secret key names may differ slightly in your cluster — adjust accordingly.) @@ -82,7 +82,7 @@ kubectl get secret -n nanohub dt-postgres-secret -o jsonpath='{.data.superuser-p In a separate terminal: ```bash -kubectl port-forward -n nanohub svc/dt-db-v2-srv 5432:5432 +kubectl port-forward -n ncn-digitaltwins-zchen svc/dt-db-v2-srv 5432:5432 ``` Skip this step and set `SKIP_ADMIN_BOOTSTRAP=true` if `ngholiza` is already diff --git a/api/tests/test_e2e_admin_flow.py b/api/tests/test_e2e_admin_flow.py index a1b4be3..0a6ad63 100644 --- a/api/tests/test_e2e_admin_flow.py +++ b/api/tests/test_e2e_admin_flow.py @@ -303,7 +303,7 @@ def phase_02_admin_bootstrap(cfg: Config, api: Api, fixtures: Fixtures, report: detail=( f"Could not reach Postgres at {cfg.pg_host}:{cfg.pg_port}: {exc}\n" "If dt-db-v2 is ClusterIP-only, run in another terminal:\n" - " kubectl port-forward -n nanohub svc/dt-db-v2-srv 5432:5432\n" + " kubectl port-forward -n ncn-digitaltwins-zchen svc/dt-db-v2-srv 5432:5432\n" "then rerun this script. Or set SKIP_ADMIN_BOOTSTRAP=true if\n" "you only need header-level admin (UI login will still be\n" "researcher until the row is updated)." @@ -879,7 +879,7 @@ def _diagnose_rls_leak(cfg: Config, fixtures: Fixtures) -> str: except Exception as exc: return ( f"Could not reach Postgres at {cfg.pg_host}:{cfg.pg_port} for diagnostics: {exc}\n" - "Ensure `kubectl port-forward -n nanohub svc/dt-db-v2-srv 5432:5432` is running." + "Ensure `kubectl port-forward -n ncn-digitaltwins-zchen svc/dt-db-v2-srv 5432:5432` is running." ) try: diff --git a/api/tests/test_retrain.py b/api/tests/test_retrain.py new file mode 100644 index 0000000..488904e --- /dev/null +++ b/api/tests/test_retrain.py @@ -0,0 +1,62 @@ +import os +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +import numpy as np + + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +from training import retrain + + +class _DummyEstimator: + def fit(self, X, y): + return self + + def predict(self, X): + return np.zeros(len(X), dtype=float) + + +class RetrainCostControlTests(unittest.TestCase): + def test_training_skips_expensive_loocv_above_limit(self): + X = np.arange(102, dtype=float).reshape(51, 2) + y = np.linspace(0.0, 1.0, 51) + + with patch.object( + retrain, "_make_estimator", side_effect=lambda _use_gpr: _DummyEstimator(), + ) as make_estimator: + result = retrain._train_one_target( + X_raw=X, + y=y, + features=["feature_1", "feature_2"], + target="etch_rate", + gpr_row_limit=400, + force_rf=False, + loocv_max_rows=50, + ) + + # One holdout fit and one final fit. Without the cap this would make + # 51 additional Gaussian-process fits for LOOCV. + self.assertEqual(make_estimator.call_count, 2) + self.assertNotIn("r2_loocv", result["metrics"]) + self.assertNotIn("rmse_loocv", result["metrics"]) + + def test_invalid_environment_limit_uses_safe_default(self): + with patch.dict(os.environ, {"RETRAIN_LOOCV_MAX_ROWS": "invalid"}): + self.assertEqual( + retrain._configured_loocv_max_rows(), + retrain._DEFAULT_LOOCV_MAX_ROWS, + ) + + def test_negative_environment_limit_disables_loocv(self): + with patch.dict(os.environ, {"RETRAIN_LOOCV_MAX_ROWS": "-1"}): + self.assertEqual(retrain._configured_loocv_max_rows(), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/training/retrain.py b/api/training/retrain.py index 22edb2e..7b6e537 100644 --- a/api/training/retrain.py +++ b/api/training/retrain.py @@ -46,6 +46,7 @@ # Target-independent retrain advisory lock key. Any 64-bit int works. _ADVISORY_LOCK_KEY = 784512341 +_DEFAULT_LOOCV_MAX_ROWS = 50 # ── Helpers ────────────────────────────────────────────────────────────────── @@ -58,6 +59,20 @@ def _setup_logging() -> None: ) +def _configured_loocv_max_rows() -> int: + raw_limit = os.getenv( + "RETRAIN_LOOCV_MAX_ROWS", str(_DEFAULT_LOOCV_MAX_ROWS), + ) + try: + return max(0, int(raw_limit)) + except ValueError: + logger.warning( + "Invalid RETRAIN_LOOCV_MAX_ROWS=%r; using %d", + raw_limit, _DEFAULT_LOOCV_MAX_ROWS, + ) + return _DEFAULT_LOOCV_MAX_ROWS + + def _build_gpr_kernel(): return ( ConstantKernel(1.0, constant_value_bounds=(1e-3, 1e3)) @@ -83,6 +98,10 @@ def _loocv_scores( if len(y) == 0: return {} if len(y) > max_n: + logger.info( + "Retrain: skipping LOOCV for %d rows (configured limit=%d)", + len(y), max_n, + ) return {} loo = LeaveOneOut() preds = np.zeros_like(y, dtype=float) @@ -153,6 +172,7 @@ def _train_one_target( target: str, gpr_row_limit: int, force_rf: bool, + loocv_max_rows: int, ) -> dict[str, Any]: """Fit one model for one target, return {bundle, metrics, model_kind}.""" n = len(y) @@ -187,11 +207,13 @@ def _train_one_target( # LOOCV on the full set when affordable. if use_gpr: loo_scores = _loocv_scores( - lambda: _make_estimator(True), X_scaled, y, max_n=200, + lambda: _make_estimator(True), X_scaled, y, + max_n=loocv_max_rows, ) else: loo_scores = _loocv_scores( - lambda: _make_estimator(False), X_raw, y, max_n=400, + lambda: _make_estimator(False), X_raw, y, + max_n=loocv_max_rows, ) metrics.update(loo_scores) @@ -300,12 +322,18 @@ def run_retrain( domain_id: str = "etcher", gpr_row_limit: int = 400, force_rf: bool = False, + loocv_max_rows: Optional[int] = None, ) -> dict[str, Any]: """Train a fresh model per target, save, promote, update state.""" from model_registry import ( get_retrain_state, promote_model, save_model, update_retrain_state, ) + if loocv_max_rows is None: + loocv_max_rows = _configured_loocv_max_rows() + else: + loocv_max_rows = max(0, loocv_max_rows) + snapshot_id, snapshot_hash, num_rows = _current_snapshot() state = get_retrain_state(domain_id) or {} if ( @@ -353,6 +381,7 @@ def run_retrain( trained = _train_one_target( X_raw=X_raw, y=y, features=valid_features, target=target, gpr_row_limit=gpr_row_limit, force_rf=force_rf, + loocv_max_rows=loocv_max_rows, ) metrics = trained["metrics"] @@ -439,6 +468,11 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: help="Domain id (default: etcher).") p.add_argument("--rf-only", action="store_true", help="Skip GPR and always train Random Forest.") + p.add_argument( + "--loocv-max-rows", type=int, default=None, + help="Run full leave-one-out validation only at or below this row " + "count (default: RETRAIN_LOOCV_MAX_ROWS or 50).", + ) return p.parse_args(argv) @@ -450,6 +484,7 @@ def main(argv: Optional[list[str]] = None) -> int: force=bool(ns.force), domain_id=ns.domain, force_rf=bool(ns.rf_only), + loocv_max_rows=ns.loocv_max_rows, ) # Summarise to stdout for CronJob logs. status = out.get("status", "error") diff --git a/geddes/README.md b/geddes/README.md index 98975b8..fc264a9 100644 --- a/geddes/README.md +++ b/geddes/README.md @@ -24,10 +24,10 @@ it cleanly separates the database container and the API container. ### Deployment Steps: 1. **Create the Deployment:** - In Rancher: **Workloads → Deployments → Create** - - Name: `dt-postgrest` (Namespace: `nanohub`) + - Name: `dt-postgrest` (Namespace: `ncn-digitaltwins-zchen`) - Image: `geddes-registry.rcac.purdue.edu/docker-hub-cache/postgrest/postgrest:v13.0.6` 2. **Set Environment Variables:** - - `PGRST_DB_URI` = `postgres://glance:glance_secret@dt-postgres:5432/logger` + - `PGRST_DB_URI` = `postgres://glance:glance_secret@dt-postgres.nanohub.svc.cluster.local:5432/logger` - `PGRST_DB_SCHEMAS` = `public` - `PGRST_DB_ANON_ROLE` = `glance_public` - `PGRST_JWT_SECRET` = `test_secret_that_is_at_least_32_characters_long` diff --git a/geddes/k8s/01-secrets.yaml.example b/geddes/k8s/01-secrets.yaml.example index d5a3076..89f4b45 100644 --- a/geddes/k8s/01-secrets.yaml.example +++ b/geddes/k8s/01-secrets.yaml.example @@ -3,7 +3,7 @@ kind: Secret metadata: name: dt-web-secret # Change this if your actual target namespace is different (e.g. ncn-something) - namespace: nanohub + namespace: ncn-digitaltwins-zchen type: Opaque stringData: # The full public URL of the platform (no trailing slash) @@ -31,11 +31,11 @@ apiVersion: v1 kind: Secret metadata: name: dt-api-secret - namespace: nanohub + namespace: ncn-digitaltwins-zchen type: Opaque stringData: # ── PostgreSQL (V2 multi-tenant database) ── - PG_HOST: "dt-db-v2-srv.nanohub.svc.cluster.local" + PG_HOST: "dt-db-v2-srv.ncn-digitaltwins-zchen.svc.cluster.local" PG_PORT: "5432" PG_DB: "digital_twin" PG_USER: "api_client" diff --git a/geddes/k8s/02-api.yaml b/geddes/k8s/02-api.yaml index 422b7c6..d301e5d 100644 --- a/geddes/k8s/02-api.yaml +++ b/geddes/k8s/02-api.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: dt-api - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: replicas: 1 selector: @@ -18,8 +18,15 @@ spec: containers: - name: dt-api # Notice we are using the 'sdx' namespace on the registry now - image: geddes-registry.rcac.purdue.edu/sdx/dt-api:v52 + image: geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-retrain-loocv-20260714@sha256:52e16ffbbdf71ba87470d8dbf686e6304955501d5b74eaf4c41ba981e14239dc imagePullPolicy: Always + resources: + requests: + cpu: "100m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" ports: - containerPort: 8000 env: @@ -38,7 +45,7 @@ apiVersion: v1 kind: Service metadata: name: dt-api - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: type: ClusterIP ports: diff --git a/geddes/k8s/03-web.yaml b/geddes/k8s/03-web.yaml index d5f7383..7117b10 100644 --- a/geddes/k8s/03-web.yaml +++ b/geddes/k8s/03-web.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: dt-web - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: replicas: 1 selector: @@ -26,6 +26,13 @@ spec: # Notice we are using the 'sdx' namespace on the registry now image: geddes-registry.rcac.purdue.edu/sdx/dt-web:v53 imagePullPolicy: Always + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" ports: - containerPort: 3000 envFrom: @@ -37,7 +44,7 @@ apiVersion: v1 kind: Service metadata: name: dt-web - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: type: ClusterIP ports: diff --git a/geddes/k8s/04-ingress.yaml b/geddes/k8s/04-ingress.yaml index 3e3c38b..70bab7f 100644 --- a/geddes/k8s/04-ingress.yaml +++ b/geddes/k8s/04-ingress.yaml @@ -5,7 +5,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-nanohub-services - namespace: nanohub + namespace: ncn-digitaltwins-zchen annotations: nginx.ingress.kubernetes.io/proxy-buffering: "off" nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" @@ -78,7 +78,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-nanohub-postgrest - namespace: nanohub + namespace: ncn-digitaltwins-zchen annotations: nginx.ingress.kubernetes.io/rewrite-target: /$2 spec: diff --git a/geddes/k8s/05-postgres.yaml b/geddes/k8s/05-postgres.yaml index 91bfcc0..8a80970 100644 --- a/geddes/k8s/05-postgres.yaml +++ b/geddes/k8s/05-postgres.yaml @@ -2,10 +2,11 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: dt-db-v2-pvc - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: accessModes: - ReadWriteOnce + storageClassName: geddes-standard-singlenode resources: requests: storage: 20Gi @@ -14,7 +15,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: dt-postgres-init - namespace: nanohub + namespace: ncn-digitaltwins-zchen data: 00-bootstrap-api-client.sh: | #!/bin/sh @@ -396,11 +397,13 @@ apiVersion: apps/v1 kind: Deployment metadata: name: dt-db-v2 - namespace: nanohub + namespace: ncn-digitaltwins-zchen labels: app: dt-db-v2 spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: app: dt-db-v2 @@ -412,6 +415,13 @@ spec: containers: - name: postgres image: postgres:15 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1" + memory: "1Gi" ports: - containerPort: 5432 env: @@ -457,7 +467,7 @@ apiVersion: v1 kind: Service metadata: name: dt-db-v2-srv - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: selector: app: dt-db-v2 diff --git a/geddes/k8s/07-postgrest-public-ingress.example.yaml b/geddes/k8s/07-postgrest-public-ingress.example.yaml index ff6db31..4468e00 100644 --- a/geddes/k8s/07-postgrest-public-ingress.example.yaml +++ b/geddes/k8s/07-postgrest-public-ingress.example.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-postgrest-public - namespace: nanohub + namespace: ncn-digitaltwins-zchen annotations: # Restrict access to the Azure Function's outbound IPs. # Replace with a comma-separated allowlist from: diff --git a/geddes/k8s/07-postgrest-public-ingress.yaml b/geddes/k8s/07-postgrest-public-ingress.yaml index 73989c2..466d9bd 100644 --- a/geddes/k8s/07-postgrest-public-ingress.yaml +++ b/geddes/k8s/07-postgrest-public-ingress.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-postgrest-public - namespace: nanohub + namespace: ncn-digitaltwins-zchen annotations: # Removed whitelist to allow dynamic Azure outbound IPs # nginx.ingress.kubernetes.io/whitelist-source-range: "20.127.117.91/32" diff --git a/geddes/k8s/08-retrain-cron.yaml b/geddes/k8s/08-retrain-cron.yaml index 301ee5a..a735cfa 100644 --- a/geddes/k8s/08-retrain-cron.yaml +++ b/geddes/k8s/08-retrain-cron.yaml @@ -22,12 +22,13 @@ apiVersion: batch/v1 kind: CronJob metadata: name: dt-api-retrain - namespace: nanohub + namespace: ncn-digitaltwins-zchen labels: app: dt-api component: retrain spec: schedule: "0 2 * * *" # 02:00 UTC every day + suspend: false timeZone: "Etc/UTC" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 @@ -48,7 +49,7 @@ spec: - name: sdx-registry-secret containers: - name: retrain - image: geddes-registry.rcac.purdue.edu/sdx/dt-api:v50 + image: geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-retrain-loocv-20260714@sha256:52e16ffbbdf71ba87470d8dbf686e6304955501d5b74eaf4c41ba981e14239dc imagePullPolicy: Always command: - python @@ -64,6 +65,10 @@ spec: value: INFO - name: AUTO_RETRAIN_DOMAIN value: etcher + # Full LOOCV scales poorly for Gaussian-process models. Holdout + # validation and the final model fit still run above this limit. + - name: RETRAIN_LOOCV_MAX_ROWS + value: "50" resources: requests: cpu: "500m" diff --git a/geddes/k8s/pilots/db3/README.md b/geddes/k8s/pilots/db3/README.md index a2c3f53..f993212 100644 --- a/geddes/k8s/pilots/db3/README.md +++ b/geddes/k8s/pilots/db3/README.md @@ -43,7 +43,7 @@ script rejects tag-only images. For example: ```sh export KUBECONFIG=/tmp/geddes-kubeconfig.yaml export KUBE_CONTEXT=geddes -export NAMESPACE=nanohub +export NAMESPACE=ncn-digitaltwins-zchen API_IMAGE='geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-@sha256:' WEB_IMAGE='geddes-registry.rcac.purdue.edu/sdx/dt-web:db3-pilot-@sha256:' @@ -60,7 +60,7 @@ The script rejects non-pilot tags, unexpected repositories, and deployments that have moved beyond the recorded v52/v53 baseline. A ConfigMap checksum in the API pod template guarantees configuration changes trigger a rollout. -The package is intentionally fixed to the manifest namespace `nanohub`; the +The package is intentionally fixed to the manifest namespace `ncn-digitaltwins-zchen`; the scripts reject any other `NAMESPACE` value instead of silently deploying a cross-namespace mix of resources. `KUBE_CONTEXT` is explicit and configurable. diff --git a/geddes/k8s/pilots/db3/configmap.yaml b/geddes/k8s/pilots/db3/configmap.yaml index 4ae3522..1b3072e 100644 --- a/geddes/k8s/pilots/db3/configmap.yaml +++ b/geddes/k8s/pilots/db3/configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: dt-db3-pilot-config - namespace: nanohub + namespace: ncn-digitaltwins-zchen labels: app.kubernetes.io/part-of: dt-db3-pilot app.kubernetes.io/managed-by: dt-db3-pilot-scripts diff --git a/geddes/k8s/pilots/db3/deploy.sh b/geddes/k8s/pilots/db3/deploy.sh index 4dc77d0..fbddccc 100755 --- a/geddes/k8s/pilots/db3/deploy.sh +++ b/geddes/k8s/pilots/db3/deploy.sh @@ -3,7 +3,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" KUBE_CONTEXT="${KUBE_CONTEXT:-geddes}" -NAMESPACE="${NAMESPACE:-nanohub}" +NAMESPACE="${NAMESPACE:-ncn-digitaltwins-zchen}" ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" API_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-api" @@ -25,7 +25,7 @@ Usage: Environment: KUBECONFIG kubeconfig file used by kubectl (recommended: set explicitly) KUBE_CONTEXT target context (default: geddes) - NAMESPACE target namespace (default: nanohub) + NAMESPACE target namespace (default: ncn-digitaltwins-zchen) ROLLOUT_TIMEOUT kubectl rollout timeout (default: 300s) Only digest-pinned images are accepted. A branch-only tag may precede the @@ -126,8 +126,8 @@ config_checksum="$(shasum -a 256 "$SCRIPT_DIR/configmap.yaml" | awk '{print $1}' KUBECTL=(kubectl --context "$KUBE_CONTEXT" --namespace "$NAMESPACE") -if [[ "$NAMESPACE" != "nanohub" ]]; then - echo "These manifests are explicitly scoped to namespace nanohub; got: $NAMESPACE" >&2 +if [[ "$NAMESPACE" != "ncn-digitaltwins-zchen" ]]; then + echo "These manifests are explicitly scoped to namespace ncn-digitaltwins-zchen; got: $NAMESPACE" >&2 exit 2 fi diff --git a/geddes/k8s/pilots/db3/ingress.yaml b/geddes/k8s/pilots/db3/ingress.yaml index c4546fb..05b3de0 100644 --- a/geddes/k8s/pilots/db3/ingress.yaml +++ b/geddes/k8s/pilots/db3/ingress.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-db3-pilot-upload - namespace: nanohub + namespace: ncn-digitaltwins-zchen labels: app.kubernetes.io/part-of: dt-db3-pilot app.kubernetes.io/managed-by: dt-db3-pilot-scripts diff --git a/geddes/k8s/pilots/db3/pvc.yaml b/geddes/k8s/pilots/db3/pvc.yaml index 85c9055..168cb77 100644 --- a/geddes/k8s/pilots/db3/pvc.yaml +++ b/geddes/k8s/pilots/db3/pvc.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: dt-db3-pilot-uploads - namespace: nanohub + namespace: ncn-digitaltwins-zchen labels: app.kubernetes.io/part-of: dt-db3-pilot app.kubernetes.io/managed-by: dt-db3-pilot-scripts diff --git a/geddes/k8s/pilots/db3/rollback.sh b/geddes/k8s/pilots/db3/rollback.sh index 1aba1c2..a3f41cd 100755 --- a/geddes/k8s/pilots/db3/rollback.sh +++ b/geddes/k8s/pilots/db3/rollback.sh @@ -2,7 +2,7 @@ set -euo pipefail KUBE_CONTEXT="${KUBE_CONTEXT:-geddes}" -NAMESPACE="${NAMESPACE:-nanohub}" +NAMESPACE="${NAMESPACE:-ncn-digitaltwins-zchen}" ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" # These are the exact images running before the pilot. Keep them digest-pinned @@ -25,7 +25,7 @@ Usage: Environment: KUBECONFIG kubeconfig file used by kubectl (recommended: set explicitly) KUBE_CONTEXT target context (default: geddes) - NAMESPACE target namespace (default: nanohub) + NAMESPACE target namespace (default: ncn-digitaltwins-zchen) ROLLOUT_TIMEOUT kubectl rollout timeout (default: 300s) The default rollback disables the route, removes the pilot configuration and @@ -76,8 +76,8 @@ fi KUBECTL=(kubectl --context "$KUBE_CONTEXT" --namespace "$NAMESPACE") -if [[ "$NAMESPACE" != "nanohub" ]]; then - echo "These manifests are explicitly scoped to namespace nanohub; got: $NAMESPACE" >&2 +if [[ "$NAMESPACE" != "ncn-digitaltwins-zchen" ]]; then + echo "These manifests are explicitly scoped to namespace ncn-digitaltwins-zchen; got: $NAMESPACE" >&2 exit 2 fi