diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..72e2298 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +.git +.github +.agents +.codex + +**/__pycache__ +**/*.py[cod] +**/.pytest_cache +**/.venv +**/node_modules +**/.next + +**/.env +**/.env.* +**/local.settings.json +**/appsettings.json +**/*kubeconfig* + +incoming +**/*.db +**/*.db3 +**/*.sqlite +**/*.sqlite3 +nanohub/confidential +nanohub/config +nanohub/data +uploader/staging + +.DS_Store +*.log diff --git a/.gitignore b/.gitignore index e8f91a9..df29bd7 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,8 @@ debug_*.py !api/tests/test_phase3_security.py !api/tests/test_e2e_admin_flow.py !api/tests/test_e2e_platform.py +!api/tests/test_glance_db3_trace_import.py +!api/tests/test_retrain.py # Development files temp.py @@ -90,6 +92,11 @@ process_etcher_data/OUTPUT_DATASET.csv *.db *.sqlite *.sqlite3 +*.db3 + +# Local source exports and conversion scratch data. GLANCE DB3 files can contain +# proprietary equipment traces and must never be committed. +/incoming/ # Cache directories .cache/ @@ -122,6 +129,8 @@ temp_azure/ *.ps1 *.sh !scripts/check_deployment_state.sh +!geddes/k8s/pilots/db3/deploy.sh +!geddes/k8s/pilots/db3/rollback.sh # VS Code settings .vscode/settings.json diff --git a/DB3_UPLOAD_PIPELINE_REVIEW.md b/DB3_UPLOAD_PIPELINE_REVIEW.md new file mode 100644 index 0000000..c60657b --- /dev/null +++ b/DB3_UPLOAD_PIPELINE_REVIEW.md @@ -0,0 +1,135 @@ +# Temporary GLANCE DB3 Test Import + +## Decision confirmed with Paul + +One catalog row represents one physical GLANCE machine run. It does **not** +represent one timestamped sample. + +Every sample row and every parameter value from that run is retained in a linked +time-series trace. The source absolute timestamp remains authoritative. Relative +seconds are calculated only when a user reads or plots one or more traces, with +each run independently starting at its earliest sample. + +No input/output summaries are invented in this branch. The physical catalog run +has empty scalar input/output objects until Paul and the team define scientifically +valid summary rules. + +## Test-only scope + +Uploading a `.db3` is a temporary way to test this data model and plotting flow. +It is opt-in and must not be treated as the long-term GLANCE integration. A future +production integration should read from the authoritative GLANCE service or an +approved export pipeline rather than depend on users moving SQLite files around. + +DB3 upload is default-off. It requires all three temporary pilot controls: + +- `DT_ENABLE_DB3_UPLOAD=true`; +- the authenticated nanoHUB ID in `DT_DB3_ALLOWED_USER_IDS`; +- `DT_DB3_EQUIPMENT_TOOL_MAP` explicitly mapping the selected registered + equipment ID to the GLANCE tool name embedded in the export. + +```json +{ + "": ["VLN-11304-CTC-PM1"] +} +``` + +This mapping is intentional. It prevents an export from one machine from being +silently attached to another machine as more GLANCE tools are tested. The default +DB3 limit is 64 MB and can be adjusted with `DT_DB3_UPLOAD_MAX_MB`. The UI reads +an authenticated runtime capability endpoint, so users outside the allowlist do +not see DB3 as an accepted upload type or see DB3 upload records. + +## Workflow + +1. The user selects an approved equipment registration and uploads one `.db3`. +2. The API streams the file to upload storage in bounded chunks. It does not read + the complete DB3 into one in-memory bytes object and does not preprocess inside + the async request handler. +3. The upload appears as `success`. The user selects a disposable test project and + clicks **Process**. +4. The existing synchronous background task reads SQLite in read-only mode and + extracts one object per physical run. A malformed run is reported and skipped + without discarding other valid physical runs. +5. One database transaction upserts the physical catalog parents and replaces all + linked samples. A retry is idempotent by source upload plus GLANCE run ID. +6. The run detail page retrieves a trace through + `GET /dataset/v2/runs/{run_id}/trace` and can compare up to six runs using an + absolute/source or dynamically calculated relative time axis. + +Processing uses an in-process background task for this pilot. A timestamped +processing lease is refreshed by a heartbeat while work is active. If the +worker disappears, polling returns the abandoned DB3 to a retryable state after +the configured stale interval; trace synchronization remains idempotent. + +The old 52 generated input/output CSV records are gone. A single DB3 now requires +one Process action. + +## Persisted model + +`equipment_runs` stores one parent per physical run, including: + +- lot name for display; +- GLANCE run, tool, and recipe identifiers; +- tool, recipe, material, start/end, and status provenance; +- source filename, upload ID, size, and SHA-256; +- parameter metadata keyed by the stable GLANCE parameter ID (`p`). + +`equipment_run_trace_samples` stores every sample linked to its parent: + +- source sample-record ID; +- source GLANCE tool ID recorded on the sample; +- deterministic sample index; +- exact `sample_time_raw` text; +- parsed timezone-naive wall time for ordering and relative-time arithmetic; +- a JSON object containing every value present at that timestamp. + +The source export does not declare a timezone, so the API reports +`timestamp_timezone: null` and never labels the values UTC. Missing values remain +null and empty sample rows remain `{}` so plots show real gaps. + +Lot name is not a technical identity. Paul's sample contains repeated lot names, +including many physical runs named `FM-2'`. The durable source identity is the +data-upload UUID plus GLANCE `idruns`; lot name remains searchable display data. + +## Paul's sample expectation + +For `incoming/paul_db3_pipeline/MyDatabaseSample.db3`, the extractor asserts: + +- 27 rows in the source `runs` table; +- 26 physical runs with sample records, therefore 26 catalog parents; +- 11,697 linked sample rows; +- 11,696 samples with values and one retained empty-value sample; +- 242 source parameter definitions; +- 795,328 parameter values; +- one source tool: `VLN-11304-CTC-PM1`. + +These counts intentionally differ from the discarded CSV conversion, which +removed the empty sample and expanded samples into catalog rows. + +## Verification + +The focused extractor test builds a small SQLite fixture at runtime. It covers +duplicate lot names, fractional and irregular timestamps, missing values, an +empty-value sample, a no-sample run, source checksum/provenance, and per-run +failure isolation. + +```bash +api/.venv/bin/python -m pytest -q api/tests/test_glance_db3_trace_import.py +api/.venv/bin/python -m py_compile \ + api/glance_db3/trace_import.py \ + api/data_loader_pg.py \ + api/routers/upload.py \ + api/routers/dataset_v2.py + +cd web +npx tsc --noEmit +npm run lint +``` + +The local `incoming/` directory and all `*.db3` files are ignored because source +exports can contain proprietary equipment traces. No generated CSV/intermediate +directories are created. The Geddes pilot stores raw uploads on its own encrypted +PVC, isolated from the normal deployment manifest. The removable deployment and +digest-pinned rollback procedure live in `geddes/k8s/pilots/db3/`; that package +must not be merged into the final production platform. diff --git a/api/FAIR_ARCHIVAL_WALKTHROUGH.md b/api/FAIR_ARCHIVAL_WALKTHROUGH.md index c310b6f..687e0d9 100644 --- a/api/FAIR_ARCHIVAL_WALKTHROUGH.md +++ b/api/FAIR_ARCHIVAL_WALKTHROUGH.md @@ -55,7 +55,7 @@ Comprehensive rewrite with researcher-friendly guide: setup, usage, architecture 1. **Run the migration**: ```bash - kubectl exec -n nanohub dt-db-v2-0 -- psql -U postgres -d digital_twin -f /tmp/add_fair_columns.sql + kubectl exec -n ncn-digitaltwins-zchen dt-db-v2-0 -- psql -U postgres -d digital_twin -f /tmp/add_fair_columns.sql ``` 2. **Add PURR env vars** to the API deployment: diff --git a/api/data_loader_pg.py b/api/data_loader_pg.py index d3682a3..44c6faf 100644 --- a/api/data_loader_pg.py +++ b/api/data_loader_pg.py @@ -51,7 +51,7 @@ def is_mock_db_enabled() -> bool: def _default_pg_host() -> str: if os.getenv("KUBERNETES_SERVICE_HOST"): - return "dt-db-v2-srv.nanohub.svc.cluster.local" + return "dt-db-v2-srv.ncn-digitaltwins-zchen.svc.cluster.local" return "localhost" @@ -164,6 +164,11 @@ def _shape_run_record(row) -> Dict[str, Any]: raw_payload = rec.pop("raw_payload_json", None) if raw_payload is not None: rec["raw_payload"] = raw_payload + # Canonical etcher rows do not use the generic GLANCE trace table. Keeping + # these keys in the shared response shape lets clients identify trace-backed + # generic runs without special-casing an absent property. + rec.setdefault("source_run_id", None) + rec.setdefault("trace_sample_count", 0) return rec @@ -413,6 +418,92 @@ def ensure_equipment_runs_pg() -> None: conn.close() +def ensure_equipment_run_trace_samples_pg() -> None: + """Create the per-sample trace table used by generic equipment runs. + + GLANCE timestamps currently arrive without a timezone. ``sample_time`` is + therefore deliberately ``TIMESTAMP WITHOUT TIME ZONE``: it supports sorting + and relative-time arithmetic without claiming an offset the source did not + provide. ``sample_time_raw`` preserves the exact source text (including any + future fractional seconds or explicit offset) for provenance and display. + """ + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_index INT NOT NULL, + source_sample_id BIGINT NOT NULL, + source_tool_id BIGINT, + sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + sample_time_raw TEXT NOT NULL, + values_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, sample_index), + UNIQUE (equipment_run_id, source_sample_id), + CHECK (jsonb_typeof(values_json) = 'object') + ) + """ + ) + cur.execute( + "ALTER TABLE equipment_run_trace_samples " + "ADD COLUMN IF NOT EXISTS source_tool_id BIGINT" + ) + cur.execute( + """ + CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time + ON equipment_run_trace_samples( + equipment_run_id, sample_time, sample_index + ) + """ + ) + cur.execute( + "ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY" + ) + cur.execute("GRANT SELECT ON equipment_run_trace_samples TO api_client") + cur.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_trace_samples' + AND policyname = 'equipment_run_trace_visibility_policy' + ) THEN + CREATE POLICY equipment_run_trace_visibility_policy + ON equipment_run_trace_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; + END + $$; + """ + ) + conn.commit() + finally: + conn.close() + + def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: """SELECT that shapes generic equipment_runs into the same record format used by the etcher /runs endpoints (so the catalog can render both).""" @@ -431,6 +522,11 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: r.upload_filename, r.source, r.project_id, + NULLIF(r.raw_payload_json #>> '{{glance,run_id}}', '') AS source_run_id, + NULLIF( + r.raw_payload_json #>> '{{glance,run_start_time}}', '' + ) AS source_run_start_time, + COALESCE(trace_counts.sample_count, 0)::int AS trace_sample_count, ''::text AS execution_request_id, p.name AS project_name, r.equipment_id AS equipment_id, @@ -441,6 +537,11 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: FROM equipment_runs r LEFT JOIN projects p ON p.id = r.project_id LEFT JOIN equipment_metadata em ON em.domain_id = r.equipment_id + LEFT JOIN LATERAL ( + SELECT COUNT(*)::int AS sample_count + FROM equipment_run_trace_samples trace_sample + WHERE trace_sample.equipment_run_id = r.id + ) trace_counts ON true """ @@ -455,6 +556,13 @@ def _shape_equipment_run_record(row) -> Dict[str, Any]: if hasattr(rec[ts_field], "isoformat") else str(rec[ts_field]) ) + source_run_start_time = rec.pop("source_run_start_time", None) + if rec.get("source") == "glance_db3" and source_run_start_time: + # equipment_runs.run_date is TIMESTAMPTZ, so inserting a naive GLANCE + # wall time makes Postgres attach the session timezone. Present the exact + # source text instead; raw trace metadata explicitly reports that its + # timezone is unknown rather than fabricating an offset for the catalog. + rec["run_date"] = str(source_run_start_time) # Expose the validated input set-points under the same "features" key the # etcher rows use, and surface measured outputs alongside. rec["features"] = rec.pop("inputs_json", {}) or {} @@ -464,6 +572,17 @@ def _shape_equipment_run_record(row) -> Dict[str, Any]: rec.setdefault("avg_etch_rate", None) rec.setdefault("range_etch_rate", None) rec.setdefault("range_nm", None) + source_run_id = rec.get("source_run_id") + if source_run_id not in (None, ""): + try: + rec["source_run_id"] = int(source_run_id) + except (TypeError, ValueError): + # GLANCE uses numeric ids today, but preserving an unexpected source + # value is safer than hiding provenance from a future importer. + rec["source_run_id"] = str(source_run_id) + else: + rec["source_run_id"] = None + rec["trace_sample_count"] = int(rec.get("trace_sample_count") or 0) rec["equipment_name"] = rec.get("equipment_name") or rec.get("equipment_id") or "" raw_payload = rec.pop("raw_payload_json", None) if raw_payload is not None: @@ -558,6 +677,420 @@ def sync_equipment_runs_pg( conn.close() +def _normalize_trace_parameter_key(value: Any) -> str: + """Return a stable JSON key for a GLANCE parameter identifier.""" + text = str(value or "").strip() + if not text: + return "" + match = re.fullmatch(r"[pP]?(\d+)", text) + if match: + return f"p{int(match.group(1))}" + return text + + +def _parse_trace_sample_time(value: Any, *, source_sample_id: int) -> tuple[datetime, str]: + """Parse a source timestamp without inventing a timezone. + + The exact input text is returned alongside a timezone-naive datetime for the + Postgres ``TIMESTAMP WITHOUT TIME ZONE`` column. If a future source includes + an explicit offset, normalize the arithmetic value to UTC before dropping + ``tzinfo`` while retaining the exact offset-bearing source text. Naive source + values remain naive; no equipment timezone is guessed. + """ + if isinstance(value, datetime): + raw = value.isoformat() + parsed = value + else: + raw = str(value or "").strip() + if not raw: + raise ValueError( + f"GLANCE sample {source_sample_id} is missing an absolute timestamp" + ) + candidate = raw[:-1] + "+00:00" if raw.endswith("Z") else raw + try: + parsed = datetime.fromisoformat(candidate) + except ValueError as exc: + raise ValueError( + f"GLANCE sample {source_sample_id} has an invalid timestamp {raw!r}" + ) from exc + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc) + return parsed.replace(tzinfo=None), raw + + +def sync_equipment_run_traces_pg( + *, + equipment_id: str, + project_id: str, + upload_id: str, + upload_filename: str, + runs: List[Dict[str, Any]], +) -> int: + """Atomically upsert physical GLANCE runs and their complete sample traces. + + One item in ``runs`` becomes one ``equipment_runs`` catalog record. Each item + carries ``source_run_id``, optional scalar ``inputs``/``outputs`` and ``raw`` + provenance, ``parameters`` metadata, and ``samples``. A sample accepts + ``source_sample_id`` (or ``sample_record_id``), ``sample_time_raw`` (or + ``sample_time``/``timestamp``), and a ``values`` mapping keyed by stable + parameter ids such as ``p130``. + + Parent rows and samples share one transaction. For DB3 imports ``row_index`` + stores the GLANCE source run id, making ``(upload_id, row_index)`` the stable + ``(upload, source run)`` identity. Reprocessing replaces traces in place, + preserves catalog run ids even when another source run is skipped or later + recovers, and removes stale rows from an earlier conversion. + """ + from psycopg2.extras import execute_values + + if not str(equipment_id or "").strip(): + raise ValueError("equipment_id is required for GLANCE trace ingestion") + if not str(project_id or "").strip(): + raise ValueError("project_id is required for GLANCE trace ingestion") + if not str(upload_id or "").strip(): + raise ValueError("upload_id is required for idempotent GLANCE trace ingestion") + if not runs: + raise ValueError("No physical GLANCE runs with trace samples were found") + + normalized_runs: list[dict[str, Any]] = [] + seen_source_run_ids: set[int] = set() + + for run in runs: + source_run_value = run.get("source_run_id", run.get("run_id")) + try: + source_run_id = int(source_run_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"Invalid or missing GLANCE source run id: {source_run_value!r}" + ) from exc + if source_run_id in seen_source_run_ids: + raise ValueError(f"Duplicate GLANCE source run id {source_run_id}") + seen_source_run_ids.add(source_run_id) + + raw_parameters = run.get("parameters", run.get("trace_parameters", [])) or [] + parameters: list[dict[str, Any]] = [] + parameter_by_key: dict[str, dict[str, Any]] = {} + for parameter in raw_parameters: + if not isinstance(parameter, dict): + raise ValueError( + f"GLANCE run {source_run_id} contains invalid parameter metadata" + ) + source_parameter_id = parameter.get( + "source_parameter_id", + parameter.get("source_id", parameter.get("idparameters")), + ) + key = _normalize_trace_parameter_key( + parameter.get("key", source_parameter_id) + ) + if not key: + raise ValueError( + f"GLANCE run {source_run_id} contains a parameter without a stable key" + ) + if key in parameter_by_key: + raise ValueError( + f"GLANCE run {source_run_id} contains duplicate parameter key {key!r}" + ) + metadata = { + "key": key, + "name": str( + parameter.get( + "name", + parameter.get("display_name", parameter.get("source_name", key)), + ) + or key + ), + "unit": str(parameter.get("unit") or ""), + } + if source_parameter_id not in (None, ""): + try: + metadata["source_parameter_id"] = int(source_parameter_id) + except (TypeError, ValueError): + metadata["source_parameter_id"] = str(source_parameter_id) + source_name = parameter.get("source_name") + if source_name not in (None, ""): + metadata["source_name"] = str(source_name) + parameter_by_key[key] = metadata + parameters.append(metadata) + + raw_samples = run.get("samples", run.get("trace_samples", [])) or [] + if not raw_samples: + raise ValueError(f"GLANCE run {source_run_id} has no trace samples") + samples: list[dict[str, Any]] = [] + seen_source_sample_ids: set[int] = set() + seen_sample_indices: set[int] = set() + for fallback_index, sample in enumerate(raw_samples): + if not isinstance(sample, dict): + raise ValueError( + f"GLANCE run {source_run_id} contains an invalid trace sample" + ) + sample_id_value = sample.get( + "source_sample_id", sample.get("sample_record_id") + ) + try: + source_sample_id = int(sample_id_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"GLANCE run {source_run_id} has an invalid sample id {sample_id_value!r}" + ) from exc + if source_sample_id in seen_source_sample_ids: + raise ValueError( + f"GLANCE run {source_run_id} repeats sample id {source_sample_id}" + ) + seen_source_sample_ids.add(source_sample_id) + + source_tool_value = sample.get( + "source_tool_id", sample.get("glance_tool_id") + ) + if source_tool_value in (None, ""): + source_tool_id = None + else: + try: + source_tool_id = int(source_tool_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"GLANCE sample {source_sample_id} has an invalid source tool id " + f"{source_tool_value!r}" + ) from exc + if not ( + -9_223_372_036_854_775_808 + <= source_tool_id + <= 9_223_372_036_854_775_807 + ): + raise ValueError( + f"GLANCE sample {source_sample_id} source tool id is outside " + "the PostgreSQL BIGINT range" + ) + + sample_index_value = sample.get("sample_index", fallback_index) + try: + sample_index = int(sample_index_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"GLANCE sample {source_sample_id} has invalid index {sample_index_value!r}" + ) from exc + if sample_index < 0 or sample_index in seen_sample_indices: + raise ValueError( + f"GLANCE run {source_run_id} has invalid or duplicate sample index {sample_index}" + ) + seen_sample_indices.add(sample_index) + + raw_time_value = sample.get( + "sample_time_raw", + sample.get("sample_time", sample.get("timestamp")), + ) + sample_time, sample_time_raw = _parse_trace_sample_time( + raw_time_value, + source_sample_id=source_sample_id, + ) + raw_values = sample.get("values", sample.get("values_json", {})) or {} + if not isinstance(raw_values, dict): + raise ValueError( + f"GLANCE sample {source_sample_id} values must be an object" + ) + values: dict[str, Any] = {} + for raw_key, raw_value in raw_values.items(): + key = _normalize_trace_parameter_key(raw_key) + if not key: + raise ValueError( + f"GLANCE sample {source_sample_id} contains a blank parameter key" + ) + if key in values: + raise ValueError( + f"GLANCE sample {source_sample_id} repeats parameter key {key!r}" + ) + values[key] = raw_value + if key not in parameter_by_key: + fallback_metadata = {"key": key, "name": key, "unit": ""} + parameter_by_key[key] = fallback_metadata + parameters.append(fallback_metadata) + samples.append( + { + "sample_index": sample_index, + "source_sample_id": source_sample_id, + "source_tool_id": source_tool_id, + "sample_time": sample_time, + "sample_time_raw": sample_time_raw, + "values": _json_safe_payload(values), + } + ) + + samples.sort(key=lambda sample: sample["sample_index"]) + raw_payload_value = run.get("raw", run.get("raw_payload", {})) or {} + if not isinstance(raw_payload_value, dict): + raise ValueError(f"GLANCE run {source_run_id} raw provenance must be an object") + raw_payload = _json_safe_payload(raw_payload_value) + glance_metadata = raw_payload.get("glance") + if not isinstance(glance_metadata, dict): + glance_metadata = {} + glance_metadata["run_id"] = source_run_id + source_run_start_time = run.get("run_start_time") + if source_run_start_time in (None, ""): + source_run_start_time = glance_metadata.get("run_start_time") + if source_run_start_time in (None, ""): + source_run_start_time = run.get("run_date") + if source_run_start_time not in (None, ""): + glance_metadata["run_start_time"] = ( + source_run_start_time.isoformat() + if hasattr(source_run_start_time, "isoformat") + else str(source_run_start_time) + ) + raw_payload["glance"] = glance_metadata + + trace_metadata = raw_payload.get("trace") + if not isinstance(trace_metadata, dict): + trace_metadata = {} + trace_metadata.update( + { + "sample_count": len(samples), + "parameters": parameters, + "timestamp_timezone": run.get("timestamp_timezone"), + "start_time": min(samples, key=lambda sample: sample["sample_time"])[ + "sample_time_raw" + ], + "end_time": max(samples, key=lambda sample: sample["sample_time"])[ + "sample_time_raw" + ], + } + ) + raw_payload["trace"] = trace_metadata + raw_payload["source_upload_id"] = str(upload_id) + raw_payload["source_db3_filename"] = str(upload_filename or "") + + normalized_runs.append( + { + "source_run_id": source_run_id, + "lotname": str(run.get("lotname", run.get("lot_name", "")) or ""), + "run_date": run.get("run_date", run.get("run_start_time")) or None, + "is_outlier": bool(run.get("is_outlier", False)), + "is_calibration_recipe": bool( + run.get("is_calibration_recipe", False) + ), + "outlier_type": str(run.get("outlier_type", "") or ""), + "inputs": _json_safe_payload(run.get("inputs", {}) or {}), + "outputs": _json_safe_payload(run.get("outputs", {}) or {}), + "raw": raw_payload, + "samples": samples, + } + ) + + # Stable ordering keeps each source run attached to the same upload row and + # therefore the same exposed catalog id on a retry. + normalized_runs.sort(key=lambda run: run["source_run_id"]) + + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", + (f"equipment_run_trace_sync|{upload_id}",), + ) + parent_values = [] + for run in normalized_runs: + parent_values.append( + ( + equipment_id, + project_id, + run["lotname"], + run["run_date"], + run["is_outlier"], + run["is_calibration_recipe"], + run["outlier_type"], + Json(run["inputs"]), + Json(run["outputs"]), + Json(run["raw"]), + upload_id, + run["source_run_id"], + upload_filename, + "glance_db3", + ) + ) + + returned_parents = execute_values( + cur, + """ + INSERT INTO equipment_runs ( + equipment_id, project_id, lotname, run_date, + is_outlier, is_calibration_recipe, outlier_type, + inputs_json, outputs_json, raw_payload_json, + upload_id, row_index, upload_filename, source + ) VALUES %s + ON CONFLICT (upload_id, row_index) WHERE upload_id IS NOT NULL + DO UPDATE SET + equipment_id = EXCLUDED.equipment_id, + project_id = EXCLUDED.project_id, + lotname = EXCLUDED.lotname, + run_date = EXCLUDED.run_date, + is_outlier = EXCLUDED.is_outlier, + is_calibration_recipe = EXCLUDED.is_calibration_recipe, + outlier_type = EXCLUDED.outlier_type, + inputs_json = EXCLUDED.inputs_json, + outputs_json = EXCLUDED.outputs_json, + raw_payload_json = EXCLUDED.raw_payload_json, + upload_filename = EXCLUDED.upload_filename, + source = EXCLUDED.source + RETURNING id, row_index + """, + parent_values, + fetch=True, + ) + parent_id_by_source_run = { + int(source_run_id): int(parent_id) + for parent_id, source_run_id in returned_parents + } + if len(parent_id_by_source_run) != len(normalized_runs): + raise RuntimeError("Could not resolve every GLANCE catalog run after upsert") + + parent_ids = list(parent_id_by_source_run.values()) + cur.execute( + "DELETE FROM equipment_run_trace_samples " + "WHERE equipment_run_id = ANY(%s)", + (parent_ids,), + ) + trace_values = [] + for run in normalized_runs: + parent_id = parent_id_by_source_run[run["source_run_id"]] + for sample in run["samples"]: + trace_values.append( + ( + parent_id, + sample["sample_index"], + sample["source_sample_id"], + sample["source_tool_id"], + sample["sample_time"], + sample["sample_time_raw"], + Json(sample["values"]), + ) + ) + execute_values( + cur, + """ + INSERT INTO equipment_run_trace_samples ( + equipment_run_id, sample_index, source_sample_id, source_tool_id, + sample_time, sample_time_raw, values_json + ) VALUES %s + """, + trace_values, + page_size=500, + ) + # Cascading deletion removes samples belonging to a stale physical + # run if a deterministic re-conversion yields fewer runs. + cur.execute( + """ + DELETE FROM equipment_runs + WHERE upload_id = %s + AND NOT (id = ANY(%s)) + """, + (upload_id, parent_ids), + ) + conn.commit() + return len(normalized_runs) + except Exception: + conn.rollback() + raise + finally: + conn.close() + + def merge_equipment_runs_pg( *, equipment_id: str, @@ -975,6 +1508,220 @@ def get_run_detail_pg( logger.error(f"Error fetching run {run_id} from Postgres: {e}") return None + +def get_equipment_run_trace_pg( + run_id: int, + *, + parameters: Optional[List[str]] = None, + nanohub_user_id: Optional[str] = None, + is_admin: bool = False, +) -> Optional[Dict[str, Any]]: + """Fetch a generic equipment run's complete trace under catalog RLS. + + Absolute source timestamp text is always returned. ``relative_seconds`` is + derived at query time from the earliest sample in the complete run, before + optional parameter selection, so selecting a sparse parameter can never move + the run's zero point. Every sample remains in the response; missing selected + values are represented as JSON null to preserve plot gaps. + """ + if run_id < EQUIPMENT_RUN_ID_OFFSET: + return None + + requested_keys: list[str] = [] + seen_requested: set[str] = set() + for requested in parameters or []: + # Accept both repeated query parameters and a comma-separated value for + # simple clients while exposing one canonical list internally. + for value in str(requested or "").split(","): + key = _normalize_trace_parameter_key(value) + if key and key not in seen_requested: + requested_keys.append(key) + seen_requested.add(key) + + conn = get_pg_superuser_connection() if is_admin else get_pg_connection(nanohub_user_id) + try: + internal_run_id = run_id - EQUIPMENT_RUN_ID_OFFSET + with conn.cursor(cursor_factory=RealDictCursor) as cur: + # Selecting the parent first applies equipment_runs RLS and avoids + # distinguishing a private run from one that does not exist. + cur.execute( + """ + SELECT + r.id, + r.lotname, + r.raw_payload_json, + r.source + FROM equipment_runs r + WHERE r.id = %s + LIMIT 1 + """, + (internal_run_id,), + ) + parent = cur.fetchone() + if not parent: + conn.commit() + return None + + cur.execute( + """ + SELECT + sample_index, + source_sample_id, + source_tool_id, + sample_time, + sample_time_raw, + values_json, + EXTRACT( + EPOCH FROM ( + sample_time - MIN(sample_time) OVER () + ) + )::double precision AS relative_seconds + FROM equipment_run_trace_samples + WHERE equipment_run_id = %s + ORDER BY sample_time, sample_index + """, + (internal_run_id,), + ) + sample_rows = [dict(row) for row in cur.fetchall()] + conn.commit() + + if not sample_rows: + return None + + raw_payload = parent.get("raw_payload_json") or {} + if not isinstance(raw_payload, dict): + raw_payload = {} + glance_metadata = raw_payload.get("glance") or {} + if not isinstance(glance_metadata, dict): + glance_metadata = {} + trace_metadata = raw_payload.get("trace") or {} + if not isinstance(trace_metadata, dict): + trace_metadata = {} + + parameter_records: list[dict[str, Any]] = [] + parameter_by_key: dict[str, dict[str, Any]] = {} + for raw_parameter in trace_metadata.get("parameters", []) or []: + if not isinstance(raw_parameter, dict): + continue + key = _normalize_trace_parameter_key( + raw_parameter.get( + "key", + raw_parameter.get( + "source_parameter_id", raw_parameter.get("source_id") + ), + ) + ) + if not key or key in parameter_by_key: + continue + parameter = { + "key": key, + "name": str( + raw_parameter.get( + "name", raw_parameter.get("display_name", key) + ) + or key + ), + "unit": str(raw_parameter.get("unit") or ""), + } + if raw_parameter.get("source_parameter_id") not in (None, ""): + parameter["source_parameter_id"] = raw_parameter[ + "source_parameter_id" + ] + if raw_parameter.get("source_name") not in (None, ""): + parameter["source_name"] = str(raw_parameter["source_name"]) + parameter_by_key[key] = parameter + parameter_records.append(parameter) + + # Be resilient to older trace rows whose parameter metadata was missing: + # stable JSON keys still make the values retrievable and self-consistent. + for sample_row in sample_rows: + values = sample_row.get("values_json") or {} + if not isinstance(values, dict): + values = {} + normalized_values: dict[str, Any] = {} + for raw_key, value in values.items(): + key = _normalize_trace_parameter_key(raw_key) + if not key: + continue + normalized_values[key] = value + if key not in parameter_by_key: + parameter = {"key": key, "name": key, "unit": ""} + parameter_by_key[key] = parameter + parameter_records.append(parameter) + sample_row["values_json"] = normalized_values + + if requested_keys: + unknown = [key for key in requested_keys if key not in parameter_by_key] + if unknown: + raise ValueError( + "Unknown trace parameter key(s): " + ", ".join(unknown) + ) + selected_keys = requested_keys + response_parameters = [parameter_by_key[key] for key in selected_keys] + else: + selected_keys = [parameter["key"] for parameter in parameter_records] + response_parameters = parameter_records + + samples: list[dict[str, Any]] = [] + for sample_row in sample_rows: + values = sample_row["values_json"] + raw_timestamp = str(sample_row.get("sample_time_raw") or "").strip() + if not raw_timestamp: + parsed_timestamp = sample_row.get("sample_time") + raw_timestamp = ( + parsed_timestamp.isoformat() + if hasattr(parsed_timestamp, "isoformat") + else str(parsed_timestamp or "") + ) + samples.append( + { + "sample_index": int(sample_row["sample_index"]), + "sample_record_id": int(sample_row["source_sample_id"]), + "source_tool_id": ( + int(sample_row["source_tool_id"]) + if sample_row.get("source_tool_id") is not None + else None + ), + "timestamp": raw_timestamp, + "relative_seconds": float(sample_row["relative_seconds"] or 0.0), + "values": {key: values.get(key) for key in selected_keys}, + } + ) + + source_run_value = glance_metadata.get("run_id") + try: + source_run_id: Any = int(source_run_value) + except (TypeError, ValueError): + source_run_id = source_run_value + + min_row = min(sample_rows, key=lambda row: row["sample_time"]) + max_row = max(sample_rows, key=lambda row: row["sample_time"]) + return { + "run_id": run_id, + "source_run_id": source_run_id, + "lot_name": str(parent.get("lotname") or ""), + "sample_count": len(samples), + "start_time": str(min_row.get("sample_time_raw") or ""), + "end_time": str(max_row.get("sample_time_raw") or ""), + "timestamp_timezone": trace_metadata.get("timestamp_timezone"), + "parameters": response_parameters, + "samples": samples, + } + except MissingRuntimeConfiguration as exc: + logger.error( + "Postgres runtime configuration missing while fetching run trace: %s", + exc, + ) + raise RuntimeError("Postgres credentials are not configured") from exc + except ValueError: + raise + except Exception as exc: + logger.error("Error fetching trace for run %s: %s", run_id, exc) + return None + finally: + conn.close() + + def get_projects_list_pg(nanohub_user_id: Optional[str] = None, is_admin: bool = False) -> List[Dict[str, Any]]: """ Fetch projects from Postgres. diff --git a/api/glance_db3/__init__.py b/api/glance_db3/__init__.py new file mode 100644 index 0000000..ba9b2e0 --- /dev/null +++ b/api/glance_db3/__init__.py @@ -0,0 +1,31 @@ +"""Read temporary GLANCE DB3 test uploads as physical runs with full traces.""" + +from .trace_import import ( + Db3TraceExtraction, + Db3TraceImportResult, + GlanceDb3Error, + GlanceDb3SchemaError, + GlanceParameter, + GlanceRunTrace, + GlanceSourceTool, + GlanceTraceSample, + RunTraceFailure, + extract_db3_trace, + extract_db3_run_traces, + parameter_key, +) + +__all__ = [ + "Db3TraceExtraction", + "Db3TraceImportResult", + "GlanceDb3Error", + "GlanceDb3SchemaError", + "GlanceParameter", + "GlanceRunTrace", + "GlanceSourceTool", + "GlanceTraceSample", + "RunTraceFailure", + "extract_db3_trace", + "extract_db3_run_traces", + "parameter_key", +] diff --git a/api/glance_db3/trace_import.py b/api/glance_db3/trace_import.py new file mode 100644 index 0000000..bdcbc7f --- /dev/null +++ b/api/glance_db3/trace_import.py @@ -0,0 +1,566 @@ +"""Read GLANCE SQLite exports as one complete time-series trace per run. + +This module deliberately does not conform GLANCE data to the platform's +row-oriented CSV upload templates. A GLANCE run is the catalog-level object; +its sample records remain an attached trace. Source timestamps and parameter +names are therefore retained exactly as SQLite returned them. +""" + +from __future__ import annotations + +import base64 +import hashlib +import math +import sqlite3 +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any + + +_REQUIRED_TABLES = frozenset( + {"runs", "samplerecord", "data", "parameters", "tools", "recipes"} +) + + +class GlanceDb3Error(ValueError): + """Base error for a DB3 file that cannot be extracted safely.""" + + +class GlanceDb3SchemaError(GlanceDb3Error): + """Raised when a file is SQLite but not the expected GLANCE schema.""" + + +def _optional_text(value: Any) -> str | None: + """Return SQLite text without normalizing timestamps or other metadata.""" + if value is None: + return None + return value if isinstance(value, str) else str(value) + + +def _required_int(value: Any, field_name: str) -> int: + if value is None or isinstance(value, bool): + raise GlanceDb3Error(f"{field_name} must be an integer, got {value!r}") + try: + numeric = Decimal(str(value)) + except (InvalidOperation, TypeError, ValueError) as exc: + raise GlanceDb3Error( + f"{field_name} must be an integer, got {value!r}" + ) from exc + if not numeric.is_finite() or numeric != numeric.to_integral_value(): + raise GlanceDb3Error(f"{field_name} must be an integer, got {value!r}") + return int(numeric) + + +def _optional_int(value: Any, field_name: str) -> int | None: + if value is None: + return None + return _required_int(value, field_name) + + +def parameter_key(parameter_id: int) -> str: + """Return the stable, source-ID-based key used in every ``values_json``.""" + return f"p{parameter_id}" + + +def _json_value(value: Any) -> Any: + """Convert uncommon SQLite values without making the result non-JSON-safe.""" + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float): + # Strict JSON has no NaN or infinities. Preserve their source meaning as + # text rather than silently converting them to null. + return value if math.isfinite(value) else str(value) + if isinstance(value, bytes): + return "base64:" + base64.b64encode(value).decode("ascii") + return str(value) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _connect_readonly(path: Path) -> sqlite3.Connection: + # Path.as_uri() safely escapes spaces and URI metacharacters in filenames. + connection = sqlite3.connect( + f"{path.resolve().as_uri()}?mode=ro&immutable=1", + uri=True, + ) + connection.row_factory = sqlite3.Row + return connection + + +@dataclass(frozen=True) +class GlanceParameter: + key: str + glance_parameter_id: int + raw_name: str | None + unit: str | None + + def to_dict(self) -> dict[str, Any]: + return { + "key": self.key, + "glance_parameter_id": self.glance_parameter_id, + "raw_name": self.raw_name, + "unit": self.unit, + } + + def to_sync_dict(self) -> dict[str, Any]: + return { + "key": self.key, + "source_parameter_id": self.glance_parameter_id, + # No display-name transformation is performed during extraction, so + # the source name is also the honest initial UI label. + "name": self.raw_name, + "source_name": self.raw_name, + "unit": self.unit, + } + + +@dataclass(frozen=True) +class GlanceSourceTool: + glance_tool_id: int + name: str | None + description: str | None + + def to_dict(self) -> dict[str, Any]: + return { + "glance_tool_id": self.glance_tool_id, + "name": self.name, + "description": self.description, + } + + +@dataclass(frozen=True) +class GlanceTraceSample: + glance_sample_record_id: int + glance_run_id: int + glance_tool_id: int | None + sample_time: str | None + values_json: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + return { + "glance_sample_record_id": self.glance_sample_record_id, + "glance_run_id": self.glance_run_id, + "glance_tool_id": self.glance_tool_id, + "sample_time": self.sample_time, + "values_json": dict(self.values_json), + } + + def to_sync_dict(self, sample_index: int) -> dict[str, Any]: + return { + "source_sample_id": self.glance_sample_record_id, + "sample_index": sample_index, + "source_tool_id": self.glance_tool_id, + "sample_time_raw": self.sample_time, + "values_json": dict(self.values_json), + } + + +@dataclass(frozen=True) +class GlanceRunTrace: + glance_run_id: int + glance_tool_id: int | None + tool_name: str | None + lot_name: str | None + material_name: str | None + glance_recipe_id: int | None + recipe_name: str | None + run_start_time: str | None + run_end_time: str | None + run_status: str | None + parameter_metadata: tuple[GlanceParameter, ...] + samples: tuple[GlanceTraceSample, ...] + + @property + def sample_count(self) -> int: + return len(self.samples) + + def provenance_dict(self) -> dict[str, Any]: + """Return compact run metadata suitable for ``raw_payload_json``.""" + return { + "glance_run_id": self.glance_run_id, + "glance_tool_id": self.glance_tool_id, + "tool_name": self.tool_name, + "lot_name": self.lot_name, + "material_name": self.material_name, + "glance_recipe_id": self.glance_recipe_id, + "recipe_name": self.recipe_name, + "run_start_time": self.run_start_time, + "run_end_time": self.run_end_time, + "run_status": self.run_status, + "sample_count": self.sample_count, + } + + def to_dict(self) -> dict[str, Any]: + result = self.provenance_dict() + result["parameter_metadata"] = [ + parameter.to_dict() for parameter in self.parameter_metadata + ] + result["samples"] = [sample.to_dict() for sample in self.samples] + return result + + def to_sync_dict(self, project_id: str | None = None) -> dict[str, Any]: + """Shape this run for the dedicated trace persistence backend.""" + return { + "project_id": project_id, + "source_run_id": self.glance_run_id, + "lotname": self.lot_name or "", + "run_start_time": self.run_start_time, + "run_date": self.run_start_time, + "inputs": {}, + "outputs": {}, + "raw": {"glance": self.provenance_dict()}, + "parameters": [ + parameter.to_sync_dict() for parameter in self.parameter_metadata + ], + "samples": [ + sample.to_sync_dict(sample_index) + for sample_index, sample in enumerate(self.samples) + ], + "source": "glance_db3", + } + + +@dataclass(frozen=True) +class RunTraceFailure: + glance_run_id: int | str | None + error: str + + def to_dict(self) -> dict[str, Any]: + return { + "glance_run_id": self.glance_run_id, + "error": self.error, + } + + +@dataclass(frozen=True) +class Db3TraceImportResult: + source_filename: str + source_size_bytes: int + source_sha256: str + source_tools: tuple[GlanceSourceTool, ...] + parameters: tuple[GlanceParameter, ...] + runs: tuple[GlanceRunTrace, ...] + failures: tuple[RunTraceFailure, ...] + + @property + def run_count(self) -> int: + return len(self.runs) + + @property + def sample_count(self) -> int: + return sum(run.sample_count for run in self.runs) + + def to_dict(self) -> dict[str, Any]: + return { + "source_filename": self.source_filename, + "source_size_bytes": self.source_size_bytes, + "source_sha256": self.source_sha256, + "run_count": self.run_count, + "sample_count": self.sample_count, + "source_tools": [tool.to_dict() for tool in self.source_tools], + "parameters": [parameter.to_dict() for parameter in self.parameters], + "runs": [run.to_dict() for run in self.runs], + "failures": [failure.to_dict() for failure in self.failures], + } + + +def _validate_schema(connection: sqlite3.Connection) -> None: + rows = connection.execute( + "SELECT name FROM sqlite_master WHERE type IN ('table', 'view')" + ).fetchall() + table_names = {str(row["name"]) for row in rows} + missing = sorted(_REQUIRED_TABLES - table_names) + if missing: + raise GlanceDb3SchemaError( + "DB3 file is missing required GLANCE tables: " + ", ".join(missing) + ) + + +def _load_parameters( + connection: sqlite3.Connection, +) -> tuple[tuple[GlanceParameter, ...], dict[int, GlanceParameter]]: + rows = connection.execute( + """ + SELECT idparameters, name, unit + FROM parameters + ORDER BY idparameters + """ + ).fetchall() + parameters: list[GlanceParameter] = [] + by_id: dict[int, GlanceParameter] = {} + for row in rows: + parameter_id = _required_int(row["idparameters"], "parameters.idparameters") + if parameter_id in by_id: + raise GlanceDb3SchemaError( + f"Duplicate parameters.idparameters value: {parameter_id}" + ) + parameter = GlanceParameter( + key=parameter_key(parameter_id), + glance_parameter_id=parameter_id, + raw_name=_optional_text(row["name"]), + unit=_optional_text(row["unit"]), + ) + parameters.append(parameter) + by_id[parameter_id] = parameter + return tuple(parameters), by_id + + +def _load_source_tools( + connection: sqlite3.Connection, +) -> tuple[GlanceSourceTool, ...]: + rows = connection.execute( + """ + SELECT idtools, name, description + FROM tools + ORDER BY idtools + """ + ).fetchall() + tools: list[GlanceSourceTool] = [] + seen_ids: set[int] = set() + for row in rows: + tool_id = _required_int(row["idtools"], "tools.idtools") + if tool_id in seen_ids: + raise GlanceDb3SchemaError(f"Duplicate tools.idtools value: {tool_id}") + seen_ids.add(tool_id) + tools.append( + GlanceSourceTool( + glance_tool_id=tool_id, + name=_optional_text(row["name"]), + description=_optional_text(row["description"]), + ) + ) + return tuple(tools) + + +def _run_failure_id(value: Any) -> int | str | None: + if value is None: + return None + try: + return _required_int(value, "runs.idruns") + except GlanceDb3Error: + return str(value) + + +def _extract_run( + connection: sqlite3.Connection, + run_row: sqlite3.Row, + parameter_by_id: dict[int, GlanceParameter], +) -> GlanceRunTrace: + raw_run_id = run_row["idruns"] + run_id = _required_int(raw_run_id, "runs.idruns") + run_tool_id = _optional_int(run_row["idtools"], "runs.idtools") + recipe_id = _optional_int(run_row["idrecipes"], "runs.idrecipes") + + rows = connection.execute( + """ + SELECT + s.idsamplerecord, + s.idruns, + s.idtools AS sample_tool_id, + s.time AS sample_time, + d.idsamplerecord AS data_sample_record_id, + d.idparameters, + d.value + FROM samplerecord s + LEFT JOIN data d ON d.idsamplerecord = s.idsamplerecord + WHERE s.idruns = ? + ORDER BY s.time, s.idsamplerecord, d.idparameters + """, + (raw_run_id,), + ).fetchall() + if not rows: + # The caller selects only runs with samples. Treat a disappearing run as + # a source consistency failure instead of creating an empty catalog run. + raise GlanceDb3Error(f"GLANCE run {run_id} has no sample rows") + + samples: list[GlanceTraceSample] = [] + used_parameter_ids: set[int] = set() + row_index = 0 + while row_index < len(rows): + first = rows[row_index] + sample_id = _required_int( + first["idsamplerecord"], "samplerecord.idsamplerecord" + ) + sample_run_id = _required_int(first["idruns"], "samplerecord.idruns") + if sample_run_id != run_id: + raise GlanceDb3Error( + f"Sample {sample_id} belongs to run {sample_run_id}, expected {run_id}" + ) + + values: dict[str, Any] = {} + while row_index < len(rows): + row = rows[row_index] + current_sample_id = _required_int( + row["idsamplerecord"], "samplerecord.idsamplerecord" + ) + if current_sample_id != sample_id: + break + if row["data_sample_record_id"] is not None: + parameter_id = _required_int( + row["idparameters"], "data.idparameters" + ) + key = parameter_key(parameter_id) + if key in values: + raise GlanceDb3Error( + f"Sample {sample_id} contains duplicate parameter {parameter_id}" + ) + values[key] = _json_value(row["value"]) + used_parameter_ids.add(parameter_id) + if parameter_id not in parameter_by_id: + parameter_by_id[parameter_id] = GlanceParameter( + key=key, + glance_parameter_id=parameter_id, + raw_name=None, + unit=None, + ) + row_index += 1 + + samples.append( + GlanceTraceSample( + glance_sample_record_id=sample_id, + glance_run_id=run_id, + glance_tool_id=_optional_int( + first["sample_tool_id"], "samplerecord.idtools" + ), + sample_time=_optional_text(first["sample_time"]), + values_json=values, + ) + ) + + used_parameters = tuple( + parameter_by_id[parameter_id] + for parameter_id in sorted(used_parameter_ids) + ) + return GlanceRunTrace( + glance_run_id=run_id, + glance_tool_id=run_tool_id, + tool_name=_optional_text(run_row["toolname"]), + lot_name=_optional_text(run_row["lotname"]), + material_name=_optional_text(run_row["materialname"]), + glance_recipe_id=recipe_id, + recipe_name=_optional_text(run_row["recipename"]), + run_start_time=_optional_text(run_row["starttime"]), + run_end_time=_optional_text(run_row["endtime"]), + run_status=_optional_text(run_row["status"]), + parameter_metadata=used_parameters, + samples=tuple(samples), + ) + + +def extract_db3_trace(db3_path: Path) -> Db3TraceImportResult: + """Extract one full, unaggregated trace for every GLANCE run with samples. + + Runs are isolated during extraction: malformed metadata or trace rows for one + run produce a ``RunTraceFailure`` while other valid runs are still returned. + File-level failures (not SQLite, missing schema, invalid parameter registry) + raise ``GlanceDb3Error`` because no run can be interpreted safely. + """ + path = Path(db3_path) + if not path.is_file(): + raise FileNotFoundError(f"DB3 file not found: {path}") + + source_size_bytes = path.stat().st_size + source_sha256 = _sha256(path) + try: + connection = _connect_readonly(path) + except sqlite3.Error as exc: + raise GlanceDb3Error(f"Could not open DB3 file: {exc}") from exc + + try: + try: + _validate_schema(connection) + parameters, parameter_by_id = _load_parameters(connection) + source_tools = _load_source_tools(connection) + run_rows = connection.execute( + """ + SELECT + r.idruns, + r.idtools, + r.lotname, + r.materialname, + r.idrecipes, + r.starttime, + r.endtime, + r.status, + t.name AS toolname, + rc.recipename + FROM runs r + LEFT JOIN tools t ON t.idtools = r.idtools + LEFT JOIN recipes rc ON rc.idrecipes = r.idrecipes + WHERE EXISTS ( + SELECT 1 + FROM samplerecord s + WHERE s.idruns = r.idruns + ) + ORDER BY r.idruns + """ + ).fetchall() + except sqlite3.Error as exc: + raise GlanceDb3SchemaError( + f"Could not read expected GLANCE schema: {exc}" + ) from exc + + traces: list[GlanceRunTrace] = [] + failures: list[RunTraceFailure] = [] + for run_row in run_rows: + try: + traces.append(_extract_run(connection, run_row, parameter_by_id)) + except (GlanceDb3Error, sqlite3.Error) as exc: + failures.append( + RunTraceFailure( + glance_run_id=_run_failure_id(run_row["idruns"]), + error=str(exc), + ) + ) + + # Include parameters discovered in data even if a malformed source did + # not declare them in the parameters table. Their raw metadata remains + # null, but their p values are not lost. + all_parameters = tuple( + parameter_by_id[parameter_id] + for parameter_id in sorted(parameter_by_id) + ) + return Db3TraceImportResult( + source_filename=path.name, + source_size_bytes=source_size_bytes, + source_sha256=source_sha256, + source_tools=source_tools, + parameters=all_parameters, + runs=tuple(traces), + failures=tuple(failures), + ) + finally: + connection.close() + + +def extract_db3_run_traces(db3_path: str | Path) -> Db3TraceImportResult: + """Backward-compatible alias for the canonical ``extract_db3_trace`` API.""" + return extract_db3_trace(Path(db3_path)) + + +# Keep the earlier development name import-compatible while callers migrate to +# the explicit ``Db3TraceImportResult`` name. +Db3TraceExtraction = Db3TraceImportResult + + +__all__ = [ + "Db3TraceExtraction", + "Db3TraceImportResult", + "GlanceDb3Error", + "GlanceDb3SchemaError", + "GlanceParameter", + "GlanceRunTrace", + "GlanceSourceTool", + "GlanceTraceSample", + "RunTraceFailure", + "extract_db3_trace", + "extract_db3_run_traces", + "parameter_key", +] diff --git a/api/main.py b/api/main.py index 318d629..5668b9c 100644 --- a/api/main.py +++ b/api/main.py @@ -44,6 +44,8 @@ "project_members", "etcher_runs", "etcher_run_files", + "equipment_runs", + "equipment_run_trace_samples", "samples", "experiment_samples", "run_samples", @@ -141,6 +143,7 @@ def _run_startup_migrations() -> None: """ from data_loader_pg import get_pg_superuser_connection from data_loader_pg import ensure_etcher_run_file_refs_pg + from data_loader_pg import ensure_equipment_run_trace_samples_pg from data_loader_pg import ensure_equipment_runs_pg from data_loader_pg import ensure_etcher_recipe_name_column_pg from metadata_pg import ( @@ -159,6 +162,8 @@ def _run_startup_migrations() -> None: ensure_sample_and_publication_tables_pg, ensure_equipment_access_table_pg, ensure_data_uploads_kind_column_pg, + ensure_data_uploads_processing_started_at_pg, + recover_stale_db3_uploads_pg, ) lock_conn = get_pg_superuser_connection() @@ -168,8 +173,9 @@ def _run_startup_migrations() -> None: lock_conn.commit() ensure_etcher_run_file_refs_pg() - ensure_equipment_runs_pg() ensure_etcher_recipe_name_column_pg() + ensure_equipment_runs_pg() + ensure_equipment_run_trace_samples_pg() ensure_experiment_proposal_generation_columns_pg() ensure_experiment_type_reuse_columns_pg() ensure_experiment_type_versioning_columns_pg() @@ -185,6 +191,8 @@ def _run_startup_migrations() -> None: ensure_sample_and_publication_tables_pg() ensure_equipment_access_table_pg() ensure_data_uploads_kind_column_pg() + ensure_data_uploads_processing_started_at_pg() + recover_stale_db3_uploads_pg() finally: try: with lock_conn.cursor() as cur: diff --git a/api/metadata_pg.py b/api/metadata_pg.py index 9ae481c..e0e04ba 100644 --- a/api/metadata_pg.py +++ b/api/metadata_pg.py @@ -1,5 +1,6 @@ import json import logging +import os import re import uuid from datetime import date, datetime @@ -5481,6 +5482,63 @@ def ensure_data_uploads_kind_column_pg() -> None: conn.close() +def ensure_data_uploads_processing_started_at_pg() -> None: + """Add a processing lease timestamp for restart-safe upload retries.""" + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + "ALTER TABLE data_uploads " + "ADD COLUMN IF NOT EXISTS processing_started_at TIMESTAMPTZ" + ) + conn.commit() + finally: + conn.close() + + +def recover_stale_db3_uploads_pg() -> None: + """Return abandoned pilot processing leases to a retryable state.""" + if os.getenv("DT_ENABLE_DB3_UPLOAD", "").strip().casefold() not in { + "1", + "true", + "yes", + "on", + }: + return + + raw_stale_minutes = os.getenv("DT_DB3_PROCESSING_STALE_MINUTES", "30").strip() + try: + stale_minutes = max(int(raw_stale_minutes), 1) + except ValueError: + stale_minutes = 30 + + conn = get_pg_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE data_uploads + SET status = 'success', + processing_started_at = NULL, + errors_json = COALESCE(errors_json, '[]'::jsonb) + || jsonb_build_array( + 'Previous DB3 processing did not finish; retry processing.' + ) + WHERE status = 'processing' + AND LOWER(filename) LIKE '%%.db3' + AND ( + processing_started_at IS NULL + OR processing_started_at < CURRENT_TIMESTAMP + - (%s * INTERVAL '1 minute') + ) + """, + (stale_minutes,), + ) + conn.commit() + finally: + conn.close() + + def create_upload_record_pg( *, equipment_id: str, @@ -5553,7 +5611,6 @@ def list_uploads_pg(user: PlatformUser) -> list[dict[str, Any]]: equipment_id, filename, stored_as, - storage_path, size_bytes, row_count, columns_json, @@ -5588,7 +5645,6 @@ def list_uploads_pg(user: PlatformUser) -> list[dict[str, Any]]: "equipment_id": record["equipment_id"], "filename": record["filename"], "stored_as": record["stored_as"], - "storage_path": record["storage_path"], "size_bytes": int(record["size_bytes"] or 0), "row_count": int(record["row_count"] or 0), "columns": _coerce_json(record.get("columns_json"), []), diff --git a/api/routers/dataset_v2.py b/api/routers/dataset_v2.py index 53f0768..fe1901e 100644 --- a/api/routers/dataset_v2.py +++ b/api/routers/dataset_v2.py @@ -8,7 +8,7 @@ from datetime import datetime, timezone from typing import Literal, Optional -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query from fastapi import Request from fastapi.responses import JSONResponse from pydantic import BaseModel @@ -18,6 +18,7 @@ add_project_member_pg, create_project_pg, delete_project_pg, + get_equipment_run_trace_pg, get_projects_list_pg, get_run_detail_pg, get_runs_list_pg, @@ -756,7 +757,7 @@ def run_detail( platform_user: PlatformUser = Depends(get_platform_user), ): """ - Return a single etcher_runs row by ``idruns``, joined with project context. + Return one visible canonical or generic catalog run with project context. RLS applies for non-admins; admins bypass via the superuser connection. Returns 404 when the run does not exist OR the caller can't see it — we @@ -778,6 +779,44 @@ def run_detail( return record +@router.get("/runs/{run_id}/trace") +def run_trace( + run_id: int, + parameters: Optional[list[str]] = Query(default=None), + platform_user: PlatformUser = Depends(get_platform_user), +): + """Return the complete timestamped trace linked to a generic equipment run. + + ``parameters`` may be repeated (``?parameters=p130¶meters=p132``) to + reduce each sample's values object. Absolute source timestamps are always + returned, and relative seconds are computed dynamically from the earliest + sample in the complete run. Missing values remain null so plots preserve + gaps instead of connecting across them. + """ + is_admin = platform_user.role == "admin" + logger.info( + "[dataset/v2/runs/%s/trace] Fetching trace for user=%s role=%s parameters=%s", + run_id, + platform_user.id, + platform_user.role, + parameters or [], + ) + try: + record = get_equipment_run_trace_pg( + run_id=run_id, + parameters=parameters, + nanohub_user_id=platform_user.id, + is_admin=is_admin, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if record is None: + # As with run detail, do not reveal whether the run is absent, private, + # canonical (non-trace), or simply has no linked trace. + raise HTTPException(status_code=404, detail=f"Trace for run {run_id} not found") + return record + + @router.post("/runs/sync") async def sync_runs( request: Request, diff --git a/api/routers/upload.py b/api/routers/upload.py index f9f938a..c2e2e00 100644 --- a/api/routers/upload.py +++ b/api/routers/upload.py @@ -1,12 +1,14 @@ """ -Data upload router — stores CSV/Excel files with lightweight structural checks. +Data upload router — stores CSV/Excel files and the temporary GLANCE DB3 test path. """ import csv -import io import hashlib +import io +import json import logging import math import os +import threading from datetime import datetime from decimal import Decimal, InvalidOperation from pathlib import Path @@ -16,12 +18,14 @@ from fastapi.responses import StreamingResponse from psycopg2.extras import Json from pydantic import BaseModel +from starlette.concurrency import run_in_threadpool from domain_configs import derive_expected_columns, load_domain_config from data_loader_pg import ( get_pg_connection, get_projects_list_pg, merge_equipment_runs_pg, + sync_equipment_run_traces_pg, sync_equipment_runs_pg, sync_runs_pg, ) @@ -30,8 +34,10 @@ create_upload_record_pg, get_equipment_pg, list_uploads_pg, + recover_stale_db3_uploads_pg, ) from security import PlatformUser, get_platform_user +from glance_db3.trace_import import extract_db3_trace logger = logging.getLogger(__name__) @@ -111,6 +117,168 @@ def resolve_uploads_dir() -> Path: # Storage directory for uploaded files UPLOADS_DIR = resolve_uploads_dir() +CSV_EXCEL_UPLOAD_LIMIT_BYTES = 50 * 1024 * 1024 +DB3_UPLOAD_CHUNK_BYTES = 1024 * 1024 + + +def _db3_upload_enabled() -> bool: + return os.getenv("DT_ENABLE_DB3_UPLOAD", "").strip().casefold() in { + "1", + "true", + "yes", + "on", + } + + +def _db3_allowed_user_ids() -> set[str]: + raw = os.getenv("DT_DB3_ALLOWED_USER_IDS", "") + return { + user_id.strip().casefold() + for user_id in raw.split(",") + if user_id.strip() + } + + +def _db3_user_is_allowed(user: PlatformUser) -> bool: + return _db3_upload_enabled() and user.id.strip().casefold() in _db3_allowed_user_ids() + + +def _assert_db3_user_allowed(user: PlatformUser) -> None: + if not _db3_upload_enabled(): + raise HTTPException( + status_code=503, + detail="The temporary DB3 upload pilot is disabled.", + ) + if user.id.strip().casefold() not in _db3_allowed_user_ids(): + raise HTTPException( + status_code=403, + detail="DB3 uploads are limited to the temporary pilot participants.", + ) + + +def _db3_upload_limit_bytes() -> int: + raw = os.getenv("DT_DB3_UPLOAD_MAX_MB", "64").strip() + try: + max_mb = int(raw) + except ValueError: + max_mb = 64 + return max(max_mb, 1) * 1024 * 1024 + + +def _db3_allowed_source_tools(equipment_id: str) -> set[str]: + """Return the explicitly configured GLANCE source tools for equipment. + + DB3 ingestion is intentionally a temporary, opt-in test path. Requiring an + equipment-to-tool map prevents a source export from being silently attached + to the wrong registered machine as more GLANCE tools are introduced. + """ + if not _db3_upload_enabled(): + raise ValueError("The temporary DB3 upload pilot is disabled.") + + raw = os.getenv("DT_DB3_EQUIPMENT_TOOL_MAP", "").strip() + if not raw: + raise ValueError( + "DB3 uploads are disabled until DT_DB3_EQUIPMENT_TOOL_MAP maps this " + "equipment ID to its GLANCE source tool name." + ) + try: + configured = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("DT_DB3_EQUIPMENT_TOOL_MAP is not valid JSON.") from exc + if not isinstance(configured, dict): + raise ValueError("DT_DB3_EQUIPMENT_TOOL_MAP must be a JSON object.") + + values = configured.get(equipment_id) + if isinstance(values, str): + values = [values] + if not isinstance(values, list): + raise ValueError( + f"DB3 uploads are not enabled for equipment '{equipment_id}'." + ) + allowed = {str(value).strip() for value in values if str(value).strip()} + if not allowed: + raise ValueError( + f"DB3 uploads are not enabled for equipment '{equipment_id}'." + ) + return allowed + + +def _db3_mapped_equipment_ids() -> list[str]: + """Return valid configured equipment IDs without exposing source tool names.""" + if not _db3_upload_enabled(): + return [] + raw = os.getenv("DT_DB3_EQUIPMENT_TOOL_MAP", "").strip() + try: + configured = json.loads(raw) if raw else {} + except json.JSONDecodeError: + return [] + if not isinstance(configured, dict): + return [] + return sorted( + str(equipment_id) + for equipment_id, values in configured.items() + if str(equipment_id).strip() + and ( + (isinstance(values, str) and values.strip()) + or ( + isinstance(values, list) + and any(str(value).strip() for value in values) + ) + ) + ) + + +def _assert_db3_pilot_project(project: dict[str, Any]) -> None: + expected_project_id = os.getenv("DT_DB3_PILOT_PROJECT_ID", "").strip() + if not expected_project_id: + raise HTTPException( + status_code=503, + detail="The temporary DB3 pilot project is not configured.", + ) + if str(project.get("id") or "").strip() != expected_project_id: + raise HTTPException( + status_code=400, + detail="DB3 traces can only be processed into the private pilot project.", + ) + if str(project.get("access") or "").strip().casefold() != "private": + raise HTTPException( + status_code=503, + detail="The configured DB3 pilot project is not private.", + ) + + +def _copy_upload_with_limit(source: Any, destination: Path, max_bytes: int) -> int: + """Stream an uploaded file to disk without buffering the whole DB3.""" + total = 0 + try: + source.seek(0) + with destination.open("wb") as target: + while True: + chunk = source.read(DB3_UPLOAD_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise ValueError("too_large") + target.write(chunk) + except Exception: + destination.unlink(missing_ok=True) + raise + return total + + +def _validate_db3_timestamp(value: Any, field_name: str) -> str: + """Validate an ISO source timestamp while returning its exact text.""" + raw = str(value or "").strip() + if not raw: + raise ValueError(f"{field_name} is missing") + candidate = raw[:-1] + "+00:00" if raw.endswith("Z") else raw + try: + datetime.fromisoformat(candidate) + except ValueError as exc: + raise ValueError(f"{field_name} is invalid: {raw!r}") from exc + return raw + QUDT_UNIT_LABEL_BY_CODE = { "CentiM3-PER-MIN": "sccm", "MilliTORR": "mTorr", @@ -350,7 +518,7 @@ def _update_upload_record( cur.execute( """ UPDATE data_uploads - SET status = %s, errors_json = %s + SET status = %s, errors_json = %s, processing_started_at = NULL WHERE id = %s """, (status, Json(errors), upload_id), @@ -359,7 +527,8 @@ def _update_upload_record( cur.execute( """ UPDATE data_uploads - SET status = %s, errors_json = %s, row_count = %s + SET status = %s, errors_json = %s, row_count = %s, + processing_started_at = NULL WHERE id = %s """, (status, Json(errors), row_count, upload_id), @@ -382,7 +551,8 @@ def _claim_upload_for_processing(upload_id: str) -> bool: cur.execute( """ UPDATE data_uploads - SET status = 'processing', errors_json = '[]'::jsonb + SET status = 'processing', errors_json = '[]'::jsonb, + processing_started_at = CURRENT_TIMESTAMP WHERE id = %s AND status <> 'processing' RETURNING id """, @@ -395,6 +565,32 @@ def _claim_upload_for_processing(upload_id: str) -> bool: conn.close() +def _touch_upload_processing_lease(upload_id: str) -> None: + conn = get_pg_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE data_uploads + SET processing_started_at = CURRENT_TIMESTAMP + WHERE id = %s AND status = 'processing' + """, + (upload_id,), + ) + conn.commit() + finally: + conn.close() + + +def _db3_processing_heartbeat_seconds() -> int: + raw = os.getenv("DT_DB3_PROCESSING_HEARTBEAT_SECONDS", "60").strip() + try: + seconds = int(raw) + except ValueError: + seconds = 60 + return min(max(seconds, 5), 300) + + def _load_upload_record(upload_id: str, user: PlatformUser) -> dict[str, Any]: conn = get_pg_connection() try: @@ -429,7 +625,12 @@ def _load_upload_record(upload_id: str, user: PlatformUser) -> dict[str, Any]: "errors": stored_issues, "kind": _normalize_kind(row[8]), } - if user.role != "admin": + if Path(record["filename"]).suffix.lower() == ".db3": + # Pilot participants deliberately share the DB3 queue so Paul, Rich, + # and the platform administrator can help one another. No same-org + # fallback applies to proprietary DB3 exports. + _assert_db3_user_allowed(user) + elif user.role != "admin": same_owner = record["owner_id"] == user.id same_org = record["owner_org"] and record["owner_org"] == user.organization if not same_owner and not same_org: @@ -836,21 +1037,102 @@ async def upload_file( # Check extension ext = Path(file.filename).suffix.lower() - if ext not in (".csv", ".xlsx", ".xls"): + if ext not in (".csv", ".xlsx", ".xls", ".db3"): raise HTTPException( status_code=400, - detail=f"Unsupported file type '{ext}'. Use CSV, XLSX, or XLS." + detail=f"Unsupported file type '{ext}'. Use CSV, XLSX, XLS, or DB3." ) - # Read file contents + if ext == ".db3": + _assert_db3_user_allowed(user) + + equipment_upload_dir = UPLOADS_DIR / equipment_id + equipment_upload_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + original_name = Path(file.filename).name + safe_name = f"{timestamp}_{original_name}" + dest = equipment_upload_dir / safe_name + + if ext == ".db3": + try: + _db3_allowed_source_tools(equipment_id) + except ValueError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + + max_size = _db3_upload_limit_bytes() + try: + size_bytes = await run_in_threadpool( + _copy_upload_with_limit, + file.file, + dest, + max_size, + ) + except ValueError as exc: + if str(exc) == "too_large": + max_mb = max_size // (1024 * 1024) + raise HTTPException( + status_code=400, + detail=f"File exceeds {max_mb} MB limit", + ) from exc + raise + + if size_bytes == 0: + dest.unlink(missing_ok=True) + raise HTTPException(status_code=400, detail="File is empty") + + upload_record = create_upload_record_pg( + equipment_id=equipment_id, + filename=original_name, + stored_as=safe_name, + storage_path=str(dest), + size_bytes=size_bytes, + row_count=0, + columns=[], + errors=[], + status="success", + user=user, + kind="all", + ) + + logger.info( + "Stored DB3 upload %s (%s bytes) for equipment %s; catalog import is pending", + original_name, + size_bytes, + equipment_id, + ) + + return { + "status": "success", + "upload_id": upload_record["upload_id"], + "equipment_id": equipment_id, + "kind": "all", + "filename": original_name, + "stored_as": safe_name, + "size_bytes": size_bytes, + "row_count": 0, + "columns": [], + "errors": None, + "message": ( + "DB3 stored for test import. Choose a project and click Process; " + "each physical GLANCE run will become one catalog run with its " + "full timestamped trace attached." + ), + "processing_status": "stored_only", + } + + # CSV/Excel validation still needs the in-memory content, but these uploads + # retain the existing, much smaller 50 MB limit. Large DB3 files use the + # streaming branch above and never become one giant Python bytes object. contents = await file.read() size_bytes = len(contents) - if size_bytes == 0: raise HTTPException(status_code=400, detail="File is empty") + if size_bytes > CSV_EXCEL_UPLOAD_LIMIT_BYTES: + max_mb = CSV_EXCEL_UPLOAD_LIMIT_BYTES // (1024 * 1024) + raise HTTPException(status_code=400, detail=f"File exceeds {max_mb} MB limit") - if size_bytes > 50 * 1024 * 1024: # 50 MB limit - raise HTTPException(status_code=400, detail="File exceeds 50 MB limit") + with dest.open("wb") as stored_file: + stored_file.write(contents) # Parse CSV to validate basic structure. # @@ -949,16 +1231,6 @@ async def upload_file( row_count = -1 columns = [] - # Store the file - equipment_upload_dir = UPLOADS_DIR / equipment_id - equipment_upload_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - safe_name = f"{timestamp}_{file.filename}" - dest = equipment_upload_dir / safe_name - - with open(dest, "wb") as f: - f.write(contents) - # Only hard errors drive failure; warnings are surfaced but stored # alongside so the upload still reaches a processable "success" state. status = "error" if errors else "success" @@ -1015,12 +1287,37 @@ async def upload_file( } +@router.get("/upload-capabilities") +def get_upload_capabilities( + user: PlatformUser = Depends(get_platform_user), +): + """Describe runtime-gated upload features for the authenticated caller.""" + mapped_equipment_ids = _db3_mapped_equipment_ids() + return { + "db3_upload": { + "enabled": _db3_user_is_allowed(user) and bool(mapped_equipment_ids), + "max_mb": _db3_upload_limit_bytes() // (1024 * 1024), + "equipment_ids": mapped_equipment_ids if _db3_user_is_allowed(user) else [], + } + } + + @router.get("/uploads") def list_uploads( user: PlatformUser = Depends(get_platform_user), ): """List all previously uploaded files.""" - return list_uploads_pg(user) + db3_allowed = _db3_user_is_allowed(user) + if db3_allowed: + recover_stale_db3_uploads_pg() + uploads = list_uploads_pg(user) + if db3_allowed: + return uploads + return [ + record + for record in uploads + if Path(str(record.get("filename") or "")).suffix.lower() != ".db3" + ] def _effective_equipment_config(equipment: dict[str, Any]) -> dict[str, Any]: @@ -1498,6 +1795,52 @@ def _execute_upload_processing( upload_record: dict[str, Any], project_id: str, prior_warnings: list[str], +) -> None: + """Execute an upload with a live processing lease for DB3 imports.""" + is_db3 = Path(upload_record["filename"]).suffix.lower() == ".db3" + if not is_db3: + _execute_upload_processing_impl( + upload_id=upload_id, + upload_record=upload_record, + project_id=project_id, + prior_warnings=prior_warnings, + ) + return + + stop_heartbeat = threading.Event() + + def heartbeat() -> None: + interval = _db3_processing_heartbeat_seconds() + while not stop_heartbeat.wait(interval): + try: + _touch_upload_processing_lease(upload_id) + except Exception: # noqa: BLE001 - recovery remains available + logger.exception("DB3 upload %s lease heartbeat failed", upload_id) + + heartbeat_thread = threading.Thread( + target=heartbeat, + name=f"db3-lease-{upload_id}", + daemon=True, + ) + heartbeat_thread.start() + try: + _execute_upload_processing_impl( + upload_id=upload_id, + upload_record=upload_record, + project_id=project_id, + prior_warnings=prior_warnings, + ) + finally: + stop_heartbeat.set() + heartbeat_thread.join(timeout=1) + + +def _execute_upload_processing_impl( + *, + upload_id: str, + upload_record: dict[str, Any], + project_id: str, + prior_warnings: list[str], ) -> None: """Parse and ingest a stored upload, updating its status to ``processed`` or ``failed``. Runs in a background task (Starlette executes sync background @@ -1505,8 +1848,9 @@ def _execute_upload_processing( can poll for progress instead of blocking on a long ingestion. The canonical etcher routes through sync_runs_pg() (the same sink as Azure). - Split input/output uploads are merged on the lot + run-date key; every other - generic upload is ingested into the equipment_runs table. + Split input/output uploads are merged on the lot + run-date key. A DB3 is + extracted here (in Starlette's background threadpool) into one generic + equipment_runs parent per physical GLANCE run plus linked trace samples. """ equipment_id = upload_record["equipment_id"] kind = _normalize_kind(upload_record.get("kind")) @@ -1520,6 +1864,148 @@ def _fail(errors: list[str], row_count: int) -> None: row_count=row_count, ) + if Path(upload_record["filename"]).suffix.lower() == ".db3": + try: + allowed_tools = _db3_allowed_source_tools(equipment_id) + extraction = extract_db3_trace(Path(upload_record["storage_path"])) + + observed_tools = { + str(run.tool_name).strip() + for run in extraction.runs + if str(run.tool_name or "").strip() + } + run_rows: list[dict[str, Any]] = [] + import_warnings = [ + f"Skipped GLANCE run {failure.glance_run_id}: {failure.error}" + for failure in extraction.failures + ] + for run in extraction.runs: + source_tool_name = str(run.tool_name or "").strip() + if not source_tool_name: + import_warnings.append( + f"Skipped GLANCE run {run.glance_run_id}: source tool name is missing" + ) + continue + if source_tool_name not in allowed_tools: + import_warnings.append( + f"Skipped GLANCE run {run.glance_run_id}: source tool " + f"'{source_tool_name}' is not mapped to equipment '{equipment_id}'" + ) + continue + try: + if not 0 <= run.glance_run_id <= 2_147_483_647: + raise ValueError("source run ID is outside the supported range") + first_sample_time = next( + ( + sample.sample_time + for sample in run.samples + if sample.sample_time + ), + None, + ) + run_date = _validate_db3_timestamp( + run.run_start_time or first_sample_time, + "run start timestamp", + ) + for sample in run.samples: + if not ( + 0 + <= sample.glance_sample_record_id + <= 9_223_372_036_854_775_807 + ): + raise ValueError( + "one or more source sample IDs are outside the supported range" + ) + _validate_db3_timestamp( + sample.sample_time, + f"sample {sample.glance_sample_record_id} timestamp", + ) + parameter_metadata = [ + { + "key": parameter.key, + "source_parameter_id": parameter.glance_parameter_id, + "name": parameter.raw_name or parameter.key, + "source_name": parameter.raw_name, + "unit": parameter.unit or "", + } + for parameter in run.parameter_metadata + ] + raw_payload = { + "glance": run.provenance_dict(), + "source_file": { + "type": "glance_db3", + "filename": extraction.source_filename, + "size_bytes": extraction.source_size_bytes, + "sha256": extraction.source_sha256, + }, + } + run_rows.append( + { + "lotname": run.lot_name or f"GLANCE run {run.glance_run_id}", + "run_date": run_date, + "inputs": {}, + "outputs": {}, + "raw": raw_payload, + "source": "glance_db3", + "source_run_id": run.glance_run_id, + "parameters": parameter_metadata, + "timestamp_timezone": None, + "samples": [ + { + "source_sample_id": sample.glance_sample_record_id, + "source_tool_id": sample.glance_tool_id, + "sample_time_raw": sample.sample_time, + "values": sample.values_json, + } + for sample in run.samples + ], + } + ) + except Exception as exc: # isolate one malformed physical run + import_warnings.append( + f"Skipped GLANCE run {run.glance_run_id}: {exc}" + ) + + if not run_rows: + _fail( + (import_warnings or ["No physical runs with trace samples were found."]), + 0, + ) + return + + inserted = sync_equipment_run_traces_pg( + equipment_id=equipment_id, + project_id=project_id, + upload_id=upload_id, + upload_filename=upload_record["filename"], + runs=run_rows, + ) + except Exception: # noqa: BLE001 - persist an actionable DB3 error + logger.exception("DB3 upload %s import failed", upload_id) + _fail( + [ + "DB3 import failed. The source file could not be processed; " + "contact the pilot administrator for the server-side details." + ], + 0, + ) + return + + logger.info( + "DB3 upload %s processed: %s physical run(s), %s trace sample(s), source tools=%s", + upload_id, + inserted, + sum(len(row["samples"]) for row in run_rows), + sorted(observed_tools), + ) + _update_upload_record( + upload_id=upload_id, + status="processed", + errors=(prior_warnings + import_warnings)[:50], + row_count=len(run_rows), + ) + return + try: if is_canonical_etcher: rows, errors = _csv_upload_to_sync_rows( @@ -1587,7 +2073,7 @@ def process_upload( user: PlatformUser = Depends(get_platform_user), ): """ - Schedule a stored CSV upload for ingestion into canonical run records. + Schedule a stored CSV or test DB3 upload for ingestion into catalog runs. Validation and ingestion can take minutes for large files, so this endpoint performs only the fast authorization/consistency checks synchronously, marks @@ -1600,8 +2086,13 @@ def process_upload( raise HTTPException(status_code=400, detail="project_id is required") project = _assert_project_visible(project_id, user) + if _db3_user_is_allowed(user): + recover_stale_db3_uploads_pg() upload_record = _load_upload_record(upload_id, user) equipment_id = upload_record["equipment_id"] + if Path(upload_record["filename"]).suffix.lower() == ".db3": + _assert_db3_user_allowed(user) + _assert_db3_pilot_project(project) # Non-blocking range warnings recorded at validation time should survive # processing so the catalog/upload UI can keep surfacing them. A "success" diff --git a/api/scripts/add_equipment_runs.sql b/api/scripts/add_equipment_runs.sql index b39e5ef..62b287e 100644 --- a/api/scripts/add_equipment_runs.sql +++ b/api/scripts/add_equipment_runs.sql @@ -66,3 +66,63 @@ BEGIN END IF; END $$; + +-- Full time-series traces are stored separately so one equipment_runs row still +-- represents one physical machine run. GLANCE timestamps are currently naive; +-- keep the parsed wall time timezone-free and preserve the exact source text. +CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_index INT NOT NULL, + source_sample_id BIGINT NOT NULL, + source_tool_id BIGINT, + sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + sample_time_raw TEXT NOT NULL, + values_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, sample_index), + UNIQUE (equipment_run_id, source_sample_id), + CHECK (jsonb_typeof(values_json) = 'object') +); + +ALTER TABLE equipment_run_trace_samples +ADD COLUMN IF NOT EXISTS source_tool_id BIGINT; + +CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time + ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); + +ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; + +GRANT SELECT ON equipment_run_trace_samples TO api_client; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_trace_samples' + AND policyname = 'equipment_run_trace_visibility_policy' + ) THEN + CREATE POLICY equipment_run_trace_visibility_policy + ON equipment_run_trace_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; +END +$$; diff --git a/api/scripts/init_db.sql b/api/scripts/init_db.sql index 1c3862d..f2a1804 100644 --- a/api/scripts/init_db.sql +++ b/api/scripts/init_db.sql @@ -487,6 +487,7 @@ CREATE TABLE IF NOT EXISTS data_uploads ( kind VARCHAR(20) NOT NULL DEFAULT 'all', owner_id VARCHAR(255) REFERENCES users(id), owner_org VARCHAR(255) DEFAULT '', + processing_started_at TIMESTAMP WITH TIME ZONE, uploaded_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); diff --git a/api/tests/E2E_README.md b/api/tests/E2E_README.md index 13008f8..c65c9ca 100644 --- a/api/tests/E2E_README.md +++ b/api/tests/E2E_README.md @@ -70,9 +70,9 @@ reachable. ### 1. Pull the shared secrets out of Kubernetes ```bash -kubectl get secret -n nanohub dt-api-secret -o jsonpath='{.data.DT_SYSTEM_TOKEN}' | base64 -d -kubectl get secret -n nanohub dt-api-secret -o jsonpath='{.data.INGESTION_TOKEN}' | base64 -d -kubectl get secret -n nanohub dt-postgres-secret -o jsonpath='{.data.superuser-password}' | base64 -d +kubectl get secret -n ncn-digitaltwins-zchen dt-api-secret -o jsonpath='{.data.DT_SYSTEM_TOKEN}' | base64 -d +kubectl get secret -n ncn-digitaltwins-zchen dt-api-secret -o jsonpath='{.data.INGESTION_TOKEN}' | base64 -d +kubectl get secret -n ncn-digitaltwins-zchen dt-postgres-secret -o jsonpath='{.data.superuser-password}' | base64 -d ``` (Secret key names may differ slightly in your cluster — adjust accordingly.) @@ -82,7 +82,7 @@ kubectl get secret -n nanohub dt-postgres-secret -o jsonpath='{.data.superuser-p In a separate terminal: ```bash -kubectl port-forward -n nanohub svc/dt-db-v2-srv 5432:5432 +kubectl port-forward -n ncn-digitaltwins-zchen svc/dt-db-v2-srv 5432:5432 ``` Skip this step and set `SKIP_ADMIN_BOOTSTRAP=true` if `ngholiza` is already diff --git a/api/tests/test_e2e_admin_flow.py b/api/tests/test_e2e_admin_flow.py index a1b4be3..0a6ad63 100644 --- a/api/tests/test_e2e_admin_flow.py +++ b/api/tests/test_e2e_admin_flow.py @@ -303,7 +303,7 @@ def phase_02_admin_bootstrap(cfg: Config, api: Api, fixtures: Fixtures, report: detail=( f"Could not reach Postgres at {cfg.pg_host}:{cfg.pg_port}: {exc}\n" "If dt-db-v2 is ClusterIP-only, run in another terminal:\n" - " kubectl port-forward -n nanohub svc/dt-db-v2-srv 5432:5432\n" + " kubectl port-forward -n ncn-digitaltwins-zchen svc/dt-db-v2-srv 5432:5432\n" "then rerun this script. Or set SKIP_ADMIN_BOOTSTRAP=true if\n" "you only need header-level admin (UI login will still be\n" "researcher until the row is updated)." @@ -879,7 +879,7 @@ def _diagnose_rls_leak(cfg: Config, fixtures: Fixtures) -> str: except Exception as exc: return ( f"Could not reach Postgres at {cfg.pg_host}:{cfg.pg_port} for diagnostics: {exc}\n" - "Ensure `kubectl port-forward -n nanohub svc/dt-db-v2-srv 5432:5432` is running." + "Ensure `kubectl port-forward -n ncn-digitaltwins-zchen svc/dt-db-v2-srv 5432:5432` is running." ) try: diff --git a/api/tests/test_glance_db3_trace_import.py b/api/tests/test_glance_db3_trace_import.py new file mode 100644 index 0000000..a7ea722 --- /dev/null +++ b/api/tests/test_glance_db3_trace_import.py @@ -0,0 +1,810 @@ +import hashlib +import io +import json +import os +import sqlite3 +import sys +import unittest +from datetime import datetime +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from fastapi import HTTPException + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +from glance_db3.trace_import import Db3TraceImportResult, extract_db3_trace +from data_loader_pg import ( + EQUIPMENT_RUN_ID_OFFSET, + _equipment_runs_base_select, + get_equipment_run_trace_pg, + _normalize_trace_parameter_key, + _parse_trace_sample_time, + sync_equipment_run_traces_pg, +) +from routers import upload as upload_router +from security import PlatformUser + + +def _write_fixture( + path: Path, + *, + include_bad_run: bool = False, + include_bad_timestamp_run: bool = False, +) -> None: + connection = sqlite3.connect(path) + try: + connection.executescript( + """ + CREATE TABLE runs ( + idruns NUMBRIC PRIMARY KEY, + idtools NUMBRIC, + lotname TEXT, + idrecipes NUMBRIC, + starttime TEXT, + endtime TEXT, + status TEXT, + materialname TEXT + ); + CREATE TABLE samplerecord ( + idsamplerecord NUMBRIC PRIMARY KEY, + idruns NUMBRIC, + idtools NUMBRIC, + time TEXT + ); + CREATE TABLE data ( + idsamplerecord NUMBRIC, + idparameters NUMBRIC, + value NUMBRIC, + PRIMARY KEY (idsamplerecord, idparameters) + ); + CREATE TABLE parameters ( + idparameters NUMBRIC PRIMARY KEY, + name TEXT, + unit TEXT + ); + CREATE TABLE tools ( + idtools NUMBRIC PRIMARY KEY, + name TEXT, + description TEXT + ); + CREATE TABLE recipes ( + idrecipes NUMBRIC PRIMARY KEY, + idtools NUMBRIC, + recipename TEXT, + recipefile TEXT, + hash TEXT, + timestamp TEXT + ); + """ + ) + connection.execute( + "INSERT INTO tools (idtools, name) VALUES (?, ?)", + (5, "VLN-11304-CTC-PM1"), + ) + connection.execute( + "INSERT INTO recipes (idrecipes, idtools, recipename) VALUES (?, ?, ?)", + (9, 5, "O2 clean-Engineering"), + ) + connection.executemany( + "INSERT INTO parameters (idparameters, name, unit) VALUES (?, ?, ?)", + [ + (1, "G Gas 1 Flow", "sccm"), + (2, "P Chamber Pressure", "mTorr"), + (7, "Unused Parameter", "V"), + ], + ) + connection.executemany( + """ + INSERT INTO runs ( + idruns, idtools, lotname, idrecipes, starttime, endtime, + status, materialname + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + ( + 101, + 5, + "LOT-REUSED", + 9, + "2026-07-01 10:00:00.000001", + "2026-07-01 10:00:10.999999", + "processed", + "Silicon", + ), + ( + 102, + 5, + "LOT-REUSED", + 9, + "2026-07-02 11:00:00.100000", + "2026-07-02 11:00:01.200000", + "processed", + "Silicon", + ), + ( + 103, + 5, + "NO-SAMPLES", + 9, + "2026-07-03 12:00:00", + "2026-07-03 12:01:00", + "aborted", + "Silicon", + ), + ( + 104, + 5, + "EMPTY-TRACE", + 9, + "2026-07-03 14:00:00.000010", + "2026-07-03 14:00:01.000020", + "processed", + "Silicon", + ), + ], + ) + connection.executemany( + """ + INSERT INTO samplerecord (idsamplerecord, idruns, idtools, time) + VALUES (?, ?, ?, ?) + """, + [ + # IDs are deliberately out of timestamp order. + (1001, 101, 5, "2026-07-01 10:00:02.875000"), + (1002, 101, 5, "2026-07-01 10:00:00.125000"), + # This intentionally has no data rows and must still survive. + (1003, 101, 5, "2026-07-01 10:00:09.000001"), + (2001, 102, 5, "2026-07-02 11:00:00.333333"), + # A run with samples but no values is still a physical run. + (4001, 104, 5, "2026-07-03 14:00:00.500015"), + ], + ) + connection.executemany( + "INSERT INTO data (idsamplerecord, idparameters, value) VALUES (?, ?, ?)", + [ + (1001, 1, 1.25), + # Explicit SQL null is distinct from an absent p2 key. + (1001, 2, None), + (1002, 2, 44.5), + (2001, 1, 2.5), + ], + ) + if include_bad_run: + connection.execute( + """ + INSERT INTO runs ( + idruns, idtools, lotname, idrecipes, starttime, endtime, + status, materialname + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + 300.5, + 5, + "MALFORMED", + 9, + "2026-07-04 13:00:00", + "2026-07-04 13:01:00", + "processed", + "Silicon", + ), + ) + connection.execute( + """ + INSERT INTO samplerecord (idsamplerecord, idruns, idtools, time) + VALUES (?, ?, ?, ?) + """, + (3001, 300.5, 5, "2026-07-04 13:00:00.555555"), + ) + if include_bad_timestamp_run: + connection.execute( + """ + INSERT INTO runs ( + idruns, idtools, lotname, idrecipes, starttime, endtime, + status, materialname + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + 105, + 5, + "BAD-TIMESTAMP", + 9, + "not-a-timestamp", + "2026-07-05 13:01:00", + "processed", + "Silicon", + ), + ) + connection.execute( + """ + INSERT INTO samplerecord (idsamplerecord, idruns, idtools, time) + VALUES (?, ?, ?, ?) + """, + (5001, 105, 5, "also-not-a-timestamp"), + ) + connection.commit() + finally: + connection.close() + + +class GlanceDb3TraceImportTests(unittest.TestCase): + def test_db3_pilot_is_default_off_and_requires_an_explicit_user_allowlist( + self, + ) -> None: + allowed_user = PlatformUser( + id="will3042", + email="will3042@purdue.edu", + name="Paul Williams", + role="equipment_owner", + organization="Birck Nanotechnology Center", + ) + other_user = PlatformUser( + id="same-org-user", + email="same-org-user@purdue.edu", + name="Same Org User", + role="researcher", + organization="Birck Nanotechnology Center", + ) + + with patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "false", + "DT_DB3_ALLOWED_USER_IDS": "will3042,hosler0", + }, + clear=False, + ): + self.assertFalse(upload_router._db3_user_is_allowed(allowed_user)) + with self.assertRaises(HTTPException) as disabled: + upload_router._assert_db3_user_allowed(allowed_user) + self.assertEqual(disabled.exception.status_code, 503) + + with patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_ALLOWED_USER_IDS": "will3042,hosler0", + }, + clear=False, + ): + self.assertTrue(upload_router._db3_user_is_allowed(allowed_user)) + self.assertFalse(upload_router._db3_user_is_allowed(other_user)) + with self.assertRaises(HTTPException) as forbidden: + upload_router._assert_db3_user_allowed(other_user) + self.assertEqual(forbidden.exception.status_code, 403) + + def test_db3_capability_exposes_only_mapped_equipment_to_pilot_users( + self, + ) -> None: + pilot_user = PlatformUser( + id="hosler0", + email="hosler0@purdue.edu", + name="Richard Hosler", + role="equipment_owner", + organization="Birck Nanotechnology Center", + ) + other_user = PlatformUser( + id="researcher", + email="researcher@purdue.edu", + name="Researcher", + role="researcher", + organization="Birck Nanotechnology Center", + ) + equipment_id = "plasmatherm_versaline_icp_rie_etcher" + with patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_ALLOWED_USER_IDS": "will3042,hosler0", + "DT_DB3_EQUIPMENT_TOOL_MAP": json.dumps( + {equipment_id: ["VLN-11304-CTC-PM1"]} + ), + }, + clear=False, + ): + pilot_capabilities = upload_router.get_upload_capabilities(pilot_user) + other_capabilities = upload_router.get_upload_capabilities(other_user) + + self.assertTrue(pilot_capabilities["db3_upload"]["enabled"]) + self.assertEqual(pilot_capabilities["db3_upload"]["max_mb"], 64) + self.assertEqual( + pilot_capabilities["db3_upload"]["equipment_ids"], + [equipment_id], + ) + self.assertFalse(other_capabilities["db3_upload"]["enabled"]) + self.assertEqual(other_capabilities["db3_upload"]["equipment_ids"], []) + + def test_db3_processing_requires_the_configured_private_project(self) -> None: + project_id = "glance_db3_pilot_deadbeef" + with patch.dict( + os.environ, + {"DT_DB3_PILOT_PROJECT_ID": project_id}, + clear=False, + ): + upload_router._assert_db3_pilot_project( + {"id": project_id, "access": "private"} + ) + + with self.assertRaises(HTTPException) as wrong_project: + upload_router._assert_db3_pilot_project( + {"id": "shared-demo", "access": "private"} + ) + self.assertEqual(wrong_project.exception.status_code, 400) + + with self.assertRaises(HTTPException) as open_project: + upload_router._assert_db3_pilot_project( + {"id": project_id, "access": "open"} + ) + self.assertEqual(open_project.exception.status_code, 503) + + def test_db3_processing_failure_does_not_expose_the_server_path(self) -> None: + updates: list[dict] = [] + equipment_id = "test-glance-equipment" + with ( + patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_EQUIPMENT_TOOL_MAP": json.dumps( + {equipment_id: ["VLN-11304-CTC-PM1"]} + ), + }, + clear=False, + ), + patch.object( + upload_router, + "extract_db3_trace", + side_effect=FileNotFoundError( + "/var/lib/dt-api/uploads/private/source.db3" + ), + ), + patch.object( + upload_router, + "_update_upload_record", + side_effect=lambda **kwargs: updates.append(kwargs), + ), + ): + upload_router._execute_upload_processing( + upload_id="00000000-0000-0000-0000-000000000099", + upload_record={ + "equipment_id": equipment_id, + "filename": "source.db3", + "storage_path": "/var/lib/dt-api/uploads/private/source.db3", + "kind": "all", + }, + project_id="glance_db3_pilot_deadbeef", + prior_warnings=[], + ) + + self.assertEqual(updates[-1]["status"], "failed") + rendered_errors = " ".join(updates[-1]["errors"]) + self.assertNotIn("/var/lib", rendered_errors) + self.assertIn("contact the pilot administrator", rendered_errors) + + def test_generic_catalog_select_renders_glance_json_paths(self) -> None: + query = _equipment_runs_base_select(include_raw_payload=True) + self.assertIn("'{glance,run_id}'", query) + self.assertIn("'{glance,run_start_time}'", query) + + def test_extracts_one_complete_trace_per_run_without_aggregation(self) -> None: + with TemporaryDirectory() as temporary_directory: + db3_path = Path(temporary_directory) / "fixture with spaces.db3" + _write_fixture(db3_path) + expected_sha256 = hashlib.sha256(db3_path.read_bytes()).hexdigest() + + result = extract_db3_trace(db3_path) + + self.assertIsInstance(result, Db3TraceImportResult) + self.assertEqual(result.source_filename, db3_path.name) + self.assertEqual(result.source_sha256, expected_sha256) + self.assertEqual( + hashlib.sha256(db3_path.read_bytes()).hexdigest(), + expected_sha256, + ) + self.assertEqual(result.source_size_bytes, db3_path.stat().st_size) + self.assertEqual(result.run_count, 3) + self.assertEqual(result.sample_count, 5) + self.assertEqual(result.failures, ()) + self.assertEqual(len(result.source_tools), 1) + self.assertEqual(result.source_tools[0].glance_tool_id, 5) + self.assertEqual(result.source_tools[0].name, "VLN-11304-CTC-PM1") + + first, second, empty_trace = result.runs + self.assertEqual( + (first.glance_run_id, second.glance_run_id, empty_trace.glance_run_id), + (101, 102, 104), + ) + # Lot names are display metadata and must not merge physical runs. + self.assertEqual(first.lot_name, "LOT-REUSED") + self.assertEqual(second.lot_name, "LOT-REUSED") + self.assertEqual(first.glance_tool_id, 5) + self.assertEqual(first.tool_name, "VLN-11304-CTC-PM1") + self.assertEqual(first.glance_recipe_id, 9) + self.assertEqual(first.recipe_name, "O2 clean-Engineering") + self.assertEqual(first.material_name, "Silicon") + self.assertEqual(first.run_status, "processed") + self.assertEqual(first.run_start_time, "2026-07-01 10:00:00.000001") + self.assertEqual(first.run_end_time, "2026-07-01 10:00:10.999999") + + self.assertEqual( + [sample.sample_time for sample in first.samples], + [ + "2026-07-01 10:00:00.125000", + "2026-07-01 10:00:02.875000", + "2026-07-01 10:00:09.000001", + ], + ) + self.assertEqual( + [sample.glance_sample_record_id for sample in first.samples], + [1002, 1001, 1003], + ) + self.assertEqual(first.samples[0].values_json, {"p2": 44.5}) + self.assertNotIn("p1", first.samples[0].values_json) + self.assertEqual(first.samples[1].values_json, {"p1": 1.25, "p2": None}) + self.assertEqual(first.samples[2].values_json, {}) + self.assertEqual(empty_trace.sample_count, 1) + self.assertEqual(empty_trace.samples[0].values_json, {}) + self.assertEqual(empty_trace.parameter_metadata, ()) + + metadata = {item.key: item for item in first.parameter_metadata} + self.assertEqual(metadata["p1"].raw_name, "G Gas 1 Flow") + self.assertEqual(metadata["p1"].unit, "sccm") + self.assertEqual(metadata["p2"].raw_name, "P Chamber Pressure") + self.assertEqual(metadata["p2"].unit, "mTorr") + self.assertNotIn("p7", metadata) + self.assertEqual( + [parameter.key for parameter in result.parameters], + ["p1", "p2", "p7"], + ) + + serialized = result.to_dict() + json.dumps(serialized, allow_nan=False) + self.assertNotIn("relative_sample_time", json.dumps(serialized)) + self.assertNotIn("run_date", serialized["runs"][0]["samples"][0]) + sync_row = first.to_sync_dict(project_id="project-test") + self.assertEqual(sync_row["project_id"], "project-test") + self.assertEqual(sync_row["lotname"], "LOT-REUSED") + self.assertEqual(sync_row["run_date"], first.run_start_time) + self.assertEqual(sync_row["source"], "glance_db3") + self.assertEqual(sync_row["source_run_id"], 101) + self.assertEqual(sync_row["parameters"][0]["key"], "p1") + self.assertEqual(sync_row["parameters"][0]["source_parameter_id"], 1) + self.assertEqual(sync_row["parameters"][0]["name"], "G Gas 1 Flow") + self.assertEqual(sync_row["samples"][0]["source_sample_id"], 1002) + self.assertEqual(sync_row["samples"][0]["source_tool_id"], 5) + self.assertEqual(sync_row["samples"][0]["sample_index"], 0) + self.assertEqual(sync_row["samples"][2]["values_json"], {}) + + def test_a_malformed_run_does_not_discard_valid_run_traces(self) -> None: + with TemporaryDirectory() as temporary_directory: + db3_path = Path(temporary_directory) / "partial.db3" + _write_fixture(db3_path, include_bad_run=True) + + result = extract_db3_trace(db3_path) + + self.assertEqual( + [run.glance_run_id for run in result.runs], + [101, 102, 104], + ) + self.assertEqual(len(result.failures), 1) + self.assertEqual(result.failures[0].glance_run_id, "300.5") + self.assertIn("runs.idruns must be an integer", result.failures[0].error) + + def test_background_processing_syncs_physical_runs_and_keeps_partial_success( + self, + ) -> None: + with TemporaryDirectory() as temporary_directory: + db3_path = Path(temporary_directory) / "partial.db3" + _write_fixture( + db3_path, + include_bad_run=True, + include_bad_timestamp_run=True, + ) + sync_calls: list[dict] = [] + upload_updates: list[dict] = [] + + def capture_sync(**kwargs): + sync_calls.append(kwargs) + return len(kwargs["runs"]) + + equipment_id = "test-glance-equipment" + tool_map = json.dumps({equipment_id: ["VLN-11304-CTC-PM1"]}) + with ( + patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_EQUIPMENT_TOOL_MAP": tool_map, + }, + clear=False, + ), + patch.object( + upload_router, + "sync_equipment_run_traces_pg", + side_effect=capture_sync, + ), + patch.object( + upload_router, + "_update_upload_record", + side_effect=lambda **kwargs: upload_updates.append(kwargs), + ), + ): + upload_router._execute_upload_processing( + upload_id="00000000-0000-0000-0000-000000000001", + upload_record={ + "equipment_id": equipment_id, + "filename": db3_path.name, + "storage_path": str(db3_path), + "kind": "all", + }, + project_id="project-test", + prior_warnings=[], + ) + + self.assertEqual(len(sync_calls), 1) + synced_runs = sync_calls[0]["runs"] + self.assertEqual( + [run["source_run_id"] for run in synced_runs], + [101, 102, 104], + ) + self.assertEqual( + [run["lotname"] for run in synced_runs[:2]], + ["LOT-REUSED", "LOT-REUSED"], + ) + self.assertEqual(synced_runs[0]["parameters"][0]["name"], "G Gas 1 Flow") + self.assertEqual(synced_runs[-1]["samples"][0]["values"], {}) + self.assertEqual(synced_runs[-1]["samples"][0]["source_tool_id"], 5) + self.assertEqual( + synced_runs[0]["raw"]["source_file"]["sha256"], + hashlib.sha256(db3_path.read_bytes()).hexdigest(), + ) + self.assertEqual(upload_updates[-1]["status"], "processed") + self.assertEqual(upload_updates[-1]["row_count"], 3) + self.assertTrue( + any("Skipped GLANCE run 300.5" in warning for warning in upload_updates[-1]["errors"]) + ) + self.assertTrue( + any( + "Skipped GLANCE run 105" in warning + and "timestamp" in warning + for warning in upload_updates[-1]["errors"] + ) + ) + + def test_db3_copy_is_chunked_and_removes_an_oversized_partial_file(self) -> None: + with TemporaryDirectory() as temporary_directory: + destination = Path(temporary_directory) / "stored.db3" + payload = b"db3-test-payload" + copied = upload_router._copy_upload_with_limit( + io.BytesIO(payload), + destination, + len(payload), + ) + self.assertEqual(copied, len(payload)) + self.assertEqual(destination.read_bytes(), payload) + + with self.assertRaisesRegex(ValueError, "too_large"): + upload_router._copy_upload_with_limit( + io.BytesIO(payload), + destination, + len(payload) - 1, + ) + self.assertFalse(destination.exists()) + + def test_trace_sync_uses_source_run_identity_and_normalizes_offset_math(self) -> None: + class FakeCursor: + def __init__(self): + self.executions: list[tuple[str, object]] = [] + + def execute(self, query, params=None): + self.executions.append((query, params)) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class FakeConnection: + def __init__(self): + self.cursor_instance = FakeCursor() + self.committed = False + self.rolled_back = False + self.closed = False + + def cursor(self, cursor_factory=None): + return self.cursor_instance + + def commit(self): + self.committed = True + + def rollback(self): + self.rolled_back = True + + def close(self): + self.closed = True + + connection = FakeConnection() + value_batches: list[tuple[str, list[tuple]]] = [] + + def capture_execute_values(cursor, query, values, **kwargs): + rows = list(values) + value_batches.append((query, rows)) + if "INSERT INTO equipment_runs" in query: + return [ + (7_000 + index, row[11]) + for index, row in enumerate(rows) + ] + return None + + runs = [ + { + "source_run_id": source_run_id, + "lotname": "LOT-REUSED", + "run_start_time": f"2026-07-01 10:00:{source_run_id}.000001", + "run_date": f"2026-07-01 10:00:{source_run_id}.000001", + "inputs": {}, + "outputs": {}, + "raw": {"glance": {"run_start_time": "source-exact"}}, + "parameters": [ + { + "key": "p1", + "source_parameter_id": 1, + "name": "Gas flow", + "unit": "sccm", + } + ], + "samples": [ + { + "source_sample_id": source_run_id * 100, + "source_tool_id": 5, + "sample_time_raw": ( + f"2026-07-01 10:00:{source_run_id}.500001" + ), + "values": {}, + } + ], + } + for source_run_id in (22, 11) + ] + + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=connection, + ), + patch( + "psycopg2.extras.execute_values", + side_effect=capture_execute_values, + ), + ): + inserted = sync_equipment_run_traces_pg( + equipment_id="equipment-test", + project_id="project-test", + upload_id="00000000-0000-0000-0000-000000000002", + upload_filename="source.db3", + runs=runs, + ) + + self.assertEqual(inserted, 2) + parent_rows = value_batches[0][1] + self.assertEqual([row[11] for row in parent_rows], [11, 22]) + trace_rows = value_batches[1][1] + self.assertEqual([row[0] for row in trace_rows], [7_000, 7_001]) + self.assertEqual([row[2] for row in trace_rows], [1_100, 2_200]) + self.assertEqual([row[3] for row in trace_rows], [5, 5]) + self.assertEqual(trace_rows[0][6].adapted, {}) + self.assertTrue(connection.committed) + self.assertFalse(connection.rolled_back) + self.assertTrue(connection.closed) + + self.assertEqual(_normalize_trace_parameter_key("P001"), "p1") + parsed, raw = _parse_trace_sample_time( + "2026-07-01T10:00:00.123456-05:00", + source_sample_id=1, + ) + self.assertEqual(raw, "2026-07-01T10:00:00.123456-05:00") + self.assertEqual(parsed.isoformat(), "2026-07-01T15:00:00.123456") + + def test_trace_read_keeps_empty_samples_as_parameter_gaps(self) -> None: + parent = { + "id": 7, + "lotname": "LOT-TRACE", + "source": "glance_db3", + "raw_payload_json": { + "glance": {"run_id": 101}, + "trace": { + "timestamp_timezone": None, + "parameters": [ + { + "key": "p1", + "name": "Gas flow", + "unit": "sccm", + "source_parameter_id": 1, + } + ], + }, + }, + } + sample_rows = [ + { + "sample_index": 0, + "source_sample_id": 1001, + "source_tool_id": 5, + "sample_time": datetime(2026, 7, 1, 10, 0, 0, 125000), + "sample_time_raw": "2026-07-01 10:00:00.125000", + "values_json": {"p1": 1.25}, + "relative_seconds": 0.0, + }, + { + "sample_index": 1, + "source_sample_id": 1002, + "source_tool_id": 5, + "sample_time": datetime(2026, 7, 1, 10, 0, 2, 875000), + "sample_time_raw": "2026-07-01 10:00:02.875000", + "values_json": {}, + "relative_seconds": 2.75, + }, + ] + + class ReadCursor: + def __init__(self): + self.mode = "" + + def execute(self, query, params=None): + self.mode = "samples" if "trace_samples" in query else "parent" + + def fetchone(self): + return parent if self.mode == "parent" else None + + def fetchall(self): + return sample_rows if self.mode == "samples" else [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class ReadConnection: + def __init__(self): + self.closed = False + + def cursor(self, cursor_factory=None): + return ReadCursor() + + def commit(self): + pass + + def close(self): + self.closed = True + + connection = ReadConnection() + with patch("data_loader_pg.get_pg_connection", return_value=connection): + result = get_equipment_run_trace_pg( + EQUIPMENT_RUN_ID_OFFSET + 7, + parameters=["p1"], + nanohub_user_id="researcher-test", + ) + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(result["source_run_id"], 101) + self.assertEqual(result["sample_count"], 2) + self.assertEqual(result["samples"][0]["source_tool_id"], 5) + self.assertEqual(result["samples"][0]["values"], {"p1": 1.25}) + self.assertEqual(result["samples"][1]["values"], {"p1": None}) + self.assertEqual(result["samples"][1]["relative_seconds"], 2.75) + self.assertEqual( + result["samples"][1]["timestamp"], + "2026-07-01 10:00:02.875000", + ) + self.assertIsNone(result["timestamp_timezone"]) + self.assertTrue(connection.closed) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_phase3_security.py b/api/tests/test_phase3_security.py index 1de8b3e..6b8c855 100644 --- a/api/tests/test_phase3_security.py +++ b/api/tests/test_phase3_security.py @@ -766,6 +766,114 @@ def test_upload_records_equipment_association(self): ) self.assertEqual(create_upload_record_pg.call_args.kwargs["user"].id, "owner-1") + def test_upload_listing_never_exposes_server_storage_paths(self): + now = datetime.now(timezone.utc) + connection = FakeConnection( + [ + { + "contains": "FROM data_uploads", + "fetchall": [ + { + "id": "00000000-0000-0000-0000-000000000001", + "equipment_id": "etcher_01", + "filename": "runs.csv", + "stored_as": "stored-runs.csv", + "storage_path": "/private/server/path/stored-runs.csv", + "size_bytes": 12, + "row_count": 1, + "columns_json": ["LOTNAME"], + "errors_json": [], + "status": "success", + "kind": "all", + "owner_id": "owner-1", + "owner_org": "Birck", + "uploaded_at": now, + } + ], + } + ] + ) + user = metadata_pg.PlatformUser( + id="owner-1", + email="owner@example.com", + name="Equipment Owner", + role="equipment_owner", + organization="Birck", + ) + + with patch.object(metadata_pg, "get_pg_connection", return_value=connection): + records = metadata_pg.list_uploads_pg(user) + + self.assertEqual(len(records), 1) + self.assertNotIn("storage_path", records[0]) + self.assertTrue(connection.closed) + + def test_db3_upload_is_rejected_when_temporary_pilot_is_disabled(self): + with patch.dict( + os.environ, + {"DT_ENABLE_DB3_UPLOAD": "false"}, + clear=False, + ), patch.object( + upload, + "get_equipment_pg", + return_value={ + "domain_id": "etcher_01", + "owner_id": "owner-1", + "owner_org": "Birck", + "status": "approved", + "columns": [], + "config_json": {}, + }, + ), patch.object(upload, "create_upload_record_pg") as create_record: + response = self.client.post( + "/api/data/upload", + headers=OWNER_HEADERS, + data={"equipment_id": "etcher_01"}, + files={"file": ("source.db3", b"sqlite", "application/vnd.sqlite3")}, + ) + + self.assertEqual(response.status_code, 503) + self.assertEqual( + response.json()["detail"], + "The temporary DB3 upload pilot is disabled.", + ) + self.assertEqual(create_record.call_count, 0) + + def test_db3_upload_rejects_a_same_org_user_outside_the_allowlist(self): + with patch.dict( + os.environ, + { + "DT_ENABLE_DB3_UPLOAD": "true", + "DT_DB3_ALLOWED_USER_IDS": "will3042,hosler0", + "DT_DB3_EQUIPMENT_TOOL_MAP": '{"etcher_01":["tool-1"]}', + }, + clear=False, + ), patch.object( + upload, + "get_equipment_pg", + return_value={ + "domain_id": "etcher_01", + "owner_id": "owner-1", + "owner_org": "Birck", + "status": "approved", + "columns": [], + "config_json": {}, + }, + ), patch.object(upload, "create_upload_record_pg") as create_record: + response = self.client.post( + "/api/data/upload", + headers=OWNER_HEADERS, + data={"equipment_id": "etcher_01"}, + files={"file": ("source.db3", b"sqlite", "application/vnd.sqlite3")}, + ) + + self.assertEqual(response.status_code, 403) + self.assertEqual( + response.json()["detail"], + "DB3 uploads are limited to the temporary pilot participants.", + ) + self.assertEqual(create_record.call_count, 0) + def test_upload_reports_missing_required_columns(self): with patch.object( upload, diff --git a/api/tests/test_retrain.py b/api/tests/test_retrain.py new file mode 100644 index 0000000..488904e --- /dev/null +++ b/api/tests/test_retrain.py @@ -0,0 +1,62 @@ +import os +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +import numpy as np + + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +from training import retrain + + +class _DummyEstimator: + def fit(self, X, y): + return self + + def predict(self, X): + return np.zeros(len(X), dtype=float) + + +class RetrainCostControlTests(unittest.TestCase): + def test_training_skips_expensive_loocv_above_limit(self): + X = np.arange(102, dtype=float).reshape(51, 2) + y = np.linspace(0.0, 1.0, 51) + + with patch.object( + retrain, "_make_estimator", side_effect=lambda _use_gpr: _DummyEstimator(), + ) as make_estimator: + result = retrain._train_one_target( + X_raw=X, + y=y, + features=["feature_1", "feature_2"], + target="etch_rate", + gpr_row_limit=400, + force_rf=False, + loocv_max_rows=50, + ) + + # One holdout fit and one final fit. Without the cap this would make + # 51 additional Gaussian-process fits for LOOCV. + self.assertEqual(make_estimator.call_count, 2) + self.assertNotIn("r2_loocv", result["metrics"]) + self.assertNotIn("rmse_loocv", result["metrics"]) + + def test_invalid_environment_limit_uses_safe_default(self): + with patch.dict(os.environ, {"RETRAIN_LOOCV_MAX_ROWS": "invalid"}): + self.assertEqual( + retrain._configured_loocv_max_rows(), + retrain._DEFAULT_LOOCV_MAX_ROWS, + ) + + def test_negative_environment_limit_disables_loocv(self): + with patch.dict(os.environ, {"RETRAIN_LOOCV_MAX_ROWS": "-1"}): + self.assertEqual(retrain._configured_loocv_max_rows(), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/training/retrain.py b/api/training/retrain.py index 22edb2e..7b6e537 100644 --- a/api/training/retrain.py +++ b/api/training/retrain.py @@ -46,6 +46,7 @@ # Target-independent retrain advisory lock key. Any 64-bit int works. _ADVISORY_LOCK_KEY = 784512341 +_DEFAULT_LOOCV_MAX_ROWS = 50 # ── Helpers ────────────────────────────────────────────────────────────────── @@ -58,6 +59,20 @@ def _setup_logging() -> None: ) +def _configured_loocv_max_rows() -> int: + raw_limit = os.getenv( + "RETRAIN_LOOCV_MAX_ROWS", str(_DEFAULT_LOOCV_MAX_ROWS), + ) + try: + return max(0, int(raw_limit)) + except ValueError: + logger.warning( + "Invalid RETRAIN_LOOCV_MAX_ROWS=%r; using %d", + raw_limit, _DEFAULT_LOOCV_MAX_ROWS, + ) + return _DEFAULT_LOOCV_MAX_ROWS + + def _build_gpr_kernel(): return ( ConstantKernel(1.0, constant_value_bounds=(1e-3, 1e3)) @@ -83,6 +98,10 @@ def _loocv_scores( if len(y) == 0: return {} if len(y) > max_n: + logger.info( + "Retrain: skipping LOOCV for %d rows (configured limit=%d)", + len(y), max_n, + ) return {} loo = LeaveOneOut() preds = np.zeros_like(y, dtype=float) @@ -153,6 +172,7 @@ def _train_one_target( target: str, gpr_row_limit: int, force_rf: bool, + loocv_max_rows: int, ) -> dict[str, Any]: """Fit one model for one target, return {bundle, metrics, model_kind}.""" n = len(y) @@ -187,11 +207,13 @@ def _train_one_target( # LOOCV on the full set when affordable. if use_gpr: loo_scores = _loocv_scores( - lambda: _make_estimator(True), X_scaled, y, max_n=200, + lambda: _make_estimator(True), X_scaled, y, + max_n=loocv_max_rows, ) else: loo_scores = _loocv_scores( - lambda: _make_estimator(False), X_raw, y, max_n=400, + lambda: _make_estimator(False), X_raw, y, + max_n=loocv_max_rows, ) metrics.update(loo_scores) @@ -300,12 +322,18 @@ def run_retrain( domain_id: str = "etcher", gpr_row_limit: int = 400, force_rf: bool = False, + loocv_max_rows: Optional[int] = None, ) -> dict[str, Any]: """Train a fresh model per target, save, promote, update state.""" from model_registry import ( get_retrain_state, promote_model, save_model, update_retrain_state, ) + if loocv_max_rows is None: + loocv_max_rows = _configured_loocv_max_rows() + else: + loocv_max_rows = max(0, loocv_max_rows) + snapshot_id, snapshot_hash, num_rows = _current_snapshot() state = get_retrain_state(domain_id) or {} if ( @@ -353,6 +381,7 @@ def run_retrain( trained = _train_one_target( X_raw=X_raw, y=y, features=valid_features, target=target, gpr_row_limit=gpr_row_limit, force_rf=force_rf, + loocv_max_rows=loocv_max_rows, ) metrics = trained["metrics"] @@ -439,6 +468,11 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: help="Domain id (default: etcher).") p.add_argument("--rf-only", action="store_true", help="Skip GPR and always train Random Forest.") + p.add_argument( + "--loocv-max-rows", type=int, default=None, + help="Run full leave-one-out validation only at or below this row " + "count (default: RETRAIN_LOOCV_MAX_ROWS or 50).", + ) return p.parse_args(argv) @@ -450,6 +484,7 @@ def main(argv: Optional[list[str]] = None) -> int: force=bool(ns.force), domain_id=ns.domain, force_rf=bool(ns.rf_only), + loocv_max_rows=ns.loocv_max_rows, ) # Summarise to stdout for CronJob logs. status = out.get("status", "error") diff --git a/geddes/README.md b/geddes/README.md index 98975b8..fc264a9 100644 --- a/geddes/README.md +++ b/geddes/README.md @@ -24,10 +24,10 @@ it cleanly separates the database container and the API container. ### Deployment Steps: 1. **Create the Deployment:** - In Rancher: **Workloads → Deployments → Create** - - Name: `dt-postgrest` (Namespace: `nanohub`) + - Name: `dt-postgrest` (Namespace: `ncn-digitaltwins-zchen`) - Image: `geddes-registry.rcac.purdue.edu/docker-hub-cache/postgrest/postgrest:v13.0.6` 2. **Set Environment Variables:** - - `PGRST_DB_URI` = `postgres://glance:glance_secret@dt-postgres:5432/logger` + - `PGRST_DB_URI` = `postgres://glance:glance_secret@dt-postgres.nanohub.svc.cluster.local:5432/logger` - `PGRST_DB_SCHEMAS` = `public` - `PGRST_DB_ANON_ROLE` = `glance_public` - `PGRST_JWT_SECRET` = `test_secret_that_is_at_least_32_characters_long` diff --git a/geddes/k8s/01-secrets.yaml.example b/geddes/k8s/01-secrets.yaml.example index d5a3076..89f4b45 100644 --- a/geddes/k8s/01-secrets.yaml.example +++ b/geddes/k8s/01-secrets.yaml.example @@ -3,7 +3,7 @@ kind: Secret metadata: name: dt-web-secret # Change this if your actual target namespace is different (e.g. ncn-something) - namespace: nanohub + namespace: ncn-digitaltwins-zchen type: Opaque stringData: # The full public URL of the platform (no trailing slash) @@ -31,11 +31,11 @@ apiVersion: v1 kind: Secret metadata: name: dt-api-secret - namespace: nanohub + namespace: ncn-digitaltwins-zchen type: Opaque stringData: # ── PostgreSQL (V2 multi-tenant database) ── - PG_HOST: "dt-db-v2-srv.nanohub.svc.cluster.local" + PG_HOST: "dt-db-v2-srv.ncn-digitaltwins-zchen.svc.cluster.local" PG_PORT: "5432" PG_DB: "digital_twin" PG_USER: "api_client" diff --git a/geddes/k8s/02-api.yaml b/geddes/k8s/02-api.yaml index 422b7c6..d301e5d 100644 --- a/geddes/k8s/02-api.yaml +++ b/geddes/k8s/02-api.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: dt-api - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: replicas: 1 selector: @@ -18,8 +18,15 @@ spec: containers: - name: dt-api # Notice we are using the 'sdx' namespace on the registry now - image: geddes-registry.rcac.purdue.edu/sdx/dt-api:v52 + image: geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-retrain-loocv-20260714@sha256:52e16ffbbdf71ba87470d8dbf686e6304955501d5b74eaf4c41ba981e14239dc imagePullPolicy: Always + resources: + requests: + cpu: "100m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" ports: - containerPort: 8000 env: @@ -38,7 +45,7 @@ apiVersion: v1 kind: Service metadata: name: dt-api - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: type: ClusterIP ports: diff --git a/geddes/k8s/03-web.yaml b/geddes/k8s/03-web.yaml index d5f7383..7117b10 100644 --- a/geddes/k8s/03-web.yaml +++ b/geddes/k8s/03-web.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: dt-web - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: replicas: 1 selector: @@ -26,6 +26,13 @@ spec: # Notice we are using the 'sdx' namespace on the registry now image: geddes-registry.rcac.purdue.edu/sdx/dt-web:v53 imagePullPolicy: Always + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" ports: - containerPort: 3000 envFrom: @@ -37,7 +44,7 @@ apiVersion: v1 kind: Service metadata: name: dt-web - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: type: ClusterIP ports: diff --git a/geddes/k8s/04-ingress.yaml b/geddes/k8s/04-ingress.yaml index 3e3c38b..70bab7f 100644 --- a/geddes/k8s/04-ingress.yaml +++ b/geddes/k8s/04-ingress.yaml @@ -5,7 +5,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-nanohub-services - namespace: nanohub + namespace: ncn-digitaltwins-zchen annotations: nginx.ingress.kubernetes.io/proxy-buffering: "off" nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" @@ -78,7 +78,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-nanohub-postgrest - namespace: nanohub + namespace: ncn-digitaltwins-zchen annotations: nginx.ingress.kubernetes.io/rewrite-target: /$2 spec: diff --git a/geddes/k8s/05-postgres.yaml b/geddes/k8s/05-postgres.yaml index 1481a62..8a80970 100644 --- a/geddes/k8s/05-postgres.yaml +++ b/geddes/k8s/05-postgres.yaml @@ -2,10 +2,11 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: dt-db-v2-pvc - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: accessModes: - ReadWriteOnce + storageClassName: geddes-standard-singlenode resources: requests: storage: 20Gi @@ -14,7 +15,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: dt-postgres-init - namespace: nanohub + namespace: ncn-digitaltwins-zchen data: 00-bootstrap-api-client.sh: | #!/bin/sh @@ -118,6 +119,100 @@ data: ) ); + -- Generic catalog rows: one row represents one physical equipment run. + CREATE TABLE IF NOT EXISTS equipment_runs ( + id BIGSERIAL PRIMARY KEY, + equipment_id VARCHAR(255) NOT NULL, + project_id VARCHAR(50) REFERENCES projects(id) ON DELETE SET NULL, + lotname VARCHAR(255), + run_date TIMESTAMP WITH TIME ZONE, + is_outlier BOOLEAN DEFAULT false, + is_calibration_recipe BOOLEAN DEFAULT false, + outlier_type TEXT DEFAULT '', + inputs_json JSONB NOT NULL DEFAULT '{}'::jsonb, + outputs_json JSONB NOT NULL DEFAULT '{}'::jsonb, + raw_payload_json JSONB NOT NULL DEFAULT '{}'::jsonb, + upload_id UUID, + row_index INT, + upload_filename VARCHAR(255) DEFAULT '', + source VARCHAR(100) DEFAULT 'data_upload', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + ); + + CREATE INDEX IF NOT EXISTS idx_equipment_runs_equipment_id + ON equipment_runs(equipment_id); + CREATE INDEX IF NOT EXISTS idx_equipment_runs_project_id + ON equipment_runs(project_id); + CREATE INDEX IF NOT EXISTS idx_equipment_runs_upload_id + ON equipment_runs(upload_id); + CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row + ON equipment_runs(upload_id, row_index) + WHERE upload_id IS NOT NULL; + + ALTER TABLE equipment_runs ENABLE ROW LEVEL SECURITY; + GRANT SELECT ON equipment_runs TO api_client; + + CREATE POLICY equipment_run_visibility_policy ON equipment_runs + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM projects p + WHERE p.id = equipment_runs.project_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + + -- Complete GLANCE traces linked to those physical run rows. + CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_index INT NOT NULL, + source_sample_id BIGINT NOT NULL, + source_tool_id BIGINT, + sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + sample_time_raw TEXT NOT NULL, + values_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, sample_index), + UNIQUE (equipment_run_id, source_sample_id), + CHECK (jsonb_typeof(values_json) = 'object') + ); + + CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time + ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); + + ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; + GRANT SELECT ON equipment_run_trace_samples TO api_client; + + CREATE POLICY equipment_run_trace_visibility_policy + ON equipment_run_trace_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + CREATE TABLE IF NOT EXISTS users ( id VARCHAR(255) PRIMARY KEY, name VARCHAR(255) NOT NULL, @@ -282,6 +377,7 @@ data: kind VARCHAR(20) NOT NULL DEFAULT 'all', owner_id VARCHAR(255) REFERENCES users(id), owner_org VARCHAR(255) DEFAULT '', + processing_started_at TIMESTAMP WITH TIME ZONE, uploaded_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); @@ -301,11 +397,13 @@ apiVersion: apps/v1 kind: Deployment metadata: name: dt-db-v2 - namespace: nanohub + namespace: ncn-digitaltwins-zchen labels: app: dt-db-v2 spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: app: dt-db-v2 @@ -317,6 +415,13 @@ spec: containers: - name: postgres image: postgres:15 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1" + memory: "1Gi" ports: - containerPort: 5432 env: @@ -362,7 +467,7 @@ apiVersion: v1 kind: Service metadata: name: dt-db-v2-srv - namespace: nanohub + namespace: ncn-digitaltwins-zchen spec: selector: app: dt-db-v2 diff --git a/geddes/k8s/07-postgrest-public-ingress.example.yaml b/geddes/k8s/07-postgrest-public-ingress.example.yaml index ff6db31..4468e00 100644 --- a/geddes/k8s/07-postgrest-public-ingress.example.yaml +++ b/geddes/k8s/07-postgrest-public-ingress.example.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-postgrest-public - namespace: nanohub + namespace: ncn-digitaltwins-zchen annotations: # Restrict access to the Azure Function's outbound IPs. # Replace with a comma-separated allowlist from: diff --git a/geddes/k8s/07-postgrest-public-ingress.yaml b/geddes/k8s/07-postgrest-public-ingress.yaml index 73989c2..466d9bd 100644 --- a/geddes/k8s/07-postgrest-public-ingress.yaml +++ b/geddes/k8s/07-postgrest-public-ingress.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dt-postgrest-public - namespace: nanohub + namespace: ncn-digitaltwins-zchen annotations: # Removed whitelist to allow dynamic Azure outbound IPs # nginx.ingress.kubernetes.io/whitelist-source-range: "20.127.117.91/32" diff --git a/geddes/k8s/08-retrain-cron.yaml b/geddes/k8s/08-retrain-cron.yaml index 301ee5a..a735cfa 100644 --- a/geddes/k8s/08-retrain-cron.yaml +++ b/geddes/k8s/08-retrain-cron.yaml @@ -22,12 +22,13 @@ apiVersion: batch/v1 kind: CronJob metadata: name: dt-api-retrain - namespace: nanohub + namespace: ncn-digitaltwins-zchen labels: app: dt-api component: retrain spec: schedule: "0 2 * * *" # 02:00 UTC every day + suspend: false timeZone: "Etc/UTC" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 @@ -48,7 +49,7 @@ spec: - name: sdx-registry-secret containers: - name: retrain - image: geddes-registry.rcac.purdue.edu/sdx/dt-api:v50 + image: geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-retrain-loocv-20260714@sha256:52e16ffbbdf71ba87470d8dbf686e6304955501d5b74eaf4c41ba981e14239dc imagePullPolicy: Always command: - python @@ -64,6 +65,10 @@ spec: value: INFO - name: AUTO_RETRAIN_DOMAIN value: etcher + # Full LOOCV scales poorly for Gaussian-process models. Holdout + # validation and the final model fit still run above this limit. + - name: RETRAIN_LOOCV_MAX_ROWS + value: "50" resources: requests: cpu: "500m" diff --git a/geddes/k8s/pilots/db3/LIVE_TEST_2026-07-13.md b/geddes/k8s/pilots/db3/LIVE_TEST_2026-07-13.md new file mode 100644 index 0000000..813e14f --- /dev/null +++ b/geddes/k8s/pilots/db3/LIVE_TEST_2026-07-13.md @@ -0,0 +1,105 @@ +# Geddes DB3 pilot live validation — 2026-07-13 + +This is an operational record for the temporary, branch-only Paul/Rich pilot. +It is not a production-platform specification and must not be merged into the +final platform. The retained file and rows below are intentionally still live +for Paul and Rich; do not delete them until their testing is complete. + +## Deployed pilot + +- Branch: `feat/db3-upload-pipeline` +- Namespace: `nanohub` +- API: `geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-5f23169@sha256:71113340698a18721c91ec31a042f3b4834ea01ff9a6b6038f7ab868383a6acc` +- Web: `geddes-registry.rcac.purdue.edu/sdx/dt-web:db3-pilot-be5289b@sha256:e2917e858ab0da5ba399676358527769f30db99e2d8e13480a7dc4129860e01d` +- Private project: `glance_db3_paul_rich_pilot_89f3f0b8` +- Equipment: `plasmatherm_versaline_icp_rie_etcher` +- Allowed source tool: `VLN-11304-CTC-PM1` +- Members/pilot allowlist: `ngholiza`, `will3042`, `hosler0` +- Storage: encrypted 5 GiB RWX PVC `dt-db3-pilot-uploads` +- Upload ingress exception: exact path `/api/dt/data/upload`, 70 MiB; the + application limit remains 64 MiB. + +Both deployments were 1/1 Ready with zero restarts after import. The public +health endpoint returned HTTP 200. Server-side dry runs of both the exact live +deploy and the pinned rollback passed without changing resources. + +## Retained validation upload + +- Upload UUID: `014864df-70a2-4137-a245-80210a54494b` +- Source filename: `MyDatabaseSample.db3` +- Stored filename: `20260714_003147_169913_MyDatabaseSample.db3` +- Size: 40,203,264 bytes +- SHA-256: `8c9472ce37d63c5bbcae129476f0b7f164db191ad286ec36054dcc6402a4b706` +- Status: `processed`, row count 26, zero errors + +The real file traversed the public authenticated Auth.js/Next.js/Ingress/API +path. Its source and PVC copies have the same byte count and SHA-256. The upload +API response does not expose its server-side storage path. + +## Imported data + +Direct PostgreSQL aggregates for the upload UUID: + +- 26 physical `equipment_runs`, all attached to the private pilot project +- 11,697 linked `equipment_run_trace_samples` +- 795,328 retained parameter values +- one retained empty-value sample (`{}`) +- one source tool, `VLN-11304-CTC-PM1` (`source_tool_id=3`) +- 68 distinct parameter keys/source IDs used by the retained traces + +The source DB3 registry contains 242 definitions; the 68 definitions actually +used by these traces are persisted with each run. No unused parameter registry +rows are presented as trace columns. + +Catalog and trace API verification used run `5000000022` (source run `11204`): +the detail and full trace returned HTTP 200 with 35 samples and 68 parameters. +A one-parameter trace returned all 35 timestamps with one value slot per sample, +and relative time covered 0.0 through 36.0 seconds. + +## Authorization and retry checks + +- Paul (`will3042`): private project visible; 26 runs and all 11,697 samples + visible; DB3 capability enabled; retained DB3 upload visible. +- Rich (`hosler0`): the same project, run, trace, capability, and upload access. +- Active nonmember (`denphi`): project absent; zero pilot runs/samples; DB3 + capability disabled; DB3 upload absent; direct detail and trace requests for + run `5000000022` returned HTTP 404. + +RLS is enabled on both parent and trace tables with project-membership/open +project SELECT policies. The parent upload/run uniqueness constraint and trace +source-sample uniqueness constraint, JSON-object check, and trace-time index are +present. + +Two simultaneous retries of the processed upload returned one HTTP 200 and one +HTTP 409. The accepted retry completed with the same 26 runs, 11,697 samples, +and 795,328 values, demonstrating the live claim guard and idempotent trace +replacement. + +## Code/build checks + +- Focused DB3 Python suite: 11 passed. +- DB3-named phase-3 security checks: 2 passed. +- TypeScript type check: passed. +- Next.js production build, locally and in the image build: passed. +- Deployment and rollback scripts: `bash -n` and Geddes server-side dry runs + passed. + +The broader phase-3 security file currently reports 21 passed and five failures +in existing experiment/project-creation tests outside the DB3 paths. One reaches +an unmocked local PostgreSQL connection; the others assert older validation or +module-patching behavior. The changed DB3 tests pass. ESLint likewise retains +seven errors and three warnings in unrelated pre-existing pages; changed files +do not add lint findings. + +## Rollback and cleanup identity + +The ordinary rollback restores these exact pre-pilot images while retaining the +PVC: + +- API v52: `sha256:b99e0ed5fca6ec908c41dd96b9351f400677f2a494be3bbe964284096f3443c6` +- Web v53: `sha256:8bfda58813952bbfca463dd6c901de9a73f4ef0b0f2f2d74d72fc626b164bd75` + +Use upload UUID `014864df-70a2-4137-a245-80210a54494b` for reviewed database +cleanup. The raw PVC and PostgreSQL rows are independent; removing one does not +remove the other. Follow the guarded steps in `README.md` only after Paul and +Rich confirm the retained pilot data is no longer needed. diff --git a/geddes/k8s/pilots/db3/README.md b/geddes/k8s/pilots/db3/README.md new file mode 100644 index 0000000..f993212 --- /dev/null +++ b/geddes/k8s/pilots/db3/README.md @@ -0,0 +1,148 @@ +# Temporary Geddes DB3 pilot + +This package exists only for the branch-only Paul/Rich DB3 pilot on Geddes. It +must not be merged into or treated as the final production platform. The final +platform does not include DB3 upload support. Pilot images must use a distinct +`db3-pilot-*` tag and an immutable digest; never retag `v52`, `v53`, or a normal +release tag with pilot code. + +The package deliberately isolates the temporary operational changes: + +- DB3 is enabled only by `dt-db3-pilot-config` and limited to nanoHUB users + `ngholiza`, `will3042` (Paul), and `hosler0` (Rich). +- Only GLANCE tool `VLN-11304-CTC-PM1` may map to approved equipment + `plasmatherm_versaline_icp_rie_etcher`. +- The app accepts at most 64 MiB. A separate exact-path ingress permits 70 MiB + for multipart overhead only at `/api/dt/data/upload`. +- Raw files use a 5 GiB encrypted, `ReadWriteMany` pilot PVC whose storage class + deletes the backing volume when the claim is deleted. +- A DB3 processing lease is refreshed every 60 seconds while work is active. It + becomes retryable after 30 minutes without a heartbeat if an API restart + interrupts the in-process task. Trace writes are idempotent. +- Existing `dt-api-secret` and `dt-web-secret` references are preserved. No + secret value is stored in this directory. + +Use a dedicated private pilot project containing only Paul, Rich, and the pilot +administrator. The allowlist is an additional boundary, not a replacement for +normal project membership and equipment authorization. + +The live pilot is pinned to private project +`glance_db3_paul_rich_pilot_89f3f0b8` (`GLANCE DB3 Paul-Rich Pilot`), tied to +Paul's approved Versaline ICP RIE registration. Its members are `ngholiza`, +`will3042`, and `hosler0`. + +The retained real-file validation and cleanup identifiers are recorded in +[`LIVE_TEST_2026-07-13.md`](LIVE_TEST_2026-07-13.md). + +## Preflight and deploy + +Build and push API/web images from this branch with a unique tag, then resolve +each pushed tag to its registry digest. Pass the full immutable references; the +script rejects tag-only images. For example: + +```sh +export KUBECONFIG=/tmp/geddes-kubeconfig.yaml +export KUBE_CONTEXT=geddes +export NAMESPACE=ncn-digitaltwins-zchen + +API_IMAGE='geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-@sha256:' +WEB_IMAGE='geddes-registry.rcac.purdue.edu/sdx/dt-web:db3-pilot-@sha256:' + +./deploy.sh --api-image "$API_IMAGE" --web-image "$WEB_IMAGE" --dry-run +./deploy.sh --api-image "$API_IMAGE" --web-image "$WEB_IMAGE" +``` + +`--dry-run` performs server-side validation of the ConfigMap, PVC, exact-path +Ingress, and both deployment patches without changing the cluster. Deployment +applies storage/config first, rolls out and validates the API before the web +image, then exposes the large-body ingress only after both are ready. +The script rejects non-pilot tags, unexpected repositories, and deployments +that have moved beyond the recorded v52/v53 baseline. A ConfigMap checksum in +the API pod template guarantees configuration changes trigger a rollout. + +The package is intentionally fixed to the manifest namespace `ncn-digitaltwins-zchen`; the +scripts reject any other `NAMESPACE` value instead of silently deploying a +cross-namespace mix of resources. `KUBE_CONTEXT` is explicit and configurable. + +After deployment, verify the normal site and upload a real DB3 through the +browser-facing `/api/dt/data/upload` path. Confirm the created runs and trace +samples are visible only inside the private pilot project. Record the upload +UUIDs and project ID; they are the safest cleanup keys. + +## Rollback + +The rollback is pinned to the exact pre-pilot images observed on Geddes: + +- API v52: `sha256:b99e0ed5fca6ec908c41dd96b9351f400677f2a494be3bbe964284096f3443c6` +- Web v53: `sha256:8bfda58813952bbfca463dd6c901de9a73f4ef0b0f2f2d74d72fc626b164bd75` + +Validate, then roll back the route and workloads: + +```sh +./rollback.sh --dry-run +./rollback.sh +``` + +The default rollback immediately removes the large-body ingress, restores the +two pre-pilot image digests, removes the ConfigMap reference and PVC mount from +`dt-api`, waits for the rollout, and deletes the pilot ConfigMap. It intentionally +retains the PVC so rollback cannot erase Paul/Rich uploads by accident. + +Do not delete the PVC while a DB3 request is processing. Once the rollback has +finished, needed raw files have been copied elsewhere, and the upload UUIDs have +been recorded, complete resource removal with an explicit acknowledgement: + +```sh +CONFIRM_NO_DB3_UPLOAD_PROCESSING=yes ./rollback.sh --delete-pvc +``` + +The second command is cleanup-only: it refuses to run while a pilot image, +ConfigMap reference, or volume mount remains, and it never changes deployment +images. This prevents a delayed PVC cleanup from downgrading a later platform +release. The encrypted storage class uses a `Delete` reclaim policy, so PVC +deletion erases the backing volume rather than archiving it. + +## Processed database cleanup + +The raw DB3 file on the PVC and the processed PostgreSQL rows are independent. +Deleting the PVC does not remove database rows. Cleanup must be based on the +recorded pilot upload UUIDs, reviewed before execution, and run by an authorized +database administrator. A safe transaction shape is: + +```sql +BEGIN; + +CREATE TEMP TABLE db3_pilot_upload_ids ( + id uuid PRIMARY KEY +) ON COMMIT DROP; + +-- Replace these placeholders only with UUIDs recorded from this pilot. +INSERT INTO db3_pilot_upload_ids (id) VALUES + (''); + +-- Inspect the exact scope before either DELETE. +SELECT id, equipment_id, project_id, upload_id, upload_filename +FROM equipment_runs +WHERE upload_id IN (SELECT id FROM db3_pilot_upload_ids); + +SELECT id, equipment_id, filename, owner_id, uploaded_at +FROM data_uploads +WHERE id IN (SELECT id FROM db3_pilot_upload_ids); + +-- Deleting parent runs cascades to equipment_run_trace_samples. +DELETE FROM equipment_runs +WHERE upload_id IN (SELECT id FROM db3_pilot_upload_ids); + +DELETE FROM data_uploads +WHERE id IN (SELECT id FROM db3_pilot_upload_ids); + +COMMIT; +``` + +Remove the dedicated private pilot project only after checking it contains no +non-pilot work. Project deletion alone is not DB3 cleanup because the +`equipment_runs.project_id` foreign key uses `ON DELETE SET NULL`. + +The additive `equipment_run_trace_samples` table and its index/RLS policy may +remain after rollback; they are inert without pilot code and data. Do not drop +the table as part of the operational rollback. diff --git a/geddes/k8s/pilots/db3/configmap.yaml b/geddes/k8s/pilots/db3/configmap.yaml new file mode 100644 index 0000000..1b3072e --- /dev/null +++ b/geddes/k8s/pilots/db3/configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: dt-db3-pilot-config + namespace: ncn-digitaltwins-zchen + labels: + app.kubernetes.io/part-of: dt-db3-pilot + app.kubernetes.io/managed-by: dt-db3-pilot-scripts +data: + # This path is deliberately disabled by default in application code. These + # values opt in only the temporary Geddes pilot deployment. + DT_ENABLE_DB3_UPLOAD: "true" + DT_DB3_ALLOWED_USER_IDS: "ngholiza,will3042,hosler0" + DT_DB3_EQUIPMENT_TOOL_MAP: '{"plasmatherm_versaline_icp_rie_etcher":["VLN-11304-CTC-PM1"]}' + DT_DB3_PILOT_PROJECT_ID: "glance_db3_paul_rich_pilot_89f3f0b8" + DT_DB3_UPLOAD_MAX_MB: "64" + DT_DB3_PROCESSING_STALE_MINUTES: "30" + DT_DB3_PROCESSING_HEARTBEAT_SECONDS: "60" + DT_UPLOADS_DIR: "/var/lib/dt-api/uploads" diff --git a/geddes/k8s/pilots/db3/deploy.sh b/geddes/k8s/pilots/db3/deploy.sh new file mode 100755 index 0000000..fbddccc --- /dev/null +++ b/geddes/k8s/pilots/db3/deploy.sh @@ -0,0 +1,276 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +KUBE_CONTEXT="${KUBE_CONTEXT:-geddes}" +NAMESPACE="${NAMESPACE:-ncn-digitaltwins-zchen}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" + +API_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-api" +WEB_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-web" +BASELINE_API_TAG="$API_REPOSITORY:v52" +BASELINE_WEB_TAG="$WEB_REPOSITORY:v53" +BASELINE_API_DIGEST="$API_REPOSITORY@sha256:b99e0ed5fca6ec908c41dd96b9351f400677f2a494be3bbe964284096f3443c6" +BASELINE_WEB_DIGEST="$WEB_REPOSITORY@sha256:8bfda58813952bbfca463dd6c901de9a73f4ef0b0f2f2d74d72fc626b164bd75" + +API_IMAGE="" +WEB_IMAGE="" +DRY_RUN=false + +usage() { + cat <<'USAGE' +Usage: + deploy.sh --api-image IMAGE@sha256:DIGEST --web-image IMAGE@sha256:DIGEST [--dry-run] + +Environment: + KUBECONFIG kubeconfig file used by kubectl (recommended: set explicitly) + KUBE_CONTEXT target context (default: geddes) + NAMESPACE target namespace (default: ncn-digitaltwins-zchen) + ROLLOUT_TIMEOUT kubectl rollout timeout (default: 300s) + +Only digest-pinned images are accepted. A branch-only tag may precede the +digest, for example dt-api:db3-pilot-abc123@sha256:... +USAGE +} + +while (($#)); do + case "$1" in + --api-image) + [[ $# -ge 2 ]] || { usage >&2; exit 2; } + API_IMAGE="$2" + shift 2 + ;; + --web-image) + [[ $# -ge 2 ]] || { usage >&2; exit 2; } + WEB_IMAGE="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +require_command() { + command -v "$1" >/dev/null 2>&1 || { + echo "Required command not found: $1" >&2 + exit 1 + } +} + +validate_image() { + local label="$1" + local image="$2" + local repository="$3" + if [[ ! "$image" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "$label must be pinned to a sha256 digest: $image" >&2 + exit 2 + fi + local tagged_image="${image%@sha256:*}" + local tag="${tagged_image#"$repository:"}" + if [[ "$tagged_image" == "$image" \ + || "$tagged_image" != "$repository:"* \ + || ! "$tag" =~ ^db3-pilot-[a-zA-Z0-9._-]+$ ]]; then + echo "$label must use $repository:db3-pilot-@sha256:" >&2 + exit 2 + fi +} + +validate_current_image() { + local label="$1" + local image="$2" + local baseline_tag="$3" + local baseline_digest="$4" + local repository="$5" + if [[ "$image" == "$baseline_tag" || "$image" == "$baseline_digest" ]]; then + return + fi + if [[ "$image" == "$repository":db3-pilot-* ]]; then + return + fi + echo "Refusing pilot deploy: $label is not on the recorded baseline or an existing DB3 pilot image: $image" >&2 + exit 1 +} + +require_command kubectl +require_command jq +require_command shasum + +[[ -n "$API_IMAGE" && -n "$WEB_IMAGE" ]] || { + usage >&2 + exit 2 +} +validate_image "--api-image" "$API_IMAGE" "$API_REPOSITORY" +validate_image "--web-image" "$WEB_IMAGE" "$WEB_REPOSITORY" + +config_json="$(kubectl create --dry-run=client -o json \ + -f "$SCRIPT_DIR/configmap.yaml")" +pilot_project_id="$(jq -r '.data.DT_DB3_PILOT_PROJECT_ID // ""' \ + <<<"$config_json")" +if [[ -z "$pilot_project_id" \ + || "$pilot_project_id" == REPLACE_WITH_* ]]; then + echo "DT_DB3_PILOT_PROJECT_ID must name the existing private pilot project" >&2 + exit 2 +fi +config_checksum="$(shasum -a 256 "$SCRIPT_DIR/configmap.yaml" | awk '{print $1}')" + +KUBECTL=(kubectl --context "$KUBE_CONTEXT" --namespace "$NAMESPACE") + +if [[ "$NAMESPACE" != "ncn-digitaltwins-zchen" ]]; then + echo "These manifests are explicitly scoped to namespace ncn-digitaltwins-zchen; got: $NAMESPACE" >&2 + exit 2 +fi + +if ! kubectl config get-contexts "$KUBE_CONTEXT" -o name 2>/dev/null \ + | grep -Fxq "$KUBE_CONTEXT"; then + echo "kubectl context does not exist: $KUBE_CONTEXT" >&2 + exit 1 +fi + +echo "Target: context=$KUBE_CONTEXT namespace=$NAMESPACE" +echo "API image: $API_IMAGE" +echo "Web image: $WEB_IMAGE" + +api_json="$("${KUBECTL[@]}" get deployment dt-api -o json)" +web_json="$("${KUBECTL[@]}" get deployment dt-web -o json)" + +current_api_image="$(jq -r '.spec.template.spec.containers[] | select(.name == "dt-api") | .image' <<<"$api_json")" +current_web_image="$(jq -r '.spec.template.spec.containers[] | select(.name == "dt-web") | .image' <<<"$web_json")" +validate_current_image "dt-api" "$current_api_image" "$BASELINE_API_TAG" \ + "$BASELINE_API_DIGEST" "$API_REPOSITORY" +validate_current_image "dt-web" "$current_web_image" "$BASELINE_WEB_TAG" \ + "$BASELINE_WEB_DIGEST" "$WEB_REPOSITORY" + +jq -e '.spec.template.spec.containers + | any(.name == "dt-api")' >/dev/null <<<"$api_json" || { + echo "deployment/dt-api has no container named dt-api" >&2 + exit 1 +} +jq -e '.spec.template.spec.containers + | any(.name == "dt-web")' >/dev/null <<<"$web_json" || { + echo "deployment/dt-web has no container named dt-web" >&2 + exit 1 +} +jq -e '.spec.template.spec.containers[] + | select(.name == "dt-api") + | (.envFrom // []) + | any(.secretRef.name == "dt-api-secret")' >/dev/null <<<"$api_json" || { + echo "Refusing to patch: dt-api-secret is not present in dt-api envFrom" >&2 + exit 1 +} + +"${KUBECTL[@]}" get secret dt-api-secret >/dev/null +kubectl --context "$KUBE_CONTEXT" get storageclass \ + ceph-multinode-encrypted >/dev/null + +api_env_from="$(jq -c ' + [.spec.template.spec.containers[] + | select(.name == "dt-api") + | (.envFrom // [])][0] + | if any(.configMapRef.name == "dt-db3-pilot-config") + then . + else . + [{"configMapRef":{"name":"dt-db3-pilot-config"}}] + end' <<<"$api_json")" +api_mounts="$(jq -c ' + [.spec.template.spec.containers[] + | select(.name == "dt-api") + | (.volumeMounts // [])][0] + | if any(.name == "dt-db3-pilot-uploads") + then . + else . + [{ + "name":"dt-db3-pilot-uploads", + "mountPath":"/var/lib/dt-api/uploads" + }] + end' <<<"$api_json")" +api_volumes="$(jq -c ' + (.spec.template.spec.volumes // []) + | if any(.name == "dt-db3-pilot-uploads") + then . + else . + [{ + "name":"dt-db3-pilot-uploads", + "persistentVolumeClaim":{"claimName":"dt-db3-pilot-uploads"} + }] + end' <<<"$api_json")" + +api_patch="$(jq -nc \ + --arg image "$API_IMAGE" \ + --arg config_checksum "$config_checksum" \ + --argjson env_from "$api_env_from" \ + --argjson mounts "$api_mounts" \ + --argjson volumes "$api_volumes" ' + {"spec":{"template":{ + "metadata":{"annotations":{ + "dt.purdue.edu/db3-pilot-config-sha256":$config_checksum + }}, + "spec":{ + "containers":[{ + "name":"dt-api", + "image":$image, + "envFrom":$env_from, + "volumeMounts":$mounts + }], + "volumes":$volumes + }}}}' )" +web_patch="$(jq -nc --arg image "$WEB_IMAGE" ' + {"spec":{"template":{"spec":{"containers":[{ + "name":"dt-web", + "image":$image + }]}}}}' )" + +echo "Validating pilot resources and deployment patches on the API server..." +"${KUBECTL[@]}" apply --dry-run=server -o name \ + -f "$SCRIPT_DIR/configmap.yaml" \ + -f "$SCRIPT_DIR/pvc.yaml" \ + -f "$SCRIPT_DIR/ingress.yaml" +"${KUBECTL[@]}" patch deployment dt-api --type=strategic \ + --patch "$api_patch" --dry-run=server -o name +"${KUBECTL[@]}" patch deployment dt-web --type=strategic \ + --patch "$web_patch" --dry-run=server -o name + +if [[ "$DRY_RUN" == true ]]; then + echo "Dry run complete; no resources were changed." + exit 0 +fi + +# The upload route is exposed only after both pilot workloads become ready. +"${KUBECTL[@]}" apply \ + -f "$SCRIPT_DIR/configmap.yaml" \ + -f "$SCRIPT_DIR/pvc.yaml" +"${KUBECTL[@]}" patch deployment dt-api --type=strategic \ + --patch "$api_patch" +"${KUBECTL[@]}" rollout status deployment/dt-api \ + --timeout="$ROLLOUT_TIMEOUT" + +# API startup installs/validates the additive trace schema and capability +# before the pilot UI can direct a user into the new workflow. +"${KUBECTL[@]}" patch deployment dt-web --type=strategic \ + --patch "$web_patch" +"${KUBECTL[@]}" rollout status deployment/dt-web \ + --timeout="$ROLLOUT_TIMEOUT" +"${KUBECTL[@]}" apply -f "$SCRIPT_DIR/ingress.yaml" + +actual_api_image="$("${KUBECTL[@]}" get deployment dt-api \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="dt-api")].image}')" +actual_web_image="$("${KUBECTL[@]}" get deployment dt-web \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="dt-web")].image}')" +[[ "$actual_api_image" == "$API_IMAGE" ]] || { + echo "dt-api image verification failed: $actual_api_image" >&2 + exit 1 +} +[[ "$actual_web_image" == "$WEB_IMAGE" ]] || { + echo "dt-web image verification failed: $actual_web_image" >&2 + exit 1 +} + +echo "DB3 pilot deployed. The 70 MiB ingress exception is limited to:" +echo " https://dt-nanohub.geddes.rcac.purdue.edu/api/dt/data/upload" diff --git a/geddes/k8s/pilots/db3/ingress.yaml b/geddes/k8s/pilots/db3/ingress.yaml new file mode 100644 index 0000000..05b3de0 --- /dev/null +++ b/geddes/k8s/pilots/db3/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: dt-db3-pilot-upload + namespace: ncn-digitaltwins-zchen + labels: + app.kubernetes.io/part-of: dt-db3-pilot + app.kubernetes.io/managed-by: dt-db3-pilot-scripts + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "70m" + nginx.ingress.kubernetes.io/proxy-request-buffering: "off" + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" +spec: + rules: + - host: dt-nanohub.geddes.rcac.purdue.edu + http: + paths: + - path: /api/dt/data/upload + pathType: Exact + backend: + service: + name: dt-web + port: + number: 3000 diff --git a/geddes/k8s/pilots/db3/pvc.yaml b/geddes/k8s/pilots/db3/pvc.yaml new file mode 100644 index 0000000..168cb77 --- /dev/null +++ b/geddes/k8s/pilots/db3/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: dt-db3-pilot-uploads + namespace: ncn-digitaltwins-zchen + labels: + app.kubernetes.io/part-of: dt-db3-pilot + app.kubernetes.io/managed-by: dt-db3-pilot-scripts +spec: + accessModes: + - ReadWriteMany + storageClassName: ceph-multinode-encrypted + resources: + requests: + storage: 5Gi diff --git a/geddes/k8s/pilots/db3/rollback.sh b/geddes/k8s/pilots/db3/rollback.sh new file mode 100755 index 0000000..a3f41cd --- /dev/null +++ b/geddes/k8s/pilots/db3/rollback.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +set -euo pipefail + +KUBE_CONTEXT="${KUBE_CONTEXT:-geddes}" +NAMESPACE="${NAMESPACE:-ncn-digitaltwins-zchen}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" + +# These are the exact images running before the pilot. Keep them digest-pinned +# even though v52/v53 are retained here as human-readable provenance. +ROLLBACK_API_IMAGE="geddes-registry.rcac.purdue.edu/sdx/dt-api@sha256:b99e0ed5fca6ec908c41dd96b9351f400677f2a494be3bbe964284096f3443c6" +ROLLBACK_WEB_IMAGE="geddes-registry.rcac.purdue.edu/sdx/dt-web@sha256:8bfda58813952bbfca463dd6c901de9a73f4ef0b0f2f2d74d72fc626b164bd75" +BASELINE_API_TAG="geddes-registry.rcac.purdue.edu/sdx/dt-api:v52" +BASELINE_WEB_TAG="geddes-registry.rcac.purdue.edu/sdx/dt-web:v53" +API_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-api" +WEB_REPOSITORY="geddes-registry.rcac.purdue.edu/sdx/dt-web" + +DRY_RUN=false +DELETE_PVC=false + +usage() { + cat <<'USAGE' +Usage: + rollback.sh [--dry-run] [--delete-pvc] + +Environment: + KUBECONFIG kubeconfig file used by kubectl (recommended: set explicitly) + KUBE_CONTEXT target context (default: geddes) + NAMESPACE target namespace (default: ncn-digitaltwins-zchen) + ROLLOUT_TIMEOUT kubectl rollout timeout (default: 300s) + +The default rollback disables the route, removes the pilot configuration and +mount from dt-api, and restores the pre-pilot v52/v53 image digests. It retains +the PVC to protect uploaded files. After that rollback, erase the PVC in a +separate cleanup-only run with --delete-pvc and +CONFIRM_NO_DB3_UPLOAD_PROCESSING=yes after recording/backing up needed data. +USAGE +} + +while (($#)); do + case "$1" in + --dry-run) + DRY_RUN=true + shift + ;; + --delete-pvc) + DELETE_PVC=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +require_command() { + command -v "$1" >/dev/null 2>&1 || { + echo "Required command not found: $1" >&2 + exit 1 + } +} + +require_command kubectl +require_command jq + +if [[ "$DELETE_PVC" == true \ + && "${CONFIRM_NO_DB3_UPLOAD_PROCESSING:-}" != "yes" ]]; then + echo "PVC deletion requires CONFIRM_NO_DB3_UPLOAD_PROCESSING=yes" >&2 + exit 2 +fi + +KUBECTL=(kubectl --context "$KUBE_CONTEXT" --namespace "$NAMESPACE") + +if [[ "$NAMESPACE" != "ncn-digitaltwins-zchen" ]]; then + echo "These manifests are explicitly scoped to namespace ncn-digitaltwins-zchen; got: $NAMESPACE" >&2 + exit 2 +fi + +if ! kubectl config get-contexts "$KUBE_CONTEXT" -o name 2>/dev/null \ + | grep -Fxq "$KUBE_CONTEXT"; then + echo "kubectl context does not exist: $KUBE_CONTEXT" >&2 + exit 1 +fi + +echo "Target: context=$KUBE_CONTEXT namespace=$NAMESPACE" +echo "Rollback API image (pre-pilot v52): $ROLLBACK_API_IMAGE" +echo "Rollback web image (pre-pilot v53): $ROLLBACK_WEB_IMAGE" + +api_json="$("${KUBECTL[@]}" get deployment dt-api -o json)" +web_json="$("${KUBECTL[@]}" get deployment dt-web -o json)" + +current_api_image="$(jq -r '.spec.template.spec.containers[] | select(.name == "dt-api") | .image' <<<"$api_json")" +current_web_image="$(jq -r '.spec.template.spec.containers[] | select(.name == "dt-web") | .image' <<<"$web_json")" + +if [[ "$DELETE_PVC" == true ]]; then + if [[ "$current_api_image" == "$API_REPOSITORY":db3-pilot-* \ + || "$current_web_image" == "$WEB_REPOSITORY":db3-pilot-* ]]; then + echo "Refusing PVC cleanup while a pilot image is still deployed; run the normal rollback first" >&2 + exit 1 + fi + if jq -e ' + (.spec.template.spec.volumes // []) + | any(.name == "dt-db3-pilot-uploads")' >/dev/null <<<"$api_json"; then + echo "Refusing PVC cleanup: dt-api still references the pilot volume" >&2 + exit 1 + fi + if jq -e ' + .spec.template.spec.containers[] | select(.name == "dt-api") + | ((.envFrom // []) | any(.configMapRef.name == "dt-db3-pilot-config")) + or ((.volumeMounts // []) | any(.name == "dt-db3-pilot-uploads"))' \ + >/dev/null <<<"$api_json"; then + echo "Refusing PVC cleanup: dt-api still has pilot configuration or mounts" >&2 + exit 1 + fi + if [[ "$DRY_RUN" == true ]]; then + echo "Dry run complete; PVC cleanup preconditions pass and no resources were changed." + exit 0 + fi + "${KUBECTL[@]}" delete ingress dt-db3-pilot-upload --ignore-not-found + "${KUBECTL[@]}" delete configmap dt-db3-pilot-config --ignore-not-found + "${KUBECTL[@]}" delete pvc dt-db3-pilot-uploads --ignore-not-found + echo "Pilot PVC deleted; the encrypted storage class has Delete reclaim policy." + exit 0 +fi + +validate_rollback_source() { + local label="$1" + local image="$2" + local baseline_tag="$3" + local baseline_digest="$4" + local repository="$5" + if [[ "$image" == "$baseline_tag" || "$image" == "$baseline_digest" \ + || "$image" == "$repository":db3-pilot-* ]]; then + return + fi + echo "Refusing rollback: $label is a later or unrelated release, not the recorded baseline/pilot: $image" >&2 + exit 1 +} + +validate_rollback_source "dt-api" "$current_api_image" "$BASELINE_API_TAG" \ + "$ROLLBACK_API_IMAGE" "$API_REPOSITORY" +validate_rollback_source "dt-web" "$current_web_image" "$BASELINE_WEB_TAG" \ + "$ROLLBACK_WEB_IMAGE" "$WEB_REPOSITORY" + +api_index="$(jq -r ' + [.spec.template.spec.containers[].name] | index("dt-api")' <<<"$api_json")" +web_index="$(jq -r ' + [.spec.template.spec.containers[].name] | index("dt-web")' <<<"$web_json")" +[[ "$api_index" != "null" ]] || { + echo "deployment/dt-api has no container named dt-api" >&2 + exit 1 +} +[[ "$web_index" != "null" ]] || { + echo "deployment/dt-web has no container named dt-web" >&2 + exit 1 +} + +api_env_from="$(jq -c --argjson i "$api_index" ' + (.spec.template.spec.containers[$i].envFrom // []) + | map(select(.configMapRef.name != "dt-db3-pilot-config"))' \ + <<<"$api_json")" +api_mounts="$(jq -c --argjson i "$api_index" ' + (.spec.template.spec.containers[$i].volumeMounts // []) + | map(select(.name != "dt-db3-pilot-uploads"))' <<<"$api_json")" +api_volumes="$(jq -c ' + (.spec.template.spec.volumes // []) + | map(select(.name != "dt-db3-pilot-uploads"))' <<<"$api_json")" +api_has_env_from="$(jq -c --argjson i "$api_index" ' + .spec.template.spec.containers[$i] | has("envFrom")' <<<"$api_json")" +api_has_mounts="$(jq -c --argjson i "$api_index" ' + .spec.template.spec.containers[$i] | has("volumeMounts")' <<<"$api_json")" +api_has_volumes="$(jq -c ' + .spec.template.spec | has("volumes")' <<<"$api_json")" +api_has_checksum="$(jq -c ' + (.spec.template.metadata.annotations // {}) + | has("dt.purdue.edu/db3-pilot-config-sha256")' <<<"$api_json")" + +# JSON Patch replaces the complete arrays after filtering only the named pilot +# entries. This preserves every unrelated live envFrom, mount, and volume. +api_patch="$(jq -nc \ + --argjson i "$api_index" \ + --arg image "$ROLLBACK_API_IMAGE" \ + --argjson env_from "$api_env_from" \ + --argjson mounts "$api_mounts" \ + --argjson volumes "$api_volumes" \ + --argjson has_env_from "$api_has_env_from" \ + --argjson has_mounts "$api_has_mounts" \ + --argjson has_volumes "$api_has_volumes" \ + --argjson has_checksum "$api_has_checksum" ' + [ + {"op":"replace", + "path":("/spec/template/spec/containers/" + ($i|tostring) + "/image"), + "value":$image} + ] + + (if $has_env_from then [( + {"op":(if $env_from == [] then "remove" else "replace" end), + "path":("/spec/template/spec/containers/" + ($i|tostring) + "/envFrom")} + + (if $env_from == [] then {} else {"value":$env_from} end) + )] else [] end) + + (if $has_mounts then [( + {"op":(if $mounts == [] then "remove" else "replace" end), + "path":("/spec/template/spec/containers/" + ($i|tostring) + "/volumeMounts")} + + (if $mounts == [] then {} else {"value":$mounts} end) + )] else [] end) + + (if $has_volumes then [( + {"op":(if $volumes == [] then "remove" else "replace" end), + "path":"/spec/template/spec/volumes"} + + (if $volumes == [] then {} else {"value":$volumes} end) + )] else [] end) + + (if $has_checksum then [{ + "op":"remove", + "path":"/spec/template/metadata/annotations/dt.purdue.edu~1db3-pilot-config-sha256" + }] else [] end)' )" +web_patch="$(jq -nc \ + --argjson i "$web_index" \ + --arg image "$ROLLBACK_WEB_IMAGE" ' + [{ + "op":"replace", + "path":("/spec/template/spec/containers/" + ($i|tostring) + "/image"), + "value":$image + }]' )" + +echo "Validating rollback patches on the API server..." +"${KUBECTL[@]}" patch deployment dt-api --type=json \ + --patch "$api_patch" --dry-run=server -o name +"${KUBECTL[@]}" patch deployment dt-web --type=json \ + --patch "$web_patch" --dry-run=server -o name + +if [[ "$DRY_RUN" == true ]]; then + echo "Dry run complete; no resources were changed." + exit 0 +fi + +# Stop admitting new large uploads before changing either workload. +"${KUBECTL[@]}" delete ingress dt-db3-pilot-upload --ignore-not-found +"${KUBECTL[@]}" patch deployment dt-api --type=json --patch "$api_patch" +"${KUBECTL[@]}" patch deployment dt-web --type=json --patch "$web_patch" + +"${KUBECTL[@]}" rollout status deployment/dt-api \ + --timeout="$ROLLOUT_TIMEOUT" +"${KUBECTL[@]}" rollout status deployment/dt-web \ + --timeout="$ROLLOUT_TIMEOUT" + +"${KUBECTL[@]}" delete configmap dt-db3-pilot-config --ignore-not-found + +echo "Pilot workload rollback complete. PVC retained for data safety:" +echo " persistentvolumeclaim/dt-db3-pilot-uploads" +echo "After DB/file retention decisions, erase it with:" +echo " CONFIRM_NO_DB3_UPLOAD_PROCESSING=yes $0 --delete-pvc" diff --git a/geddes/k8s/postgres/schema_v2.sql b/geddes/k8s/postgres/schema_v2.sql index f7a9517..0653384 100644 --- a/geddes/k8s/postgres/schema_v2.sql +++ b/geddes/k8s/postgres/schema_v2.sql @@ -145,6 +145,50 @@ CREATE POLICY equipment_run_visibility_policy ON equipment_runs ) ); +-- Complete parameter traces linked to one physical generic equipment run. +-- sample_time intentionally has no timezone because current GLANCE DB3 exports +-- do not declare one; sample_time_raw preserves the exact source timestamp. +CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_index INT NOT NULL, + source_sample_id BIGINT NOT NULL, + source_tool_id BIGINT, + sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + sample_time_raw TEXT NOT NULL, + values_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, sample_index), + UNIQUE (equipment_run_id, source_sample_id), + CHECK (jsonb_typeof(values_json) = 'object') +); + +CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time +ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); + +ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; +GRANT SELECT ON equipment_run_trace_samples TO api_client; + +CREATE POLICY equipment_run_trace_visibility_policy + ON equipment_run_trace_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + CREATE TABLE IF NOT EXISTS users ( id VARCHAR(255) PRIMARY KEY, -- NanoHUB Subject ID or Email name VARCHAR(255) NOT NULL, @@ -312,6 +356,7 @@ CREATE TABLE IF NOT EXISTS data_uploads ( kind VARCHAR(20) NOT NULL DEFAULT 'all', owner_id VARCHAR(255) REFERENCES users(id), owner_org VARCHAR(255) DEFAULT '', + processing_started_at TIMESTAMP WITH TIME ZONE, uploaded_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); diff --git a/web/app/api/dt/[...path]/route.ts b/web/app/api/dt/[...path]/route.ts index d18b329..e1f1312 100644 --- a/web/app/api/dt/[...path]/route.ts +++ b/web/app/api/dt/[...path]/route.ts @@ -41,6 +41,10 @@ const BLOCKED_RESPONSE_HEADERS = new Set([ export const runtime = "nodejs"; +type StreamingRequestInit = RequestInit & { + duplex?: "half"; +}; + function buildTargetUrl(path: string[], request: NextRequest): string { const sanitizedPath = path.map(encodeURIComponent).join("/"); const suffix = request.nextUrl.search || ""; @@ -70,7 +74,7 @@ async function resolveSession() { async function buildRequestInit( request: NextRequest, session: NonNullable>>, -): Promise { +): Promise { const headers = new Headers(); request.headers.forEach((value, key) => { @@ -86,14 +90,18 @@ async function buildRequestInit( headers.set("X-User-Role", session.user.role || "researcher"); headers.set("X-User-Org", session.user.organization || "Purdue University"); - const init: RequestInit = { + const init: StreamingRequestInit = { method: request.method, headers, cache: "no-store", }; if (request.method !== "GET" && request.method !== "HEAD") { - init.body = await request.arrayBuffer(); + // Keep large uploads as a stream all the way to FastAPI. Buffering here + // used Next's 10 MiB proxy clone limit and made otherwise-valid DB3 files + // fail before the API could enforce its own configured size limit. + init.body = request.body; + init.duplex = "half"; } return init; diff --git a/web/app/data/catalog/[id]/page.tsx b/web/app/data/catalog/[id]/page.tsx index 95bdb31..a5eac0a 100644 --- a/web/app/data/catalog/[id]/page.tsx +++ b/web/app/data/catalog/[id]/page.tsx @@ -4,6 +4,7 @@ import { use, useEffect, useState } from "react"; import Link from "next/link"; import { cn } from "@/lib/utils"; import { getV2Run, type V2Run } from "@/lib/api-client"; +import { RunTraceExplorer } from "@/components/run-trace-explorer"; import { ArrowLeft, Cpu, @@ -40,6 +41,13 @@ function formatNumber(value: number | null | undefined, digits = 2): string { function formatTimestamp(value: string | null | undefined): string { if (!value) return "—"; + const sourceText = value.trim(); + const hasExplicitTimezone = /(?:z|[+-]\d{2}:?\d{2})$/i.test(sourceText); + if (!hasExplicitTimezone) { + // GLANCE currently exports naive wall times. Keep that exact clock value + // visible instead of letting the browser silently attach its own zone. + return `${sourceText.replace("T", " ")} (source time; timezone not recorded)`; + } const d = new Date(value); if (Number.isNaN(d.getTime())) return value; return d.toLocaleString("en-US", { @@ -107,10 +115,12 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string ); } + const isGlanceTrace = run.source === "glance_db3"; const badges: { label: string; className: string }[] = []; + if (isGlanceTrace) badges.push({ label: "Imported GLANCE Trace", className: "text-violet-500 bg-violet-500/10" }); if (run.is_calibration) badges.push({ label: "Calibration", className: "text-blue-500 bg-blue-500/10" }); if (run.is_outlier) badges.push({ label: "Outlier", className: "text-amber-500 bg-amber-500/10" }); - if (badges.length === 0) badges.push({ label: "Clean Validated", className: "text-emerald-500 bg-emerald-500/10" }); + if (!isGlanceTrace && badges.length === 0) badges.push({ label: "Clean Validated", className: "text-emerald-500 bg-emerald-500/10" }); const accessKey = (run.project_access || "private") as keyof typeof ACCESS_STYLE; const access = ACCESS_STYLE[accessKey] || ACCESS_STYLE.private; @@ -121,6 +131,8 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string const genericOutputs = run.outputs && Object.keys(run.outputs).length > 0 ? run.outputs : null; const isEtcher = run.equipment_id === "etcher" || run.avg_etch_rate !== null; + const glanceProvenance = run.raw_payload?.glance; + const sourceFile = run.raw_payload?.source_file; return (
@@ -325,6 +337,10 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string
+ {(run.trace_sample_count ?? 0) > 0 && ( + + )} + {/* Provenance */}

@@ -332,9 +348,53 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string

-
Source run id (idruns)
+
Catalog run ID
{run.run_id}
+ {run.source_run_id !== null && run.source_run_id !== undefined && ( +
+
Source GLANCE run ID
+
{run.source_run_id}
+
+ )} + {(glanceProvenance?.tool_name || glanceProvenance?.glance_tool_id != null) && ( +
+
Source GLANCE tool
+
+ {glanceProvenance.tool_name || "—"} + {glanceProvenance.glance_tool_id != null && ( + + ID {glanceProvenance.glance_tool_id} + + )} +
+
+ )} + {(glanceProvenance?.recipe_name || glanceProvenance?.glance_recipe_id != null) && ( +
+
Source recipe
+
+ {glanceProvenance.recipe_name || "—"} + {glanceProvenance.glance_recipe_id != null && ( + + ID {glanceProvenance.glance_recipe_id} + + )} +
+
+ )} + {glanceProvenance?.material_name && ( +
+
Source material
+
{glanceProvenance.material_name}
+
+ )} + {glanceProvenance?.run_status && ( +
+
Source run status
+
{glanceProvenance.run_status}
+
+ )}
Execution request ID
{run.execution_request_id || "—"}
@@ -351,6 +411,30 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string
Equipment id
{run.equipment_id || "—"}
+ {glanceProvenance?.run_end_time && ( +
+
Source run end
+
+ {formatTimestamp(glanceProvenance.run_end_time)} +
+
+ )} + {sourceFile?.filename && ( +
+
Source DB3
+
+ {sourceFile.filename} +
+
+ )} + {sourceFile?.sha256 && ( +
+
Source SHA-256
+
+ {sourceFile.sha256} +
+
+ )}
{(run.file_refs || []).length > 0 && (
diff --git a/web/app/data/catalog/page.tsx b/web/app/data/catalog/page.tsx index 5e077c5..e07b024 100644 --- a/web/app/data/catalog/page.tsx +++ b/web/app/data/catalog/page.tsx @@ -14,6 +14,10 @@ import { Loader2, } from "lucide-react"; +const isGlanceTrace = (run: V2Run) => run.source === "glance_db3"; +const isCleanValidated = (run: V2Run) => + !isGlanceTrace(run) && !run.is_outlier && !run.is_calibration; + export default function DataCatalogPage() { const [runs, setRuns] = useState([]); const [loading, setLoading] = useState(true); @@ -83,9 +87,10 @@ export default function DataCatalogPage() { const matchesSearch = r.lot_name?.toLowerCase().includes(q); const matchesOutlier = filterOutliers === "all" ? true : - filterOutliers === "clean" ? (!r.is_outlier && !r.is_calibration) : + filterOutliers === "clean" ? isCleanValidated(r) : filterOutliers === "outlier" ? r.is_outlier : - r.is_calibration; + filterOutliers === "calibration" ? r.is_calibration : + isGlanceTrace(r); return matchesSearch && matchesOutlier; }); @@ -129,7 +134,7 @@ export default function DataCatalogPage() {

Total Runs Mapped

-

{runs.filter(r => !r.is_outlier && !r.is_calibration).length}

+

{runs.filter(isCleanValidated).length}

Clean Production Runs

@@ -155,7 +160,7 @@ export default function DataCatalogPage() { />
- {(["all", "clean", "outlier", "calibration"] as const).map((level) => ( + {(["all", "clean", "outlier", "calibration", "glance"] as const).map((level) => ( ))}
@@ -177,16 +182,21 @@ export default function DataCatalogPage() { {filtered.map((r) => { // Show every flag as its own badge so a row that is both a // calibration and an outlier renders both pills (rather than - // hiding one behind a priority order). Clean rows get a single - // "Clean Validated" pill. + // hiding one behind a priority order). A GLANCE trace has not + // undergone scientific validation, so it gets a neutral import + // badge and is kept out of the clean/validated classification. + const glanceTrace = isGlanceTrace(r); const badges: { label: string; className: string }[] = []; + if (glanceTrace) { + badges.push({ label: "Imported GLANCE Trace", className: "text-violet-500 bg-violet-500/10" }); + } if (r.is_calibration) { badges.push({ label: "Calibration", className: "text-blue-500 bg-blue-500/10" }); } if (r.is_outlier) { badges.push({ label: "Outlier", className: "text-amber-500 bg-amber-500/10" }); } - if (badges.length === 0) { + if (!glanceTrace && badges.length === 0) { badges.push({ label: "Clean Validated", className: "text-emerald-500 bg-emerald-500/10" }); } diff --git a/web/app/data/upload/page.tsx b/web/app/data/upload/page.tsx index fc3a70c..139a2a2 100644 --- a/web/app/data/upload/page.tsx +++ b/web/app/data/upload/page.tsx @@ -4,13 +4,16 @@ import { useState, useRef, useEffect } from "react"; import { cn } from "@/lib/utils"; import { getEquipmentListSimple, + getUploadCapabilities, getUploads, getV2Projects, processUpload, uploadFile, type EquipmentSimple, type PersistedUpload, + type UploadResult, type UploadKind, + type UploadCapabilities, type V2Project, } from "@/lib/api-client"; import { @@ -41,6 +44,10 @@ interface UploadedFile { timestamp: string; } +function isDb3File(filename: string): boolean { + return filename.toLowerCase().endsWith(".db3"); +} + function mapPersistedUpload(upload: PersistedUpload): UploadedFile { return { id: upload.id, @@ -62,16 +69,60 @@ function mapPersistedUpload(upload: PersistedUpload): UploadedFile { errors: upload.errors.length > 0 ? upload.errors : undefined, message: upload.status === "processed" - ? "Processed into the data catalog." + ? isDb3File(upload.filename) + ? `Imported ${upload.row_count.toLocaleString()} physical GLANCE runs with linked traces.` + : "Processed into the data catalog." : upload.status === "success" - ? "Stored upload record. Choose a project and process it to add rows to the catalog." - : "Stored upload record with structural validation issues.", + ? isDb3File(upload.filename) + ? "DB3 stored for test import. Choose a project and process its physical runs and traces." + : "Stored upload record. Choose a project and process it to add rows to the catalog." + : upload.status === "stored" + ? "Stored for provenance; no catalog processing is available for this record." + : upload.status === "processing" + ? isDb3File(upload.filename) + ? "Importing physical GLANCE runs and their timestamped traces..." + : "Processing rows into the data catalog..." + : upload.status === "failed" || upload.status === "error" + ? "Processing failed. Review the errors below." + : "Upload status is unavailable.", timestamp: upload.uploaded_at, }; } +function mapUploadResult( + upload: UploadResult, + fallbackTimestamp: string, + fallbackType = "text/csv", +): UploadedFile { + return { + id: upload.upload_id, + name: upload.filename, + size: upload.size_bytes, + type: fallbackType, + status: + upload.status === "processed" + ? "processed" + : upload.status === "processing" + ? "processing" + : upload.status === "failed" + ? "failed" + : upload.status === "stored" + ? "stored" + : upload.status === "success" + ? "success" + : "error", + equipmentId: upload.equipment_id, + kind: upload.kind ?? "all", + rows: upload.row_count > 0 ? upload.row_count : undefined, + columns: upload.columns, + errors: upload.errors || undefined, + message: upload.message, + timestamp: fallbackTimestamp, + }; +} + function shouldRestoreUpload(upload: PersistedUpload): boolean { - return upload.status !== "error" && upload.status !== "failed"; + return Boolean(upload.id); } export default function DataUploadPage() { @@ -84,24 +135,34 @@ export default function DataUploadPage() { const [selectedProject, setSelectedProject] = useState(""); const [processingUploadId, setProcessingUploadId] = useState(null); const [uploadKind, setUploadKind] = useState("input"); + const [uploadCapabilities, setUploadCapabilities] = useState({ + db3_upload: { enabled: false, max_mb: 64, equipment_ids: [] }, + }); const fileInputRef = useRef(null); // The canonical etcher is ingested as one combined file; every other // registered tool uses paired input/output templates. const isEtcher = selectedEquipment === "etcher"; const effectiveKind: UploadKind = isEtcher ? "all" : uploadKind; + const db3EnabledForSelectedEquipment = + uploadCapabilities.db3_upload.enabled && + uploadCapabilities.db3_upload.equipment_ids.includes(selectedEquipment); useEffect(() => { const fetchEquipment = async () => { - const [equipmentData, uploadData, projectData] = await Promise.all([ + const [equipmentData, uploadData, projectData, capabilities] = await Promise.all([ getEquipmentListSimple(), getUploads(), getV2Projects(), + getUploadCapabilities(), ]); setEquipment(equipmentData ?? []); setFiles((uploadData ?? []).filter(shouldRestoreUpload).map(mapPersistedUpload)); setProjects(projectData ?? []); + if (capabilities) { + setUploadCapabilities(capabilities); + } setEquipmentLoading(false); if (equipmentData && equipmentData.length > 0) { @@ -176,6 +237,25 @@ export default function DataUploadPage() { for (const rawFile of Array.from(fileList)) { const ts = new Date().toISOString(); + if (isDb3File(rawFile.name) && !db3EnabledForSelectedEquipment) { + setFiles((prev) => [ + { + name: rawFile.name, + size: rawFile.size, + type: rawFile.type || "application/vnd.sqlite3", + status: "error", + equipmentId: selectedEquipment, + kind: "all", + timestamp: ts, + errors: [ + "DB3 import is not enabled for this user and selected equipment.", + ], + }, + ...prev, + ]); + continue; + } + const fileKind: UploadKind = isDb3File(rawFile.name) ? "all" : effectiveKind; // Add file in "uploading" state setFiles((prev) => [ @@ -185,7 +265,7 @@ export default function DataUploadPage() { type: rawFile.type || "text/csv", status: "uploading", equipmentId: selectedEquipment, - kind: effectiveKind, + kind: fileKind, timestamp: ts, }, ...prev, @@ -195,33 +275,33 @@ export default function DataUploadPage() { setTimeout(() => { setFiles((prev) => prev.map((f) => - f.name === rawFile.name && f.timestamp === ts ? { ...f, status: "validating" } : f + f.name === rawFile.name && + f.timestamp === ts && + f.status === "uploading" + ? { ...f, status: "validating" } + : f ) ); }, 400); // Actually send to backend - const result = await uploadFile(rawFile, selectedEquipment, effectiveKind); + const uploadResponse = await uploadFile(rawFile, selectedEquipment, fileKind); + const result = uploadResponse.data; setFiles((prev) => prev.map((f) => { if (f.name !== rawFile.name || f.timestamp !== ts) return f; if (!result) { - return { ...f, status: "error", errors: ["API unavailable — is the FastAPI server running?"] }; + return { + ...f, + status: "error", + errors: [uploadResponse.error || "Upload failed."], + }; } - return { - ...f, - id: result.upload_id, - status: result.status === "success" ? "success" : "error", - equipmentId: result.equipment_id, - kind: result.kind ?? effectiveKind, - rows: result.row_count > 0 ? result.row_count : undefined, - columns: result.columns, - errors: result.errors || undefined, - message: result.message, - }; + return mapUploadResult(result, ts, rawFile.type || "text/csv"); }) ); + } }; @@ -304,6 +384,7 @@ export default function DataUploadPage() {

Upload Data

Store CSV or Excel files against registered equipment and run lightweight structural validation. + {uploadCapabilities.db3_upload.enabled && " A temporary DB3 pilot is enabled for approved participants."}

@@ -442,7 +523,7 @@ export default function DataUploadPage() { ref={fileInputRef} type="file" multiple - accept=".csv,.xlsx,.xls" + accept={db3EnabledForSelectedEquipment ? ".csv,.xlsx,.xls,.db3" : ".csv,.xlsx,.xls"} onChange={handleFileSelect} className="hidden" /> @@ -452,7 +533,9 @@ export default function DataUploadPage() {

Drop files here or click to browse

- Supports CSV, XLSX, XLS — max 50 MB per file + {db3EnabledForSelectedEquipment + ? `CSV/Excel max 50 MB; temporary DB3 pilot max ${uploadCapabilities.db3_upload.max_mb} MB` + : "CSV/Excel max 50 MB"}

@@ -461,6 +544,11 @@ export default function DataUploadPage() { Excel + {db3EnabledForSelectedEquipment && ( + + DB3 pilot + + )}
@@ -528,13 +616,17 @@ export default function DataUploadPage() { {file.status === "validating" && (

Checking required columns and row structure...

)} - {(file.status === "success" || file.status === "processed") && ( + {(file.status === "success" || file.status === "processed" || file.status === "stored") && (

{file.message ?? `Stored successfully${file.rows ? ` after checking ${file.rows} data rows` : ""}.`}

)} {file.status === "processing" && ( -

Processing rows into the data catalog...

+

+ {isDb3File(file.name) + ? "Importing physical GLANCE runs and their timestamped traces..." + : "Processing rows into the data catalog..."} +

)} {(file.status === "error" || file.status === "failed") && file.errors && (
@@ -545,7 +637,7 @@ export default function DataUploadPage() { ))}
)} - {(file.status === "success" || file.status === "processed") && file.errors && ( + {(file.status === "success" || file.status === "processed" || file.status === "stored") && file.errors && (
{file.errors.map((warning, j) => (

@@ -601,7 +693,8 @@ export default function DataUploadPage() {

3. Processing

- Processing runs in the background and the status updates here automatically — no need to reload. Input and output files are merged by lot name and run date. Excel files are stored for provenance but are not ingested automatically. + Processing runs in the background and the status updates here automatically — no need to reload. Input and output CSV files are merged by lot name and run date. Excel files are stored for provenance but are not ingested automatically. + {uploadCapabilities.db3_upload.enabled && " The temporary DB3 pilot creates one catalog row per physical GLANCE run and stores every timestamped sample as its linked trace."}

diff --git a/web/components/run-trace-explorer.tsx b/web/components/run-trace-explorer.tsx new file mode 100644 index 0000000..4d43cdf --- /dev/null +++ b/web/components/run-trace-explorer.tsx @@ -0,0 +1,494 @@ +"use client"; + +import dynamic from "next/dynamic"; +import { useEffect, useMemo, useState } from "react"; +import { Activity, AlertCircle, Clock3, Loader2 } from "lucide-react"; + +import { ErrorBoundary } from "@/components/ErrorBoundary"; +import { + getV2Runs, + getV2RunTrace, + type V2Run, + type V2RunTrace, + type V2RunTraceParameter, +} from "@/lib/api-client"; +import { cn } from "@/lib/utils"; + +const Plot = dynamic(() => import("react-plotly.js"), { ssr: false }); + +const MAX_SELECTED_RUNS = 6; +const TRACE_COLORS = [ + "#8b5cf6", + "#10b981", + "#f59e0b", + "#3b82f6", + "#ef4444", + "#ec4899", + "#14b8a6", + "#f97316", +]; + +type AxisMode = "relative" | "absolute"; + +interface RunTraceExplorerProps { + run: V2Run; +} + +interface ComparisonResult { + requestKey: string; + traces: V2RunTrace[]; + warning: string | null; +} + +function sortParameters( + parameters: V2RunTraceParameter[], +): V2RunTraceParameter[] { + return [...parameters].sort((left, right) => { + const byName = left.name.localeCompare(right.name, undefined, { + numeric: true, + sensitivity: "base", + }); + return byName || left.key.localeCompare(right.key, undefined, { numeric: true }); + }); +} + +function parameterLabel(parameter: V2RunTraceParameter): string { + const unit = parameter.unit?.trim() ? ` (${parameter.unit.trim()})` : ""; + return `${parameter.name}${unit} · ${parameter.key}`; +} + +function traceLabel(trace: V2RunTrace): string { + const lot = trace.lot_name || `Catalog run ${trace.run_id}`; + return trace.source_run_id === null || trace.source_run_id === undefined + ? lot + : `${lot} · GLANCE ${trace.source_run_id}`; +} + +function finiteValue(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function colorForRun(runId: number): string { + return TRACE_COLORS[Math.abs(runId) % TRACE_COLORS.length]; +} + +export function RunTraceExplorer({ run }: RunTraceExplorerProps) { + const [currentTrace, setCurrentTrace] = useState(null); + const [peerRuns, setPeerRuns] = useState([]); + const [selectedRunIds, setSelectedRunIds] = useState([run.run_id]); + const [selectedParameter, setSelectedParameter] = useState(""); + const [axisMode, setAxisMode] = useState("relative"); + const [comparisonResult, setComparisonResult] = + useState(null); + const [initialLoading, setInitialLoading] = useState(true); + const [traceError, setTraceError] = useState(null); + + const selectedPeerIds = useMemo( + () => selectedRunIds.filter((runId) => runId !== run.run_id), + [run.run_id, selectedRunIds], + ); + const comparisonRequestKey = `${selectedParameter}:${selectedPeerIds.join(",")}`; + + useEffect(() => { + let cancelled = false; + + async function loadInitialTrace() { + const [trace, visibleRuns] = await Promise.all([ + getV2RunTrace(run.run_id), + run.project_id + ? getV2Runs({ + project_id: run.project_id, + include_outliers: true, + limit: 200, + }) + : Promise.resolve([]), + ]); + + if (cancelled) return; + + if (!trace) { + setTraceError("The complete time-series trace could not be loaded."); + setInitialLoading(false); + return; + } + + const parameters = sortParameters(trace.parameters); + setCurrentTrace(trace); + setSelectedParameter(parameters[0]?.key ?? ""); + setPeerRuns( + (visibleRuns ?? []) + .filter( + (candidate) => + candidate.run_id !== run.run_id && + candidate.project_id === run.project_id && + candidate.equipment_id === run.equipment_id && + (candidate.trace_sample_count ?? 0) > 0, + ) + .sort((left, right) => + (right.run_date ?? "").localeCompare(left.run_date ?? ""), + ), + ); + setInitialLoading(false); + } + + void loadInitialTrace(); + return () => { + cancelled = true; + }; + }, [run.equipment_id, run.project_id, run.run_id]); + + useEffect(() => { + let cancelled = false; + + if (!selectedParameter || selectedPeerIds.length === 0) return; + + async function loadPeerTraces() { + const results = await Promise.all( + selectedPeerIds.map(async (runId) => ({ + runId, + trace: await getV2RunTrace(runId, [selectedParameter]), + })), + ); + + if (cancelled) return; + + const available = results + .map((result) => result.trace) + .filter((trace): trace is V2RunTrace => trace !== null); + const missing = results.filter((result) => result.trace === null); + + setComparisonResult({ + requestKey: comparisonRequestKey, + traces: available, + warning: + missing.length > 0 + ? `${missing.length} selected run${missing.length === 1 ? "" : "s"} could not be loaded.` + : null, + }); + } + + void loadPeerTraces(); + return () => { + cancelled = true; + }; + }, [comparisonRequestKey, selectedParameter, selectedPeerIds]); + + const comparisonIsCurrent = comparisonResult?.requestKey === comparisonRequestKey; + const comparisonWarning = + comparisonResult?.requestKey === comparisonRequestKey + ? comparisonResult.warning + : null; + const comparisonLoading = + Boolean(selectedParameter) && + selectedPeerIds.length > 0 && + !comparisonIsCurrent; + + const parameterOptions = useMemo( + () => sortParameters(currentTrace?.parameters ?? []), + [currentTrace], + ); + const parameter = parameterOptions.find( + (candidate) => candidate.key === selectedParameter, + ); + const displayedTraces = useMemo( + () => { + if (!currentTrace) return []; + const matchingPeerTraces = + comparisonResult?.requestKey === comparisonRequestKey + ? comparisonResult.traces + : []; + return [currentTrace, ...matchingPeerTraces]; + }, + [comparisonRequestKey, comparisonResult, currentTrace], + ); + + const plotData = useMemo(() => { + if (!selectedParameter) return []; + + return displayedTraces.map((trace) => ({ + x: trace.samples.map((sample) => + axisMode === "relative" ? sample.relative_seconds : sample.timestamp, + ), + y: trace.samples.map((sample) => + finiteValue(sample.values[selectedParameter]), + ), + text: trace.samples.map( + (sample) => + `Absolute: ${sample.timestamp}
` + + `Relative: ${sample.relative_seconds.toFixed(3)} s
` + + `Sample record: ${sample.sample_record_id ?? "—"}`, + ), + type: "scattergl" as const, + mode: "lines" as const, + name: traceLabel(trace), + legendgroup: String(trace.run_id), + connectgaps: false, + line: { color: colorForRun(trace.run_id), width: 1.6 }, + hovertemplate: + "%{fullData.name}
Value: %{y}
%{text}", + })); + }, [axisMode, displayedTraces, selectedParameter]); + + const runsWithoutValues = useMemo(() => { + if (!selectedParameter) return []; + return displayedTraces.filter( + (trace) => + !trace.samples.some( + (sample) => finiteValue(sample.values[selectedParameter]) !== null, + ), + ); + }, [displayedTraces, selectedParameter]); + + function togglePeerRun(runId: number) { + setSelectedRunIds((current) => { + if (current.includes(runId)) { + return current.filter((selectedId) => selectedId !== runId); + } + if (current.length >= MAX_SELECTED_RUNS) return current; + return [...current, runId]; + }); + } + + if (initialLoading) { + return ( +
+
+ Loading complete run trace… +
+
+ ); + } + + if (traceError || !currentTrace) { + return ( +
+
+ +
+

Time-series trace unavailable

+

+ {traceError ?? "No trace samples were returned for this run."} +

+
+
+
+ ); + } + + return ( +
+
+
+
+ +

+ Complete parameter trace +

+
+

+ Every recorded sample is retained. Relative time is calculated from each + run's first absolute sample only when displayed. +

+
+
+ {currentTrace.sample_count.toLocaleString()} samples +
+
+ +
+ + +
+ + Time axis + +
+ {(["relative", "absolute"] as const).map((mode) => ( + + ))} +
+
+
+ +
+
+

+ Compare runs ({selectedRunIds.length}/{MAX_SELECTED_RUNS}) +

+ {selectedRunIds.length >= MAX_SELECTED_RUNS && ( +

Comparison limit reached

+ )} +
+
+ + {peerRuns.map((peer) => { + const selected = selectedRunIds.includes(peer.run_id); + const disabled = !selected && selectedRunIds.length >= MAX_SELECTED_RUNS; + return ( + + ); + })} + {peerRuns.length === 0 && ( +

+ No other trace-backed runs from this equipment are available in this + project. +

+ )} +
+
+ +
+ {comparisonLoading && ( +

+ Loading selected run + traces… +

+ )} + {comparisonWarning && ( +

+ {comparisonWarning} +

+ )} + {runsWithoutValues.length > 0 && ( +

+ {`${runsWithoutValues.map(traceLabel).join(", ")} ${ + runsWithoutValues.length === 1 ? "has" : "have" + } no numeric values for this parameter.`} +

+ )} +
+ + {parameter ? ( + + + + ) : ( +
+ No trace parameters were reported for this run. +
+ )} + +
+ +

+ {axisMode === "relative" + ? "Relative seconds are derived at read time from each run's first absolute sample; they are not stored as source data." + : currentTrace.timestamp_timezone + ? `Source timestamps are shown in ${currentTrace.timestamp_timezone}.` + : "GLANCE did not record a source timezone. Source clock values are shown as recorded."} +

+
+
+ ); +} + +export default RunTraceExplorer; diff --git a/web/lib/api-client.ts b/web/lib/api-client.ts index 62b58e3..4ce3b18 100644 --- a/web/lib/api-client.ts +++ b/web/lib/api-client.ts @@ -150,6 +150,8 @@ export interface V2RunFileRef { export interface V2Run { run_id: number; + source_run_id?: number | string | null; + trace_sample_count?: number; execution_request_id: string; lot_name: string; run_date: string | null; @@ -169,6 +171,31 @@ export interface V2Run { pi_name: string | null; project_access: V2Project["access"] | null; file_refs: V2RunFileRef[]; + source?: string; + upload_filename?: string; + raw_payload?: { + glance?: { + glance_run_id?: number | string | null; + glance_tool_id?: number | string | null; + tool_name?: string | null; + lot_name?: string | null; + material_name?: string | null; + glance_recipe_id?: number | string | null; + recipe_name?: string | null; + run_start_time?: string | null; + run_end_time?: string | null; + run_status?: string | null; + sample_count?: number | null; + run_id?: number | string | null; + }; + source_file?: { + type?: string | null; + filename?: string | null; + size_bytes?: number | null; + sha256?: string | null; + }; + [key: string]: unknown; + }; } export function getV2Projects() { @@ -474,6 +501,44 @@ export function getV2Run(runId: number | string) { return apiFetch(`/dataset/v2/runs/${encodeURIComponent(String(runId))}`); } +export interface V2RunTraceParameter { + key: string; + name: string; + unit: string | null; +} + +export interface V2RunTraceSample { + sample_record_id: number | string | null; + source_tool_id: number | string | null; + timestamp: string; + relative_seconds: number; + values: Record; +} + +export interface V2RunTrace { + run_id: number; + source_run_id: number | string | null; + lot_name: string; + sample_count: number; + start_time: string | null; + end_time: string | null; + timestamp_timezone: string | null; + parameters: V2RunTraceParameter[]; + samples: V2RunTraceSample[]; +} + +export function getV2RunTrace( + runId: number | string, + parameters: string[] = [], +) { + const query = new URLSearchParams(); + parameters.forEach((parameter) => query.append("parameters", parameter)); + const suffix = query.size > 0 ? `?${query.toString()}` : ""; + return apiFetch( + `/dataset/v2/runs/${encodeURIComponent(String(runId))}/trace${suffix}`, + ); +} + export interface ParityData { source: string; model_type?: string; @@ -1251,7 +1316,6 @@ export interface PersistedUpload { equipment_id: string; filename: string; stored_as: string; - storage_path: string; size_bytes: number; row_count: number; columns: string[]; @@ -1263,6 +1327,14 @@ export interface PersistedUpload { uploaded_at: string; } +export interface UploadCapabilities { + db3_upload: { + enabled: boolean; + max_mb: number; + equipment_ids: string[]; + }; +} + export interface ProcessUploadResult { status: "processing" | "processed" | "failed" | string; upload_id: string; @@ -1274,11 +1346,16 @@ export interface ProcessUploadResult { message: string; } +export interface UploadFileResponse { + data: UploadResult | null; + error: string | null; +} + export async function uploadFile( file: File, equipmentId: string, kind: UploadKind = "all", -): Promise { +): Promise { try { const formData = new FormData(); formData.append("file", file); @@ -1292,12 +1369,27 @@ export async function uploadFile( }); if (!response.ok) { - return null; + let error = `Upload failed (HTTP ${response.status}).`; + try { + const body = (await response.json()) as { detail?: unknown }; + if (typeof body.detail === "string" && body.detail.trim()) { + error = body.detail; + } + } catch { + // The response was not JSON; retain the status-based message. + } + return { data: null, error }; } - return (await response.json()) as UploadResult; + return { + data: (await response.json()) as UploadResult, + error: null, + }; } catch { - return null; + return { + data: null, + error: "API unavailable — is the FastAPI server running?", + }; } } @@ -1305,6 +1397,10 @@ export function getUploads() { return apiFetch("/data/uploads"); } +export function getUploadCapabilities() { + return apiFetch("/data/upload-capabilities"); +} + export function processUpload(uploadId: string, projectId: string) { return apiJson( `/data/uploads/${encodeURIComponent(uploadId)}/process`, diff --git a/web/next.config.ts b/web/next.config.ts index 1327cdb..454bb34 100644 --- a/web/next.config.ts +++ b/web/next.config.ts @@ -3,6 +3,12 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { // Required for Docker / Geddes Kubernetes deployment output: "standalone", + experimental: { + // Preserve headroom for any authenticated proxied request that is not on + // the explicitly streamed upload route. The API retains the authoritative + // per-file limits. + proxyClientMaxBodySize: "70mb", + }, }; export default nextConfig; diff --git a/web/proxy.ts b/web/proxy.ts index 9e1bded..fbabe2c 100644 --- a/web/proxy.ts +++ b/web/proxy.ts @@ -92,9 +92,10 @@ export async function proxy(req: NextRequest) { export const config = { // Match every request except Next.js internals and the file extensions - // we already treat as public. Keeping the matcher tight avoids an - // unnecessary getToken() call on every static asset request. + // we already treat as public. The upload route is excluded from this outer + // proxy because Next clones matching request bodies before route handling. + // Its catch-all route still calls auth() and remains the trust boundary. matcher: [ - "/((?!_next/static|_next/image|_next/data|favicon.ico).*)", + "/((?!api/dt/data/upload(?:/|$)|_next/static|_next/image|_next/data|favicon.ico).*)", ], };