diff --git a/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md b/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md index a6e9a9c..41eb93f 100644 --- a/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md +++ b/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md @@ -649,3 +649,134 @@ maintenance operation: it reads the authoritative project plus global Postgres history, disables the CSV fallback, fits against the current snapshot instead of reusing a stale active model, and fails the durable follow-up when no proposal batch can be produced. + +## 16. Tool 3 production activation record + +On 2026-08-06 the Tool 3 direct-PostgreSQL connector was activated for the +registered equipment +`updated_plasmatherm_versaline_deep_silicon_etcher` and the private project +`glance_db3_dse_paul_rich_pilot_705fa109`. The source tool was verified directly +as `VLN-11304-CTC-PM1` (`idtools=3`). The mapping is stored only in the Geddes +Secret; no credential or deployable mapping JSON is committed to Git. + +The initial image exposed an entry-point defect: launching +`scripts/run_glance_poll.py` as a file did not place the sibling `common` +package on Python's import path. The container now launches +`python -m scripts.run_glance_poll` from `/app/azure`. The first failed Job was +stopped by its bounded retry policy before it could query GLANCE or write DT +data. + +The activation gates produced the following evidence: + +- Approved source run `1` created one DT identity with 77 distinct samples and + 7 distinct events. Its acknowledged cursor advanced from 0 to 1. +- An identical run-1 replay returned `no_data`, proving the pilot cursor was + durable and did not create a duplicate. +- The complete 13-run facility allowlist committed 13 distinct source + identities, 6,044 samples, 153,696 parameter values, and 103 events. No run + was rejected and no local connector failure occurred. +- A fresh source audit found 3,517 Tool 3 runs, a current maximum run ID of + 12,164, and a latest start time of 2026-08-06 12:02:00.619830 in the source's + timezone-naive timestamp convention. +- Current run 12,164 committed 351 samples, 23,868 values, and 6 events. It + produced two non-fatal registry warnings; unmatched source signals remain + visible under their preserved parameter IDs rather than being silently + aliased or discarded. +- The exact normal-mode canary idempotently refreshed run 12,164, discovered + new run 12,165, and advanced the live cursor to 12,165. Each source run still + had exactly one DT identity. + +The live CronJob has no backfill allowlist, uses run 12,164 as its bounded live +baseline, and runs on `*/10 * * * *` with `concurrencyPolicy: Forbid`. Automatic +executions at 17:30, 17:40, and 17:50 UTC all succeeded with zero rejected runs +and zero local failures. They advanced the cursor through run 12,167 while +preserving exactly one DT identity for each of runs 12,164 through 12,167. + +This activation does not authorize or perform an unbounded historical import. +The approximately 3,500 earlier Tool 3 runs remain a separate backfill project +and must be copied in reviewed, bounded batches with source/destination counts. +The Git CronJob manifest remains suspended and placeholder-only by design; +production activation and its secret-backed mapping are operational state. + +### 16.1 Permanent-project cutover + +On 2026-08-07 the Tool 3 production mapping was cut over from the temporary +DB3 pilot to the private permanent project `glance_live_dse_0297fa75` (GLANCE +Live DSE). The Secret retains the required equipment-keyed mapping for +`updated_plasmatherm_versaline_deep_silicon_etcher`, source tool 3, and live +baseline run 12,164. No connection credential or deployable mapping value is +stored in Git. + +The scheduler was suspended with no active Job during the change. The 13 +direct live identities from source runs 12,164 through 12,183 were then +replayed through the production ingestion endpoint. The stable source identity +upsert moved those rows to GLANCE Live DSE without inserting duplicates. The +13 approved historical canary identities (runs 1 through 169) and all 39 +`glance_db3` pilot records remained in the temporary project. + +The post-cutover audit recorded: + +- 13 permanent-project live identities, 10,822 samples, and 79 events; +- 13 temporary-project direct canary identities, 6,044 samples, and 103 + events; +- unchanged combined totals of 26 direct identities, 16,866 samples, and 182 + events; +- no duplicate direct GLANCE source identities; and +- successful row-level visibility for both `hosler0` and `will3042`, including + all 1,772 samples from source run 12,183. + +One initial replay Job rejected the configuration before querying GLANCE or +writing DT data because the mapping had temporarily been serialized as a JSON +array. The mapping was restored to the connector's required equipment-keyed +object, independently parsed by the API, and the replacement exact-ID replay +completed with 13 accepted runs and zero local failures or rejections. + +A normal-mode overlap poll and the first automatically scheduled poll after +resume both completed successfully with cursor 12,183, zero rejected runs, and +zero local failures. The CronJob remains active on `*/10 * * * *`. This +cutover does not move the DB3 pilot data and does not authorize an unbounded +historical Tool 3 backfill. + +## 17. Trace export and equipment recipe viewer + +The production acceptance test was completed by facility staff using Tool 3 +run 12,213 (`PolyEtch_7loops`) on 2026-08-09. Digital Twin preserved the run in +GLANCE Live DSE with 87 trace samples, 68 parameters, six events, and recipe +2,964. Fifty-nine parameters carried GLANCE units; parameters without a source +unit must be labeled `unit not provided` rather than assigned an inferred unit. + +Both the run-detail trace explorer and cross-run Trace Analysis must display a +prominent Y-axis parameter/unit label and an explicit relative-seconds or +absolute-source-time X-axis label. They must export the selected parameter to +CSV with catalog/source run identity, equipment/tool identity, parameter +identity, source timestamp, derived relative seconds, sample identity, unit, +and the original recorded value. Edge smoothing and trendlines are display-only +and must never alter the CSV. Plotly's image action remains available for PNG +graph export. + +The Recipe Viewer is equipment-scoped. It must not compare or combine recipes +from different equipment. Within an equipment item, `idrecipes` is the source +identity; recipe name, source hash, and source timestamp distinguish versions +with repeated names. Recipe discovery, detail, and export use the same project +row-level visibility as Catalog, return 404 for both absent and inaccessible +recipes, and never return the Base64 recipe file to the browser. + +The server decodes `recipes.recipefile` from strict Base64 into bounded UTF-8 +XML, rejects DTD/entity declarations, verifies the connector-recorded decoded +size and SHA-256, and caps recipe size, steps, attributes, and cell length. +Unknown attributes are preserved after the standard GLANCE rows with a visible +warning; unknown match-mode codes are preserved rather than guessed. The +decoded matrix and CSV were verified byte-for-byte against the facility's +recipe-2,964 fixture: 11 steps, 123 parameter rows, and a 9,454-byte CSV. +Facility source files remain external validation fixtures and are not committed +to Git. + +Production deployment completed on 2026-08-11 from commit `371da73`. The API +is pinned to `sha256:a9cd36d051b2aada3dd5e7f843cbaac83998711dbd205f66935357e3a0bde483` +and the web application is pinned to +`sha256:77251decd1d31729309749331e0801fd071cac315d2a9612ad7ba104930b12f9`. +An authenticated, non-admin acceptance request as facility user `hosler0` +resolved recipe 2,964 to run 12,213, returned 11 steps and 123 parameter rows, +exposed no Base64 source, and produced the exact 9,454-byte reference CSV. The +10-minute GLANCE CronJob remained enabled and completed successfully after the +rollout. diff --git a/api/data_loader_pg.py b/api/data_loader_pg.py index b4579c6..5c51454 100644 --- a/api/data_loader_pg.py +++ b/api/data_loader_pg.py @@ -2968,6 +2968,314 @@ def get_equipment_run_trace_pg( conn.close() +def _recipe_scalar(value: Any) -> Any: + """Return a JSON-safe scalar while preserving GLANCE source text.""" + + if value is None: + return None + if hasattr(value, "isoformat"): + return value.isoformat() + return value + + +def list_recipe_equipment_pg( + *, + nanohub_user_id: Optional[str] = None, + is_admin: bool = False, +) -> List[Dict[str, Any]]: + """List equipment with recipe files visible under catalog RLS.""" + + conn = get_pg_superuser_connection() if is_admin else get_pg_connection( + nanohub_user_id + ) + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + WITH visible_recipes AS ( + SELECT + r.equipment_id, + NULLIF( + r.raw_payload_json #>> + '{glance,recipe,source_recipe_id}', + '' + ) AS source_recipe_id + FROM equipment_runs r + WHERE COALESCE(NULLIF(r.source_system, ''), r.source) = 'glance' + AND NULLIF( + r.raw_payload_json #>> + '{glance,recipe,raw,recipefile}', + '' + ) IS NOT NULL + ) + SELECT + visible.equipment_id, + COALESCE( + metadata.equipment_name, + visible.equipment_id + ) AS equipment_name, + COUNT(DISTINCT visible.source_recipe_id)::int AS recipe_count + FROM visible_recipes visible + LEFT JOIN equipment_metadata metadata + ON metadata.domain_id = visible.equipment_id + WHERE visible.source_recipe_id IS NOT NULL + GROUP BY visible.equipment_id, metadata.equipment_name + ORDER BY COALESCE(metadata.equipment_name, visible.equipment_id) + """ + ) + rows = [dict(row) for row in cur.fetchall()] + conn.commit() + return rows + finally: + conn.close() + + +def list_equipment_recipes_pg( + *, + equipment_id: str, + nanohub_user_id: Optional[str] = None, + is_admin: bool = False, + limit: int = 500, + offset: int = 0, +) -> List[Dict[str, Any]]: + """List the latest visible metadata for each GLANCE recipe identity.""" + + normalized_equipment_id = str(equipment_id or "").strip() + if not normalized_equipment_id: + raise ValueError("equipment_id is required") + safe_limit = max(1, min(int(limit), 1_000)) + safe_offset = max(0, int(offset)) + conn = get_pg_superuser_connection() if is_admin else get_pg_connection( + nanohub_user_id + ) + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + f""" + WITH visible AS ( + SELECT + r.id, + r.equipment_id, + r.source_tool_id, + r.source_run_id, + r.run_date, + r.project_id, + NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,source_recipe_id}}', + '' + ) AS source_recipe_id, + COALESCE( + NULLIF( + r.raw_payload_json #>> '{{glance,recipe,name}}', + '' + ), + NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,raw,recipename}}', + '' + ) + ) AS recipe_name, + COALESCE( + NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,source_hash}}', + '' + ), + NULLIF( + r.raw_payload_json #>> '{{glance,recipe,raw,hash}}', + '' + ) + ) AS source_hash, + COALESCE( + NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,timestamp}}', + '' + ), + NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,raw,timestamp}}', + '' + ) + ) AS recipe_timestamp, + NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,file_sha256}}', + '' + ) AS file_sha256, + NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,file_size_bytes}}', + '' + ) AS file_size_bytes + FROM equipment_runs r + WHERE r.equipment_id = %s + AND COALESCE(NULLIF(r.source_system, ''), r.source) = 'glance' + AND NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,raw,recipefile}}', + '' + ) IS NOT NULL + ), ranked AS ( + SELECT + visible.*, + COUNT(*) OVER ( + PARTITION BY equipment_id, source_recipe_id + )::int AS run_count, + ROW_NUMBER() OVER ( + PARTITION BY equipment_id, source_recipe_id + ORDER BY run_date DESC NULLS LAST, id DESC + ) AS recipe_rank + FROM visible + WHERE source_recipe_id IS NOT NULL + ) + SELECT + equipment_id, + source_tool_id, + source_recipe_id, + recipe_name, + source_hash, + recipe_timestamp, + file_sha256, + file_size_bytes, + run_count, + run_date AS latest_run_at, + source_run_id AS latest_source_run_id, + (id + {EQUIPMENT_RUN_ID_OFFSET}) AS latest_catalog_run_id, + project_id AS latest_project_id + FROM ranked + WHERE recipe_rank = 1 + ORDER BY recipe_timestamp DESC NULLS LAST, + source_recipe_id::bigint DESC + LIMIT %s OFFSET %s + """, + (normalized_equipment_id, safe_limit, safe_offset), + ) + rows = [dict(row) for row in cur.fetchall()] + conn.commit() + + recipes: list[dict[str, Any]] = [] + for row in rows: + try: + source_recipe_id: Any = int(row.get("source_recipe_id")) + except (TypeError, ValueError): + source_recipe_id = str(row.get("source_recipe_id") or "") + try: + file_size_bytes = int(row.get("file_size_bytes")) + except (TypeError, ValueError): + file_size_bytes = None + recipes.append( + { + **row, + "source_recipe_id": source_recipe_id, + "file_size_bytes": file_size_bytes, + "latest_run_at": _recipe_scalar(row.get("latest_run_at")), + "latest_catalog_run_id": int( + row.get("latest_catalog_run_id") + ), + "latest_source_run_id": ( + int(row["latest_source_run_id"]) + if row.get("latest_source_run_id") is not None + else None + ), + } + ) + return recipes + finally: + conn.close() + + +def get_equipment_recipe_pg( + *, + equipment_id: str, + source_recipe_id: int, + nanohub_user_id: Optional[str] = None, + is_admin: bool = False, +) -> Optional[Dict[str, Any]]: + """Fetch one encoded recipe from the newest visible run using it.""" + + normalized_equipment_id = str(equipment_id or "").strip() + if not normalized_equipment_id: + raise ValueError("equipment_id is required") + if int(source_recipe_id) < 1: + raise ValueError("source_recipe_id must be positive") + + conn = get_pg_superuser_connection() if is_admin else get_pg_connection( + nanohub_user_id + ) + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + f""" + SELECT + r.equipment_id, + r.source_tool_id, + r.source_run_id AS latest_source_run_id, + r.run_date AS latest_run_at, + r.project_id AS latest_project_id, + (r.id + {EQUIPMENT_RUN_ID_OFFSET}) AS latest_catalog_run_id, + r.raw_payload_json #> + '{{glance,recipe}}' AS recipe_json + FROM equipment_runs r + WHERE r.equipment_id = %s + AND COALESCE(NULLIF(r.source_system, ''), r.source) = 'glance' + AND NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,source_recipe_id}}', + '' + ) = %s + AND NULLIF( + r.raw_payload_json #>> + '{{glance,recipe,raw,recipefile}}', + '' + ) IS NOT NULL + ORDER BY r.run_date DESC NULLS LAST, r.id DESC + LIMIT 1 + """, + (normalized_equipment_id, str(int(source_recipe_id))), + ) + row = cur.fetchone() + conn.commit() + if not row: + return None + + record = dict(row) + recipe = record.pop("recipe_json", None) or {} + if not isinstance(recipe, dict): + return None + raw_recipe = recipe.get("raw") or {} + if not isinstance(raw_recipe, dict): + raw_recipe = {} + encoded_recipe = raw_recipe.get("recipefile") + if not isinstance(encoded_recipe, str) or not encoded_recipe.strip(): + return None + + return { + **record, + "source_recipe_id": int(source_recipe_id), + "name": str( + recipe.get("name") or raw_recipe.get("recipename") or "" + ), + "source_hash": recipe.get("source_hash") or raw_recipe.get("hash"), + "timestamp": _recipe_scalar( + recipe.get("timestamp") or raw_recipe.get("timestamp") + ), + "file_sha256": recipe.get("file_sha256"), + "file_size_bytes": recipe.get("file_size_bytes"), + "encoded_recipe": encoded_recipe, + "latest_run_at": _recipe_scalar(record.get("latest_run_at")), + "latest_catalog_run_id": int(record["latest_catalog_run_id"]), + "latest_source_run_id": ( + int(record["latest_source_run_id"]) + if record.get("latest_source_run_id") is not None + else None + ), + } + finally: + conn.close() + + def get_projects_list_pg(nanohub_user_id: Optional[str] = None, is_admin: bool = False) -> List[Dict[str, Any]]: """ Fetch projects from Postgres. diff --git a/api/glance_recipe.py b/api/glance_recipe.py new file mode 100644 index 0000000..775362a --- /dev/null +++ b/api/glance_recipe.py @@ -0,0 +1,281 @@ +"""Safe, deterministic decoding for GLANCE equipment recipe files. + +GLANCE stores ``recipes.recipefile`` as Base64-encoded XML. The native +viewer exports a transposed CSV: one column per recipe step and one row per +recipe parameter. This module reproduces that representation without +returning the encoded source file to browsers. +""" + +from __future__ import annotations + +import base64 +import binascii +import csv +import hashlib +import io +import re +import xml.etree.ElementTree as ET +from typing import Any + + +MAX_ENCODED_RECIPE_BYTES = 8 * 1024 * 1024 +MAX_DECODED_RECIPE_BYTES = 6 * 1024 * 1024 +MAX_RECIPE_STEPS = 1_000 +MAX_STEP_ATTRIBUTES = 2_000 +MAX_CELL_CHARACTERS = 250_000 + + +class GlanceRecipeDecodeError(ValueError): + """Raised when an encoded GLANCE recipe cannot be decoded safely.""" + + +ROW_ORDER: list[str] = [ + "Pressure", + "Type", + "Time", + "PIDLoopPressureControl", + "ZeroProcessMano", + "StepOverPercent", + "PressureControlMode", + "ThrottlePosition", + "Temperature1", + "Temperature2", + "Temperature3", + "Temperature4", + "Temperature5", + "UseCurrentTemps", + "WaitForTempCompliance", + "Description", + "EndBy", + "EndpointRecipe", + "OnEPNotFound", + "MinEPTime", + "FilmThickness", + "ElectrodePosition", + "PinsDownDelay", +] +for number in range(1, 17): + ROW_ORDER.extend((f"Gas{number}", f"Gas{number}Dump")) +for rf_name, matchbox_name in (("RF1", "MB1"), ("RF2", "MB2"), ("RF3", "MB3")): + ROW_ORDER.extend( + ( + f"{rf_name}Setpoint", + f"{rf_name}MaxRef", + f"{rf_name}MaxImp", + f"{rf_name}Waveform", + f"{rf_name}Pulse", + f"{rf_name}PulsePeriod", + f"{rf_name}PulseOnTime", + f"{matchbox_name}Load", + f"{matchbox_name}Tune", + f"{matchbox_name}MatchMode", + ) + ) +ROW_ORDER.extend(f"RfTableLine{number}" for number in range(10)) +ROW_ORDER.extend( + ( + "Restart", + "LoopDestination", + "LoopTerminationMode", + "LoopIterations", + "LoopTime", + "LoopEndpointRecipe", + "LoopOverPercent", + "HeliumMode", + "HeliumPressureSetpoint", + "HeliumFlowSetpoint", + "CheckSealMode", + "FixedDurationChuck", + "HeliumStepOverride", + "PreDechuckHePressure", + "PostDechuckHePressure", + "DechuckHeFlow", + "EChuckSetpoint", + "GateStartEnabled", + "GateEndEnabled", + "GateStartTime", + "GateEndTime", + "GateStartRelativeToStart", + "GateEndRelativeToStart", + "AbortRecipeName", + "AbortRecipeCategory", + "XYPlatformMove", + "XYPlatformXPosition", + "XYPlatformYPosition", + ) +) + +_TWO_FIELD_INDEX_ZERO = {f"RfTableLine{number}" for number in range(10)} +_MULTI_BOOLEAN_INDEX_ZERO = {"UseCurrentTemps"} +_MATCH_MODE_ATTRIBUTES = {"MB1MatchMode", "MB2MatchMode", "MB3MatchMode"} +_MATCH_MODE_LABELS = { + "0": "Auto", + "1": "Fixed", + "2": "Man", + "3": "Hold", + "4": "Preset", +} +_STEP_TAG = re.compile(r"^Step(\d+)$") + + +def _local_tag(tag: Any) -> str: + value = str(tag or "") + return value.rsplit("}", 1)[-1] + + +def _decode_value(attribute: str, raw_value: str, warnings: set[str]) -> str: + if len(raw_value) > MAX_CELL_CHARACTERS: + raise GlanceRecipeDecodeError( + f"Recipe attribute {attribute!r} exceeds the safe cell-size limit" + ) + + if attribute in _MATCH_MODE_ATTRIBUTES: + # The native exporter leaves an absent match-mode cell truly empty; + # other empty attributes are represented by one space below. + if raw_value == "": + return "" + if raw_value not in _MATCH_MODE_LABELS: + warnings.add( + f"Unknown {attribute} code {raw_value!r} was preserved as recorded." + ) + return _MATCH_MODE_LABELS.get(raw_value, raw_value) + + if raw_value == "": + # This matches the GLANCE viewer CSV rather than collapsing a recorded + # empty attribute into an absent CSV cell. + return " " + + if "," in raw_value: + fields = raw_value.split(",") + if attribute in _TWO_FIELD_INDEX_ZERO or attribute in _MULTI_BOOLEAN_INDEX_ZERO: + return fields[0] + if len(fields) >= 3: + return fields[2] + + return raw_value + + +def decode_glance_recipe(encoded_recipe: str) -> dict[str, Any]: + """Decode one Base64 GLANCE recipe into a viewer-compatible matrix. + + The parser rejects DTD/entity declarations and enforces explicit size, + step-count, and attribute-count bounds before returning any data. Unknown + attributes are appended after the viewer's known row order and reported as + warnings so future GLANCE schema additions cannot disappear silently. + """ + + if not isinstance(encoded_recipe, str) or not encoded_recipe.strip(): + raise GlanceRecipeDecodeError("The GLANCE recipe file is empty") + + normalized = encoded_recipe.strip() + if len(normalized) > MAX_ENCODED_RECIPE_BYTES: + raise GlanceRecipeDecodeError("The encoded GLANCE recipe exceeds the size limit") + + try: + xml_bytes = base64.b64decode(normalized, validate=True) + except (binascii.Error, ValueError) as exc: + raise GlanceRecipeDecodeError( + "The GLANCE recipe file is not valid Base64" + ) from exc + + if len(xml_bytes) > MAX_DECODED_RECIPE_BYTES: + raise GlanceRecipeDecodeError("The decoded GLANCE recipe exceeds the size limit") + lowered = xml_bytes.lower() + if b" MAX_STEP_ATTRIBUTES: + raise GlanceRecipeDecodeError( + f"Recipe step {number} exceeds the attribute-count limit" + ) + seen_numbers.add(number) + numbered_steps.append((number, element)) + + if not numbered_steps: + raise GlanceRecipeDecodeError("The XML does not contain any recipe steps") + if len(numbered_steps) > MAX_RECIPE_STEPS: + raise GlanceRecipeDecodeError("The recipe exceeds the step-count limit") + numbered_steps.sort(key=lambda item: item[0]) + + known_attributes = set(ROW_ORDER) + unknown_attributes: list[str] = [] + seen_unknown: set[str] = set() + for _, step in numbered_steps: + for attribute in step.attrib: + if attribute == "Name" or attribute in known_attributes or attribute in seen_unknown: + continue + seen_unknown.add(attribute) + unknown_attributes.append(attribute) + + warning_set: set[str] = set() + if unknown_attributes: + warning_set.add( + "Unknown recipe attributes were preserved after the standard GLANCE rows: " + + ", ".join(unknown_attributes) + ) + + steps = [ + { + "index": index, + "source_number": source_number, + "source_tag": _local_tag(element.tag), + "name": str(element.get("Name", "")), + "label": f"{index}.\n{element.get('Name', '')}", + } + for index, (source_number, element) in enumerate(numbered_steps, start=1) + ] + row_names = [*ROW_ORDER, *unknown_attributes] + rows = [] + for attribute in row_names: + rows.append( + { + "parameter": attribute, + "values": [ + _decode_value(attribute, element.get(attribute, ""), warning_set) + for _, element in numbered_steps + ], + } + ) + + return { + "steps": steps, + "rows": rows, + "warnings": sorted(warning_set), + "decoded_size_bytes": len(xml_bytes), + "file_sha256": hashlib.sha256(xml_bytes).hexdigest(), + } + + +def glance_recipe_csv(decoded_recipe: dict[str, Any]) -> str: + """Serialize a decoded recipe using the native GLANCE CSV conventions.""" + + steps = decoded_recipe.get("steps") or [] + rows = decoded_recipe.get("rows") or [] + output = io.StringIO(newline="") + header = [str(step.get("label") or "") for step in steps] + quoted_header = ",".join( + '"' + value.replace('"', '""') + '"' for value in header + ) + output.write("," + quoted_header + "\r\n") + writer = csv.writer(output, quoting=csv.QUOTE_ALL, lineterminator="\r\n") + for row in rows: + writer.writerow([row.get("parameter", ""), *(row.get("values") or [])]) + return output.getvalue() diff --git a/api/routers/dataset_v2.py b/api/routers/dataset_v2.py index 498dd57..c964027 100644 --- a/api/routers/dataset_v2.py +++ b/api/routers/dataset_v2.py @@ -10,7 +10,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query from fastapi import Request -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, Response from pydantic import BaseModel from pydantic import ValidationError @@ -20,16 +20,24 @@ create_project_pg, delete_project_pg, get_equipment_run_trace_pg, + get_equipment_recipe_pg, get_glance_ingestion_project_pg, get_projects_list_pg, get_run_detail_pg, get_runs_list_pg, + list_equipment_recipes_pg, list_project_members_pg, + list_recipe_equipment_pg, remove_project_member_pg, record_glance_ingestion_audit_pg, sync_equipment_run_traces_pg, sync_runs_pg, ) +from glance_recipe import ( + GlanceRecipeDecodeError, + decode_glance_recipe, + glance_recipe_csv, +) from domain_configs import load_domain_config, summarize_domain_config from metadata_pg import ( add_project_experiment_pg, @@ -777,7 +785,6 @@ async def export_ml_data( nanohub_user_id=platform_user.id, is_admin=is_admin, ) - from fastapi.responses import Response return Response( content=csv_content, media_type="text/csv", @@ -799,6 +806,133 @@ async def export_ml_data( raise HTTPException(status_code=500, detail=str(exc)) +def _decoded_recipe_record(record: dict) -> dict: + """Decode and integrity-check a database recipe without exposing Base64.""" + + encoded_recipe = record.get("encoded_recipe") + try: + decoded = decode_glance_recipe(encoded_recipe) + except GlanceRecipeDecodeError as exc: + raise HTTPException( + status_code=422, + detail=f"Stored GLANCE recipe could not be decoded: {exc}", + ) from exc + + declared_sha = str(record.get("file_sha256") or "").strip().lower() + if declared_sha and declared_sha != decoded["file_sha256"]: + raise HTTPException( + status_code=422, + detail="Stored GLANCE recipe failed its SHA-256 integrity check", + ) + declared_size = record.get("file_size_bytes") + if declared_size not in (None, ""): + try: + size_matches = int(declared_size) == decoded["decoded_size_bytes"] + except (TypeError, ValueError): + size_matches = False + if not size_matches: + raise HTTPException( + status_code=422, + detail="Stored GLANCE recipe failed its file-size integrity check", + ) + + public_record = { + key: value for key, value in record.items() if key != "encoded_recipe" + } + return {**public_record, **decoded} + + +@router.get("/recipes/equipment-options") +def recipe_equipment_options( + platform_user: PlatformUser = Depends(get_platform_user), +): + """List equipment for which the caller can see at least one recipe.""" + + return list_recipe_equipment_pg( + nanohub_user_id=platform_user.id, + is_admin=platform_user.role == "admin", + ) + + +@router.get("/recipes") +def equipment_recipes( + equipment_id: str = Query(min_length=1, max_length=255), + limit: int = Query(default=500, ge=1, le=1_000), + offset: int = Query(default=0, ge=0), + platform_user: PlatformUser = Depends(get_platform_user), +): + """List version-aware GLANCE recipe metadata for one equipment item.""" + + try: + return list_equipment_recipes_pg( + equipment_id=equipment_id, + nanohub_user_id=platform_user.id, + is_admin=platform_user.role == "admin", + limit=limit, + offset=offset, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/recipes/{source_recipe_id}/export") +def export_equipment_recipe( + source_recipe_id: int, + equipment_id: str = Query(min_length=1, max_length=255), + platform_user: PlatformUser = Depends(get_platform_user), +): + """Export one visible GLANCE recipe in native-viewer-compatible CSV.""" + + try: + record = get_equipment_recipe_pg( + equipment_id=equipment_id, + source_recipe_id=source_recipe_id, + nanohub_user_id=platform_user.id, + is_admin=platform_user.role == "admin", + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if record is None: + raise HTTPException(status_code=404, detail="Recipe not found") + decoded = _decoded_recipe_record(record) + content = glance_recipe_csv(decoded) + return Response( + content=content, + media_type="text/csv; charset=utf-8", + headers={ + "Content-Disposition": ( + "attachment; " + f"filename=glance_recipe_{int(source_recipe_id)}.csv" + ), + "X-Content-Type-Options": "nosniff", + }, + ) + + +@router.get("/recipes/{source_recipe_id}") +def equipment_recipe_detail( + source_recipe_id: int, + equipment_id: str = Query(min_length=1, max_length=255), + platform_user: PlatformUser = Depends(get_platform_user), +): + """Return one decoded recipe matrix under normal project visibility.""" + + try: + record = get_equipment_recipe_pg( + equipment_id=equipment_id, + source_recipe_id=source_recipe_id, + nanohub_user_id=platform_user.id, + is_admin=platform_user.role == "admin", + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if record is None: + # Do not distinguish an unknown recipe from one visible only through a + # private project the caller cannot access. + raise HTTPException(status_code=404, detail="Recipe not found") + return _decoded_recipe_record(record) + + @router.get("/runs/{run_id}") def run_detail( run_id: int, diff --git a/api/tests/test_glance_recipe.py b/api/tests/test_glance_recipe.py new file mode 100644 index 0000000..bd915e4 --- /dev/null +++ b/api/tests/test_glance_recipe.py @@ -0,0 +1,89 @@ +import base64 +import csv +import io +import sys +import unittest +from pathlib import Path + + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +from glance_recipe import ( # noqa: E402 + GlanceRecipeDecodeError, + decode_glance_recipe, + glance_recipe_csv, +) + + +def encoded(xml: str) -> str: + return base64.b64encode(xml.encode("utf-8")).decode("ascii") + + +class GlanceRecipeDecoderTests(unittest.TestCase): + def test_decodes_viewer_matrix_and_csv_conventions(self): + xml = ( + '' + '' + '' + '' + '' + ) + result = decode_glance_recipe(encoded(xml)) + + self.assertEqual([step["name"] for step in result["steps"]], ["< Initial >", "Etch"]) + rows = {row["parameter"]: row["values"] for row in result["rows"]} + self.assertEqual(rows["Pressure"], ["10", "20"]) + self.assertEqual(rows["Description"], [" ", "main"]) + self.assertEqual(rows["MB1MatchMode"], ["Auto", "Man"]) + self.assertEqual(rows["RfTableLine0"], ["5", "6"]) + self.assertEqual(rows["UseCurrentTemps"], ["True", "False"]) + self.assertEqual(result["warnings"], []) + + csv_text = glance_recipe_csv(result) + self.assertTrue(csv_text.startswith(',"1.\n< Initial >","2.\nEtch"\r\n')) + parsed = list(csv.reader(io.StringIO(csv_text, newline=""))) + self.assertEqual(parsed[0], ["", "1.\n< Initial >", "2.\nEtch"]) + self.assertEqual(len(parsed), len(result["rows"]) + 1) + + def test_unknown_attributes_are_preserved_and_reported(self): + xml = '' + result = decode_glance_recipe(encoded(xml)) + + self.assertEqual(result["rows"][-1], { + "parameter": "FutureSetting", + "values": ["enabled"], + }) + self.assertIn("FutureSetting", result["warnings"][0]) + + def test_unknown_match_mode_is_preserved_and_reported(self): + xml = '' + result = decode_glance_recipe(encoded(xml)) + rows = {row["parameter"]: row["values"] for row in result["rows"]} + self.assertEqual(rows["MB1MatchMode"], ["99"]) + self.assertIn("Unknown MB1MatchMode code", result["warnings"][0]) + + def test_rejects_invalid_base64(self): + with self.assertRaisesRegex(GlanceRecipeDecodeError, "valid Base64"): + decode_glance_recipe("not-base64!") + + def test_rejects_dtd_or_entity_declarations(self): + xml = ']>' + with self.assertRaisesRegex(GlanceRecipeDecodeError, "DTD"): + decode_glance_recipe(encoded(xml)) + + def test_rejects_duplicate_step_numbers(self): + xml = '' + with self.assertRaisesRegex(GlanceRecipeDecodeError, "duplicated"): + decode_glance_recipe(encoded(xml)) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_glance_recipe_routes.py b/api/tests/test_glance_recipe_routes.py new file mode 100644 index 0000000..0bfd10b --- /dev/null +++ b/api/tests/test_glance_recipe_routes.py @@ -0,0 +1,173 @@ +import base64 +import hashlib +import os +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +from fastapi.testclient import TestClient + + +os.environ.setdefault("DT_SYSTEM_TOKEN", "test-system-token") +os.environ.setdefault("INGESTION_TOKEN", "test-ingestion-token") +os.environ.setdefault("PG_PASS", "test-api-password") +os.environ.setdefault("PG_SUPER_PASS", "test-super-password") + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +import main # noqa: E402 +from glance_recipe import decode_glance_recipe, glance_recipe_csv # noqa: E402 +from routers import dataset_v2 # noqa: E402 + + +RESEARCHER_HEADERS = { + "X-System-Token": os.environ["DT_SYSTEM_TOKEN"], + "X-User-Id": "researcher-1", + "X-User-Email": "researcher@example.com", + "X-User-Name": "Researcher", + "X-User-Role": "researcher", + "X-User-Org": "Birck", +} + + +def recipe_record(): + xml_bytes = ( + '' + '' + ).encode("utf-8") + return { + "equipment_id": "equipment-a", + "source_tool_id": 3, + "source_recipe_id": 2964, + "name": "Recipe A", + "source_hash": "source-hash", + "timestamp": "2026-08-09T11:00:00", + "file_sha256": hashlib.sha256(xml_bytes).hexdigest(), + "file_size_bytes": len(xml_bytes), + "encoded_recipe": base64.b64encode(xml_bytes).decode("ascii"), + "latest_run_at": "2026-08-09T11:30:00", + "latest_source_run_id": 12213, + "latest_catalog_run_id": 5_000_002_265, + "latest_project_id": "private-project", + } + + +class GlanceRecipeRouteTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = TestClient(main.app) + + def test_equipment_options_use_authenticated_identity(self): + with patch.object( + dataset_v2, + "list_recipe_equipment_pg", + return_value=[ + { + "equipment_id": "equipment-a", + "equipment_name": "Equipment A", + "recipe_count": 2, + } + ], + ) as list_options: + response = self.client.get( + "/api/dataset/v2/recipes/equipment-options", + headers=RESEARCHER_HEADERS, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()[0]["recipe_count"], 2) + list_options.assert_called_once_with( + nanohub_user_id="researcher-1", + is_admin=False, + ) + + def test_recipe_list_is_equipment_scoped(self): + with patch.object( + dataset_v2, + "list_equipment_recipes_pg", + return_value=[], + ) as list_recipes: + response = self.client.get( + "/api/dataset/v2/recipes?equipment_id=equipment-a", + headers=RESEARCHER_HEADERS, + ) + + self.assertEqual(response.status_code, 200) + list_recipes.assert_called_once_with( + equipment_id="equipment-a", + nanohub_user_id="researcher-1", + is_admin=False, + limit=500, + offset=0, + ) + + def test_recipe_detail_decodes_but_does_not_expose_base64(self): + stored = recipe_record() + with patch.object( + dataset_v2, + "get_equipment_recipe_pg", + return_value=stored, + ): + response = self.client.get( + "/api/dataset/v2/recipes/2964?equipment_id=equipment-a", + headers=RESEARCHER_HEADERS, + ) + + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertEqual(payload["source_recipe_id"], 2964) + self.assertEqual([step["name"] for step in payload["steps"]], ["Start", "Etch"]) + self.assertNotIn("encoded_recipe", payload) + self.assertNotIn(stored["encoded_recipe"], response.text) + + def test_recipe_export_matches_decoder_csv(self): + stored = recipe_record() + expected = glance_recipe_csv(decode_glance_recipe(stored["encoded_recipe"])) + with patch.object( + dataset_v2, + "get_equipment_recipe_pg", + return_value=stored, + ): + response = self.client.get( + "/api/dataset/v2/recipes/2964/export?equipment_id=equipment-a", + headers=RESEARCHER_HEADERS, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, expected.encode("utf-8")) + self.assertIn("glance_recipe_2964.csv", response.headers["content-disposition"]) + + def test_private_or_missing_recipe_is_indistinguishable(self): + with patch.object( + dataset_v2, + "get_equipment_recipe_pg", + return_value=None, + ): + response = self.client.get( + "/api/dataset/v2/recipes/2964?equipment_id=equipment-a", + headers=RESEARCHER_HEADERS, + ) + self.assertEqual(response.status_code, 404) + self.assertEqual(response.json()["detail"], "Recipe not found") + + def test_integrity_mismatch_is_rejected(self): + stored = recipe_record() + stored["file_sha256"] = "0" * 64 + with patch.object( + dataset_v2, + "get_equipment_recipe_pg", + return_value=stored, + ): + response = self.client.get( + "/api/dataset/v2/recipes/2964?equipment_id=equipment-a", + headers=RESEARCHER_HEADERS, + ) + self.assertEqual(response.status_code, 422) + self.assertIn("SHA-256", response.json()["detail"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/azure/Dockerfile.glance b/azure/Dockerfile.glance index 94aec89..e131908 100644 --- a/azure/Dockerfile.glance +++ b/azure/Dockerfile.glance @@ -14,4 +14,4 @@ COPY scripts ./scripts USER 10001 -CMD ["python", "scripts/run_glance_poll.py"] +CMD ["python", "-m", "scripts.run_glance_poll"] diff --git a/geddes/k8s/02-api.yaml b/geddes/k8s/02-api.yaml index 3eba378..66b45aa 100644 --- a/geddes/k8s/02-api.yaml +++ b/geddes/k8s/02-api.yaml @@ -18,7 +18,7 @@ spec: containers: - name: dt-api # Notice we are using the 'sdx' namespace on the registry now - image: geddes-registry.rcac.purdue.edu/sdx/dt-api:db3-pilot-retrain-loocv-20260714@sha256:52e16ffbbdf71ba87470d8dbf686e6304955501d5b74eaf4c41ba981e14239dc + image: geddes-registry.rcac.purdue.edu/sdx/dt-api:recipe-371da73@sha256:a9cd36d051b2aada3dd5e7f843cbaac83998711dbd205f66935357e3a0bde483 imagePullPolicy: Always resources: requests: diff --git a/geddes/k8s/03-web.yaml b/geddes/k8s/03-web.yaml index 7117b10..2aac4f7 100644 --- a/geddes/k8s/03-web.yaml +++ b/geddes/k8s/03-web.yaml @@ -24,7 +24,7 @@ spec: containers: - name: dt-web # Notice we are using the 'sdx' namespace on the registry now - image: geddes-registry.rcac.purdue.edu/sdx/dt-web:v53 + image: geddes-registry.rcac.purdue.edu/sdx/dt-web:recipe-371da73@sha256:77251decd1d31729309749331e0801fd071cac315d2a9612ad7ba104930b12f9 imagePullPolicy: Always resources: requests: diff --git a/geddes/k8s/glance-ingestion-cronjob.yaml b/geddes/k8s/glance-ingestion-cronjob.yaml index 351960d..c01c75e 100644 --- a/geddes/k8s/glance-ingestion-cronjob.yaml +++ b/geddes/k8s/glance-ingestion-cronjob.yaml @@ -47,6 +47,8 @@ spec: - name: connector image: geddes-registry.rcac.purdue.edu/sdx/dt-glance-ingestion:REPLACE_WITH_DIGEST imagePullPolicy: IfNotPresent + workingDir: /app/azure + command: ["python", "-m", "scripts.run_glance_poll"] env: - name: GLANCE_SOURCE_MODE value: postgres diff --git a/web/app/data/catalog/[id]/page.tsx b/web/app/data/catalog/[id]/page.tsx index a5eac0a..af70098 100644 --- a/web/app/data/catalog/[id]/page.tsx +++ b/web/app/data/catalog/[id]/page.tsx @@ -18,6 +18,7 @@ import { Globe, Activity, AlertTriangle, + BookOpen, } from "lucide-react"; const FEATURE_DISPLAY: Record = { @@ -115,9 +116,12 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string ); } - const isGlanceTrace = run.source === "glance_db3"; + const isGlanceTrace = run.source === "glance" || run.source === "glance_db3"; const badges: { label: string; className: string }[] = []; - if (isGlanceTrace) badges.push({ label: "Imported GLANCE Trace", className: "text-violet-500 bg-violet-500/10" }); + if (isGlanceTrace) badges.push({ + label: run.source === "glance" ? "Live GLANCE Trace" : "Imported GLANCE Trace", + className: "text-violet-500 bg-violet-500/10", + }); if (run.is_calibration) badges.push({ label: "Calibration", className: "text-blue-500 bg-blue-500/10" }); if (run.is_outlier) badges.push({ label: "Outlier", className: "text-amber-500 bg-amber-500/10" }); if (!isGlanceTrace && badges.length === 0) badges.push({ label: "Clean Validated", className: "text-emerald-500 bg-emerald-500/10" }); @@ -132,6 +136,8 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string run.outputs && Object.keys(run.outputs).length > 0 ? run.outputs : null; const isEtcher = run.equipment_id === "etcher" || run.avg_etch_rate !== null; const glanceProvenance = run.raw_payload?.glance; + const liveRecipe = glanceProvenance?.recipe; + const liveRecipeId = liveRecipe?.source_recipe_id; const sourceFile = run.raw_payload?.source_file; return ( @@ -196,6 +202,28 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string )} + {run.source === "glance" && + run.equipment_id && + liveRecipeId !== null && + liveRecipeId !== undefined && ( +
+
+

+ Recipe {String(liveRecipeId)} · {liveRecipe?.name || "Unnamed recipe"} +

+

+ View this equipment-specific recipe version using its GLANCE ID, hash, and timestamp. +

+
+ + Open recipe viewer + +
+ )} + {/* Project & ownership */}

diff --git a/web/app/data/catalog/page.tsx b/web/app/data/catalog/page.tsx index e07b024..6b02457 100644 --- a/web/app/data/catalog/page.tsx +++ b/web/app/data/catalog/page.tsx @@ -14,7 +14,8 @@ import { Loader2, } from "lucide-react"; -const isGlanceTrace = (run: V2Run) => run.source === "glance_db3"; +const isGlanceTrace = (run: V2Run) => + run.source === "glance" || run.source === "glance_db3"; const isCleanValidated = (run: V2Run) => !isGlanceTrace(run) && !run.is_outlier && !run.is_calibration; @@ -188,7 +189,10 @@ export default function DataCatalogPage() { const glanceTrace = isGlanceTrace(r); const badges: { label: string; className: string }[] = []; if (glanceTrace) { - badges.push({ label: "Imported GLANCE Trace", className: "text-violet-500 bg-violet-500/10" }); + badges.push({ + label: r.source === "glance" ? "Live GLANCE Trace" : "Imported GLANCE Trace", + className: "text-violet-500 bg-violet-500/10", + }); } if (r.is_calibration) { badges.push({ label: "Calibration", className: "text-blue-500 bg-blue-500/10" }); diff --git a/web/app/data/recipes/page.tsx b/web/app/data/recipes/page.tsx new file mode 100644 index 0000000..3a4c830 --- /dev/null +++ b/web/app/data/recipes/page.tsx @@ -0,0 +1,17 @@ +import { Suspense } from "react"; + +import { RecipeViewer } from "@/components/recipe-viewer"; + +export default function RecipeViewerPage() { + return ( + + Loading recipe viewer… +

+ } + > + + + ); +} diff --git a/web/components/cross-run-trace-analysis.tsx b/web/components/cross-run-trace-analysis.tsx index b8b7897..b9784ad 100644 --- a/web/components/cross-run-trace-analysis.tsx +++ b/web/components/cross-run-trace-analysis.tsx @@ -2,7 +2,7 @@ import dynamic from "next/dynamic"; import { useEffect, useMemo, useRef, useState } from "react"; -import { Activity, AlertCircle, Loader2 } from "lucide-react"; +import { Activity, AlertCircle, Download, Loader2 } from "lucide-react"; import { ErrorBoundary } from "@/components/ErrorBoundary"; import { @@ -17,6 +17,13 @@ import { formatTrendMetric, trimRepeatedEdgeZeros, } from "@/lib/trace-plot"; +import { + buildTraceCsv, + downloadCsv, + safeDownloadStem, + traceAxisLabel, + traceParameterUnit, +} from "@/lib/trace-export"; const Plot = dynamic(() => import("react-plotly.js"), { ssr: false }); const MAX_SELECTED_RUNS = 10; @@ -278,6 +285,18 @@ export function CrossRunTraceAnalysis() { if (next.length === 0) setWarning(null); } + function exportRawTraceCsv() { + if (!selected || visibleTraces.length === 0) return; + const csv = buildTraceCsv(visibleTraces, selected); + const runSuffix = `${visibleTraces.length}-run${ + visibleTraces.length === 1 ? "" : "s" + }`; + downloadCsv( + csv, + `${safeDownloadStem(`trace-${selected.key}-${runSuffix}`)}.csv`, + ); + } + return (
@@ -374,6 +393,15 @@ export function CrossRunTraceAnalysis() { /> Apply trendline +
{selectedIds.length > 0 && warning && ( @@ -398,6 +426,25 @@ export function CrossRunTraceAnalysis() {

)} + {selected && visibleTraces.length > 0 && ( +
+ + Y-axis: {traceAxisLabel(selected)} + + + X-axis:{" "} + {axisMode === "relative" + ? "Seconds from each run's first sample" + : "GLANCE source timestamp"} + + {!traceParameterUnit(selected) && ( + + GLANCE did not provide a unit for this parameter. + + )} +
+ )} + {selected && visibleTraces.length ? ( diff --git a/web/components/recipe-viewer.tsx b/web/components/recipe-viewer.tsx new file mode 100644 index 0000000..085e61d --- /dev/null +++ b/web/components/recipe-viewer.tsx @@ -0,0 +1,416 @@ +"use client"; + +import Link from "next/link"; +import { useSearchParams } from "next/navigation"; +import { useEffect, useMemo, useState } from "react"; +import { + AlertCircle, + BookOpen, + Download, + ExternalLink, + FileCode2, + Loader2, + Search, +} from "lucide-react"; + +import { + getV2EquipmentRecipe, + getV2EquipmentRecipeExportUrl, + getV2EquipmentRecipes, + getV2RecipeEquipmentOptions, + type V2EquipmentRecipeDetail, + type V2EquipmentRecipeSummary, + type V2RecipeEquipmentOption, +} from "@/lib/api-client"; +import { cn } from "@/lib/utils"; + +function sourceTimestamp(value: string | null | undefined): string { + if (!value) return "Not recorded"; + const normalized = value.replace("T", " "); + return /(?:z|[+-]\d{2}:?\d{2})$/i.test(value) + ? new Date(value).toLocaleString() + : `${normalized} (source time)`; +} + +function fileSize(value: number | null | undefined): string { + if (typeof value !== "number" || !Number.isFinite(value)) return "Unknown"; + if (value < 1_024) return `${value.toLocaleString()} bytes`; + return `${(value / 1_024).toFixed(1)} KiB`; +} + +function shortHash(value: string | null | undefined): string { + if (!value) return "No source hash"; + return value.length > 18 ? `${value.slice(0, 10)}…${value.slice(-6)}` : value; +} + +export function RecipeViewer() { + const searchParams = useSearchParams(); + const requestedEquipment = searchParams.get("equipment")?.trim() ?? ""; + const requestedRecipe = Number(searchParams.get("recipe")); + const [equipment, setEquipment] = useState([]); + const [selectedEquipment, setSelectedEquipment] = useState(""); + const [recipes, setRecipes] = useState([]); + const [selectedRecipeId, setSelectedRecipeId] = useState(null); + const [detail, setDetail] = useState(null); + const [search, setSearch] = useState(""); + const [loadingEquipment, setLoadingEquipment] = useState(true); + const [loadingRecipes, setLoadingRecipes] = useState(false); + const [loadingDetail, setLoadingDetail] = useState(false); + const [downloading, setDownloading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + void getV2RecipeEquipmentOptions().then((records) => { + if (cancelled) return; + const available = records ?? []; + setEquipment(available); + const requestedIsVisible = available.some( + (item) => item.equipment_id === requestedEquipment, + ); + setSelectedEquipment( + requestedIsVisible ? requestedEquipment : available[0]?.equipment_id ?? "", + ); + setLoadingEquipment(false); + }); + return () => { + cancelled = true; + }; + }, [requestedEquipment]); + + useEffect(() => { + let cancelled = false; + setRecipes([]); + setDetail(null); + setSelectedRecipeId(null); + setError(null); + if (!selectedEquipment) return; + setLoadingRecipes(true); + void getV2EquipmentRecipes(selectedEquipment).then((records) => { + if (cancelled) return; + const available = records ?? []; + setRecipes(available); + const requestedIsVisible = + Number.isInteger(requestedRecipe) && + requestedRecipe > 0 && + available.some((item) => item.source_recipe_id === requestedRecipe); + setSelectedRecipeId( + requestedIsVisible + ? requestedRecipe + : available[0]?.source_recipe_id ?? null, + ); + setLoadingRecipes(false); + }); + return () => { + cancelled = true; + }; + }, [requestedRecipe, selectedEquipment]); + + useEffect(() => { + let cancelled = false; + setDetail(null); + setError(null); + if (!selectedEquipment || selectedRecipeId === null) return; + setLoadingDetail(true); + void getV2EquipmentRecipe(selectedEquipment, selectedRecipeId).then( + (record) => { + if (cancelled) return; + setDetail(record); + if (!record) { + setError( + "This recipe could not be decoded or is no longer visible to your account.", + ); + } + setLoadingDetail(false); + }, + ); + return () => { + cancelled = true; + }; + }, [selectedEquipment, selectedRecipeId]); + + const filteredRecipes = useMemo(() => { + const query = search.trim().toLowerCase(); + if (!query) return recipes; + return recipes.filter((recipe) => + [ + recipe.recipe_name, + recipe.source_recipe_id, + recipe.source_hash, + recipe.recipe_timestamp, + ].some((value) => String(value ?? "").toLowerCase().includes(query)), + ); + }, [recipes, search]); + + async function exportRecipeCsv() { + if (!detail || downloading) return; + setDownloading(true); + setError(null); + try { + const response = await fetch( + getV2EquipmentRecipeExportUrl( + detail.equipment_id, + detail.source_recipe_id, + ), + { cache: "no-store" }, + ); + if (!response.ok) { + throw new Error(`Recipe export failed with status ${response.status}`); + } + const blob = await response.blob(); + const contentDisposition = response.headers.get("content-disposition") ?? ""; + const filename = + contentDisposition.match(/filename=([^;]+)/i)?.[1]?.replaceAll('"', "") ?? + `glance_recipe_${detail.source_recipe_id}.csv`; + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = filename; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); + } catch (reason) { + setError( + reason instanceof Error ? reason.message : "Recipe export failed.", + ); + } finally { + setDownloading(false); + } + } + + return ( +
+
+
+ +

GLANCE Recipe Viewer

+
+

+ Inspect versioned recipes for one equipment item at a time. Recipe IDs, + source hashes, and timestamps remain visible so identical names are never + treated as the same version. +

+
+ +
+ +

+ Only recipes linked to projects you can already access are listed. +

+
+ + {loadingEquipment ? ( +
+ Loading recipe equipment… +
+ ) : equipment.length === 0 ? ( +
+ +

No visible GLANCE recipes

+

+ Ask a project PI to grant access to a project containing live GLANCE + runs. +

+
+ ) : ( +
+
+
+ + setSearch(event.target.value)} + placeholder="Search name, ID, hash, or date" + className="w-full rounded-md border border-[hsl(var(--border))] bg-[hsl(var(--background))] py-2 pl-9 pr-3 text-sm" + /> +
+ {loadingRecipes ? ( +

+ Loading recipes… +

+ ) : ( +
+ {filteredRecipes.map((recipe) => ( + + ))} + {filteredRecipes.length === 0 && ( +

+ No recipe matches this search. +

+ )} +
+ )} +
+ +
+ {loadingDetail ? ( +
+ Safely decoding recipe… +
+ ) : error && !detail ? ( +
+

+ {error} +

+
+ ) : detail ? ( +
+ {error && ( +

+ {error} +

+ )} +
+
+

+ Recipe {detail.source_recipe_id} +

+

+ {detail.name || `Recipe ${detail.source_recipe_id}`} +

+

+ {detail.steps.length.toLocaleString()} steps ·{" "} + {detail.rows.length.toLocaleString()} parameters ·{" "} + {fileSize(detail.decoded_size_bytes)} decoded +

+
+ +
+ +
+
+
Version timestamp
+
+ {sourceTimestamp(detail.timestamp)} +
+
+
+
GLANCE source hash
+
{detail.source_hash || "—"}
+
+
+
Decoded SHA-256
+
{detail.file_sha256 || "—"}
+
+
+
Latest visible run
+
+ + GLANCE {detail.latest_source_run_id ?? "—"} + + +
+
+
+ + {detail.warnings.length > 0 && ( +
+ {detail.warnings.map((warning) => ( +

{warning}

+ ))} +
+ )} + +
+ + + + + {detail.steps.map((step) => ( + + ))} + + + + {detail.rows.map((row) => ( + + + {row.values.map((value, index) => ( + + ))} + + ))} + +
+ Parameter + + + Step {step.index} + + {step.name || "Unnamed"} +
+ {row.parameter} + + {value === " " ? — : value} +
+
+
+ ) : ( +
+ Select a recipe to inspect it. +
+ )} +
+
+ )} +
+ ); +} diff --git a/web/components/run-trace-explorer.tsx b/web/components/run-trace-explorer.tsx index 70f42af..423e8be 100644 --- a/web/components/run-trace-explorer.tsx +++ b/web/components/run-trace-explorer.tsx @@ -2,7 +2,13 @@ import dynamic from "next/dynamic"; import { useEffect, useMemo, useRef, useState } from "react"; -import { Activity, AlertCircle, Clock3, Loader2 } from "lucide-react"; +import { + Activity, + AlertCircle, + Clock3, + Download, + Loader2, +} from "lucide-react"; import { ErrorBoundary } from "@/components/ErrorBoundary"; import { @@ -17,6 +23,13 @@ import { formatTrendMetric, trimRepeatedEdgeZeros, } from "@/lib/trace-plot"; +import { + buildTraceCsv, + downloadCsv, + safeDownloadStem, + traceAxisLabel, + traceParameterUnit, +} from "@/lib/trace-export"; import { cn } from "@/lib/utils"; const Plot = dynamic(() => import("react-plotly.js"), { ssr: false }); @@ -422,6 +435,18 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { }); } + function exportRawTraceCsv() { + if (!parameter || displayedTraces.length === 0) return; + const csv = buildTraceCsv(displayedTraces, parameter); + const runSuffix = `${displayedTraces.length}-run${ + displayedTraces.length === 1 ? "" : "s" + }`; + downloadCsv( + csv, + `${safeDownloadStem(`trace-${parameter.key}-${runSuffix}`)}.csv`, + ); + } + if (initialLoading) { return (
@@ -535,6 +560,15 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { /> Apply trendline + Plot-only; source and stored values remain unchanged. @@ -634,6 +668,24 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { : ""}

)} + {parameter && ( +
+ + Y-axis: {traceAxisLabel(parameter)} + + + X-axis:{" "} + {axisMode === "relative" + ? "Seconds from each run's first sample" + : "GLANCE source timestamp"} + + {!traceParameterUnit(parameter) && ( + + GLANCE did not provide a unit for this parameter. + + )} +
+ )}
{parameter ? ( @@ -643,7 +695,7 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { layout={{ autosize: true, height: 480, - margin: { l: 70, r: 25, t: 115, b: 105 }, + margin: { l: 95, r: 25, t: 115, b: 120 }, paper_bgcolor: "rgba(0,0,0,0)", plot_bgcolor: "rgba(0,0,0,0)", font: { color: "#94a3b8", family: "Inter, system-ui, sans-serif" }, @@ -655,17 +707,21 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { axisMode === "relative" ? "Seconds from first sample in each run" : "GLANCE source timestamp", + standoff: 18, + font: { color: "#cbd5e1", size: 13 }, }, + automargin: true, type: axisMode === "relative" ? "linear" : "date", gridcolor: "rgba(148,163,184,0.12)", zeroline: false, }, yaxis: { title: { - text: parameter.unit - ? `${parameter.name} (${parameter.unit})` - : parameter.name, + text: traceAxisLabel(parameter), + standoff: 18, + font: { color: "#cbd5e1", size: 13 }, }, + automargin: true, gridcolor: "rgba(148,163,184,0.12)", zeroline: false, }, @@ -706,6 +762,11 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { displayModeBar: true, displaylogo: false, scrollZoom: true, + toImageButtonOptions: { + format: "png", + filename: safeDownloadStem(`trace-${parameter.key}`), + scale: 2, + }, }} useResizeHandler style={{ width: "100%" }} diff --git a/web/components/sidebar.tsx b/web/components/sidebar.tsx index 4ac1973..6c03127 100644 --- a/web/components/sidebar.tsx +++ b/web/components/sidebar.tsx @@ -30,6 +30,7 @@ import { Globe, GitBranch, Sparkles, + BookOpen, } from "lucide-react"; import { useTheme } from "next-themes"; import { useState } from "react"; @@ -58,6 +59,7 @@ const NAV_SECTIONS = [ { id: "ingestion", label: "Ingestion", href: "/data/monitor", icon: Activity }, { id: "catalog", label: "Catalog", href: "/data/catalog", icon: Search }, { id: "analysis", label: "Trace Analysis", href: "/data/analysis", icon: Activity }, + { id: "recipes", label: "Recipe Viewer", href: "/data/recipes", icon: BookOpen }, ], }, { diff --git a/web/lib/api-client.ts b/web/lib/api-client.ts index 9e2d943..f2fb2a3 100644 --- a/web/lib/api-client.ts +++ b/web/lib/api-client.ts @@ -188,6 +188,26 @@ export interface V2Run { run_status?: string | null; sample_count?: number | null; run_id?: number | string | null; + run?: { + idruns?: number | string | null; + idtools?: number | string | null; + idrecipes?: number | string | null; + lotname?: string | null; + materialname?: string | null; + starttime?: string | null; + endtime?: string | null; + status?: string | null; + [key: string]: unknown; + }; + recipe?: { + source_recipe_id?: number | string | null; + name?: string | null; + source_hash?: string | null; + timestamp?: string | null; + file_sha256?: string | null; + file_size_bytes?: number | null; + [key: string]: unknown; + }; }; source_file?: { type?: string | null; @@ -563,6 +583,91 @@ export function getV2RunTrace( ); } +export interface V2RecipeEquipmentOption { + equipment_id: string; + equipment_name: string; + recipe_count: number; +} + +export interface V2EquipmentRecipeSummary { + equipment_id: string; + source_tool_id: number | string | null; + source_recipe_id: number; + recipe_name: string | null; + source_hash: string | null; + recipe_timestamp: string | null; + file_sha256: string | null; + file_size_bytes: number | null; + run_count: number; + latest_run_at: string | null; + latest_source_run_id: number | null; + latest_catalog_run_id: number; + latest_project_id: string | null; +} + +export interface V2DecodedRecipeStep { + index: number; + source_number: number; + source_tag: string; + name: string; + label: string; +} + +export interface V2DecodedRecipeRow { + parameter: string; + values: string[]; +} + +export interface V2EquipmentRecipeDetail { + equipment_id: string; + source_tool_id: number | string | null; + source_recipe_id: number; + name: string; + source_hash: string | null; + timestamp: string | null; + file_sha256: string | null; + file_size_bytes: number | null; + latest_run_at: string | null; + latest_source_run_id: number | null; + latest_catalog_run_id: number; + latest_project_id: string | null; + steps: V2DecodedRecipeStep[]; + rows: V2DecodedRecipeRow[]; + warnings: string[]; + decoded_size_bytes: number; +} + +export function getV2RecipeEquipmentOptions() { + return apiFetch( + "/dataset/v2/recipes/equipment-options", + ); +} + +export function getV2EquipmentRecipes(equipmentId: string) { + const query = new URLSearchParams({ equipment_id: equipmentId }); + return apiFetch( + `/dataset/v2/recipes?${query.toString()}`, + ); +} + +export function getV2EquipmentRecipe( + equipmentId: string, + sourceRecipeId: number, +) { + const query = new URLSearchParams({ equipment_id: equipmentId }); + return apiFetch( + `/dataset/v2/recipes/${encodeURIComponent(String(sourceRecipeId))}?${query.toString()}`, + ); +} + +export function getV2EquipmentRecipeExportUrl( + equipmentId: string, + sourceRecipeId: number, +): string { + const query = new URLSearchParams({ equipment_id: equipmentId }); + return `${API_URL}/dataset/v2/recipes/${encodeURIComponent(String(sourceRecipeId))}/export?${query.toString()}`; +} + export interface ParityData { source: string; model_type?: string; diff --git a/web/lib/auth-context.tsx b/web/lib/auth-context.tsx index 7ebb1ba..1c5cfc3 100644 --- a/web/lib/auth-context.tsx +++ b/web/lib/auth-context.tsx @@ -82,7 +82,7 @@ export const ROLE_PERMISSIONS: Record< visibleNavSections: ["main", "data", "ml", "admin"], visibleNavItems: [ "dashboard", "assistant", "equipment", "projects", "experiments", "processes", "templates", "library", "public_library", "optimize", "analytics", - "samples", "upload", "ingestion", "catalog", "analysis", "parity", "importance", + "samples", "upload", "ingestion", "catalog", "analysis", "recipes", "parity", "importance", "convergence", "proposals", "users", "reviews", "execution_queue", "settings", ], }, @@ -98,7 +98,7 @@ export const ROLE_PERMISSIONS: Record< visibleNavSections: ["main", "data", "ml", "admin"], visibleNavItems: [ "dashboard", "assistant", "equipment", "projects", "experiments", "processes", "templates", "library", "public_library", "optimize", "analytics", - "samples", "upload", "ingestion", "catalog", "analysis", "parity", "importance", + "samples", "upload", "ingestion", "catalog", "analysis", "recipes", "parity", "importance", "convergence", "proposals", "settings", ], }, @@ -114,7 +114,7 @@ export const ROLE_PERMISSIONS: Record< visibleNavSections: ["main", "data", "ml", "admin"], visibleNavItems: [ "dashboard", "assistant", "equipment", "projects", "experiments", "processes", "templates", "library", "public_library", "analytics", - "samples", "upload", "ingestion", "catalog", "analysis", "execution_queue", "settings", + "samples", "upload", "ingestion", "catalog", "analysis", "recipes", "execution_queue", "settings", ], }, researcher: { @@ -129,7 +129,7 @@ export const ROLE_PERMISSIONS: Record< visibleNavSections: ["main", "data", "ml", "admin"], visibleNavItems: [ "dashboard", "assistant", "equipment", "projects", "experiments", "processes", "templates", "library", "public_library", "optimize", "analytics", - "samples", "upload", "catalog", "analysis", "parity", "importance", "convergence", + "samples", "upload", "catalog", "analysis", "recipes", "parity", "importance", "convergence", "proposals", "settings", ], }, diff --git a/web/lib/trace-export.ts b/web/lib/trace-export.ts new file mode 100644 index 0000000..18a72fa --- /dev/null +++ b/web/lib/trace-export.ts @@ -0,0 +1,105 @@ +import type { + V2RunTrace, + V2RunTraceParameter, +} from "@/lib/api-client"; + +const TRACE_CSV_COLUMNS = [ + "catalog_run_id", + "glance_run_id", + "lot_name", + "source_system", + "source_tool_id", + "parameter_key", + "source_parameter_id", + "parameter_name", + "unit", + "sample_record_id", + "source_timestamp", + "relative_seconds", + "recorded_value", +] as const; + +function csvCell(value: unknown): string { + if (value === null || value === undefined) return ""; + const text = + typeof value === "boolean" ? (value ? "true" : "false") : String(value); + return `"${text.replaceAll('"', '""')}"`; +} + +export function traceParameterUnit( + parameter: V2RunTraceParameter | null | undefined, +): string { + return parameter?.unit?.trim() || ""; +} + +export function traceAxisLabel( + parameter: V2RunTraceParameter | null | undefined, +): string { + if (!parameter) return "Recorded value (unit not provided)"; + const name = + parameter.registered && parameter.registered_name + ? parameter.registered_name + : parameter.name || parameter.key; + const unit = traceParameterUnit(parameter); + return unit ? `${name} (${unit})` : `${name} (unit not provided)`; +} + +export function buildTraceCsv( + traces: V2RunTrace[], + selectedParameter: V2RunTraceParameter, +): string { + const rows: unknown[][] = [Array.from(TRACE_CSV_COLUMNS)]; + + traces.forEach((trace) => { + const traceParameter = + trace.parameters.find( + (parameter) => parameter.key === selectedParameter.key, + ) ?? selectedParameter; + const parameterName = + traceParameter.registered && traceParameter.registered_name + ? traceParameter.registered_name + : traceParameter.name; + const unit = traceParameterUnit(traceParameter); + + trace.samples.forEach((sample) => { + rows.push([ + trace.run_id, + trace.source_run_id, + trace.lot_name, + trace.source_system ?? "", + trace.source_tool_id ?? sample.source_tool_id, + selectedParameter.key, + traceParameter.source_parameter_id ?? "", + parameterName, + unit, + sample.sample_record_id, + sample.timestamp, + sample.relative_seconds, + sample.values[selectedParameter.key], + ]); + }); + }); + + return `${rows.map((row) => row.map(csvCell).join(",")).join("\r\n")}\r\n`; +} + +export function safeDownloadStem(value: string): string { + const normalized = value + .trim() + .replace(/[^a-zA-Z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80); + return normalized || "trace"; +} + +export function downloadCsv(content: string, filename: string): void { + const blob = new Blob([content], { type: "text/csv;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = filename; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); +}