diff --git a/api/data_loader_pg.py b/api/data_loader_pg.py index bb0b11e..d3682a3 100644 --- a/api/data_loader_pg.py +++ b/api/data_loader_pg.py @@ -117,6 +117,7 @@ def _run_base_select(*, include_raw_payload: bool = False) -> str: r.is_outlier, r.is_calibration_recipe AS is_calibration, r.outlier_type, + r.recipe_name, r.avg_etch_rate, r.range_etch_rate, r.range_nm, @@ -301,6 +302,25 @@ def ensure_etcher_run_file_refs_pg() -> None: conn.close() +def ensure_etcher_recipe_name_column_pg() -> None: + """Add recipe_name to etcher_runs if the column does not yet exist. + + Idempotent — safe to call on every startup. + """ + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + ALTER TABLE etcher_runs + ADD COLUMN IF NOT EXISTS recipe_name VARCHAR(255) DEFAULT '' + """ + ) + conn.commit() + finally: + conn.close() + + def ensure_equipment_runs_pg() -> None: """Create the generic equipment_runs table, RLS policy, and grants. @@ -1456,6 +1476,7 @@ def sync_runs_pg(runs_data: List[Dict[str, Any]]): bool(row.get("is_outlier", False)), bool(row.get("is_calibration_recipe", False)), str(row.get("outlier_type", "")), + str(row.get("recipe_name", "")), float(row.get("AvgEtchRate")) if row.get("AvgEtchRate") is not None else None, float(row.get("RangeEtchRate")) if row.get("RangeEtchRate") is not None else None, row.get("Range_nm"), @@ -1501,6 +1522,7 @@ def sync_runs_pg(runs_data: List[Dict[str, Any]]): insert_query = """ INSERT INTO etcher_runs ( idruns, project_id, execution_request_id, lotname, run_date, is_outlier, is_calibration_recipe, outlier_type, + recipe_name, avg_etch_rate, range_etch_rate, range_nm, etch_avgo2flow, etch_avg_rf1_pow, etch_avg_rf2_pow, etch_avgpres, etch_avgcf4flow, raw_payload_json @@ -1513,6 +1535,7 @@ def sync_runs_pg(runs_data: List[Dict[str, Any]]): is_outlier = EXCLUDED.is_outlier, is_calibration_recipe = EXCLUDED.is_calibration_recipe, outlier_type = EXCLUDED.outlier_type, + recipe_name = EXCLUDED.recipe_name, avg_etch_rate = EXCLUDED.avg_etch_rate, range_etch_rate = EXCLUDED.range_etch_rate, range_nm = EXCLUDED.range_nm, diff --git a/api/main.py b/api/main.py index f71f096..318d629 100644 --- a/api/main.py +++ b/api/main.py @@ -142,6 +142,7 @@ def _run_startup_migrations() -> None: from data_loader_pg import get_pg_superuser_connection from data_loader_pg import ensure_etcher_run_file_refs_pg from data_loader_pg import ensure_equipment_runs_pg + from data_loader_pg import ensure_etcher_recipe_name_column_pg from metadata_pg import ( ensure_experiment_proposal_generation_columns_pg, ensure_experiment_type_reuse_columns_pg, @@ -168,6 +169,7 @@ def _run_startup_migrations() -> None: ensure_etcher_run_file_refs_pg() ensure_equipment_runs_pg() + ensure_etcher_recipe_name_column_pg() ensure_experiment_proposal_generation_columns_pg() ensure_experiment_type_reuse_columns_pg() ensure_experiment_type_versioning_columns_pg() diff --git a/api/routers/upload.py b/api/routers/upload.py index d1ffbd2..f9f938a 100644 --- a/api/routers/upload.py +++ b/api/routers/upload.py @@ -136,6 +136,7 @@ def resolve_uploads_dir() -> Path: "is_calibration_recipe": ("is_calibration_recipe", "is_calibration", "calibration"), "outlier_type": ("outlier_type", "outlier_reason"), "execution_request_id": ("execution_request_id", "request_id"), + "recipe_name": ("recipe_name", "RecipeName", "recipe"), } REQUIRED_SYNC_COLUMNS = ( @@ -554,6 +555,7 @@ def _csv_upload_to_sync_rows( _upload_value_ci(raw_row, resolve, "is_calibration_recipe") ), "outlier_type": str(_upload_value_ci(raw_row, resolve, "outlier_type") or ""), + "recipe_name": str(_upload_value_ci(raw_row, resolve, "recipe_name") or ""), "execution_request_id": str(_upload_value_ci(raw_row, resolve, "execution_request_id") or ""), "manual_upload_id": upload_id, "manual_upload_filename": upload_record["filename"], @@ -714,6 +716,11 @@ def add(name: Any, *, role: str = "") -> None: for column_name in derive_expected_columns(config): add(column_name) + # Append recipe_name metadata column after the legacy fallback so that the + # `len(columns) <= 2` gate still fires for configs with no declared + # features/parameters/outputs. + add("recipe_name", role="metadata") + return columns, value_types diff --git a/api/scripts/add_recipe_name.sql b/api/scripts/add_recipe_name.sql new file mode 100644 index 0000000..7767169 --- /dev/null +++ b/api/scripts/add_recipe_name.sql @@ -0,0 +1,7 @@ +-- Migration: add recipe_name column to etcher_runs +-- This column stores the Glance recipe identifier (e.g. "O2 clean-Engineering", +-- "Si etch") so operators no longer need to strip it during data cleaning. +-- The column is optional; existing rows default to empty string. + +ALTER TABLE etcher_runs +ADD COLUMN IF NOT EXISTS recipe_name VARCHAR(255) DEFAULT ''; diff --git a/api/scripts/init_db.sql b/api/scripts/init_db.sql index dc704e0..1c3862d 100644 --- a/api/scripts/init_db.sql +++ b/api/scripts/init_db.sql @@ -57,6 +57,7 @@ CREATE TABLE IF NOT EXISTS etcher_runs ( is_outlier BOOLEAN DEFAULT false, is_calibration_recipe BOOLEAN DEFAULT false, outlier_type TEXT, + recipe_name VARCHAR(255) DEFAULT '', -- Targets avg_etch_rate DOUBLE PRECISION, diff --git a/azure/process_etcher_data/__init__.py b/azure/process_etcher_data/__init__.py index 5fd822f..77ab7cc 100644 --- a/azure/process_etcher_data/__init__.py +++ b/azure/process_etcher_data/__init__.py @@ -424,8 +424,36 @@ def match_lot_and_file(lot, base): # Calibration drift baseline (from thesis example) CALIBRATION_BASELINE_IDRUN = 1063 -# Recipe cache -RECIPE_CACHE = {} +# Recipe caches +RECIPE_CACHE = {} # recipe_id -> parsed setpoint dict (or None) +RECIPE_ROW_CACHE = {} # recipe_id -> raw /recipes row (or None) + +def get_recipe_row(recipe_id): + """Fetch and cache the full Glance recipes row for a given idrecipes. + + A single ``select=*`` call provides both the ``recipefile`` blob (used for + setpoint parsing) and the human-readable name columns, so name and + setpoint lookups share one recipe-table request instead of each issuing + their own (which would fetch the large recipe XML twice per recipe). + Returns the row dict, or None if the recipe is missing or the fetch fails. + """ + if recipe_id is None: + return None + if recipe_id in RECIPE_ROW_CACHE: + return RECIPE_ROW_CACHE[recipe_id] + + row = None + try: + recipes = api_get_all("/recipes", {"idrecipes": f"eq.{recipe_id}", "select": "*"}) + if recipes: + row = recipes[0] + else: + logging.warning(f"No recipe found for idrecipes={recipe_id}") + except Exception as e: + logging.warning(f"Failed to fetch recipe row for idrecipes={recipe_id}: {e}") + + RECIPE_ROW_CACHE[recipe_id] = row + return row def get_recipe_setpoints(recipe_id): """ @@ -439,15 +467,13 @@ def get_recipe_setpoints(recipe_id): return RECIPE_CACHE[recipe_id] try: - # Get recipe file from PostgREST - recipes = api_get_all("/recipes", {"idrecipes": f"eq.{recipe_id}", "select": "idrecipes,recipefile"}) + # Reuse the shared recipe-row cache so we don't re-fetch the recipe file + recipe_data = get_recipe_row(recipe_id) - if not recipes: - logging.warning(f"No recipe found for idrecipes={recipe_id}") + if not recipe_data: RECIPE_CACHE[recipe_id] = None return None - recipe_data = recipes[0] recipe_file = recipe_data.get('recipefile') if not recipe_file: @@ -556,6 +582,59 @@ def check_comma_separated(dframe): RECIPE_CACHE[recipe_id] = None return None +# Recipe name cache (idrecipes -> human-readable recipe name) +RECIPE_NAME_CACHE = {} + +# Ordered list of columns in the Glance `recipes` table that may hold the +# human-readable recipe name (e.g. "O2 clean-Engineering", "Si etch"). The +# exact column varies across Logger schema revisions, so we probe in priority +# order and use the first non-empty value we find. +RECIPE_NAME_COLUMN_CANDIDATES = ( + "name", + "recipename", + "recipe_name", + "recipe", + "title", + "description", +) + +def _extract_recipe_name(recipe_row): + """Pick the most likely human-readable name from a Glance recipes row.""" + for field in RECIPE_NAME_COLUMN_CANDIDATES: + value = recipe_row.get(field) + if value not in (None, ""): + return str(value).strip() + return "" + +def get_recipe_name(recipe_id): + """Fetch the human-readable recipe name for a Glance idrecipes value. + + Results are cached per recipe_id. Returns '' when the recipe or a + recognizable name column cannot be found, so live ingestion degrades + gracefully (matching the empty-string default used elsewhere) rather than + failing when recipe metadata is unavailable. + """ + if recipe_id is None: + return "" + if recipe_id in RECIPE_NAME_CACHE: + return RECIPE_NAME_CACHE[recipe_id] + + name = "" + # Reuse the shared recipe-row cache; this avoids a second /recipes fetch + # (and re-downloading the large recipefile blob) when setpoints are also + # resolved for the same recipe. + recipe_row = get_recipe_row(recipe_id) + if recipe_row: + name = _extract_recipe_name(recipe_row) + if not name: + logging.warning( + f"Recipe {recipe_id} has no recognizable name column " + f"(checked {RECIPE_NAME_COLUMN_CANDIDATES})" + ) + + RECIPE_NAME_CACHE[recipe_id] = name + return name + def detect_spike_within_5pct(series, setpoint, t_start=None, tolerance_factor=0.05): """ Return first timestamp (>= t_start if given) where value is within ±5% of setpoint. @@ -791,7 +870,11 @@ def compute_all_step_averages(vals_df, param_ids, runs_df=None): # Get recipe ID for this run recipe_id = recipe_lookup.get(rid) - + + # Resolve the human-readable recipe name from Glance so it can be + # carried into final_df and persisted alongside the run. + recipe_name = get_recipe_name(recipe_id) + # Try thesis method (recipe setpoint-based) first setpoint_dict = None bounds = None @@ -866,6 +949,7 @@ def compute_all_step_averages(vals_df, param_ids, runs_df=None): row = { 'idruns': rid, 'LOTNAME': lotname, + 'recipe_name': recipe_name, 'run_start_time': start_time, 'run_end_time': end_time, 'is_outlier': is_outlier, diff --git a/geddes/k8s/05-postgres.yaml b/geddes/k8s/05-postgres.yaml index 9b512ae..1481a62 100644 --- a/geddes/k8s/05-postgres.yaml +++ b/geddes/k8s/05-postgres.yaml @@ -67,6 +67,7 @@ data: is_outlier BOOLEAN DEFAULT false, is_calibration_recipe BOOLEAN DEFAULT false, outlier_type TEXT, + recipe_name VARCHAR(255) DEFAULT '', avg_etch_rate DOUBLE PRECISION, range_etch_rate DOUBLE PRECISION, range_nm DOUBLE PRECISION, diff --git a/geddes/k8s/postgres/schema_v2.sql b/geddes/k8s/postgres/schema_v2.sql index 23ea44a..f7a9517 100644 --- a/geddes/k8s/postgres/schema_v2.sql +++ b/geddes/k8s/postgres/schema_v2.sql @@ -40,6 +40,7 @@ CREATE TABLE IF NOT EXISTS etcher_runs ( is_outlier BOOLEAN DEFAULT false, is_calibration_recipe BOOLEAN DEFAULT false, outlier_type TEXT, + recipe_name VARCHAR(255) DEFAULT '', avg_etch_rate DOUBLE PRECISION, range_etch_rate DOUBLE PRECISION, range_nm DOUBLE PRECISION,