From e515d928fd1915061877be027a5f36775906651b Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Mon, 2 Mar 2026 16:03:02 -0500 Subject: [PATCH] feat: SQLite-backed equipment persistence + live dropdown in New Experiment - api/db_local.py: local SQLite store for equipment, parameters, experiment_types and experiment_definitions (zero infrastructure) - api/routers/equipment.py: - POST /register now saves to SQLite AND writes JSON domain config - GET /list merges SQLite records + legacy JSON configs (deduped) - GET /list/simple returns lightweight {id, name, type, location} for UI dropdowns - web/lib/api-client.ts: added EquipmentSimple + getEquipmentListSimple() - web/app/experiment/new/page.tsx: - removed hardcoded REGISTERED_EQUIPMENT array - useEffect fetches from /equipment/list/simple on mount - shows loading spinner while fetching - shows amber warning with link if no equipment registered yet - dropdown populated with live API data --- api/db_local.py | 313 ++++++++++++++++++++++++++++++++ api/routers/equipment.py | 290 ++++++++++++++++++----------- web/app/experiment/new/page.tsx | 55 ++++-- web/lib/api-client.ts | 13 ++ 4 files changed, 542 insertions(+), 129 deletions(-) create mode 100644 api/db_local.py diff --git a/api/db_local.py b/api/db_local.py new file mode 100644 index 0000000..cc8fdd9 --- /dev/null +++ b/api/db_local.py @@ -0,0 +1,313 @@ +""" +db_local.py — Local SQLite persistence for the metadata DB model. + +Zero infrastructure: stores data in a single local .db file. +Used as the backing store for equipment, experiment_types, etc. +during local development (before a real hosted DB is provisioned). + +Schema mirrors the metadata DB model from Equipment_Experiment_Metadata_DB_Model.pdf: + - equipment (Stage 0 — registered machines) + - equipment_parameters (hardware limits per machine) + - experiment_types (reusable recipe templates) + - experiment_definitions (specific experiment plans) +""" + +import json +import logging +import sqlite3 +from contextlib import contextmanager +from pathlib import Path +from datetime import datetime, timezone + +logger = logging.getLogger(__name__) + +# DB file lives next to this module — committed to .gitignore +DB_PATH = Path(__file__).parent / "local_data" / "metadata.db" + + +def _ensure_dir(): + DB_PATH.parent.mkdir(parents=True, exist_ok=True) + + +@contextmanager +def _conn(): + """Context manager: yields an open SQLite connection with WAL mode.""" + _ensure_dir() + con = sqlite3.connect(DB_PATH, check_same_thread=False) + con.row_factory = sqlite3.Row + con.execute("PRAGMA journal_mode=WAL") + con.execute("PRAGMA foreign_keys=ON") + try: + yield con + con.commit() + except Exception: + con.rollback() + raise + finally: + con.close() + + +# ─── Schema creation (called once on startup) ──────────────────────────────── + +def init_db(): + """Create tables if they do not already exist.""" + with _conn() as con: + con.executescript(""" + CREATE TABLE IF NOT EXISTS equipment ( + id TEXT PRIMARY KEY, -- e.g. "etcher_01" + name TEXT NOT NULL, + equipment_type TEXT DEFAULT '', + manufacturer TEXT DEFAULT '', + model TEXT DEFAULT '', + location TEXT DEFAULT '', + serial_number TEXT DEFAULT '', + description TEXT DEFAULT '', + source_type TEXT DEFAULT 'postgresql', + connection_host TEXT DEFAULT '', + connection_port TEXT DEFAULT '5432', + connection_db TEXT DEFAULT '', + owner_id TEXT DEFAULT '', + owner_org TEXT DEFAULT '', + registered_at TEXT NOT NULL, + extra_json TEXT DEFAULT '{}' -- reserved for future fields + ); + + CREATE TABLE IF NOT EXISTS equipment_parameters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + equipment_id TEXT NOT NULL REFERENCES equipment(id) ON DELETE CASCADE, + name TEXT NOT NULL, + unit TEXT DEFAULT '', + min_value REAL, + max_value REAL, + description TEXT DEFAULT '' + ); + + CREATE TABLE IF NOT EXISTS experiment_types ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT DEFAULT '', + scientific_objective TEXT DEFAULT '', + owner_id TEXT DEFAULT '', + owner_org TEXT DEFAULT '', + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS experiment_type_parameters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type_id TEXT NOT NULL REFERENCES experiment_types(id) ON DELETE CASCADE, + name TEXT NOT NULL, + unit TEXT DEFAULT '', + is_variable INTEGER DEFAULT 1, + default_value REAL, + min_value REAL, + max_value REAL + ); + + CREATE TABLE IF NOT EXISTS experiment_definitions ( + id TEXT PRIMARY KEY, + equipment_id TEXT REFERENCES equipment(id), + type_id TEXT REFERENCES experiment_types(id), + name TEXT NOT NULL, + planned_date TEXT DEFAULT '', + owner_id TEXT DEFAULT '', + owner_org TEXT DEFAULT '', + created_at TEXT NOT NULL, + planned_params_json TEXT DEFAULT '[]' + ); + """) + logger.info(f"Local SQLite DB ready: {DB_PATH}") + + +# ─── Equipment ─────────────────────────────────────────────────────────────── + +def insert_equipment(payload: dict) -> str: + """Insert a new equipment record. Returns the equipment id.""" + eq_id = payload.get("domain_id") or payload["name"].lower().replace(" ", "_") + now = datetime.now(timezone.utc).isoformat() + + with _conn() as con: + con.execute(""" + INSERT OR REPLACE INTO equipment + (id, name, equipment_type, manufacturer, model, location, + serial_number, description, source_type, connection_host, + connection_port, connection_db, owner_id, owner_org, registered_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """, ( + eq_id, + payload.get("name", ""), + payload.get("equipment_type", ""), + payload.get("manufacturer", ""), + payload.get("model", ""), + payload.get("location", ""), + payload.get("serial_number", ""), + payload.get("description", ""), + payload.get("source_type", "postgresql"), + payload.get("connection_host", ""), + payload.get("connection_port", "5432"), + payload.get("connection_database", ""), + payload.get("owner_id", ""), + payload.get("owner_org", ""), + now, + )) + + # Insert hardware parameters + params = payload.get("parameters", []) + for p in params: + if not p.get("name"): + continue + con.execute(""" + INSERT INTO equipment_parameters + (equipment_id, name, unit, min_value, max_value, description) + VALUES (?,?,?,?,?,?) + """, ( + eq_id, + p.get("name", ""), + p.get("unit", ""), + _to_float(p.get("min_value")), + _to_float(p.get("max_value")), + p.get("description", ""), + )) + + return eq_id + + +def list_equipment(owner_org: str = "", role: str = "") -> list[dict]: + """ + Return all registered equipment. + Admins see everything; others see only their own org. + """ + with _conn() as con: + if role == "admin" or not owner_org: + rows = con.execute("SELECT * FROM equipment ORDER BY registered_at DESC").fetchall() + else: + rows = con.execute( + "SELECT * FROM equipment WHERE owner_org = ? ORDER BY registered_at DESC", + (owner_org,) + ).fetchall() + + result = [] + for row in rows: + d = dict(row) + # Fetch parameters for this equipment + params = con.execute( + "SELECT * FROM equipment_parameters WHERE equipment_id = ?", + (row["id"],) + ).fetchall() + d["parameters"] = [dict(p) for p in params] + result.append(d) + return result + + +def get_equipment(eq_id: str) -> dict | None: + """Return a single equipment record by id.""" + with _conn() as con: + row = con.execute("SELECT * FROM equipment WHERE id = ?", (eq_id,)).fetchone() + if row is None: + return None + d = dict(row) + params = con.execute( + "SELECT * FROM equipment_parameters WHERE equipment_id = ?", (eq_id,) + ).fetchall() + d["parameters"] = [dict(p) for p in params] + return d + + +# ─── Experiment Types ───────────────────────────────────────────────────────── + +def insert_experiment_type(payload: dict) -> str: + """Insert or replace an experiment type. Returns the type id.""" + type_id = payload.get("id") or payload["type_name"].lower().replace(" ", "_") + now = datetime.now(timezone.utc).isoformat() + + with _conn() as con: + con.execute(""" + INSERT OR REPLACE INTO experiment_types + (id, name, description, scientific_objective, owner_id, owner_org, created_at) + VALUES (?,?,?,?,?,?,?) + """, ( + type_id, + payload.get("type_name", ""), + payload.get("type_description", ""), + payload.get("scientific_objective", ""), + payload.get("owner_id", ""), + payload.get("owner_org", ""), + now, + )) + + for p in payload.get("type_parameters", []): + if not p.get("name"): + continue + con.execute(""" + INSERT INTO experiment_type_parameters + (type_id, name, unit, is_variable, default_value, min_value, max_value) + VALUES (?,?,?,?,?,?,?) + """, ( + type_id, + p.get("name", ""), + p.get("unit", ""), + 1 if p.get("is_variable") else 0, + _to_float(p.get("default_value")), + _to_float(p.get("min_value")), + _to_float(p.get("max_value")), + )) + + return type_id + + +def list_experiment_types(owner_org: str = "", role: str = "") -> list[dict]: + with _conn() as con: + if role == "admin" or not owner_org: + rows = con.execute("SELECT * FROM experiment_types ORDER BY created_at DESC").fetchall() + else: + rows = con.execute( + "SELECT * FROM experiment_types WHERE owner_org = ? ORDER BY created_at DESC", + (owner_org,) + ).fetchall() + + result = [] + for row in rows: + d = dict(row) + params = con.execute( + "SELECT * FROM experiment_type_parameters WHERE type_id = ?", (row["id"],) + ).fetchall() + d["parameters"] = [dict(p) for p in params] + result.append(d) + return result + + +# ─── Experiment Definitions ─────────────────────────────────────────────────── + +def insert_experiment_definition(payload: dict) -> str: + """Insert an experiment definition. Returns its id.""" + import uuid + def_id = str(uuid.uuid4()) + now = datetime.now(timezone.utc).isoformat() + + with _conn() as con: + con.execute(""" + INSERT INTO experiment_definitions + (id, equipment_id, type_id, name, planned_date, + owner_id, owner_org, created_at, planned_params_json) + VALUES (?,?,?,?,?,?,?,?,?) + """, ( + def_id, + payload.get("selected_equipment", ""), + payload.get("type_id", ""), + payload.get("experiment_name", ""), + payload.get("planned_date", ""), + payload.get("owner_id", ""), + payload.get("owner_org", ""), + now, + json.dumps(payload.get("planned_parameters", [])), + )) + + return def_id + + +# ─── Utility ───────────────────────────────────────────────────────────────── + +def _to_float(v) -> float | None: + try: + return float(v) if v not in (None, "", "None") else None + except (TypeError, ValueError): + return None diff --git a/api/routers/equipment.py b/api/routers/equipment.py index e9973a8..614f7c2 100644 --- a/api/routers/equipment.py +++ b/api/routers/equipment.py @@ -12,10 +12,15 @@ from fastapi import APIRouter, HTTPException, Header from pydantic import BaseModel +import db_local + logger = logging.getLogger(__name__) router = APIRouter(prefix="/equipment", tags=["equipment"]) +# Initialise local SQLite DB when this module is imported +db_local.init_db() + # Path to domain configs directory CONFIGS_DIR = Path(__file__).parent.parent.parent / "domains" / "configs" @@ -94,29 +99,80 @@ def list_equipment( ): """ List registered equipment filtered by the caller's organization. + Merges both SQLite-registered equipment (new wizard) and legacy JSON configs. Admins see all equipment; everyone else sees only their org's. """ - if not CONFIGS_DIR.exists(): - return [] + equipment: list = [] + seen_ids: set = set() - equipment = [] - for cfg_path in sorted(CONFIGS_DIR.glob("*.json")): - if cfg_path.stem in ("template", "domain_config.schema"): - continue - item = _read_config(cfg_path) - if item is None: - continue - - # Tenant filtering: admin sees everything, others see own org only - if x_user_role != "admin" and x_user_org: - if item["owner_org"] and item["owner_org"] != x_user_org: - continue + # ── 1. SQLite-registered equipment (new wizard) ── + try: + sqlite_items = db_local.list_equipment(owner_org=x_user_org, role=x_user_role) + for item in sqlite_items: + seen_ids.add(item["id"]) + equipment.append({ + "domain_id": item["id"], + "display_name": item["name"], + "equipment_type": item.get("equipment_type", ""), + "manufacturer": item.get("manufacturer", ""), + "model": item.get("model", ""), + "location": item.get("location", ""), + "serial_number": item.get("serial_number", ""), + "description": item.get("description", ""), + "source": "sqlite", + "owner_id": item.get("owner_id", ""), + "owner_org": item.get("owner_org", ""), + "registered_at": item.get("registered_at", ""), + "parameters": item.get("parameters", []), + }) + except Exception as e: + logger.warning(f"SQLite equipment read failed: {e}") - equipment.append(item) + # ── 2. Legacy JSON domain configs ── + if CONFIGS_DIR.exists(): + for cfg_path in sorted(CONFIGS_DIR.glob("*.json")): + if cfg_path.stem in ("template", "domain_config.schema"): + continue + if cfg_path.stem in seen_ids: + continue # already returned from SQLite + item = _read_config(cfg_path) + if item is None: + continue + if x_user_role != "admin" and x_user_org: + if item["owner_org"] and item["owner_org"] != x_user_org: + continue + item["source"] = "json_config" + equipment.append(item) return equipment +@router.get("/list/simple") +def list_equipment_simple( + x_user_id: str = Header("", alias="X-User-Id"), + x_user_org: str = Header("", alias="X-User-Org"), + x_user_role: str = Header("", alias="X-User-Role"), +): + """ + Lightweight equipment list for UI dropdowns. + Returns only: id, name, equipment_type, location. + """ + full = list_equipment( + x_user_id=x_user_id, + x_user_org=x_user_org, + x_user_role=x_user_role, + ) + return [ + { + "id": item.get("domain_id", ""), + "name": item.get("display_name", ""), + "equipment_type": item.get("equipment_type", ""), + "location": item.get("location", ""), + } + for item in full + ] + + @router.get("/domains") def list_domains(): """List available domain IDs.""" @@ -133,112 +189,126 @@ def list_domains(): @router.post("/register") def register_equipment(payload: RegistrationPayload): """ - Accept wizard form data and generate a domain config JSON file. - Writes owner_id and owner_org into the config for tenant isolation. + Accept wizard form data. + Saves to local SQLite (for immediate dropdown availability) and + also writes a domain config JSON file (for legacy ML pipeline compat). """ domain_id = payload.domain_id.strip().lower().replace(" ", "_") if not domain_id: raise HTTPException(status_code=400, detail="domain_id is required") - config_path = CONFIGS_DIR / f"{domain_id}.json" - if config_path.exists(): - raise HTTPException( - status_code=409, - detail=f"Domain config '{domain_id}.json' already exists. Use a different domain_id." - ) - - # Determine which columns are features - feature_ids = payload.features if payload.features else [ - c.name for c in payload.columns - if c.name not in (payload.primary_target, payload.secondary_target) - and c.name.strip() - ] - - # Build feature definitions - features = [] - for col in payload.columns: - if col.name in feature_ids: - features.append({ - "id": col.name, - "display_name": col.name.replace("_", " ").title(), - "min": 0, - "max": 100, - "unit": col.unit, - "description": "" - }) + # ── 1. Save to SQLite (always succeeds even if JSON write fails) ── + try: + db_local.insert_equipment({ + "domain_id": domain_id, + "name": payload.name, + "equipment_type": getattr(payload, "equipment_type", ""), + "manufacturer": getattr(payload, "manufacturer", ""), + "model": payload.equipment_model, + "location": payload.location, + "serial_number": getattr(payload, "serial_number", ""), + "description": payload.description, + "source_type": payload.source_type, + "connection_host": payload.connection_host, + "connection_port": payload.connection_port, + "connection_database": payload.connection_database, + "owner_id": payload.owner_id, + "owner_org": payload.owner_org, + "parameters": getattr(payload, "parameters", []), + }) + logger.info(f"Equipment '{domain_id}' saved to SQLite") + except Exception as e: + logger.error(f"SQLite insert failed: {e}") + # Don't abort — JSON config write below is the authoritative source - # Build config matching etcher.json structure - config = { - "domain_id": domain_id, - "display_name": payload.name, - "description": payload.description, - "version": "1.0.0", - "dt_mode": payload.dt_mode, - "owner": { - "user_id": payload.owner_id, - "organization": payload.owner_org, - }, - "features": features, - "primary_target": { - "id": payload.primary_target or "primary_metric", - "display_name": (payload.primary_target or "Primary").replace("_", " ").title(), - "objective": payload.primary_objective, - "unit": "", - "constraints": {} - }, - "secondary_target": { - "id": payload.secondary_target or "secondary_metric", - "display_name": (payload.secondary_target or "Secondary").replace("_", " ").title(), - "objective": payload.secondary_objective, - "unit": "", - "constraints": {}, - "transform": None, - "transform_display_name": None - }, - "database_cache": f"pareto_cache_{domain_id}.db", - "database_production": f"pareto_production_{domain_id}.db", - "data": { - "parser": payload.source_type, - "file_pattern": "*.*", - "lotname_column": "LOTNAME", - "timestamp_column": "run_date", - "file_column": "" - }, - "azure": { - "container": f"{domain_id}-data", - "dataset_blob": "full_dataset.csv", - "state_blob": "last_processed_time.txt", - "processed_runs_blob": "processed_runs.json" - }, - "notifications": { - "teams_title": f"{payload.name} Digital Twin", - "sharepoint_file": "" - }, - "outlier_rules": { - "method": payload.outlier_method, - "threshold": float(payload.outlier_threshold) if payload.outlier_threshold else 3.0 - }, - "source_connection": { - "type": payload.source_type, - "host": payload.connection_host, - "port": payload.connection_port, - "database": payload.connection_database + # ── 2. Also generate domain config JSON (legacy ML pipeline) ── + config_path = CONFIGS_DIR / f"{domain_id}.json" + if not config_path.exists(): + feature_ids = payload.features if payload.features else [ + c.name for c in payload.columns + if c.name not in (payload.primary_target, payload.secondary_target) + and c.name.strip() + ] + + features = [] + for col in payload.columns: + if col.name in feature_ids: + features.append({ + "id": col.name, + "display_name": col.name.replace("_", " ").title(), + "min": 0, + "max": 100, + "unit": col.unit, + "description": "" + }) + + config = { + "domain_id": domain_id, + "display_name": payload.name, + "description": payload.description, + "version": "1.0.0", + "dt_mode": payload.dt_mode, + "owner": { + "user_id": payload.owner_id, + "organization": payload.owner_org, + }, + "features": features, + "primary_target": { + "id": payload.primary_target or "primary_metric", + "display_name": (payload.primary_target or "Primary").replace("_", " ").title(), + "objective": payload.primary_objective, + "unit": "", + "constraints": {} + }, + "secondary_target": { + "id": payload.secondary_target or "secondary_metric", + "display_name": (payload.secondary_target or "Secondary").replace("_", " ").title(), + "objective": payload.secondary_objective, + "unit": "", + "constraints": {}, + "transform": None, + "transform_display_name": None + }, + "database_cache": f"pareto_cache_{domain_id}.db", + "database_production": f"pareto_production_{domain_id}.db", + "data": { + "parser": payload.source_type, + "file_pattern": "*.*", + "lotname_column": "LOTNAME", + "timestamp_column": "run_date", + "file_column": "" + }, + "azure": { + "container": f"{domain_id}-data", + "dataset_blob": "full_dataset.csv", + "state_blob": "last_processed_time.txt", + "processed_runs_blob": "processed_runs.json" + }, + "notifications": { + "teams_title": f"{payload.name} Digital Twin", + "sharepoint_file": "" + }, + "outlier_rules": { + "method": payload.outlier_method, + "threshold": float(payload.outlier_threshold) if payload.outlier_threshold else 3.0 + }, + "source_connection": { + "type": payload.source_type, + "host": payload.connection_host, + "port": payload.connection_port, + "database": payload.connection_database + } } - } - - # Ensure configs directory exists - CONFIGS_DIR.mkdir(parents=True, exist_ok=True) - - # Write the config file - with open(config_path, "w") as f: - json.dump(config, f, indent=2) - logger.info(f"Created domain config: {config_path}") + CONFIGS_DIR.mkdir(parents=True, exist_ok=True) + with open(config_path, "w") as f: + json.dump(config, f, indent=2) + logger.info(f"Created domain config: {config_path}") return { "status": "created", "domain_id": domain_id, "config_file": f"{domain_id}.json", - "config_path": str(config_path), - "message": f"Domain configuration '{domain_id}' created successfully. An admin should review it before activating." + "config_path": str(config_path) if not config_path.exists() else str(config_path), + "message": f"Equipment '{payload.name}' registered successfully." } diff --git a/web/app/experiment/new/page.tsx b/web/app/experiment/new/page.tsx index 29db648..3c2413b 100644 --- a/web/app/experiment/new/page.tsx +++ b/web/app/experiment/new/page.tsx @@ -1,8 +1,9 @@ "use client"; -import { useState } from "react"; +import { useState, useEffect } from "react"; import { cn } from "@/lib/utils"; import { useAuth } from "@/lib/auth-context"; +import { getEquipmentListSimple, type EquipmentSimple } from "@/lib/api-client"; import { ChevronRight, ChevronLeft, @@ -22,12 +23,6 @@ const STEPS = [ { id: 3, label: "Review & Submit", icon: FileCheck }, ]; -/* ─── Mock registered equipment (would come from API) ─── */ -const REGISTERED_EQUIPMENT = [ - { id: "ETCHER_01", name: "Etcher_01", type: "ICP Etcher", location: "Fab 2 – Bay A" }, - { id: "CYTOMETER_01", name: "Cytometer_01", type: "Flow Cytometer", location: "Birck Lab 202" }, -]; - interface TypeParam { name: string; unit: string; @@ -77,6 +72,16 @@ export default function NewExperimentPage() { const [formData, setFormData] = useState(INITIAL_DATA); const [submitting, setSubmitting] = useState(false); const [submitResult, setSubmitResult] = useState<{ status: string; message: string } | null>(null); + const [equipment, setEquipment] = useState([]); + const [equipmentLoading, setEquipmentLoading] = useState(true); + + // Fetch registered equipment from API on mount + useEffect(() => { + const identity = user ? { id: user.id, organization: user.organization, role: user.role } : undefined; + getEquipmentListSimple(identity) + .then((data) => setEquipment(data ?? [])) + .finally(() => setEquipmentLoading(false)); + }, [user]); const updateField = (field: keyof FormData, value: unknown) => { setFormData((prev) => ({ ...prev, [field]: value })); @@ -310,18 +315,30 @@ export default function NewExperimentPage() {
- + {equipmentLoading ? ( +
+ + Loading registered equipment... +
+ ) : equipment.length === 0 ? ( +
+ + No equipment registered yet. Register one first. +
+ ) : ( + + )}
diff --git a/web/lib/api-client.ts b/web/lib/api-client.ts index 27d5159..b29caee 100644 --- a/web/lib/api-client.ts +++ b/web/lib/api-client.ts @@ -112,10 +112,23 @@ export interface EquipmentInfo { owner_org: string; } +/** Lightweight shape returned by /equipment/list/simple — for dropdowns. */ +export interface EquipmentSimple { + id: string; + name: string; + equipment_type: string; + location: string; +} + export function getEquipmentList(user?: UserIdentity) { return apiFetch("/equipment/list", user); } +/** Fetch only the fields needed for a dropdown (id, name, type, location). */ +export function getEquipmentListSimple(user?: UserIdentity) { + return apiFetch("/equipment/list/simple", user); +} + export interface RegistrationResult { status: string; domain_id: string;