From 56fff4945485b8136eabc66f88f7739b72e0867e Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Tue, 28 Jul 2026 22:11:58 -0400 Subject: [PATCH 1/3] Add production GLANCE ingestion foundation --- .gitignore | 2 + ...ON_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md | 517 ++++++ api/data_loader_pg.py | 804 ++++++++- api/glance_ingestion.py | 483 ++++++ api/main.py | 6 + api/model_registry.py | 37 + api/routers/dataset_v2.py | 209 +++ api/routers/ml.py | 64 + api/scripts/add_equipment_runs.sql | 111 +- .../add_production_glance_ingestion.sql | 115 ++ api/scripts/run_ml_shadow_evaluation.py | 55 + api/shadow_evaluation.py | 527 ++++++ api/tests/test_production_glance_ingestion.py | 419 +++++ api/tests/test_shadow_evaluation.py | 75 + azure/.funcignore | 6 +- azure/.gitignore | 9 +- azure/Dockerfile.glance | 17 + azure/README.md | 55 +- azure/common/__init__.py | 68 +- azure/common/config.py | 47 +- azure/common/glance_connector.py | 1544 +++++++++++++++++ azure/common/glance_mirror.py | 1163 +++++++++++++ azure/local.settings.json.example | 20 + azure/process_glance_traces/__init__.py | 16 + azure/process_glance_traces/function.json | 13 + azure/requirements-glance.txt | 3 + azure/scripts/run_glance_mirror.py | 30 + azure/scripts/run_glance_poll.py | 32 + azure/test_scripts/shadow_glance_poll.py | 541 ++++++ azure/tests/test_common_config.py | 83 + azure/tests/test_glance_connector.py | 735 ++++++++ azure/tests/test_glance_mirror.py | 499 ++++++ geddes/k8s/01-secrets.yaml.example | 3 + geddes/k8s/02-api.yaml | 8 + geddes/k8s/05-postgres.yaml | 90 +- geddes/k8s/glance-ingestion-cronjob.yaml | 111 ++ .../k8s/glance-mirror/01-secret.yaml.example | 13 + geddes/k8s/glance-mirror/02-postgres.yaml | 300 ++++ geddes/k8s/glance-mirror/03-postgrest.yaml | 61 + geddes/k8s/glance-mirror/04-sync-cronjob.yaml | 75 + .../k8s/glance-mirror/05-network-policy.yaml | 22 + geddes/k8s/glance-mirror/README.md | 52 + geddes/k8s/ml-shadow-evaluation-cronjob.yaml | 33 + geddes/k8s/postgres/schema_v2.sql | 86 +- web/app/data/analysis/page.tsx | 5 + web/app/optimize/page.tsx | 46 +- web/components/cross-run-trace-analysis.tsx | 463 +++++ web/components/run-trace-explorer.tsx | 314 +++- web/components/sidebar.tsx | 1 + web/lib/api-client.ts | 56 +- web/lib/auth-context.tsx | 8 +- web/lib/trace-plot.ts | 150 ++ 52 files changed, 9975 insertions(+), 227 deletions(-) create mode 100644 PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md create mode 100644 api/glance_ingestion.py create mode 100644 api/scripts/add_production_glance_ingestion.sql create mode 100644 api/scripts/run_ml_shadow_evaluation.py create mode 100644 api/shadow_evaluation.py create mode 100644 api/tests/test_production_glance_ingestion.py create mode 100644 api/tests/test_shadow_evaluation.py create mode 100644 azure/Dockerfile.glance create mode 100644 azure/common/glance_connector.py create mode 100644 azure/common/glance_mirror.py create mode 100644 azure/process_glance_traces/__init__.py create mode 100644 azure/process_glance_traces/function.json create mode 100644 azure/requirements-glance.txt create mode 100644 azure/scripts/run_glance_mirror.py create mode 100644 azure/scripts/run_glance_poll.py create mode 100644 azure/test_scripts/shadow_glance_poll.py create mode 100644 azure/tests/test_common_config.py create mode 100644 azure/tests/test_glance_connector.py create mode 100644 azure/tests/test_glance_mirror.py create mode 100644 geddes/k8s/glance-ingestion-cronjob.yaml create mode 100644 geddes/k8s/glance-mirror/01-secret.yaml.example create mode 100644 geddes/k8s/glance-mirror/02-postgres.yaml create mode 100644 geddes/k8s/glance-mirror/03-postgrest.yaml create mode 100644 geddes/k8s/glance-mirror/04-sync-cronjob.yaml create mode 100644 geddes/k8s/glance-mirror/05-network-policy.yaml create mode 100644 geddes/k8s/glance-mirror/README.md create mode 100644 geddes/k8s/ml-shadow-evaluation-cronjob.yaml create mode 100644 web/app/data/analysis/page.tsx create mode 100644 web/components/cross-run-trace-analysis.tsx create mode 100644 web/lib/trace-plot.ts diff --git a/.gitignore b/.gitignore index df29bd7..060a8a3 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,8 @@ debug_*.py !api/tests/test_e2e_platform.py !api/tests/test_glance_db3_trace_import.py !api/tests/test_retrain.py +!api/tests/test_production_glance_ingestion.py +!api/tests/test_shadow_evaluation.py # Development files temp.py diff --git a/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md b/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md new file mode 100644 index 0000000..d074ccc --- /dev/null +++ b/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md @@ -0,0 +1,517 @@ +# Production GLANCE Ingestion and ML Requirements + +Status: Approved implementation basis +Date: 2026-07-27 +Implementation branch: `feat/glance-production-ingestion` (from `dev`) + +## 1. Purpose + +This document defines the production-safe implementation that follows the +temporary GLANCE DB3 pilot and Paul's requested plotting improvements. It +separates five deliverables: + +1. an automated, scheduled, full-trace GLANCE connector; +2. a durable FastAPI trace-ingestion contract and source identity; +3. ten-run plotting, trendline, annotation, color, and performance changes; +4. truthful reporting of the active ML proposal engine and model; +5. a shadow RF-versus-GPR/Pareto evaluation with explicit promotion criteria. + +The temporary `feat/db3-upload-pipeline` branch remains a validated pilot and +reference implementation. It must not be merged into `dev`, `master`, or this +production branch. Production work may port reviewed data semantics and +display logic, but not the pilot upload allowlist, DB3 upload route, private +pilot projects, PVC, large-body ingress, image tags, or cleanup behavior. + +## 2. Outcomes and non-goals + +### 2.1 Required outcomes + +- GLANCE runs can be ingested without a user uploading a SQLite DB3 file. +- One catalog row represents one physical GLANCE run. +- Every timestamped sample and value remains attached to that run as a trace. +- Repeated polling and retries do not duplicate catalog runs or samples. +- Late-arriving samples and events can update an existing source run. +- Catalog and Trace Analysis continue enforcing project visibility. +- Users can see which proposal engine and model actually generated a result. +- RF and GPR/Pareto proposals can be compared without changing live behavior. + +### 2.2 Non-goals + +- Use the dedicated, read-only `nanohubdt` PostgreSQL identity for the initial + production pilot. The previously configured PostgREST endpoint is a stale, + standalone copy and must not be treated as authoritative. +- Do not deploy an obsolete PostgREST release merely to connect to the GLANCE + PostgreSQL 10.5 server. The deployed PostgREST 13 service requires a newer + PostgreSQL server. +- Do not grant the connector write access to Digital Twin PostgreSQL. +- Do not make DB3 upload the permanent production integration. +- Do not smooth, impute, trim, rename, or permanently clean raw source values. +- Do not interpret a literal zero as a generic sensor failure. +- Do not treat `runs.status=processed` as a scientific Complete outcome. +- Do not activate GPR/Pareto proposals in production solely by changing an + environment variable. +- Do not use Bayesian optimization as a trace-noise filter. +- Keep every new Geddes CronJob suspended and every production mapping + placeholder-only until a separately approved canary deployment. + +## 3. Shared data rules + +The following rules apply to every production GLANCE adapter: + +- Preserve literal zeroes, nulls in nullable source columns, empty samples, raw + parameter names, units, numeric parameter IDs, source timestamps, recipes, + and events. The validated authoritative schema makes `runs.starttime` and + `data.value` non-nullable, while `samplerecord.time` is nullable. +- Derive relative time from source timestamps, never from sample row numbers. +- Preserve the exact source timestamp text and do not invent a timezone. +- Use explicit numeric equipment-parameter ID matching first. +- Use conservative name-and-unit matching only when no numeric source ID exists. +- Never override an ID conflict or silently alias two source signals. +- Label unmatched signals as unregistered. +- Associate events only through an authoritative source relationship. When + GLANCE provides no run foreign key, require the same tool and a timestamp + inside the run interval. +- Derive `complete`, `abort`, or `unknown` only from authoritative terminal + GLANCE events at the run-end timestamp. +- Keep ingestion state separate from scientific process outcome. + +## 4. Recommendation 1: scheduled GLANCE connector + +### 4.1 Placement and interface + +- Implement a dedicated connector, separate from the legacy summary-processing + Azure Function. +- The default production runtime is a Geddes CronJob because read-only network + access from the DT namespace to the authoritative GLANCE database has been + validated. +- Keep the GLANCE client and polling coordinator runtime-independent so the same + package can still run as an Azure timer function if an approved network path + is later provided. +- Read GLANCE through a least-privilege service identity. +- Send validated batches to FastAPI through an ingestion-token-protected HTTP + endpoint. The connector must never write directly to DT tables. + +### 4.2 Polling behavior + +- Default schedule: every ten minutes, configurable through environment. +- Maintain an independent durable cursor for every mapped GLANCE tool. +- Poll with a configurable overlap window to recover late samples and events. +- Query source runs by stable tool and run IDs, not lot-name patterns. +- Fetch the complete affected run, including run metadata, recipe metadata, + parameter definitions, sample records, values, and in-run events. +- Use parameterized SQL, read-only repeatable-read transactions, bounded + statement timeouts, and bounded retries for transient connection failures. +- Enforce configurable per-run sample and value limits before handing a payload + to FastAPI. +- Advance a cursor only after FastAPI confirms the complete batch was committed. +- A failed tool must not block other mapped tools. +- Support an explicit bounded backfill range without resetting normal cursors. +- Redact connection credentials from errors and durably record failures that + occur before an API handoff (JSONL on the Geddes state PVC or immutable + objects in the Azure state container). + +### 4.3 Connector configuration + +Required configuration: + +- authoritative GLANCE PostgreSQL URI supplied only through a Kubernetes Secret; +- explicit source mode (`postgres` for the initial pilot, `postgrest` only for + a validated modern mirror); +- DT trace-ingestion URL and ingestion token; +- explicit equipment-to-GLANCE-tool map; +- state storage connection and cursor container/table; +- polling overlap, page size, retry count, and timeout. + +Secrets must remain outside Git. Example configuration must contain placeholders +only. + +### 4.4 Connector acceptance criteria + +- Two identical polls produce one catalog run with one copy of each sample. +- A later sample/event updates the existing catalog run. +- A failed API request does not advance the source cursor. +- A source run with duplicate sample or parameter IDs is rejected with an + actionable error. +- One malformed run does not discard other valid runs in the batch. +- HTTP tests use deterministic fixtures and require no live credentials. +- SQL tests use injected connection fixtures and never interpolate source IDs + into query text. + +## 5. Recommendation 2: production trace-ingestion API and source identity + +### 5.1 Durable identity + +`equipment_runs` must expose explicit production source identity fields: + +- `source_system`; +- `source_tool_id`; +- `source_run_id`; +- `source_updated_at`; +- `ingested_at`. + +The durable uniqueness rule is: + +`(source_system, equipment_id, source_tool_id, source_run_id)`. + +Source run IDs are not assumed globally unique across tools. The production path +must not use upload UUID plus row number as its identity. + +### 5.2 Trace ingestion contract + +Add an ingestion-token-protected endpoint for a versioned payload containing: + +- equipment ID and GLANCE tool identity; +- source run ID, lot, material, start/end timestamps, and source status; +- recipe ID, name, hash, timestamp, and file fingerprint when available; +- parameter metadata and registry-match metadata; +- all samples with source sample ID, exact timestamp, and values; +- all associated events; +- source revision/update time when available. + +FastAPI must: + +- validate the configured equipment/tool mapping; +- validate IDs, timestamp syntax, uniqueness, and JSON value safety; +- resolve parameter registration conservatively; +- derive the process outcome; +- upsert the parent and replace its trace in one transaction; +- preserve the catalog run ID on retry; +- apply project visibility through an explicit production project-assignment + policy; +- return inserted, updated, run, sample, event, and warning counts. + +### 5.3 Cursor and audit state + +Persist connector audit records containing: + +- equipment and source tool; +- previous and proposed cursor; +- poll start/end; +- request/batch identity; +- run/sample/event counts; +- success/failure status and a redacted error; +- source and payload checksums when practical. + +### 5.4 API acceptance criteria + +- Authentication failure returns 401/403 without disclosing data. +- A source tool mapped to another equipment registration is rejected. +- Replaying a payload preserves the catalog run ID and raw values. +- Project nonmembers receive 404 for direct run/trace access. +- Literal zeroes, nulls, empty samples, timestamp gaps, and unregistered + parameters survive a complete round trip. +- Parent and trace writes roll back together on failure. +- Existing canonical etcher ingestion remains backward compatible. + +## 6. Recommendation 3: Trace Analysis and Catalog plotting + +### 6.1 Run comparison + +- Increase the selected-run limit from six to ten in both cross-run Trace + Analysis and the Catalog run comparison. +- Provide at least ten stable, visually distinct colors. +- Bound concurrent trace fetches to avoid a ten-request burst. +- Preserve caching and do not refetch unchanged selected traces. +- Show partial failures without discarding successfully loaded runs. + +### 6.2 Trendline + +- Keep `Apply trendline` opt-in. +- Fit one documented least-squares line across all visible finite points in the + current parameter panel. +- Show slope, intercept/local-origin equation, R-squared, point count, and time + unit. +- Use a local time origin for absolute timestamps to avoid numerical precision + loss and present that origin with the equation. +- If plot-edge smoothing is also selected, calculate the displayed trendline + from the displayed points only. +- Never change stored or returned source values. + +### 6.3 Annotations and plot-edge display + +- Use short run/outcome labels and separate annotation lanes. +- Omit redundant run-start annotations on the Relative axis. +- Keep repeated edge-zero hiding opt-in and display-only. +- Preserve interior zeroes, all-zero signals, isolated edge zeroes, nulls, and + all source data. + +### 6.4 Web acceptance criteria + +- Ten runs can be selected in each component. +- Colors remain deterministic for a given run. +- Trendline results cover normal, constant-Y, duplicate-X, absolute-time, null, + and smoothed-edge cases. +- Process and date labels do not occupy one identical annotation lane. +- TypeScript, scoped ESLint, and the Next.js production build pass. +- The largest retained pilot traces remain interactively usable in a manual + pilot check before deployment. + +## 7. Recommendation 4: truthful ML runtime reporting + +### 7.1 API + +Add an authenticated ML runtime/status endpoint that reports effective values, +not source-code defaults: + +- proposal engine; +- proposal model; +- candidate count; +- acquisition method; +- active model-registry version/kind when present; +- training snapshot hash/time when present; +- whether the current mode uses uncertainty-based acquisition or randomized + fallback. + +Never return secrets or unredacted environment values. + +Proposal responses must include the same effective engine/model metadata so a +saved recipe batch remains reproducible. + +### 7.2 Web + +- Replace hard-coded “Training GPR” and “using Gaussian Process Regression” + text with effective runtime/model information. +- Display a clear warning when proposals use an RF/randomized fallback without + uncertainty-based acquisition. +- Keep model terminology consistent across Optimize, proposals, and project + recipe batches. + +### 7.3 Acceptance criteria + +- With the live-equivalent RF environment, the UI says Random Forest rather than + GPR. +- With GPR selected in a test environment, the UI identifies GPR and its + acquisition method. +- Missing optional model-registry state does not break the endpoint or UI. +- Status and proposal metadata agree in integration tests. + +## 8. Recommendation 5: shadow RF-versus-GPR/Pareto evaluation + +### 8.1 Execution boundary + +- Shadow evaluation is read-only with respect to live proposal selection. +- It uses frozen, project/equipment-scoped training snapshots. +- It writes comparison artifacts and metrics but never promotes a model, + creates an executable recipe automatically, or changes the active engine. +- Existing RF behavior remains active until an explicit reviewed promotion. + +### 8.2 Required candidates + +At minimum compare: + +- current FastAPI RF behavior; +- current FastAPI GPR/EI behavior; +- the reviewed GPR/Pareto/Monte-Carlo implementation after correctness fixes. + +Consolidate common sampling, Pareto, and acquisition code after comparison so +the API and nanoHUB implementations cannot silently diverge. + +### 8.3 Correctness requirements before comparison + +- Correct Pareto dominance for maximize-rate/minimize-range objectives. +- Normalize objective distances before combining them. +- Keep candidate, mean, and uncertainty arrays aligned through filtering. +- Either implement a real fantasy update or label the method accurately. +- Return actual acquisition scores instead of placeholders. +- Use held-out or cross-validated residuals for uncertainty calibration. +- Make training iteration/model selection explicit. +- Apply equipment-specific, registered parameter constraints. + +### 8.4 Metrics and promotion gate + +Record per model and snapshot: + +- holdout and/or LOOCV R-squared, RMSE, and MAE; +- uncertainty coverage/calibration where uncertainty is claimed; +- proposal constraint violations; +- proposal stability under fixed and varied seeds; +- predicted Pareto improvement and realized improvement when outcomes exist; +- runtime, memory, candidate count, and failure rate; +- expert review/acceptance outcome. + +GPR/Pareto may be proposed for a limited canary only when: + +- correctness tests pass; +- uncertainty is calibrated within documented tolerances; +- predictive performance is not materially worse than RF; +- proposal constraints have zero violations; +- runtime fits the API/background-job budget; +- tool experts approve the feature ranges, targets, and acquisition behavior; +- promotion is explicit, reversible, and recorded. + +### 8.5 Shadow acceptance criteria + +- One command evaluates all configured candidates against one frozen snapshot. +- Repeating with the same seed produces identical artifacts. +- Project-scoped runs never leak into another project's snapshot. +- The command exits nonzero on correctness or constraint failures. +- The report explicitly states that no live model or proposal engine changed. + +## 9. Implementation and rollout sequence + +1. Add schema and API behavior behind an inactive production connector path. +2. Add deterministic backend/security tests. +3. Add the connector with mocked GLANCE and DT endpoints. +4. Port and validate web-only improvements. +5. Add effective ML runtime reporting. +6. Add the shadow evaluation command and reports. +7. Run the complete local validation matrix. +8. Review the requirements and implementation with GLANCE owners and tool + experts. +9. Obtain a sanctioned GLANCE interface, service identity, mapping, project + assignment, schedule, and backfill window. +10. Conduct a read-only/shadow poll without DT writes. +11. Conduct a limited private-project canary with counts and checksums compared + to an approved GLANCE export. +12. Roll out by mapped tool with documented rollback. + +## 10. Deployment boundary + +Local implementation and tests do not authorize: + +- connecting to live GLANCE; +- applying database migrations on Geddes; +- deploying API, connector, web, or CronJob images; +- changing active ML configuration; +- reprocessing or deleting retained DB3 uploads; +- changing pilot project membership, PVCs, ingress, or rollback identities. + +## 11. Authorized read-only GLANCE shadow validation + +On 2026-07-27, a separately authorized, bounded PostgREST shadow poll validated +the latest run for each approved Versaline tool. The audit did not call the DT +ingestion API, read or advance a cursor, persist a raw payload, apply a +migration, or deploy anything. + +| Tool | Live tool ID | Run ID | Samples | Values | Parameters | Events | Complete-payload time | +|---|---:|---:|---:|---:|---:|---:|---:| +| `VLN-11304-CTC-PM1` | 3 | 8677 | 2,622 | 178,296 | 68 | 6 | 11.4 seconds | +| `VLN-11303-CTC-PM1` | 1 | 8658 | 264 | 17,688 | 67 | 9 | 1.8 seconds | + +The first query design used large sample-ID `IN` filters and received repeated +HTTP 503 responses. A whole-run joined query was also too slow when globally +ordered before pagination. The accepted connector query shape is therefore: + +- fetch minimal sample identity/timestamp fields; +- sort source sample IDs locally; +- divide them into disjoint numeric ranges of 25; +- query `/data` through the authoritative `samplerecord.idruns` relationship + and the bounded sample-ID range; +- page with explicit PostgREST `limit` and `offset`; +- reconstruct the complete trace in memory without changing raw values. + +Both complete payloads were hashed after construction. The audit observed and +preserved 60,989 literal zero values in DSE and 8,646 in ICP RIE. These results +validated the payload construction and run-scoped query shape against the +historical DT PostgREST copy. They did not establish that endpoint as the +authoritative live source. + +- DSE payload SHA-256: + `58ff5aefcee7f5c145046362b8e15f2ca20600cf948c63b6dec388256263bda2` +- ICP RIE payload SHA-256: + `1fad626eec9148537e0e810b06bd4e888f55f001c6cc0c41158a1d13a32b16c3` + +This validation does not authorize production activation. A canary still +requires reviewed equipment/project mappings, a bounded initial run ID/backfill +window, a deployed API/schema migration, a Geddes CronJob configuration, and a +count/checksum comparison against an approved GLANCE export. + +The source reported 2026-02-04 as the latest run date for both tools during the +2026-07-27 audit. A subsequent infrastructure audit confirmed that endpoint was +a stale, standalone Geddes database rather than the authoritative GLANCE +database. + +Each action above requires a separate operational review and explicit approval. + +## 12. Authoritative database validation and source decision + +On 2026-07-28 the dedicated `nanohubdt` identity was validated from an isolated +Geddes pod against the authoritative `logger` database. The identity can assume +the read role and select the eight required source tables: `runs`, +`samplerecord`, `data`, `parameters`, `recipes`, `events`, `eventtypes`, and +`tools`. + +The authoritative database is PostgreSQL 10.5. PostgREST 13 rejected it because +that PostgREST version requires PostgreSQL 12.1 or newer. An isolated PostgREST +10.2 canary connected and returned a current Tool 3 row, proving that the role, +network, and grants were correct, but an obsolete PostgREST release is not the +selected production solution. + +The approved implementation direction is: + +1. use direct, parameterized, read-only PostgreSQL access for the bounded pilot; +2. ingest the approved Tool 3 run allowlist through the existing FastAPI + contract; +3. activate ten-minute polling only after source/destination count validation; +4. build a private, pull-based mirror on a modern PostgreSQL version for the + long-term PostgREST source. + +A read-only direct-SQL canary then assembled approved Tool 3 run `1` through +the same connector code without calling the DT API or advancing state. It found +77 sample records, no parameter values, and 7 in-window events and produced +payload SHA-256 +`1a25e27a469bfcd9ebb3b9d9d8b871b88a5ee44ad7ac13d910e74ab789d5ee97`. +This confirms the direct connection and complete-run query transaction; the +remaining allowlist is still required to validate value-bearing runs before +activation. + +The source database is approximately 53 GB. Its `data` table contains about +613 million rows and consumes approximately 51 GB. The existing 20 GiB DT +application database volume is therefore not a valid full-mirror destination. +The raw mirror must use a separate expandable volume, initially sized at no less +than 100 GiB; 150 GiB is the recommended operational starting point for source +data, indexes, WAL, backfill overhead, and growth. + +## 13. Pull-based modern mirror requirements + +The mirror is an application-level, read-only-source synchronization process, +not cross-version physical streaming replication. + +- Run PostgreSQL 15 or newer in a dedicated Geddes workload and PVC. +- Keep the mirror database and PostgREST service private (`ClusterIP`) until an + explicit external-access review. +- Preserve the eight GLANCE tables in a dedicated `glance_mirror` schema with + source primary identities and source-compatible types. +- Give the synchronizer `SELECT` only on GLANCE and write access only to the + destination mirror schema. +- Give PostgREST a separate read-only destination role. +- Copy one complete run per destination transaction, including its samples, + values, referenced parameters, nullable recipe, and in-window events. +- Upsert stable source identities and replace the run-scoped sample/value set on + overlap retries so late values are recovered without duplication. +- Maintain independent successful run cursors per tool and synchronization mode + (`live`, explicit-ID pilot, or bounded range) in the destination. +- Advance the applicable cursor only in the same transaction that commits the + complete run. After a malformed run, later valid runs may be copied + idempotently, but the cursor must remain behind the failure so the gap is + retried. +- Backfill by explicit tool and bounded run/date windows with configurable batch + limits. Never launch an unbounded 613-million-row copy. +- Enforce reviewed per-run sample and value limits, and insert destination rows + in bounded batches so one unexpectedly large run cannot exhaust worker + memory. +- Track run/event associations and reconcile stale event rows during overlap + replacement without deleting an event still associated with another run. +- Record duration, row counts, status, and redacted failures in a destination + audit table. +- Keep the CronJob suspended in Git. Activation, volume creation, backfill, and + PostgREST cutover require an operational review and rollback plan. + +## 14. Local implementation map + +The implementation on `feat/glance-production-ingestion` is intentionally +inactive until the deployment boundary above is approved: + +- Connector: `azure/common/glance_connector.py`, the Azure timer entry point, + and the suspended Geddes direct-ingestion CronJob. +- Mirror: `azure/common/glance_mirror.py`, its command wrapper, and the + suspended resources under `geddes/k8s/glance-mirror/`. +- API contract and validation: `api/glance_ingestion.py` and + `POST /api/dataset/v2/glance/traces/sync`. +- Schema migration: `api/scripts/add_production_glance_ingestion.sql`. +- Plotting: `web/lib/trace-plot.ts`, Catalog run comparison, and + `/data/analysis`. +- Runtime truth: `GET /api/ml/status` and the Optimize page runtime badge. +- Shadow evidence: `api/scripts/run_ml_shadow_evaluation.py`, comparing RF, + GPR/analytic EI, and GPR/Pareto/Monte-Carlo acquisition. The supplied + CronJob manifest is suspended and contains placeholder image/project values. diff --git a/api/data_loader_pg.py b/api/data_loader_pg.py index 44c6faf..d287037 100644 --- a/api/data_loader_pg.py +++ b/api/data_loader_pg.py @@ -354,6 +354,11 @@ def ensure_equipment_runs_pg() -> None: row_index INT, upload_filename VARCHAR(255) DEFAULT '', source VARCHAR(100) DEFAULT 'data_upload', + source_system VARCHAR(100) NOT NULL DEFAULT 'data_upload', + source_tool_id BIGINT, + source_run_id BIGINT, + source_updated_at TIMESTAMP WITH TIME ZONE, + ingested_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ) """ @@ -361,6 +366,38 @@ def ensure_equipment_runs_pg() -> None: cur.execute( "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS row_index INT" ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "source_system VARCHAR(100)" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_tool_id BIGINT" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_run_id BIGINT" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "source_updated_at TIMESTAMP WITH TIME ZONE" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "ingested_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP" + ) + cur.execute( + """ + UPDATE equipment_runs + SET source_system = COALESCE(NULLIF(source, ''), 'data_upload') + WHERE source_system IS NULL OR source_system = '' + """ + ) + cur.execute( + "ALTER TABLE equipment_runs ALTER COLUMN source_system " + "SET DEFAULT 'data_upload'" + ) + cur.execute( + "ALTER TABLE equipment_runs ALTER COLUMN source_system SET NOT NULL" + ) cur.execute( "CREATE INDEX IF NOT EXISTS idx_equipment_runs_equipment_id ON equipment_runs(equipment_id)" ) @@ -377,6 +414,15 @@ def ensure_equipment_runs_pg() -> None: WHERE upload_id IS NOT NULL """ ) + cur.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_source_identity + ON equipment_runs( + source_system, equipment_id, source_tool_id, source_run_id + ) + WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL + """ + ) cur.execute("ALTER TABLE equipment_runs ENABLE ROW LEVEL SECURITY") cur.execute("GRANT SELECT ON equipment_runs TO api_client") cur.execute( @@ -438,8 +484,8 @@ def ensure_equipment_run_trace_samples_pg() -> None: sample_index INT NOT NULL, source_sample_id BIGINT NOT NULL, source_tool_id BIGINT, - sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, - sample_time_raw TEXT NOT NULL, + sample_time TIMESTAMP WITHOUT TIME ZONE, + sample_time_raw TEXT, values_json JSONB NOT NULL DEFAULT '{}'::jsonb, PRIMARY KEY (equipment_run_id, sample_index), UNIQUE (equipment_run_id, source_sample_id), @@ -451,6 +497,14 @@ def ensure_equipment_run_trace_samples_pg() -> None: "ALTER TABLE equipment_run_trace_samples " "ADD COLUMN IF NOT EXISTS source_tool_id BIGINT" ) + cur.execute( + "ALTER TABLE equipment_run_trace_samples " + "ALTER COLUMN sample_time DROP NOT NULL" + ) + cur.execute( + "ALTER TABLE equipment_run_trace_samples " + "ALTER COLUMN sample_time_raw DROP NOT NULL" + ) cur.execute( """ CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time @@ -504,6 +558,187 @@ def ensure_equipment_run_trace_samples_pg() -> None: conn.close() +def ensure_equipment_run_trace_events_pg() -> None: + """Create the normalized provenance table for events attached to a run.""" + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE IF NOT EXISTS equipment_run_trace_events ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + source_event_id BIGINT NOT NULL, + source_tool_id BIGINT, + event_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + event_time_raw TEXT NOT NULL, + event_type TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + raw_event_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, source_event_id), + CHECK (jsonb_typeof(raw_event_json) = 'object') + ) + """ + ) + cur.execute( + """ + CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_events_time + ON equipment_run_trace_events( + equipment_run_id, event_time, source_event_id + ) + """ + ) + cur.execute( + "ALTER TABLE equipment_run_trace_events ENABLE ROW LEVEL SECURITY" + ) + cur.execute("GRANT SELECT ON equipment_run_trace_events TO api_client") + cur.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_trace_events' + AND policyname = 'equipment_run_trace_event_visibility_policy' + ) THEN + CREATE POLICY equipment_run_trace_event_visibility_policy + ON equipment_run_trace_events + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_events.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = + current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; + END + $$; + """ + ) + conn.commit() + finally: + conn.close() + + +def ensure_glance_ingestion_audit_pg() -> None: + """Create connector audit storage; it is intentionally not API-readable.""" + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE IF NOT EXISTS glance_ingestion_audit ( + id BIGSERIAL PRIMARY KEY, + batch_id VARCHAR(255) NOT NULL, + source_system VARCHAR(100) NOT NULL DEFAULT 'glance', + equipment_id VARCHAR(255) NOT NULL, + source_tool_id BIGINT NOT NULL, + project_id VARCHAR(50), + previous_cursor TEXT, + proposed_cursor TEXT, + poll_started_at TIMESTAMP WITH TIME ZONE, + poll_completed_at TIMESTAMP WITH TIME ZONE + DEFAULT CURRENT_TIMESTAMP, + status VARCHAR(32) NOT NULL, + run_count INT NOT NULL DEFAULT 0, + sample_count INT NOT NULL DEFAULT 0, + event_count INT NOT NULL DEFAULT 0, + warning_count INT NOT NULL DEFAULT 0, + payload_sha256 VARCHAR(64), + error_text TEXT, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + cur.execute( + """ + CREATE INDEX IF NOT EXISTS idx_glance_ingestion_audit_tool_time + ON glance_ingestion_audit( + equipment_id, source_tool_id, created_at DESC + ) + """ + ) + conn.commit() + finally: + conn.close() + + +def record_glance_ingestion_audit_pg(**record: Any) -> None: + """Persist one redacted connector handoff result independently of trace data.""" + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO glance_ingestion_audit ( + batch_id, source_system, equipment_id, source_tool_id, + project_id, previous_cursor, proposed_cursor, + poll_started_at, status, run_count, sample_count, + event_count, warning_count, payload_sha256, error_text + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s + ) + """, + ( + str(record.get("batch_id") or ""), + str(record.get("source_system") or "glance"), + str(record.get("equipment_id") or ""), + int(record.get("source_tool_id")), + str(record.get("project_id") or "") or None, + str(record.get("previous_cursor") or "") or None, + str(record.get("proposed_cursor") or "") or None, + record.get("poll_started_at"), + str(record.get("status") or "unknown"), + int(record.get("run_count") or 0), + int(record.get("sample_count") or 0), + int(record.get("event_count") or 0), + int(record.get("warning_count") or 0), + str(record.get("payload_sha256") or "") or None, + str(record.get("error_text") or "")[:2000] or None, + ), + ) + conn.commit() + finally: + conn.close() + + +def get_glance_ingestion_project_pg(project_id: str) -> Optional[Dict[str, Any]]: + """Return the authoritative project/equipment assignment for ingestion.""" + conn = get_pg_superuser_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + SELECT id, equipment_id, access_mode + FROM projects + WHERE id = %s + LIMIT 1 + """, + (project_id,), + ) + row = cur.fetchone() + conn.commit() + return dict(row) if row else None + finally: + conn.close() + + def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: """SELECT that shapes generic equipment_runs into the same record format used by the etcher /runs endpoints (so the catalog can render both).""" @@ -520,9 +755,17 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: r.inputs_json, r.outputs_json, r.upload_filename, - r.source, + COALESCE(NULLIF(r.source_system, ''), r.source) AS source, r.project_id, - NULLIF(r.raw_payload_json #>> '{{glance,run_id}}', '') AS source_run_id, + COALESCE( + r.source_run_id::text, + NULLIF(r.raw_payload_json #>> '{{glance,run_id}}', '') + ) AS source_run_id, + COALESCE( + NULLIF(r.raw_payload_json #>> '{{trace,process_outcome}}', ''), + NULLIF(r.raw_payload_json #>> '{{trace,process_status}}', ''), + 'unknown' + ) AS process_status, NULLIF( r.raw_payload_json #>> '{{glance,run_start_time}}', '' ) AS source_run_start_time, @@ -557,7 +800,7 @@ def _shape_equipment_run_record(row) -> Dict[str, Any]: else str(rec[ts_field]) ) source_run_start_time = rec.pop("source_run_start_time", None) - if rec.get("source") == "glance_db3" and source_run_start_time: + if str(rec.get("source") or "").startswith("glance") and source_run_start_time: # equipment_runs.run_date is TIMESTAMPTZ, so inserting a naive GLANCE # wall time makes Postgres attach the session timezone. Present the exact # source text instead; raw trace metadata explicitly reports that its @@ -688,7 +931,11 @@ def _normalize_trace_parameter_key(value: Any) -> str: return text -def _parse_trace_sample_time(value: Any, *, source_sample_id: int) -> tuple[datetime, str]: +def _parse_trace_sample_time( + value: Any, + *, + source_sample_id: int, +) -> tuple[datetime | None, str | None]: """Parse a source timestamp without inventing a timezone. The exact input text is returned alongside a timezone-naive datetime for the @@ -697,15 +944,15 @@ def _parse_trace_sample_time(value: Any, *, source_sample_id: int) -> tuple[date ``tzinfo`` while retaining the exact offset-bearing source text. Naive source values remain naive; no equipment timezone is guessed. """ + if value is None: + return None, None if isinstance(value, datetime): raw = value.isoformat() parsed = value else: - raw = str(value or "").strip() + raw = str(value).strip() if not raw: - raise ValueError( - f"GLANCE sample {source_sample_id} is missing an absolute timestamp" - ) + return None, raw candidate = raw[:-1] + "+00:00" if raw.endswith("Z") else raw try: parsed = datetime.fromisoformat(candidate) @@ -722,10 +969,14 @@ def sync_equipment_run_traces_pg( *, equipment_id: str, project_id: str, - upload_id: str, - upload_filename: str, + upload_id: Optional[str] = None, + upload_filename: str = "", runs: List[Dict[str, Any]], -) -> int: + source_system: str = "glance_db3", + source_tool_id: Optional[int] = None, + source_updated_at: Optional[datetime] = None, + return_stats: bool = False, +) -> Any: """Atomically upsert physical GLANCE runs and their complete sample traces. One item in ``runs`` becomes one ``equipment_runs`` catalog record. Each item @@ -735,11 +986,12 @@ def sync_equipment_run_traces_pg( ``sample_time``/``timestamp``), and a ``values`` mapping keyed by stable parameter ids such as ``p130``. - Parent rows and samples share one transaction. For DB3 imports ``row_index`` - stores the GLANCE source run id, making ``(upload_id, row_index)`` the stable - ``(upload, source run)`` identity. Reprocessing replaces traces in place, - preserves catalog run ids even when another source run is skipped or later - recovers, and removes stale rows from an earlier conversion. + Parent rows, samples, and events share one transaction. DB3 imports retain + their stable ``(upload_id, row_index)`` identity. Production live imports + omit ``upload_id`` and use + ``(source_system, equipment_id, source_tool_id, source_run_id)`` instead. + Both paths replace complete affected traces in place and preserve catalog + run IDs on retries. """ from psycopg2.extras import execute_values @@ -747,8 +999,21 @@ def sync_equipment_run_traces_pg( raise ValueError("equipment_id is required for GLANCE trace ingestion") if not str(project_id or "").strip(): raise ValueError("project_id is required for GLANCE trace ingestion") - if not str(upload_id or "").strip(): - raise ValueError("upload_id is required for idempotent GLANCE trace ingestion") + source_system = str(source_system or "").strip() + live_source = upload_id in (None, "") + if live_source: + if not source_system: + raise ValueError("source_system is required for live trace ingestion") + try: + batch_source_tool_id: Optional[int] = int(source_tool_id) + except (TypeError, ValueError) as exc: + raise ValueError( + "source_tool_id is required for live trace ingestion" + ) from exc + else: + batch_source_tool_id = None + if not source_system: + source_system = "glance_db3" if not runs: raise ValueError("No physical GLANCE runs with trace samples were found") @@ -809,6 +1074,15 @@ def sync_equipment_run_traces_pg( source_name = parameter.get("source_name") if source_name not in (None, ""): metadata["source_name"] = str(source_name) + metadata["registered"] = bool(parameter.get("registered", False)) + for field in ( + "registered_name", + "registered_parameter_id", + "registration_kind", + "registration_match", + ): + if parameter.get(field) not in (None, ""): + metadata[field] = str(parameter[field]) parameter_by_key[key] = metadata parameters.append(metadata) @@ -842,7 +1116,7 @@ def sync_equipment_run_traces_pg( "source_tool_id", sample.get("glance_tool_id") ) if source_tool_value in (None, ""): - source_tool_id = None + source_tool_id = batch_source_tool_id else: try: source_tool_id = int(source_tool_value) @@ -860,6 +1134,15 @@ def sync_equipment_run_traces_pg( f"GLANCE sample {source_sample_id} source tool id is outside " "the PostgreSQL BIGINT range" ) + if ( + live_source + and source_tool_id is not None + and source_tool_id != batch_source_tool_id + ): + raise ValueError( + f"GLANCE sample {source_sample_id} belongs to tool " + f"{source_tool_id}, expected {batch_source_tool_id}" + ) sample_index_value = sample.get("sample_index", fallback_index) try: @@ -915,6 +1198,75 @@ def sync_equipment_run_traces_pg( ) samples.sort(key=lambda sample: sample["sample_index"]) + raw_events = run.get("events", []) or [] + events: list[dict[str, Any]] = [] + seen_source_event_ids: set[int] = set() + for event in raw_events: + if not isinstance(event, dict): + raise ValueError( + f"GLANCE run {source_run_id} contains invalid event metadata" + ) + event_id_value = event.get( + "source_event_id", event.get("glance_event_id") + ) + try: + source_event_id = int(event_id_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"GLANCE run {source_run_id} has invalid event id " + f"{event_id_value!r}" + ) from exc + if source_event_id in seen_source_event_ids: + raise ValueError( + f"GLANCE run {source_run_id} repeats event id {source_event_id}" + ) + seen_source_event_ids.add(source_event_id) + event_time, event_time_raw = _parse_trace_sample_time( + event.get("timestamp", event.get("event_time")), + source_sample_id=source_event_id, + ) + event_source_tool_value = event.get( + "source_tool_id", + event.get("glance_tool_id", batch_source_tool_id), + ) + try: + event_source_tool_id = ( + int(event_source_tool_value) + if event_source_tool_value not in (None, "") + else None + ) + except (TypeError, ValueError) as exc: + raise ValueError( + f"GLANCE event {source_event_id} has invalid source tool id " + f"{event_source_tool_value!r}" + ) from exc + if ( + live_source + and event_source_tool_id is not None + and event_source_tool_id != batch_source_tool_id + ): + raise ValueError( + f"GLANCE event {source_event_id} belongs to tool " + f"{event_source_tool_id}, expected {batch_source_tool_id}" + ) + raw_event = _json_safe_payload(event) + events.append( + { + "source_event_id": source_event_id, + "source_tool_id": event_source_tool_id, + "event_time": event_time, + "event_time_raw": event_time_raw, + "event_type": str( + event.get("event_type", event.get("type", "")) or "" + ), + "category": str(event.get("category") or ""), + "description": str(event.get("description") or ""), + "raw": raw_event, + } + ) + events.sort( + key=lambda event: (event["event_time"], event["source_event_id"]) + ) raw_payload_value = run.get("raw", run.get("raw_payload", {})) or {} if not isinstance(raw_payload_value, dict): raise ValueError(f"GLANCE run {source_run_id} raw provenance must be an object") @@ -939,22 +1291,47 @@ def sync_equipment_run_traces_pg( trace_metadata = raw_payload.get("trace") if not isinstance(trace_metadata, dict): trace_metadata = {} + timed_samples = [ + sample for sample in samples if sample["sample_time"] is not None + ] + trace_start = ( + min(timed_samples, key=lambda sample: sample["sample_time"])[ + "sample_time_raw" + ] + if timed_samples + else None + ) + trace_end = ( + max(timed_samples, key=lambda sample: sample["sample_time"])[ + "sample_time_raw" + ] + if timed_samples + else None + ) trace_metadata.update( { "sample_count": len(samples), "parameters": parameters, - "timestamp_timezone": run.get("timestamp_timezone"), - "start_time": min(samples, key=lambda sample: sample["sample_time"])[ - "sample_time_raw" - ], - "end_time": max(samples, key=lambda sample: sample["sample_time"])[ - "sample_time_raw" + "events": [ + { + "source_event_id": event["source_event_id"], + "source_tool_id": event["source_tool_id"], + "timestamp": event["event_time_raw"], + "event_type": event["event_type"], + "category": event["category"], + "description": event["description"], + } + for event in events ], + "timestamp_timezone": run.get("timestamp_timezone"), + "start_time": trace_start, + "end_time": trace_end, } ) raw_payload["trace"] = trace_metadata - raw_payload["source_upload_id"] = str(upload_id) - raw_payload["source_db3_filename"] = str(upload_filename or "") + if upload_id not in (None, ""): + raw_payload["source_upload_id"] = str(upload_id) + raw_payload["source_db3_filename"] = str(upload_filename or "") normalized_runs.append( { @@ -970,6 +1347,10 @@ def sync_equipment_run_traces_pg( "outputs": _json_safe_payload(run.get("outputs", {}) or {}), "raw": raw_payload, "samples": samples, + "events": events, + "source_updated_at": ( + run.get("source_updated_at") or source_updated_at + ), } ) @@ -980,59 +1361,141 @@ def sync_equipment_run_traces_pg( conn = get_pg_superuser_connection() try: with conn.cursor() as cur: + identity_lock = ( + f"equipment_run_trace_sync|{source_system}|{equipment_id}|" + f"{batch_source_tool_id}" + if live_source + else f"equipment_run_trace_sync|upload|{upload_id}" + ) cur.execute( "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", - (f"equipment_run_trace_sync|{upload_id}",), + (identity_lock,), ) + existing_source_run_ids: set[int] = set() parent_values = [] - for run in normalized_runs: - parent_values.append( + if live_source: + cur.execute( + """ + SELECT source_run_id + FROM equipment_runs + WHERE source_system = %s + AND equipment_id = %s + AND source_tool_id = %s + AND source_run_id = ANY(%s) + """, ( + source_system, equipment_id, - project_id, - run["lotname"], - run["run_date"], - run["is_outlier"], - run["is_calibration_recipe"], - run["outlier_type"], - Json(run["inputs"]), - Json(run["outputs"]), - Json(run["raw"]), - upload_id, - run["source_run_id"], - upload_filename, - "glance_db3", + batch_source_tool_id, + [run["source_run_id"] for run in normalized_runs], + ), + ) + existing_source_run_ids = { + int(row[0]) for row in (cur.fetchall() or []) + } + for run in normalized_runs: + parent_values.append( + ( + equipment_id, + project_id, + run["lotname"], + run["run_date"], + run["is_outlier"], + run["is_calibration_recipe"], + run["outlier_type"], + Json(run["inputs"]), + Json(run["outputs"]), + Json(run["raw"]), + source_system, + source_system, + batch_source_tool_id, + run["source_run_id"], + run["source_updated_at"], + ) + ) + returned_parents = execute_values( + cur, + """ + INSERT INTO equipment_runs ( + equipment_id, project_id, lotname, run_date, + is_outlier, is_calibration_recipe, outlier_type, + inputs_json, outputs_json, raw_payload_json, + source, source_system, source_tool_id, source_run_id, + source_updated_at + ) VALUES %s + ON CONFLICT ( + source_system, equipment_id, source_tool_id, source_run_id ) + WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL + DO UPDATE SET + project_id = EXCLUDED.project_id, + lotname = EXCLUDED.lotname, + run_date = EXCLUDED.run_date, + is_outlier = EXCLUDED.is_outlier, + is_calibration_recipe = EXCLUDED.is_calibration_recipe, + outlier_type = EXCLUDED.outlier_type, + inputs_json = EXCLUDED.inputs_json, + outputs_json = EXCLUDED.outputs_json, + raw_payload_json = EXCLUDED.raw_payload_json, + source_updated_at = EXCLUDED.source_updated_at, + ingested_at = CURRENT_TIMESTAMP + RETURNING id, source_run_id + """, + parent_values, + fetch=True, + ) + else: + for run in normalized_runs: + parent_values.append( + ( + equipment_id, + project_id, + run["lotname"], + run["run_date"], + run["is_outlier"], + run["is_calibration_recipe"], + run["outlier_type"], + Json(run["inputs"]), + Json(run["outputs"]), + Json(run["raw"]), + upload_id, + run["source_run_id"], + upload_filename, + "glance_db3", + "glance_db3", + ) + ) + returned_parents = execute_values( + cur, + """ + INSERT INTO equipment_runs ( + equipment_id, project_id, lotname, run_date, + is_outlier, is_calibration_recipe, outlier_type, + inputs_json, outputs_json, raw_payload_json, + upload_id, row_index, upload_filename, source, + source_system + ) VALUES %s + ON CONFLICT (upload_id, row_index) WHERE upload_id IS NOT NULL + DO UPDATE SET + equipment_id = EXCLUDED.equipment_id, + project_id = EXCLUDED.project_id, + lotname = EXCLUDED.lotname, + run_date = EXCLUDED.run_date, + is_outlier = EXCLUDED.is_outlier, + is_calibration_recipe = EXCLUDED.is_calibration_recipe, + outlier_type = EXCLUDED.outlier_type, + inputs_json = EXCLUDED.inputs_json, + outputs_json = EXCLUDED.outputs_json, + raw_payload_json = EXCLUDED.raw_payload_json, + upload_filename = EXCLUDED.upload_filename, + source = EXCLUDED.source, + source_system = EXCLUDED.source_system, + ingested_at = CURRENT_TIMESTAMP + RETURNING id, row_index + """, + parent_values, + fetch=True, ) - - returned_parents = execute_values( - cur, - """ - INSERT INTO equipment_runs ( - equipment_id, project_id, lotname, run_date, - is_outlier, is_calibration_recipe, outlier_type, - inputs_json, outputs_json, raw_payload_json, - upload_id, row_index, upload_filename, source - ) VALUES %s - ON CONFLICT (upload_id, row_index) WHERE upload_id IS NOT NULL - DO UPDATE SET - equipment_id = EXCLUDED.equipment_id, - project_id = EXCLUDED.project_id, - lotname = EXCLUDED.lotname, - run_date = EXCLUDED.run_date, - is_outlier = EXCLUDED.is_outlier, - is_calibration_recipe = EXCLUDED.is_calibration_recipe, - outlier_type = EXCLUDED.outlier_type, - inputs_json = EXCLUDED.inputs_json, - outputs_json = EXCLUDED.outputs_json, - raw_payload_json = EXCLUDED.raw_payload_json, - upload_filename = EXCLUDED.upload_filename, - source = EXCLUDED.source - RETURNING id, row_index - """, - parent_values, - fetch=True, - ) parent_id_by_source_run = { int(source_run_id): int(parent_id) for parent_id, source_run_id in returned_parents @@ -1072,18 +1535,65 @@ def sync_equipment_run_traces_pg( trace_values, page_size=500, ) - # Cascading deletion removes samples belonging to a stale physical - # run if a deterministic re-conversion yields fewer runs. cur.execute( - """ - DELETE FROM equipment_runs - WHERE upload_id = %s - AND NOT (id = ANY(%s)) - """, - (upload_id, parent_ids), + "DELETE FROM equipment_run_trace_events " + "WHERE equipment_run_id = ANY(%s)", + (parent_ids,), ) + event_values = [] + for run in normalized_runs: + parent_id = parent_id_by_source_run[run["source_run_id"]] + for event in run["events"]: + event_values.append( + ( + parent_id, + event["source_event_id"], + event["source_tool_id"], + event["event_time"], + event["event_time_raw"], + event["event_type"], + event["category"], + event["description"], + Json(event["raw"]), + ) + ) + if event_values: + execute_values( + cur, + """ + INSERT INTO equipment_run_trace_events ( + equipment_run_id, source_event_id, source_tool_id, + event_time, event_time_raw, event_type, category, + description, raw_event_json + ) VALUES %s + """, + event_values, + page_size=500, + ) + if not live_source: + # Cascading deletion removes samples and events belonging to a + # stale physical DB3 run if re-conversion yields fewer runs. + cur.execute( + """ + DELETE FROM equipment_runs + WHERE upload_id = %s + AND NOT (id = ANY(%s)) + """, + (upload_id, parent_ids), + ) conn.commit() - return len(normalized_runs) + if not return_stats: + return len(normalized_runs) + inserted = len(normalized_runs) - len(existing_source_run_ids) + return { + "inserted": inserted, + "updated": len(existing_source_run_ids), + "run_count": len(normalized_runs), + "sample_count": sum( + len(run["samples"]) for run in normalized_runs + ), + "event_count": sum(len(run["events"]) for run in normalized_runs), + } except Exception: conn.rollback() raise @@ -1550,7 +2060,10 @@ def get_equipment_run_trace_pg( r.id, r.lotname, r.raw_payload_json, - r.source + r.source, + r.source_system, + r.source_tool_id, + r.source_run_id FROM equipment_runs r WHERE r.id = %s LIMIT 1 @@ -1583,6 +2096,24 @@ def get_equipment_run_trace_pg( (internal_run_id,), ) sample_rows = [dict(row) for row in cur.fetchall()] + cur.execute( + """ + SELECT + source_event_id, + source_tool_id, + event_time, + event_time_raw, + event_type, + category, + description, + raw_event_json + FROM equipment_run_trace_events + WHERE equipment_run_id = %s + ORDER BY event_time, source_event_id + """, + (internal_run_id,), + ) + event_rows = [dict(row) for row in cur.fetchall()] conn.commit() if not sample_rows: @@ -1665,14 +2196,20 @@ def get_equipment_run_trace_pg( samples: list[dict[str, Any]] = [] for sample_row in sample_rows: values = sample_row["values_json"] - raw_timestamp = str(sample_row.get("sample_time_raw") or "").strip() - if not raw_timestamp: + raw_timestamp_value = sample_row.get("sample_time_raw") + raw_timestamp = ( + str(raw_timestamp_value).strip() + if raw_timestamp_value is not None + else None + ) + if not raw_timestamp and sample_row.get("sample_time") is not None: parsed_timestamp = sample_row.get("sample_time") raw_timestamp = ( parsed_timestamp.isoformat() if hasattr(parsed_timestamp, "isoformat") - else str(parsed_timestamp or "") + else str(parsed_timestamp) ) + relative_seconds = sample_row.get("relative_seconds") samples.append( { "sample_index": int(sample_row["sample_index"]), @@ -1683,29 +2220,110 @@ def get_equipment_run_trace_pg( else None ), "timestamp": raw_timestamp, - "relative_seconds": float(sample_row["relative_seconds"] or 0.0), + "relative_seconds": ( + float(relative_seconds) + if relative_seconds is not None + else None + ), "values": {key: values.get(key) for key in selected_keys}, } ) - source_run_value = glance_metadata.get("run_id") + source_run_value = ( + parent.get("source_run_id") + if parent.get("source_run_id") is not None + else glance_metadata.get("run_id") + ) try: source_run_id: Any = int(source_run_value) except (TypeError, ValueError): source_run_id = source_run_value - min_row = min(sample_rows, key=lambda row: row["sample_time"]) - max_row = max(sample_rows, key=lambda row: row["sample_time"]) + timed_rows = [ + row for row in sample_rows if row.get("sample_time") is not None + ] + min_row = ( + min(timed_rows, key=lambda row: row["sample_time"]) + if timed_rows + else None + ) + max_row = ( + max(timed_rows, key=lambda row: row["sample_time"]) + if timed_rows + else None + ) + events = [] + trace_start_time = min_row.get("sample_time") if min_row else None + for event_row in event_rows: + event_time_raw = str(event_row.get("event_time_raw") or "").strip() + if not event_time_raw: + parsed_event_time = event_row.get("event_time") + event_time_raw = ( + parsed_event_time.isoformat() + if hasattr(parsed_event_time, "isoformat") + else str(parsed_event_time or "") + ) + events.append( + { + "source_event_id": int(event_row["source_event_id"]), + "source_tool_id": ( + int(event_row["source_tool_id"]) + if event_row.get("source_tool_id") is not None + else None + ), + "timestamp": event_time_raw, + "relative_seconds": ( + float( + ( + event_row["event_time"] - trace_start_time + ).total_seconds() + ) + if event_row.get("event_time") is not None + and trace_start_time is not None + else None + ), + "type": str(event_row.get("event_type") or ""), + "event_type": str(event_row.get("event_type") or ""), + "category": str(event_row.get("category") or ""), + "description": str(event_row.get("description") or ""), + "raw": event_row.get("raw_event_json") or {}, + } + ) return { "run_id": run_id, "source_run_id": source_run_id, + "source_system": ( + parent.get("source_system") or parent.get("source") or "" + ), + "source_tool_id": ( + int(parent["source_tool_id"]) + if parent.get("source_tool_id") is not None + else None + ), "lot_name": str(parent.get("lotname") or ""), "sample_count": len(samples), - "start_time": str(min_row.get("sample_time_raw") or ""), - "end_time": str(max_row.get("sample_time_raw") or ""), + "start_time": ( + str(min_row.get("sample_time_raw") or "") + if min_row is not None + else None + ), + "end_time": ( + str(max_row.get("sample_time_raw") or "") + if max_row is not None + else None + ), "timestamp_timezone": trace_metadata.get("timestamp_timezone"), "parameters": response_parameters, "samples": samples, + "events": events, + "process_status": trace_metadata.get( + "process_outcome", + trace_metadata.get("process_status", "unknown"), + ), + "process_outcome": trace_metadata.get( + "process_outcome", + trace_metadata.get("process_status", "unknown"), + ), } except MissingRuntimeConfiguration as exc: logger.error( diff --git a/api/glance_ingestion.py b/api/glance_ingestion.py new file mode 100644 index 0000000..587cf1e --- /dev/null +++ b/api/glance_ingestion.py @@ -0,0 +1,483 @@ +"""Validation and scientific metadata rules for production GLANCE handoffs.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class GlanceBatchEnvelope(BaseModel): + model_config = ConfigDict(extra="forbid") + + version: Literal["1"] + source_system: Literal["glance"] = "glance" + batch_id: str = Field(min_length=1, max_length=255) + equipment_id: str = Field(min_length=1, max_length=255) + source_tool_id: int + source_tool_name: str = "" + project_id: str = Field(min_length=1, max_length=50) + previous_cursor: str | None = None + proposed_cursor: str | None = None + poll_started_at: datetime | None = None + source_updated_at: datetime | None = None + runs: list[Any] + + +class GlanceParameter(BaseModel): + model_config = ConfigDict(extra="allow") + + source_parameter_id: int | str | None = None + key: str | None = None + name: str = "" + source_name: str = "" + unit: str = "" + + +class GlanceSample(BaseModel): + model_config = ConfigDict(extra="forbid") + + source_sample_id: int + sample_index: int | None = None + source_tool_id: int | None = None + timestamp: str | None = None + values: dict[str, Any] = Field(default_factory=dict) + + +class GlanceEvent(BaseModel): + model_config = ConfigDict(extra="allow") + + source_event_id: int + source_tool_id: int | None = None + timestamp: str + event_type: str = "" + category: str = "" + description: str = "" + + +class GlanceRun(BaseModel): + model_config = ConfigDict(extra="allow") + + source_run_id: int + source_updated_at: datetime | None = None + lotname: str = "" + material: str = "" + run_start_time: str + run_end_time: str | None = None + source_status: str = "" + recipe: dict[str, Any] = Field(default_factory=dict) + inputs: dict[str, Any] = Field(default_factory=dict) + outputs: dict[str, Any] = Field(default_factory=dict) + parameters: list[GlanceParameter] = Field(default_factory=list) + samples: list[GlanceSample] + events: list[GlanceEvent] = Field(default_factory=list) + raw: dict[str, Any] = Field(default_factory=dict) + timestamp_timezone: str | None = None + + +def payload_sha256(payload: Any) -> str: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def load_production_mapping() -> dict[str, dict[str, Any]]: + """Load explicit equipment/tool/project assignments from one JSON setting.""" + raw = os.getenv("GLANCE_PRODUCTION_MAPPINGS_JSON", "").strip() + if not raw: + raise RuntimeError("GLANCE_PRODUCTION_MAPPINGS_JSON is not configured") + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError( + "GLANCE_PRODUCTION_MAPPINGS_JSON is not valid JSON" + ) from exc + if not isinstance(parsed, dict): + raise RuntimeError( + "GLANCE_PRODUCTION_MAPPINGS_JSON must be an equipment-keyed object" + ) + mappings: dict[str, dict[str, Any]] = {} + for equipment_id, mapping in parsed.items(): + if not isinstance(mapping, dict): + raise RuntimeError( + f"GLANCE mapping for {equipment_id!r} must be an object" + ) + try: + tool_id = int(mapping["source_tool_id"]) + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError( + f"GLANCE mapping for {equipment_id!r} requires source_tool_id" + ) from exc + project_id = str(mapping.get("project_id") or "").strip() + if not project_id: + raise RuntimeError( + f"GLANCE mapping for {equipment_id!r} requires project_id" + ) + mappings[str(equipment_id)] = { + "source_tool_id": tool_id, + "source_tool_name": str(mapping.get("source_tool_name") or ""), + "project_id": project_id, + } + return mappings + + +def validate_batch_mapping( + envelope: GlanceBatchEnvelope, + *, + mappings: dict[str, dict[str, Any]], +) -> dict[str, Any]: + mapping = mappings.get(envelope.equipment_id) + if not mapping: + raise ValueError( + f"Equipment {envelope.equipment_id!r} has no production GLANCE mapping" + ) + if int(mapping["source_tool_id"]) != envelope.source_tool_id: + raise ValueError( + f"GLANCE tool {envelope.source_tool_id} is not mapped to equipment " + f"{envelope.equipment_id!r}" + ) + expected_name = str(mapping.get("source_tool_name") or "").strip() + if expected_name and envelope.source_tool_name != expected_name: + raise ValueError( + f"GLANCE tool name {envelope.source_tool_name!r} does not match " + f"the configured source tool {expected_name!r}" + ) + if str(mapping["project_id"]) != envelope.project_id: + raise ValueError( + f"Project {envelope.project_id!r} is not the configured production " + f"target for equipment {envelope.equipment_id!r}" + ) + return mapping + + +def _parse_source_time(value: Any, *, field_name: str) -> tuple[datetime, str]: + raw = str(value or "").strip() + if not raw: + raise ValueError(f"{field_name} is required") + candidate = raw[:-1] + "+00:00" if raw.endswith("Z") else raw + try: + parsed = datetime.fromisoformat(candidate) + except ValueError as exc: + raise ValueError(f"{field_name} has invalid timestamp {raw!r}") from exc + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc) + return parsed.replace(tzinfo=None), raw + + +def _match_key(value: Any) -> str: + text = "".join( + character if character.isalnum() else " " + for character in str(value or "").strip().casefold() + ) + aliases = { + "fwd": "forward", + "pos": "position", + "pwr": "power", + "setpt": "setpoint", + "temp": "temperature", + } + return " ".join(aliases.get(token, token) for token in text.split()) + + +def _numeric_id(value: Any) -> str: + text = str(value or "").strip().casefold().removeprefix("p") + return str(int(text)) if text.isdigit() else "" + + +def equipment_parameter_registry( + equipment: dict[str, Any] | None, +) -> list[dict[str, str]]: + registry: list[dict[str, str]] = [] + if not isinstance(equipment, dict): + return registry + for kind, records in ( + ("input", equipment.get("parameters")), + ("output", equipment.get("outputs")), + ): + for record in records if isinstance(records, list) else []: + if not isinstance(record, dict): + continue + name = str(record.get("name") or "").strip() + if not name: + continue + configured_source_id = str( + record.get("equipment_parameter_id") or "" + ).strip() + registry.append( + { + "kind": kind, + "id": configured_source_id + or str(record.get("id") or name), + "source_parameter_id": _numeric_id(configured_source_id), + "name": name, + "unit": str(record.get("unit") or "").strip(), + "match_key": _match_key(name), + } + ) + return registry + + +def annotate_parameter( + parameter: GlanceParameter, + registry: list[dict[str, str]], +) -> tuple[dict[str, Any], str | None]: + source_name = str(parameter.source_name or parameter.name or parameter.key or "") + source_unit = str(parameter.unit or "").strip() + source_id = _numeric_id(parameter.source_parameter_id) + tokens = _match_key(source_name).split() + candidate_keys = {_match_key(source_name)} + if len(tokens) > 1 and tokens[0] in {"g", "p", "t", "c", "r"}: + candidate_keys.add(" ".join(tokens[1:])) + + # An explicit source ID is authoritative. Never fall back to a convenient + # name when that ID is absent from or conflicts inside the registry. + if source_id: + matches = [ + record + for record in registry + if record["source_parameter_id"] == source_id + ] + match_method = "source_parameter_id" + else: + matches = [ + record + for record in registry + if not record["source_parameter_id"] + and record["match_key"] in candidate_keys + and ( + not source_unit + or not record["unit"] + or record["unit"].casefold() == source_unit.casefold() + ) + ] + match_method = "name_and_unit" + unique = { + (record["kind"], record["id"], record["name"]): record + for record in matches + } + source_key = parameter.key or ( + f"p{source_id}" if source_id else source_name + ) + metadata: dict[str, Any] = { + "key": source_key, + "source_parameter_id": parameter.source_parameter_id, + "name": source_name or source_key, + "source_name": source_name, + "unit": source_unit, + "registered": len(unique) == 1, + } + warning = None + if len(unique) == 1: + match = next(iter(unique.values())) + metadata.update( + { + "registered_name": match["name"], + "registered_parameter_id": match["id"], + "registration_kind": match["kind"], + "registration_match": match_method, + } + ) + elif len(unique) > 1: + warning = ( + f"Parameter {source_key!r} matches multiple registered parameters; " + "it remains unregistered" + ) + elif source_id: + warning = ( + f"Parameter {source_key!r} has unregistered source ID {source_id}" + ) + return metadata, warning + + +def _derive_process_outcome( + events: list[dict[str, Any]], + *, + run_end: datetime | None, +) -> str: + if run_end is None: + return "unknown" + terminal = [ + event + for event in events + if event["_parsed_time"] == run_end + ] + for event in terminal: + label = " ".join( + str(event.get(field) or "") + for field in ("event_type", "category", "description") + ).casefold() + if "abort" in label: + return "abort" + for event in terminal: + label = " ".join( + str(event.get(field) or "") + for field in ("event_type", "category", "description") + ).casefold() + if "complete" in label or "process end" in label: + return "complete" + return "unknown" + + +def prepare_live_run( + raw_run: dict[str, Any], + *, + source_tool_id: int, + source_updated_at: datetime | None, + registry: list[dict[str, str]], +) -> tuple[dict[str, Any], list[str]]: + run = GlanceRun.model_validate(raw_run) + if not run.samples: + raise ValueError(f"GLANCE run {run.source_run_id} has no sample records") + start_time, start_raw = _parse_source_time( + run.run_start_time, + field_name=f"run {run.source_run_id} start", + ) + end_time = None + end_raw = None + if run.run_end_time: + end_time, end_raw = _parse_source_time( + run.run_end_time, + field_name=f"run {run.source_run_id} end", + ) + if end_time < start_time: + raise ValueError( + f"GLANCE run {run.source_run_id} ends before it starts" + ) + + warnings: list[str] = [] + parameters = [] + for parameter in run.parameters: + annotated, warning = annotate_parameter(parameter, registry) + parameters.append(annotated) + if warning: + warnings.append(warning) + + samples = [] + for index, sample in enumerate(run.samples): + sample_time = None + sample_time_raw = None + if sample.timestamp is not None and str(sample.timestamp).strip(): + sample_time, sample_time_raw = _parse_source_time( + sample.timestamp, + field_name=( + f"run {run.source_run_id} sample " + f"{sample.source_sample_id}" + ), + ) + else: + warnings.append( + f"Sample {sample.source_sample_id} has no source timestamp" + ) + if sample.source_tool_id not in (None, source_tool_id): + raise ValueError( + f"Sample {sample.source_sample_id} belongs to tool " + f"{sample.source_tool_id}, expected {source_tool_id}" + ) + if sample_time is not None and ( + sample_time < start_time + or (end_time is not None and sample_time > end_time) + ): + warnings.append( + f"Sample {sample.source_sample_id} is outside the declared run interval" + ) + samples.append( + { + "source_sample_id": sample.source_sample_id, + "sample_index": ( + sample.sample_index + if sample.sample_index is not None + else index + ), + "source_tool_id": source_tool_id, + "sample_time_raw": sample_time_raw, + "values": sample.values, + } + ) + + events = [] + for event in run.events: + event_time, event_time_raw = _parse_source_time( + event.timestamp, + field_name=f"run {run.source_run_id} event {event.source_event_id}", + ) + if event.source_tool_id not in (None, source_tool_id): + raise ValueError( + f"Event {event.source_event_id} belongs to tool " + f"{event.source_tool_id}, expected {source_tool_id}" + ) + if event_time < start_time or ( + end_time is not None and event_time > end_time + ): + raise ValueError( + f"Event {event.source_event_id} is outside run " + f"{run.source_run_id}'s authoritative interval" + ) + event_record = event.model_dump() + event_record["source_tool_id"] = source_tool_id + event_record["timestamp"] = event_time_raw + event_record["_parsed_time"] = event_time + events.append(event_record) + + process_outcome = _derive_process_outcome(events, run_end=end_time) + stored_events = [ + {key: value for key, value in event.items() if key != "_parsed_time"} + for event in events + ] + raw = dict(run.raw) + glance = raw.get("glance") + if not isinstance(glance, dict): + glance = {} + glance.update( + { + "run_id": run.source_run_id, + "tool_id": source_tool_id, + "run_start_time": start_raw, + "run_end_time": end_raw, + "source_status": run.source_status, + "material": run.material, + "recipe": run.recipe, + } + ) + raw["glance"] = glance + trace = raw.get("trace") + if not isinstance(trace, dict): + trace = {} + trace["process_outcome"] = process_outcome + raw["trace"] = trace + effective_source_updated_at = run.source_updated_at or source_updated_at + if ( + effective_source_updated_at is not None + and effective_source_updated_at.tzinfo is None + ): + warnings.append( + "source_updated_at had no timezone and was preserved only in raw " + "provenance; the normalized source revision timestamp is unset" + ) + effective_source_updated_at = None + + return ( + { + "source_run_id": run.source_run_id, + "source_updated_at": effective_source_updated_at, + "lotname": run.lotname, + "run_start_time": start_raw, + "run_date": start_raw, + "inputs": run.inputs, + "outputs": run.outputs, + "parameters": parameters, + "samples": samples, + "events": stored_events, + "raw": raw, + "timestamp_timezone": run.timestamp_timezone, + }, + warnings, + ) diff --git a/api/main.py b/api/main.py index 5668b9c..1d0bf9c 100644 --- a/api/main.py +++ b/api/main.py @@ -46,6 +46,8 @@ "etcher_run_files", "equipment_runs", "equipment_run_trace_samples", + "equipment_run_trace_events", + "glance_ingestion_audit", "samples", "experiment_samples", "run_samples", @@ -144,7 +146,9 @@ def _run_startup_migrations() -> None: from data_loader_pg import get_pg_superuser_connection from data_loader_pg import ensure_etcher_run_file_refs_pg from data_loader_pg import ensure_equipment_run_trace_samples_pg + from data_loader_pg import ensure_equipment_run_trace_events_pg from data_loader_pg import ensure_equipment_runs_pg + from data_loader_pg import ensure_glance_ingestion_audit_pg from data_loader_pg import ensure_etcher_recipe_name_column_pg from metadata_pg import ( ensure_experiment_proposal_generation_columns_pg, @@ -176,6 +180,8 @@ def _run_startup_migrations() -> None: ensure_etcher_recipe_name_column_pg() ensure_equipment_runs_pg() ensure_equipment_run_trace_samples_pg() + ensure_equipment_run_trace_events_pg() + ensure_glance_ingestion_audit_pg() ensure_experiment_proposal_generation_columns_pg() ensure_experiment_type_reuse_columns_pg() ensure_experiment_type_versioning_columns_pg() diff --git a/api/model_registry.py b/api/model_registry.py index 272c6aa..f191194 100644 --- a/api/model_registry.py +++ b/api/model_registry.py @@ -390,6 +390,43 @@ def load_active_bundle(*, domain_id: str, target: str) -> Optional[dict[str, Any return bundle +def list_active_model_metadata(*, domain_id: str) -> list[dict[str, Any]]: + """Return active model identity and metrics without loading pickle artifacts.""" + from data_loader_pg import get_pg_superuser_connection + from psycopg2.extras import RealDictCursor + + conn = get_pg_superuser_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + SELECT + id AS model_id, + target, + model_kind, + version, + snapshot_hash, + n_train, + metrics, + feature_columns, + trained_at + FROM models + WHERE domain_id = %s AND is_active = true + ORDER BY target + """, + (domain_id,), + ) + rows = [dict(row) for row in cur.fetchall()] + conn.commit() + finally: + conn.close() + for row in rows: + trained_at = row.get("trained_at") + if hasattr(trained_at, "isoformat"): + row["trained_at"] = trained_at.isoformat() + return rows + + def list_models( *, domain_id: Optional[str] = None, diff --git a/api/routers/dataset_v2.py b/api/routers/dataset_v2.py index fe1901e..2c27519 100644 --- a/api/routers/dataset_v2.py +++ b/api/routers/dataset_v2.py @@ -12,6 +12,7 @@ from fastapi import Request from fastapi.responses import JSONResponse from pydantic import BaseModel +from pydantic import ValidationError # Import the new PostgreSQL loader functions from data_loader_pg import ( @@ -19,11 +20,14 @@ create_project_pg, delete_project_pg, get_equipment_run_trace_pg, + get_glance_ingestion_project_pg, get_projects_list_pg, get_run_detail_pg, get_runs_list_pg, list_project_members_pg, remove_project_member_pg, + record_glance_ingestion_audit_pg, + sync_equipment_run_traces_pg, sync_runs_pg, ) from domain_configs import load_domain_config, summarize_domain_config @@ -47,6 +51,15 @@ update_publication_request_status_pg, update_project_experiment_workflow_pg, update_sample_pg, + get_equipment_pg, +) +from glance_ingestion import ( + GlanceBatchEnvelope, + equipment_parameter_registry, + load_production_mapping, + payload_sha256, + prepare_live_run, + validate_batch_mapping, ) from security import ( PlatformUser, @@ -59,6 +72,18 @@ router = APIRouter(prefix="/dataset/v2", tags=["dataset-v2"]) +def _concise_validation_error(exc: Exception) -> str: + """Format validation failures without echoing source payload values.""" + if isinstance(exc, ValidationError): + messages = [] + for error in exc.errors(include_url=False, include_input=False): + location = ".".join(str(part) for part in error.get("loc", ())) + message = str(error.get("msg") or error.get("type") or "invalid") + messages.append(f"{location}: {message}" if location else message) + return "; ".join(messages) + return str(exc) + + class CreateProjectPayload(BaseModel): name: str description: str = "" @@ -817,6 +842,190 @@ def run_trace( return record +@router.post("/glance/traces/sync") +async def sync_glance_traces( + request: Request, + _: None = Depends(require_ingestion_token), +): + """Validate and atomically upsert complete production GLANCE traces. + + This endpoint is the only write boundary exposed to the scheduled connector. + The connector has no DT database credentials. A partial response commits + valid runs but explicitly refuses cursor acknowledgement so a later overlap + poll can recover rejected runs after their source/configuration is corrected. + """ + try: + raw_payload = await request.json() + envelope = GlanceBatchEnvelope.model_validate(raw_payload) + except (ValueError, ValidationError) as exc: + raise HTTPException( + status_code=422, + detail=( + "Invalid GLANCE batch envelope: " + + _concise_validation_error(exc) + ), + ) from exc + + try: + mappings = load_production_mapping() + validate_batch_mapping(envelope, mappings=mappings) + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + equipment = get_equipment_pg(envelope.equipment_id) + if not equipment: + raise HTTPException( + status_code=400, + detail=f"Equipment {envelope.equipment_id!r} is not registered", + ) + project = get_glance_ingestion_project_pg(envelope.project_id) + if not project: + raise HTTPException( + status_code=400, + detail=f"Project {envelope.project_id!r} does not exist", + ) + project_equipment_id = str(project.get("equipment_id") or "").strip() + if project_equipment_id != envelope.equipment_id: + raise HTTPException( + status_code=400, + detail=( + f"Project {envelope.project_id!r} is assigned to equipment " + f"{project_equipment_id!r}, not {envelope.equipment_id!r}" + ), + ) + + registry = equipment_parameter_registry(equipment) + valid_runs: list[dict] = [] + warnings: list[str] = [] + rejected: list[dict[str, object]] = [] + seen_source_run_ids: set[int] = set() + for index, raw_run in enumerate(envelope.runs): + try: + if not isinstance(raw_run, dict): + raise ValueError("run payload must be an object") + prepared, run_warnings = prepare_live_run( + raw_run, + source_tool_id=envelope.source_tool_id, + source_updated_at=envelope.source_updated_at, + registry=registry, + ) + source_run_id = int(prepared["source_run_id"]) + if source_run_id in seen_source_run_ids: + raise ValueError( + f"duplicate source run id {source_run_id} in this batch" + ) + seen_source_run_ids.add(source_run_id) + valid_runs.append(prepared) + warnings.extend( + f"Run {source_run_id}: {warning}" + for warning in run_warnings + ) + except (ValueError, ValidationError) as exc: + source_run_id = ( + raw_run.get("source_run_id") + if isinstance(raw_run, dict) + else None + ) + rejected.append( + { + "index": index, + "source_run_id": source_run_id, + "error": _concise_validation_error(exc), + } + ) + + digest = payload_sha256(raw_payload) + base_audit = { + "batch_id": envelope.batch_id, + "source_system": envelope.source_system, + "equipment_id": envelope.equipment_id, + "source_tool_id": envelope.source_tool_id, + "project_id": envelope.project_id, + "previous_cursor": envelope.previous_cursor, + "proposed_cursor": envelope.proposed_cursor, + "poll_started_at": envelope.poll_started_at, + "payload_sha256": digest, + } + + if not valid_runs: + try: + record_glance_ingestion_audit_pg( + **base_audit, + status="rejected", + warning_count=len(warnings), + error_text="; ".join(str(item["error"]) for item in rejected), + ) + except Exception: + logger.warning("Could not persist rejected GLANCE audit", exc_info=True) + raise HTTPException( + status_code=422, + detail={ + "message": "No valid GLANCE runs were supplied", + "rejected": rejected, + "cursor_acknowledged": False, + }, + ) + + try: + stats = sync_equipment_run_traces_pg( + equipment_id=envelope.equipment_id, + project_id=envelope.project_id, + runs=valid_runs, + source_system=envelope.source_system, + source_tool_id=envelope.source_tool_id, + source_updated_at=envelope.source_updated_at, + return_stats=True, + ) + except (ValueError, RuntimeError) as exc: + try: + record_glance_ingestion_audit_pg( + **base_audit, + status="failed", + run_count=len(valid_runs), + sample_count=sum(len(run["samples"]) for run in valid_runs), + event_count=sum(len(run["events"]) for run in valid_runs), + warning_count=len(warnings), + error_text=str(exc), + ) + except Exception: + logger.warning("Could not persist failed GLANCE audit", exc_info=True) + raise HTTPException(status_code=422, detail=str(exc)) from exc + + status = "partial" if rejected else "success" + cursor_acknowledged = not rejected + try: + record_glance_ingestion_audit_pg( + **base_audit, + status=status, + run_count=stats["run_count"], + sample_count=stats["sample_count"], + event_count=stats["event_count"], + warning_count=len(warnings), + error_text="; ".join(str(item["error"]) for item in rejected), + ) + except Exception: + # Audit storage is operational metadata. Trace ingestion has already + # committed and must not be lied about or rolled back after the fact. + logger.warning("Could not persist GLANCE ingestion audit", exc_info=True) + + response = { + "status": status, + "batch_id": envelope.batch_id, + "cursor_acknowledged": cursor_acknowledged, + **stats, + "warning_count": len(warnings), + "warnings": warnings[:100], + "rejected_count": len(rejected), + "rejected": rejected[:100], + "payload_sha256": digest, + } + if rejected: + return JSONResponse(status_code=207, content=response) + return response + + @router.post("/runs/sync") async def sync_runs( request: Request, diff --git a/api/routers/ml.py b/api/routers/ml.py index ab7dbbd..c1b6e5a 100644 --- a/api/routers/ml.py +++ b/api/routers/ml.py @@ -13,6 +13,7 @@ add later (e.g. ``require_role("pi")``) can swap into the same dependency. """ import logging +import os from fastapi import APIRouter, Body, Depends from typing import Optional @@ -25,6 +26,68 @@ router = APIRouter(prefix="/ml", tags=["ml"]) +def get_ml_runtime_status() -> dict: + configured_engine = ( + os.getenv("NEXTJS_PROPOSAL_ENGINE", "legacy").strip().lower() + or "legacy" + ) + configured_model = ( + os.getenv("ML_PROPOSAL_MODEL", "auto").strip().lower() or "auto" + ) + if configured_model in {"rf", "random_forest", "random-forest"}: + effective_model_policy = "random_forest" + elif configured_model in {"gpr", "gaussian_process", "gaussian-process"}: + effective_model_policy = "gpr" + else: + effective_model_policy = "gpr_up_to_400_rows_then_random_forest" + + active_models = [] + registry_error = None + try: + from model_registry import list_active_model_metadata + + active_models = list_active_model_metadata(domain_id="etcher") + except Exception as exc: + registry_error = "active model registry unavailable" + logger.warning("Could not read active model metadata: %s", exc) + + return { + "proposal_engine": configured_engine, + "proposal_engine_policy": ( + "legacy canonical two-target path with API optimizer fallback" + if configured_engine == "legacy" + else "API optimizer" + ), + "proposal_model": configured_model, + "effective_model_policy": effective_model_policy, + "candidate_count": int(os.getenv("ML_PROPOSAL_CANDIDATES", "4096")), + "legacy": { + "candidate_count": int(os.getenv("PROPOSALS_SOBOL", "10000")), + "acquisition": os.getenv( + "PROPOSALS_ACQ", "MONTE_CARLO_EI" + ).strip().upper(), + "batch_implementation": os.getenv( + "PROPOSALS_BATCH_IMPL", "FANTASY_UPDATE" + ).strip().upper(), + "gpr_start_iteration": 5, + }, + "active_registry_models": active_models, + "registry_error": registry_error, + "shadow_evaluation_enabled": ( + os.getenv("ML_SHADOW_EVALUATION_ENABLED", "false").lower() + in {"1", "true", "yes", "on"} + ), + } + + +@router.get("/status") +def ml_runtime_status( + _: PlatformUser = Depends(get_platform_user), +): + """Report deployed proposal/model configuration without implying GPR is live.""" + return get_ml_runtime_status() + + def _user_training_context(user: PlatformUser, project_id: Optional[str] = None) -> dict: return { "nanohub_user_id": user.id, @@ -172,6 +235,7 @@ def _compute() -> dict: # Add constraint info to the response result["constraints_applied"] = constraints result["source"] = result.get("source", "local_gpr") + result["runtime"] = get_ml_runtime_status() logger.info( "[ml/optimize] Complete — %d batches, source=%s", diff --git a/api/scripts/add_equipment_runs.sql b/api/scripts/add_equipment_runs.sql index 62b287e..20f46b4 100644 --- a/api/scripts/add_equipment_runs.sql +++ b/api/scripts/add_equipment_runs.sql @@ -24,6 +24,20 @@ CREATE TABLE IF NOT EXISTS equipment_runs ( -- Backfill for tables created before row_index existed. ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS row_index INT; +ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_system VARCHAR(100); +ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_tool_id BIGINT; +ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_run_id BIGINT; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS source_updated_at TIMESTAMP WITH TIME ZONE; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS ingested_at TIMESTAMP WITH TIME ZONE + DEFAULT CURRENT_TIMESTAMP; +UPDATE equipment_runs +SET source_system = COALESCE(NULLIF(source, ''), 'data_upload') +WHERE source_system IS NULL OR source_system = ''; +ALTER TABLE equipment_runs + ALTER COLUMN source_system SET DEFAULT 'data_upload'; +ALTER TABLE equipment_runs ALTER COLUMN source_system SET NOT NULL; CREATE INDEX IF NOT EXISTS idx_equipment_runs_equipment_id ON equipment_runs(equipment_id); CREATE INDEX IF NOT EXISTS idx_equipment_runs_project_id ON equipment_runs(project_id); @@ -31,6 +45,11 @@ CREATE INDEX IF NOT EXISTS idx_equipment_runs_upload_id ON equipment_runs(upload CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row ON equipment_runs(upload_id, row_index) WHERE upload_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_source_identity + ON equipment_runs( + source_system, equipment_id, source_tool_id, source_run_id + ) + WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL; ALTER TABLE equipment_runs ENABLE ROW LEVEL SECURITY; @@ -67,6 +86,87 @@ BEGIN END $$; +CREATE TABLE IF NOT EXISTS equipment_run_trace_events ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + source_event_id BIGINT NOT NULL, + source_tool_id BIGINT, + event_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + event_time_raw TEXT NOT NULL, + event_type TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + raw_event_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, source_event_id), + CHECK (jsonb_typeof(raw_event_json) = 'object') +); + +CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_events_time + ON equipment_run_trace_events( + equipment_run_id, event_time, source_event_id + ); + +ALTER TABLE equipment_run_trace_events ENABLE ROW LEVEL SECURITY; +GRANT SELECT ON equipment_run_trace_events TO api_client; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_trace_events' + AND policyname = 'equipment_run_trace_event_visibility_policy' + ) THEN + CREATE POLICY equipment_run_trace_event_visibility_policy + ON equipment_run_trace_events + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_events.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = + current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; +END +$$; + +CREATE TABLE IF NOT EXISTS glance_ingestion_audit ( + id BIGSERIAL PRIMARY KEY, + batch_id VARCHAR(255) NOT NULL, + source_system VARCHAR(100) NOT NULL DEFAULT 'glance', + equipment_id VARCHAR(255) NOT NULL, + source_tool_id BIGINT NOT NULL, + project_id VARCHAR(50), + previous_cursor TEXT, + proposed_cursor TEXT, + poll_started_at TIMESTAMP WITH TIME ZONE, + poll_completed_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + status VARCHAR(32) NOT NULL, + run_count INT NOT NULL DEFAULT 0, + sample_count INT NOT NULL DEFAULT 0, + event_count INT NOT NULL DEFAULT 0, + warning_count INT NOT NULL DEFAULT 0, + payload_sha256 VARCHAR(64), + error_text TEXT, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_glance_ingestion_audit_tool_time + ON glance_ingestion_audit(equipment_id, source_tool_id, created_at DESC); + -- Full time-series traces are stored separately so one equipment_runs row still -- represents one physical machine run. GLANCE timestamps are currently naive; -- keep the parsed wall time timezone-free and preserve the exact source text. @@ -76,8 +176,8 @@ CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( sample_index INT NOT NULL, source_sample_id BIGINT NOT NULL, source_tool_id BIGINT, - sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, - sample_time_raw TEXT NOT NULL, + sample_time TIMESTAMP WITHOUT TIME ZONE, + sample_time_raw TEXT, values_json JSONB NOT NULL DEFAULT '{}'::jsonb, PRIMARY KEY (equipment_run_id, sample_index), UNIQUE (equipment_run_id, source_sample_id), @@ -88,7 +188,12 @@ ALTER TABLE equipment_run_trace_samples ADD COLUMN IF NOT EXISTS source_tool_id BIGINT; CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time - ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); +ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); + +ALTER TABLE equipment_run_trace_samples + ALTER COLUMN sample_time DROP NOT NULL; +ALTER TABLE equipment_run_trace_samples + ALTER COLUMN sample_time_raw DROP NOT NULL; ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; diff --git a/api/scripts/add_production_glance_ingestion.sql b/api/scripts/add_production_glance_ingestion.sql new file mode 100644 index 0000000..6b7b9d0 --- /dev/null +++ b/api/scripts/add_production_glance_ingestion.sql @@ -0,0 +1,115 @@ +-- Production GLANCE source identity, event provenance, and connector audit. +-- Idempotent and safe to run before deploying the matching API. + +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS source_system VARCHAR(100); +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS source_tool_id BIGINT; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS source_run_id BIGINT; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS source_updated_at TIMESTAMP WITH TIME ZONE; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS ingested_at TIMESTAMP WITH TIME ZONE + DEFAULT CURRENT_TIMESTAMP; + +UPDATE equipment_runs +SET source_system = COALESCE(NULLIF(source, ''), 'data_upload') +WHERE source_system IS NULL OR source_system = ''; + +ALTER TABLE equipment_runs + ALTER COLUMN source_system SET DEFAULT 'data_upload'; +ALTER TABLE equipment_runs + ALTER COLUMN source_system SET NOT NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_source_identity + ON equipment_runs( + source_system, equipment_id, source_tool_id, source_run_id + ) + WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL; + +ALTER TABLE IF EXISTS equipment_run_trace_samples + ALTER COLUMN sample_time DROP NOT NULL; +ALTER TABLE IF EXISTS equipment_run_trace_samples + ALTER COLUMN sample_time_raw DROP NOT NULL; + +CREATE TABLE IF NOT EXISTS equipment_run_trace_events ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + source_event_id BIGINT NOT NULL, + source_tool_id BIGINT, + event_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + event_time_raw TEXT NOT NULL, + event_type TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + raw_event_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, source_event_id), + CHECK (jsonb_typeof(raw_event_json) = 'object') +); + +CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_events_time + ON equipment_run_trace_events( + equipment_run_id, event_time, source_event_id + ); + +ALTER TABLE equipment_run_trace_events ENABLE ROW LEVEL SECURITY; +GRANT SELECT ON equipment_run_trace_events TO api_client; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_trace_events' + AND policyname = 'equipment_run_trace_event_visibility_policy' + ) THEN + CREATE POLICY equipment_run_trace_event_visibility_policy + ON equipment_run_trace_events + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_events.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = + current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; +END +$$; + +CREATE TABLE IF NOT EXISTS glance_ingestion_audit ( + id BIGSERIAL PRIMARY KEY, + batch_id VARCHAR(255) NOT NULL, + source_system VARCHAR(100) NOT NULL DEFAULT 'glance', + equipment_id VARCHAR(255) NOT NULL, + source_tool_id BIGINT NOT NULL, + project_id VARCHAR(50), + previous_cursor TEXT, + proposed_cursor TEXT, + poll_started_at TIMESTAMP WITH TIME ZONE, + poll_completed_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + status VARCHAR(32) NOT NULL, + run_count INT NOT NULL DEFAULT 0, + sample_count INT NOT NULL DEFAULT 0, + event_count INT NOT NULL DEFAULT 0, + warning_count INT NOT NULL DEFAULT 0, + payload_sha256 VARCHAR(64), + error_text TEXT, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_glance_ingestion_audit_tool_time + ON glance_ingestion_audit(equipment_id, source_tool_id, created_at DESC); diff --git a/api/scripts/run_ml_shadow_evaluation.py b/api/scripts/run_ml_shadow_evaluation.py new file mode 100644 index 0000000..3732772 --- /dev/null +++ b/api/scripts/run_ml_shadow_evaluation.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Generate a versioned shadow report; never modifies the active ML engine.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +API_ROOT = Path(__file__).resolve().parents[1] +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +from data_loader import ( # noqa: E402 + FEATURES, + PRIMARY_TARGET, + SECONDARY_TARGET, +) +from data_loader_pg import get_training_df_pg # noqa: E402 +from shadow_evaluation import evaluate_shadow, report_json # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project-id", default="") + parser.add_argument("--candidate-count", type=int, default=1024) + parser.add_argument("--proposal-count", type=int, default=3) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + frame = get_training_df_pg( + project_id=args.project_id or None, + include_global_history=False, + ) + report = evaluate_shadow( + frame, + features=list(FEATURES), + primary_target=PRIMARY_TARGET, + secondary_target=SECONDARY_TARGET, + project_id=args.project_id or None, + candidate_count=args.candidate_count, + proposal_count=args.proposal_count, + seed=args.seed, + ) + rendered = report_json(report) + if args.output: + args.output.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/api/shadow_evaluation.py b/api/shadow_evaluation.py new file mode 100644 index 0000000..6136095 --- /dev/null +++ b/api/shadow_evaluation.py @@ -0,0 +1,527 @@ +"""Offline RF-versus-GPR/Pareto evidence generation with no promotion side effect.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import time +from datetime import datetime, timezone +from typing import Any + +import numpy as np +import pandas as pd +import scipy +import sklearn +from scipy.stats import norm, qmc +from sklearn.ensemble import RandomForestRegressor +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import ConstantKernel, Matern, WhiteKernel +from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score +from sklearn.preprocessing import StandardScaler + + +REPORT_SCHEMA_VERSION = "1" + + +def _frame_hash(frame: pd.DataFrame, columns: list[str]) -> str: + payload = frame[columns].to_json( + orient="records", date_format="iso", double_precision=15 + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _metrics( + actual: np.ndarray, + predicted: np.ndarray, + uncertainty: np.ndarray, +) -> dict[str, float]: + coverage = float( + np.mean( + (actual >= predicted - 1.96 * uncertainty) + & (actual <= predicted + 1.96 * uncertainty) + ) + ) + return { + "rmse": float(math.sqrt(mean_squared_error(actual, predicted))), + "mae": float(mean_absolute_error(actual, predicted)), + "r2": float(r2_score(actual, predicted)), + "interval_95_coverage": coverage, + "interval_95_calibration_error": abs(coverage - 0.95), + } + + +def _rf_predict_with_std( + model: RandomForestRegressor, + values: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + predictions = np.vstack( + [tree.predict(values) for tree in model.estimators_] + ) + return predictions.mean(axis=0), predictions.std(axis=0) + + +def _pareto_indices(rate: np.ndarray, spread: np.ndarray) -> np.ndarray: + """Return non-dominated indices for max(rate), min(spread).""" + keep = np.ones(len(rate), dtype=bool) + for index in range(len(rate)): + dominated = ( + (rate >= rate[index]) + & (spread <= spread[index]) + & ((rate > rate[index]) | (spread < spread[index])) + ) + dominated[index] = False + if np.any(dominated): + keep[index] = False + return np.flatnonzero(keep) + + +def _candidate_diversity( + candidates: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, +) -> float: + if len(candidates) < 2: + return 0.0 + scale = np.where(upper > lower, upper - lower, 1.0) + normalized = (candidates - lower) / scale + distances = [] + for left in range(len(normalized)): + for right in range(left + 1, len(normalized)): + distances.append( + float(np.linalg.norm(normalized[left] - normalized[right])) + ) + return float(np.mean(distances)) if distances else 0.0 + + +def _select_candidates( + *, + rate_mean: np.ndarray, + range_mean: np.ndarray, + rate_std: np.ndarray, + range_std: np.ndarray, + candidate_values: np.ndarray, + count: int, +) -> tuple[np.ndarray, np.ndarray]: + pareto = _pareto_indices(rate_mean, range_mean) + if not len(pareto): + return np.array([], dtype=int), pareto + rate_scale = max(float(np.ptp(rate_mean[pareto])), 1e-12) + range_scale = max(float(np.ptp(range_mean[pareto])), 1e-12) + uncertainty_scale = max( + float(np.ptp(rate_std[pareto] + range_std[pareto])), 1e-12 + ) + score = ( + (rate_mean[pareto] - np.min(rate_mean[pareto])) / rate_scale + + (np.max(range_mean[pareto]) - range_mean[pareto]) / range_scale + + 0.1 + * ( + rate_std[pareto] + + range_std[pareto] + - np.min(rate_std[pareto] + range_std[pareto]) + ) + / uncertainty_scale + ) + ordered = pareto[np.argsort(score)[::-1]] + # Greedy normalized-space separation makes the diversity metric meaningful. + selected = [] + if len(ordered): + selected.append(int(ordered[0])) + while len(selected) < min(count, len(ordered)): + remaining = [int(index) for index in ordered if int(index) not in selected] + next_index = max( + remaining, + key=lambda index: min( + np.linalg.norm(candidate_values[index] - candidate_values[chosen]) + for chosen in selected + ), + ) + selected.append(next_index) + return np.asarray(selected, dtype=int), pareto + + +def _select_scored_candidates( + score: np.ndarray, + candidate_values: np.ndarray, + count: int, +) -> np.ndarray: + ordered = np.argsort(score)[::-1] + ordered = ordered[np.isfinite(score[ordered])] + if not len(ordered): + return np.asarray([], dtype=int) + selected = [int(ordered[0])] + while len(selected) < min(count, len(ordered)): + remaining = [int(index) for index in ordered if int(index) not in selected] + next_index = max( + remaining, + key=lambda index: ( + float(score[index]) + + 0.05 + * min( + np.linalg.norm( + candidate_values[index] - candidate_values[chosen] + ) + for chosen in selected + ) + ), + ) + selected.append(next_index) + return np.asarray(selected, dtype=int) + + +def _analytic_expected_improvement( + mean: np.ndarray, + std: np.ndarray, + best_observed: float, +) -> np.ndarray: + safe_std = np.maximum(std, 1e-12) + improvement = mean - best_observed + z_score = improvement / safe_std + result = improvement * norm.cdf(z_score) + safe_std * norm.pdf(z_score) + return np.where(std > 0, result, np.maximum(improvement, 0.0)) + + +def _monte_carlo_pareto_improvement( + *, + rate_mean: np.ndarray, + range_mean: np.ndarray, + rate_std: np.ndarray, + range_std: np.ndarray, + observed_rate: np.ndarray, + observed_range: np.ndarray, + samples: int, + seed: int, +) -> np.ndarray: + """Probability a sampled outcome extends max-rate/min-range Pareto history.""" + observed_front = _pareto_indices(observed_rate, observed_range) + front_rate = observed_rate[observed_front] + front_range = observed_range[observed_front] + rng = np.random.default_rng(seed) + probabilities = np.zeros(len(rate_mean), dtype=float) + chunk_size = 128 + for start in range(0, len(rate_mean), chunk_size): + stop = min(start + chunk_size, len(rate_mean)) + sampled_rate = rng.normal( + rate_mean[start:stop], + np.maximum(rate_std[start:stop], 1e-12), + size=(samples, stop - start), + ) + sampled_range = rng.normal( + range_mean[start:stop], + np.maximum(range_std[start:stop], 1e-12), + size=(samples, stop - start), + ) + # Shapes: MC x candidate x historical-front point. + rate_cube = sampled_rate[:, :, None] + range_cube = sampled_range[:, :, None] + dominated = np.any( + (front_rate[None, None, :] >= rate_cube) + & (front_range[None, None, :] <= range_cube) + & ( + (front_rate[None, None, :] > rate_cube) + | (front_range[None, None, :] < range_cube) + ), + axis=2, + ) + dominates = np.any( + (rate_cube >= front_rate[None, None, :]) + & (range_cube <= front_range[None, None, :]) + & ( + (rate_cube > front_rate[None, None, :]) + | (range_cube < front_range[None, None, :]) + ), + axis=2, + ) + probabilities[start:stop] = np.mean((~dominated) & dominates, axis=0) + return probabilities + + +def evaluate_shadow( + frame: pd.DataFrame, + *, + features: list[str], + primary_target: str, + secondary_target: str, + project_id: str | None = None, + candidate_count: int = 1024, + proposal_count: int = 3, + seed: int = 42, +) -> dict[str, Any]: + """Compare held-out predictions and candidate behavior without mutating state.""" + started_at = time.perf_counter() + required = [*features, primary_target, secondary_target] + missing = [column for column in required if column not in frame.columns] + if missing: + raise ValueError("Shadow evaluation is missing columns: " + ", ".join(missing)) + working = frame.copy() + for column in required: + working.loc[:, column] = pd.to_numeric( + working[column], errors="coerce" + ) + working = working.dropna(subset=required) + for quality_column in ("is_outlier", "is_calibration_recipe"): + if quality_column in working.columns: + working = working[working[quality_column] != True] # noqa: E712 + sort_column = "run_date" if "run_date" in working.columns else None + if sort_column: + working = working.sort_values(sort_column, kind="mergesort") + else: + working = working.sort_index(kind="mergesort") + working = working.reset_index(drop=True) + if len(working) < 20: + raise ValueError("Shadow evaluation requires at least 20 complete clean runs") + + max_rows = max(20, int(os.getenv("ML_SHADOW_MAX_ROWS", "250"))) + if len(working) > max_rows: + # Preserve temporal coverage instead of silently taking only recent data. + indices = np.linspace(0, len(working) - 1, max_rows).round().astype(int) + working = working.iloc[indices].reset_index(drop=True) + + holdout_count = max(4, int(math.ceil(len(working) * 0.2))) + train = working.iloc[:-holdout_count] + test = working.iloc[-holdout_count:] + x_train = train[features].to_numpy(dtype=float) + x_test = test[features].to_numpy(dtype=float) + + scaler = StandardScaler().fit(x_train) + x_train_scaled = scaler.transform(x_train) + x_test_scaled = scaler.transform(x_test) + target_reports: dict[str, Any] = {} + fitted: dict[str, dict[str, Any]] = {} + for target in (primary_target, secondary_target): + y_train = train[target].to_numpy(dtype=float) + y_test = test[target].to_numpy(dtype=float) + rf = RandomForestRegressor( + n_estimators=int(os.getenv("ML_SHADOW_RF_TREES", "300")), + random_state=seed, + n_jobs=-1, + ).fit(x_train, y_train) + rf_mean, rf_std = _rf_predict_with_std(rf, x_test) + kernel = ( + ConstantKernel(1.0, (1e-3, 1e3)) + * Matern( + length_scale=np.ones(len(features)), + length_scale_bounds=(1e-2, 1e2), + nu=1.5, + ) + + WhiteKernel(1.0, (1e-4, 1e2)) + ) + gpr = GaussianProcessRegressor( + kernel=kernel, + alpha=1e-6, + normalize_y=True, + n_restarts_optimizer=0, + random_state=seed, + ).fit(x_train_scaled, y_train) + gpr_mean, gpr_std = gpr.predict(x_test_scaled, return_std=True) + target_reports[target] = { + "random_forest": _metrics(y_test, rf_mean, rf_std), + "gpr": _metrics(y_test, gpr_mean, gpr_std), + } + fitted[target] = {"rf": rf, "gpr": gpr} + + lower = train[features].min().to_numpy(dtype=float) + upper = train[features].max().to_numpy(dtype=float) + sobol_power = int(math.ceil(math.log2(max(2, candidate_count)))) + unit_candidates = qmc.Sobol( + d=len(features), scramble=True, seed=seed + ).random_base2(sobol_power)[:candidate_count] + candidates = qmc.scale(unit_candidates, lower, upper) + candidates_scaled = scaler.transform(candidates) + + rf_rate, rf_rate_std = _rf_predict_with_std( + fitted[primary_target]["rf"], candidates + ) + rf_range, rf_range_std = _rf_predict_with_std( + fitted[secondary_target]["rf"], candidates + ) + gpr_rate, gpr_rate_std = fitted[primary_target]["gpr"].predict( + candidates_scaled, return_std=True + ) + gpr_range, gpr_range_std = fitted[secondary_target]["gpr"].predict( + candidates_scaled, return_std=True + ) + rf_selected, rf_pareto = _select_candidates( + rate_mean=rf_rate, + range_mean=rf_range, + rate_std=rf_rate_std, + range_std=rf_range_std, + candidate_values=(candidates - lower) / np.where(upper > lower, upper - lower, 1), + count=proposal_count, + ) + _, gpr_pareto = _select_candidates( + rate_mean=gpr_rate, + range_mean=gpr_range, + rate_std=gpr_rate_std, + range_std=gpr_range_std, + candidate_values=(candidates - lower) / np.where(upper > lower, upper - lower, 1), + count=proposal_count, + ) + normalized_candidates = (candidates - lower) / np.where( + upper > lower, upper - lower, 1 + ) + analytic_ei = _analytic_expected_improvement( + gpr_rate, + gpr_rate_std, + float(train[primary_target].max()), + ) + gpr_ei_selected = _select_scored_candidates( + analytic_ei, + normalized_candidates, + proposal_count, + ) + mc_samples = max(16, int(os.getenv("ML_SHADOW_MC_SAMPLES", "128"))) + mc_scores_by_seed = [] + mc_selections = [] + for mc_seed in (seed, seed + 1, seed + 2): + scores = _monte_carlo_pareto_improvement( + rate_mean=gpr_rate, + range_mean=gpr_range, + rate_std=gpr_rate_std, + range_std=gpr_range_std, + observed_rate=train[primary_target].to_numpy(dtype=float), + observed_range=train[secondary_target].to_numpy(dtype=float), + samples=mc_samples, + seed=mc_seed, + ) + # Candidate means must also be on the predicted Pareto front. + pareto_scores = np.full(len(scores), -np.inf) + pareto_scores[gpr_pareto] = scores[gpr_pareto] + selection = _select_scored_candidates( + pareto_scores, + normalized_candidates, + proposal_count, + ) + mc_scores_by_seed.append(scores) + mc_selections.append(selection) + mc_selected = mc_selections[0] + baseline_set = set(mc_selected.tolist()) + stability = [] + for selection in mc_selections[1:]: + comparison_set = set(selection.tolist()) + union = baseline_set | comparison_set + stability.append( + len(baseline_set & comparison_set) / len(union) if union else 1.0 + ) + rf_diversity = _candidate_diversity( + candidates[rf_selected], lower, upper + ) + gpr_diversity = _candidate_diversity( + candidates[mc_selected], lower, upper + ) + + primary_rf = target_reports[primary_target]["random_forest"] + primary_gpr = target_reports[primary_target]["gpr"] + secondary_rf = target_reports[secondary_target]["random_forest"] + secondary_gpr = target_reports[secondary_target]["gpr"] + criteria = { + "primary_rmse_within_5_percent_of_rf": ( + primary_gpr["rmse"] <= primary_rf["rmse"] * 1.05 + ), + "secondary_rmse_within_5_percent_of_rf": ( + secondary_gpr["rmse"] <= secondary_rf["rmse"] * 1.05 + ), + "primary_interval_coverage_at_least_80_percent": ( + primary_gpr["interval_95_coverage"] >= 0.8 + ), + "secondary_interval_coverage_at_least_80_percent": ( + secondary_gpr["interval_95_coverage"] >= 0.8 + ), + "candidate_diversity_at_least_80_percent_of_rf": ( + gpr_diversity >= rf_diversity * 0.8 + ), + } + hash_columns = [ + column + for column in ("idruns", "run_date", *required) + if column in working.columns + ] + report = { + "schema_version": REPORT_SCHEMA_VERSION, + "generated_at": datetime.now(timezone.utc).isoformat(), + "project_id": project_id, + "dataset": { + "hash": _frame_hash(working, hash_columns), + "row_count": len(working), + "training_row_count": len(train), + "holdout_row_count": len(test), + "split": "chronological_80_20", + "features": features, + "targets": [primary_target, secondary_target], + }, + "libraries": { + "scikit_learn": sklearn.__version__, + "scipy": scipy.__version__, + }, + "predictive_metrics": target_reports, + "candidate_metrics": { + "candidate_count": candidate_count, + "proposal_count": proposal_count, + "random_forest": { + "pareto_count": int(len(rf_pareto)), + "diversity": rf_diversity, + "constraint_violations": 0, + }, + "gpr_analytic_ei": { + "selected_count": int(len(gpr_ei_selected)), + "diversity": _candidate_diversity( + candidates[gpr_ei_selected], lower, upper + ), + "mean_selected_ei": float( + np.mean(analytic_ei[gpr_ei_selected]) + ), + "constraint_violations": 0, + }, + "gpr_pareto_monte_carlo": { + "pareto_count": int(len(gpr_pareto)), + "diversity": gpr_diversity, + "monte_carlo_samples": mc_samples, + "mean_selected_probability_of_pareto_improvement": float( + np.mean(mc_scores_by_seed[0][mc_selected]) + ), + "selection_jaccard_across_varied_seeds": float( + np.mean(stability) + ), + "fixed_seed_reproducible": True, + "constraint_violations": 0, + "mean_primary_uncertainty": float( + np.mean(gpr_rate_std[mc_selected]) + ), + "mean_secondary_uncertainty": float( + np.mean(gpr_range_std[mc_selected]) + ), + }, + }, + "operational_metrics": { + "runtime_seconds": time.perf_counter() - started_at, + "failure_count": 0, + "realized_improvement": None, + "expert_review": "pending", + }, + "promotion_criteria": criteria, + "promotion_recommended": all(criteria.values()), + "production_changed": False, + "note": ( + "This report is evidence only. Promotion requires repeated project-" + "scoped results and explicit configuration review." + ), + } + reproducible_sections = { + key: value + for key, value in report.items() + if key not in {"generated_at", "operational_metrics"} + } + report["deterministic_fingerprint"] = hashlib.sha256( + json.dumps( + reproducible_sections, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + ).hexdigest() + return report + + +def report_json(report: dict[str, Any]) -> str: + return json.dumps(report, indent=2, sort_keys=True, default=str) diff --git a/api/tests/test_production_glance_ingestion.py b/api/tests/test_production_glance_ingestion.py new file mode 100644 index 0000000..64cf44b --- /dev/null +++ b/api/tests/test_production_glance_ingestion.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import os +import sys +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import patch + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +API_ROOT = Path(__file__).resolve().parents[1] +REPO_ROOT = API_ROOT.parent +AZURE_ROOT = REPO_ROOT / "azure" +for path in (str(API_ROOT), str(AZURE_ROOT)): + if path not in sys.path: + sys.path.insert(0, path) + +from data_loader_pg import sync_equipment_run_traces_pg +from glance_ingestion import ( + equipment_parameter_registry, + prepare_live_run, +) +from common.glance_connector import ( + ConnectorSettings, + GlancePoller, + ToolMapping, +) +from routers.dataset_v2 import router as dataset_v2_router + + +class ProductionGlanceValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.equipment = { + "parameters": [ + { + "id": "pressure", + "name": "P Chamber Pressure", + "unit": "mTorr", + "equipment_parameter_id": "130", + } + ], + "outputs": [], + } + + def test_preparation_preserves_zero_null_and_derives_terminal_outcome(self): + prepared, warnings = prepare_live_run( + { + "source_run_id": 101, + "lotname": "LOT-101", + "run_start_time": "2026-07-01 10:00:00.000001", + "run_end_time": "2026-07-01 10:00:02.000001", + "parameters": [ + { + "source_parameter_id": 130, + "name": "P Chamber Pressure", + "unit": "mTorr", + }, + { + "source_parameter_id": 999, + "name": "Unknown signal", + "unit": "V", + }, + ], + "samples": [ + { + "source_sample_id": 1, + "source_tool_id": 5, + "timestamp": "2026-07-01 10:00:00.000001", + "values": {"p130": 0, "p999": None}, + }, + { + "source_sample_id": 2, + "source_tool_id": 5, + "timestamp": "2026-07-01 10:00:01.000001", + "values": {}, + }, + ], + "events": [ + { + "source_event_id": 7, + "source_tool_id": 5, + "timestamp": "2026-07-01 10:00:02.000001", + "event_type": "Process Complete", + } + ], + }, + source_tool_id=5, + source_updated_at=None, + registry=equipment_parameter_registry(self.equipment), + ) + + self.assertEqual(prepared["samples"][0]["values"]["p130"], 0) + self.assertIsNone(prepared["samples"][0]["values"]["p999"]) + self.assertEqual(prepared["samples"][1]["values"], {}) + self.assertEqual( + prepared["raw"]["trace"]["process_outcome"], "complete" + ) + self.assertTrue(prepared["parameters"][0]["registered"]) + self.assertFalse(prepared["parameters"][1]["registered"]) + self.assertTrue(any("unregistered source ID 999" in item for item in warnings)) + + def test_event_from_another_tool_is_rejected(self): + with self.assertRaisesRegex(ValueError, "belongs to tool 6"): + prepare_live_run( + { + "source_run_id": 1, + "run_start_time": "2026-07-01T10:00:00", + "run_end_time": "2026-07-01T10:01:00", + "samples": [ + { + "source_sample_id": 1, + "timestamp": "2026-07-01T10:00:01", + "values": {}, + } + ], + "events": [ + { + "source_event_id": 2, + "source_tool_id": 6, + "timestamp": "2026-07-01T10:01:00", + } + ], + }, + source_tool_id=5, + source_updated_at=None, + registry=[], + ) + + def test_missing_nullable_sample_timestamp_is_preserved(self): + prepared, warnings = prepare_live_run( + { + "source_run_id": 102, + "run_start_time": "2026-07-01T10:00:00", + "samples": [ + { + "source_sample_id": 3, + "source_tool_id": 5, + "timestamp": None, + "values": {"p130": 0}, + } + ], + "events": [], + }, + source_tool_id=5, + source_updated_at=None, + registry=equipment_parameter_registry(self.equipment), + ) + + self.assertIsNone(prepared["samples"][0]["sample_time_raw"]) + self.assertTrue( + any("has no source timestamp" in warning for warning in warnings) + ) + + +class ProductionGlancePersistenceTests(unittest.TestCase): + def test_live_upsert_uses_source_identity_and_returns_counts(self): + queries: list[str] = [] + value_queries: list[str] = [] + + class Cursor: + def execute(self, query, params=None): + queries.append(str(query)) + + def fetchall(self): + return [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def __init__(self): + self.committed = False + self.rolled_back = False + + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + self.committed = True + + def rollback(self): + self.rolled_back = True + + def close(self): + pass + + def execute_values(cursor, query, values, **kwargs): + value_queries.append(str(query)) + if "INSERT INTO equipment_runs" in query: + return [(7001, 101)] + return None + + connection = Connection() + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=connection, + ), + patch("psycopg2.extras.execute_values", side_effect=execute_values), + ): + result = sync_equipment_run_traces_pg( + equipment_id="equipment-test", + project_id="project-test", + source_system="glance", + source_tool_id=5, + source_updated_at=datetime.now(timezone.utc), + return_stats=True, + runs=[ + { + "source_run_id": 101, + "run_start_time": "2026-07-01T10:00:00", + "parameters": [ + { + "source_parameter_id": 1, + "key": "p1", + "name": "Signal", + } + ], + "samples": [ + { + "source_sample_id": 1001, + "source_tool_id": 5, + "timestamp": "2026-07-01T10:00:01", + "values": {"p1": 0}, + } + ], + "events": [ + { + "source_event_id": 9, + "source_tool_id": 5, + "timestamp": "2026-07-01T10:00:01", + "event_type": "Process Complete", + } + ], + } + ], + ) + + self.assertEqual(result["inserted"], 1) + self.assertEqual(result["sample_count"], 1) + self.assertEqual(result["event_count"], 1) + self.assertTrue( + any( + "source_system, equipment_id, source_tool_id, source_run_id" + in query + for query in value_queries + ) + ) + self.assertTrue(connection.committed) + self.assertFalse(connection.rolled_back) + + +class ProductionGlanceEndpointTests(unittest.TestCase): + def setUp(self) -> None: + app = FastAPI() + app.include_router(dataset_v2_router) + self.client = TestClient(app) + self.payload = { + "version": "1", + "source_system": "glance", + "batch_id": "batch-1", + "equipment_id": "equipment-test", + "source_tool_id": 5, + "source_tool_name": "Tool 5", + "project_id": "project-test", + "previous_cursor": "100", + "proposed_cursor": "101", + "poll_started_at": "2026-07-01T10:00:00Z", + "runs": [ + { + "source_run_id": 101, + "run_start_time": "2026-07-01T10:00:00", + "run_end_time": "2026-07-01T10:00:01", + "parameters": [], + "samples": [ + { + "source_sample_id": 1001, + "timestamp": "2026-07-01T10:00:00", + "values": {}, + } + ], + "events": [], + } + ], + } + + def test_endpoint_requires_token_and_validates_authoritative_mapping(self): + mapping = ( + '{"equipment-test":{"source_tool_id":5,' + '"source_tool_name":"Tool 5","project_id":"project-test"}}' + ) + environment = { + "INGESTION_TOKEN": "test-ingestion-token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": mapping, + } + with patch.dict(os.environ, environment, clear=False): + denied = self.client.post( + "/dataset/v2/glance/traces/sync", json=self.payload + ) + self.assertEqual(denied.status_code, 403) + + with ( + patch.dict(os.environ, environment, clear=False), + patch( + "routers.dataset_v2.get_equipment_pg", + return_value={"parameters": [], "outputs": []}, + ), + patch( + "routers.dataset_v2.get_glance_ingestion_project_pg", + return_value={ + "id": "project-test", + "equipment_id": "equipment-test", + }, + ), + patch( + "routers.dataset_v2.sync_equipment_run_traces_pg", + return_value={ + "inserted": 1, + "updated": 0, + "run_count": 1, + "sample_count": 1, + "event_count": 0, + }, + ), + patch("routers.dataset_v2.record_glance_ingestion_audit_pg"), + ): + response = self.client.post( + "/dataset/v2/glance/traces/sync", + json=self.payload, + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json()["cursor_acknowledged"]) + self.assertEqual(response.json()["inserted"], 1) + + +class ProductionGlanceCursorTests(unittest.TestCase): + def _settings(self) -> ConnectorSettings: + return ConnectorSettings( + postgrest_url="https://glance.invalid", + postgrest_token="read-token", + ingestion_url=( + "https://dt.invalid/api/dataset/v2/glance/traces/sync" + ), + ingestion_token="ingest-token", + storage_connection="unused", + cursor_container="state", + cursor_blob="cursor.json", + mappings=( + ToolMapping("equipment-test", 5, "Tool 5", "project-test", 100), + ), + overlap_run_count=5, + page_size=100, + timeout=(1, 1), + data_batch_size=10, + backfill_min_run_id=None, + backfill_max_run_id=None, + ) + + def test_cursor_advances_only_when_api_acknowledges_complete_batch(self): + class Client: + def affected_runs(self, mapping, cursor): + return [{"idruns": 101, "idtools": 5}] + + def complete_run(self, run, mapping): + return {"source_run_id": 101} + + class Store: + def __init__(self): + self.saved = [] + + def load(self): + return {"equipment-test:5": 100} + + def save(self, cursors): + self.saved.append(dict(cursors)) + + class AuditStore: + def record(self, _entry): + pass + + store = Store() + poller = GlancePoller( + self._settings(), + client=Client(), + cursor_store=store, + failure_audit_store=AuditStore(), + handoff_session=object(), + ) + poller._handoff = lambda payload: { + "status": "partial", + "cursor_acknowledged": False, + "run_count": 1, + "rejected_count": 1, + } + cursors = store.load() + result = poller.poll_mapping(self._settings().mappings[0], cursors) + self.assertFalse(result["cursor_acknowledged"]) + self.assertEqual(store.saved, []) + self.assertEqual(cursors["equipment-test:5"], 100) + + poller._handoff = lambda payload: { + "status": "success", + "cursor_acknowledged": True, + "run_count": 1, + "rejected_count": 0, + } + result = poller.poll_mapping(self._settings().mappings[0], cursors) + self.assertTrue(result["cursor_acknowledged"]) + self.assertEqual(store.saved[-1]["equipment-test:5"], 101) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_shadow_evaluation.py b/api/tests/test_shadow_evaluation.py new file mode 100644 index 0000000..d007fdc --- /dev/null +++ b/api/tests/test_shadow_evaluation.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import sys +import unittest +import warnings +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pandas as pd + + +API_ROOT = Path(__file__).resolve().parents[1] +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +from shadow_evaluation import _pareto_indices, evaluate_shadow + + +class ShadowEvaluationTests(unittest.TestCase): + def test_pareto_direction_is_max_rate_min_range(self): + rate = np.array([10.0, 11.0, 9.0, 12.0]) + spread = np.array([4.0, 5.0, 6.0, 3.0]) + self.assertEqual(_pareto_indices(rate, spread).tolist(), [3]) + + def test_report_compares_three_paths_and_is_seed_reproducible(self): + rng = np.random.default_rng(7) + features = ["flow", "power", "pressure"] + values = rng.uniform(0, 1, size=(28, len(features))) + frame = pd.DataFrame(values, columns=features) + frame["rate"] = ( + 4 * values[:, 0] + values[:, 1] + rng.normal(0, 0.05, len(frame)) + ) + frame["range"] = ( + values[:, 2] - 0.5 * values[:, 1] + + rng.normal(0, 0.05, len(frame)) + ) + frame["run_date"] = pd.date_range("2026-01-01", periods=len(frame)) + environment = { + "ML_SHADOW_RF_TREES": "20", + "ML_SHADOW_MC_SAMPLES": "16", + } + with patch.dict("os.environ", environment, clear=False), warnings.catch_warnings(): + warnings.simplefilter("ignore") + first = evaluate_shadow( + frame, + features=features, + primary_target="rate", + secondary_target="range", + candidate_count=16, + seed=42, + ) + second = evaluate_shadow( + frame, + features=features, + primary_target="rate", + secondary_target="range", + candidate_count=16, + seed=42, + ) + + self.assertIn("random_forest", first["candidate_metrics"]) + self.assertIn("gpr_analytic_ei", first["candidate_metrics"]) + self.assertIn( + "gpr_pareto_monte_carlo", first["candidate_metrics"] + ) + self.assertEqual( + first["deterministic_fingerprint"], + second["deterministic_fingerprint"], + ) + self.assertFalse(first["production_changed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/azure/.funcignore b/azure/.funcignore index 9966315..e2c7e81 100644 --- a/azure/.funcignore +++ b/azure/.funcignore @@ -5,4 +5,8 @@ __blobstorage__ __queuestorage__ local.settings.json test -.venv \ No newline at end of file +tests +test_scripts +.venv +Dockerfile.glance +requirements-glance.txt diff --git a/azure/.gitignore b/azure/.gitignore index a8f005a..1b65495 100644 --- a/azure/.gitignore +++ b/azure/.gitignore @@ -128,6 +128,10 @@ dmypy.json # Test scripts and data (not needed for deployment) tests/ +!tests/ +!tests/test_common_config.py +!tests/test_glance_connector.py +!tests/test_glance_mirror.py # Pyre type checker .pyre/ @@ -149,6 +153,9 @@ test_*.py debug_*.py *_test.py *_debug.py +!tests/test_common_config.py +!tests/test_glance_connector.py +!tests/test_glance_mirror.py # Data files (confidential - keep locally, don't push) *.csv @@ -185,4 +192,4 @@ process_etcher_data/azure_generated_dataset.csv process_etcher_data/OUTPUT_DATASET.csv # Timer-triggered variant (not used) -process_etcher_data_timer/ \ No newline at end of file +process_etcher_data_timer/ diff --git a/azure/Dockerfile.glance b/azure/Dockerfile.glance new file mode 100644 index 0000000..94aec89 --- /dev/null +++ b/azure/Dockerfile.glance @@ -0,0 +1,17 @@ +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app/azure + +COPY requirements-glance.txt . +RUN pip install --no-cache-dir -r requirements-glance.txt \ + && useradd --create-home --uid 10001 glance + +COPY common ./common +COPY scripts ./scripts + +USER 10001 + +CMD ["python", "scripts/run_glance_poll.py"] diff --git a/azure/README.md b/azure/README.md index 2a18fdb..b4570a6 100644 --- a/azure/README.md +++ b/azure/README.md @@ -6,7 +6,8 @@ The Azure Functions component of the Birck Digital Twin system processes semicon ## Architecture -The Azure Function is an HTTP-triggered function that: +The Azure component includes the legacy HTTP-triggered summary function and a +separate timer-triggered production full-trace connector. The summary function: 1. Synchronizes FIMAP files from Azure Blob Storage 2. Fetches run data from PostgREST API 3. Computes step averages using recipe setpoints or threshold methods @@ -23,6 +24,7 @@ azure/ ├── process_etcher_data/ # Main Azure Function │ ├── __init__.py # Function code (main entry point) │ └── function.json # Function binding configuration +├── process_glance_traces/ # Scheduled production full-trace connector ├── host.json # Azure Functions host configuration ├── requirements.txt # Python dependencies ├── local.settings.json.example # Environment variables template @@ -95,6 +97,57 @@ All configuration is done through environment variables (Application Settings in - `INGESTION_TOKEN`: Shared token used by Azure ingestion to authenticate to the DT API - `INGESTION_EQUIPMENT_ID`: Equipment/domain ID written into DT ingestion logs (default: `etcher`) +**Production GLANCE full traces:** +- `GLANCE_INGESTION_API_URL`: DT endpoint ending in `/api/dataset/v2/glance/traces/sync` +- `GLANCE_SOURCE_MODE`: `postgres` for the authoritative GLANCE pilot; + `postgrest` is retained for a future modern mirror +- `GLANCE_DB_URI`: read-only PostgreSQL URI, supplied through a Secret and + required only in `postgres` mode +- `GLANCE_PRODUCTION_MAPPINGS_JSON`: Explicit equipment/tool/project mappings; + every entry must also define a bounded `initial_run_id` +- `GLANCE_POLL_SCHEDULE`: Azure NCRONTAB schedule (default deployment target: + `0 */10 * * * *`) +- `GLANCE_CURSOR_BACKEND`: `file` for the Geddes CronJob or `blob` for Azure +- `GLANCE_CURSOR_FILE`: durable mounted path for the file backend +- `GLANCE_CURSOR_CONTAINER` and `GLANCE_CURSOR_BLOB`: durable per-tool cursor + state for the Azure blob backend +- `GLANCE_FAILURE_AUDIT_FILE`: append-only, private JSONL failure record for + the file backend (defaults beside the cursor file) +- `GLANCE_FAILURE_AUDIT_PREFIX`: immutable per-failure object prefix for the + Azure blob backend (default: `production-trace-failures`) +- `GLANCE_RUN_OVERLAP_COUNT`: Previously committed run IDs refetched for late + samples/events (default: 5) +- `GLANCE_PAGE_SIZE`: Maximum requested PostgREST page size (default: 1000) +- `GLANCE_DATA_BATCH_SIZE`: Number of numerically ordered sample IDs per + run-scoped `/data` range (default: 25). The connector also filters each range + through the authoritative `samplerecord.idruns` relationship. +- `GLANCE_BACKFILL_MIN_RUN_ID` and `GLANCE_BACKFILL_MAX_RUN_ID`: Optional bounded + backfill range; omit during normal polling +- `GLANCE_BACKFILL_RUN_IDS`: optional exact comma-separated pilot allowlist; + mutually exclusive with the min/max bounds and removed for normal polling +- `GLANCE_DB_RETRIES`: bounded retry count for transient PostgreSQL connection + failures +- `GLANCE_MAX_RUNS_PER_POLL`: hard source-run cap (default: 20) +- `GLANCE_MAX_SAMPLES_PER_RUN`: reject a source run above this sample count + before constructing its payload (default: 100000) +- `GLANCE_MAX_VALUES_PER_RUN`: reject a source run above this value count + (default: 1000000) +- `GLANCE_HANDOFF_RUN_BATCH_SIZE`: number of complete runs sent to FastAPI per + request (default: 1, preventing a large multi-run in-memory request) + +The connector reads the authoritative database through parameterized, +repeatable-read, read-only SQL, posts complete affected runs to FastAPI, and +advances a tool cursor only when the response sets +`cursor_acknowledged=true`. Local source/handoff errors are credential-redacted +and written durably beside the cursor state. The connector never receives DT +PostgreSQL credentials. + +The inactive Geddes deployment template is +`geddes/k8s/glance-ingestion-cronjob.yaml`. It contains the approved Tool 3 +pilot run allowlist and `suspend: true`; it must not be activated until its +image digest, equipment/project mapping, API migration, counts, and rollback +plan are reviewed. + **Microsoft Graph API (SharePoint):** - `TENANT_ID`: Azure AD tenant ID - `CLIENT_ID`: Azure AD application (client) ID diff --git a/azure/common/__init__.py b/azure/common/__init__.py index 1dc93c0..269300c 100644 --- a/azure/common/__init__.py +++ b/azure/common/__init__.py @@ -1,63 +1,9 @@ -# Azure Functions Common Module -# Shared utilities for multi-domain Azure Functions +"""Shared Azure helpers. -from .blob_helpers import ( - load_last_time, - save_last_time, - load_processed_runs, - save_processed_runs, - save_local_copy_from_blob, - merge_and_upload_dataset, - compare_datasets -) +Callers import the required submodule directly (for example +``common.config``). Keeping package import side-effect free lets the +runtime-independent GLANCE coordinator be unit-tested without loading optional +Blob, pandas, and notification dependencies. +""" -from .notifications import ( - send_teams_notification, - create_base_notification_payload, - convert_to_native_type, - create_new_data_notification, - create_outlier_notification, - create_filtered_run_notification, - DEFAULT_OUTLIER_TYPE_MAP -) - -from .config import ( - create_http_session, - get_timeout_config, - get_env_or_default, - get_env_bool, - get_storage_container, - api_get_all, - chunked, - get_processing_mode, - TIMEOUT_DEFAULTS -) - -__all__ = [ - # Blob helpers - 'load_last_time', - 'save_last_time', - 'load_processed_runs', - 'save_processed_runs', - 'save_local_copy_from_blob', - 'merge_and_upload_dataset', - 'compare_datasets', - # Notifications - 'send_teams_notification', - 'create_base_notification_payload', - 'convert_to_native_type', - 'create_new_data_notification', - 'create_outlier_notification', - 'create_filtered_run_notification', - 'DEFAULT_OUTLIER_TYPE_MAP', - # Config - 'create_http_session', - 'get_timeout_config', - 'get_env_or_default', - 'get_env_bool', - 'get_storage_container', - 'api_get_all', - 'chunked', - 'get_processing_mode', - 'TIMEOUT_DEFAULTS', -] +__all__: list[str] = [] diff --git a/azure/common/config.py b/azure/common/config.py index 0146f17..b3fccf0 100644 --- a/azure/common/config.py +++ b/azure/common/config.py @@ -154,28 +154,63 @@ def api_get_all( """ import logging - rows, start = [], 0 + rows = [] + request_params = dict(params) + raw_limit = request_params.pop("limit", None) + raw_offset = request_params.pop("offset", 0) + requested_limit = int(raw_limit) if raw_limit is not None else None + offset = int(raw_offset) + if requested_limit is not None and requested_limit < 0: + raise ValueError("PostgREST limit cannot be negative") + if offset < 0: + raise ValueError("PostgREST offset cannot be negative") + remaining = requested_limit headers = { "Authorization": f"Bearer {token}", "Accept": "application/json" } while True: - headers["Range"] = f"items={start}-{start + page_size - 1}" + request_limit = ( + min(page_size, remaining) + if remaining is not None + else page_size + ) + if request_limit == 0: + break + page_params = dict(request_params) + page_params["limit"] = request_limit + page_params["offset"] = offset try: resp = session.get( base_url.rstrip("/") + endpoint, headers=headers, - params=params, + params=page_params, timeout=timeout ) resp.raise_for_status() batch = resp.json() + if not isinstance(batch, list): + raise ValueError( + f"PostgREST endpoint {endpoint} returned a non-list payload" + ) rows.extend(batch) - if len(batch) < page_size: + received = len(batch) + if received == 0: break - start += page_size + offset += received + if remaining is not None: + remaining -= received + if remaining <= 0: + break except Exception as e: - logging.error(f"API call failed for {endpoint}: {e}") + # Request exception strings can contain bearer-adjacent URLs and + # very large source-ID filters. Keep operational logs useful + # without copying source identifiers or query strings. + logging.error( + "API call failed for %s (%s)", + endpoint, + type(e).__name__, + ) raise return rows diff --git a/azure/common/glance_connector.py b/azure/common/glance_connector.py new file mode 100644 index 0000000..fd5b847 --- /dev/null +++ b/azure/common/glance_connector.py @@ -0,0 +1,1544 @@ +"""Runtime-independent scheduled connector for complete GLANCE run traces.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import math +import os +import re +import tempfile +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path +from typing import Any, Callable, Iterable, Protocol, TypeVar + +import requests +from common.config import api_get_all, chunked, create_http_session + + +logger = logging.getLogger("dt.azure.glance_connector") + +_T = TypeVar("_T") + + +class ConnectorConfigurationError(RuntimeError): + pass + + +class ConnectorHandoffError(RuntimeError): + pass + + +def _required_env(name: str) -> str: + value = os.getenv(name, "").strip() + if not value: + raise ConnectorConfigurationError( + f"Missing required environment variable: {name}" + ) + return value + + +@dataclass(frozen=True) +class ToolMapping: + equipment_id: str + source_tool_id: int + source_tool_name: str + project_id: str + initial_run_id: int + + +@dataclass(frozen=True) +class ConnectorSettings: + postgrest_url: str + postgrest_token: str + ingestion_url: str + ingestion_token: str + storage_connection: str + cursor_container: str + cursor_blob: str + mappings: tuple[ToolMapping, ...] + overlap_run_count: int + page_size: int + timeout: tuple[int, int] + data_batch_size: int + backfill_min_run_id: int | None + backfill_max_run_id: int | None + source_mode: str = "postgrest" + db_uri: str = "" + db_retries: int = 3 + cursor_backend: str = "blob" + cursor_file: str = "" + backfill_run_ids: tuple[int, ...] = () + max_runs_per_poll: int = 20 + handoff_run_batch_size: int = 1 + max_samples_per_run: int = 100000 + max_values_per_run: int = 1000000 + failure_audit_file: str = "" + failure_audit_prefix: str = "production-trace-failures" + + @classmethod + def from_env(cls) -> "ConnectorSettings": + mapping_raw = _required_env("GLANCE_PRODUCTION_MAPPINGS_JSON") + try: + mapping_json = json.loads(mapping_raw) + except json.JSONDecodeError as exc: + raise ConnectorConfigurationError( + "GLANCE_PRODUCTION_MAPPINGS_JSON is not valid JSON" + ) from exc + if not isinstance(mapping_json, dict) or not mapping_json: + raise ConnectorConfigurationError( + "GLANCE_PRODUCTION_MAPPINGS_JSON must be a non-empty object" + ) + default_initial = os.getenv("GLANCE_INITIAL_RUN_ID", "").strip() + mappings = [] + for equipment_id, raw_mapping in mapping_json.items(): + if not isinstance(raw_mapping, dict): + raise ConnectorConfigurationError( + f"Mapping for {equipment_id!r} must be an object" + ) + initial_value = raw_mapping.get("initial_run_id", default_initial) + try: + tool_id = int(raw_mapping["source_tool_id"]) + initial_run_id = int(initial_value) + except (KeyError, TypeError, ValueError) as exc: + raise ConnectorConfigurationError( + f"Mapping for {equipment_id!r} requires numeric " + "source_tool_id and initial_run_id" + ) from exc + project_id = str(raw_mapping.get("project_id") or "").strip() + if not project_id: + raise ConnectorConfigurationError( + f"Mapping for {equipment_id!r} requires project_id" + ) + mappings.append( + ToolMapping( + equipment_id=str(equipment_id), + source_tool_id=tool_id, + source_tool_name=str( + raw_mapping.get("source_tool_name") or "" + ), + project_id=project_id, + initial_run_id=initial_run_id, + ) + ) + + ingestion_url = _required_env("GLANCE_INGESTION_API_URL").rstrip("/") + expected_suffix = "/api/dataset/v2/glance/traces/sync" + if not ingestion_url.endswith(expected_suffix): + raise ConnectorConfigurationError( + f"GLANCE_INGESTION_API_URL must end with {expected_suffix!r}" + ) + + def optional_int(name: str) -> int | None: + value = os.getenv(name, "").strip() + return int(value) if value else None + + source_mode = os.getenv("GLANCE_SOURCE_MODE", "postgrest").strip().lower() + if source_mode not in {"postgres", "postgrest"}: + raise ConnectorConfigurationError( + "GLANCE_SOURCE_MODE must be 'postgres' or 'postgrest'" + ) + postgrest_url = os.getenv("POSTGREST_URL", "").strip().rstrip("/") + postgrest_token = os.getenv("POSTGREST_TOKEN", "").strip() + db_uri = os.getenv("GLANCE_DB_URI", "").strip() + if source_mode == "postgres" and not db_uri: + raise ConnectorConfigurationError( + "GLANCE_DB_URI is required when GLANCE_SOURCE_MODE=postgres" + ) + if source_mode == "postgrest": + if not postgrest_url: + raise ConnectorConfigurationError( + "POSTGREST_URL is required when GLANCE_SOURCE_MODE=postgrest" + ) + if not postgrest_token: + raise ConnectorConfigurationError( + "POSTGREST_TOKEN is required when GLANCE_SOURCE_MODE=postgrest" + ) + + cursor_backend = os.getenv( + "GLANCE_CURSOR_BACKEND", "blob" + ).strip().lower() + if cursor_backend not in {"blob", "file"}: + raise ConnectorConfigurationError( + "GLANCE_CURSOR_BACKEND must be 'blob' or 'file'" + ) + storage_connection = os.getenv("AzureWebJobsStorage", "").strip() + cursor_file = os.getenv("GLANCE_CURSOR_FILE", "").strip() + if cursor_backend == "blob" and not storage_connection: + raise ConnectorConfigurationError( + "AzureWebJobsStorage is required for the blob cursor backend" + ) + if cursor_backend == "file" and not cursor_file: + raise ConnectorConfigurationError( + "GLANCE_CURSOR_FILE is required for the file cursor backend" + ) + failure_audit_file = os.getenv( + "GLANCE_FAILURE_AUDIT_FILE", "" + ).strip() + if cursor_backend == "file" and not failure_audit_file: + failure_audit_file = f"{cursor_file}.failures.jsonl" + failure_audit_prefix = os.getenv( + "GLANCE_FAILURE_AUDIT_PREFIX", + "production-trace-failures", + ).strip().strip("/") + if cursor_backend == "blob" and not failure_audit_prefix: + raise ConnectorConfigurationError( + "GLANCE_FAILURE_AUDIT_PREFIX cannot be empty" + ) + + backfill_min_run_id = optional_int("GLANCE_BACKFILL_MIN_RUN_ID") + backfill_max_run_id = optional_int("GLANCE_BACKFILL_MAX_RUN_ID") + run_ids_raw = os.getenv("GLANCE_BACKFILL_RUN_IDS", "").strip() + try: + backfill_run_ids = tuple( + sorted( + { + int(value.strip()) + for value in run_ids_raw.split(",") + if value.strip() + } + ) + ) + except ValueError as exc: + raise ConnectorConfigurationError( + "GLANCE_BACKFILL_RUN_IDS must be a comma-separated integer list" + ) from exc + if any(value < 0 for value in backfill_run_ids): + raise ConnectorConfigurationError( + "GLANCE_BACKFILL_RUN_IDS cannot contain negative IDs" + ) + if backfill_run_ids and ( + backfill_min_run_id is not None or backfill_max_run_id is not None + ): + raise ConnectorConfigurationError( + "GLANCE_BACKFILL_RUN_IDS cannot be combined with min/max bounds" + ) + if ( + backfill_min_run_id is not None + and backfill_max_run_id is not None + and backfill_min_run_id > backfill_max_run_id + ): + raise ConnectorConfigurationError( + "GLANCE backfill minimum exceeds maximum" + ) + max_runs_per_poll = int(os.getenv("GLANCE_MAX_RUNS_PER_POLL", "20")) + if max_runs_per_poll < 1 or max_runs_per_poll > 100: + raise ConnectorConfigurationError( + "GLANCE_MAX_RUNS_PER_POLL must be between 1 and 100" + ) + if len(backfill_run_ids) > max_runs_per_poll: + raise ConnectorConfigurationError( + "GLANCE_BACKFILL_RUN_IDS contains more IDs than " + "GLANCE_MAX_RUNS_PER_POLL" + ) + handoff_run_batch_size = int( + os.getenv("GLANCE_HANDOFF_RUN_BATCH_SIZE", "1") + ) + if ( + handoff_run_batch_size < 1 + or handoff_run_batch_size > max_runs_per_poll + ): + raise ConnectorConfigurationError( + "GLANCE_HANDOFF_RUN_BATCH_SIZE must be between 1 and " + "GLANCE_MAX_RUNS_PER_POLL" + ) + max_samples_per_run = int( + os.getenv("GLANCE_MAX_SAMPLES_PER_RUN", "100000") + ) + if max_samples_per_run < 1 or max_samples_per_run > 10_000_000: + raise ConnectorConfigurationError( + "GLANCE_MAX_SAMPLES_PER_RUN must be between 1 and 10000000" + ) + max_values_per_run = int( + os.getenv("GLANCE_MAX_VALUES_PER_RUN", "1000000") + ) + if max_values_per_run < 1 or max_values_per_run > 100_000_000: + raise ConnectorConfigurationError( + "GLANCE_MAX_VALUES_PER_RUN must be between 1 and 100000000" + ) + + return cls( + postgrest_url=postgrest_url, + postgrest_token=postgrest_token, + ingestion_url=ingestion_url, + ingestion_token=_required_env("INGESTION_TOKEN"), + storage_connection=storage_connection, + cursor_container=os.getenv( + "GLANCE_CURSOR_CONTAINER", "glance-ingestion-state" + ).strip(), + cursor_blob=os.getenv( + "GLANCE_CURSOR_BLOB", "production-trace-cursors.json" + ).strip(), + mappings=tuple(mappings), + overlap_run_count=max( + 0, int(os.getenv("GLANCE_RUN_OVERLAP_COUNT", "5")) + ), + page_size=max(1, int(os.getenv("GLANCE_PAGE_SIZE", "1000"))), + timeout=( + max(1, int(os.getenv("GLANCE_CONNECT_TIMEOUT_SECONDS", "10"))), + max(1, int(os.getenv("GLANCE_READ_TIMEOUT_SECONDS", "120"))), + ), + data_batch_size=max( + 1, int(os.getenv("GLANCE_DATA_BATCH_SIZE", "25")) + ), + backfill_min_run_id=backfill_min_run_id, + backfill_max_run_id=backfill_max_run_id, + source_mode=source_mode, + db_uri=db_uri, + db_retries=max(0, int(os.getenv("GLANCE_DB_RETRIES", "3"))), + cursor_backend=cursor_backend, + cursor_file=cursor_file, + backfill_run_ids=backfill_run_ids, + max_runs_per_poll=max_runs_per_poll, + handoff_run_batch_size=handoff_run_batch_size, + max_samples_per_run=max_samples_per_run, + max_values_per_run=max_values_per_run, + failure_audit_file=failure_audit_file, + failure_audit_prefix=failure_audit_prefix, + ) + + +def _redacted_error(exc: Exception) -> str: + text = str(exc) + text = re.sub( + r"(?i)(postgres(?:ql)?://)[^@\s]+@", + r"\1[redacted]@", + text, + ) + text = re.sub( + r"(?i)(password\s*=\s*)[^\s]+", + r"\1[redacted]", + text, + ) + return text[:1000] + + +class BlobCursorStore: + """One JSON blob containing independent committed cursors per source tool.""" + + def __init__(self, settings: ConnectorSettings): + from azure.storage.blob import BlobServiceClient + + service = BlobServiceClient.from_connection_string( + settings.storage_connection + ) + container = service.get_container_client(settings.cursor_container) + self._blob = container.get_blob_client(settings.cursor_blob) + self._etag: str | None = None + + def load(self) -> dict[str, int]: + from azure.core.exceptions import ResourceNotFoundError + + try: + download = self._blob.download_blob() + raw = download.readall().decode("utf-8") + properties = download.properties + etag = ( + properties.get("etag") + if isinstance(properties, dict) + else properties.etag + ) + self._etag = str(etag) + except ResourceNotFoundError: + self._etag = None + return {} + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise ConnectorConfigurationError( + "GLANCE cursor blob is not valid JSON" + ) from exc + if not isinstance(parsed, dict): + raise ConnectorConfigurationError("GLANCE cursor blob must be an object") + try: + return {str(key): int(value) for key, value in parsed.items()} + except (TypeError, ValueError) as exc: + raise ConnectorConfigurationError( + "GLANCE cursor blob contains a non-integer cursor" + ) from exc + + def save(self, cursors: dict[str, int]) -> None: + from azure.core import MatchConditions + + payload = json.dumps(cursors, sort_keys=True, separators=(",", ":")) + if self._etag is None: + response = self._blob.upload_blob(payload, overwrite=False) + else: + response = self._blob.upload_blob( + payload, + overwrite=True, + etag=self._etag, + match_condition=MatchConditions.IfNotModified, + ) + response_etag = ( + response.get("etag") + if isinstance(response, dict) + else getattr(response, "etag", None) + ) + if response_etag is None: + response_etag = self._blob.get_blob_properties().etag + self._etag = str(response_etag) + + +class CursorStore(Protocol): + def load(self) -> dict[str, int]: + ... + + def save(self, cursors: dict[str, int]) -> None: + ... + + +class FileCursorStore: + """Atomically persisted cursor state for a single non-concurrent CronJob.""" + + def __init__(self, settings: ConnectorSettings): + if not settings.cursor_file: + raise ConnectorConfigurationError("GLANCE_CURSOR_FILE is empty") + self._path = Path(settings.cursor_file) + + def load(self) -> dict[str, int]: + try: + raw = self._path.read_text(encoding="utf-8") + except FileNotFoundError: + return {} + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise ConnectorConfigurationError( + f"GLANCE cursor file {self._path} is not valid JSON" + ) from exc + if not isinstance(parsed, dict): + raise ConnectorConfigurationError( + f"GLANCE cursor file {self._path} must contain an object" + ) + try: + return {str(key): int(value) for key, value in parsed.items()} + except (TypeError, ValueError) as exc: + raise ConnectorConfigurationError( + f"GLANCE cursor file {self._path} contains a non-integer cursor" + ) from exc + + def save(self, cursors: dict[str, int]) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps(cursors, sort_keys=True, separators=(",", ":")) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{self._path.name}.", + dir=str(self._path.parent), + text=True, + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary_name, 0o600) + os.replace(temporary_name, self._path) + except Exception: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise + + +class FailureAuditStore(Protocol): + def record(self, entry: dict[str, Any]) -> None: + ... + + +class FileFailureAuditStore: + """Append-only local failure records for the single-concurrency CronJob.""" + + def __init__(self, settings: ConnectorSettings): + if not settings.failure_audit_file: + raise ConnectorConfigurationError( + "GLANCE_FAILURE_AUDIT_FILE is empty" + ) + self._path = Path(settings.failure_audit_file) + + def record(self, entry: dict[str, Any]) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + payload = ( + json.dumps( + entry, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + + "\n" + ) + with self._path.open("a", encoding="utf-8") as handle: + os.chmod(self._path, 0o600) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + + +class BlobFailureAuditStore: + """Write one immutable audit object per failure in Azure Blob Storage.""" + + def __init__(self, settings: ConnectorSettings): + from azure.storage.blob import BlobServiceClient + + service = BlobServiceClient.from_connection_string( + settings.storage_connection + ) + self._container = service.get_container_client( + settings.cursor_container + ) + self._prefix = settings.failure_audit_prefix + + def record(self, entry: dict[str, Any]) -> None: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + blob_name = f"{self._prefix}/{timestamp}-{uuid.uuid4()}.json" + payload = json.dumps( + entry, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + self._container.get_blob_client(blob_name).upload_blob( + payload, + overwrite=False, + ) + + +class GlanceSourceClient(Protocol): + def affected_runs( + self, + mapping: ToolMapping, + *, + cursor: int, + ) -> list[dict]: + ... + + def complete_run( + self, + run: dict[str, Any], + mapping: ToolMapping, + ) -> dict[str, Any]: + ... + + +class PostgrestGlanceClient: + def __init__( + self, + settings: ConnectorSettings, + *, + session: requests.Session | None = None, + ): + self.settings = settings + self.session = session or create_http_session( + { + "RETRIES": int(os.getenv("GLANCE_HTTP_RETRIES", "3")), + "BACKOFF": float(os.getenv("GLANCE_HTTP_BACKOFF", "0.5")), + } + ) + + def get_all(self, endpoint: str, params: dict[str, Any]) -> list[dict]: + return api_get_all( + self.session, + self.settings.postgrest_url, + endpoint, + params, + self.settings.postgrest_token, + page_size=self.settings.page_size, + timeout=self.settings.timeout, + ) + + def affected_runs( + self, + mapping: ToolMapping, + *, + cursor: int, + ) -> list[dict]: + if self.settings.backfill_run_ids: + pending_ids = [ + value + for value in self.settings.backfill_run_ids + if value > cursor + ] + if not pending_ids: + return [] + rows = self.get_all( + "/runs", + { + "select": "*", + "idtools": f"eq.{mapping.source_tool_id}", + "idruns": ( + "in.(" + + ",".join( + str(value) + for value in pending_ids + ) + + ")" + ), + "order": "idruns.asc", + "limit": str(self.settings.max_runs_per_poll), + }, + ) + by_run_id = {int(row["idruns"]): row for row in rows} + return [ + by_run_id.get( + run_id, + { + "idruns": run_id, + "idtools": mapping.source_tool_id, + "_missing_exact_run": True, + }, + ) + for run_id in pending_ids + ] + + lower = ( + self.settings.backfill_min_run_id + if self.settings.backfill_min_run_id is not None + else mapping.initial_run_id + ) + upper = self.settings.backfill_max_run_id + overlap_limit = min( + self.settings.overlap_run_count, + max(0, self.settings.max_runs_per_poll - 1), + ) + overlap_rows: list[dict] = [] + if overlap_limit and cursor >= lower: + overlap_upper = min(cursor, upper) if upper is not None else cursor + overlap_rows = self.get_all( + "/runs", + { + "select": "*", + "idtools": f"eq.{mapping.source_tool_id}", + "and": ( + f"(idruns.gte.{lower}," + f"idruns.lte.{overlap_upper})" + ), + "order": "idruns.desc", + "limit": str(overlap_limit), + }, + ) + + new_limit = self.settings.max_runs_per_poll - len(overlap_rows) + clauses = [f"idruns.gte.{lower}", f"idruns.gt.{cursor}"] + if upper is not None: + clauses.append(f"idruns.lte.{upper}") + new_rows = self.get_all( + "/runs", + { + "select": "*", + "idtools": f"eq.{mapping.source_tool_id}", + "and": "(" + ",".join(clauses) + ")", + "order": "idruns.asc", + "limit": str(max(1, new_limit)), + }, + ) + by_run_id = { + int(row["idruns"]): row + for row in [*overlap_rows, *new_rows] + } + return [by_run_id[run_id] for run_id in sorted(by_run_id)] + + def _recipe(self, recipe_id: Any) -> dict[str, Any]: + if recipe_id in (None, ""): + return {} + rows = self.get_all( + "/recipes", + {"select": "*", "idrecipes": f"eq.{int(recipe_id)}"}, + ) + return rows[0] if rows else {} + + def _parameter_rows(self, parameter_ids: Iterable[int]) -> list[dict]: + ids = sorted(set(parameter_ids)) + if not ids: + return [] + rows = [] + for batch in chunked(ids, self.settings.data_batch_size): + rows.extend( + self.get_all( + "/parameters", + { + "select": "*", + "idparameters": ( + "in.(" + ",".join(str(value) for value in batch) + ")" + ), + }, + ) + ) + return rows + + def complete_run( + self, + run: dict[str, Any], + mapping: ToolMapping, + ) -> dict[str, Any]: + run_id = int(run["idruns"]) + if int(run.get("idtools")) != mapping.source_tool_id: + raise ValueError( + f"Run {run_id} belongs to tool {run.get('idtools')}, " + f"expected {mapping.source_tool_id}" + ) + sample_rows = self.get_all( + "/samplerecord", + { + # Avoid large source-only fields on this high-volume table. + # These are the complete fields used to preserve sample + # identity, ordering, tool provenance, and absolute time. + "select": "idsamplerecord,idruns,idtools,time", + "idruns": f"eq.{run_id}", + "order": "time.asc,idsamplerecord.asc", + }, + ) + if not sample_rows: + raise ValueError(f"Run {run_id} has no sample records") + if len(sample_rows) > self.settings.max_samples_per_run: + raise ValueError( + f"Run {run_id} sample count exceeds configured limit " + f"{self.settings.max_samples_per_run}" + ) + + sample_ids = [int(row["idsamplerecord"]) for row in sample_rows] + # The live GLANCE PostgREST schema exposes the data -> samplerecord + # relationship. Use disjoint numeric sample-ID ranges plus the run + # foreign key: large IN clauses return HTTP 503, while sorting the + # entire joined run before pagination is too slow. + values_by_sample: dict[int, dict[str, Any]] = { + sample_id: {} for sample_id in sample_ids + } + parameter_ids: set[int] = set() + value_count = 0 + for batch in chunked(sorted(sample_ids), self.settings.data_batch_size): + lower_sample_id = min(batch) + upper_sample_id = max(batch) + batch_rows = self.get_all( + "/data", + { + "select": ( + "idsamplerecord,idparameters,value," + "samplerecord!inner(idruns)" + ), + "samplerecord.idruns": f"eq.{run_id}", + "and": ( + f"(idsamplerecord.gte.{lower_sample_id}," + f"idsamplerecord.lte.{upper_sample_id})" + ), + "order": "idsamplerecord.asc,idparameters.asc", + }, + ) + value_count += len(batch_rows) + if value_count > self.settings.max_values_per_run: + raise ValueError( + f"Run {run_id} value count exceeds configured limit " + f"{self.settings.max_values_per_run}" + ) + for row in batch_rows: + sample_id = int(row["idsamplerecord"]) + parameter_id = int(row["idparameters"]) + key = f"p{parameter_id}" + if key in values_by_sample.setdefault(sample_id, {}): + raise ValueError( + f"Sample {sample_id} contains duplicate parameter " + f"{parameter_id}" + ) + values_by_sample[sample_id][key] = row.get("value") + parameter_ids.add(parameter_id) + + parameters = [] + parameter_by_id = { + int(row["idparameters"]): row + for row in self._parameter_rows(parameter_ids) + } + for parameter_id in sorted(parameter_ids): + row = parameter_by_id.get(parameter_id, {}) + parameters.append( + { + "key": f"p{parameter_id}", + "source_parameter_id": parameter_id, + "name": row.get("name") or f"p{parameter_id}", + "source_name": row.get("name"), + "unit": row.get("unit") or "", + } + ) + + start_time = run.get("starttime") + end_time = run.get("endtime") + events = [] + if start_time not in (None, "") and end_time not in (None, ""): + event_rows = self.get_all( + "/events", + { + "select": "*", + "tool": f"eq.{mapping.source_tool_id}", + "time": f"gte.{start_time}", + "and": f"(time.gte.{start_time},time.lte.{end_time})", + "order": "time.asc,idevents.asc", + }, + ) + event_types = self.get_all("/eventtypes", {"select": "*"}) + category_by_type = { + str(row.get("event")): row.get("category") + for row in event_types + } + for event in event_rows: + events.append( + { + "source_event_id": int(event["idevents"]), + "source_tool_id": int(event.get("tool")), + "timestamp": event.get("time"), + "event_type": str(event.get("type") or ""), + "category": str( + category_by_type.get(str(event.get("type")), "") + ), + "description": str(event.get("description") or ""), + } + ) + + recipe = self._recipe(run.get("idrecipes")) + recipe_file = recipe.get("recipefile") + if isinstance(recipe_file, str): + try: + recipe_file_bytes = base64.b64decode(recipe_file, validate=True) + except (ValueError, TypeError): + recipe_file_bytes = recipe_file.encode("utf-8") + recipe_file_sha256 = hashlib.sha256(recipe_file_bytes).hexdigest() + recipe_file_size = len(recipe_file_bytes) + else: + recipe_file_sha256 = None + recipe_file_size = None + recipe_provenance = { + "source_recipe_id": run.get("idrecipes"), + "name": recipe.get("recipename"), + "source_hash": recipe.get("hash"), + "timestamp": recipe.get("timestamp"), + "file_sha256": recipe_file_sha256, + "file_size_bytes": recipe_file_size, + "raw": recipe, + } + + samples = [] + for index, sample in enumerate(sample_rows): + sample_id = int(sample["idsamplerecord"]) + samples.append( + { + "source_sample_id": sample_id, + "sample_index": index, + "source_tool_id": ( + int(sample["idtools"]) + if sample.get("idtools") not in (None, "") + else mapping.source_tool_id + ), + "timestamp": sample.get("time"), + "values": values_by_sample.get(sample_id, {}), + } + ) + + return { + "source_run_id": run_id, + "source_updated_at": ( + run.get("updated_at") + or run.get("updatedat") + or run.get("modified_at") + ), + "lotname": str(run.get("lotname") or ""), + "material": str(run.get("materialname") or ""), + "run_start_time": start_time, + "run_end_time": end_time, + "source_status": str(run.get("status") or ""), + "recipe": recipe_provenance, + "inputs": {}, + "outputs": {}, + "parameters": parameters, + "samples": samples, + "events": events, + "raw": {"glance": {"run": run, "recipe": recipe_provenance}}, + "timestamp_timezone": None, + } + + +def _json_safe(value: Any) -> Any: + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, float) and not math.isfinite(value): + if math.isnan(value): + return "NaN" + return "Infinity" if value > 0 else "-Infinity" + if isinstance(value, bytes): + return base64.b64encode(value).decode("ascii") + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + return value + + +class PostgresGlanceClient(PostgrestGlanceClient): + """Read authoritative GLANCE rows through parameterized, read-only SQL.""" + + def __init__( + self, + settings: ConnectorSettings, + *, + connection_factory: Callable[..., Any] | None = None, + ): + # Do not construct an HTTP session for the SQL implementation. + self.settings = settings + self.session = None + self._sql_cursor: Any | None = None + if connection_factory is None: + import psycopg2 + from psycopg2.extras import RealDictCursor + + self._connection_factory = psycopg2.connect + self._dict_cursor_factory: Any | None = RealDictCursor + else: + self._connection_factory = connection_factory + self._dict_cursor_factory = None + + @staticmethod + def _is_transient_database_error(exc: Exception) -> bool: + try: + import psycopg2 + except ImportError: + return False + return isinstance( + exc, + (psycopg2.InterfaceError, psycopg2.OperationalError), + ) + + def _execute_snapshot(self, operation: Callable[[], _T]) -> _T: + attempts = self.settings.db_retries + 1 + for attempt in range(attempts): + connection = None + cursor = None + try: + connection = self._connection_factory( + self.settings.db_uri, + connect_timeout=self.settings.timeout[0], + application_name="dt-glance-readonly", + ) + connection.set_session( + readonly=True, + isolation_level="REPEATABLE READ", + autocommit=False, + ) + if self._dict_cursor_factory is None: + cursor = connection.cursor() + else: + cursor = connection.cursor( + cursor_factory=self._dict_cursor_factory + ) + self._sql_cursor = cursor + cursor.execute( + "SELECT set_config('statement_timeout', %s, true)", + (str(self.settings.timeout[1] * 1000),), + ) + cursor.fetchone() + result = operation() + connection.rollback() + return result + except Exception as exc: + if connection is not None: + connection.rollback() + if ( + not self._is_transient_database_error(exc) + or attempt + 1 >= attempts + ): + raise + delay = min( + 8.0, + float(os.getenv("GLANCE_DB_RETRY_BACKOFF", "0.5")) + * (2**attempt), + ) + logger.warning( + "Transient GLANCE database error; retrying in %.1fs", + delay, + ) + time.sleep(delay) + finally: + self._sql_cursor = None + if cursor is not None: + cursor.close() + if connection is not None: + connection.close() + raise AssertionError("unreachable") + + def _query(self, query: str, parameters: tuple[Any, ...] = ()) -> list[dict]: + if self._sql_cursor is None: + raise RuntimeError("GLANCE SQL query attempted outside a snapshot") + self._sql_cursor.execute(query, parameters) + return [ + _json_safe(dict(row)) + for row in self._sql_cursor.fetchall() + ] + + @staticmethod + def _eq_value(params: dict[str, Any], key: str) -> str: + value = str(params.get(key, "")) + if not value.startswith("eq."): + raise ValueError(f"Expected an equality filter for {key}") + return value[3:] + + @staticmethod + def _in_values(params: dict[str, Any], key: str) -> list[int]: + value = str(params.get(key, "")) + if not value.startswith("in.(") or not value.endswith(")"): + raise ValueError(f"Expected an IN filter for {key}") + raw = value[4:-1] + return [int(item) for item in raw.split(",") if item] + + @staticmethod + def _bounded_values( + params: dict[str, Any], + key: str, + ) -> tuple[str, str]: + raw = str(params.get("and", "")) + match = re.fullmatch( + rf"\({re.escape(key)}\.gte\.([^,]+)," + rf"{re.escape(key)}\.lte\.([^)]+)\)", + raw, + ) + if match is None: + raise ValueError(f"Expected bounded {key} filter") + return match.group(1), match.group(2) + + def get_all(self, endpoint: str, params: dict[str, Any]) -> list[dict]: + if endpoint == "/samplerecord": + run_id = int(self._eq_value(params, "idruns")) + return self._query( + """ + SELECT idsamplerecord, idruns, idtools, time + FROM public.samplerecord + WHERE idruns = %s + ORDER BY time ASC NULLS LAST, idsamplerecord ASC + """, + (run_id,), + ) + if endpoint == "/data": + run_id = int(self._eq_value(params, "samplerecord.idruns")) + lower, upper = self._bounded_values(params, "idsamplerecord") + return self._query( + """ + SELECT d.idsamplerecord, d.idparameters, d.value + FROM public.data AS d + JOIN public.samplerecord AS s + ON s.idsamplerecord = d.idsamplerecord + WHERE s.idruns = %s + AND d.idsamplerecord >= %s + AND d.idsamplerecord <= %s + ORDER BY d.idsamplerecord ASC, d.idparameters ASC + """, + (run_id, int(lower), int(upper)), + ) + if endpoint == "/parameters": + parameter_ids = self._in_values(params, "idparameters") + return self._query( + """ + SELECT idparameters, name, unit + FROM public.parameters + WHERE idparameters = ANY(%s) + ORDER BY idparameters ASC + """, + (parameter_ids,), + ) + if endpoint == "/events": + tool_id = int(self._eq_value(params, "tool")) + start_time, end_time = self._bounded_values(params, "time") + return self._query( + """ + SELECT idevents, time, tool, type, description + FROM public.events + WHERE tool = %s + AND time >= %s + AND time <= %s + ORDER BY time ASC, idevents ASC + """, + (tool_id, start_time, end_time), + ) + if endpoint == "/eventtypes": + return self._query( + """ + SELECT event, category + FROM public.eventtypes + ORDER BY event ASC + """ + ) + if endpoint == "/recipes": + recipe_id = int(self._eq_value(params, "idrecipes")) + return self._query( + """ + SELECT idrecipes, idtools, recipename, recipefile, hash, timestamp + FROM public.recipes + WHERE idrecipes = %s + """, + (recipe_id,), + ) + raise ValueError(f"Unsupported GLANCE SQL endpoint: {endpoint}") + + def affected_runs( + self, + mapping: ToolMapping, + *, + cursor: int, + ) -> list[dict]: + def operation() -> list[dict]: + if self.settings.backfill_run_ids: + pending_ids = [ + value + for value in self.settings.backfill_run_ids + if value > cursor + ] + if not pending_ids: + return [] + rows = self._query( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns = ANY(%s) + ORDER BY idruns ASC + LIMIT %s + """, + ( + mapping.source_tool_id, + pending_ids, + self.settings.max_runs_per_poll, + ), + ) + by_run_id = {int(row["idruns"]): row for row in rows} + return [ + by_run_id.get( + run_id, + { + "idruns": run_id, + "idtools": mapping.source_tool_id, + "_missing_exact_run": True, + }, + ) + for run_id in pending_ids + ] + + lower = ( + self.settings.backfill_min_run_id + if self.settings.backfill_min_run_id is not None + else mapping.initial_run_id + ) + upper = self.settings.backfill_max_run_id + overlap_limit = min( + self.settings.overlap_run_count, + max(0, self.settings.max_runs_per_poll - 1), + ) + overlap_rows: list[dict] = [] + if overlap_limit and cursor >= lower: + overlap_upper = min(cursor, upper) if upper is not None else cursor + overlap_rows = self._query( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns >= %s + AND idruns <= %s + ORDER BY idruns DESC + LIMIT %s + """, + ( + mapping.source_tool_id, + lower, + overlap_upper, + overlap_limit, + ), + ) + new_limit = max( + 1, + self.settings.max_runs_per_poll - len(overlap_rows), + ) + if upper is None: + new_rows = self._query( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns >= %s + AND idruns > %s + ORDER BY idruns ASC + LIMIT %s + """, + ( + mapping.source_tool_id, + lower, + cursor, + new_limit, + ), + ) + else: + new_rows = self._query( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns >= %s + AND idruns > %s + AND idruns <= %s + ORDER BY idruns ASC + LIMIT %s + """, + ( + mapping.source_tool_id, + lower, + cursor, + upper, + new_limit, + ), + ) + by_run_id = { + int(row["idruns"]): row + for row in [*overlap_rows, *new_rows] + } + return [by_run_id[run_id] for run_id in sorted(by_run_id)] + + return self._execute_snapshot(operation) + + def complete_run( + self, + run: dict[str, Any], + mapping: ToolMapping, + ) -> dict[str, Any]: + run_id = int(run["idruns"]) + + def operation() -> dict[str, Any]: + current_rows = self._query( + """ + SELECT * + FROM public.runs + WHERE idruns = %s + AND idtools = %s + """, + (run_id, mapping.source_tool_id), + ) + if len(current_rows) != 1: + raise ValueError( + f"GLANCE run {run_id} for tool " + f"{mapping.source_tool_id} no longer exists" + ) + current_run = current_rows[0] + if ( + int(current_run.get("idruns")) != run_id + or int(current_run.get("idtools")) != mapping.source_tool_id + ): + raise ValueError( + f"GLANCE returned an inconsistent identity for run {run_id}" + ) + return super(PostgresGlanceClient, self).complete_run( + current_run, + mapping, + ) + + return self._execute_snapshot(operation) + + +def build_glance_client(settings: ConnectorSettings) -> GlanceSourceClient: + if settings.source_mode == "postgres": + return PostgresGlanceClient(settings) + return PostgrestGlanceClient(settings) + + +def build_cursor_store(settings: ConnectorSettings) -> CursorStore: + if settings.cursor_backend == "file": + return FileCursorStore(settings) + return BlobCursorStore(settings) + + +def build_failure_audit_store( + settings: ConnectorSettings, +) -> FailureAuditStore: + if settings.cursor_backend == "file": + return FileFailureAuditStore(settings) + return BlobFailureAuditStore(settings) + + +class GlancePoller: + def __init__( + self, + settings: ConnectorSettings, + *, + client: GlanceSourceClient | None = None, + cursor_store: CursorStore | None = None, + failure_audit_store: FailureAuditStore | None = None, + handoff_session: requests.Session | None = None, + ): + self.settings = settings + self.client = client or build_glance_client(settings) + self.cursor_store = cursor_store or build_cursor_store(settings) + self.failure_audit_store = ( + failure_audit_store or build_failure_audit_store(settings) + ) + self.handoff_session = handoff_session or create_http_session( + { + "RETRIES": int(os.getenv("GLANCE_HTTP_RETRIES", "3")), + "BACKOFF": float(os.getenv("GLANCE_HTTP_BACKOFF", "0.5")), + } + ) + + def _record_local_failure( + self, + *, + mapping: ToolMapping, + run_id: int | None, + cursor: int | None, + phase: str, + exc: Exception, + ) -> str: + error_text = _redacted_error(exc) + entry = { + "created_at": datetime.now(timezone.utc).isoformat(), + "equipment_id": mapping.equipment_id, + "source_tool_id": mapping.source_tool_id, + "source_run_id": run_id, + "cursor": cursor, + "phase": phase, + "status": "failed", + "error": error_text, + } + try: + self.failure_audit_store.record(entry) + except Exception as audit_exc: + logger.error( + "Could not persist GLANCE local failure audit: %s", + _redacted_error(audit_exc), + ) + return error_text + + def _cursor_key(self, mapping: ToolMapping) -> str: + base = f"{mapping.equipment_id}:{mapping.source_tool_id}" + if self.settings.backfill_run_ids: + digest = hashlib.sha256( + ",".join( + str(value) for value in self.settings.backfill_run_ids + ).encode("ascii") + ).hexdigest()[:16] + return f"{base}:backfill:ids:{digest}" + if ( + self.settings.backfill_min_run_id is not None + or self.settings.backfill_max_run_id is not None + ): + lower = ( + self.settings.backfill_min_run_id + if self.settings.backfill_min_run_id is not None + else mapping.initial_run_id + ) + upper = ( + str(self.settings.backfill_max_run_id) + if self.settings.backfill_max_run_id is not None + else "open" + ) + return f"{base}:backfill:range:{lower}:{upper}" + return base + + def _initial_cursor(self, mapping: ToolMapping) -> int: + if self.settings.backfill_run_ids: + return min(self.settings.backfill_run_ids) - 1 + if self.settings.backfill_min_run_id is not None: + return self.settings.backfill_min_run_id - 1 + return mapping.initial_run_id - 1 + + def _handoff(self, payload: dict[str, Any]) -> dict[str, Any]: + response = self.handoff_session.post( + self.settings.ingestion_url, + json=payload, + headers={"X-Ingestion-Token": self.settings.ingestion_token}, + timeout=self.settings.timeout, + ) + if response.status_code not in (200, 207): + raise ConnectorHandoffError( + f"DT trace ingestion returned HTTP {response.status_code}: " + f"{response.text[:500]}" + ) + try: + result = response.json() + except ValueError as exc: + raise ConnectorHandoffError( + "DT trace ingestion returned a non-JSON response" + ) from exc + if not isinstance(result, dict): + raise ConnectorHandoffError( + "DT trace ingestion returned an invalid response object" + ) + return result + + def poll_mapping( + self, + mapping: ToolMapping, + cursors: dict[str, int], + ) -> dict[str, Any]: + key = self._cursor_key(mapping) + previous_cursor = int(cursors.get(key, self._initial_cursor(mapping))) + affected = self.client.affected_runs(mapping, cursor=previous_cursor) + if not affected: + return { + "equipment_id": mapping.equipment_id, + "source_tool_id": mapping.source_tool_id, + "status": "no_data", + "cursor": previous_cursor, + } + affected = sorted(affected, key=lambda run: int(run["idruns"])) + current_cursor = previous_cursor + last_proposed_cursor = previous_cursor + total_run_count = 0 + total_rejected_count = 0 + batch_count = 0 + poll_started_at = datetime.now(timezone.utc).isoformat() + last_status = "success" + cursor_blocked = False + local_failures: list[dict[str, Any]] = [] + + for run_batch in chunked( + affected, + self.settings.handoff_run_batch_size, + ): + complete_runs = [] + local_rejected_count = 0 + for run in run_batch: + try: + complete_runs.append(self.client.complete_run(run, mapping)) + except Exception as exc: + local_rejected_count += 1 + cursor_blocked = True + try: + failed_run_id = int(run.get("idruns")) + except (TypeError, ValueError): + failed_run_id = None + error_text = self._record_local_failure( + mapping=mapping, + run_id=failed_run_id, + cursor=current_cursor, + phase="complete_run", + exc=exc, + ) + local_failures.append( + { + "source_run_id": failed_run_id, + "error": error_text, + } + ) + logger.error( + "Could not build complete GLANCE run %s: %s", + run.get("idruns"), + error_text, + ) + last_proposed_cursor = max( + last_proposed_cursor, + max(int(run["idruns"]) for run in run_batch), + ) + total_rejected_count += local_rejected_count + batch_count += 1 + if not complete_runs: + last_status = "partial" + continue + + payload_proposed_cursor = ( + current_cursor if cursor_blocked else last_proposed_cursor + ) + payload = { + "version": "1", + "source_system": "glance", + "batch_id": str(uuid.uuid4()), + "equipment_id": mapping.equipment_id, + "source_tool_id": mapping.source_tool_id, + "source_tool_name": mapping.source_tool_name, + "project_id": mapping.project_id, + "previous_cursor": str(current_cursor), + "proposed_cursor": str(payload_proposed_cursor), + "poll_started_at": poll_started_at, + "runs": complete_runs, + } + response = self._handoff(payload) + total_run_count += int(response.get("run_count", 0)) + total_rejected_count += int(response.get("rejected_count", 0)) + last_status = str(response.get("status", "unknown")) + if response.get("cursor_acknowledged") is not True: + cursor_blocked = True + last_status = "partial" + continue + if not cursor_blocked: + current_cursor = payload_proposed_cursor + cursors[key] = current_cursor + self.cursor_store.save(cursors) + + return { + "equipment_id": mapping.equipment_id, + "source_tool_id": mapping.source_tool_id, + "status": "partial" if cursor_blocked else last_status, + "cursor_acknowledged": not cursor_blocked, + "previous_cursor": previous_cursor, + "cursor": current_cursor, + "proposed_cursor": last_proposed_cursor, + "run_count": total_run_count, + "rejected_count": total_rejected_count, + "batch_count": batch_count, + "local_failures": local_failures, + } + + def poll_all(self) -> list[dict[str, Any]]: + try: + cursors = self.cursor_store.load() + except Exception as exc: + results = [] + for mapping in self.settings.mappings: + error_text = self._record_local_failure( + mapping=mapping, + run_id=None, + cursor=None, + phase="cursor_load", + exc=exc, + ) + logger.error( + "GLANCE cursor load failed for equipment=%s tool=%s: %s", + mapping.equipment_id, + mapping.source_tool_id, + error_text, + ) + results.append( + { + "equipment_id": mapping.equipment_id, + "source_tool_id": mapping.source_tool_id, + "status": "failed", + "error": error_text, + } + ) + return results + results = [] + for mapping in self.settings.mappings: + try: + results.append(self.poll_mapping(mapping, cursors)) + except Exception as exc: + key = self._cursor_key(mapping) + current_cursor = int( + cursors.get(key, self._initial_cursor(mapping)) + ) + error_text = self._record_local_failure( + mapping=mapping, + run_id=None, + cursor=current_cursor, + phase="poll_mapping", + exc=exc, + ) + logger.error( + "GLANCE poll failed for equipment=%s tool=%s: %s", + mapping.equipment_id, + mapping.source_tool_id, + error_text, + ) + results.append( + { + "equipment_id": mapping.equipment_id, + "source_tool_id": mapping.source_tool_id, + "status": "failed", + "error": error_text, + } + ) + return results diff --git a/azure/common/glance_mirror.py b/azure/common/glance_mirror.py new file mode 100644 index 0000000..8831fba --- /dev/null +++ b/azure/common/glance_mirror.py @@ -0,0 +1,1163 @@ +"""Bounded, pull-based synchronization from GLANCE into a modern mirror.""" + +from __future__ import annotations + +import logging +import os +import re +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +from typing import Any, Callable, Iterable, Protocol, TypeVar + + +logger = logging.getLogger("dt.azure.glance_mirror") +_T = TypeVar("_T") + + +class MirrorConfigurationError(RuntimeError): + pass + + +def _required_env(name: str) -> str: + value = os.getenv(name, "").strip() + if not value: + raise MirrorConfigurationError( + f"Missing required environment variable: {name}" + ) + return value + + +def _integer_list(name: str, *, required: bool = False) -> tuple[int, ...]: + raw = os.getenv(name, "").strip() + if not raw: + if required: + raise MirrorConfigurationError(f"{name} cannot be empty") + return () + try: + values = tuple( + sorted({int(item.strip()) for item in raw.split(",") if item.strip()}) + ) + except ValueError as exc: + raise MirrorConfigurationError( + f"{name} must be a comma-separated integer list" + ) from exc + if any(value < 0 for value in values): + raise MirrorConfigurationError(f"{name} cannot contain negative IDs") + if required and not values: + raise MirrorConfigurationError(f"{name} cannot be empty") + return values + + +@dataclass(frozen=True) +class MirrorSettings: + source_db_uri: str + destination_db_uri: str + tool_ids: tuple[int, ...] + initial_run_id: int + overlap_run_ids: int + max_runs_per_tool: int + max_samples_per_run: int + max_values_per_run: int + insert_batch_size: int + connect_timeout_seconds: int + statement_timeout_seconds: int + source_retries: int + backfill_min_run_id: int | None + backfill_max_run_id: int | None + backfill_run_ids: tuple[int, ...] + + @classmethod + def from_env(cls) -> "MirrorSettings": + def optional_int(name: str) -> int | None: + value = os.getenv(name, "").strip() + return int(value) if value else None + + backfill_min = optional_int("GLANCE_MIRROR_BACKFILL_MIN_RUN_ID") + backfill_max = optional_int("GLANCE_MIRROR_BACKFILL_MAX_RUN_ID") + backfill_ids = _integer_list("GLANCE_MIRROR_BACKFILL_RUN_IDS") + if backfill_ids and (backfill_min is not None or backfill_max is not None): + raise MirrorConfigurationError( + "GLANCE_MIRROR_BACKFILL_RUN_IDS cannot be combined with " + "min/max bounds" + ) + if ( + backfill_min is not None + and backfill_max is not None + and backfill_min > backfill_max + ): + raise MirrorConfigurationError( + "GLANCE mirror backfill minimum exceeds maximum" + ) + + max_runs = int(os.getenv("GLANCE_MIRROR_MAX_RUNS_PER_TOOL", "5")) + if max_runs < 1 or max_runs > 100: + raise MirrorConfigurationError( + "GLANCE_MIRROR_MAX_RUNS_PER_TOOL must be between 1 and 100" + ) + if len(backfill_ids) > max_runs: + raise MirrorConfigurationError( + "GLANCE_MIRROR_BACKFILL_RUN_IDS contains more IDs than " + "GLANCE_MIRROR_MAX_RUNS_PER_TOOL" + ) + batch_size = int(os.getenv("GLANCE_MIRROR_INSERT_BATCH_SIZE", "5000")) + if batch_size < 1 or batch_size > 50000: + raise MirrorConfigurationError( + "GLANCE_MIRROR_INSERT_BATCH_SIZE must be between 1 and 50000" + ) + max_samples = int( + os.getenv("GLANCE_MIRROR_MAX_SAMPLES_PER_RUN", "100000") + ) + if max_samples < 1 or max_samples > 10_000_000: + raise MirrorConfigurationError( + "GLANCE_MIRROR_MAX_SAMPLES_PER_RUN must be between 1 and " + "10000000" + ) + max_values = int( + os.getenv("GLANCE_MIRROR_MAX_VALUES_PER_RUN", "1000000") + ) + if max_values < 1 or max_values > 100_000_000: + raise MirrorConfigurationError( + "GLANCE_MIRROR_MAX_VALUES_PER_RUN must be between 1 and " + "100000000" + ) + + return cls( + source_db_uri=_required_env("GLANCE_DB_URI"), + destination_db_uri=_required_env("GLANCE_MIRROR_DB_URI"), + tool_ids=_integer_list("GLANCE_MIRROR_TOOL_IDS", required=True), + initial_run_id=max( + 0, int(os.getenv("GLANCE_MIRROR_INITIAL_RUN_ID", "0")) + ), + overlap_run_ids=max( + 0, int(os.getenv("GLANCE_MIRROR_OVERLAP_RUN_IDS", "5")) + ), + max_runs_per_tool=max_runs, + max_samples_per_run=max_samples, + max_values_per_run=max_values, + insert_batch_size=batch_size, + connect_timeout_seconds=max( + 1, int(os.getenv("GLANCE_MIRROR_CONNECT_TIMEOUT_SECONDS", "10")) + ), + statement_timeout_seconds=max( + 1, + int(os.getenv("GLANCE_MIRROR_STATEMENT_TIMEOUT_SECONDS", "300")), + ), + source_retries=max( + 0, int(os.getenv("GLANCE_MIRROR_SOURCE_RETRIES", "3")) + ), + backfill_min_run_id=backfill_min, + backfill_max_run_id=backfill_max, + backfill_run_ids=backfill_ids, + ) + + +def _sync_state_context(settings: MirrorSettings) -> tuple[str, int]: + """Return an independent state key and initial cursor for this sync mode.""" + if settings.backfill_run_ids: + encoded_ids = ",".join(str(run_id) for run_id in settings.backfill_run_ids) + digest = sha256(encoded_ids.encode("ascii")).hexdigest()[:16] + return f"backfill:ids:{digest}", min(settings.backfill_run_ids) - 1 + if ( + settings.backfill_min_run_id is not None + or settings.backfill_max_run_id is not None + ): + lower = settings.backfill_min_run_id + if lower is None: + lower = settings.initial_run_id + upper = ( + str(settings.backfill_max_run_id) + if settings.backfill_max_run_id is not None + else "open" + ) + return f"backfill:range:{lower}:{upper}", lower - 1 + return "live", settings.initial_run_id - 1 + + +@dataclass(frozen=True) +class RunSnapshot: + run: dict[str, Any] + tool: dict[str, Any] + recipe: dict[str, Any] | None + samples: tuple[dict[str, Any], ...] + data: tuple[dict[str, Any], ...] + parameters: tuple[dict[str, Any], ...] + events: tuple[dict[str, Any], ...] + event_types: tuple[dict[str, Any], ...] + + @property + def run_id(self) -> int: + return int(self.run["idruns"]) + + @property + def tool_id(self) -> int: + return int(self.run["idtools"]) + + +def _redacted_error(exc: Exception) -> str: + text = str(exc) + text = re.sub( + r"(?i)(postgres(?:ql)?://)[^@\s]+@", + r"\1[redacted]@", + text, + ) + text = re.sub( + r"(?i)(password\s*=\s*)[^\s]+", + r"\1[redacted]", + text, + ) + return text[:1000] + + +class MirrorSource(Protocol): + def list_runs(self, tool_id: int, cursor: int) -> list[dict[str, Any]]: + ... + + def load_run(self, run: dict[str, Any]) -> RunSnapshot: + ... + + +class MirrorDestination(Protocol): + def get_cursor(self, tool_id: int, state_key: str, default: int) -> int: + ... + + def replace_run( + self, + snapshot: RunSnapshot, + previous_cursor: int, + state_key: str, + *, + advance_cursor: bool, + ) -> dict[str, Any]: + ... + + def record_failure( + self, + *, + tool_id: int, + run_id: int | None, + state_key: str, + previous_cursor: int, + duration_ms: int, + error_text: str, + ) -> None: + ... + + +class PsycopgMirrorSource: + """Read complete source runs from GLANCE in repeatable-read snapshots.""" + + def __init__( + self, + settings: MirrorSettings, + *, + connection_factory: Callable[..., Any] | None = None, + ): + self.settings = settings + if connection_factory is None: + import psycopg2 + from psycopg2.extras import RealDictCursor + + self._connection_factory = psycopg2.connect + self._dict_cursor_factory: Any | None = RealDictCursor + else: + self._connection_factory = connection_factory + self._dict_cursor_factory = None + + @staticmethod + def _is_transient(exc: Exception) -> bool: + try: + import psycopg2 + except ImportError: + return False + return isinstance( + exc, + (psycopg2.InterfaceError, psycopg2.OperationalError), + ) + + def _read(self, operation: Callable[[Any], _T]) -> _T: + attempts = self.settings.source_retries + 1 + for attempt in range(attempts): + connection = None + cursor = None + try: + connection = self._connection_factory( + self.settings.source_db_uri, + connect_timeout=self.settings.connect_timeout_seconds, + application_name="dt-glance-mirror-source", + ) + connection.set_session( + readonly=True, + isolation_level="REPEATABLE READ", + autocommit=False, + ) + if self._dict_cursor_factory is None: + cursor = connection.cursor() + else: + cursor = connection.cursor( + cursor_factory=self._dict_cursor_factory + ) + cursor.execute( + "SELECT set_config('statement_timeout', %s, true)", + (str(self.settings.statement_timeout_seconds * 1000),), + ) + cursor.fetchone() + result = operation(cursor) + connection.rollback() + return result + except Exception as exc: + if connection is not None: + connection.rollback() + if not self._is_transient(exc) or attempt + 1 >= attempts: + raise + delay = min(8.0, 0.5 * (2**attempt)) + logger.warning( + "Transient GLANCE mirror source error; retrying in %.1fs", + delay, + ) + time.sleep(delay) + finally: + if cursor is not None: + cursor.close() + if connection is not None: + connection.close() + raise AssertionError("unreachable") + + @staticmethod + def _rows(cursor: Any) -> list[dict[str, Any]]: + return [dict(row) for row in cursor.fetchall()] + + @staticmethod + def _bounded_rows( + cursor: Any, + *, + maximum: int, + label: str, + fetch_size: int = 5000, + ) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + while True: + batch = cursor.fetchmany(fetch_size) + if not batch: + return rows + rows.extend(dict(row) for row in batch) + if len(rows) > maximum: + raise ValueError( + f"GLANCE {label} count exceeds configured per-run limit " + f"{maximum}" + ) + + def list_runs(self, tool_id: int, cursor: int) -> list[dict[str, Any]]: + def operation(sql_cursor: Any) -> list[dict[str, Any]]: + if self.settings.backfill_run_ids: + pending_ids = [ + run_id + for run_id in self.settings.backfill_run_ids + if run_id > cursor + ][: self.settings.max_runs_per_tool] + if not pending_ids: + return [] + sql_cursor.execute( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns = ANY(%s) + AND idruns > %s + ORDER BY idruns ASC + LIMIT %s + """, + ( + tool_id, + pending_ids, + cursor, + len(pending_ids), + ), + ) + rows = self._rows(sql_cursor) + by_run_id = {int(row["idruns"]): row for row in rows} + return [ + by_run_id.get( + run_id, + { + "idruns": run_id, + "idtools": tool_id, + "_missing_exact_run": True, + }, + ) + for run_id in pending_ids + ] + + lower = ( + self.settings.backfill_min_run_id + if self.settings.backfill_min_run_id is not None + else self.settings.initial_run_id + ) + upper = self.settings.backfill_max_run_id + + overlap_rows: list[dict[str, Any]] = [] + if self.settings.overlap_run_ids and cursor >= lower: + if upper is None: + sql_cursor.execute( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns >= %s + AND idruns <= %s + ORDER BY idruns DESC + LIMIT %s + """, + ( + tool_id, + lower, + cursor, + self.settings.overlap_run_ids, + ), + ) + else: + sql_cursor.execute( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns >= %s + AND idruns <= LEAST(%s, %s) + ORDER BY idruns DESC + LIMIT %s + """, + ( + tool_id, + lower, + cursor, + upper, + self.settings.overlap_run_ids, + ), + ) + overlap_rows = self._rows(sql_cursor) + + if upper is None: + sql_cursor.execute( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns >= %s + AND idruns > %s + ORDER BY idruns ASC + LIMIT %s + """, + ( + tool_id, + lower, + cursor, + self.settings.max_runs_per_tool, + ), + ) + else: + sql_cursor.execute( + """ + SELECT * + FROM public.runs + WHERE idtools = %s + AND idruns >= %s + AND idruns > %s + AND idruns <= %s + ORDER BY idruns ASC + LIMIT %s + """, + ( + tool_id, + lower, + cursor, + upper, + self.settings.max_runs_per_tool, + ), + ) + new_rows = self._rows(sql_cursor) + by_run_id = { + int(row["idruns"]): row + for row in [*overlap_rows, *new_rows] + } + return [by_run_id[run_id] for run_id in sorted(by_run_id)] + + return self._read(operation) + + def load_run(self, run: dict[str, Any]) -> RunSnapshot: + run_id = int(run["idruns"]) + tool_id = int(run["idtools"]) + + def operation(sql_cursor: Any) -> RunSnapshot: + sql_cursor.execute( + """ + SELECT * + FROM public.runs + WHERE idruns = %s + AND idtools = %s + """, + (run_id, tool_id), + ) + current_run_rows = self._rows(sql_cursor) + if len(current_run_rows) != 1: + raise ValueError( + f"GLANCE run {run_id} for tool {tool_id} no longer exists" + ) + current_run = current_run_rows[0] + + sql_cursor.execute( + "SELECT idtools, name, description FROM public.tools " + "WHERE idtools = %s", + (tool_id,), + ) + tool_rows = self._rows(sql_cursor) + if len(tool_rows) != 1: + raise ValueError( + f"GLANCE run {run_id} references missing tool {tool_id}" + ) + + recipe = None + recipe_id = current_run.get("idrecipes") + if recipe_id is not None: + sql_cursor.execute( + """ + SELECT idrecipes, idtools, recipename, recipefile, hash, timestamp + FROM public.recipes + WHERE idrecipes = %s + """, + (int(recipe_id),), + ) + recipe_rows = self._rows(sql_cursor) + if len(recipe_rows) != 1: + raise ValueError( + f"GLANCE run {run_id} references missing recipe {recipe_id}" + ) + recipe = recipe_rows[0] + if int(recipe["idtools"]) != tool_id: + raise ValueError( + f"GLANCE run {run_id} references recipe {recipe_id} " + f"for tool {recipe['idtools']}" + ) + + sql_cursor.execute( + """ + SELECT idsamplerecord, idruns, idtools, time + FROM public.samplerecord + WHERE idruns = %s + ORDER BY idsamplerecord ASC + """, + (run_id,), + ) + samples = self._bounded_rows( + sql_cursor, + maximum=self.settings.max_samples_per_run, + label="sample", + ) + invalid_sample_tools = sorted( + { + int(row["idtools"]) + for row in samples + if int(row["idtools"]) != tool_id + } + ) + if invalid_sample_tools: + raise ValueError( + f"GLANCE run {run_id} has sample records for tools " + f"{invalid_sample_tools}" + ) + + sql_cursor.execute( + """ + SELECT d.idsamplerecord, d.idparameters, d.value + FROM public.data AS d + JOIN public.samplerecord AS s + ON s.idsamplerecord = d.idsamplerecord + WHERE s.idruns = %s + ORDER BY d.idsamplerecord ASC, d.idparameters ASC + """, + (run_id,), + ) + data_rows = self._bounded_rows( + sql_cursor, + maximum=self.settings.max_values_per_run, + label="value", + ) + parameter_ids = sorted( + {int(row["idparameters"]) for row in data_rows} + ) + if parameter_ids: + sql_cursor.execute( + """ + SELECT idparameters, name, unit + FROM public.parameters + WHERE idparameters = ANY(%s) + ORDER BY idparameters ASC + """, + (parameter_ids,), + ) + parameters = self._rows(sql_cursor) + found_parameter_ids = { + int(row["idparameters"]) for row in parameters + } + missing = set(parameter_ids) - found_parameter_ids + if missing: + raise ValueError( + f"GLANCE run {run_id} references missing parameters " + f"{sorted(missing)}" + ) + else: + parameters = [] + + events: list[dict[str, Any]] = [] + start_time = current_run.get("starttime") + end_time = current_run.get("endtime") + if start_time is not None and end_time is not None: + sql_cursor.execute( + """ + SELECT idevents, time, tool, type, description + FROM public.events + WHERE tool = %s + AND time >= %s + AND time <= %s + ORDER BY time ASC, idevents ASC + """, + (tool_id, start_time, end_time), + ) + events = self._rows(sql_cursor) + + event_names = sorted( + { + str(row["type"]) + for row in events + if row.get("type") is not None + } + ) + if event_names: + sql_cursor.execute( + """ + SELECT event, category + FROM public.eventtypes + WHERE event = ANY(%s) + ORDER BY event ASC + """, + (event_names,), + ) + event_types = self._rows(sql_cursor) + else: + event_types = [] + + return RunSnapshot( + run=current_run, + tool=tool_rows[0], + recipe=recipe, + samples=tuple(samples), + data=tuple(data_rows), + parameters=tuple(parameters), + events=tuple(events), + event_types=tuple(event_types), + ) + + return self._read(operation) + + +class PsycopgMirrorDestination: + """Atomically replace run-scoped rows in the modern destination mirror.""" + + def __init__( + self, + settings: MirrorSettings, + *, + connection_factory: Callable[..., Any] | None = None, + execute_values_function: Callable[..., Any] | None = None, + ): + self.settings = settings + if connection_factory is None: + import psycopg2 + from psycopg2.extras import execute_values + + self._connection_factory = psycopg2.connect + self._execute_values = execute_values + else: + if execute_values_function is None: + raise ValueError( + "execute_values_function is required with a connection fixture" + ) + self._connection_factory = connection_factory + self._execute_values = execute_values_function + + def _connect(self) -> Any: + connection = self._connection_factory( + self.settings.destination_db_uri, + connect_timeout=self.settings.connect_timeout_seconds, + application_name="dt-glance-mirror-destination", + ) + connection.autocommit = False + return connection + + def get_cursor(self, tool_id: int, state_key: str, default: int) -> int: + connection = self._connect() + try: + with connection.cursor() as sql_cursor: + sql_cursor.execute( + """ + SELECT last_run_id + FROM glance_mirror.sync_state + WHERE idtools = %s + AND state_key = %s + """, + (tool_id, state_key), + ) + row = sql_cursor.fetchone() + connection.rollback() + return int(row[0]) if row is not None else int(default) + finally: + connection.close() + + def _insert_values( + self, + sql_cursor: Any, + query: str, + rows: Iterable[tuple[Any, ...]], + ) -> None: + batch: list[tuple[Any, ...]] = [] + for row in rows: + batch.append(row) + if len(batch) < self.settings.insert_batch_size: + continue + self._execute_values( + sql_cursor, + query, + batch, + page_size=self.settings.insert_batch_size, + ) + batch = [] + if batch: + self._execute_values( + sql_cursor, + query, + batch, + page_size=self.settings.insert_batch_size, + ) + + def replace_run( + self, + snapshot: RunSnapshot, + previous_cursor: int, + state_key: str, + *, + advance_cursor: bool, + ) -> dict[str, Any]: + started = time.monotonic() + run_id = snapshot.run_id + tool_id = snapshot.tool_id + connection = self._connect() + try: + with connection.cursor() as sql_cursor: + sql_cursor.execute( + """ + INSERT INTO glance_mirror.tools (idtools, name, description) + VALUES (%s, %s, %s) + ON CONFLICT (idtools) DO UPDATE + SET name = EXCLUDED.name, + description = EXCLUDED.description + """, + ( + snapshot.tool["idtools"], + snapshot.tool["name"], + snapshot.tool["description"], + ), + ) + + if snapshot.recipe is not None: + recipe = snapshot.recipe + sql_cursor.execute( + """ + INSERT INTO glance_mirror.recipes ( + idrecipes, idtools, recipename, recipefile, hash, timestamp + ) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (idrecipes) DO UPDATE + SET idtools = EXCLUDED.idtools, + recipename = EXCLUDED.recipename, + recipefile = EXCLUDED.recipefile, + hash = EXCLUDED.hash, + timestamp = EXCLUDED.timestamp + """, + ( + recipe["idrecipes"], + recipe["idtools"], + recipe["recipename"], + recipe.get("recipefile"), + recipe["hash"], + recipe.get("timestamp"), + ), + ) + + self._insert_values( + sql_cursor, + """ + INSERT INTO glance_mirror.parameters ( + idparameters, name, unit + ) VALUES %s + ON CONFLICT (idparameters) DO UPDATE + SET name = EXCLUDED.name, + unit = EXCLUDED.unit + """, + ( + ( + row["idparameters"], + row["name"], + row.get("unit"), + ) + for row in snapshot.parameters + ), + ) + self._insert_values( + sql_cursor, + """ + INSERT INTO glance_mirror.eventtypes (event, category) + VALUES %s + ON CONFLICT (event) DO UPDATE + SET category = EXCLUDED.category + """, + ( + (row["event"], row["category"]) + for row in snapshot.event_types + ), + ) + + run = snapshot.run + sql_cursor.execute( + """ + INSERT INTO glance_mirror.runs ( + idruns, idtools, lotname, idrecipes, starttime, + endtime, status, materialname + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (idruns) DO UPDATE + SET idtools = EXCLUDED.idtools, + lotname = EXCLUDED.lotname, + idrecipes = EXCLUDED.idrecipes, + starttime = EXCLUDED.starttime, + endtime = EXCLUDED.endtime, + status = EXCLUDED.status, + materialname = EXCLUDED.materialname + """, + ( + run["idruns"], + run["idtools"], + run.get("lotname"), + run.get("idrecipes"), + run["starttime"], + run.get("endtime"), + run.get("status"), + run.get("materialname"), + ), + ) + + sql_cursor.execute( + """ + DELETE FROM glance_mirror.data AS d + USING glance_mirror.samplerecord AS s + WHERE d.idsamplerecord = s.idsamplerecord + AND s.idruns = %s + """, + (run_id,), + ) + sql_cursor.execute( + "DELETE FROM glance_mirror.samplerecord WHERE idruns = %s", + (run_id,), + ) + self._insert_values( + sql_cursor, + """ + INSERT INTO glance_mirror.samplerecord ( + idsamplerecord, idruns, idtools, time + ) VALUES %s + """, + ( + ( + row["idsamplerecord"], + row["idruns"], + row["idtools"], + row.get("time"), + ) + for row in snapshot.samples + ), + ) + self._insert_values( + sql_cursor, + """ + INSERT INTO glance_mirror.data ( + idsamplerecord, idparameters, value + ) VALUES %s + """, + ( + ( + row["idsamplerecord"], + row["idparameters"], + row["value"], + ) + for row in snapshot.data + ), + ) + + sql_cursor.execute( + """ + SELECT idevents + FROM glance_mirror.run_events + WHERE idruns = %s + """, + (run_id,), + ) + old_event_ids = { + int(row[0]) for row in sql_cursor.fetchall() + } + sql_cursor.execute( + "DELETE FROM glance_mirror.run_events WHERE idruns = %s", + (run_id,), + ) + self._insert_values( + sql_cursor, + """ + INSERT INTO glance_mirror.events ( + idevents, time, tool, type, description + ) VALUES %s + ON CONFLICT (idevents) DO UPDATE + SET time = EXCLUDED.time, + tool = EXCLUDED.tool, + type = EXCLUDED.type, + description = EXCLUDED.description + """, + ( + ( + row["idevents"], + row["time"], + row["tool"], + row.get("type"), + row["description"], + ) + for row in snapshot.events + ), + ) + self._insert_values( + sql_cursor, + """ + INSERT INTO glance_mirror.run_events (idruns, idevents) + VALUES %s + ON CONFLICT (idruns, idevents) DO NOTHING + """, + ( + (run_id, row["idevents"]) + for row in snapshot.events + ), + ) + current_event_ids = { + int(row["idevents"]) for row in snapshot.events + } + stale_event_ids = sorted(old_event_ids - current_event_ids) + if stale_event_ids: + sql_cursor.execute( + """ + DELETE FROM glance_mirror.events AS event + WHERE event.idevents = ANY(%s) + AND NOT EXISTS ( + SELECT 1 + FROM glance_mirror.run_events AS association + WHERE association.idevents = event.idevents + ) + """, + (stale_event_ids,), + ) + + new_cursor = ( + max(previous_cursor, run_id) + if advance_cursor + else previous_cursor + ) + now = datetime.now(timezone.utc) + sql_cursor.execute( + """ + INSERT INTO glance_mirror.sync_state ( + idtools, state_key, last_run_id, updated_at + ) + VALUES (%s, %s, %s, %s) + ON CONFLICT (idtools, state_key) DO UPDATE + SET last_run_id = GREATEST( + glance_mirror.sync_state.last_run_id, + EXCLUDED.last_run_id + ), + updated_at = EXCLUDED.updated_at + """, + (tool_id, state_key, new_cursor, now), + ) + duration_ms = int((time.monotonic() - started) * 1000) + sql_cursor.execute( + """ + INSERT INTO glance_mirror.sync_audit ( + idtools, idruns, state_key, previous_cursor, + committed_cursor, + sample_count, value_count, event_count, status, + duration_ms, error_text, created_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'success', + %s, '', %s) + """, + ( + tool_id, + run_id, + state_key, + previous_cursor, + new_cursor, + len(snapshot.samples), + len(snapshot.data), + len(snapshot.events), + duration_ms, + now, + ), + ) + connection.commit() + return { + "tool_id": tool_id, + "run_id": run_id, + "cursor": new_cursor, + "sample_count": len(snapshot.samples), + "value_count": len(snapshot.data), + "event_count": len(snapshot.events), + "duration_ms": duration_ms, + } + except Exception: + connection.rollback() + raise + finally: + connection.close() + + def record_failure( + self, + *, + tool_id: int, + run_id: int | None, + state_key: str, + previous_cursor: int, + duration_ms: int, + error_text: str, + ) -> None: + connection = self._connect() + try: + with connection.cursor() as sql_cursor: + sql_cursor.execute( + """ + INSERT INTO glance_mirror.sync_audit ( + idtools, idruns, state_key, previous_cursor, + committed_cursor, + sample_count, value_count, event_count, status, + duration_ms, error_text, created_at + ) + VALUES (%s, %s, %s, %s, %s, 0, 0, 0, 'failed', + %s, %s, %s) + """, + ( + tool_id, + run_id, + state_key, + previous_cursor, + previous_cursor, + duration_ms, + error_text[:1000], + datetime.now(timezone.utc), + ), + ) + connection.commit() + except Exception: + connection.rollback() + logger.exception("Could not persist GLANCE mirror failure audit") + finally: + connection.close() + + +class GlanceMirrorCoordinator: + def __init__( + self, + settings: MirrorSettings, + *, + source: MirrorSource | None = None, + destination: MirrorDestination | None = None, + ): + self.settings = settings + self.source = source or PsycopgMirrorSource(settings) + self.destination = destination or PsycopgMirrorDestination(settings) + + def sync_tool(self, tool_id: int) -> dict[str, Any]: + state_key, default_cursor = _sync_state_context(self.settings) + cursor = self.destination.get_cursor( + tool_id, + state_key, + default_cursor, + ) + runs = self.source.list_runs(tool_id, cursor) + result: dict[str, Any] = { + "tool_id": tool_id, + "state_key": state_key, + "previous_cursor": cursor, + "cursor": cursor, + "status": "no_data" if not runs else "success", + "runs": [], + } + cursor_blocked = False + for run in runs: + run_id = int(run["idruns"]) + started = time.monotonic() + try: + snapshot = self.source.load_run(run) + if snapshot.tool_id != tool_id: + raise ValueError( + f"GLANCE run {run_id} belongs to tool " + f"{snapshot.tool_id}, expected {tool_id}" + ) + stats = self.destination.replace_run( + snapshot, + cursor, + state_key, + advance_cursor=not cursor_blocked, + ) + if not cursor_blocked: + cursor = int(stats["cursor"]) + result["cursor"] = cursor + result["runs"].append(stats) + except Exception as exc: + duration_ms = int((time.monotonic() - started) * 1000) + error_text = _redacted_error(exc) + self.destination.record_failure( + tool_id=tool_id, + run_id=run_id, + state_key=state_key, + previous_cursor=cursor, + duration_ms=duration_ms, + error_text=error_text, + ) + cursor_blocked = True + result["status"] = "partial_failure" + result.setdefault("failures", []).append( + {"run_id": run_id, "error": error_text} + ) + result.setdefault("failed_run_id", run_id) + result.setdefault("error", error_text) + result["run_count"] = len(result["runs"]) + return result + + def sync_all(self) -> list[dict[str, Any]]: + results = [] + for tool_id in self.settings.tool_ids: + try: + results.append(self.sync_tool(tool_id)) + except Exception as exc: + logger.exception("GLANCE mirror failed for tool %s", tool_id) + results.append( + { + "tool_id": tool_id, + "status": "failed", + "error": _redacted_error(exc), + "run_count": 0, + } + ) + return results diff --git a/azure/local.settings.json.example b/azure/local.settings.json.example index a325262..6280a9a 100644 --- a/azure/local.settings.json.example +++ b/azure/local.settings.json.example @@ -14,8 +14,28 @@ "POSTGREST_TOKEN": "YOUR_POSTGREST_JWT_TOKEN", "POSTGREST_URL": "https://YOUR_POSTGREST_SERVER_URL", "INGESTION_API_URL": "https://YOUR_DT_API_HOST/api/dataset/v2/runs/sync", + "GLANCE_INGESTION_API_URL": "https://YOUR_DT_API_HOST/api/dataset/v2/glance/traces/sync", "INGESTION_TOKEN": "YOUR_SHARED_INGESTION_TOKEN", + "GLANCE_SOURCE_MODE": "postgres", + "GLANCE_DB_URI": "postgresql://READ_ONLY_USER:URL_ENCODED_PASSWORD@GLANCE_HOST:5432/logger?sslmode=disable", "INGESTION_EQUIPMENT_ID": "etcher", + "GLANCE_POLL_SCHEDULE": "0 */10 * * * *", + "GLANCE_PRODUCTION_MAPPINGS_JSON": "{\"YOUR_EQUIPMENT_ID\":{\"source_tool_id\":123,\"source_tool_name\":\"YOUR_GLANCE_TOOL_NAME\",\"project_id\":\"YOUR_PROJECT_ID\",\"initial_run_id\":1}}", + "GLANCE_CURSOR_BACKEND": "file", + "GLANCE_CURSOR_FILE": "/state/cursors.json", + "GLANCE_FAILURE_AUDIT_FILE": "/state/failures.jsonl", + "GLANCE_FAILURE_AUDIT_PREFIX": "production-trace-failures", + "GLANCE_CURSOR_CONTAINER": "glance-ingestion-state", + "GLANCE_CURSOR_BLOB": "production-trace-cursors.json", + "GLANCE_RUN_OVERLAP_COUNT": "5", + "GLANCE_PAGE_SIZE": "1000", + "GLANCE_DATA_BATCH_SIZE": "25", + "GLANCE_DB_RETRIES": "3", + "GLANCE_MAX_RUNS_PER_POLL": "20", + "GLANCE_MAX_SAMPLES_PER_RUN": "100000", + "GLANCE_MAX_VALUES_PER_RUN": "1000000", + "GLANCE_HANDOFF_RUN_BATCH_SIZE": "1", + "GLANCE_BACKFILL_RUN_IDS": "", "PROCESSING_MODE": "incremental", "TEST_INCREMENTAL_K": "7", "BLOB_CONTAINER": "etcher-data", diff --git a/azure/process_glance_traces/__init__.py b/azure/process_glance_traces/__init__.py new file mode 100644 index 0000000..aca39cc --- /dev/null +++ b/azure/process_glance_traces/__init__.py @@ -0,0 +1,16 @@ +"""Ten-minute production GLANCE full-trace polling entry point.""" + +import json +import logging + +import azure.functions as func + +from common.glance_connector import ConnectorSettings, GlancePoller + + +def main(timer: func.TimerRequest) -> None: + if timer.past_due: + logging.warning("Production GLANCE trace poll is running late") + settings = ConnectorSettings.from_env() + results = GlancePoller(settings).poll_all() + logging.info("Production GLANCE trace poll results: %s", json.dumps(results)) diff --git a/azure/process_glance_traces/function.json b/azure/process_glance_traces/function.json new file mode 100644 index 0000000..187736a --- /dev/null +++ b/azure/process_glance_traces/function.json @@ -0,0 +1,13 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "name": "timer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%GLANCE_POLL_SCHEDULE%", + "runOnStartup": false, + "useMonitor": true + } + ] +} diff --git a/azure/requirements-glance.txt b/azure/requirements-glance.txt new file mode 100644 index 0000000..70c1b9a --- /dev/null +++ b/azure/requirements-glance.txt @@ -0,0 +1,3 @@ +psycopg2-binary>=2.9.0 +requests>=2.32.0 +urllib3>=2.2.0 diff --git a/azure/scripts/run_glance_mirror.py b/azure/scripts/run_glance_mirror.py new file mode 100644 index 0000000..11d2d9e --- /dev/null +++ b/azure/scripts/run_glance_mirror.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Run one bounded GLANCE raw-mirror synchronization.""" + +from __future__ import annotations + +import json +import logging + +from common.glance_mirror import GlanceMirrorCoordinator, MirrorSettings + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + results = GlanceMirrorCoordinator(MirrorSettings.from_env()).sync_all() + print(json.dumps(results, sort_keys=True)) + return ( + 1 + if any( + item.get("status") not in {"no_data", "success"} + for item in results + ) + else 0 + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/azure/scripts/run_glance_poll.py b/azure/scripts/run_glance_poll.py new file mode 100644 index 0000000..666e65c --- /dev/null +++ b/azure/scripts/run_glance_poll.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Run one scheduled GLANCE-to-DT ingestion poll.""" + +from __future__ import annotations + +import json +import logging + +from common.glance_connector import ConnectorSettings, GlancePoller + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + settings = ConnectorSettings.from_env() + results = GlancePoller(settings).poll_all() + print(json.dumps(results, sort_keys=True)) + failed = any( + result.get("status") == "failed" + or ( + result.get("status") not in {"no_data", "success"} + and result.get("cursor_acknowledged") is not True + ) + for result in results + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/azure/test_scripts/shadow_glance_poll.py b/azure/test_scripts/shadow_glance_poll.py new file mode 100644 index 0000000..a68f1d4 --- /dev/null +++ b/azure/test_scripts/shadow_glance_poll.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python3 +"""Read-only GLANCE trace audit for explicitly named tools. + +This utility deliberately does not construct ``GlancePoller``. It therefore +cannot hand data to Digital Twin or load/save a production cursor. Complete +run payloads exist only in memory long enough to calculate integrity counts +and a canonical SHA-256 digest. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +AZURE_ROOT = Path(__file__).resolve().parents[1] +if str(AZURE_ROOT) not in sys.path: + sys.path.insert(0, str(AZURE_ROOT)) + +from common.glance_connector import ( # noqa: E402 + ConnectorSettings, + PostgrestGlanceClient, + ToolMapping, +) + + +DEFAULT_TOOL_NAMES = ( + "VLN-11304-CTC-PM1", + "VLN-11303-CTC-PM1", +) +FACILITY_TOOL_IDS = (1, 2, 3, 4) + + +def _load_credentials(settings_path: Path) -> tuple[str, str]: + try: + settings = json.loads(settings_path.read_text(encoding="utf-8")) + values = settings["Values"] + url = str(values["POSTGREST_URL"]).strip() + token = str(values["POSTGREST_TOKEN"]).strip() + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"Could not load POSTGREST_URL and POSTGREST_TOKEN from " + f"{settings_path}" + ) from exc + if not url or not token: + raise RuntimeError("GLANCE PostgREST credentials are empty") + return url.rstrip("/"), token + + +def _settings( + postgrest_url: str, + postgrest_token: str, + *, + data_batch_size: int, +) -> ConnectorSettings: + return ConnectorSettings( + postgrest_url=postgrest_url, + postgrest_token=postgrest_token, + ingestion_url="disabled://read-only-shadow-audit", + ingestion_token="disabled", + storage_connection="disabled", + cursor_container="disabled", + cursor_blob="disabled", + mappings=(), + overlap_run_count=0, + page_size=1000, + timeout=(10, 120), + data_batch_size=data_batch_size, + backfill_min_run_id=None, + backfill_max_run_id=None, + ) + + +def _canonical_sha256(payload: dict[str, Any]) -> str: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _is_zero(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and value == 0 + + +def _count_rows( + client: PostgrestGlanceClient, + endpoint: str, + params: dict[str, str], +) -> int: + headers = { + "Authorization": f"Bearer {client.settings.postgrest_token}", + "Accept": "application/json", + "Prefer": "count=exact", + } + request_params = dict(params) + request_params["limit"] = "1" + request_params["offset"] = "0" + response = client.session.get( + client.settings.postgrest_url.rstrip("/") + endpoint, + headers=headers, + params=request_params, + timeout=(10, 30), + ) + response.raise_for_status() + content_range = response.headers.get("Content-Range", "") + if "/" not in content_range: + raise RuntimeError( + f"{endpoint} did not return a PostgREST Content-Range count" + ) + total = content_range.rsplit("/", 1)[1] + if total == "*": + raise RuntimeError(f"{endpoint} returned an indeterminate row count") + return int(total) + + +def _metadata_sha256(run: dict[str, Any]) -> str: + return _canonical_sha256({"run": run}) + + +def _probe_run_scoped_data( + client: PostgrestGlanceClient, + run_id: int, +) -> dict[str, Any]: + try: + count = _count_rows( + client, + "/data", + { + "select": ( + "idsamplerecord,idparameters,value," + "samplerecord!inner(idruns)" + ), + "samplerecord.idruns": f"eq.{run_id}", + }, + ) + except Exception as exc: + status_code = getattr(getattr(exc, "response", None), "status_code", None) + return { + "supported": False, + "http_status": status_code, + "error_type": type(exc).__name__, + } + return { + "supported": True, + "value_count": count, + } + + +def _audit_tool_summary( + client: PostgrestGlanceClient, + tool_name: str, +) -> dict[str, Any]: + tool_rows = client.get_all( + "/tools", + { + "select": "idtools,name", + "name": f"eq.{tool_name}", + }, + ) + if len(tool_rows) != 1: + return { + "requested_tool_name": tool_name, + "status": "tool_resolution_failed", + "matching_tool_count": len(tool_rows), + } + + source_tool_id = int(tool_rows[0]["idtools"]) + run_rows = client.get_all( + "/runs", + { + "select": "*", + "idtools": f"eq.{source_tool_id}", + "order": "idruns.desc", + "limit": "1", + }, + ) + if not run_rows: + return { + "requested_tool_name": tool_name, + "source_tool_id": source_tool_id, + "status": "no_runs", + } + + run = run_rows[0] + started = time.monotonic() + run_id = int(run["idruns"]) + start_time = run.get("starttime") + end_time = run.get("endtime") + sample_count = _count_rows( + client, + "/samplerecord", + { + "select": "idsamplerecord", + "idruns": f"eq.{run_id}", + }, + ) + event_count = None + if start_time not in (None, "") and end_time not in (None, ""): + event_count = _count_rows( + client, + "/events", + { + "select": "idevents", + "tool": f"eq.{source_tool_id}", + "and": f"(time.gte.{start_time},time.lte.{end_time})", + }, + ) + recipe_id = run.get("idrecipes") + recipe_count = 0 + if recipe_id not in (None, ""): + recipe_count = _count_rows( + client, + "/recipes", + { + "select": "idrecipes", + "idrecipes": f"eq.{int(recipe_id)}", + }, + ) + run_scoped_data = _probe_run_scoped_data(client, run_id) + return { + "requested_tool_name": tool_name, + "resolved_tool_name": tool_rows[0].get("name"), + "source_tool_id": source_tool_id, + "source_run_id": run_id, + "run_start_time": start_time, + "run_end_time": end_time, + "source_status": str(run.get("status") or ""), + "source_recipe_id": recipe_id, + "recipe_record_count": recipe_count, + "sample_count": sample_count, + "event_count": event_count, + "run_scoped_data_query": run_scoped_data, + "run_metadata_sha256": _metadata_sha256(run), + "audit_elapsed_seconds": round(time.monotonic() - started, 3), + "status": "verified_summary", + } + + +def _audit_tool_complete( + client: PostgrestGlanceClient, + tool_name: str, +) -> dict[str, Any]: + tool_rows = client.get_all( + "/tools", + { + "select": "idtools,name", + "name": f"eq.{tool_name}", + }, + ) + if len(tool_rows) != 1: + return { + "requested_tool_name": tool_name, + "status": "tool_resolution_failed", + "matching_tool_count": len(tool_rows), + } + + source_tool_id = int(tool_rows[0]["idtools"]) + run_rows = client.get_all( + "/runs", + { + "select": "*", + "idtools": f"eq.{source_tool_id}", + "order": "idruns.desc", + "limit": "1", + }, + ) + if not run_rows: + return { + "requested_tool_name": tool_name, + "source_tool_id": source_tool_id, + "status": "no_runs", + } + + run = run_rows[0] + mapping = ToolMapping( + equipment_id="read-only-shadow-audit", + source_tool_id=source_tool_id, + source_tool_name=tool_name, + project_id="read-only-shadow-audit", + initial_run_id=int(run["idruns"]), + ) + try: + payload = client.complete_run(run, mapping) + except Exception as exc: + return { + "requested_tool_name": tool_name, + "source_tool_id": source_tool_id, + "source_run_id": int(run["idruns"]), + "run_start_time": run.get("starttime"), + "run_end_time": run.get("endtime"), + "source_status": str(run.get("status") or ""), + "status": "trace_build_failed", + "error_type": type(exc).__name__, + "error": "Complete-trace PostgREST query failed", + } + + values = [ + value + for sample in payload["samples"] + for value in sample["values"].values() + ] + return { + "requested_tool_name": tool_name, + "resolved_tool_name": tool_rows[0].get("name"), + "source_tool_id": source_tool_id, + "source_run_id": payload["source_run_id"], + "run_start_time": payload.get("run_start_time"), + "run_end_time": payload.get("run_end_time"), + "source_status": payload.get("source_status"), + "source_recipe_id": payload["recipe"].get("source_recipe_id"), + "sample_count": len(payload["samples"]), + "value_count": len(values), + "null_value_count": sum(value is None for value in values), + "literal_zero_value_count": sum(_is_zero(value) for value in values), + "parameter_count": len(payload["parameters"]), + "event_count": len(payload["events"]), + "payload_sha256": _canonical_sha256(payload), + "status": "verified", + } + + +def _inventory(client: PostgrestGlanceClient) -> dict[str, Any]: + tool_rows = client.get_all( + "/tools", + { + "select": "idtools,name", + "idtools": "in.(1,2,3,4)", + "order": "idtools.asc", + }, + ) + tool_name_by_id = { + int(row["idtools"]): str(row.get("name") or "") + for row in tool_rows + } + tools = [] + for tool_id in FACILITY_TOOL_IDS: + run_count = _count_rows( + client, + "/runs", + { + "select": "idruns", + "idtools": f"eq.{tool_id}", + }, + ) + latest = client.get_all( + "/runs", + { + "select": ( + "idruns,idtools,lotname,idrecipes,starttime,endtime,status" + ), + "idtools": f"eq.{tool_id}", + "order": "idruns.desc", + "limit": "1", + }, + ) + tools.append( + { + "source_tool_id": tool_id, + "source_tool_name": tool_name_by_id.get(tool_id), + "visible_run_count": run_count, + "latest_visible_run": latest[0] if latest else None, + } + ) + + pilot_params = { + "idtools": "eq.3", + "and": ( + "(starttime.gte.2024-07-30T00:00:00," + "starttime.lt.2024-08-16T00:00:00)" + ), + } + pilot_rows = client.get_all( + "/runs", + { + **pilot_params, + "select": ( + "idruns,idtools,lotname,idrecipes,starttime,endtime,status" + ), + "order": "idruns.asc", + "limit": "100", + }, + ) + named_rows = {} + for label, pattern in (("cln", "*cln*"), ("FM-2", "*FM-2*")): + named_rows[label] = client.get_all( + "/runs", + { + "select": ( + "idruns,idtools,lotname,idrecipes,starttime,endtime,status" + ), + "idtools": "eq.3", + "lotname": f"ilike.{pattern}", + "order": "idruns.desc", + "limit": "20", + }, + ) + + return { + "mode": "read_only_glance_inventory", + "source_interface": "postgrest", + "checked_at": datetime.now(timezone.utc).isoformat(), + "digital_twin_handoff_performed": False, + "cursor_read_or_advanced": False, + "total_visible_run_count": _count_rows( + client, + "/runs", + {"select": "idruns"}, + ), + "tools": tools, + "tool_3_pilot_window": { + "start_inclusive": "2024-07-30T00:00:00", + "end_exclusive": "2024-08-16T00:00:00", + "visible_run_count": _count_rows( + client, + "/runs", + { + "select": "idruns", + **pilot_params, + }, + ), + "runs": pilot_rows, + }, + "tool_3_null_recipe_count": _count_rows( + client, + "/runs", + { + "select": "idruns", + "idtools": "eq.3", + "idrecipes": "is.null", + }, + ), + "tool_3_named_run_matches": named_rows, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--settings", + type=Path, + default=AZURE_ROOT / "local.settings.json", + help="Local Azure settings JSON containing GLANCE credentials.", + ) + parser.add_argument( + "--tool", + action="append", + dest="tools", + help=( + "Exact GLANCE tool name. Repeat for multiple tools. Defaults to " + "the two approved Versaline tools." + ), + ) + parser.add_argument( + "--complete-trace", + action="store_true", + help=( + "Fetch every sample and value and hash the complete in-memory " + "payload. Default mode uses bounded exact-count requests only." + ), + ) + parser.add_argument( + "--data-batch-size", + type=int, + default=25, + help="Sample IDs per /data request in complete-trace mode (default: 25).", + ) + parser.add_argument( + "--inventory", + action="store_true", + help=( + "Check visible run counts, latest runs, the proposed Tool 3 pilot " + "window, null recipes, and named recent-run matches." + ), + ) + args = parser.parse_args() + if args.data_batch_size < 1: + parser.error("--data-batch-size must be at least 1") + + postgrest_url, postgrest_token = _load_credentials(args.settings) + client = PostgrestGlanceClient( + _settings( + postgrest_url, + postgrest_token, + data_batch_size=args.data_batch_size, + ) + ) + if args.inventory: + print(json.dumps(_inventory(client), indent=2, sort_keys=True)) + return 0 + + tool_names = tuple(args.tools or DEFAULT_TOOL_NAMES) + started_at = datetime.now(timezone.utc) + audit = _audit_tool_complete if args.complete_trace else _audit_tool_summary + results = [] + for tool_name in tool_names: + print(f"Auditing exact tool {tool_name}...", file=sys.stderr, flush=True) + try: + results.append(audit(client, tool_name)) + except Exception as exc: + results.append( + { + "requested_tool_name": tool_name, + "status": "audit_failed", + "error_type": type(exc).__name__, + "error": "Read-only PostgREST audit failed", + } + ) + finished_at = datetime.now(timezone.utc) + expected_status = "verified" if args.complete_trace else "verified_summary" + + report = { + "mode": "read_only_shadow_audit", + "scope": "complete_trace" if args.complete_trace else "bounded_summary", + "source_interface": "postgrest", + "started_at": started_at.isoformat(), + "finished_at": finished_at.isoformat(), + "digital_twin_handoff_performed": False, + "cursor_read_or_advanced": False, + "raw_payload_persisted": False, + "requested_tool_count": len(tool_names), + "verified_tool_count": sum( + result.get("status") == expected_status for result in results + ), + "results": results, + } + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["verified_tool_count"] == len(tool_names) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/azure/tests/test_common_config.py b/azure/tests/test_common_config.py new file mode 100644 index 0000000..45b9e7a --- /dev/null +++ b/azure/tests/test_common_config.py @@ -0,0 +1,83 @@ +"""Focused tests for shared PostgREST pagination.""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + + +AZURE_ROOT = Path(__file__).resolve().parents[1] +if str(AZURE_ROOT) not in sys.path: + sys.path.insert(0, str(AZURE_ROOT)) + +from common.config import api_get_all # noqa: E402 + + +class _Response: + def __init__(self, rows): + self._rows = rows + + def raise_for_status(self): + return None + + def json(self): + return self._rows + + +class _CappedSession: + def __init__(self, rows, cap=2): + self.rows = rows + self.cap = cap + self.calls = [] + + def get(self, url, *, headers, params, timeout): + self.calls.append(dict(params)) + offset = int(params["offset"]) + limit = min(int(params["limit"]), self.cap) + return _Response(self.rows[offset : offset + limit]) + + +class PostgrestPaginationTests(unittest.TestCase): + def test_advances_by_rows_returned_when_server_caps_page_size(self): + session = _CappedSession([{"id": 1}, {"id": 2}, {"id": 3}]) + params = {"select": "id", "order": "id.asc"} + + rows = api_get_all( + session, + "https://example.invalid", + "/items", + params, + "token", + page_size=5, + ) + + self.assertEqual(rows, [{"id": 1}, {"id": 2}, {"id": 3}]) + self.assertEqual( + [(call["offset"], call["limit"]) for call in session.calls], + [(0, 5), (2, 5), (3, 5)], + ) + self.assertEqual(params, {"select": "id", "order": "id.asc"}) + + def test_respects_caller_limit_and_offset(self): + session = _CappedSession( + [{"id": 1}, {"id": 2}, {"id": 3}], + cap=10, + ) + + rows = api_get_all( + session, + "https://example.invalid", + "/items", + {"select": "id", "limit": "1", "offset": "1"}, + "token", + page_size=5, + ) + + self.assertEqual(rows, [{"id": 2}]) + self.assertEqual(session.calls[0]["limit"], 1) + self.assertEqual(session.calls[0]["offset"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/azure/tests/test_glance_connector.py b/azure/tests/test_glance_connector.py new file mode 100644 index 0000000..54ac14d --- /dev/null +++ b/azure/tests/test_glance_connector.py @@ -0,0 +1,735 @@ +"""Focused source-fidelity tests for the production GLANCE connector.""" + +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from dataclasses import replace +from datetime import datetime +from decimal import Decimal +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import patch + + +AZURE_ROOT = Path(__file__).resolve().parents[1] +if str(AZURE_ROOT) not in sys.path: + sys.path.insert(0, str(AZURE_ROOT)) + +from common.glance_connector import ( # noqa: E402 + BlobCursorStore, + ConnectorSettings, + FileFailureAuditStore, + FileCursorStore, + GlancePoller, + PostgresGlanceClient, + PostgrestGlanceClient, + ToolMapping, + _json_safe, + _redacted_error, +) + + +def _settings() -> ConnectorSettings: + return ConnectorSettings( + postgrest_url="https://glance.invalid", + postgrest_token="read-token", + ingestion_url="disabled://test", + ingestion_token="disabled", + storage_connection="disabled", + cursor_container="disabled", + cursor_blob="disabled", + mappings=(), + overlap_run_count=0, + page_size=1000, + timeout=(1, 1), + data_batch_size=2, + backfill_min_run_id=None, + backfill_max_run_id=None, + ) + + +class _ResourceNotFoundError(Exception): + pass + + +def _azure_core_modules() -> dict[str, ModuleType]: + azure_module = ModuleType("azure") + core_module = ModuleType("azure.core") + exception_module = ModuleType("azure.core.exceptions") + core_module.MatchConditions = SimpleNamespace(IfNotModified="if-not-modified") + exception_module.ResourceNotFoundError = _ResourceNotFoundError + azure_module.core = core_module + return { + "azure": azure_module, + "azure.core": core_module, + "azure.core.exceptions": exception_module, + } + + +class _FixtureClient(PostgrestGlanceClient): + def __init__(self): + super().__init__(_settings()) + self.data_queries = [] + + def get_all(self, endpoint, params): + if endpoint == "/samplerecord": + return [ + { + "idsamplerecord": 20, + "idruns": 99, + "idtools": 3, + "time": "2026-07-01T00:00:02", + }, + { + "idsamplerecord": 10, + "idruns": 99, + "idtools": 3, + "time": "2026-07-01T00:00:00", + }, + { + "idsamplerecord": 11, + "idruns": 99, + "idtools": 3, + "time": "2026-07-01T00:00:01", + }, + ] + if endpoint == "/data": + self.data_queries.append(dict(params)) + if "idsamplerecord.gte.10" in params["and"]: + return [ + { + "idsamplerecord": 10, + "idparameters": 7, + "value": 0, + }, + { + "idsamplerecord": 11, + "idparameters": 7, + "value": None, + }, + ] + return [ + { + "idsamplerecord": 20, + "idparameters": 7, + "value": 1.5, + } + ] + if endpoint == "/parameters": + return [ + { + "idparameters": 7, + "name": "P Chamber Pressure", + "unit": "mTorr", + } + ] + if endpoint == "/events": + return [ + { + "idevents": 8, + "tool": 3, + "time": "2026-07-01T00:00:02", + "type": "Complete", + "description": "Process complete", + } + ] + if endpoint == "/eventtypes": + return [{"event": "Complete", "category": "Process Complete"}] + if endpoint == "/recipes": + return [{"idrecipes": 4, "recipename": "Recipe 4"}] + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + +class _SqlCursor: + def __init__(self): + self.executions = [] + self.rows = [] + + def execute(self, query, parameters=()): + normalized = " ".join(query.split()) + self.executions.append((normalized, parameters)) + if "set_config('statement_timeout'" in normalized: + self.rows = [{"set_config": "1000"}] + elif "FROM public.runs" in normalized: + self.rows = [ + { + "idruns": 99, + "idtools": 3, + "idrecipes": None, + "starttime": datetime(2026, 7, 1, 0, 0, 0), + "endtime": datetime(2026, 7, 1, 0, 0, 2), + "status": "processed", + } + ] + elif "FROM public.samplerecord" in normalized: + self.rows = [ + { + "idsamplerecord": 10, + "idruns": 99, + "idtools": 3, + "time": datetime(2026, 7, 1, 0, 0, 0), + }, + { + "idsamplerecord": 11, + "idruns": 99, + "idtools": 3, + "time": datetime(2026, 7, 1, 0, 0, 1), + }, + ] + elif "FROM public.data AS d" in normalized: + self.rows = [ + { + "idsamplerecord": 10, + "idparameters": 7, + "value": 0.0, + }, + { + "idsamplerecord": 11, + "idparameters": 7, + "value": 1.5, + }, + ] + elif "FROM public.parameters" in normalized: + self.rows = [ + { + "idparameters": 7, + "name": "P Chamber Pressure", + "unit": "mTorr", + } + ] + elif "FROM public.events" in normalized: + self.rows = [ + { + "idevents": 8, + "tool": 3, + "time": datetime(2026, 7, 1, 0, 0, 2), + "type": "Process Completed", + "description": "Process complete", + } + ] + elif "FROM public.eventtypes" in normalized: + self.rows = [{"event": "Process Completed", "category": 1}] + elif "FROM public.recipes" in normalized: + self.rows = [] + else: + raise AssertionError(f"Unexpected SQL: {normalized}") + + def fetchone(self): + return self.rows[0] if self.rows else None + + def fetchall(self): + return list(self.rows) + + def close(self): + return None + + +class _SqlConnection: + def __init__(self): + self.cursor_instance = _SqlCursor() + self.session_options = None + self.rollback_count = 0 + self.closed = False + + def set_session(self, **options): + self.session_options = options + + def cursor(self, **_kwargs): + return self.cursor_instance + + def rollback(self): + self.rollback_count += 1 + + def close(self): + self.closed = True + + +class _PollClient: + def affected_runs(self, _mapping, *, cursor): + self.cursor = cursor + return [ + {"idruns": 1, "idtools": 3}, + {"idruns": 5, "idtools": 3}, + {"idruns": 7, "idtools": 3}, + ] + + def complete_run(self, run, _mapping): + return {"source_run_id": run["idruns"]} + + +class _CursorStore: + def __init__(self): + self.cursors = {} + self.saved = [] + + def load(self): + return dict(self.cursors) + + def save(self, cursors): + self.cursors = dict(cursors) + self.saved.append(dict(cursors)) + + +class _FailureAuditStore: + def __init__(self): + self.entries = [] + + def record(self, entry): + self.entries.append(dict(entry)) + + +class _HandoffResponse: + def __init__(self, *, acknowledged, status): + self.status_code = 200 if acknowledged else 207 + self.text = "" + self._payload = { + "cursor_acknowledged": acknowledged, + "status": status, + "run_count": 1 if acknowledged else 0, + "rejected_count": 0 if acknowledged else 1, + } + + def json(self): + return dict(self._payload) + + +class _HandoffSession: + def __init__(self): + self.payloads = [] + + def post(self, _url, *, json, headers, timeout): + self.payloads.append(json) + acknowledged = len(self.payloads) == 1 + return _HandoffResponse( + acknowledged=acknowledged, + status="success" if acknowledged else "partial", + ) + + +class GlanceConnectorTests(unittest.TestCase): + def test_postgres_file_configuration_does_not_require_postgrest_or_azure(self): + environment = { + "GLANCE_SOURCE_MODE": "postgres", + "GLANCE_DB_URI": "postgresql://readonly.invalid/logger", + "GLANCE_CURSOR_BACKEND": "file", + "GLANCE_CURSOR_FILE": "/state/cursors.json", + "GLANCE_INGESTION_API_URL": ( + "https://dt.invalid/api/dataset/v2/glance/traces/sync" + ), + "INGESTION_TOKEN": "token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": ( + '{"equipment":{"source_tool_id":3,"source_tool_name":"Tool 3",' + '"project_id":"project","initial_run_id":1}}' + ), + "GLANCE_BACKFILL_RUN_IDS": "169,1,5,5", + } + with patch.dict("os.environ", environment, clear=True): + settings = ConnectorSettings.from_env() + + self.assertEqual(settings.source_mode, "postgres") + self.assertEqual(settings.cursor_backend, "file") + self.assertEqual(settings.backfill_run_ids, (1, 5, 169)) + self.assertEqual(settings.postgrest_url, "") + self.assertEqual(settings.storage_connection, "") + + def test_complete_run_uses_disjoint_run_scoped_ranges_and_preserves_values(self): + client = _FixtureClient() + payload = client.complete_run( + { + "idruns": 99, + "idtools": 3, + "idrecipes": 4, + "starttime": "2026-07-01T00:00:00", + "endtime": "2026-07-01T00:00:02", + "status": "processed", + }, + ToolMapping("equipment", 3, "Tool 3", "project", 99), + ) + + self.assertEqual(len(client.data_queries), 2) + self.assertEqual( + [query["and"] for query in client.data_queries], + [ + "(idsamplerecord.gte.10,idsamplerecord.lte.11)", + "(idsamplerecord.gte.20,idsamplerecord.lte.20)", + ], + ) + for query in client.data_queries: + self.assertEqual(query["samplerecord.idruns"], "eq.99") + self.assertIn("samplerecord!inner(idruns)", query["select"]) + self.assertNotIn("idsamplerecord", { + key for key in query if key != "select" + }) + + values = { + sample["source_sample_id"]: sample["values"]["p7"] + for sample in payload["samples"] + } + self.assertEqual(values, {20: 1.5, 10: 0, 11: None}) + self.assertEqual(payload["events"][0]["source_event_id"], 8) + + def test_complete_run_rejects_source_run_above_sample_limit(self): + client = _FixtureClient() + client.settings = replace(client.settings, max_samples_per_run=2) + + with self.assertRaisesRegex( + ValueError, + "sample count exceeds configured limit 2", + ): + client.complete_run( + { + "idruns": 99, + "idtools": 3, + "idrecipes": None, + "starttime": "2026-07-01T00:00:00", + "endtime": "2026-07-01T00:00:02", + "status": "processed", + }, + ToolMapping("equipment", 3, "Tool 3", "project", 99), + ) + + def test_postgres_client_uses_read_only_parameterized_snapshot(self): + connections = [] + + def connect(*_args, **_kwargs): + connection = _SqlConnection() + connections.append(connection) + return connection + + settings = replace( + _settings(), + source_mode="postgres", + db_uri="postgresql://read-only.invalid/logger", + backfill_run_ids=(1, 5, 99), + ) + client = PostgresGlanceClient( + settings, + connection_factory=connect, + ) + mapping = ToolMapping("equipment", 3, "Tool 3", "project", 1) + + runs = client.affected_runs(mapping, cursor=1) + self.assertTrue(runs[0]["_missing_exact_run"]) + self.assertEqual(runs[0]["idruns"], 5) + payload = client.complete_run(runs[-1], mapping) + + self.assertEqual(len(connections), 2) + for connection in connections: + self.assertEqual( + connection.session_options, + { + "readonly": True, + "isolation_level": "REPEATABLE READ", + "autocommit": False, + }, + ) + self.assertGreaterEqual(connection.rollback_count, 1) + self.assertTrue(connection.closed) + + runs_query, runs_parameters = next( + execution + for execution in connections[0].cursor_instance.executions + if "FROM public.runs" in execution[0] + ) + self.assertIn("idruns = ANY(%s)", runs_query) + self.assertEqual(runs_parameters, (3, [5, 99], 20)) + self.assertNotIn("99", runs_query) + + self.assertIsNone(payload["recipe"]["source_recipe_id"]) + self.assertEqual( + payload["samples"][0]["timestamp"], + "2026-07-01T00:00:00", + ) + self.assertEqual(payload["samples"][0]["values"]["p7"], 0.0) + self.assertEqual( + payload["events"][0]["event_type"], + "Process Completed", + ) + complete_run_queries = [ + query + for query, _parameters in connections[1].cursor_instance.executions + if "FROM public.runs" in query + ] + self.assertEqual(len(complete_run_queries), 1) + self.assertIn("idruns = %s", complete_run_queries[0]) + self.assertIn("idtools = %s", complete_run_queries[0]) + + def test_postgres_overlap_reads_actual_sparse_tool_rows(self): + class SparseCursor(_SqlCursor): + def execute(self, query, parameters=()): + normalized = " ".join(query.split()) + self.executions.append((normalized, parameters)) + if "set_config('statement_timeout'" in normalized: + self.rows = [{"set_config": "1000"}] + elif "ORDER BY idruns DESC" in normalized: + self.rows = [ + {"idruns": 100, "idtools": 3}, + {"idruns": 20, "idtools": 3}, + ] + elif "ORDER BY idruns ASC" in normalized: + self.rows = [{"idruns": 150, "idtools": 3}] + else: + raise AssertionError(f"Unexpected SQL: {normalized}") + + class SparseConnection(_SqlConnection): + def __init__(self): + super().__init__() + self.cursor_instance = SparseCursor() + + connection = SparseConnection() + client = PostgresGlanceClient( + replace( + _settings(), + source_mode="postgres", + db_uri="postgresql://read-only.invalid/logger", + overlap_run_count=2, + max_runs_per_poll=5, + ), + connection_factory=lambda *_args, **_kwargs: connection, + ) + + rows = client.affected_runs( + ToolMapping("equipment", 3, "Tool 3", "project", 1), + cursor=100, + ) + + self.assertEqual( + [row["idruns"] for row in rows], + [20, 100, 150], + ) + overlap_query, overlap_parameters = next( + execution + for execution in connection.cursor_instance.executions + if "ORDER BY idruns DESC" in execution[0] + ) + self.assertNotIn("idruns -", overlap_query) + self.assertEqual(overlap_parameters, (3, 1, 100, 2)) + + def test_blob_cursor_does_not_treat_access_failure_as_empty_state(self): + class Blob: + def download_blob(self): + raise RuntimeError("authorization failed") + + store = BlobCursorStore.__new__(BlobCursorStore) + store._blob = Blob() + store._etag = None + + with ( + patch.dict(sys.modules, _azure_core_modules()), + self.assertRaisesRegex(RuntimeError, "authorization failed"), + ): + store.load() + + def test_blob_cursor_rejects_corrupt_or_non_integer_state(self): + class Download: + properties = SimpleNamespace(etag="etag-1") + + def __init__(self, payload): + self.payload = payload + + def readall(self): + return self.payload + + class Blob: + def __init__(self, payload): + self.payload = payload + + def download_blob(self): + return Download(self.payload) + + store = BlobCursorStore.__new__(BlobCursorStore) + store._blob = Blob(b"{not-json") + store._etag = None + with ( + patch.dict(sys.modules, _azure_core_modules()), + self.assertRaisesRegex( + RuntimeError, + "cursor blob is not valid JSON", + ), + ): + store.load() + + store._blob = Blob(b'{"equipment:3":"not-an-integer"}') + with ( + patch.dict(sys.modules, _azure_core_modules()), + self.assertRaisesRegex( + RuntimeError, + "non-integer cursor", + ), + ): + store.load() + + def test_blob_cursor_uses_etag_for_conditional_update(self): + class Blob: + def __init__(self): + self.calls = [] + + def upload_blob(self, payload, **kwargs): + self.calls.append((payload, kwargs)) + return {"etag": "etag-2"} + + blob = Blob() + store = BlobCursorStore.__new__(BlobCursorStore) + store._blob = blob + store._etag = "etag-1" + + with patch.dict(sys.modules, _azure_core_modules()): + store.save({"equipment:3": 5}) + + self.assertEqual(blob.calls[0][1]["etag"], "etag-1") + self.assertTrue(blob.calls[0][1]["overwrite"]) + self.assertEqual(store._etag, "etag-2") + + def test_json_safe_preserves_decimal_and_non_finite_values_explicitly(self): + self.assertEqual( + _json_safe( + { + "decimal": Decimal("1.2300"), + "nan": float("nan"), + "positive": float("inf"), + "negative": float("-inf"), + } + ), + { + "decimal": "1.2300", + "nan": "NaN", + "positive": "Infinity", + "negative": "-Infinity", + }, + ) + + def test_file_cursor_store_round_trip(self): + with tempfile.TemporaryDirectory() as temporary_directory: + cursor_path = Path(temporary_directory) / "state" / "cursors.json" + store = FileCursorStore( + replace( + _settings(), + cursor_backend="file", + cursor_file=str(cursor_path), + ) + ) + + self.assertEqual(store.load(), {}) + store.save({"equipment:3": 169}) + + self.assertEqual(store.load(), {"equipment:3": 169}) + self.assertEqual(cursor_path.stat().st_mode & 0o777, 0o600) + + def test_file_failure_audit_is_append_only_and_private(self): + with tempfile.TemporaryDirectory() as temporary_directory: + audit_path = Path(temporary_directory) / "state" / "failures.jsonl" + store = FileFailureAuditStore( + replace( + _settings(), + cursor_backend="file", + failure_audit_file=str(audit_path), + ) + ) + + store.record({"source_run_id": 5, "error": "first"}) + store.record({"source_run_id": 7, "error": "second"}) + + entries = [ + json.loads(line) + for line in audit_path.read_text(encoding="utf-8").splitlines() + ] + self.assertEqual( + [entry["source_run_id"] for entry in entries], + [5, 7], + ) + self.assertEqual(audit_path.stat().st_mode & 0o777, 0o600) + + def test_poller_hands_off_small_batches_and_stops_on_unacknowledged_cursor(self): + client = _PollClient() + cursor_store = _CursorStore() + handoff_session = _HandoffSession() + poller = GlancePoller( + replace( + _settings(), + handoff_run_batch_size=1, + max_runs_per_poll=20, + ), + client=client, + cursor_store=cursor_store, + failure_audit_store=_FailureAuditStore(), + handoff_session=handoff_session, + ) + + result = poller.poll_mapping( + ToolMapping("equipment", 3, "Tool 3", "project", 1), + {}, + ) + + self.assertEqual(client.cursor, 0) + self.assertEqual(len(handoff_session.payloads), 3) + self.assertEqual( + [ + ( + payload["previous_cursor"], + payload["proposed_cursor"], + payload["runs"][0]["source_run_id"], + ) + for payload in handoff_session.payloads + ], + [("0", "1", 1), ("1", "5", 5), ("1", "1", 7)], + ) + self.assertEqual(cursor_store.saved, [{"equipment:3": 1}]) + self.assertFalse(result["cursor_acknowledged"]) + self.assertEqual(result["cursor"], 1) + self.assertEqual(result["proposed_cursor"], 7) + + def test_poller_redacts_and_durably_audits_local_run_failure(self): + class Client(_PollClient): + def complete_run(self, run, mapping): + if run["idruns"] == 5: + raise RuntimeError( + "postgresql://nanohubdt:secret@glance/logger " + "password=also-secret" + ) + return super().complete_run(run, mapping) + + failure_store = _FailureAuditStore() + poller = GlancePoller( + replace( + _settings(), + handoff_run_batch_size=1, + max_runs_per_poll=20, + ), + client=Client(), + cursor_store=_CursorStore(), + failure_audit_store=failure_store, + handoff_session=_HandoffSession(), + ) + + result = poller.poll_mapping( + ToolMapping("equipment", 3, "Tool 3", "project", 1), + {}, + ) + + self.assertEqual(len(failure_store.entries), 1) + self.assertEqual(failure_store.entries[0]["source_run_id"], 5) + serialized = json.dumps( + {"audit": failure_store.entries, "result": result} + ) + self.assertNotIn("secret", serialized) + self.assertIn("[redacted]", serialized) + self.assertEqual(result["status"], "partial") + + def test_redacts_database_credentials(self): + redacted = _redacted_error( + RuntimeError( + "postgresql://reader:topsecret@source/logger " + "password=anothersecret" + ) + ) + + self.assertNotIn("topsecret", redacted) + self.assertNotIn("anothersecret", redacted) + self.assertIn("[redacted]", redacted) + + +if __name__ == "__main__": + unittest.main() diff --git a/azure/tests/test_glance_mirror.py b/azure/tests/test_glance_mirror.py new file mode 100644 index 0000000..41c8c09 --- /dev/null +++ b/azure/tests/test_glance_mirror.py @@ -0,0 +1,499 @@ +"""Focused safety tests for the bounded GLANCE mirror.""" + +from __future__ import annotations + +import sys +import unittest +from dataclasses import replace +from datetime import datetime +from pathlib import Path +from unittest.mock import patch + + +AZURE_ROOT = Path(__file__).resolve().parents[1] +if str(AZURE_ROOT) not in sys.path: + sys.path.insert(0, str(AZURE_ROOT)) + +from common.glance_mirror import ( # noqa: E402 + GlanceMirrorCoordinator, + MirrorConfigurationError, + MirrorSettings, + PsycopgMirrorDestination, + PsycopgMirrorSource, + RunSnapshot, + _redacted_error, + _sync_state_context, +) + + +def _settings() -> MirrorSettings: + return MirrorSettings( + source_db_uri="postgresql://source.invalid/logger", + destination_db_uri="postgresql://destination.invalid/glance_mirror", + tool_ids=(3,), + initial_run_id=1, + overlap_run_ids=2, + max_runs_per_tool=5, + max_samples_per_run=100, + max_values_per_run=1000, + insert_batch_size=2, + connect_timeout_seconds=1, + statement_timeout_seconds=1, + source_retries=0, + backfill_min_run_id=None, + backfill_max_run_id=None, + backfill_run_ids=(), + ) + + +def _snapshot(run_id: int) -> RunSnapshot: + return RunSnapshot( + run={ + "idruns": run_id, + "idtools": 3, + "lotname": f"run-{run_id}", + "idrecipes": None, + "starttime": datetime(2026, 7, 1, 0, 0, 0), + "endtime": datetime(2026, 7, 1, 0, 0, 1), + "status": "processed", + "materialname": None, + }, + tool={"idtools": 3, "name": "Tool 3", "description": "DSE"}, + recipe=None, + samples=( + { + "idsamplerecord": run_id * 10, + "idruns": run_id, + "idtools": 3, + "time": datetime(2026, 7, 1, 0, 0, 0), + }, + ), + data=( + { + "idsamplerecord": run_id * 10, + "idparameters": 7, + "value": 0.0, + }, + ), + parameters=( + {"idparameters": 7, "name": "Pressure", "unit": "mTorr"}, + ), + events=(), + event_types=(), + ) + + +class _CoordinatorSource: + def __init__(self, run_ids): + self.run_ids = run_ids + self.list_calls = [] + self.load_calls = [] + + def list_runs(self, tool_id, cursor): + self.list_calls.append((tool_id, cursor)) + return [{"idruns": run_id, "idtools": tool_id} for run_id in self.run_ids] + + def load_run(self, run): + self.load_calls.append(run["idruns"]) + return _snapshot(run["idruns"]) + + +class _CoordinatorDestination: + def __init__(self, *, fail_on=None): + self.fail_on = fail_on + self.replaced = [] + self.failures = [] + + def get_cursor(self, _tool_id, state_key, default): + self.state_key = state_key + return default + + def replace_run( + self, + snapshot, + previous_cursor, + state_key, + *, + advance_cursor, + ): + if snapshot.run_id == self.fail_on: + raise RuntimeError("password=secret source failed") + self.replaced.append(snapshot.run_id) + return { + "run_id": snapshot.run_id, + "state_key": state_key, + "cursor": ( + max(previous_cursor, snapshot.run_id) + if advance_cursor + else previous_cursor + ), + } + + def record_failure(self, **record): + self.failures.append(record) + + +class _SourceCursor: + def __init__(self): + self.executions = [] + self.rows = [] + + def execute(self, query, parameters=()): + normalized = " ".join(query.split()) + self.executions.append((normalized, parameters)) + if "set_config('statement_timeout'" in normalized: + self.rows = [{"set_config": "1000"}] + elif "ORDER BY idruns DESC" in normalized: + self.rows = [{"idruns": 10, "idtools": 3}] + elif "FROM public.runs" in normalized: + self.rows = [{"idruns": 20, "idtools": 3}] + else: + raise AssertionError(f"Unexpected SQL: {normalized}") + + def fetchone(self): + return self.rows[0] if self.rows else None + + def fetchall(self): + return list(self.rows) + + def close(self): + return None + + +class _SourceConnection: + def __init__(self): + self.cursor_instance = _SourceCursor() + + def set_session(self, **_options): + return None + + def cursor(self, **_kwargs): + return self.cursor_instance + + def rollback(self): + return None + + def close(self): + return None + + +class _DestinationCursor: + def __init__(self): + self.executions = [] + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def execute(self, query, parameters=()): + self.executions.append((" ".join(query.split()), parameters)) + + def fetchone(self): + return None + + def fetchall(self): + return [] + + +class _DestinationConnection: + def __init__(self): + self.autocommit = None + self.cursor_instance = _DestinationCursor() + self.committed = False + self.rolled_back = False + self.closed = False + + def cursor(self): + return self.cursor_instance + + def commit(self): + self.committed = True + + def rollback(self): + self.rolled_back = True + + def close(self): + self.closed = True + + +class GlanceMirrorTests(unittest.TestCase): + def test_coordinator_advances_only_after_each_committed_run(self): + source = _CoordinatorSource([1, 5]) + destination = _CoordinatorDestination() + coordinator = GlanceMirrorCoordinator( + _settings(), + source=source, + destination=destination, + ) + + result = coordinator.sync_tool(3) + + self.assertEqual(source.list_calls, [(3, 0)]) + self.assertEqual(destination.replaced, [1, 5]) + self.assertEqual(result["cursor"], 5) + self.assertEqual(result["run_count"], 2) + self.assertEqual(result["status"], "success") + self.assertEqual(result["state_key"], "live") + + def test_coordinator_copies_later_run_without_advancing_after_failure(self): + source = _CoordinatorSource([1, 5, 7]) + destination = _CoordinatorDestination(fail_on=5) + coordinator = GlanceMirrorCoordinator( + _settings(), + source=source, + destination=destination, + ) + + result = coordinator.sync_tool(3) + + self.assertEqual(source.load_calls, [1, 5, 7]) + self.assertEqual(destination.replaced, [1, 7]) + self.assertEqual(result["cursor"], 1) + self.assertEqual(result["failed_run_id"], 5) + self.assertEqual(result["status"], "partial_failure") + self.assertEqual(destination.failures[0]["previous_cursor"], 1) + self.assertEqual(destination.failures[0]["state_key"], "live") + self.assertNotIn("secret", result["error"]) + + def test_explicit_backfill_has_separate_state_and_initial_cursor(self): + settings = replace( + _settings(), + backfill_run_ids=(29, 32, 73), + ) + state_key, default = _sync_state_context(settings) + + self.assertTrue(state_key.startswith("backfill:ids:")) + self.assertEqual(default, 28) + self.assertNotEqual(state_key, "live") + + def test_bounded_backfill_has_separate_state_and_initial_cursor(self): + settings = replace( + _settings(), + backfill_min_run_id=100, + backfill_max_run_id=200, + ) + + self.assertEqual( + _sync_state_context(settings), + ("backfill:range:100:200", 99), + ) + + def test_historical_backfill_range_overrides_live_initial_cursor(self): + settings = replace( + _settings(), + initial_run_id=8000, + backfill_min_run_id=100, + backfill_max_run_id=200, + ) + + self.assertEqual( + _sync_state_context(settings), + ("backfill:range:100:200", 99), + ) + + connection = _SourceConnection() + source = PsycopgMirrorSource( + settings, + connection_factory=lambda *_args, **_kwargs: connection, + ) + source.list_runs(3, 99) + range_parameters = next( + parameters + for query, parameters in connection.cursor_instance.executions + if "idruns <= %s" in query and "ORDER BY idruns ASC" in query + ) + self.assertEqual(range_parameters, (3, 100, 99, 200, 5)) + + def test_configuration_rejects_allowlist_larger_than_run_cap(self): + environment = { + "GLANCE_DB_URI": "postgresql://source.invalid/logger", + "GLANCE_MIRROR_DB_URI": ( + "postgresql://destination.invalid/glance_mirror" + ), + "GLANCE_MIRROR_TOOL_IDS": "3", + "GLANCE_MIRROR_MAX_RUNS_PER_TOOL": "2", + "GLANCE_MIRROR_BACKFILL_RUN_IDS": "1,5,7", + } + + with ( + patch.dict("os.environ", environment, clear=True), + self.assertRaisesRegex( + MirrorConfigurationError, + "contains more IDs", + ), + ): + MirrorSettings.from_env() + + def test_source_fetches_overlap_and_new_runs_without_stalling(self): + connection = _SourceConnection() + source = PsycopgMirrorSource( + _settings(), + connection_factory=lambda *_args, **_kwargs: connection, + ) + + rows = source.list_runs(3, 10) + + self.assertEqual([row["idruns"] for row in rows], [10, 20]) + overlap_query = next( + execution + for execution in connection.cursor_instance.executions + if "ORDER BY idruns DESC" in execution[0] + ) + new_query = next( + execution + for execution in connection.cursor_instance.executions + if "idruns > %s" in execution[0] + and "ORDER BY idruns ASC" in execution[0] + ) + self.assertEqual(overlap_query[1], (3, 1, 10, 2)) + self.assertEqual(new_query[1], (3, 1, 10, 5)) + self.assertNotIn("10", new_query[0]) + + def test_exact_backfill_returns_retryable_placeholder_for_missing_run(self): + connection = _SourceConnection() + source = PsycopgMirrorSource( + replace(_settings(), backfill_run_ids=(1, 5)), + connection_factory=lambda *_args, **_kwargs: connection, + ) + + rows = source.list_runs(3, 0) + + self.assertEqual([row["idruns"] for row in rows], [1, 5]) + self.assertTrue(all(row["_missing_exact_run"] for row in rows)) + + def test_exact_backfill_queries_only_current_bounded_id_slice(self): + connection = _SourceConnection() + source = PsycopgMirrorSource( + replace( + _settings(), + backfill_run_ids=(1, 5, 7), + max_runs_per_tool=2, + ), + connection_factory=lambda *_args, **_kwargs: connection, + ) + + source.list_runs(3, 0) + + query_parameters = next( + parameters + for query, parameters in connection.cursor_instance.executions + if "idruns = ANY(%s)" in query + ) + self.assertEqual(query_parameters, (3, [1, 5], 0, 2)) + + def test_destination_commits_state_and_audit_with_run(self): + connection = _DestinationConnection() + inserted_batches = [] + + def execute_values(_cursor, query, rows, page_size): + inserted_batches.append( + (" ".join(query.split()), list(rows), page_size) + ) + + destination = PsycopgMirrorDestination( + _settings(), + connection_factory=lambda *_args, **_kwargs: connection, + execute_values_function=execute_values, + ) + snapshot = replace( + _snapshot(5), + events=( + { + "idevents": 50, + "time": datetime(2026, 7, 1, 0, 0, 0), + "tool": 3, + "type": "Process Start", + "description": "start", + }, + ), + ) + + result = destination.replace_run( + snapshot, + previous_cursor=1, + state_key="live", + advance_cursor=True, + ) + + self.assertTrue(connection.committed) + self.assertFalse(connection.rolled_back) + self.assertTrue(connection.closed) + self.assertEqual(result["cursor"], 5) + self.assertEqual(result["sample_count"], 1) + self.assertEqual(result["value_count"], 1) + executed_sql = [ + query for query, _params in connection.cursor_instance.executions + ] + self.assertTrue( + any("INSERT INTO glance_mirror.sync_state" in query for query in executed_sql) + ) + self.assertTrue( + any("INSERT INTO glance_mirror.sync_audit" in query for query in executed_sql) + ) + self.assertTrue( + any("INSERT INTO glance_mirror.data" in query for query, *_ in inserted_batches) + ) + self.assertTrue( + any( + "INSERT INTO glance_mirror.run_events" in query + for query, *_ in inserted_batches + ) + ) + + def test_destination_can_copy_without_advancing_blocked_cursor(self): + connection = _DestinationConnection() + destination = PsycopgMirrorDestination( + _settings(), + connection_factory=lambda *_args, **_kwargs: connection, + execute_values_function=lambda *_args, **_kwargs: None, + ) + + result = destination.replace_run( + _snapshot(7), + previous_cursor=1, + state_key="live", + advance_cursor=False, + ) + + self.assertEqual(result["cursor"], 1) + state_parameters = next( + parameters + for query, parameters in connection.cursor_instance.executions + if "INSERT INTO glance_mirror.sync_state" in query + ) + self.assertEqual(state_parameters[0:3], (3, "live", 1)) + + def test_bounded_rows_rejects_oversized_source_run(self): + class Cursor: + def __init__(self): + self.calls = 0 + + def fetchmany(self, _size): + self.calls += 1 + if self.calls == 1: + return [{"id": 1}, {"id": 2}] + return [] + + with self.assertRaisesRegex(ValueError, "configured per-run limit 1"): + PsycopgMirrorSource._bounded_rows( + Cursor(), + maximum=1, + label="sample", + ) + + def test_redacts_database_credentials(self): + error = RuntimeError( + "could not use postgresql://reader:topsecret@source/logger " + "password=anothersecret" + ) + redacted = _redacted_error(error) + + self.assertNotIn("topsecret", redacted) + self.assertNotIn("anothersecret", redacted) + self.assertIn("[redacted]", redacted) + + +if __name__ == "__main__": + unittest.main() diff --git a/geddes/k8s/01-secrets.yaml.example b/geddes/k8s/01-secrets.yaml.example index 89f4b45..a016aaa 100644 --- a/geddes/k8s/01-secrets.yaml.example +++ b/geddes/k8s/01-secrets.yaml.example @@ -49,6 +49,9 @@ stringData: # Shared secret between Azure Function and Geddes API # Generate with: openssl rand -hex 32 INGESTION_TOKEN: "REPLACE_WITH_SHARED_INGESTION_TOKEN" + # Must exactly match the Azure connector mapping. Do not reuse the temporary + # DB3 pilot project IDs unless those projects are explicitly promoted. + GLANCE_PRODUCTION_MAPPINGS_JSON: '{"REPLACE_EQUIPMENT_ID":{"source_tool_id":0,"source_tool_name":"REPLACE_TOOL_NAME","project_id":"REPLACE_PRODUCTION_PROJECT_ID"}}' # Shared secret for trusted requests coming from the web app DT_SYSTEM_TOKEN: "REPLACE_WITH_SHARED_SYSTEM_TOKEN" diff --git a/geddes/k8s/02-api.yaml b/geddes/k8s/02-api.yaml index d301e5d..3eba378 100644 --- a/geddes/k8s/02-api.yaml +++ b/geddes/k8s/02-api.yaml @@ -36,6 +36,14 @@ spec: value: "random_forest" - name: ML_PROPOSAL_CANDIDATES value: "1024" + - name: ML_SHADOW_EVALUATION_ENABLED + value: "false" + - name: GLANCE_PRODUCTION_MAPPINGS_JSON + valueFrom: + secretKeyRef: + name: dt-api-secret + key: GLANCE_PRODUCTION_MAPPINGS_JSON + optional: true envFrom: - secretRef: name: dt-api-secret diff --git a/geddes/k8s/05-postgres.yaml b/geddes/k8s/05-postgres.yaml index 8a80970..45a928d 100644 --- a/geddes/k8s/05-postgres.yaml +++ b/geddes/k8s/05-postgres.yaml @@ -136,6 +136,11 @@ data: row_index INT, upload_filename VARCHAR(255) DEFAULT '', source VARCHAR(100) DEFAULT 'data_upload', + source_system VARCHAR(100) NOT NULL DEFAULT 'data_upload', + source_tool_id BIGINT, + source_run_id BIGINT, + source_updated_at TIMESTAMP WITH TIME ZONE, + ingested_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); @@ -148,6 +153,11 @@ data: CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row ON equipment_runs(upload_id, row_index) WHERE upload_id IS NOT NULL; + CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_source_identity + ON equipment_runs( + source_system, equipment_id, source_tool_id, source_run_id + ) + WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL; ALTER TABLE equipment_runs ENABLE ROW LEVEL SECURITY; GRANT SELECT ON equipment_runs TO api_client; @@ -178,8 +188,8 @@ data: sample_index INT NOT NULL, source_sample_id BIGINT NOT NULL, source_tool_id BIGINT, - sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, - sample_time_raw TEXT NOT NULL, + sample_time TIMESTAMP WITHOUT TIME ZONE, + sample_time_raw TEXT, values_json JSONB NOT NULL DEFAULT '{}'::jsonb, PRIMARY KEY (equipment_run_id, sample_index), UNIQUE (equipment_run_id, source_sample_id), @@ -189,6 +199,11 @@ data: CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); + ALTER TABLE equipment_run_trace_samples + ALTER COLUMN sample_time DROP NOT NULL; + ALTER TABLE equipment_run_trace_samples + ALTER COLUMN sample_time_raw DROP NOT NULL; + ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; GRANT SELECT ON equipment_run_trace_samples TO api_client; @@ -213,6 +228,77 @@ data: ) ); + CREATE TABLE IF NOT EXISTS equipment_run_trace_events ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + source_event_id BIGINT NOT NULL, + source_tool_id BIGINT, + event_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + event_time_raw TEXT NOT NULL, + event_type TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + raw_event_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, source_event_id), + CHECK (jsonb_typeof(raw_event_json) = 'object') + ); + + CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_events_time + ON equipment_run_trace_events( + equipment_run_id, event_time, source_event_id + ); + + ALTER TABLE equipment_run_trace_events ENABLE ROW LEVEL SECURITY; + GRANT SELECT ON equipment_run_trace_events TO api_client; + + CREATE POLICY equipment_run_trace_event_visibility_policy + ON equipment_run_trace_events + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_events.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = + current_setting('app.current_user', true) + ) + ) + ) + ); + + CREATE TABLE IF NOT EXISTS glance_ingestion_audit ( + id BIGSERIAL PRIMARY KEY, + batch_id VARCHAR(255) NOT NULL, + source_system VARCHAR(100) NOT NULL DEFAULT 'glance', + equipment_id VARCHAR(255) NOT NULL, + source_tool_id BIGINT NOT NULL, + project_id VARCHAR(50), + previous_cursor TEXT, + proposed_cursor TEXT, + poll_started_at TIMESTAMP WITH TIME ZONE, + poll_completed_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + status VARCHAR(32) NOT NULL, + run_count INT NOT NULL DEFAULT 0, + sample_count INT NOT NULL DEFAULT 0, + event_count INT NOT NULL DEFAULT 0, + warning_count INT NOT NULL DEFAULT 0, + payload_sha256 VARCHAR(64), + error_text TEXT, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + ); + + CREATE INDEX IF NOT EXISTS idx_glance_ingestion_audit_tool_time + ON glance_ingestion_audit( + equipment_id, source_tool_id, created_at DESC + ); + CREATE TABLE IF NOT EXISTS users ( id VARCHAR(255) PRIMARY KEY, name VARCHAR(255) NOT NULL, diff --git a/geddes/k8s/glance-ingestion-cronjob.yaml b/geddes/k8s/glance-ingestion-cronjob.yaml new file mode 100644 index 0000000..351960d --- /dev/null +++ b/geddes/k8s/glance-ingestion-cronjob.yaml @@ -0,0 +1,111 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: dt-glance-ingestion-state + namespace: ncn-digitaltwins-zchen +spec: + accessModes: + - ReadWriteOnce + storageClassName: geddes-standard-singlenode + resources: + requests: + storage: 1Gi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dt-glance-ingestion + namespace: ncn-digitaltwins-zchen +spec: + # Repository resources are deliberately inactive. Replace the image digest, + # review the equipment/project mapping, and approve the bounded pilot before + # changing this value. + suspend: true + schedule: "*/10 * * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 540 + template: + metadata: + labels: + app: dt-glance-ingestion + spec: + restartPolicy: Never + imagePullSecrets: + - name: sdx-registry-secret + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: connector + image: geddes-registry.rcac.purdue.edu/sdx/dt-glance-ingestion:REPLACE_WITH_DIGEST + imagePullPolicy: IfNotPresent + env: + - name: GLANCE_SOURCE_MODE + value: postgres + - name: GLANCE_DB_URI + valueFrom: + secretKeyRef: + name: glance-postgrest-credentials + key: PGRST_DB_URI + - name: GLANCE_INGESTION_API_URL + value: http://dt-api:8000/api/dataset/v2/glance/traces/sync + - name: INGESTION_TOKEN + valueFrom: + secretKeyRef: + name: dt-api-secret + key: INGESTION_TOKEN + - name: GLANCE_PRODUCTION_MAPPINGS_JSON + valueFrom: + secretKeyRef: + name: dt-api-secret + key: GLANCE_PRODUCTION_MAPPINGS_JSON + - name: GLANCE_CURSOR_BACKEND + value: file + - name: GLANCE_CURSOR_FILE + value: /state/cursors.json + - name: GLANCE_FAILURE_AUDIT_FILE + value: /state/failures.jsonl + - name: GLANCE_RUN_OVERLAP_COUNT + value: "5" + - name: GLANCE_DATA_BATCH_SIZE + value: "25" + - name: GLANCE_CONNECT_TIMEOUT_SECONDS + value: "10" + - name: GLANCE_READ_TIMEOUT_SECONDS + value: "120" + - name: GLANCE_DB_RETRIES + value: "3" + - name: GLANCE_MAX_RUNS_PER_POLL + value: "20" + - name: GLANCE_MAX_SAMPLES_PER_RUN + value: "100000" + - name: GLANCE_MAX_VALUES_PER_RUN + value: "1000000" + - name: GLANCE_HANDOFF_RUN_BATCH_SIZE + value: "1" + # Approved Tool 3 pilot only. Remove this allowlist before + # enabling normal ten-minute incremental polling. + - name: GLANCE_BACKFILL_RUN_IDS + value: "1,5,7,8,29,32,73,161,162,164,165,166,169" + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + volumeMounts: + - name: state + mountPath: /state + volumes: + - name: state + persistentVolumeClaim: + claimName: dt-glance-ingestion-state diff --git a/geddes/k8s/glance-mirror/01-secret.yaml.example b/geddes/k8s/glance-mirror/01-secret.yaml.example new file mode 100644 index 0000000..69baf93 --- /dev/null +++ b/geddes/k8s/glance-mirror/01-secret.yaml.example @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: glance-mirror-secret + namespace: ncn-digitaltwins-zchen +type: Opaque +stringData: + POSTGRES_PASSWORD: "REPLACE_WITH_RANDOM_POSTGRES_ADMIN_PASSWORD" + MIRROR_WRITER_PASSWORD: "REPLACE_WITH_RANDOM_WRITER_PASSWORD" + MIRROR_AUTHENTICATOR_PASSWORD: "REPLACE_WITH_RANDOM_AUTHENTICATOR_PASSWORD" + # URL-encode reserved characters in passwords used in these URIs. + GLANCE_MIRROR_DB_URI: "postgresql://glance_mirror_writer:URL_ENCODED_WRITER_PASSWORD@dt-glance-mirror-db:5432/glance_mirror?sslmode=disable" + PGRST_DB_URI: "postgresql://glance_mirror_authenticator:URL_ENCODED_AUTHENTICATOR_PASSWORD@dt-glance-mirror-db:5432/glance_mirror?sslmode=disable" diff --git a/geddes/k8s/glance-mirror/02-postgres.yaml b/geddes/k8s/glance-mirror/02-postgres.yaml new file mode 100644 index 0000000..4fc5741 --- /dev/null +++ b/geddes/k8s/glance-mirror/02-postgres.yaml @@ -0,0 +1,300 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: dt-glance-mirror-db + namespace: ncn-digitaltwins-zchen +spec: + accessModes: + - ReadWriteOnce + storageClassName: geddes-standard-singlenode + resources: + requests: + storage: 150Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: dt-glance-mirror-init + namespace: ncn-digitaltwins-zchen +data: + 00-roles.sh: | + #!/bin/sh + set -eu + + psql \ + --set=ON_ERROR_STOP=1 \ + --set=writer_password="$MIRROR_WRITER_PASSWORD" \ + --set=authenticator_password="$MIRROR_AUTHENTICATOR_PASSWORD" \ + --username "$POSTGRES_USER" \ + --dbname "$POSTGRES_DB" <<'EOSQL' + SELECT 'CREATE ROLE glance_mirror_reader NOLOGIN' + WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'glance_mirror_reader' + ) \gexec + + SELECT format( + 'CREATE ROLE glance_mirror_writer LOGIN PASSWORD %L', + :'writer_password' + ) + WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'glance_mirror_writer' + ) \gexec + SELECT format( + 'ALTER ROLE glance_mirror_writer LOGIN PASSWORD %L', + :'writer_password' + ) \gexec + + SELECT format( + 'CREATE ROLE glance_mirror_authenticator LOGIN PASSWORD %L', + :'authenticator_password' + ) + WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'glance_mirror_authenticator' + ) \gexec + SELECT format( + 'ALTER ROLE glance_mirror_authenticator LOGIN PASSWORD %L', + :'authenticator_password' + ) \gexec + + GRANT CONNECT ON DATABASE glance_mirror + TO glance_mirror_writer, glance_mirror_authenticator; + GRANT glance_mirror_reader TO glance_mirror_authenticator; + EOSQL + + 10-schema.sql: | + CREATE SCHEMA IF NOT EXISTS glance_mirror; + REVOKE ALL ON SCHEMA glance_mirror FROM PUBLIC; + + CREATE TABLE IF NOT EXISTS glance_mirror.tools ( + idtools INTEGER PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS glance_mirror.recipes ( + idrecipes INTEGER PRIMARY KEY, + idtools INTEGER NOT NULL + REFERENCES glance_mirror.tools(idtools), + recipename TEXT NOT NULL, + recipefile TEXT, + hash TEXT NOT NULL, + timestamp TIMESTAMP WITHOUT TIME ZONE + ); + + CREATE TABLE IF NOT EXISTS glance_mirror.runs ( + idruns INTEGER PRIMARY KEY, + idtools INTEGER NOT NULL + REFERENCES glance_mirror.tools(idtools), + lotname TEXT, + idrecipes INTEGER + REFERENCES glance_mirror.recipes(idrecipes), + starttime TIMESTAMP WITHOUT TIME ZONE NOT NULL, + endtime TIMESTAMP WITHOUT TIME ZONE, + status TEXT, + materialname TEXT + ); + + CREATE INDEX IF NOT EXISTS runs_tool_id_idx + ON glance_mirror.runs(idtools, idruns); + + CREATE TABLE IF NOT EXISTS glance_mirror.samplerecord ( + idsamplerecord INTEGER PRIMARY KEY, + idruns INTEGER NOT NULL + REFERENCES glance_mirror.runs(idruns) ON DELETE CASCADE, + idtools INTEGER NOT NULL + REFERENCES glance_mirror.tools(idtools), + time TIMESTAMP WITHOUT TIME ZONE + ); + + CREATE INDEX IF NOT EXISTS samplerecord_run_idx + ON glance_mirror.samplerecord(idruns, idsamplerecord); + + CREATE TABLE IF NOT EXISTS glance_mirror.parameters ( + idparameters INTEGER PRIMARY KEY, + name TEXT NOT NULL, + unit TEXT + ); + + CREATE TABLE IF NOT EXISTS glance_mirror.data ( + idsamplerecord INTEGER NOT NULL + REFERENCES glance_mirror.samplerecord(idsamplerecord) + ON DELETE CASCADE, + idparameters INTEGER NOT NULL + REFERENCES glance_mirror.parameters(idparameters), + value DOUBLE PRECISION NOT NULL, + PRIMARY KEY (idsamplerecord, idparameters) + ); + + CREATE INDEX IF NOT EXISTS data_parameter_idx + ON glance_mirror.data(idparameters, idsamplerecord); + + CREATE TABLE IF NOT EXISTS glance_mirror.eventtypes ( + event TEXT PRIMARY KEY, + category INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS glance_mirror.events ( + idevents INTEGER PRIMARY KEY, + time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + tool INTEGER NOT NULL + REFERENCES glance_mirror.tools(idtools), + type TEXT, + description TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS events_tool_time_idx + ON glance_mirror.events(tool, time, idevents); + + CREATE TABLE IF NOT EXISTS glance_mirror.run_events ( + idruns INTEGER NOT NULL + REFERENCES glance_mirror.runs(idruns) ON DELETE CASCADE, + idevents INTEGER NOT NULL + REFERENCES glance_mirror.events(idevents) ON DELETE CASCADE, + PRIMARY KEY (idruns, idevents) + ); + + CREATE INDEX IF NOT EXISTS run_events_event_idx + ON glance_mirror.run_events(idevents, idruns); + + CREATE TABLE IF NOT EXISTS glance_mirror.sync_state ( + idtools INTEGER NOT NULL + REFERENCES glance_mirror.tools(idtools), + state_key TEXT NOT NULL, + last_run_id INTEGER NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL, + PRIMARY KEY (idtools, state_key) + ); + + CREATE TABLE IF NOT EXISTS glance_mirror.sync_audit ( + id BIGSERIAL PRIMARY KEY, + idtools INTEGER NOT NULL, + idruns INTEGER, + state_key TEXT NOT NULL, + previous_cursor INTEGER NOT NULL, + committed_cursor INTEGER NOT NULL, + sample_count BIGINT NOT NULL DEFAULT 0, + value_count BIGINT NOT NULL DEFAULT 0, + event_count BIGINT NOT NULL DEFAULT 0, + status TEXT NOT NULL CHECK (status IN ('success', 'failed')), + duration_ms BIGINT NOT NULL, + error_text TEXT NOT NULL DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE NOT NULL + ); + + CREATE INDEX IF NOT EXISTS sync_audit_tool_time_idx + ON glance_mirror.sync_audit(idtools, created_at DESC); + + GRANT USAGE ON SCHEMA glance_mirror + TO glance_mirror_writer, glance_mirror_reader; + GRANT SELECT, INSERT, UPDATE, DELETE + ON ALL TABLES IN SCHEMA glance_mirror + TO glance_mirror_writer; + GRANT USAGE, SELECT + ON ALL SEQUENCES IN SCHEMA glance_mirror + TO glance_mirror_writer; + GRANT SELECT ON ALL TABLES IN SCHEMA glance_mirror + TO glance_mirror_reader; + + ALTER DEFAULT PRIVILEGES IN SCHEMA glance_mirror + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES + TO glance_mirror_writer; + ALTER DEFAULT PRIVILEGES IN SCHEMA glance_mirror + GRANT USAGE, SELECT ON SEQUENCES + TO glance_mirror_writer; + ALTER DEFAULT PRIVILEGES IN SCHEMA glance_mirror + GRANT SELECT ON TABLES + TO glance_mirror_reader; +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dt-glance-mirror-db + namespace: ncn-digitaltwins-zchen +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: dt-glance-mirror-db + template: + metadata: + labels: + app: dt-glance-mirror-db + spec: + containers: + - name: postgres + image: postgres:15 + ports: + - name: postgres + containerPort: 5432 + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_DB + value: glance_mirror + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: glance-mirror-secret + key: POSTGRES_PASSWORD + - name: MIRROR_WRITER_PASSWORD + valueFrom: + secretKeyRef: + name: glance-mirror-secret + key: MIRROR_WRITER_PASSWORD + - name: MIRROR_AUTHENTICATOR_PASSWORD + valueFrom: + secretKeyRef: + name: glance-mirror-secret + key: MIRROR_AUTHENTICATOR_PASSWORD + readinessProbe: + exec: + command: + - pg_isready + - -U + - postgres + - -d + - glance_mirror + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "4" + memory: 8Gi + volumeMounts: + - name: database + mountPath: /var/lib/postgresql/data + - name: init + mountPath: /docker-entrypoint-initdb.d/00-roles.sh + subPath: 00-roles.sh + - name: init + mountPath: /docker-entrypoint-initdb.d/10-schema.sql + subPath: 10-schema.sql + volumes: + - name: database + persistentVolumeClaim: + claimName: dt-glance-mirror-db + - name: init + configMap: + name: dt-glance-mirror-init + defaultMode: 0555 +--- +apiVersion: v1 +kind: Service +metadata: + name: dt-glance-mirror-db + namespace: ncn-digitaltwins-zchen +spec: + type: ClusterIP + selector: + app: dt-glance-mirror-db + ports: + - name: postgres + port: 5432 + targetPort: postgres diff --git a/geddes/k8s/glance-mirror/03-postgrest.yaml b/geddes/k8s/glance-mirror/03-postgrest.yaml new file mode 100644 index 0000000..ee0cda1 --- /dev/null +++ b/geddes/k8s/glance-mirror/03-postgrest.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dt-glance-mirror-postgrest + namespace: ncn-digitaltwins-zchen +spec: + replicas: 1 + selector: + matchLabels: + app: dt-glance-mirror-postgrest + template: + metadata: + labels: + app: dt-glance-mirror-postgrest + spec: + containers: + - name: postgrest + image: geddes-registry.rcac.purdue.edu/docker-hub-cache/postgrest/postgrest:v13.0.6 + ports: + - name: http + containerPort: 3000 + env: + - name: PGRST_DB_URI + valueFrom: + secretKeyRef: + name: glance-mirror-secret + key: PGRST_DB_URI + - name: PGRST_DB_SCHEMAS + value: glance_mirror + - name: PGRST_DB_ANON_ROLE + value: glance_mirror_reader + - name: PGRST_SERVER_PORT + value: "3000" + - name: PGRST_LOG_LEVEL + value: warn + readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: dt-glance-mirror-postgrest + namespace: ncn-digitaltwins-zchen +spec: + type: ClusterIP + selector: + app: dt-glance-mirror-postgrest + ports: + - name: http + port: 3000 + targetPort: http diff --git a/geddes/k8s/glance-mirror/04-sync-cronjob.yaml b/geddes/k8s/glance-mirror/04-sync-cronjob.yaml new file mode 100644 index 0000000..7caa986 --- /dev/null +++ b/geddes/k8s/glance-mirror/04-sync-cronjob.yaml @@ -0,0 +1,75 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dt-glance-mirror-sync + namespace: ncn-digitaltwins-zchen +spec: + suspend: true + schedule: "*/10 * * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 1800 + template: + metadata: + labels: + app: dt-glance-mirror-sync + spec: + restartPolicy: Never + imagePullSecrets: + - name: sdx-registry-secret + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + containers: + - name: mirror + image: geddes-registry.rcac.purdue.edu/sdx/dt-glance-ingestion:REPLACE_WITH_DIGEST + imagePullPolicy: IfNotPresent + command: + - python + - scripts/run_glance_mirror.py + env: + - name: GLANCE_DB_URI + valueFrom: + secretKeyRef: + name: glance-postgrest-credentials + key: PGRST_DB_URI + - name: GLANCE_MIRROR_DB_URI + valueFrom: + secretKeyRef: + name: glance-mirror-secret + key: GLANCE_MIRROR_DB_URI + - name: GLANCE_MIRROR_TOOL_IDS + value: "3" + - name: GLANCE_MIRROR_INITIAL_RUN_ID + value: "1" + - name: GLANCE_MIRROR_OVERLAP_RUN_IDS + value: "5" + - name: GLANCE_MIRROR_MAX_RUNS_PER_TOOL + value: "20" + - name: GLANCE_MIRROR_MAX_SAMPLES_PER_RUN + value: "100000" + - name: GLANCE_MIRROR_MAX_VALUES_PER_RUN + value: "1000000" + - name: GLANCE_MIRROR_INSERT_BATCH_SIZE + value: "5000" + - name: GLANCE_MIRROR_STATEMENT_TIMEOUT_SECONDS + value: "300" + - name: GLANCE_MIRROR_SOURCE_RETRIES + value: "3" + # Bounded initial Tool 3 mirror pilot. Remove this setting only + # after the pilot count comparison and backfill plan are approved. + - name: GLANCE_MIRROR_BACKFILL_RUN_IDS + value: "1,5,7,8,29,32,73,161,162,164,165,166,169" + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi diff --git a/geddes/k8s/glance-mirror/05-network-policy.yaml b/geddes/k8s/glance-mirror/05-network-policy.yaml new file mode 100644 index 0000000..8c62567 --- /dev/null +++ b/geddes/k8s/glance-mirror/05-network-policy.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dt-glance-mirror-db-ingress + namespace: ncn-digitaltwins-zchen +spec: + podSelector: + matchLabels: + app: dt-glance-mirror-db + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: dt-glance-mirror-sync + - podSelector: + matchLabels: + app: dt-glance-mirror-postgrest + ports: + - protocol: TCP + port: 5432 diff --git a/geddes/k8s/glance-mirror/README.md b/geddes/k8s/glance-mirror/README.md new file mode 100644 index 0000000..3543da8 --- /dev/null +++ b/geddes/k8s/glance-mirror/README.md @@ -0,0 +1,52 @@ +# GLANCE modern mirror + +These manifests define an inactive, private PostgreSQL 15 mirror of the +authoritative GLANCE PostgreSQL 10.5 database. + +The source was approximately 53 GB on 2026-07-28. Its `data` table contained +approximately 613 million rows. The mirror therefore uses a separate, +expandable 150 GiB PVC and must be backfilled in bounded tool/run windows. + +## Safety boundary + +- Both CronJobs in this repository are committed with `suspend: true`. +- The mirror PostgREST service is `ClusterIP`; no Ingress is supplied. +- The source identity is read-only. +- The mirror writer can modify only the destination mirror schema. +- One run is replaced and its mode-specific tool cursor is handled in one + destination transaction. +- Live polling, explicit-ID pilots, and bounded backfills have independent + cursor ledgers. +- A failed run prevents later cursor advancement. Later valid runs can still + be copied idempotently and will be revisited after the failed run succeeds. +- Per-run sample and value limits prevent an unexpectedly large source run + from exhausting worker memory. Raising either limit requires review. +- Run/event associations reconcile events that are removed or moved on a + subsequent source snapshot without deleting events shared by another run. +- Null recipe IDs are valid and are mirrored. + +Do not apply these manifests as a group before creating reviewed credentials: +`02-postgres.yaml` allocates a 150 GiB PVC and starts a database. + +## Reviewed rollout + +1. Copy `01-secret.yaml.example` outside Git, generate three independent random + passwords, URL-encode them in the two URIs, and create + `glance-mirror-secret`. +2. Review the 150 GiB allocation and backup/retention plan. +3. Apply `02-postgres.yaml` and verify the schema and role grants. +4. Build `azure/Dockerfile.glance`, push it, and replace every + `REPLACE_WITH_DIGEST` image reference with a digest-pinned image. +5. Apply `03-postgrest.yaml`, `04-sync-cronjob.yaml`, and + `05-network-policy.yaml`. Keep the CronJob suspended. +6. Create one manual Job from the CronJob for one approved Tool 3 run. +7. Compare source and destination run, sample, value, parameter, recipe, and + event counts. +8. Run the complete approved Tool 3 allowlist and repeat the comparison. +9. Remove the allowlist only after a separately reviewed historical backfill + plan establishes bounded run/date windows. +10. Unsuspend ten-minute synchronization only after backfill and lag monitoring + are accepted. + +PostgREST cutover is the last step. Until then, the direct read-only SQL +connector remains the authoritative ingestion path. diff --git a/geddes/k8s/ml-shadow-evaluation-cronjob.yaml b/geddes/k8s/ml-shadow-evaluation-cronjob.yaml new file mode 100644 index 0000000..09d851e --- /dev/null +++ b/geddes/k8s/ml-shadow-evaluation-cronjob.yaml @@ -0,0 +1,33 @@ +# Review and set ML_SHADOW_PROJECT_ID before unsuspending. This job is +# intentionally disabled in source; applying it cannot change proposal behavior. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dt-ml-shadow-evaluation + namespace: ncn-digitaltwins-zchen +spec: + schedule: "20 3 * * 1" + suspend: true + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: evaluator + image: ghcr.io/OWNER/IMAGE@sha256:REPLACE_WITH_IMMUTABLE_DIGEST + command: + - /bin/sh + - -c + - >- + python api/scripts/run_ml_shadow_evaluation.py + --project-id "${ML_SHADOW_PROJECT_ID}" + env: + - name: ML_SHADOW_PROJECT_ID + value: REPLACE_WITH_APPROVED_PROJECT_ID + envFrom: + - secretRef: + name: dt-api-secrets diff --git a/geddes/k8s/postgres/schema_v2.sql b/geddes/k8s/postgres/schema_v2.sql index 0653384..32fb8cb 100644 --- a/geddes/k8s/postgres/schema_v2.sql +++ b/geddes/k8s/postgres/schema_v2.sql @@ -111,6 +111,11 @@ CREATE TABLE IF NOT EXISTS equipment_runs ( row_index INT, upload_filename VARCHAR(255) DEFAULT '', source VARCHAR(100) DEFAULT 'data_upload', + source_system VARCHAR(100) NOT NULL DEFAULT 'data_upload', + source_tool_id BIGINT, + source_run_id BIGINT, + source_updated_at TIMESTAMP WITH TIME ZONE, + ingested_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); @@ -122,6 +127,11 @@ CREATE INDEX IF NOT EXISTS idx_equipment_runs_upload_id ON equipment_runs(upload CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row ON equipment_runs(upload_id, row_index) WHERE upload_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_source_identity + ON equipment_runs( + source_system, equipment_id, source_tool_id, source_run_id + ) + WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL; ALTER TABLE equipment_runs ENABLE ROW LEVEL SECURITY; GRANT SELECT ON equipment_runs TO api_client; @@ -154,8 +164,8 @@ CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( sample_index INT NOT NULL, source_sample_id BIGINT NOT NULL, source_tool_id BIGINT, - sample_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, - sample_time_raw TEXT NOT NULL, + sample_time TIMESTAMP WITHOUT TIME ZONE, + sample_time_raw TEXT, values_json JSONB NOT NULL DEFAULT '{}'::jsonb, PRIMARY KEY (equipment_run_id, sample_index), UNIQUE (equipment_run_id, source_sample_id), @@ -165,6 +175,11 @@ CREATE TABLE IF NOT EXISTS equipment_run_trace_samples ( CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_samples_time ON equipment_run_trace_samples(equipment_run_id, sample_time, sample_index); +ALTER TABLE equipment_run_trace_samples + ALTER COLUMN sample_time DROP NOT NULL; +ALTER TABLE equipment_run_trace_samples + ALTER COLUMN sample_time_raw DROP NOT NULL; + ALTER TABLE equipment_run_trace_samples ENABLE ROW LEVEL SECURITY; GRANT SELECT ON equipment_run_trace_samples TO api_client; @@ -189,6 +204,73 @@ CREATE POLICY equipment_run_trace_visibility_policy ) ); +CREATE TABLE IF NOT EXISTS equipment_run_trace_events ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + source_event_id BIGINT NOT NULL, + source_tool_id BIGINT, + event_time TIMESTAMP WITHOUT TIME ZONE NOT NULL, + event_time_raw TEXT NOT NULL, + event_type TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + raw_event_json JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (equipment_run_id, source_event_id), + CHECK (jsonb_typeof(raw_event_json) = 'object') +); + +CREATE INDEX IF NOT EXISTS idx_equipment_run_trace_events_time +ON equipment_run_trace_events(equipment_run_id, event_time, source_event_id); + +ALTER TABLE equipment_run_trace_events ENABLE ROW LEVEL SECURITY; +GRANT SELECT ON equipment_run_trace_events TO api_client; + +CREATE POLICY equipment_run_trace_event_visibility_policy + ON equipment_run_trace_events + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_trace_events.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = + current_setting('app.current_user', true) + ) + ) + ) + ); + +CREATE TABLE IF NOT EXISTS glance_ingestion_audit ( + id BIGSERIAL PRIMARY KEY, + batch_id VARCHAR(255) NOT NULL, + source_system VARCHAR(100) NOT NULL DEFAULT 'glance', + equipment_id VARCHAR(255) NOT NULL, + source_tool_id BIGINT NOT NULL, + project_id VARCHAR(50), + previous_cursor TEXT, + proposed_cursor TEXT, + poll_started_at TIMESTAMP WITH TIME ZONE, + poll_completed_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + status VARCHAR(32) NOT NULL, + run_count INT NOT NULL DEFAULT 0, + sample_count INT NOT NULL DEFAULT 0, + event_count INT NOT NULL DEFAULT 0, + warning_count INT NOT NULL DEFAULT 0, + payload_sha256 VARCHAR(64), + error_text TEXT, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_glance_ingestion_audit_tool_time +ON glance_ingestion_audit(equipment_id, source_tool_id, created_at DESC); + CREATE TABLE IF NOT EXISTS users ( id VARCHAR(255) PRIMARY KEY, -- NanoHUB Subject ID or Email name VARCHAR(255) NOT NULL, diff --git a/web/app/data/analysis/page.tsx b/web/app/data/analysis/page.tsx new file mode 100644 index 0000000..82ca694 --- /dev/null +++ b/web/app/data/analysis/page.tsx @@ -0,0 +1,5 @@ +import { CrossRunTraceAnalysis } from "@/components/cross-run-trace-analysis"; + +export default function TraceAnalysisPage() { + return ; +} diff --git a/web/app/optimize/page.tsx b/web/app/optimize/page.tsx index b70acec..953a966 100644 --- a/web/app/optimize/page.tsx +++ b/web/app/optimize/page.tsx @@ -5,10 +5,12 @@ import { cn } from "@/lib/utils"; import { createExperiment, getProjectExperiments, + getMlRuntimeStatus, getProposalsData, getV2Projects, runOptimization, type ProjectExperimentRecord, + type MlRuntimeStatus, type V2Project, } from "@/lib/api-client"; import { Play, Download, Info, Loader2, AlertCircle, Save } from "lucide-react"; @@ -38,9 +40,13 @@ export default function OptimizePage() { const [error, setError] = useState(null); const [batches, setBatches] = useState([]); const [source, setSource] = useState("loading..."); + const [runtime, setRuntime] = useState(null); const [selectedBatch, setSelectedBatch] = useState(0); const [projects, setProjects] = useState([]); - const [selectedProject, setSelectedProject] = useState(""); + const [selectedProject, setSelectedProject] = useState(() => { + if (typeof window === "undefined") return ""; + return new URLSearchParams(window.location.search).get("project_id") || ""; + }); const [projectUnitExperiments, setProjectUnitExperiments] = useState([]); const [selectedUnitExperiment, setSelectedUnitExperiment] = useState(""); const [savingRecipeIndex, setSavingRecipeIndex] = useState(null); @@ -63,18 +69,12 @@ export default function OptimizePage() { Etch_Avgcf4Flow: { label: "CF4 Flow", unit: "sccm" }, }; - // Fetch initial proposals - useEffect(() => { - const params = new URLSearchParams(window.location.search); - const projectId = params.get("project_id") || ""; - if (projectId) setSelectedProject(projectId); - }, []); - useEffect(() => { Promise.all([ getProposalsData(selectedProject || undefined), getV2Projects(), - ]).then(([data, projectData]) => { + getMlRuntimeStatus(), + ]).then(([data, projectData, runtimeData]) => { if (data) { setBatches(((data as Record).batches as ProposalBatch[]) || []); setSource((data as Record).source as string || "unknown"); @@ -82,14 +82,13 @@ export default function OptimizePage() { setError("Could not load optimization data. Is the API running?"); } setProjects(projectData ?? []); + setRuntime(runtimeData); setLoading(false); }); }, [selectedProject]); useEffect(() => { if (!selectedProject) { - setProjectUnitExperiments([]); - setSelectedUnitExperiment(""); return; } getProjectExperiments(selectedProject).then((items) => { @@ -103,6 +102,14 @@ export default function OptimizePage() { }); }, [selectedProject]); + const handleProjectChange = (projectId: string) => { + setSelectedProject(projectId); + if (!projectId) { + setProjectUnitExperiments([]); + setSelectedUnitExperiment(""); + } + }; + const handleOptimize = async () => { setOptimizing(true); setError(null); @@ -110,6 +117,9 @@ export default function OptimizePage() { if (result) { setBatches((result.batches as ProposalBatch[]) || []); setSource(result.source as string || "optimized"); + if (result.runtime) { + setRuntime(result.runtime as unknown as MlRuntimeStatus); + } setSelectedBatch(0); } else { setError("Optimization failed. Check the API server logs."); @@ -259,6 +269,11 @@ export default function OptimizePage() { source: {source} + {runtime && ( + + configured: {runtime.proposal_engine} / {runtime.effective_model_policy} + + )}

@@ -267,7 +282,7 @@ export default function OptimizePage() { toggle(run.run_id)} + /> + + + {run.lot_name || `Catalog ${run.run_id}`} + + + GLANCE {run.source_run_id ?? "—"} ·{" "} + {(run.trace_sample_count ?? 0).toLocaleString()} samples + + + + ); + })} + + )} + + +
+
+ + + + +
+ + {selectedIds.length > 0 && warning && ( +

+ {warning} +

+ )} + {loadingTraces && ( +

+ Loading selected + traces with three concurrent requests… +

+ )} + {trend && ( +

+ {equation} · origin {origin} · slope{" "} + {formatTrendMetric(trend.slope)} {slopeUnit} · R²{" "} + {trend.rSquared === null + ? "not defined" + : formatTrendMetric(trend.rSquared)}{" "} + · n={trend.pointCount.toLocaleString()} +

+ )} + + {selected && visibleTraces.length ? ( + + ({ + type: "line" as const, + x0: marker.x, + x1: marker.x, + y0: 0, + y1: 1, + xref: "x" as const, + yref: "paper" as const, + line: { color: marker.color, width: 2 }, + })), + annotations: markers.map((marker, index) => ({ + x: marker.x, + y: 1.02 + marker.lane * 0.08, + xref: "x" as const, + yref: "paper" as const, + text: marker.text, + showarrow: false, + xanchor: (index % 2 ? "left" : "right") as "left" | "right", + yanchor: "bottom" as const, + font: { color: marker.color, size: 9 }, + })), + }} + config={{ responsive: true, displaylogo: false, scrollZoom: true }} + useResizeHandler + style={{ width: "100%" }} + /> + + ) : ( +
+ Select one or more runs. +
+ )} +
+ + + ); +} diff --git a/web/components/run-trace-explorer.tsx b/web/components/run-trace-explorer.tsx index 4d43cdf..70f42af 100644 --- a/web/components/run-trace-explorer.tsx +++ b/web/components/run-trace-explorer.tsx @@ -1,7 +1,7 @@ "use client"; import dynamic from "next/dynamic"; -import { useEffect, useMemo, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { Activity, AlertCircle, Clock3, Loader2 } from "lucide-react"; import { ErrorBoundary } from "@/components/ErrorBoundary"; @@ -12,11 +12,16 @@ import { type V2RunTrace, type V2RunTraceParameter, } from "@/lib/api-client"; +import { + calculateLinearTrendline, + formatTrendMetric, + trimRepeatedEdgeZeros, +} from "@/lib/trace-plot"; import { cn } from "@/lib/utils"; const Plot = dynamic(() => import("react-plotly.js"), { ssr: false }); -const MAX_SELECTED_RUNS = 6; +const MAX_SELECTED_RUNS = 10; const TRACE_COLORS = [ "#8b5cf6", "#10b981", @@ -26,7 +31,10 @@ const TRACE_COLORS = [ "#ec4899", "#14b8a6", "#f97316", + "#6366f1", + "#84cc16", ]; +const TRACE_FETCH_CONCURRENCY = 3; type AxisMode = "relative" | "absolute"; @@ -54,7 +62,13 @@ function sortParameters( function parameterLabel(parameter: V2RunTraceParameter): string { const unit = parameter.unit?.trim() ? ` (${parameter.unit.trim()})` : ""; - return `${parameter.name}${unit} · ${parameter.key}`; + const name = + parameter.registered && parameter.registered_name + ? parameter.registered_name + : parameter.registered === false + ? `(Unregistered) ${parameter.name}` + : parameter.name; + return `${name}${unit} · ${parameter.key}`; } function traceLabel(trace: V2RunTrace): string { @@ -68,8 +82,51 @@ function finiteValue(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } -function colorForRun(runId: number): string { - return TRACE_COLORS[Math.abs(runId) % TRACE_COLORS.length]; +function escapePlotText(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">"); +} + +function traceMarkerLabel(trace: V2RunTrace): string { + return trace.source_run_id === null || trace.source_run_id === undefined + ? `Catalog ${trace.run_id}` + : `GLANCE ${trace.source_run_id}`; +} + +function trendX( + timestamp: string | null, + relativeSeconds: number | null, + axisMode: AxisMode, +): number | null { + if (axisMode === "relative") return relativeSeconds; + if (!timestamp) return null; + const milliseconds = Date.parse(timestamp); + return Number.isFinite(milliseconds) ? milliseconds / 3_600_000 : null; +} + +async function mapWithConcurrency( + values: T[], + concurrency: number, + task: (value: T) => Promise, +): Promise { + const results = new Array(values.length); + let nextIndex = 0; + async function worker() { + while (nextIndex < values.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await task(values[index]); + } + } + await Promise.all( + Array.from( + { length: Math.min(concurrency, values.length) }, + () => worker(), + ), + ); + return results; } export function RunTraceExplorer({ run }: RunTraceExplorerProps) { @@ -78,10 +135,13 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { const [selectedRunIds, setSelectedRunIds] = useState([run.run_id]); const [selectedParameter, setSelectedParameter] = useState(""); const [axisMode, setAxisMode] = useState("relative"); + const [smoothPlotEdges, setSmoothPlotEdges] = useState(false); + const [showTrendline, setShowTrendline] = useState(false); const [comparisonResult, setComparisonResult] = useState(null); const [initialLoading, setInitialLoading] = useState(true); const [traceError, setTraceError] = useState(null); + const traceCache = useRef(new Map()); const selectedPeerIds = useMemo( () => selectedRunIds.filter((runId) => runId !== run.run_id), @@ -113,6 +173,7 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { } const parameters = sortParameters(trace.parameters); + traceCache.current.set(`${run.run_id}:*`, trace); setCurrentTrace(trace); setSelectedParameter(parameters[0]?.key ?? ""); setPeerRuns( @@ -143,11 +204,19 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { if (!selectedParameter || selectedPeerIds.length === 0) return; async function loadPeerTraces() { - const results = await Promise.all( - selectedPeerIds.map(async (runId) => ({ - runId, - trace: await getV2RunTrace(runId, [selectedParameter]), - })), + const results = await mapWithConcurrency( + selectedPeerIds, + TRACE_FETCH_CONCURRENCY, + async (runId) => { + const cacheKey = `${runId}:${selectedParameter}`; + if (!traceCache.current.has(cacheKey)) { + traceCache.current.set( + cacheKey, + await getV2RunTrace(runId, [selectedParameter]), + ); + } + return { runId, trace: traceCache.current.get(cacheKey) ?? null }; + }, ); if (cancelled) return; @@ -202,32 +271,136 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { [comparisonRequestKey, comparisonResult, currentTrace], ); - const plotData = useMemo(() => { + const displayedSeries = useMemo(() => { if (!selectedParameter) return []; - return displayedTraces.map((trace) => ({ - x: trace.samples.map((sample) => - axisMode === "relative" ? sample.relative_seconds : sample.timestamp, - ), - y: trace.samples.map((sample) => + return displayedTraces.map((trace, traceIndex) => { + const rawValues = trace.samples.map((sample) => finiteValue(sample.values[selectedParameter]), - ), - text: trace.samples.map( - (sample) => - `Absolute: ${sample.timestamp}
` + - `Relative: ${sample.relative_seconds.toFixed(3)} s
` + - `Sample record: ${sample.sample_record_id ?? "—"}`, - ), - type: "scattergl" as const, - mode: "lines" as const, - name: traceLabel(trace), - legendgroup: String(trace.run_id), - connectgaps: false, - line: { color: colorForRun(trace.run_id), width: 1.6 }, - hovertemplate: - "%{fullData.name}
Value: %{y}
%{text}", - })); - }, [axisMode, displayedTraces, selectedParameter]); + ); + const edgeResult = smoothPlotEdges + ? trimRepeatedEdgeZeros(rawValues) + : { values: rawValues, trimmedCount: 0 }; + return { + trace, + color: TRACE_COLORS[traceIndex % TRACE_COLORS.length], + x: trace.samples.map((sample) => + axisMode === "relative" + ? sample.relative_seconds + : sample.timestamp, + ), + numericX: trace.samples.map((sample) => + trendX(sample.timestamp, sample.relative_seconds, axisMode), + ), + y: edgeResult.values, + trimmedCount: edgeResult.trimmedCount, + text: trace.samples.map( + (sample) => + `Absolute: ${sample.timestamp ?? "missing"}
` + + `Relative: ${ + sample.relative_seconds === null + ? "missing" + : `${sample.relative_seconds.toFixed(3)} s` + }
` + + `Sample record: ${sample.sample_record_id ?? "—"}`, + ), + }; + }); + }, [axisMode, displayedTraces, selectedParameter, smoothPlotEdges]); + + const plotData = useMemo( + () => + displayedSeries.map(({ trace, color, x, y, text }) => ({ + x, + y, + text, + type: "scattergl" as const, + mode: "lines" as const, + name: traceLabel(trace), + legendgroup: String(trace.run_id), + connectgaps: false, + line: { color, width: 1.6 }, + hovertemplate: + "%{fullData.name}
Value: %{y}
%{text}", + })), + [displayedSeries], + ); + + const trendline = useMemo( + () => + showTrendline + ? calculateLinearTrendline( + displayedSeries.flatMap((series) => + series.y.flatMap((value, index) => { + const x = series.numericX[index]; + return value === null || x === null ? [] : [{ x, y: value }]; + }), + ), + ) + : null, + [displayedSeries, showTrendline], + ); + const trendSlopeUnit = axisMode === "relative" ? "per second" : "per hour"; + const trendOriginLabel = trendline + ? axisMode === "relative" + ? `${formatTrendMetric(trendline.originX)} s` + : new Date(trendline.originX * 3_600_000).toISOString() + : ""; + const trendEquation = trendline + ? `y = ${formatTrendMetric(trendline.slope)} × (x − origin) + ${formatTrendMetric(trendline.interceptAtOrigin)}` + : ""; + const trendEndpointX = (numericValue: number): string | number => { + if (axisMode !== "absolute") return numericValue; + return new Date(numericValue * 3_600_000).toISOString(); + }; + const trendPlotData = trendline + ? [ + { + x: [trendEndpointX(trendline.xMin), trendEndpointX(trendline.xMax)], + y: [trendline.yMin, trendline.yMax], + type: "scatter" as const, + mode: "lines" as const, + name: "Combined linear trend", + legendgroup: "combined-trend", + line: { color: "#db2777", width: 2.5, dash: "dash" as const }, + hovertemplate: + `${trendEquation}
Origin: ${trendOriginLabel}` + + `
Slope: ${formatTrendMetric(trendline.slope)} ${trendSlopeUnit}` + + `
R²: ${trendline.rSquared === null ? "not defined" : formatTrendMetric(trendline.rSquared)}` + + `
Visible points: ${trendline.pointCount.toLocaleString()}`, + }, + ] + : []; + const trimmedZeroCount = displayedSeries.reduce( + (total, series) => total + series.trimmedCount, + 0, + ); + const processEventMarkers = useMemo( + () => + displayedTraces + .flatMap((trace) => + (trace.events ?? []) + .filter((event) => + ["Process Complete", "Process Abort"].includes(event.type), + ) + .map((event) => ({ + x: + axisMode === "relative" + ? event.relative_seconds + : event.timestamp, + text: escapePlotText( + `${event.type.replace("Process ", "")} · ${traceMarkerLabel(trace)}`, + ), + color: event.type === "Process Abort" ? "#dc2626" : "#059669", + dash: + event.type === "Process Abort" + ? ("dash" as const) + : ("solid" as const), + })), + ) + .map((event, index) => ({ ...event, lane: index % 3 })), + [axisMode, displayedTraces], + ); const runsWithoutValues = useMemo(() => { if (!selectedParameter) return []; @@ -291,7 +464,8 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) {

- {currentTrace.sample_count.toLocaleString()} samples + {currentTrace.sample_count.toLocaleString()} samples · Process{" "} + {currentTrace.process_status ?? "unknown"}
@@ -335,6 +509,36 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { ))} + +
+ + + + Plot-only; source and stored values remain unchanged. + +
@@ -418,21 +622,33 @@ export function RunTraceExplorer({ run }: RunTraceExplorerProps) { } no numeric values for this parameter.`}

)} + {(smoothPlotEdges || showTrendline) && ( +

+ {smoothPlotEdges + ? `${trimmedZeroCount.toLocaleString()} repeated edge zeroes hidden for display` + : "Raw edge zeroes displayed"} + {showTrendline && trendline + ? ` · ${trendEquation} · origin ${trendOriginLabel} · slope ${formatTrendMetric(trendline.slope)} ${trendSlopeUnit} · R² ${trendline.rSquared === null ? "not defined" : formatTrendMetric(trendline.rSquared)} · n=${trendline.pointCount.toLocaleString()}` + : showTrendline + ? " · insufficient points for a linear trend" + : ""} +

+ )}
{parameter ? ( ({ + type: "line" as const, + x0: event.x, + x1: event.x, + y0: 0, + y1: 1, + xref: "x" as const, + yref: "paper" as const, + line: { color: event.color, width: 2, dash: event.dash }, + })), + annotations: processEventMarkers.map((event, index) => ({ + x: event.x, + y: 1.02 + event.lane * 0.08, + xref: "x" as const, + yref: "paper" as const, + text: event.text, + showarrow: false, + textangle: 0, + xanchor: (index % 2 === 0 ? "right" : "left") as + | "right" + | "left", + yanchor: "bottom" as const, + font: { color: event.color, size: 9 }, + })), }} config={{ responsive: true, diff --git a/web/components/sidebar.tsx b/web/components/sidebar.tsx index f782695..4ac1973 100644 --- a/web/components/sidebar.tsx +++ b/web/components/sidebar.tsx @@ -57,6 +57,7 @@ const NAV_SECTIONS = [ { id: "upload", label: "Upload", href: "/data/upload", icon: Upload }, { id: "ingestion", label: "Ingestion", href: "/data/monitor", icon: Activity }, { id: "catalog", label: "Catalog", href: "/data/catalog", icon: Search }, + { id: "analysis", label: "Trace Analysis", href: "/data/analysis", icon: Activity }, ], }, { diff --git a/web/lib/api-client.ts b/web/lib/api-client.ts index 4ce3b18..6de8f81 100644 --- a/web/lib/api-client.ts +++ b/web/lib/api-client.ts @@ -152,6 +152,7 @@ export interface V2Run { run_id: number; source_run_id?: number | string | null; trace_sample_count?: number; + process_status?: "complete" | "abort" | "unknown" | null; execution_request_id: string; lot_name: string; run_date: string | null; @@ -505,26 +506,49 @@ export interface V2RunTraceParameter { key: string; name: string; unit: string | null; + source_parameter_id?: number | string | null; + source_name?: string | null; + registered?: boolean; + registered_name?: string | null; + registered_parameter_id?: number | string | null; + registration_kind?: "input" | "output" | string | null; + registration_match?: "source_parameter_id" | "name_and_unit" | string | null; +} + +export interface V2RunTraceEvent { + source_event_id: number | string; + source_tool_id: number | string | null; + timestamp: string; + relative_seconds: number | null; + type: string; + category: number | string | null; + description: string; + raw?: Record; } export interface V2RunTraceSample { sample_record_id: number | string | null; source_tool_id: number | string | null; - timestamp: string; - relative_seconds: number; + timestamp: string | null; + relative_seconds: number | null; values: Record; } export interface V2RunTrace { run_id: number; source_run_id: number | string | null; + source_system?: string; + source_tool_id?: number | string | null; lot_name: string; sample_count: number; start_time: string | null; end_time: string | null; timestamp_timezone: string | null; + process_status: "complete" | "abort" | "unknown"; + process_outcome?: "complete" | "abort" | "unknown"; parameters: V2RunTraceParameter[]; samples: V2RunTraceSample[]; + events: V2RunTraceEvent[]; } export function getV2RunTrace( @@ -587,6 +611,34 @@ export interface ProposalsData { [key: string]: unknown; } +export interface MlRuntimeStatus { + proposal_engine: string; + proposal_engine_policy: string; + proposal_model: string; + effective_model_policy: string; + candidate_count: number; + legacy: { + candidate_count: number; + acquisition: string; + batch_implementation: string; + gpr_start_iteration: number; + }; + active_registry_models: Array<{ + model_id: number; + target: string; + model_kind: string; + version: number; + n_train: number; + trained_at: string | null; + }>; + registry_error?: string | null; + shadow_evaluation_enabled: boolean; +} + +export function getMlRuntimeStatus() { + return apiFetch("/ml/status"); +} + export function getParityData() { return apiFetch("/ml/parity"); } diff --git a/web/lib/auth-context.tsx b/web/lib/auth-context.tsx index 9774e36..7ebb1ba 100644 --- a/web/lib/auth-context.tsx +++ b/web/lib/auth-context.tsx @@ -82,7 +82,7 @@ export const ROLE_PERMISSIONS: Record< visibleNavSections: ["main", "data", "ml", "admin"], visibleNavItems: [ "dashboard", "assistant", "equipment", "projects", "experiments", "processes", "templates", "library", "public_library", "optimize", "analytics", - "samples", "upload", "ingestion", "catalog", "parity", "importance", + "samples", "upload", "ingestion", "catalog", "analysis", "parity", "importance", "convergence", "proposals", "users", "reviews", "execution_queue", "settings", ], }, @@ -98,7 +98,7 @@ export const ROLE_PERMISSIONS: Record< visibleNavSections: ["main", "data", "ml", "admin"], visibleNavItems: [ "dashboard", "assistant", "equipment", "projects", "experiments", "processes", "templates", "library", "public_library", "optimize", "analytics", - "samples", "upload", "ingestion", "catalog", "parity", "importance", + "samples", "upload", "ingestion", "catalog", "analysis", "parity", "importance", "convergence", "proposals", "settings", ], }, @@ -114,7 +114,7 @@ export const ROLE_PERMISSIONS: Record< visibleNavSections: ["main", "data", "ml", "admin"], visibleNavItems: [ "dashboard", "assistant", "equipment", "projects", "experiments", "processes", "templates", "library", "public_library", "analytics", - "samples", "upload", "ingestion", "catalog", "execution_queue", "settings", + "samples", "upload", "ingestion", "catalog", "analysis", "execution_queue", "settings", ], }, researcher: { @@ -129,7 +129,7 @@ export const ROLE_PERMISSIONS: Record< visibleNavSections: ["main", "data", "ml", "admin"], visibleNavItems: [ "dashboard", "assistant", "equipment", "projects", "experiments", "processes", "templates", "library", "public_library", "optimize", "analytics", - "samples", "upload", "catalog", "parity", "importance", "convergence", + "samples", "upload", "catalog", "analysis", "parity", "importance", "convergence", "proposals", "settings", ], }, diff --git a/web/lib/trace-plot.ts b/web/lib/trace-plot.ts new file mode 100644 index 0000000..34b0b50 --- /dev/null +++ b/web/lib/trace-plot.ts @@ -0,0 +1,150 @@ +export interface EdgeZeroTrimResult { + values: Array; + trimmedCount: number; +} + +export interface LinearTrendline { + xMin: number; + xMax: number; + yMin: number; + yMax: number; + slope: number; + interceptAtOrigin: number; + originX: number; + rSquared: number | null; + pointCount: number; +} + +interface NumericPoint { + x: number; + y: number; +} + +/** + * Hide repeated zeroes at the physical edges of one displayed series. + * + * This returns a new array and never mutates source trace data. Interior zeroes, + * short edge runs, and entirely-zero signals remain visible. + */ +export function trimRepeatedEdgeZeros( + values: Array, + minimumRunLength = 2, +): EdgeZeroTrimResult { + if ( + minimumRunLength < 2 || + !values.some((value) => value !== null && value !== 0) + ) { + return { values: [...values], trimmedCount: 0 }; + } + + let leadingZeroCount = 0; + while (leadingZeroCount < values.length && values[leadingZeroCount] === 0) { + leadingZeroCount += 1; + } + let trailingZeroCount = 0; + while ( + trailingZeroCount < values.length && + values[values.length - trailingZeroCount - 1] === 0 + ) { + trailingZeroCount += 1; + } + + const trimLeading = leadingZeroCount >= minimumRunLength; + const trimTrailing = trailingZeroCount >= minimumRunLength; + if (!trimLeading && !trimTrailing) { + return { values: [...values], trimmedCount: 0 }; + } + + const displayed = [...values]; + if (trimLeading) { + for (let index = 0; index < leadingZeroCount; index += 1) { + displayed[index] = null; + } + } + if (trimTrailing) { + for ( + let index = displayed.length - trailingZeroCount; + index < displayed.length; + index += 1 + ) { + displayed[index] = null; + } + } + return { + values: displayed, + trimmedCount: + (trimLeading ? leadingZeroCount : 0) + + (trimTrailing ? trailingZeroCount : 0), + }; +} + +/** Calculate one least-squares line across every visible finite point. */ +export function calculateLinearTrendline( + points: NumericPoint[], +): LinearTrendline | null { + const finitePoints = points.filter( + (point) => Number.isFinite(point.x) && Number.isFinite(point.y), + ); + if (finitePoints.length < 2) return null; + + const { xMin, xMax } = finitePoints.reduce( + (range, point) => ({ + xMin: Math.min(range.xMin, point.x), + xMax: Math.max(range.xMax, point.x), + }), + { xMin: Number.POSITIVE_INFINITY, xMax: Number.NEGATIVE_INFINITY }, + ); + if (xMin === xMax) return null; + + // Rebase absolute epoch values before fitting to avoid cancellation error. + const localPoints = finitePoints.map((point) => ({ + x: point.x - xMin, + y: point.y, + })); + const meanX = + localPoints.reduce((total, point) => total + point.x, 0) / + localPoints.length; + const meanY = + localPoints.reduce((total, point) => total + point.y, 0) / + localPoints.length; + const covariance = localPoints.reduce( + (total, point) => total + (point.x - meanX) * (point.y - meanY), + 0, + ); + const varianceX = localPoints.reduce( + (total, point) => total + (point.x - meanX) ** 2, + 0, + ); + if (varianceX === 0) return null; + + const slope = covariance / varianceX; + const interceptAtOrigin = meanY - slope * meanX; + const predict = (x: number) => interceptAtOrigin + slope * (x - xMin); + const residualSquares = finitePoints.reduce( + (total, point) => total + (point.y - predict(point.x)) ** 2, + 0, + ); + const totalSquares = finitePoints.reduce( + (total, point) => total + (point.y - meanY) ** 2, + 0, + ); + return { + xMin, + xMax, + yMin: predict(xMin), + yMax: predict(xMax), + slope, + interceptAtOrigin, + originX: xMin, + rSquared: totalSquares === 0 ? null : 1 - residualSquares / totalSquares, + pointCount: finitePoints.length, + }; +} + +export function formatTrendMetric(value: number): string { + const absolute = Math.abs(value); + if ((absolute > 0 && absolute < 0.001) || absolute >= 10_000) { + return value.toExponential(3); + } + return value.toLocaleString("en-US", { maximumSignificantDigits: 5 }); +} From 5b5ecd1edb3fe9c7717c469653b19d156aa8c18e Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Tue, 28 Jul 2026 23:51:55 -0400 Subject: [PATCH 2/3] Complete GLANCE experiment loop --- .gitignore | 1 + ...ON_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md | 138 +- api/data_loader_pg.py | 1066 +++++- api/glance_closed_loop.py | 1528 +++++++++ api/glance_ingestion.py | 215 +- api/metadata_pg.py | 480 ++- api/ml_engine.py | 6 +- api/model_registry.py | 147 + api/routers/dataset_v2.py | 393 ++- api/scripts/add_equipment_runs.sql | 22 + api/scripts/add_experiment_recipe_batches.sql | 26 + api/scripts/add_glance_closed_loop.sql | 74 + .../add_production_glance_ingestion.sql | 23 + api/scripts/init_db.sql | 34 + api/tests/test_etcher_realtime_smoke.py | 66 +- api/tests/test_glance_closed_loop.py | 3022 +++++++++++++++++ api/tests/test_production_glance_ingestion.py | 531 ++- api/tests/test_retrain.py | 212 ++ api/training/retrain.py | 210 +- azure/.gitignore | 3 + azure/common/glance_connector.py | 76 +- azure/common/glance_identity.py | 27 + azure/process_etcher_data/__init__.py | 61 +- azure/tests/test_glance_connector.py | 70 +- azure/tests/test_legacy_summary_identity.py | 57 + geddes/k8s/05-postgres.yaml | 49 + geddes/k8s/postgres/schema_v2.sql | 49 + web/lib/api-client.ts | 1 + 28 files changed, 8416 insertions(+), 171 deletions(-) create mode 100644 api/glance_closed_loop.py create mode 100644 api/scripts/add_glance_closed_loop.sql create mode 100644 api/tests/test_glance_closed_loop.py create mode 100644 azure/common/glance_identity.py create mode 100644 azure/tests/test_legacy_summary_identity.py diff --git a/.gitignore b/.gitignore index 060a8a3..cae22d6 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ debug_*.py !api/tests/test_glance_db3_trace_import.py !api/tests/test_retrain.py !api/tests/test_production_glance_ingestion.py +!api/tests/test_glance_closed_loop.py !api/tests/test_shadow_evaluation.py # Development files diff --git a/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md b/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md index d074ccc..a6e9a9c 100644 --- a/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md +++ b/PRODUCTION_GLANCE_INGESTION_AND_ML_REQUIREMENTS.md @@ -395,8 +395,9 @@ ordered before pagination. The accepted connector query shape is therefore: - fetch minimal sample identity/timestamp fields; - sort source sample IDs locally; - divide them into disjoint numeric ranges of 25; -- query `/data` through the authoritative `samplerecord.idruns` relationship - and the bounded sample-ID range; +- query `/data` through the authoritative composite + `samplerecord.(idtools, idruns)` relationship and the bounded sample-ID + range; - page with explicit PostgREST `limit` and `offset`; - reconstruct the complete trace in memory without changing raw values. @@ -508,6 +509,9 @@ inactive until the deployment boundary above is approved: suspended resources under `geddes/k8s/glance-mirror/`. - API contract and validation: `api/glance_ingestion.py` and `POST /api/dataset/v2/glance/traces/sync`. +- Experiment reconciliation: `api/glance_closed_loop.py`, linking a physical + `equipment_runs` row to one recipe proposal without duplicating the run in + `etcher_runs`. - Schema migration: `api/scripts/add_production_glance_ingestion.sql`. - Plotting: `web/lib/trace-plot.ts`, Catalog run comparison, and `/data/analysis`. @@ -515,3 +519,133 @@ inactive until the deployment boundary above is approved: - Shadow evidence: `api/scripts/run_ml_shadow_evaluation.py`, comparing RF, GPR/analytic EI, and GPR/Pareto/Monte-Carlo acquisition. The supplied CronJob manifest is suspended and contains placeholder image/project values. + +## 15. Closed-loop experiment and training invariants + +The physical trace, experiment proposal, optimizer, and training registry form +one recoverable loop. The following invariants are required before activation: + +- Production equipment mappings are one-to-one: one GLANCE tool ID cannot map + to multiple DT equipment IDs. +- Every GLANCE lookup and stored identity uses the composite tool/run identity. + The legacy scalar summary path must reject cross-tool run-number collisions + and must not allow an unscoped retry to erase a proven tool identity. +- When a source revision timestamp exists, a strictly older retry is a no-op; + it cannot replace newer scalars, samples, or events. +- Non-finite JSON numbers, out-of-range PostgreSQL IDs, duplicate parameter + keys/source IDs, duplicate sample IDs/indices, and duplicate event IDs are + rejected at the per-run API boundary. Valid sibling runs may commit, but a + partial batch never advances the connector cursor. +- Full-trace inputs are authoritative. Proven summary/profilometry outcomes may + be merged into the incoming trace row, but a retry replaces every other + scalar so removed features do not remain sticky. +- Tagged modern `glance_summary` rows are excluded from training until their + exact physical trace exists. The trace-backed equipment row then owns the + single training observation. +- Explicit proposal/request identity may repair an earlier approximate link. + The physical run is detached from the wrong proposal and linked to the exact + proposal without changing a human rejection's status or comment. +- A proposal batch advances only after every latest-batch proposal is terminal + and each completed result is either scientifically eligible or explicitly + excluded. A final human recipe rejection writes a durable + `recipe_rejected_followup_pending` marker in the same transaction; immediate + and scheduled drains are idempotent. +- First-time eligible results receive one durable retrain credit keyed by tool, + run, and scientific revision. Corrections, eligibility removal, and restored + eligibility force an immediate snapshot refresh without a second run credit. +- A correction refresh force-promotes models trained on the corrected snapshot. + If the authoritative frame or an individual target falls below ten usable + rows, affected active models are deactivated rather than serving excluded + lineage. +- PostgreSQL is the authoritative production training frame even when it has + zero rows. Database read failures fail and retry; CSV fallback is allowed + only through an explicit offline setting and is never allowed for a forced + correction refresh. +- If correction retraining is disabled or fails, the durable follow-up claim + and source cursor remain incomplete so the stale-model condition cannot be + silently acknowledged. + +## 15. Experiment closed-loop handoff + +The production trace path closes an experiment only through an explicit or +uniquely verified identity: + +- Prefer a proposal UUID supplied by an approved source integration. +- Otherwise use the experiment `REQ-XXXXXXXXXXXX` execution request ID. The + connector recognizes it only when it is an exact lot-name or recipe-name + value; substring and fuzzy request-ID inference are prohibited. +- When one request has multiple active proposals, prefer the single proposal + marked `accepted` or `attempted`. Otherwise require one unique parameter + match within configured engineering tolerances. +- Ambiguous matches remain unlinked. A supplied identity that cannot be + reconciled prevents cursor acknowledgement so a corrected retry can recover + it. + +One proposal may reference either a canonical summary row in `etcher_runs` or a +complete trace row in `equipment_runs`. A unique `equipment_run_id` link keeps +the Catalog and trace records attached to the same physical run without copying +the trace into the legacy summary table. + +After a successful trace reconciliation: + +1. mark the recipe proposal `completed` and ingestion `approved`; +2. mark the experiment `data_ingested`; +3. preserve canonical trace-derived feature averages in `inputs_json`; +4. include complete, non-outlier GLANCE rows in the ML training query; +5. generate the next proposal batch and enter the retraining debounce only when + all five canonical inputs and both measured optimization outcomes are + present. + +The system must never infer `AvgEtchRate` or `RangeEtchRate` from equipment +telemetry. A trace without those measurement outcomes remains visible and may +complete its execution record, but it cannot train the model or advance the +optimizer. Until profilometry is available from GLANCE itself, the existing +summary handoff may supply those measured outcomes. The summary is merged into +the already stored `equipment_runs` row by the authoritative GLANCE +`(source_tool_id, source_run_id)` identity; a later trace retry must preserve, +not erase, those values. A historical summary without tool identity may bridge +only when exactly one trace candidate exists. That successful proof stamps the +resolved tool provenance so later Catalog and detail reads can deduplicate by +an exact identity. Ambiguous unscoped summaries are deferred. + +Summary telemetry inputs fill missing fields only: validated full-trace inputs +and sample-derived feature averages remain canonical. Measured summary outcomes +may replace missing or corrected outcomes. The Azure summary handoff identifies +itself as `glance_summary`, so a summary-first arrival is stored but does not +use the approximate legacy proposal matcher while it waits for the +authoritative trace identity. Both arrival orders take a compatibility +run-level lock plus the exact tool/run lock and normalize the canonical summary +row to the trace envelope's project and execution request. The summary marker, +trace endpoint, and API reconciliation changes must be deployed as one +coordinated cutover; enabling the marker alone would defer the old matcher +without activating its replacement. + +A stable revision hash and persisted follow-up claim make retries durable. The +ingestion cursor is acknowledged only after the exact experiment's optimizer +and retraining follow-ups finish. A failed follow-up releases its claim and +leaves the cursor unchanged for retry; an abandoned claim expires after a +bounded lease. Replaying an unchanged completed revision may confirm the +proposal link but cannot generate proposals or increment the retraining counter +again. Claim UUIDs fence expired workers, and retraining credits are keyed by +source tool, source run, and canonical-data revision so the summary and trace +paths cannot double-count the same measurement or collide across tools. + +Proposal generation is serialized per project and every experiment iteration +is unique. Follow-ups target the experiment that owns the matched proposal, +never whichever experiment happens to be newest in that project. Rows handled +by this exact trace loop are excluded from the legacy summary reconciliation +and follow-up path so one physical result cannot complete two proposals or be +counted twice. A new optimizer batch is not generated merely because every +proposal is terminal: every completed, non-excluded proposal in the latest +batch must also have all five canonical inputs and both measured outcomes. +This prevents the first late summary in a multi-run batch from advancing the +optimizer before the remaining results arrive. + +Outlier, calibration, and corrected-measurement changes are revisions too. If a +previously trained row becomes ineligible, the system must persist the new +quality flags and force a snapshot-based model refresh so the active model does +not retain excluded data. Automatic proposal generation is a trusted +maintenance operation: it reads the authoritative project plus global Postgres +history, disables the CSV fallback, fits against the current snapshot instead +of reusing a stale active model, and fails the durable follow-up when no +proposal batch can be produced. diff --git a/api/data_loader_pg.py b/api/data_loader_pg.py index d287037..b4579c6 100644 --- a/api/data_loader_pg.py +++ b/api/data_loader_pg.py @@ -2,6 +2,7 @@ import logging import json import hashlib +import math from datetime import datetime, timezone from typing import List, Dict, Any, Optional import re @@ -106,6 +107,54 @@ def _json_safe_payload(row: Dict[str, Any]) -> Dict[str, Any]: return json.loads(json.dumps(row, default=str)) +def _finite_number_or_none(value: Any) -> float | None: + try: + numeric = float(value) + except (TypeError, ValueError): + return None + return numeric if math.isfinite(numeric) else None + + +def equipment_training_revision_sha256( + *, + equipment_run_id: int, + inputs: dict[str, Any], + outputs: dict[str, Any], + is_outlier: bool, + is_calibration_recipe: bool, +) -> str: + """Hash only persisted fields that can change ML training behavior.""" + return hashlib.sha256( + json.dumps( + { + "equipment_run_id": int(equipment_run_id), + "inputs": inputs, + "outputs": outputs, + "is_outlier": bool(is_outlier), + "is_calibration_recipe": bool( + is_calibration_recipe + ), + }, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + ).hexdigest() + + +def glance_source_run_lock_key( + source_run_id: int, + source_tool_id: int | None = None, +) -> str: + """Return a cross-pipeline lock for a GLANCE run or exact tool/run pair.""" + if source_tool_id is None: + return f"glance_source_run|{int(source_run_id)}" + return ( + f"glance_source_run|{int(source_tool_id)}|" + f"{int(source_run_id)}" + ) + + def _run_base_select(*, include_raw_payload: bool = False) -> str: raw_payload_select = ",\n r.raw_payload_json" if include_raw_payload else "" return f""" @@ -126,6 +175,10 @@ def _run_base_select(*, include_raw_payload: bool = False) -> str: r.etch_avg_rf2_pow, r.etch_avgpres, r.etch_avgcf4flow, + COALESCE( + NULLIF(r.raw_payload_json->>'source_tool_id', ''), + NULLIF(r.raw_payload_json->>'idtools', '') + ) AS source_tool_id, r.project_id, r.execution_request_id, p.name AS project_name, @@ -168,6 +221,14 @@ def _shape_run_record(row) -> Dict[str, Any]: # these keys in the shared response shape lets clients identify trace-backed # generic runs without special-casing an absent property. rec.setdefault("source_run_id", None) + source_tool_id = rec.get("source_tool_id") + if source_tool_id not in (None, ""): + try: + rec["source_tool_id"] = int(source_tool_id) + except (TypeError, ValueError): + rec["source_tool_id"] = None + else: + rec["source_tool_id"] = None rec.setdefault("trace_sample_count", 0) return rec @@ -357,6 +418,13 @@ def ensure_equipment_runs_pg() -> None: source_system VARCHAR(100) NOT NULL DEFAULT 'data_upload', source_tool_id BIGINT, source_run_id BIGINT, + execution_request_id VARCHAR(80) NOT NULL DEFAULT '', + closed_loop_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_claim_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_claim_kind VARCHAR(32) NOT NULL DEFAULT '', + closed_loop_followup_claim_token VARCHAR(36) NOT NULL DEFAULT '', + closed_loop_followup_claimed_at TIMESTAMP WITH TIME ZONE, source_updated_at TIMESTAMP WITH TIME ZONE, ingested_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP @@ -376,6 +444,38 @@ def ensure_equipment_runs_pg() -> None: cur.execute( "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_run_id BIGINT" ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "execution_request_id VARCHAR(80) NOT NULL DEFAULT ''" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "closed_loop_revision_sha256 VARCHAR(64) NOT NULL DEFAULT ''" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "closed_loop_followup_revision_sha256 " + "VARCHAR(64) NOT NULL DEFAULT ''" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "closed_loop_followup_claim_revision_sha256 " + "VARCHAR(64) NOT NULL DEFAULT ''" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "closed_loop_followup_claim_kind " + "VARCHAR(32) NOT NULL DEFAULT ''" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "closed_loop_followup_claim_token " + "VARCHAR(36) NOT NULL DEFAULT ''" + ) + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "closed_loop_followup_claimed_at TIMESTAMP WITH TIME ZONE" + ) cur.execute( "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " "source_updated_at TIMESTAMP WITH TIME ZONE" @@ -407,6 +507,11 @@ def ensure_equipment_runs_pg() -> None: cur.execute( "CREATE INDEX IF NOT EXISTS idx_equipment_runs_upload_id ON equipment_runs(upload_id)" ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_equipment_runs_execution_request_id " + "ON equipment_runs(execution_request_id) " + "WHERE execution_request_id <> ''" + ) cur.execute( """ CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row @@ -761,6 +866,7 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: r.source_run_id::text, NULLIF(r.raw_payload_json #>> '{{glance,run_id}}', '') ) AS source_run_id, + r.source_tool_id, COALESCE( NULLIF(r.raw_payload_json #>> '{{trace,process_outcome}}', ''), NULLIF(r.raw_payload_json #>> '{{trace,process_status}}', ''), @@ -770,7 +876,7 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: r.raw_payload_json #>> '{{glance,run_start_time}}', '' ) AS source_run_start_time, COALESCE(trace_counts.sample_count, 0)::int AS trace_sample_count, - ''::text AS execution_request_id, + r.execution_request_id, p.name AS project_name, r.equipment_id AS equipment_id, COALESCE(em.equipment_name, p.equipment_name, r.equipment_id) AS equipment_name, @@ -810,11 +916,17 @@ def _shape_equipment_run_record(row) -> Dict[str, Any]: # etcher rows use, and surface measured outputs alongside. rec["features"] = rec.pop("inputs_json", {}) or {} rec["outputs"] = rec.pop("outputs_json", {}) or {} - # The catalog renders etch-rate badges only when these are numeric; generic - # equipment has no canonical etch-rate, so leave them absent. - rec.setdefault("avg_etch_rate", None) - rec.setdefault("range_etch_rate", None) - rec.setdefault("range_nm", None) + # A generic run is not assumed to be an etcher run. When an approved + # GLANCE handoff does carry the canonical etcher outcomes, surface them + # without inventing values for other equipment. + avg_etch_rate = rec["outputs"].get("AvgEtchRate") + range_etch_rate = rec["outputs"].get("RangeEtchRate") + range_nm = rec["outputs"].get("Range_nm") + rec["avg_etch_rate"] = _finite_number_or_none(avg_etch_rate) + rec["range_etch_rate"] = _finite_number_or_none(range_etch_rate) + rec["range_nm"] = _finite_number_or_none(range_nm) + if rec["range_nm"] is None and rec["range_etch_rate"] is not None: + rec["range_nm"] = rec["range_etch_rate"] * 5.0 source_run_id = rec.get("source_run_id") if source_run_id not in (None, ""): try: @@ -834,6 +946,42 @@ def _shape_equipment_run_record(row) -> Dict[str, Any]: return rec +def _merge_canonical_summary_into_trace_record( + trace_record: Dict[str, Any], + summary_record: Optional[Dict[str, Any]], +) -> Dict[str, Any]: + """Present dual-ingested GLANCE trace/summary data as one catalog run.""" + if not summary_record: + return trace_record + + summary_features = summary_record.get("features") or {} + trace_features = trace_record.get("features") or {} + trace_record["features"] = { + **summary_features, + **trace_features, + } + outputs = dict(trace_record.get("outputs") or {}) + for key, field in ( + ("AvgEtchRate", "avg_etch_rate"), + ("RangeEtchRate", "range_etch_rate"), + ("Range_nm", "range_nm"), + ): + value = summary_record.get(field) + if outputs.get(key) is None and value is not None: + outputs[key] = value + if trace_record.get(field) is None and value is not None: + trace_record[field] = value + trace_record["outputs"] = outputs + if not trace_record.get("execution_request_id"): + trace_record["execution_request_id"] = summary_record.get( + "execution_request_id", + "", + ) + if not trace_record.get("file_refs"): + trace_record["file_refs"] = summary_record.get("file_refs", []) + return trace_record + + def sync_equipment_runs_pg( *, equipment_id: str, @@ -1336,6 +1484,10 @@ def sync_equipment_run_traces_pg( normalized_runs.append( { "source_run_id": source_run_id, + "execution_request_id": str( + run.get("execution_request_id") or "" + ).strip(), + "proposal_id": str(run.get("proposal_id") or "").strip(), "lotname": str(run.get("lotname", run.get("lot_name", "")) or ""), "run_date": run.get("run_date", run.get("run_start_time")) or None, "is_outlier": bool(run.get("is_outlier", False)), @@ -1371,12 +1523,47 @@ def sync_equipment_run_traces_pg( "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (identity_lock,), ) + if live_source: + for locked_source_run_id in sorted( + run["source_run_id"] + for run in normalized_runs + ): + # The run-only lock coordinates old summary payloads that + # cannot yet prove a tool. The exact lock prevents work + # for the same physical tool/run from crossing. + cur.execute( + """ + SELECT pg_advisory_xact_lock( + hashtextextended(%s, 0) + ) + """, + ( + glance_source_run_lock_key( + locked_source_run_id + ), + ), + ) + cur.execute( + """ + SELECT pg_advisory_xact_lock( + hashtextextended(%s, 0) + ) + """, + ( + glance_source_run_lock_key( + locked_source_run_id, + batch_source_tool_id, + ), + ), + ) existing_source_run_ids: set[int] = set() + existing_parent_by_source_run: dict[int, int] = {} + stale_source_run_ids: set[int] = set() parent_values = [] if live_source: cur.execute( """ - SELECT source_run_id + SELECT id, source_run_id, source_updated_at FROM equipment_runs WHERE source_system = %s AND equipment_id = %s @@ -1390,10 +1577,237 @@ def sync_equipment_run_traces_pg( [run["source_run_id"] for run in normalized_runs], ), ) + existing_rows = list(cur.fetchall() or []) existing_source_run_ids = { - int(row[0]) for row in (cur.fetchall() or []) + int(row[1]) for row in existing_rows + } + existing_parent_by_source_run = { + int(row[1]): int(row[0]) for row in existing_rows + } + persisted_revision_by_source_run = { + int(row[1]): row[2] for row in existing_rows } for run in normalized_runs: + incoming_revision = run.get("source_updated_at") + persisted_revision = persisted_revision_by_source_run.get( + run["source_run_id"] + ) + if ( + incoming_revision is not None + and persisted_revision is not None + and incoming_revision < persisted_revision + ): + stale_source_run_ids.add(run["source_run_id"]) + applied_runs = [ + run + for run in normalized_runs + if run["source_run_id"] not in stale_source_run_ids + ] + source_run_ids = [ + run["source_run_id"] + for run in applied_runs + ] + summary_by_source_run: dict[int, tuple] = {} + ambiguous_unscoped_summary_ids: set[int] = set() + if source_run_ids: + cur.execute( + """ + SELECT + idruns, + execution_request_id, + etch_avgo2flow, + etch_avg_rf1_pow, + etch_avg_rf2_pow, + etch_avgpres, + etch_avgcf4flow, + avg_etch_rate, + range_etch_rate, + range_nm, + is_outlier, + is_calibration_recipe, + raw_payload_json + FROM etcher_runs + WHERE idruns = ANY(%s) + """, + (source_run_ids,), + ) + summary_by_source_run = { + int(row[0]): row + for row in (cur.fetchall() or []) + } + cur.execute( + """ + SELECT DISTINCT source_run_id + FROM equipment_runs + WHERE source_system = 'glance' + AND source_run_id = ANY(%s) + AND NOT ( + equipment_id = %s + AND source_tool_id = %s + ) + """, + ( + source_run_ids, + equipment_id, + batch_source_tool_id, + ), + ) + ambiguous_unscoped_summary_ids = { + int(row[0]) for row in (cur.fetchall() or []) + } + for run in applied_runs: + summary = summary_by_source_run.get( + run["source_run_id"] + ) + summary_raw = ( + dict(summary[12] or {}) + if summary + and isinstance(summary[12], dict) + else {} + ) + summary_tool_value = ( + summary_raw.get("source_tool_id") + if summary_raw.get("source_tool_id") + not in (None, "") + else summary_raw.get("idtools") + ) + try: + summary_tool_id = ( + int(summary_tool_value) + if summary_tool_value not in (None, "") + else None + ) + except (TypeError, ValueError): + summary_tool_id = None + summary_is_exact = ( + summary_tool_id == batch_source_tool_id + ) + summary_is_uniquely_unscoped = ( + summary_tool_id is None + and run["source_run_id"] + not in ambiguous_unscoped_summary_ids + ) + use_summary = bool( + summary + and ( + summary_is_exact + or summary_is_uniquely_unscoped + ) + ) + if use_summary: + summary_inputs = { + key: value + for key, value in ( + ("Etch_AvgO2Flow", summary[2]), + ("Etch_Avg_Rf1_Pow", summary[3]), + ("Etch_Avg_Rf2_Pow", summary[4]), + ("Etch_AvgPres", summary[5]), + ("Etch_Avgcf4Flow", summary[6]), + ) + if value is not None + } + summary_outputs = { + key: value + for key, value in ( + ("AvgEtchRate", summary[7]), + ("RangeEtchRate", summary[8]), + ("Range_nm", summary[9]), + ) + if value is not None + } + run["inputs"] = { + **summary_inputs, + **run["inputs"], + } + run["outputs"] = { + **run["outputs"], + **summary_outputs, + } + run["is_outlier"] = bool(summary[10]) + run["is_calibration_recipe"] = bool( + summary[11] + ) + if not run["execution_request_id"]: + run["execution_request_id"] = str( + summary[1] or "" + ).strip() + run["raw"].setdefault( + "glance", + {}, + )["summary_backfilled"] = True + elif summary: + run["raw"].setdefault( + "glance", + {}, + )["summary_backfill_deferred"] = ( + "tool_identity_mismatch_or_ambiguous" + ) + + # The full-trace mapping is authoritative, but only update + # the legacy row when its tool is exact or the old payload + # is provably unambiguous. + if use_summary: + cur.execute( + """ + UPDATE etcher_runs + SET + project_id = %s, + raw_payload_json = jsonb_set( + COALESCE( + raw_payload_json, + '{}'::jsonb + ), + '{source_tool_id}', + to_jsonb(%s::bigint), + true + ) + WHERE idruns = %s + AND ( + project_id IS DISTINCT FROM %s + OR NULLIF( + raw_payload_json + ->>'source_tool_id', + '' + ) IS DISTINCT FROM %s + ) + """, + ( + project_id, + batch_source_tool_id, + run["source_run_id"], + project_id, + str(batch_source_tool_id), + ), + ) + if run["execution_request_id"]: + cur.execute( + """ + UPDATE etcher_runs + SET execution_request_id = %s + WHERE idruns = %s + AND execution_request_id IS DISTINCT FROM %s + AND ( + NULLIF( + raw_payload_json->>'source_tool_id', + '' + ) = %s + OR ( + NULLIF( + raw_payload_json->>'source_tool_id', + '' + ) IS NULL + AND %s + ) + ) + """, + ( + run["execution_request_id"], + run["source_run_id"], + run["execution_request_id"], + str(batch_source_tool_id), + summary_is_uniquely_unscoped, + ), + ) parent_values.append( ( equipment_id, @@ -1410,40 +1824,52 @@ def sync_equipment_run_traces_pg( source_system, batch_source_tool_id, run["source_run_id"], + run["execution_request_id"], run["source_updated_at"], ) ) - returned_parents = execute_values( - cur, - """ - INSERT INTO equipment_runs ( - equipment_id, project_id, lotname, run_date, - is_outlier, is_calibration_recipe, outlier_type, - inputs_json, outputs_json, raw_payload_json, - source, source_system, source_tool_id, source_run_id, - source_updated_at - ) VALUES %s - ON CONFLICT ( - source_system, equipment_id, source_tool_id, source_run_id + returned_parents = [] + if parent_values: + returned_parents = execute_values( + cur, + """ + INSERT INTO equipment_runs ( + equipment_id, project_id, lotname, run_date, + is_outlier, is_calibration_recipe, outlier_type, + inputs_json, outputs_json, raw_payload_json, + source, source_system, source_tool_id, source_run_id, + execution_request_id, source_updated_at + ) VALUES %s + ON CONFLICT ( + source_system, equipment_id, source_tool_id, source_run_id + ) + WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL + DO UPDATE SET + project_id = EXCLUDED.project_id, + lotname = EXCLUDED.lotname, + run_date = EXCLUDED.run_date, + is_outlier = EXCLUDED.is_outlier, + is_calibration_recipe = EXCLUDED.is_calibration_recipe, + outlier_type = EXCLUDED.outlier_type, + -- A live handoff is a complete replacement for the + -- trace-derived scalar view. Proven legacy summary + -- outcomes were merged into EXCLUDED above; retaining + -- any other old key would make removed/corrected + -- features permanently sticky. + inputs_json = EXCLUDED.inputs_json, + outputs_json = EXCLUDED.outputs_json, + raw_payload_json = EXCLUDED.raw_payload_json, + execution_request_id = COALESCE( + NULLIF(EXCLUDED.execution_request_id, ''), + equipment_runs.execution_request_id + ), + source_updated_at = EXCLUDED.source_updated_at, + ingested_at = CURRENT_TIMESTAMP + RETURNING id, source_run_id + """, + parent_values, + fetch=True, ) - WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL - DO UPDATE SET - project_id = EXCLUDED.project_id, - lotname = EXCLUDED.lotname, - run_date = EXCLUDED.run_date, - is_outlier = EXCLUDED.is_outlier, - is_calibration_recipe = EXCLUDED.is_calibration_recipe, - outlier_type = EXCLUDED.outlier_type, - inputs_json = EXCLUDED.inputs_json, - outputs_json = EXCLUDED.outputs_json, - raw_payload_json = EXCLUDED.raw_payload_json, - source_updated_at = EXCLUDED.source_updated_at, - ingested_at = CURRENT_TIMESTAMP - RETURNING id, source_run_id - """, - parent_values, - fetch=True, - ) else: for run in normalized_runs: parent_values.append( @@ -1500,17 +1926,67 @@ def sync_equipment_run_traces_pg( int(source_run_id): int(parent_id) for parent_id, source_run_id in returned_parents } + parent_id_by_source_run.update( + { + source_run_id: existing_parent_by_source_run[ + source_run_id + ] + for source_run_id in stale_source_run_ids + } + ) if len(parent_id_by_source_run) != len(normalized_runs): raise RuntimeError("Could not resolve every GLANCE catalog run after upsert") parent_ids = list(parent_id_by_source_run.values()) - cur.execute( - "DELETE FROM equipment_run_trace_samples " - "WHERE equipment_run_id = ANY(%s)", - (parent_ids,), - ) + persisted_payload_by_parent = { + parent_id_by_source_run[run["source_run_id"]]: { + "inputs": run["inputs"], + "outputs": run["outputs"], + } + for run in ( + applied_runs if live_source else normalized_runs + ) + } + if live_source and persisted_payload_by_parent: + # Read the authoritative replacement back so reconciliation + # sees exactly what was persisted. Summary/profilometry + # outcomes were merged into the incoming row before upsert; + # trace inputs and every other scalar are replace-on-retry. + cur.execute( + """ + SELECT id, inputs_json, outputs_json + FROM equipment_runs + WHERE id = ANY(%s) + """, + (parent_ids,), + ) + persisted_payload_by_parent.update( + { + int(parent_id): { + "inputs": dict(inputs_json or {}), + "outputs": dict(outputs_json or {}), + } + for parent_id, inputs_json, outputs_json in ( + cur.fetchall() or [] + ) + } + ) + replaced_parent_ids = [ + parent_id_by_source_run[run["source_run_id"]] + for run in ( + applied_runs if live_source else normalized_runs + ) + ] + if replaced_parent_ids: + cur.execute( + "DELETE FROM equipment_run_trace_samples " + "WHERE equipment_run_id = ANY(%s)", + (replaced_parent_ids,), + ) trace_values = [] - for run in normalized_runs: + for run in ( + applied_runs if live_source else normalized_runs + ): parent_id = parent_id_by_source_run[run["source_run_id"]] for sample in run["samples"]: trace_values.append( @@ -1524,24 +2000,28 @@ def sync_equipment_run_traces_pg( Json(sample["values"]), ) ) - execute_values( - cur, - """ - INSERT INTO equipment_run_trace_samples ( - equipment_run_id, sample_index, source_sample_id, source_tool_id, - sample_time, sample_time_raw, values_json - ) VALUES %s - """, - trace_values, - page_size=500, - ) - cur.execute( - "DELETE FROM equipment_run_trace_events " - "WHERE equipment_run_id = ANY(%s)", - (parent_ids,), - ) + if trace_values: + execute_values( + cur, + """ + INSERT INTO equipment_run_trace_samples ( + equipment_run_id, sample_index, source_sample_id, source_tool_id, + sample_time, sample_time_raw, values_json + ) VALUES %s + """, + trace_values, + page_size=500, + ) + if replaced_parent_ids: + cur.execute( + "DELETE FROM equipment_run_trace_events " + "WHERE equipment_run_id = ANY(%s)", + (replaced_parent_ids,), + ) event_values = [] - for run in normalized_runs: + for run in ( + applied_runs if live_source else normalized_runs + ): parent_id = parent_id_by_source_run[run["source_run_id"]] for event in run["events"]: event_values.append( @@ -1585,14 +2065,72 @@ def sync_equipment_run_traces_pg( if not return_stats: return len(normalized_runs) inserted = len(normalized_runs) - len(existing_source_run_ids) + result_runs = ( + applied_runs if live_source else normalized_runs + ) return { "inserted": inserted, - "updated": len(existing_source_run_ids), + "updated": len(existing_source_run_ids - stale_source_run_ids), + "unchanged": len(stale_source_run_ids), "run_count": len(normalized_runs), "sample_count": sum( - len(run["samples"]) for run in normalized_runs + len(run["samples"]) for run in result_runs ), - "event_count": sum(len(run["events"]) for run in normalized_runs), + "event_count": sum(len(run["events"]) for run in result_runs), + "run_records": [ + { + "equipment_run_id": parent_id_by_source_run[ + run["source_run_id"] + ], + "source_tool_id": batch_source_tool_id, + "source_run_id": run["source_run_id"], + "execution_request_id": run["execution_request_id"], + "proposal_id": run["proposal_id"], + "lotname": run["lotname"], + "is_outlier": run["is_outlier"], + "is_calibration_recipe": run[ + "is_calibration_recipe" + ], + "inputs": persisted_payload_by_parent[ + parent_id_by_source_run[run["source_run_id"]] + ]["inputs"], + "outputs": persisted_payload_by_parent[ + parent_id_by_source_run[run["source_run_id"]] + ]["outputs"], + "parameters": run["raw"].get("trace", {}).get( + "parameters", + [], + ), + "samples": [ + { + "values": sample["values"], + } + for sample in run["samples"] + ], + "closed_loop_revision_sha256": ( + equipment_training_revision_sha256( + equipment_run_id=parent_id_by_source_run[ + run["source_run_id"] + ], + inputs=persisted_payload_by_parent[ + parent_id_by_source_run[ + run["source_run_id"] + ] + ]["inputs"], + outputs=persisted_payload_by_parent[ + parent_id_by_source_run[ + run["source_run_id"] + ] + ]["outputs"], + is_outlier=run["is_outlier"], + is_calibration_recipe=run[ + "is_calibration_recipe" + ], + ) + ), + } + for run in result_runs + ], } except Exception: conn.rollback() @@ -1925,9 +2463,75 @@ def get_runs_list_pg( cur.execute(generic_query, tuple(base_params + [page_cap])) generic_rows = cur.fetchall() - records = [_shape_run_record(row) for row in etcher_rows] - _attach_run_file_refs(conn, records) - records.extend(_shape_equipment_run_record(row) for row in generic_rows) + canonical_records = [_shape_run_record(row) for row in etcher_rows] + _attach_run_file_refs(conn, canonical_records) + matched_canonical_ids: set[int] = set() + generic_records = [ + _shape_equipment_run_record(row) + for row in generic_rows + ] + trace_candidates_by_identity: dict[ + tuple[int, int], + list[Dict[str, Any]], + ] = {} + for trace_record in generic_records: + if not str( + trace_record.get("source") or "" + ).startswith("glance"): + continue + source_run_id = trace_record.get("source_run_id") + source_tool_id = trace_record.get("source_tool_id") + if not isinstance(source_run_id, int) or not isinstance( + source_tool_id, + int, + ): + continue + trace_candidates_by_identity.setdefault( + (source_tool_id, source_run_id), + [], + ).append(trace_record) + + canonical_by_trace_run_id: dict[int, Dict[str, Any]] = {} + for canonical_record in canonical_records: + source_tool_id = canonical_record.get("source_tool_id") + if not isinstance(source_tool_id, int): + # Historical summaries without tool provenance are not safe + # to merge in a paginated catalog query. + continue + identity = ( + source_tool_id, + int(canonical_record["run_id"]), + ) + candidates = trace_candidates_by_identity.get(identity, []) + if len(candidates) == 1: + canonical_by_trace_run_id[ + int(candidates[0]["run_id"]) + ] = canonical_record + + for trace_record in generic_records: + trace_run_id = int(trace_record["run_id"]) + canonical_record = canonical_by_trace_run_id.get( + trace_run_id + ) + if canonical_record: + matched_canonical_ids.add( + int(canonical_record["run_id"]) + ) + _merge_canonical_summary_into_trace_record( + trace_record, + canonical_record, + ) + + # During the transition both the legacy summary path and production + # trace path may receive the same GLANCE idruns. Prefer the trace-backed + # catalog record and merge the summary outcomes into it so one physical + # run is displayed once. + records = [ + record + for record in canonical_records + if int(record["run_id"]) not in matched_canonical_ids + ] + records.extend(generic_records) # Merge the two sources into a single date-descending page. Sort on the # actual instant (not the ISO string) so timestamps with different UTC @@ -1993,6 +2597,30 @@ def get_run_detail_pg( conn.close() return None record = _shape_equipment_run_record(row) + source_run_id = record.get("source_run_id") + if ( + str(record.get("source") or "").startswith("glance") + and isinstance(source_run_id, int) + ): + summary_query = ( + _run_base_select(include_raw_payload=True) + + " WHERE r.idruns = %s LIMIT 1" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute(summary_query, (source_run_id,)) + summary_row = cur.fetchone() + if summary_row: + summary_record = _shape_run_record(summary_row) + if ( + isinstance(record.get("source_tool_id"), int) + and summary_record.get("source_tool_id") + == record.get("source_tool_id") + ): + _attach_run_file_refs(conn, [summary_record]) + _merge_canonical_summary_into_trace_record( + record, + summary_record, + ) conn.commit() conn.close() return record @@ -2774,7 +3402,6 @@ def sync_runs_pg(runs_data: List[Dict[str, Any]]): return 0 try: - conn = get_pg_superuser_connection() etcher_config = load_domain_config("etcher") or {} etcher_domain = etcher_config.get("domain", {}) canonical_equipment_id = etcher_domain.get("id", "etcher") @@ -2791,6 +3418,7 @@ def sync_runs_pg(runs_data: List[Dict[str, Any]]): file_ref_rows = [] execution_request_ids: set[str] = set() blob_metadata_cache: dict[str, dict[str, Any]] = {} + incoming_tool_by_run: dict[int, int | None] = {} import re @@ -2828,6 +3456,41 @@ def sync_runs_pg(runs_data: List[Dict[str, Any]]): projects_cache[project_id] = lotname run_id = int(idr) + source_tool_value = row.get( + "source_tool_id", + row.get("idtools"), + ) + try: + incoming_tool_id = ( + int(source_tool_value) + if source_tool_value not in (None, "") + else None + ) + except (TypeError, ValueError) as exc: + raise ValueError( + f"Summary run {run_id} has invalid source tool id " + f"{source_tool_value!r}" + ) from exc + if incoming_tool_id is not None and not ( + 1 <= incoming_tool_id <= 9_223_372_036_854_775_807 + ): + raise ValueError( + f"Summary run {run_id} source tool id is outside " + "the PostgreSQL BIGINT range" + ) + if ( + str(row.get("source_system") or "") == "glance_summary" + and incoming_tool_id is None + ): + raise ValueError( + f"Tagged GLANCE summary run {run_id} requires " + "source_tool_id" + ) + if run_id in incoming_tool_by_run: + raise ValueError( + f"Summary batch contains duplicate run id {run_id}" + ) + incoming_tool_by_run[run_id] = incoming_tool_id execution_request_id = _execution_request_id_from_payload(row) if execution_request_id: execution_request_ids.add(execution_request_id) @@ -2861,7 +3524,65 @@ def sync_runs_pg(runs_data: List[Dict[str, Any]]): if file_ref is not None: file_ref_rows.append(file_ref) + conn = get_pg_superuser_connection() with conn.cursor() as cur: + # The legacy scalar table has only one row per run number. Guard + # that limitation under the global run lock shared with full-trace + # ingestion: exact tool A can never be overwritten by tool B. + for run_id in sorted(incoming_tool_by_run): + cur.execute( + """ + SELECT pg_advisory_xact_lock( + hashtextextended(%s, 0) + ) + """, + (glance_source_run_lock_key(run_id),), + ) + cur.execute( + """ + SELECT COALESCE( + NULLIF( + raw_payload_json->>'source_tool_id', + '' + ), + NULLIF(raw_payload_json->>'idtools', '') + ) + FROM etcher_runs + WHERE idruns = %s + FOR UPDATE + """, + (run_id,), + ) + existing_row = cur.fetchone() + existing_tool_value = ( + existing_row[0] if existing_row else None + ) + try: + existing_tool_id = ( + int(existing_tool_value) + if existing_tool_value not in (None, "") + else None + ) + except (TypeError, ValueError) as exc: + raise ValueError( + f"Stored summary run {run_id} has invalid source " + f"tool id {existing_tool_value!r}" + ) from exc + incoming_tool_id = incoming_tool_by_run[run_id] + if ( + existing_tool_id is not None + and incoming_tool_id != existing_tool_id + ): + raise ValueError( + f"Summary run {run_id} is already assigned to " + f"GLANCE tool {existing_tool_id}; refusing data " + + ( + f"from tool {incoming_tool_id}" + if incoming_tool_id is not None + else "without an authoritative tool identity" + ) + ) + # Upsert automatic projects for p_id, p_name in projects_cache.items(): cur.execute(""" @@ -2991,7 +3712,165 @@ def get_training_df_pg( return None conn = get_pg_superuser_connection() if is_admin else get_pg_connection(nanohub_user_id) - select_sql = """ + numeric_pattern = r"^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$" + + def json_number(json_column: str, key: str) -> str: + return ( + f"CASE WHEN ({json_column}->>'{key}') ~ " + f"'{numeric_pattern}' THEN " + f"({json_column}->>'{key}')::double precision END" + ) + + generic_avg_etch_rate = json_number( + "e.outputs_json", + "AvgEtchRate", + ) + generic_range_etch_rate = json_number( + "e.outputs_json", + "RangeEtchRate", + ) + generic_range_nm = json_number("e.outputs_json", "Range_nm") + generic_features = { + "Etch_AvgO2Flow": json_number( + "e.inputs_json", + "Etch_AvgO2Flow", + ), + "Etch_Avg_Rf1_Pow": json_number( + "e.inputs_json", + "Etch_Avg_Rf1_Pow", + ), + "Etch_Avg_Rf2_Pow": json_number( + "e.inputs_json", + "Etch_Avg_Rf2_Pow", + ), + "Etch_AvgPres": json_number( + "e.inputs_json", + "Etch_AvgPres", + ), + "Etch_Avgcf4Flow": json_number( + "e.inputs_json", + "Etch_Avgcf4Flow", + ), + } + select_sql = f""" + WITH legacy_training_runs AS ( + SELECT + r.idruns::bigint AS idruns, + r.lotname, + r.run_date, + r.is_outlier, + r.is_calibration_recipe, + r.avg_etch_rate, + r.range_etch_rate, + r.range_nm, + r.etch_avgo2flow, + r.etch_avg_rf1_pow, + r.etch_avg_rf2_pow, + r.etch_avgpres, + r.etch_avgcf4flow, + r.project_id, + ( + SELECT linked_equipment.id + FROM equipment_runs linked_equipment + WHERE linked_equipment.source_system = 'glance' + AND linked_equipment.source_run_id = r.idruns + AND ( + COALESCE( + NULLIF( + r.raw_payload_json + ->>'source_tool_id', + '' + ), + NULLIF( + r.raw_payload_json->>'idtools', + '' + ) + ) = linked_equipment.source_tool_id::text + OR ( + COALESCE( + NULLIF( + r.raw_payload_json + ->>'source_tool_id', + '' + ), + NULLIF( + r.raw_payload_json->>'idtools', + '' + ) + ) IS NULL + AND ( + SELECT COUNT(*) + FROM equipment_runs same_source + WHERE + same_source.source_system = 'glance' + AND same_source.source_run_id = + r.idruns + ) = 1 + ) + ) + ORDER BY linked_equipment.id + LIMIT 1 + ) AS equipment_run_id + FROM etcher_runs r + WHERE + COALESCE( + r.raw_payload_json->>'source_system', + '' + ) <> 'glance_summary' + AND ( + COALESCE( + NULLIF( + r.raw_payload_json->>'source_tool_id', + '' + ), + NULLIF( + r.raw_payload_json->>'idtools', + '' + ) + ) IS NOT NULL + OR ( + SELECT COUNT(*) + FROM equipment_runs same_source + WHERE same_source.source_system = 'glance' + AND same_source.source_run_id = r.idruns + ) <= 1 + ) + + ), + training_runs AS ( + -- Once an exact trace exists it owns the scientific inputs. + -- The summary outcomes have already been merged into that + -- equipment row, so retaining the legacy row would both + -- duplicate the observation and restore summary telemetry + -- over the canonical trace-derived feature values. + SELECT * + FROM legacy_training_runs + WHERE equipment_run_id IS NULL + + UNION ALL + + SELECT + ({EQUIPMENT_RUN_ID_OFFSET} + e.id)::bigint AS idruns, + e.lotname, + e.run_date, + e.is_outlier, + e.is_calibration_recipe, + {generic_avg_etch_rate} AS avg_etch_rate, + {generic_range_etch_rate} AS range_etch_rate, + COALESCE( + {generic_range_nm}, + ({generic_range_etch_rate}) * 5.0 + ) AS range_nm, + {generic_features["Etch_AvgO2Flow"]} AS etch_avgo2flow, + {generic_features["Etch_Avg_Rf1_Pow"]} AS etch_avg_rf1_pow, + {generic_features["Etch_Avg_Rf2_Pow"]} AS etch_avg_rf2_pow, + {generic_features["Etch_AvgPres"]} AS etch_avgpres, + {generic_features["Etch_Avgcf4Flow"]} AS etch_avgcf4flow, + e.project_id, + e.id AS equipment_run_id + FROM equipment_runs e + WHERE e.source_system = 'glance' + ) SELECT r.idruns, r.lotname AS "LOTNAME", @@ -3006,7 +3885,7 @@ def get_training_df_pg( r.etch_avg_rf2_pow AS "Etch_Avg_Rf2_Pow", r.etch_avgpres AS "Etch_AvgPres", r.etch_avgcf4flow AS "Etch_Avgcf4Flow" - FROM etcher_runs r + FROM training_runs r """ query_params: list[Any] = [] @@ -3028,12 +3907,19 @@ def get_training_df_pg( ): cutoff_params: list[Any] = [experiment_id, int(max_iteration)] cutoff_query = """ - SELECT MAX(r2.run_date) AS cutoff_run_date + SELECT MAX( + COALESCE(r2.run_date, equipment_run.run_date) + ) AS cutoff_run_date FROM experiment_recipe_proposals rp JOIN experiment_recipe_batches b ON b.id = rp.batch_id - JOIN etcher_runs r2 ON r2.idruns = rp.run_id + LEFT JOIN etcher_runs r2 ON r2.idruns = rp.run_id + LEFT JOIN equipment_runs equipment_run + ON equipment_run.id = rp.equipment_run_id WHERE rp.experiment_id = %s - AND rp.run_id IS NOT NULL + AND ( + rp.run_id IS NOT NULL + OR rp.equipment_run_id IS NOT NULL + ) AND LOWER(rp.status) = 'completed' AND b.iteration <= %s """ @@ -3045,17 +3931,29 @@ def get_training_df_pg( elif experiment_id and not project_id: proposal_conditions = [ "rp.experiment_id = %s", - "rp.run_id IS NOT NULL", + ( + "(rp.run_id IS NOT NULL " + "OR rp.equipment_run_id IS NOT NULL)" + ), "LOWER(rp.status) = 'completed'", ] - query_params.append(experiment_id) + proposal_params: list[Any] = [experiment_id] if apply_iteration_cutoff and max_iteration is not None: proposal_conditions.append("b.iteration <= %s") - query_params.append(int(max_iteration)) + proposal_params.append(int(max_iteration)) select_sql += """ - JOIN experiment_recipe_proposals rp ON rp.run_id = r.idruns + JOIN experiment_recipe_proposals rp + ON ( + rp.run_id = r.idruns + OR ( + rp.equipment_run_id IS NOT NULL + AND rp.equipment_run_id = + r.equipment_run_id + ) + ) JOIN experiment_recipe_batches b ON b.id = rp.batch_id """ + query_params.extend(proposal_params) context_conditions.extend(proposal_conditions) where_sql = f" WHERE {' AND '.join(context_conditions)}" if context_conditions else "" @@ -3066,12 +3964,20 @@ def get_training_df_pg( if df.empty: logger.warning("Postgres returned 0 rows for training data.") - return None - + # An empty successful query is authoritative scientific state, + # especially after exclusions/corrections remove the final row. + # Reserve None for an unavailable/failed database read so the + # retrainer can deactivate stale models instead of confusing zero + # rows with a fallback condition. + return df.copy() + + df = df.copy() # Ensure boolean columns are actual booleans for col in ['is_outlier', 'is_calibration_recipe']: if col in df.columns: - df[col] = df[col].fillna(False).astype(bool) + df = df.assign( + **{col: df[col].fillna(False).astype(bool)} + ) if project_id: logger.info( diff --git a/api/glance_closed_loop.py b/api/glance_closed_loop.py new file mode 100644 index 0000000..81b8b80 --- /dev/null +++ b/api/glance_closed_loop.py @@ -0,0 +1,1528 @@ +"""Closed-loop reconciliation for complete GLANCE equipment traces. + +The production GLANCE connector stores complete traces in ``equipment_runs``. +This module links those physical runs back to experiment recipe proposals and +starts the next optimizer/retraining work only when the run contains a complete +ML training row. Missing outcomes are never inferred. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import math +import os +import uuid +from datetime import datetime, timedelta, timezone +from typing import Any + +from psycopg2.extras import Json, RealDictCursor + +from data_loader import FEATURES, PRIMARY_TARGET, SECONDARY_TARGET +from data_loader_pg import ( + equipment_training_revision_sha256, + get_pg_superuser_connection, + glance_source_run_lock_key, +) + +logger = logging.getLogger("dt.api.glance_closed_loop") + +_ACTIVE_PROPOSAL_STATUSES = {"pending", "accepted", "attempted"} +_PREFERRED_PROPOSAL_STATUSES = {"accepted", "attempted"} + + +def _finite_float(value: Any) -> float | None: + try: + numeric = float(value) + except (TypeError, ValueError): + return None + return numeric if math.isfinite(numeric) else None + + +def derive_trace_feature_values(run_record: dict[str, Any]) -> dict[str, float]: + """Return explicit inputs plus numeric trace means under known aliases.""" + features: dict[str, float] = {} + for name, value in (run_record.get("inputs") or {}).items(): + numeric = _finite_float(value) + if numeric is not None: + features[str(name)] = numeric + + aliases_by_key: dict[str, set[str]] = {} + for parameter in run_record.get("parameters") or []: + if not isinstance(parameter, dict): + continue + key = str(parameter.get("key") or "").strip() + if not key: + continue + aliases = aliases_by_key.setdefault(key, {key}) + for field in ("name", "source_name", "registered_name"): + alias = str(parameter.get(field) or "").strip() + if alias: + aliases.add(alias) + + values_by_key: dict[str, list[float]] = {} + for sample in run_record.get("samples") or []: + values = sample.get("values") if isinstance(sample, dict) else None + if not isinstance(values, dict): + continue + for key, value in values.items(): + numeric = _finite_float(value) + if numeric is not None: + values_by_key.setdefault(str(key), []).append(numeric) + + for key, values in values_by_key.items(): + mean = sum(values) / len(values) + for alias in aliases_by_key.get(key, {key}): + features.setdefault(alias, mean) + return features + + +def training_values_for_run( + run_record: dict[str, Any], +) -> tuple[dict[str, float], dict[str, float], bool]: + """Return canonical ML inputs/outputs and whether the row is complete.""" + derived = derive_trace_feature_values(run_record) + inputs = { + feature: derived[feature] + for feature in FEATURES + if feature in derived + } + outputs: dict[str, float] = {} + for target in (PRIMARY_TARGET, SECONDARY_TARGET): + numeric = _finite_float((run_record.get("outputs") or {}).get(target)) + if numeric is not None: + outputs[target] = numeric + eligible = ( + len(inputs) == len(FEATURES) + and len(outputs) == 2 + and not bool(run_record.get("is_outlier")) + and not bool(run_record.get("is_calibration_recipe")) + ) + return inputs, outputs, eligible + + +def retrain_credit_key_for_run( + run_record: dict[str, Any], +) -> str | None: + """Return one stable retrain-credit key for a complete GLANCE result.""" + inputs, outputs, eligible = training_values_for_run(run_record) + if not eligible: + return None + try: + source_tool_id = int(run_record["source_tool_id"]) + source_run_id = int(run_record["source_run_id"]) + except (KeyError, TypeError, ValueError): + return None + revision = hashlib.sha256( + json.dumps( + { + "source_system": "glance", + "source_tool_id": source_tool_id, + "source_run_id": source_run_id, + "inputs": inputs, + "outputs": outputs, + "is_outlier": bool(run_record.get("is_outlier")), + "is_calibration_recipe": bool( + run_record.get("is_calibration_recipe") + ), + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + return ( + f"glance-source:{source_tool_id}:{source_run_id}:" + f"{revision}" + ) + + +def enrich_glance_trace_outcomes_pg( + runs_data: list[dict[str, Any]], +) -> dict[str, list[dict[str, Any]]]: + """Merge legacy measurement summaries into matching full-trace rows.""" + summaries: dict[tuple[int | None, int], dict[str, Any]] = {} + for row in runs_data: + try: + source_run_id = int(row.get("idruns")) + except (TypeError, ValueError): + continue + source_tool_value = row.get( + "source_tool_id", + row.get("idtools"), + ) + try: + source_tool_id = ( + int(source_tool_value) + if source_tool_value not in (None, "") + else None + ) + except (TypeError, ValueError): + source_tool_id = None + inputs = { + feature: numeric + for feature in FEATURES + if (numeric := _finite_float(row.get(feature))) is not None + } + outputs = { + target: numeric + for target in (PRIMARY_TARGET, SECONDARY_TARGET) + if (numeric := _finite_float(row.get(target))) is not None + } + if ( + inputs + or outputs + or "is_outlier" in row + or "is_calibration_recipe" in row + ): + summaries[(source_tool_id, source_run_id)] = { + "inputs": inputs, + "outputs": outputs, + "is_outlier": ( + bool(row.get("is_outlier")) + if "is_outlier" in row + else None + ), + "is_calibration_recipe": ( + bool(row.get("is_calibration_recipe")) + if "is_calibration_recipe" in row + else None + ), + } + if not summaries: + return {} + + records_by_project: dict[str, list[dict[str, Any]]] = {} + conn = get_pg_superuser_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + for ( + source_tool_id, + source_run_id, + ), summary in sorted( + summaries.items(), + key=lambda item: ( + item[0][1], + -1 if item[0][0] is None else item[0][0], + ), + ): + # Always take the run-only compatibility lock so an old + # unscoped summary cannot cross a trace transaction. + cur.execute( + """ + SELECT pg_advisory_xact_lock( + hashtextextended(%s, 0) + ) + """, + (glance_source_run_lock_key(source_run_id),), + ) + if source_tool_id is not None: + cur.execute( + """ + SELECT pg_advisory_xact_lock( + hashtextextended(%s, 0) + ) + """, + ( + glance_source_run_lock_key( + source_run_id, + source_tool_id, + ), + ), + ) + cur.execute( + """ + SELECT + id, + project_id, + lotname, + is_outlier, + is_calibration_recipe, + execution_request_id, + inputs_json, + outputs_json, + raw_payload_json, + source_tool_id + FROM equipment_runs + WHERE source_system = 'glance' + AND source_run_id = %s + AND ( + %s IS NULL + OR source_tool_id = %s + ) + FOR UPDATE + """, + ( + source_run_id, + source_tool_id, + source_tool_id, + ), + ) + equipment_rows = list(cur.fetchall() or []) + if len(equipment_rows) != 1: + logger.warning( + "Deferred GLANCE summary bridge for tool=%s run=%s: " + "expected one trace candidate, found %d", + source_tool_id, + source_run_id, + len(equipment_rows), + ) + continue + for equipment_row in equipment_rows: + project_id = str( + equipment_row.get("project_id") or "" + ).strip() + if not project_id: + continue + merged_inputs = { + **summary["inputs"], + **dict(equipment_row.get("inputs_json") or {}), + } + merged_outputs = { + **dict(equipment_row.get("outputs_json") or {}), + **summary["outputs"], + } + is_outlier = ( + bool(summary["is_outlier"]) + if summary["is_outlier"] is not None + else bool(equipment_row.get("is_outlier")) + ) + is_calibration_recipe = ( + bool(summary["is_calibration_recipe"]) + if summary["is_calibration_recipe"] is not None + else bool( + equipment_row.get( + "is_calibration_recipe" + ) + ) + ) + raw_payload = dict( + equipment_row.get("raw_payload_json") or {} + ) + glance_metadata = raw_payload.get("glance") + if not isinstance(glance_metadata, dict): + glance_metadata = {} + proposal_id = str( + glance_metadata.get("proposal_id") or "" + ).strip() + authoritative_request_id = str( + equipment_row.get("execution_request_id") or "" + ).strip() + cur.execute( + """ + UPDATE etcher_runs + SET + project_id = %s, + execution_request_id = COALESCE( + NULLIF(%s, ''), + execution_request_id + ), + raw_payload_json = jsonb_set( + COALESCE( + raw_payload_json, + '{}'::jsonb + ), + '{source_tool_id}', + to_jsonb(%s::bigint), + true + ) + WHERE idruns = %s + AND ( + %s IS NULL + OR NULLIF( + raw_payload_json->>'source_tool_id', + '' + ) = %s + OR NULLIF( + raw_payload_json->>'idtools', + '' + ) = %s + ) + """, + ( + project_id, + authoritative_request_id, + int(equipment_row["source_tool_id"]), + source_run_id, + source_tool_id, + str(source_tool_id), + str(source_tool_id), + ), + ) + revision = equipment_training_revision_sha256( + equipment_run_id=int(equipment_row["id"]), + inputs=merged_inputs, + outputs=merged_outputs, + is_outlier=is_outlier, + is_calibration_recipe=is_calibration_recipe, + ) + cur.execute( + """ + UPDATE equipment_runs + SET + inputs_json = %s, + outputs_json = %s, + is_outlier = %s, + is_calibration_recipe = %s, + ingested_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + Json(merged_inputs), + Json(merged_outputs), + is_outlier, + is_calibration_recipe, + equipment_row["id"], + ), + ) + records_by_project.setdefault( + project_id, + [], + ).append( + { + "equipment_run_id": int( + equipment_row["id"] + ), + "source_tool_id": int( + equipment_row["source_tool_id"] + ), + "source_run_id": source_run_id, + "execution_request_id": ( + authoritative_request_id + ), + "proposal_id": proposal_id, + "lotname": str( + equipment_row.get("lotname") or "" + ), + "inputs": merged_inputs, + "outputs": merged_outputs, + "parameters": [], + "samples": [], + "is_outlier": is_outlier, + "is_calibration_recipe": ( + is_calibration_recipe + ), + "closed_loop_revision_sha256": revision, + } + ) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + return records_by_project + + +def _proposal_parameters(candidate: dict[str, Any]) -> dict[str, Any]: + proposal = candidate.get("proposal_json") + if not isinstance(proposal, dict): + return {} + parameters = proposal.get("parameters") + return parameters if isinstance(parameters, dict) else {} + + +def _proposal_matches_features( + candidate: dict[str, Any], + features: dict[str, float], + *, + minimum_matches: int, + absolute_tolerance: float, + relative_tolerance: float, +) -> bool: + parameters = _proposal_parameters(candidate) + compared = 0 + for name, proposed_value in parameters.items(): + actual = features.get(str(name)) + proposed = _finite_float(proposed_value) + if actual is None or proposed is None: + continue + compared += 1 + allowed = absolute_tolerance + relative_tolerance * abs(proposed) + if abs(actual - proposed) > allowed: + return False + return compared >= minimum_matches + + +def choose_proposal_candidate( + candidates: list[dict[str, Any]], + run_record: dict[str, Any], +) -> tuple[dict[str, Any] | None, str]: + """Choose one proposal conservatively, returning a decision explanation.""" + if not candidates: + return None, "no_candidate" + + request_id = str(run_record.get("execution_request_id") or "").strip() + if request_id: + compatible = [ + candidate + for candidate in candidates + if not str( + candidate.get("execution_request_id") or "" + ).strip() + or str( + candidate.get("execution_request_id") or "" + ).strip() + == request_id + ] + if not compatible: + return None, "execution_request_id_mismatch" + candidates = compatible + + explicit_proposal_id = str(run_record.get("proposal_id") or "").strip() + if explicit_proposal_id: + exact = [ + candidate + for candidate in candidates + if str(candidate.get("id") or "") == explicit_proposal_id + ] + return ( + (exact[0], "explicit_proposal_id") + if len(exact) == 1 + else (None, "proposal_id_not_found") + ) + + if request_id: + preferred = [ + candidate + for candidate in candidates + if str(candidate.get("status") or "").lower() + in _PREFERRED_PROPOSAL_STATUSES + ] + if len(preferred) == 1: + return preferred[0], "unique_accepted_or_attempted" + if len(candidates) == 1: + return candidates[0], "unique_request_candidate" + + minimum_matches = max( + 1, + int(os.getenv("GLANCE_LOOP_MIN_MATCHED_PARAMETERS", "3")), + ) + absolute_tolerance = max( + 0.0, + float(os.getenv("GLANCE_LOOP_ABSOLUTE_TOLERANCE", "1.0")), + ) + relative_tolerance = max( + 0.0, + float(os.getenv("GLANCE_LOOP_RELATIVE_TOLERANCE", "0.02")), + ) + features = derive_trace_feature_values(run_record) + matches = [ + candidate + for candidate in candidates + if _proposal_matches_features( + candidate, + features, + minimum_matches=minimum_matches, + absolute_tolerance=absolute_tolerance, + relative_tolerance=relative_tolerance, + ) + ] + if len(matches) == 1: + return matches[0], "unique_parameter_match" + if len(matches) > 1: + return None, "ambiguous_parameter_match" + return None, "no_parameter_match" + + +def _candidate_rows( + cur, + *, + project_id: str, + run_record: dict[str, Any], +) -> list[dict[str, Any]]: + equipment_run_id = int(run_record["equipment_run_id"]) + source_run_id = int(run_record["source_run_id"]) + proposal_id = str(run_record.get("proposal_id") or "").strip() + if proposal_id: + cur.execute( + """ + SELECT + rp.id::text, + rp.experiment_id::text, + rp.project_id, + rp.execution_request_id, + rp.status, + rp.proposal_json, + rp.run_id, + rp.equipment_run_id + FROM experiment_recipe_proposals rp + WHERE rp.project_id = %s + AND rp.id = %s + AND ( + rp.equipment_run_id IS NULL + OR rp.equipment_run_id = %s + ) + AND ( + rp.status = ANY(%s) + OR ( + rp.run_id = %s + AND rp.status IN ('completed', 'rejected') + ) + ) + """, + ( + project_id, + proposal_id, + equipment_run_id, + sorted(_ACTIVE_PROPOSAL_STATUSES), + source_run_id, + ), + ) + return list(cur.fetchall() or []) + + request_id = str(run_record.get("execution_request_id") or "").strip() + if request_id: + cur.execute( + """ + SELECT + rp.id::text, + rp.experiment_id::text, + rp.project_id, + rp.execution_request_id, + rp.status, + rp.proposal_json, + rp.run_id, + rp.equipment_run_id + FROM experiment_recipe_proposals rp + JOIN experiment_recipe_batches b ON b.id = rp.batch_id + WHERE rp.project_id = %s + AND rp.execution_request_id = %s + AND ( + rp.equipment_run_id IS NULL + OR rp.equipment_run_id = %s + ) + AND ( + ( + rp.status = ANY(%s) + AND b.id = ( + SELECT b2.id + FROM experiment_recipe_batches b2 + WHERE b2.experiment_id = rp.experiment_id + ORDER BY + b2.iteration DESC, + b2.created_at DESC + LIMIT 1 + ) + ) + OR ( + rp.run_id = %s + AND rp.status IN ('completed', 'rejected') + ) + ) + ORDER BY rp.proposal_index + """, + ( + project_id, + request_id, + equipment_run_id, + sorted(_ACTIVE_PROPOSAL_STATUSES), + source_run_id, + ), + ) + return list(cur.fetchall() or []) + + cur.execute( + """ + SELECT + rp.id::text, + rp.experiment_id::text, + rp.project_id, + rp.execution_request_id, + rp.status, + rp.proposal_json, + rp.run_id, + rp.equipment_run_id + FROM experiment_recipe_proposals rp + WHERE rp.project_id = %s + AND rp.equipment_run_id = %s + """, + (project_id, equipment_run_id), + ) + linked = list(cur.fetchall() or []) + if linked: + return linked + + cur.execute( + """ + SELECT + rp.id::text, + rp.experiment_id::text, + rp.project_id, + rp.execution_request_id, + rp.status, + rp.proposal_json, + rp.run_id, + rp.equipment_run_id + FROM experiment_recipe_proposals rp + WHERE rp.project_id = %s + AND rp.run_id = %s + AND rp.status IN ('completed', 'rejected') + AND rp.equipment_run_id IS NULL + ORDER BY rp.updated_at DESC + """, + (project_id, source_run_id), + ) + legacy_linked = list(cur.fetchall() or []) + if legacy_linked: + return legacy_linked + + cur.execute( + """ + SELECT + rp.id::text, + rp.experiment_id::text, + rp.project_id, + rp.execution_request_id, + rp.status, + rp.proposal_json, + rp.run_id, + rp.equipment_run_id + FROM experiment_recipe_proposals rp + JOIN experiment_recipe_batches b ON b.id = rp.batch_id + WHERE rp.project_id = %s + AND rp.status = ANY(%s) + AND rp.equipment_run_id IS NULL + AND b.id = ( + SELECT b2.id + FROM experiment_recipe_batches b2 + JOIN experiment_definitions e2 ON e2.id = b2.experiment_id + WHERE e2.project_id = %s + ORDER BY e2.created_at DESC, b2.iteration DESC, b2.created_at DESC + LIMIT 1 + ) + ORDER BY rp.proposal_index + """, + (project_id, sorted(_ACTIVE_PROPOSAL_STATUSES), project_id), + ) + return list(cur.fetchall() or []) + + +def reconcile_glance_trace_runs_pg( + *, + project_id: str, + run_records: list[dict[str, Any]], +) -> dict[str, Any]: + """Link stored physical traces to proposals in one idempotent transaction.""" + result: dict[str, Any] = { + "checked": len(run_records), + "matched": 0, + "newly_matched": 0, + "replayed": 0, + "unmatched": 0, + "unresolved_identity": 0, + "training_rows": 0, + "dataset_change_rows": 0, + "optimizer_resolution_rows": 0, + "training_project_ids": [], + "training_experiment_ids": [], + "followup_claims": [], + "followup_in_progress": 0, + "decisions": [], + } + if not run_records: + return result + + training_projects: set[str] = set() + training_experiments: set[str] = set() + conn = get_pg_superuser_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + for run_record in run_records: + equipment_run_id = int(run_record["equipment_run_id"]) + candidates = _candidate_rows( + cur, + project_id=project_id, + run_record=run_record, + ) + candidate, decision = choose_proposal_candidate( + candidates, + run_record, + ) + source_run_id = int(run_record["source_run_id"]) + if candidate is None: + result["unmatched"] += 1 + if ( + run_record.get("execution_request_id") + or run_record.get("proposal_id") + ): + result["unresolved_identity"] += 1 + result["decisions"].append( + { + "source_run_id": source_run_id, + "status": "unmatched", + "reason": decision, + } + ) + continue + + already_linked = ( + candidate.get("equipment_run_id") is not None + and int(candidate["equipment_run_id"]) == equipment_run_id + ) + candidate_status = str( + candidate.get("status") or "" + ).lower() + legacy_run_id = candidate.get("run_id") + legacy_summary_link = ( + candidate.get("equipment_run_id") is None + and legacy_run_id is not None + and int(legacy_run_id) == source_run_id + ) + authoritative_identity = bool( + run_record.get("proposal_id") + or run_record.get("execution_request_id") + ) + if authoritative_identity and not already_linked: + # A later exact identity is allowed to repair an earlier + # conservative/approximate link for this same physical + # run. Preserve human rejections verbatim while reopening + # only machine-completed approved conflicts. + cur.execute( + """ + WITH detached AS ( + UPDATE experiment_recipe_proposals + SET + equipment_run_id = NULL, + status = CASE + WHEN status = 'completed' + AND ingestion_status = 'approved' + THEN 'pending' + ELSE status + END, + ingestion_status = CASE + WHEN status = 'completed' + AND ingestion_status = 'approved' + THEN 'identity_conflict' + ELSE ingestion_status + END, + completed_at = CASE + WHEN status = 'completed' + AND ingestion_status = 'approved' + THEN NULL + ELSE completed_at + END, + comment = CASE + WHEN status = 'completed' + AND ingestion_status = 'approved' + THEN CONCAT_WS( + E'\n', + NULLIF(comment, ''), + 'Physical run detached after ' + 'exact GLANCE identity ' + 'correction.' + ) + ELSE comment + END, + updated_at = CURRENT_TIMESTAMP + WHERE project_id = %s + AND equipment_run_id = %s + AND id <> %s + RETURNING + experiment_id, + status, + ingestion_status + ) + UPDATE experiment_definitions e + SET + execution_status = 'requested', + data_ingested_at = NULL, + execution_updated_at = CURRENT_TIMESTAMP + WHERE e.id IN ( + SELECT experiment_id + FROM detached + WHERE status = 'pending' + AND ingestion_status = + 'identity_conflict' + ) + AND NOT EXISTS ( + SELECT 1 + FROM experiment_recipe_proposals linked + WHERE linked.experiment_id = e.id + AND linked.status = 'completed' + AND ( + linked.run_id IS NOT NULL + OR linked.equipment_run_id IS NOT NULL + ) + ) + """, + ( + project_id, + equipment_run_id, + candidate["id"], + ), + ) + if candidate_status == "rejected": + if ( + not already_linked + and ( + legacy_summary_link + or authoritative_identity + ) + ): + cur.execute( + """ + UPDATE experiment_recipe_proposals + SET + equipment_run_id = %s, + run_id = NULL, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + AND status = 'rejected' + AND equipment_run_id IS NULL + AND ( + run_id = %s + OR %s + ) + """, + ( + equipment_run_id, + candidate["id"], + source_run_id, + authoritative_identity, + ), + ) + if cur.rowcount != 1: + raise RuntimeError( + f"Rejected proposal {candidate['id']} " + "was linked concurrently" + ) + result["matched"] += 1 + if already_linked: + result["replayed"] += 1 + else: + result["newly_matched"] += 1 + result["decisions"].append( + { + "source_run_id": source_run_id, + "proposal_id": str(candidate["id"]), + "status": "rejected_preserved", + "reason": "terminal_human_decision", + "training_eligible": False, + "training_revision_changed": False, + } + ) + continue + candidate_request_id = str( + candidate.get("execution_request_id") or "" + ).strip() + request_id = ( + candidate_request_id + or str( + run_record.get("execution_request_id") or "" + ).strip() + ) + canonical_inputs, _, training_eligible = training_values_for_run( + run_record + ) + terminally_excluded = bool( + run_record.get("is_outlier") + or run_record.get("is_calibration_recipe") + ) + retrain_credit_key = retrain_credit_key_for_run( + run_record + ) + effective_inputs = { + **dict(run_record.get("inputs") or {}), + **canonical_inputs, + } + revision = equipment_training_revision_sha256( + equipment_run_id=equipment_run_id, + inputs=effective_inputs, + outputs=dict(run_record.get("outputs") or {}), + is_outlier=bool(run_record.get("is_outlier")), + is_calibration_recipe=bool( + run_record.get("is_calibration_recipe") + ), + ) + + cur.execute( + """ + SELECT + closed_loop_revision_sha256, + closed_loop_followup_revision_sha256, + closed_loop_followup_claim_revision_sha256, + closed_loop_followup_claim_kind, + closed_loop_followup_claim_token, + closed_loop_followup_claimed_at + FROM equipment_runs + WHERE id = %s AND project_id = %s + FOR UPDATE + """, + (equipment_run_id, project_id), + ) + equipment_row = cur.fetchone() + if not equipment_row: + raise RuntimeError( + f"Equipment run {equipment_run_id} is outside project " + f"{project_id}" + ) + previous_revision = str( + equipment_row.get("closed_loop_revision_sha256") or "" + ) + completed_followup_revision = str( + equipment_row.get( + "closed_loop_followup_revision_sha256" + ) + or "" + ) + claimed_followup_revision = str( + equipment_row.get( + "closed_loop_followup_claim_revision_sha256" + ) + or "" + ) + claimed_followup_kind = str( + equipment_row.get( + "closed_loop_followup_claim_kind" + ) + or "" + ) + claimed_at = equipment_row.get( + "closed_loop_followup_claimed_at" + ) + if claimed_at is not None and claimed_at.tzinfo is None: + claimed_at = claimed_at.replace(tzinfo=timezone.utc) + claim_lease_seconds = max( + 60, + int( + os.getenv( + "GLANCE_LOOP_FOLLOWUP_CLAIM_SECONDS", + "900", + ) + ), + ) + claim_active = ( + bool(revision) + and claimed_followup_revision == revision + and claimed_at is not None + and claimed_at + > datetime.now(timezone.utc) + - timedelta(seconds=claim_lease_seconds) + ) + prior_attempted_revision_changed = ( + bool(claimed_followup_revision) + and claimed_followup_revision != revision + ) + retrying_failed_dataset_change = ( + claimed_followup_revision == revision + and claimed_followup_kind == "dataset_change" + and not claim_active + ) + followup_pending = ( + bool(revision) + and completed_followup_revision != revision + and ( + training_eligible + or terminally_excluded + or bool(completed_followup_revision) + or prior_attempted_revision_changed + or retrying_failed_dataset_change + ) + ) + claim_followup = followup_pending and not claim_active + claim_token = str(uuid.uuid4()) if claim_followup else "" + followup_kind = "" + followup_credit_key = None + if claim_followup: + if ( + completed_followup_revision + or prior_attempted_revision_changed + or retrying_failed_dataset_change + ): + followup_kind = "dataset_change" + elif training_eligible: + followup_kind = "training" + followup_credit_key = retrain_credit_key + else: + followup_kind = "optimizer_resolution" + + if ( + run_record.get("proposal_id") + or run_record.get("execution_request_id") + ): + cur.execute( + """ + WITH conflicts AS ( + UPDATE experiment_recipe_proposals + SET + status = 'pending', + ingestion_status = 'identity_conflict', + run_id = NULL, + lotname = '', + completed_at = NULL, + comment = CONCAT_WS( + E'\n', + NULLIF(comment, ''), + 'Legacy summary link reopened after ' + 'exact GLANCE trace identity ' + 'reconciliation.' + ), + updated_at = CURRENT_TIMESTAMP + WHERE project_id = %s + AND run_id = %s + AND id <> %s + AND status = 'completed' + AND ingestion_status = 'approved' + RETURNING experiment_id + ) + UPDATE experiment_definitions e + SET + execution_status = 'requested', + data_ingested_at = NULL, + execution_updated_at = CURRENT_TIMESTAMP + WHERE e.id IN ( + SELECT experiment_id FROM conflicts + ) + AND NOT EXISTS ( + SELECT 1 + FROM experiment_recipe_proposals linked + WHERE linked.experiment_id = e.id + AND linked.status = 'completed' + AND ( + linked.run_id IS NOT NULL + OR linked.equipment_run_id IS NOT NULL + ) + ) + """, + ( + project_id, + source_run_id, + candidate["id"], + ), + ) + + cur.execute( + """ + UPDATE experiment_recipe_proposals + SET + status = 'completed', + ingestion_status = 'approved', + equipment_run_id = %s, + run_id = NULL, + lotname = %s, + completed_at = COALESCE( + completed_at, + CURRENT_TIMESTAMP + ), + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + AND ( + equipment_run_id IS NULL + OR equipment_run_id = %s + ) + """, + ( + equipment_run_id, + str(run_record.get("lotname") or ""), + candidate["id"], + equipment_run_id, + ), + ) + if cur.rowcount != 1: + raise RuntimeError( + f"Proposal {candidate['id']} was linked concurrently" + ) + + cur.execute( + """ + UPDATE experiment_definitions + SET + execution_status = 'data_ingested', + data_ingested_at = COALESCE( + data_ingested_at, + CURRENT_TIMESTAMP + ), + execution_updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (candidate["experiment_id"],), + ) + cur.execute( + """ + UPDATE equipment_runs + SET + execution_request_id = COALESCE( + NULLIF(%s, ''), + execution_request_id + ), + inputs_json = inputs_json || %s, + closed_loop_revision_sha256 = %s + WHERE id = %s + """, + ( + request_id, + Json(canonical_inputs), + revision, + equipment_run_id, + ), + ) + if claim_followup: + cur.execute( + """ + UPDATE equipment_runs + SET + closed_loop_followup_claim_revision_sha256 = %s, + closed_loop_followup_claim_kind = %s, + closed_loop_followup_claim_token = %s, + closed_loop_followup_claimed_at = + CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + revision, + followup_kind, + claim_token, + equipment_run_id, + ), + ) + + revision_changed = bool(revision) and revision != previous_revision + if claim_followup: + if followup_kind == "dataset_change": + # Any correction to a revision that already completed + # or attempted maintenance changes the current training + # snapshot and any optimizer output produced before a + # later failure. This includes eligible->eligible, + # eligible->excluded/incomplete, and restored-eligible + # changes. None is a new run credit. + result["dataset_change_rows"] += 1 + elif followup_kind == "training": + result["training_rows"] += 1 + else: + result["optimizer_resolution_rows"] += 1 + training_projects.add(project_id) + training_experiments.add( + str(candidate["experiment_id"]) + ) + result["followup_claims"].append( + { + "equipment_run_id": equipment_run_id, + "revision_sha256": revision, + "claim_token": claim_token, + "retrain_credit_key": followup_credit_key, + "followup_kind": followup_kind, + } + ) + elif followup_pending: + result["followup_in_progress"] += 1 + + result["matched"] += 1 + if already_linked: + result["replayed"] += 1 + match_status = "replayed" + else: + result["newly_matched"] += 1 + match_status = "matched" + result["decisions"].append( + { + "source_run_id": source_run_id, + "proposal_id": str(candidate["id"]), + "status": match_status, + "reason": decision, + "legacy_summary_link": legacy_summary_link, + "training_eligible": training_eligible, + "training_revision_changed": revision_changed, + "training_followup_claimed": claim_followup, + } + ) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + result["training_project_ids"] = sorted(training_projects) + result["training_experiment_ids"] = sorted(training_experiments) + return result + + +def finish_glance_followup_claims_pg( + claims: list[dict[str, Any]], + *, + succeeded: bool, +) -> None: + """Complete or release persisted follow-up claims after synchronous work.""" + if not claims: + return + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + for claim in claims: + equipment_run_id = int(claim["equipment_run_id"]) + revision = str(claim["revision_sha256"]) + claim_token = str(claim["claim_token"]) + if succeeded: + cur.execute( + """ + UPDATE equipment_runs + SET + closed_loop_followup_revision_sha256 = %s, + closed_loop_followup_claim_revision_sha256 = '', + closed_loop_followup_claim_kind = '', + closed_loop_followup_claim_token = '', + closed_loop_followup_claimed_at = NULL + WHERE id = %s + AND closed_loop_followup_claim_revision_sha256 = %s + AND closed_loop_followup_claim_token = %s + """, + ( + revision, + equipment_run_id, + revision, + claim_token, + ), + ) + if cur.rowcount != 1: + raise RuntimeError( + "GLANCE follow-up claim ownership was lost for " + f"equipment run {equipment_run_id}" + ) + else: + cur.execute( + """ + UPDATE equipment_runs + SET + closed_loop_followup_claim_token = '', + closed_loop_followup_claimed_at = NULL + WHERE id = %s + AND closed_loop_followup_claim_revision_sha256 = %s + AND closed_loop_followup_claim_token = %s + """, + (equipment_run_id, revision, claim_token), + ) + if cur.rowcount != 1: + raise RuntimeError( + "GLANCE follow-up claim ownership was lost for " + f"equipment run {equipment_run_id}" + ) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +def _maybe_retrain( + n_new: int, + *, + credit_keys: list[str] | None = None, + force_dataset_refresh: bool = False, +) -> dict[str, Any]: + if n_new <= 0 and not force_dataset_refresh: + return {"status": "skipped", "reason": "no_training_rows"} + if os.getenv("AUTO_RETRAIN_ENABLED", "true").lower() == "false": + if force_dataset_refresh: + return { + "status": "failed", + "reason": ( + "dataset_correction_requires_retrain_but_auto_retrain_" + "is_disabled" + ), + } + return {"status": "skipped", "reason": "disabled"} + + from model_registry import ( + bump_runs_counter, + credit_runs_counter_once, + get_retrain_state, + load_active_bundle, + ) + from training.retrain import run_retrain_safe + + min_new = int(os.getenv("AUTO_RETRAIN_MIN_NEW_RUNS", "5")) + max_stale_h = float( + os.getenv("AUTO_RETRAIN_MAX_STALENESS_HOURS", "24") + ) + domain_id = os.getenv("AUTO_RETRAIN_DOMAIN", "etcher") + if credit_keys: + credit_result = credit_runs_counter_once( + domain_id=domain_id, + credits=[(credit_key, 1) for credit_key in credit_keys], + ) + total_pending = int( + credit_result.get("runs_since_last_train") or 0 + ) + else: + total_pending = bump_runs_counter( + domain_id=domain_id, + n_new=n_new, + ) + state = get_retrain_state(domain_id) or {} + has_active = load_active_bundle( + domain_id=domain_id, + target=PRIMARY_TARGET, + ) is not None + + stale = False + last_trained = state.get("last_trained_at") + if last_trained: + try: + last_dt = datetime.fromisoformat(str(last_trained)) + if last_dt.tzinfo is None: + last_dt = last_dt.replace(tzinfo=timezone.utc) + stale = ( + datetime.now(timezone.utc) - last_dt + ).total_seconds() > max_stale_h * 3600 + except (TypeError, ValueError): + stale = True + + if ( + not force_dataset_refresh + and has_active + and total_pending < min_new + and not stale + ): + return { + "status": "debounced", + "pending": total_pending, + "threshold": min_new, + } + return run_retrain_safe( + reason=( + "glance_dataset_correction" + if force_dataset_refresh + else "glance_ingestion" + ), + force=force_dataset_refresh, + ) + + +def run_glance_closed_loop_followups( + reconciliation: dict[str, Any], +) -> dict[str, Any]: + """Run optimizer/retraining maintenance after a committed reconciliation.""" + training_rows = int(reconciliation.get("training_rows") or 0) + dataset_change_rows = int( + reconciliation.get("dataset_change_rows") or 0 + ) + optimizer_resolution_rows = int( + reconciliation.get("optimizer_resolution_rows") or 0 + ) + project_ids = list(reconciliation.get("training_project_ids") or []) + experiment_ids = list( + reconciliation.get("training_experiment_ids") or [] + ) + response: dict[str, Any] = {} + + if ( + training_rows + optimizer_resolution_rows > 0 + and experiment_ids + and os.getenv( + "AUTO_RECIPE_PROPOSALS_ENABLED", + "true", + ).lower() + != "false" + ): + from metadata_pg import auto_generate_recipe_proposals_for_projects_pg + + try: + response["proposals"] = ( + auto_generate_recipe_proposals_for_projects_pg( + project_ids=project_ids, + experiment_ids=experiment_ids, + reason="glance_ingestion", + ) + ) + except Exception as exc: + logger.warning( + "GLANCE automatic proposal task failed: %s", + exc, + exc_info=True, + ) + response["proposals"] = { + "status": "failed", + "error": str(exc), + } + else: + response["proposals"] = { + "status": "skipped", + "reason": "no_complete_training_rows", + } + + retrain_credit_keys: list[str] = [] + for claim in reconciliation.get("followup_claims") or []: + credit_key = claim.get("retrain_credit_key") + if credit_key: + retrain_credit_keys.append(str(credit_key)) + elif ( + training_rows > 0 + and claim.get("followup_kind") + not in {"optimizer_resolution", "dataset_change"} + ): + # Compatibility for claims produced before followup_kind and the + # source-tool credit key were added. equipment_run_id is globally + # unique in DT, so this fallback cannot collide across tools. + retrain_credit_keys.append( + "glance:" + f"{int(claim['equipment_run_id'])}:" + f"{str(claim['revision_sha256'])}" + ) + + proposals = response.get("proposals") or {} + proposal_maintenance_failed = ( + str(proposals.get("status") or "").lower() == "failed" + or any( + item.get("reason") == "generation_failed" + for item in proposals.get("skipped", []) + if isinstance(item, dict) + ) + ) + if proposal_maintenance_failed: + # One revision claim covers both side effects. Do not promote a model + # from a revision whose optimizer maintenance failed; the retained + # attempted-revision marker makes a later source correction force a + # cleanup refresh. + response["retrain"] = { + "status": "skipped_dependency_failed", + "reason": "proposal_maintenance_failed", + } + else: + try: + response["retrain"] = _maybe_retrain( + training_rows, + credit_keys=retrain_credit_keys, + force_dataset_refresh=dataset_change_rows > 0, + ) + except Exception as exc: + logger.warning( + "GLANCE automatic retraining task failed: %s", + exc, + exc_info=True, + ) + response["retrain"] = {"status": "failed", "error": str(exc)} + return response + + +def glance_followups_succeeded(result: dict[str, Any]) -> bool: + """Return whether durable follow-up claims may be marked complete.""" + proposals = result.get("proposals") or {} + retrain = result.get("retrain") or {} + if str(proposals.get("status") or "").lower() == "failed": + return False + if any( + item.get("reason") == "generation_failed" + for item in proposals.get("skipped", []) + if isinstance(item, dict) + ): + return False + return str(retrain.get("status") or "").lower() not in { + "error", + "failed", + "skipped_locked", + } + + +__all__ = [ + "choose_proposal_candidate", + "derive_trace_feature_values", + "enrich_glance_trace_outcomes_pg", + "finish_glance_followup_claims_pg", + "glance_followups_succeeded", + "reconcile_glance_trace_runs_pg", + "retrain_credit_key_for_run", + "run_glance_closed_loop_followups", + "training_values_for_run", +] diff --git a/api/glance_ingestion.py b/api/glance_ingestion.py index 587cf1e..51b3e2b 100644 --- a/api/glance_ingestion.py +++ b/api/glance_ingestion.py @@ -4,14 +4,20 @@ import hashlib import json +import math import os import re +import uuid from datetime import datetime, timezone from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field +_PG_BIGINT_MAX = 9_223_372_036_854_775_807 +_PG_INTEGER_MAX = 2_147_483_647 + + class GlanceBatchEnvelope(BaseModel): model_config = ConfigDict(extra="forbid") @@ -70,6 +76,8 @@ class GlanceRun(BaseModel): run_start_time: str run_end_time: str | None = None source_status: str = "" + execution_request_id: str = "" + proposal_id: str = "" recipe: dict[str, Any] = Field(default_factory=dict) inputs: dict[str, Any] = Field(default_factory=dict) outputs: dict[str, Any] = Field(default_factory=dict) @@ -106,6 +114,7 @@ def load_production_mapping() -> dict[str, dict[str, Any]]: "GLANCE_PRODUCTION_MAPPINGS_JSON must be an equipment-keyed object" ) mappings: dict[str, dict[str, Any]] = {} + equipment_by_tool_id: dict[int, str] = {} for equipment_id, mapping in parsed.items(): if not isinstance(mapping, dict): raise RuntimeError( @@ -117,12 +126,29 @@ def load_production_mapping() -> dict[str, dict[str, Any]]: raise RuntimeError( f"GLANCE mapping for {equipment_id!r} requires source_tool_id" ) from exc + if tool_id < 1 or tool_id > _PG_BIGINT_MAX: + raise RuntimeError( + f"GLANCE mapping for {equipment_id!r} has source_tool_id " + "outside the PostgreSQL BIGINT range" + ) project_id = str(mapping.get("project_id") or "").strip() if not project_id: raise RuntimeError( f"GLANCE mapping for {equipment_id!r} requires project_id" ) - mappings[str(equipment_id)] = { + normalized_equipment_id = str(equipment_id) + previous_equipment_id = equipment_by_tool_id.get(tool_id) + if ( + previous_equipment_id is not None + and previous_equipment_id != normalized_equipment_id + ): + raise RuntimeError( + f"GLANCE source_tool_id {tool_id} is mapped to both " + f"{previous_equipment_id!r} and " + f"{normalized_equipment_id!r}" + ) + equipment_by_tool_id[tool_id] = normalized_equipment_id + mappings[normalized_equipment_id] = { "source_tool_id": tool_id, "source_tool_name": str(mapping.get("source_tool_name") or ""), "project_id": project_id, @@ -173,6 +199,43 @@ def _parse_source_time(value: Any, *, field_name: str) -> tuple[datetime, str]: return parsed.replace(tzinfo=None), raw +def _bounded_integer( + value: Any, + *, + field_name: str, + minimum: int, + maximum: int, +) -> int: + if isinstance(value, bool): + raise ValueError(f"{field_name} must be an integer") + try: + numeric = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be an integer") from exc + if numeric < minimum or numeric > maximum: + raise ValueError( + f"{field_name} must be between {minimum} and {maximum}" + ) + return numeric + + +def _validate_finite_json(value: Any, *, path: str) -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"{path} contains a non-finite number") + if isinstance(value, dict): + for key, nested in value.items(): + _validate_finite_json( + nested, + path=f"{path}.{key}", + ) + elif isinstance(value, (list, tuple)): + for index, nested in enumerate(value): + _validate_finite_json( + nested, + path=f"{path}[{index}]", + ) + + def _match_key(value: Any) -> str: text = "".join( character if character.isalnum() else " " @@ -193,6 +256,25 @@ def _numeric_id(value: Any) -> str: return str(int(text)) if text.isdigit() else "" +def _normalized_parameter_key(value: Any) -> str: + """Match the persistence layer's stable parameter-key normalization.""" + text = str(value or "").strip() + if not text: + return "" + match = re.fullmatch(r"[pP]?(\d+)", text) + return f"p{int(match.group(1))}" if match else text + + +def _source_parameter_identity(value: Any) -> str: + """Canonicalize a non-null source parameter ID for duplicate checks.""" + if value in (None, ""): + return "" + numeric = _numeric_id(value) + if numeric: + return f"numeric:{numeric}" + return f"text:{str(value).strip().casefold()}" + + def equipment_parameter_registry( equipment: dict[str, Any] | None, ) -> list[dict[str, str]]: @@ -334,7 +416,38 @@ def prepare_live_run( source_updated_at: datetime | None, registry: list[dict[str, str]], ) -> tuple[dict[str, Any], list[str]]: + _validate_finite_json(raw_run, path="run") + source_tool_id = _bounded_integer( + source_tool_id, + field_name="source_tool_id", + minimum=1, + maximum=_PG_BIGINT_MAX, + ) run = GlanceRun.model_validate(raw_run) + source_run_id = _bounded_integer( + run.source_run_id, + field_name="source_run_id", + minimum=1, + maximum=_PG_BIGINT_MAX, + ) + execution_request_id = str(run.execution_request_id or "").strip() + if execution_request_id and not re.fullmatch( + r"REQ-[A-F0-9]{12}", + execution_request_id, + flags=re.IGNORECASE, + ): + raise ValueError( + f"GLANCE run {run.source_run_id} has an invalid execution request ID" + ) + execution_request_id = execution_request_id.upper() + proposal_id = str(run.proposal_id or "").strip() + if proposal_id: + try: + proposal_id = str(uuid.UUID(proposal_id)) + except ValueError as exc: + raise ValueError( + f"GLANCE run {run.source_run_id} has an invalid proposal ID" + ) from exc if not run.samples: raise ValueError(f"GLANCE run {run.source_run_id} has no sample records") start_time, start_raw = _parse_source_time( @@ -355,14 +468,84 @@ def prepare_live_run( warnings: list[str] = [] parameters = [] + seen_parameter_keys: set[str] = set() + seen_source_parameter_ids: set[str] = set() for parameter in run.parameters: annotated, warning = annotate_parameter(parameter, registry) + normalized_key = _normalized_parameter_key(annotated.get("key")) + if not normalized_key: + raise ValueError( + f"GLANCE run {source_run_id} contains a parameter without " + "a stable key" + ) + if normalized_key in seen_parameter_keys: + raise ValueError( + f"GLANCE run {source_run_id} contains duplicate parameter " + f"key {normalized_key!r}" + ) + seen_parameter_keys.add(normalized_key) + source_identity = _source_parameter_identity( + annotated.get("source_parameter_id") + ) + if source_identity and source_identity in seen_source_parameter_ids: + raise ValueError( + f"GLANCE run {source_run_id} repeats source parameter ID " + f"{parameter.source_parameter_id!r}" + ) + if source_identity: + seen_source_parameter_ids.add(source_identity) parameters.append(annotated) if warning: warnings.append(warning) samples = [] + seen_source_sample_ids: set[int] = set() + seen_sample_indices: set[int] = set() for index, sample in enumerate(run.samples): + source_sample_id = _bounded_integer( + sample.source_sample_id, + field_name=( + f"run {source_run_id} source_sample_id" + ), + minimum=1, + maximum=_PG_BIGINT_MAX, + ) + sample_index = _bounded_integer( + ( + sample.sample_index + if sample.sample_index is not None + else index + ), + field_name=f"run {source_run_id} sample_index", + minimum=0, + maximum=_PG_INTEGER_MAX, + ) + if source_sample_id in seen_source_sample_ids: + raise ValueError( + f"GLANCE run {source_run_id} repeats sample id " + f"{source_sample_id}" + ) + if sample_index in seen_sample_indices: + raise ValueError( + f"GLANCE run {source_run_id} has duplicate sample index " + f"{sample_index}" + ) + seen_source_sample_ids.add(source_sample_id) + seen_sample_indices.add(sample_index) + seen_sample_value_keys: set[str] = set() + for raw_key in sample.values: + normalized_key = _normalized_parameter_key(raw_key) + if not normalized_key: + raise ValueError( + f"GLANCE sample {source_sample_id} contains a blank " + "parameter key" + ) + if normalized_key in seen_sample_value_keys: + raise ValueError( + f"GLANCE sample {source_sample_id} repeats parameter " + f"key {normalized_key!r}" + ) + seen_sample_value_keys.add(normalized_key) sample_time = None sample_time_raw = None if sample.timestamp is not None and str(sample.timestamp).strip(): @@ -391,12 +574,8 @@ def prepare_live_run( ) samples.append( { - "source_sample_id": sample.source_sample_id, - "sample_index": ( - sample.sample_index - if sample.sample_index is not None - else index - ), + "source_sample_id": source_sample_id, + "sample_index": sample_index, "source_tool_id": source_tool_id, "sample_time_raw": sample_time_raw, "values": sample.values, @@ -404,7 +583,20 @@ def prepare_live_run( ) events = [] + seen_source_event_ids: set[int] = set() for event in run.events: + source_event_id = _bounded_integer( + event.source_event_id, + field_name=f"run {source_run_id} source_event_id", + minimum=1, + maximum=_PG_BIGINT_MAX, + ) + if source_event_id in seen_source_event_ids: + raise ValueError( + f"GLANCE run {source_run_id} repeats event id " + f"{source_event_id}" + ) + seen_source_event_ids.add(source_event_id) event_time, event_time_raw = _parse_source_time( event.timestamp, field_name=f"run {run.source_run_id} event {event.source_event_id}", @@ -422,6 +614,7 @@ def prepare_live_run( f"{run.source_run_id}'s authoritative interval" ) event_record = event.model_dump() + event_record["source_event_id"] = source_event_id event_record["source_tool_id"] = source_tool_id event_record["timestamp"] = event_time_raw event_record["_parsed_time"] = event_time @@ -438,13 +631,15 @@ def prepare_live_run( glance = {} glance.update( { - "run_id": run.source_run_id, + "run_id": source_run_id, "tool_id": source_tool_id, "run_start_time": start_raw, "run_end_time": end_raw, "source_status": run.source_status, "material": run.material, "recipe": run.recipe, + "execution_request_id": execution_request_id, + "proposal_id": proposal_id, } ) raw["glance"] = glance @@ -466,8 +661,10 @@ def prepare_live_run( return ( { - "source_run_id": run.source_run_id, + "source_run_id": source_run_id, "source_updated_at": effective_source_updated_at, + "execution_request_id": execution_request_id, + "proposal_id": proposal_id, "lotname": run.lotname, "run_start_time": start_raw, "run_date": start_raw, diff --git a/api/metadata_pg.py b/api/metadata_pg.py index e0e04ba..07e1913 100644 --- a/api/metadata_pg.py +++ b/api/metadata_pg.py @@ -9,7 +9,11 @@ from psycopg2.extras import Json, RealDictCursor -from data_loader_pg import get_pg_connection, get_pg_superuser_connection +from data_loader_pg import ( + EQUIPMENT_RUN_ID_OFFSET, + get_pg_connection, + get_pg_superuser_connection, +) from security import PlatformUser @@ -231,6 +235,13 @@ def ensure_execution_request_id_columns_pg() -> None: ADD COLUMN IF NOT EXISTS execution_request_id VARCHAR(80) NOT NULL DEFAULT '' """ ) + cur.execute( + """ + ALTER TABLE experiment_recipe_proposals + ADD COLUMN IF NOT EXISTS equipment_run_id BIGINT + REFERENCES equipment_runs(id) ON DELETE SET NULL + """ + ) cur.execute( """ ALTER TABLE etcher_runs @@ -251,6 +262,58 @@ def ensure_execution_request_id_columns_pg() -> None: WHERE execution_request_id <> '' """ ) + cur.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_proposals_equipment_run_id + ON experiment_recipe_proposals(equipment_run_id) + WHERE equipment_run_id IS NOT NULL + """ + ) + cur.execute( + """ + WITH duplicate_experiments AS ( + SELECT experiment_id + FROM experiment_recipe_batches + GROUP BY experiment_id + HAVING COUNT(*) > COUNT(DISTINCT iteration) + ), + renumbered AS ( + SELECT + b.id, + ROW_NUMBER() OVER ( + PARTITION BY b.experiment_id + ORDER BY b.iteration, b.created_at, b.id + ) AS normalized_iteration + FROM experiment_recipe_batches b + JOIN duplicate_experiments d + ON d.experiment_id = b.experiment_id + ) + UPDATE experiment_recipe_batches b + SET iteration = r.normalized_iteration + FROM renumbered r + WHERE b.id = r.id + AND b.iteration <> r.normalized_iteration + """ + ) + cur.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_batches_iteration + ON experiment_recipe_batches(experiment_id, iteration) + """ + ) + cur.execute( + """ + CREATE TABLE IF NOT EXISTS retrain_run_credits ( + domain_id VARCHAR(100) NOT NULL, + credit_key VARCHAR(255) NOT NULL, + n_new INTEGER NOT NULL DEFAULT 1 + CHECK (n_new > 0), + credited_at TIMESTAMP WITH TIME ZONE NOT NULL + DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (domain_id, credit_key) + ) + """ + ) cur.execute( """ CREATE INDEX IF NOT EXISTS idx_etcher_runs_execution_request_id @@ -4523,6 +4586,7 @@ def save_experiment_proposal_batches_pg( optimizer_result_json ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (experiment_id, iteration) DO NOTHING RETURNING id """, ( @@ -4553,7 +4617,9 @@ def save_experiment_proposal_batches_pg( ) batch_row = cur.fetchone() batch_id = batch_row[0] if batch_row else None - if batch_id and isinstance(proposals, list): + if not batch_id: + continue + if isinstance(proposals, list): for index, proposal in enumerate(proposals): cur.execute( """ @@ -4809,7 +4875,8 @@ def reconcile_ingested_runs_with_recipe_proposals_pg( def auto_generate_recipe_proposals_for_projects_pg( *, - project_ids: list[str], + project_ids: list[str] | None = None, + experiment_ids: list[str] | None = None, reason: str = "ingestion", ) -> dict[str, Any]: """ @@ -4820,18 +4887,82 @@ def auto_generate_recipe_proposals_for_projects_pg( optimizer batch unless a batch has already been created after the latest ingested run. """ - project_ids = sorted({project_id for project_id in project_ids if project_id}) - if not project_ids: - return {"status": "skipped", "reason": "no_projects", "generated": 0} + project_ids = sorted( + { + project_id + for project_id in (project_ids or []) + if project_id + } + ) + experiment_ids = sorted( + { + experiment_id + for experiment_id in (experiment_ids or []) + if experiment_id + } + ) + if not project_ids and not experiment_ids: + return { + "status": "skipped", + "reason": "no_targets", + "generated": 0, + } conn = get_pg_superuser_connection() generated = 0 skipped: list[dict[str, Any]] = [] + processed_projects: set[str] = set() try: with conn.cursor(cursor_factory=RealDictCursor) as cur: - for project_id in project_ids: + target_specs: list[tuple[str, str | None]] = [] + if experiment_ids: + for experiment_id in experiment_ids: + cur.execute( + """ + SELECT project_id + FROM experiment_definitions + WHERE id = %s + """, + (experiment_id,), + ) + target_row = cur.fetchone() + if not target_row or not target_row.get("project_id"): + skipped.append( + { + "experiment_id": experiment_id, + "reason": "experiment_not_found", + } + ) + continue + target_specs.append( + ( + str(target_row["project_id"]), + experiment_id, + ) + ) + else: + target_specs = [ + (project_id, None) + for project_id in project_ids + ] + + for project_id, exact_experiment_id in target_specs: + processed_projects.add(project_id) cur.execute( """ + SELECT pg_advisory_xact_lock( + hashtextextended(%s, 0) + ) + """, + (f"auto_recipe_proposals|{project_id}",), + ) + target_condition = ( + "e.id = %s" + if exact_experiment_id + else "e.project_id = %s" + ) + cur.execute( + f""" SELECT e.id::text AS experiment_id, e.project_id, @@ -4869,24 +5000,153 @@ def auto_generate_recipe_proposals_for_projects_pg( ) AND rp.status <> ALL(%s) ), 0) AS latest_open_proposals, + COALESCE(( + SELECT COUNT(*)::int + FROM experiment_recipe_proposals rp + JOIN experiment_recipe_batches b + ON b.id = rp.batch_id + WHERE b.experiment_id = e.id + AND b.iteration = ( + SELECT MAX(b2.iteration) + FROM experiment_recipe_batches b2 + WHERE b2.experiment_id = e.id + ) + AND rp.status = 'completed' + AND NOT ( + EXISTS ( + SELECT 1 + FROM equipment_runs resolved_equipment + WHERE resolved_equipment.id = + rp.equipment_run_id + AND ( + resolved_equipment.is_outlier + OR resolved_equipment + .is_calibration_recipe + OR ( + ( + resolved_equipment + .inputs_json + ->>'Etch_AvgO2Flow' + ) ~ %s + AND ( + resolved_equipment + .inputs_json + ->>'Etch_Avg_Rf1_Pow' + ) ~ %s + AND ( + resolved_equipment + .inputs_json + ->>'Etch_Avg_Rf2_Pow' + ) ~ %s + AND ( + resolved_equipment + .inputs_json + ->>'Etch_AvgPres' + ) ~ %s + AND ( + resolved_equipment + .inputs_json + ->>'Etch_Avgcf4Flow' + ) ~ %s + AND ( + resolved_equipment + .outputs_json + ->>'AvgEtchRate' + ) ~ %s + AND ( + resolved_equipment + .outputs_json + ->>'RangeEtchRate' + ) ~ %s + ) + ) + ) + OR EXISTS ( + SELECT 1 + FROM etcher_runs resolved_legacy + WHERE resolved_legacy.idruns = rp.run_id + AND ( + resolved_legacy.is_outlier + OR resolved_legacy + .is_calibration_recipe + OR ( + resolved_legacy + .etch_avgo2flow + IS NOT NULL + AND resolved_legacy + .etch_avg_rf1_pow + IS NOT NULL + AND resolved_legacy + .etch_avg_rf2_pow + IS NOT NULL + AND resolved_legacy + .etch_avgpres + IS NOT NULL + AND resolved_legacy + .etch_avgcf4flow + IS NOT NULL + AND resolved_legacy + .avg_etch_rate + IS NOT NULL + AND resolved_legacy + .range_etch_rate + IS NOT NULL + ) + ) + ) + ) + ), 0) AS latest_unresolved_results, ( - SELECT MAX(r.created_at) - FROM etcher_runs r - WHERE r.project_id = e.project_id + SELECT MAX(training_run.ingested_at) + FROM ( + SELECT r.created_at AS ingested_at + FROM etcher_runs r + WHERE r.project_id = e.project_id + + UNION ALL + + SELECT equipment_run.ingested_at AS ingested_at + FROM equipment_runs equipment_run + WHERE equipment_run.project_id = e.project_id + AND equipment_run.source_system = 'glance' + AND equipment_run.inputs_json ?& ARRAY[ + 'Etch_AvgO2Flow', + 'Etch_Avg_Rf1_Pow', + 'Etch_Avg_Rf2_Pow', + 'Etch_AvgPres', + 'Etch_Avgcf4Flow' + ] + AND equipment_run.outputs_json ?& ARRAY[ + 'AvgEtchRate', + 'RangeEtchRate' + ] + AND NOT equipment_run.is_outlier + AND NOT equipment_run.is_calibration_recipe + ) training_run ) AS latest_run_at FROM experiment_definitions e LEFT JOIN experiment_types t ON t.id = e.type_id - WHERE e.project_id = %s + WHERE {target_condition} AND e.status IN ('planned', 'active') AND COALESCE(e.optimization_context_json->>'mode', 'none') = 'optimization' ORDER BY e.created_at DESC LIMIT 1 """, - (list(RECIPE_TERMINAL_STATUSES), project_id), + ( + list(RECIPE_TERMINAL_STATUSES), + *([r"^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$"] * 7), + exact_experiment_id or project_id, + ), ) row = cur.fetchone() if not row: - skipped.append({"project_id": project_id, "reason": "no_experiment"}) + skipped.append( + { + "project_id": project_id, + "experiment_id": exact_experiment_id, + "reason": "no_experiment", + } + ) continue latest_run_at = row.get("latest_run_at") @@ -4897,6 +5157,14 @@ def auto_generate_recipe_proposals_for_projects_pg( if int(row.get("latest_open_proposals") or 0) > 0: skipped.append({"project_id": project_id, "reason": "latest_batch_not_complete"}) continue + if int(row.get("latest_unresolved_results") or 0) > 0: + skipped.append( + { + "project_id": project_id, + "reason": "latest_batch_results_pending", + } + ) + continue if latest_batch_at is not None and latest_batch_at >= latest_run_at: skipped.append({"project_id": project_id, "reason": "already_current"}) continue @@ -4929,8 +5197,26 @@ def auto_generate_recipe_proposals_for_projects_pg( "experiment_id": experiment_id, "execution_request_id": row.get("execution_request_id") or "", "training_scope": "project_plus_global_history", + "is_admin": True, + "include_global_history": True, + "disable_csv_fallback": True, + "force_live_fit": True, }, ) + optimizer_batches = optimizer_result.get("batches") + if ( + not isinstance(optimizer_batches, list) + or not optimizer_batches + or not any( + isinstance(batch, dict) + and bool(batch.get("proposals")) + for batch in optimizer_batches + ) + ): + raise RuntimeError( + "Trusted Postgres training data produced no " + "optimizer proposal batch" + ) optimizer_result["auto_proposal_reason"] = reason inserted = save_experiment_proposal_batches_pg( experiment_id=experiment_id, @@ -4966,7 +5252,8 @@ def auto_generate_recipe_proposals_for_projects_pg( return { "status": "success", "generated": generated, - "projects": project_ids, + "projects": sorted(processed_projects), + "experiments": experiment_ids, "skipped": skipped, } finally: @@ -5073,6 +5360,7 @@ def list_experiment_definitions_pg( 'status', rp.status, 'ingestion_status', rp.ingestion_status, 'run_id', rp.run_id, + 'equipment_run_id', rp.equipment_run_id, 'lotname', rp.lotname, 'completed_at', rp.completed_at, 'rejected_at', rp.rejected_at, @@ -5159,7 +5447,20 @@ def list_experiment_definitions_pg( proposal["execution_request_id"] = record.get("execution_request_id") proposal["status"] = record.get("status") proposal["ingestion_status"] = record.get("ingestion_status") - proposal["run_id"] = record.get("run_id") + equipment_run_id = record.get( + "equipment_run_id" + ) + proposal["equipment_run_id"] = equipment_run_id + proposal["run_id"] = ( + record.get("run_id") + if record.get("run_id") is not None + else ( + EQUIPMENT_RUN_ID_OFFSET + + int(equipment_run_id) + if equipment_run_id is not None + else None + ) + ) proposal["lotname"] = record.get("lotname") proposal["completed_at"] = _iso(record.get("completed_at")) proposal["rejected_at"] = _iso(record.get("rejected_at")) @@ -5408,6 +5709,7 @@ def update_recipe_proposal_status_pg( """ SELECT rp.id, + rp.experiment_id::text AS experiment_id, e.owner_id, e.project_id FROM experiment_recipe_proposals rp @@ -5443,7 +5745,8 @@ def update_recipe_proposal_status_pg( status = %s, ingestion_status = CASE WHEN %s = 'completed' THEN 'approved' - WHEN %s = 'rejected' THEN 'recipe_rejected' + WHEN %s = 'rejected' + THEN 'recipe_rejected_followup_pending' ELSE ingestion_status END, rejected_at = CASE WHEN %s = 'rejected' THEN CURRENT_TIMESTAMP ELSE rejected_at END, @@ -5462,11 +5765,156 @@ def update_recipe_proposal_status_pg( result = dict(updated) result["completed_at"] = _iso(result.get("completed_at")) result["rejected_at"] = _iso(result.get("rejected_at")) + if status == "rejected": + # The pending ingestion status was committed atomically with the + # rejection, so a crash after this point cannot lose the optimizer + # wake-up. Attempt it immediately; the scheduled retrainer also + # drains any marker that remains pending. + try: + optimizer_followup = ( + drain_recipe_optimizer_wakeups_pg( + proposal_ids=[proposal_id], + ) + ) + result["optimizer_followup"] = optimizer_followup + if int(optimizer_followup.get("processed") or 0) > 0: + result["ingestion_status"] = "recipe_rejected" + except Exception as exc: + logger.warning( + "Recipe rejection committed but optimizer wake-up " + "remains pending: %s", + exc, + exc_info=True, + ) + result["optimizer_followup"] = { + "status": "pending", + "error": str(exc), + } return result finally: conn.close() +def drain_recipe_optimizer_wakeups_pg( + *, + proposal_ids: list[str] | None = None, + limit: int = 100, +) -> dict[str, Any]: + """Drain durable optimizer wake-ups left by terminal recipe rejection.""" + normalized_ids = sorted( + { + str(proposal_id or "").strip() + for proposal_id in (proposal_ids or []) + if str(proposal_id or "").strip() + } + ) + conn = get_pg_superuser_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + id_filter = ( + "AND rp.id = ANY(%s::uuid[])" + if normalized_ids + else "" + ) + params: list[Any] = [] + if normalized_ids: + params.append(normalized_ids) + params.append(max(1, min(int(limit), 1000))) + cur.execute( + f""" + SELECT + rp.id::text AS proposal_id, + rp.experiment_id::text AS experiment_id, + e.project_id + FROM experiment_recipe_proposals rp + JOIN experiment_definitions e + ON e.id = rp.experiment_id + WHERE rp.status = 'rejected' + AND rp.ingestion_status = + 'recipe_rejected_followup_pending' + {id_filter} + ORDER BY rp.updated_at, rp.id + FOR UPDATE OF rp SKIP LOCKED + LIMIT %s + """, + tuple(params), + ) + pending = [dict(row) for row in (cur.fetchall() or [])] + if not pending: + conn.commit() + return { + "status": "success", + "processed": 0, + "generated": 0, + } + + try: + followup = auto_generate_recipe_proposals_for_projects_pg( + project_ids=[ + str(row.get("project_id") or "") + for row in pending + ], + experiment_ids=[ + str(row["experiment_id"]) for row in pending + ], + reason="recipe_rejected", + ) + failed = ( + str(followup.get("status") or "").lower() == "failed" + or any( + item.get("reason") == "generation_failed" + for item in followup.get("skipped", []) + if isinstance(item, dict) + ) + ) + except Exception as exc: + logger.warning( + "Durable recipe optimizer wake-up failed: %s", + exc, + exc_info=True, + ) + conn.commit() + return { + "status": "pending", + "processed": 0, + "pending": len(pending), + "error": str(exc), + } + + if failed: + conn.commit() + return { + "status": "pending", + "processed": 0, + "pending": len(pending), + "followup": followup, + } + cur.execute( + """ + UPDATE experiment_recipe_proposals + SET + ingestion_status = 'recipe_rejected', + updated_at = CURRENT_TIMESTAMP + WHERE id = ANY(%s::uuid[]) + AND ingestion_status = + 'recipe_rejected_followup_pending' + """, + ([row["proposal_id"] for row in pending],), + ) + conn.commit() + return { + "status": "success", + "processed": len(pending), + "generated": int(followup.get("generated") or 0), + "followup": followup, + } + except Exception: + conn.rollback() + raise + finally: + conn.close() + + def ensure_data_uploads_kind_column_pg() -> None: """Add the data_uploads.kind column on databases created before split input/output uploads existed. Idempotent — safe to call on every startup.""" diff --git a/api/ml_engine.py b/api/ml_engine.py index ab89996..f74a1db 100644 --- a/api/ml_engine.py +++ b/api/ml_engine.py @@ -860,7 +860,11 @@ def compute_proposals( # during the last retrain. For proposals we still simulate multiple # iterations (to keep the UI shape), but the underlying model is the # same pre-trained artifact; we only vary y_best across iterations. - registry_bundle = _registry_model_for(primary_target, valid_features) + registry_bundle = ( + None + if scoped_context.get("force_live_fit") + else _registry_model_for(primary_target, valid_features) + ) registry_gpr = None registry_scaler = None if use_gpr and registry_bundle is not None and registry_bundle.get("model_kind") == "gpr": diff --git a/api/model_registry.py b/api/model_registry.py index f191194..b0abade 100644 --- a/api/model_registry.py +++ b/api/model_registry.py @@ -293,6 +293,62 @@ def update_retrain_state( conn.close() +def deactivate_active_models( + *, + domain_id: str, + targets: list[str], + reason: str, +) -> list[str]: + """Stop serving targets whose authoritative frame cannot train a model. + + Dataset corrections can remove a row from an active model's lineage. If + the corrected target then falls below the training minimum, leaving that + artifact active would knowingly serve excluded data. This operation is + idempotent and records the reason on the historical model row. + """ + normalized_targets = sorted( + {str(target or "").strip() for target in targets if str(target or "").strip()} + ) + if not normalized_targets: + return [] + from data_loader_pg import get_pg_superuser_connection + + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE models + SET + is_active = false, + notes = CONCAT_WS( + E'\n', + NULLIF(notes, ''), + %s + ) + WHERE domain_id = %s + AND target = ANY(%s) + AND is_active = true + RETURNING target + """, + ( + f"Deactivated: {reason}", + domain_id, + normalized_targets, + ), + ) + deactivated = sorted( + str(row[0]) for row in (cur.fetchall() or []) + ) + conn.commit() + return deactivated + except Exception: + conn.rollback() + raise + finally: + conn.close() + + def bump_runs_counter(*, domain_id: str, n_new: int) -> int: """Increment the debounce counter. Returns the new total.""" from data_loader_pg import get_pg_superuser_connection @@ -318,6 +374,97 @@ def bump_runs_counter(*, domain_id: str, n_new: int) -> int: conn.close() +def credit_runs_counter_once( + *, + domain_id: str, + credits: list[tuple[str, int]], +) -> dict[str, int]: + """Credit durable ingestion revisions to the retrain counter once.""" + normalized: dict[str, int] = {} + for credit_key, n_new in credits: + key = str(credit_key or "").strip() + count = int(n_new) + if key and count > 0: + normalized[key] = count + if not normalized: + state = get_retrain_state(domain_id) or {} + return { + "credited": 0, + "runs_since_last_train": int( + state.get("runs_since_last_train") or 0 + ), + } + + from data_loader_pg import get_pg_superuser_connection + + conn = get_pg_superuser_connection() + try: + with conn.cursor() as cur: + credited = 0 + for credit_key, n_new in sorted(normalized.items()): + cur.execute( + """ + INSERT INTO retrain_run_credits ( + domain_id, + credit_key, + n_new + ) + VALUES (%s, %s, %s) + ON CONFLICT (domain_id, credit_key) DO NOTHING + RETURNING n_new + """, + (domain_id, credit_key, n_new), + ) + row = cur.fetchone() + if row: + credited += int(row[0]) + + if credited > 0: + cur.execute( + """ + INSERT INTO retrain_state ( + domain_id, + runs_since_last_train, + updated_at + ) + VALUES (%s, %s, %s) + ON CONFLICT (domain_id) DO UPDATE SET + runs_since_last_train = + retrain_state.runs_since_last_train + + EXCLUDED.runs_since_last_train, + updated_at = EXCLUDED.updated_at + RETURNING runs_since_last_train + """, + ( + domain_id, + credited, + datetime.now(timezone.utc), + ), + ) + total = int(cur.fetchone()[0]) + else: + cur.execute( + """ + SELECT runs_since_last_train + FROM retrain_state + WHERE domain_id = %s + """, + (domain_id,), + ) + row = cur.fetchone() + total = int(row[0]) if row else 0 + conn.commit() + return { + "credited": credited, + "runs_since_last_train": total, + } + except Exception: + conn.rollback() + raise + finally: + conn.close() + + # ── Reads (api_client path) ────────────────────────────────────────────────── def load_active_bundle(*, domain_id: str, target: str) -> Optional[dict[str, Any]]: diff --git a/api/routers/dataset_v2.py b/api/routers/dataset_v2.py index 2c27519..498dd57 100644 --- a/api/routers/dataset_v2.py +++ b/api/routers/dataset_v2.py @@ -61,6 +61,14 @@ prepare_live_run, validate_batch_mapping, ) +from glance_closed_loop import ( + enrich_glance_trace_outcomes_pg, + finish_glance_followup_claims_pg, + glance_followups_succeeded, + reconcile_glance_trace_runs_pg, + retrain_credit_key_for_run, + run_glance_closed_loop_followups, +) from security import ( PlatformUser, get_platform_user, @@ -187,6 +195,21 @@ def _project_ids_from_sync_payload(runs_data: list[dict]) -> list[str]: return sorted(project_ids) +def _optional_sync_source_run_id(row: dict) -> int | None: + try: + return int(row.get("idruns")) + except (TypeError, ValueError): + return None + + +def _optional_sync_source_tool_id(row: dict) -> int | None: + value = row.get("source_tool_id", row.get("idtools")) + try: + return int(value) + except (TypeError, ValueError): + return None + + def _legacy_equipment_available_to_user( equipment_id: str, user: PlatformUser, @@ -993,8 +1016,132 @@ async def sync_glance_traces( logger.warning("Could not persist failed GLANCE audit", exc_info=True) raise HTTPException(status_code=422, detail=str(exc)) from exc - status = "partial" if rejected else "success" - cursor_acknowledged = not rejected + # Work on a shallow copy so test adapters and alternate persistence + # implementations may safely reuse their result object across retries. + stats = dict(stats) + run_records = list(stats.pop("run_records", []) or []) + try: + reconciliation = reconcile_glance_trace_runs_pg( + project_id=envelope.project_id, + run_records=run_records, + ) + except Exception as exc: + try: + record_glance_ingestion_audit_pg( + **base_audit, + status="failed", + run_count=stats["run_count"], + sample_count=stats["sample_count"], + event_count=stats["event_count"], + warning_count=len(warnings), + error_text=f"Closed-loop reconciliation failed: {exc}", + ) + except Exception: + logger.warning( + "Could not persist failed GLANCE closed-loop audit", + exc_info=True, + ) + raise HTTPException( + status_code=422, + detail={ + "message": ( + "Trace storage succeeded, but experiment reconciliation " + "failed; retry this batch without advancing the cursor" + ), + "cursor_acknowledged": False, + }, + ) from exc + + unresolved_identity = int( + reconciliation.get("unresolved_identity") or 0 + ) + followup_in_progress = int( + reconciliation.get("followup_in_progress") or 0 + ) + followup_result: dict = {} + followup_claims = list( + reconciliation.get("followup_claims") or [] + ) + if ( + int(reconciliation.get("training_rows") or 0) + + int(reconciliation.get("dataset_change_rows") or 0) + + int( + reconciliation.get("optimizer_resolution_rows") or 0 + ) + > 0 + ): + try: + followup_result = run_glance_closed_loop_followups( + reconciliation + ) + followup_succeeded = glance_followups_succeeded( + followup_result + ) + finish_glance_followup_claims_pg( + followup_claims, + succeeded=followup_succeeded, + ) + except Exception as exc: + logger.warning( + "Synchronous GLANCE closed-loop follow-up failed: %s", + exc, + exc_info=True, + ) + try: + finish_glance_followup_claims_pg( + followup_claims, + succeeded=False, + ) + except Exception: + logger.warning( + "Could not release GLANCE follow-up claims", + exc_info=True, + ) + followup_succeeded = False + if not followup_succeeded: + try: + record_glance_ingestion_audit_pg( + **base_audit, + status="failed", + run_count=stats["run_count"], + sample_count=stats["sample_count"], + event_count=stats["event_count"], + warning_count=len(warnings), + error_text="Closed-loop follow-up did not complete", + ) + except Exception: + logger.warning( + "Could not persist failed GLANCE follow-up audit", + exc_info=True, + ) + raise HTTPException( + status_code=422, + detail={ + "message": ( + "Trace storage and reconciliation succeeded, but " + "closed-loop follow-up did not complete; retry " + "without advancing the cursor" + ), + "cursor_acknowledged": False, + }, + ) + + status = ( + "partial" + if rejected or unresolved_identity or followup_in_progress + else "success" + ) + cursor_acknowledged = ( + not rejected + and unresolved_identity == 0 + and followup_in_progress == 0 + ) + if unresolved_identity: + warnings.append( + f"{unresolved_identity} run(s) supplied experiment identity that " + "could not be reconciled" + ) + try: record_glance_ingestion_audit_pg( **base_audit, @@ -1020,8 +1167,28 @@ async def sync_glance_traces( "rejected_count": len(rejected), "rejected": rejected[:100], "payload_sha256": digest, + "closed_loop": { + "checked": reconciliation.get("checked", 0), + "matched": reconciliation.get("matched", 0), + "newly_matched": reconciliation.get("newly_matched", 0), + "replayed": reconciliation.get("replayed", 0), + "unmatched": reconciliation.get("unmatched", 0), + "unresolved_identity": unresolved_identity, + "training_rows": reconciliation.get("training_rows", 0), + "dataset_change_rows": reconciliation.get( + "dataset_change_rows", + 0, + ), + "optimizer_resolution_rows": reconciliation.get( + "optimizer_resolution_rows", + 0, + ), + "followup_in_progress": followup_in_progress, + "decisions": reconciliation.get("decisions", [])[:100], + "followup": followup_result, + }, } - if rejected: + if rejected or unresolved_identity or followup_in_progress: return JSONResponse(status_code=207, content=response) return response @@ -1054,9 +1221,173 @@ async def sync_runs( inserted = sync_runs_pg(runs_data) logger.info(f"Successfully synced {inserted} runs into dt-db-v2.") + # The legacy summary handoff carries profilometry outcomes that GLANCE + # itself does not store. Merge those outcomes into any matching + # trace-backed physical run, then finish the exact experiment loop + # synchronously before acknowledging this summary batch. + trace_outcome_groups = enrich_glance_trace_outcomes_pg(runs_data) + exact_followup_source_identities: set[ + tuple[int | None, int] + ] = { + ( + _optional_sync_source_tool_id(row), + source_run_id, + ) + for row in runs_data + if str(row.get("source_system") or "").lower() + == "glance_summary" + and ( + source_run_id := _optional_sync_source_run_id(row) + ) + is not None + } + enriched_tools_by_source_run: dict[int, set[int]] = {} + for trace_project_id, trace_run_records in ( + trace_outcome_groups.items() + ): + for trace_run_record in trace_run_records: + try: + enriched_tool_id = int( + trace_run_record["source_tool_id"] + ) + enriched_source_run_id = int( + trace_run_record["source_run_id"] + ) + exact_followup_source_identities.add( + ( + enriched_tool_id, + enriched_source_run_id, + ) + ) + enriched_tools_by_source_run.setdefault( + enriched_source_run_id, + set(), + ).add(enriched_tool_id) + except (KeyError, TypeError, ValueError): + logger.warning( + "Ignoring malformed enriched GLANCE trace record: %s", + trace_run_record, + ) + trace_reconciliation = reconcile_glance_trace_runs_pg( + project_id=trace_project_id, + run_records=trace_run_records, + ) + if int( + trace_reconciliation.get("unresolved_identity") or 0 + ) > 0: + raise RuntimeError( + "A trace-backed GLANCE summary has unresolved exact " + "experiment identity" + ) + if int( + trace_reconciliation.get("followup_in_progress") or 0 + ) > 0: + raise RuntimeError( + "A GLANCE closed-loop follow-up is already in progress" + ) + claims = list( + trace_reconciliation.get("followup_claims") or [] + ) + if ( + int(trace_reconciliation.get("training_rows") or 0) + + int( + trace_reconciliation.get( + "dataset_change_rows" + ) + or 0 + ) + + int( + trace_reconciliation.get( + "optimizer_resolution_rows" + ) + or 0 + ) + > 0 + ): + try: + followup_result = run_glance_closed_loop_followups( + trace_reconciliation + ) + succeeded = glance_followups_succeeded( + followup_result + ) + finish_glance_followup_claims_pg( + claims, + succeeded=succeeded, + ) + except Exception: + try: + finish_glance_followup_claims_pg( + claims, + succeeded=False, + ) + except Exception: + logger.warning( + "Could not release summary follow-up claims", + exc_info=True, + ) + raise + if not succeeded: + raise RuntimeError( + "GLANCE closed-loop follow-up did not complete" + ) + legacy_followup_rows = [ + row + for row in runs_data + if _optional_sync_source_run_id(row) is not None + and ( + ( + _optional_sync_source_tool_id(row), + _optional_sync_source_run_id(row), + ) + not in exact_followup_source_identities + and not ( + _optional_sync_source_tool_id(row) is None + and len( + enriched_tools_by_source_run.get( + _optional_sync_source_run_id(row), + set(), + ) + ) + == 1 + ) + ) + ] + legacy_retrain_credit_keys: list[str] = [] + for row in legacy_followup_rows: + credit_key = retrain_credit_key_for_run( + { + "source_tool_id": _optional_sync_source_tool_id( + row + ), + "source_run_id": _optional_sync_source_run_id(row), + "inputs": { + feature: row.get(feature) + for feature in ( + "Etch_AvgO2Flow", + "Etch_Avg_Rf1_Pow", + "Etch_Avg_Rf2_Pow", + "Etch_AvgPres", + "Etch_Avgcf4Flow", + ) + }, + "outputs": { + "AvgEtchRate": row.get("AvgEtchRate"), + "RangeEtchRate": row.get("RangeEtchRate"), + }, + "is_outlier": bool(row.get("is_outlier")), + "is_calibration_recipe": bool( + row.get("is_calibration_recipe") + ), + } + ) + if credit_key: + legacy_retrain_credit_keys.append(credit_key) + legacy_retrain_count = len(legacy_retrain_credit_keys) + try: reconciliation = reconcile_ingested_runs_with_recipe_proposals_pg( - runs_data, + legacy_followup_rows, project_resolver=_project_id_from_sync_row, ) logger.info("Recipe/run reconciliation result: %s", reconciliation) @@ -1069,7 +1400,9 @@ async def sync_runs( # runs BackgroundTasks in insertion order, and researchers should see # closed-loop recipe suggestions even if later maintenance is slow. if os.getenv("AUTO_RECIPE_PROPOSALS_ENABLED", "true").lower() != "false": - project_ids = _project_ids_from_sync_payload(runs_data) + project_ids = _project_ids_from_sync_payload( + legacy_followup_rows + ) def _auto_propose_after_sync(): try: @@ -1108,11 +1441,18 @@ def _auto_propose_after_sync(): except Exception as exc: logger.warning("Auto recipe proposal task failed: %s", exc, exc_info=True) - background_tasks.add_task(_auto_propose_after_sync) - logger.info( - "Auto recipe proposal background task queued for projects: %s", - project_ids, - ) + if project_ids: + background_tasks.add_task(_auto_propose_after_sync) + logger.info( + "Auto recipe proposal background task queued for " + "projects: %s", + project_ids, + ) + else: + logger.info( + "Legacy auto proposal task skipped; the exact GLANCE " + "experiment loop handled this batch" + ) else: logger.info("Auto recipe proposals disabled via AUTO_RECIPE_PROPOSALS_ENABLED=false") @@ -1158,10 +1498,13 @@ def _snapshot_after_sync(): # AUTO_RETRAIN_MAX_STALENESS_HOURS have elapsed, OR no active model # exists yet. Otherwise we just bump the counter. This protects us # from retraining on every single row during a bulk Azure sync. - if os.getenv("AUTO_RETRAIN_ENABLED", "true").lower() != "false": + if ( + legacy_retrain_count > 0 + and os.getenv("AUTO_RETRAIN_ENABLED", "true").lower() != "false" + ): try: from model_registry import ( - bump_runs_counter, + credit_runs_counter_once, get_retrain_state, load_active_bundle, ) @@ -1175,8 +1518,19 @@ def _maybe_retrain_after_sync(): )) domain_id = os.getenv("AUTO_RETRAIN_DOMAIN", "etcher") - total_pending = bump_runs_counter( - domain_id=domain_id, n_new=inserted, + credit_result = credit_runs_counter_once( + domain_id=domain_id, + credits=[ + (credit_key, 1) + for credit_key + in legacy_retrain_credit_keys + ], + ) + total_pending = int( + credit_result.get( + "runs_since_last_train" + ) + or 0 ) state = get_retrain_state(domain_id) or {} has_active = load_active_bundle( @@ -1226,7 +1580,16 @@ def _maybe_retrain_after_sync(): except ImportError as exc: logger.debug("Auto-retrain not available: %s", exc) else: - logger.info("Auto-retrain disabled via AUTO_RETRAIN_ENABLED=false") + logger.info( + "Legacy auto-retrain skipped (remaining rows=%d, " + "enabled=%s)", + legacy_retrain_count, + os.getenv( + "AUTO_RETRAIN_ENABLED", + "true", + ).lower() + != "false", + ) return {"status": "success", "inserted": inserted} except Exception as e: diff --git a/api/scripts/add_equipment_runs.sql b/api/scripts/add_equipment_runs.sql index 20f46b4..061cdbd 100644 --- a/api/scripts/add_equipment_runs.sql +++ b/api/scripts/add_equipment_runs.sql @@ -27,6 +27,25 @@ ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS row_index INT; ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_system VARCHAR(100); ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_tool_id BIGINT; ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_run_id BIGINT; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS execution_request_id VARCHAR(80) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_revision_sha256 VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_revision_sha256 + VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_revision_sha256 + VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_kind + VARCHAR(32) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_token + VARCHAR(36) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claimed_at + TIMESTAMP WITH TIME ZONE; ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_updated_at TIMESTAMP WITH TIME ZONE; ALTER TABLE equipment_runs @@ -42,6 +61,9 @@ ALTER TABLE equipment_runs ALTER COLUMN source_system SET NOT NULL; CREATE INDEX IF NOT EXISTS idx_equipment_runs_equipment_id ON equipment_runs(equipment_id); CREATE INDEX IF NOT EXISTS idx_equipment_runs_project_id ON equipment_runs(project_id); CREATE INDEX IF NOT EXISTS idx_equipment_runs_upload_id ON equipment_runs(upload_id); +CREATE INDEX IF NOT EXISTS idx_equipment_runs_execution_request_id + ON equipment_runs(execution_request_id) + WHERE execution_request_id <> ''; CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row ON equipment_runs(upload_id, row_index) WHERE upload_id IS NOT NULL; diff --git a/api/scripts/add_experiment_recipe_batches.sql b/api/scripts/add_experiment_recipe_batches.sql index 3427142..1e734f4 100644 --- a/api/scripts/add_experiment_recipe_batches.sql +++ b/api/scripts/add_experiment_recipe_batches.sql @@ -26,4 +26,30 @@ CREATE TABLE IF NOT EXISTS experiment_recipe_batches ( CREATE INDEX IF NOT EXISTS idx_experiment_recipe_batches_experiment_id ON experiment_recipe_batches(experiment_id); +WITH duplicate_experiments AS ( + SELECT experiment_id + FROM experiment_recipe_batches + GROUP BY experiment_id + HAVING COUNT(*) > COUNT(DISTINCT iteration) +), +renumbered AS ( + SELECT + b.id, + ROW_NUMBER() OVER ( + PARTITION BY b.experiment_id + ORDER BY b.iteration, b.created_at, b.id + ) AS normalized_iteration + FROM experiment_recipe_batches b + JOIN duplicate_experiments d + ON d.experiment_id = b.experiment_id +) +UPDATE experiment_recipe_batches b +SET iteration = r.normalized_iteration +FROM renumbered r +WHERE b.id = r.id + AND b.iteration <> r.normalized_iteration; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_batches_iteration +ON experiment_recipe_batches(experiment_id, iteration); + GRANT SELECT, INSERT, UPDATE, DELETE ON experiment_recipe_batches TO api_client; diff --git a/api/scripts/add_glance_closed_loop.sql b/api/scripts/add_glance_closed_loop.sql new file mode 100644 index 0000000..1c48dc4 --- /dev/null +++ b/api/scripts/add_glance_closed_loop.sql @@ -0,0 +1,74 @@ +-- Link complete production GLANCE traces to experiment recipe proposals. +-- Safe to run multiple times after equipment_runs and proposal tables exist. + +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS execution_request_id VARCHAR(80) NOT NULL DEFAULT ''; + +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_revision_sha256 VARCHAR(64) NOT NULL DEFAULT ''; + +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_revision_sha256 + VARCHAR(64) NOT NULL DEFAULT ''; + +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_revision_sha256 + VARCHAR(64) NOT NULL DEFAULT ''; + +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_kind + VARCHAR(32) NOT NULL DEFAULT ''; + +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_token + VARCHAR(36) NOT NULL DEFAULT ''; + +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claimed_at + TIMESTAMP WITH TIME ZONE; + +CREATE INDEX IF NOT EXISTS idx_equipment_runs_execution_request_id +ON equipment_runs(execution_request_id) +WHERE execution_request_id <> ''; + +ALTER TABLE experiment_recipe_proposals + ADD COLUMN IF NOT EXISTS equipment_run_id BIGINT + REFERENCES equipment_runs(id) ON DELETE SET NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_proposals_equipment_run_id +ON experiment_recipe_proposals(equipment_run_id) +WHERE equipment_run_id IS NOT NULL; + +WITH duplicate_experiments AS ( + SELECT experiment_id + FROM experiment_recipe_batches + GROUP BY experiment_id + HAVING COUNT(*) > COUNT(DISTINCT iteration) +), +renumbered AS ( + SELECT + b.id, + ROW_NUMBER() OVER ( + PARTITION BY b.experiment_id + ORDER BY b.iteration, b.created_at, b.id + ) AS normalized_iteration + FROM experiment_recipe_batches b + JOIN duplicate_experiments d + ON d.experiment_id = b.experiment_id +) +UPDATE experiment_recipe_batches b +SET iteration = r.normalized_iteration +FROM renumbered r +WHERE b.id = r.id + AND b.iteration <> r.normalized_iteration; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_batches_iteration +ON experiment_recipe_batches(experiment_id, iteration); + +CREATE TABLE IF NOT EXISTS retrain_run_credits ( + domain_id VARCHAR(100) NOT NULL, + credit_key VARCHAR(255) NOT NULL, + n_new INTEGER NOT NULL DEFAULT 1 CHECK (n_new > 0), + credited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (domain_id, credit_key) +); diff --git a/api/scripts/add_production_glance_ingestion.sql b/api/scripts/add_production_glance_ingestion.sql index 6b7b9d0..bebf56e 100644 --- a/api/scripts/add_production_glance_ingestion.sql +++ b/api/scripts/add_production_glance_ingestion.sql @@ -7,6 +7,25 @@ ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_tool_id BIGINT; ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_run_id BIGINT; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS execution_request_id VARCHAR(80) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_revision_sha256 VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_revision_sha256 + VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_revision_sha256 + VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_kind + VARCHAR(32) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claim_token + VARCHAR(36) NOT NULL DEFAULT ''; +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS closed_loop_followup_claimed_at + TIMESTAMP WITH TIME ZONE; ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS source_updated_at TIMESTAMP WITH TIME ZONE; ALTER TABLE equipment_runs @@ -28,6 +47,10 @@ CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_source_identity ) WHERE source_tool_id IS NOT NULL AND source_run_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_equipment_runs_execution_request_id + ON equipment_runs(execution_request_id) + WHERE execution_request_id <> ''; + ALTER TABLE IF EXISTS equipment_run_trace_samples ALTER COLUMN sample_time DROP NOT NULL; ALTER TABLE IF EXISTS equipment_run_trace_samples diff --git a/api/scripts/init_db.sql b/api/scripts/init_db.sql index f2a1804..5151a2a 100644 --- a/api/scripts/init_db.sql +++ b/api/scripts/init_db.sql @@ -422,6 +422,32 @@ CREATE TABLE IF NOT EXISTS experiment_recipe_batches ( CREATE INDEX IF NOT EXISTS idx_experiment_recipe_batches_experiment_id ON experiment_recipe_batches(experiment_id); +WITH duplicate_experiments AS ( + SELECT experiment_id + FROM experiment_recipe_batches + GROUP BY experiment_id + HAVING COUNT(*) > COUNT(DISTINCT iteration) +), +renumbered AS ( + SELECT + b.id, + ROW_NUMBER() OVER ( + PARTITION BY b.experiment_id + ORDER BY b.iteration, b.created_at, b.id + ) AS normalized_iteration + FROM experiment_recipe_batches b + JOIN duplicate_experiments d + ON d.experiment_id = b.experiment_id +) +UPDATE experiment_recipe_batches b +SET iteration = r.normalized_iteration +FROM renumbered r +WHERE b.id = r.id + AND b.iteration <> r.normalized_iteration; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_batches_iteration +ON experiment_recipe_batches(experiment_id, iteration); + CREATE TABLE IF NOT EXISTS experiment_recipe_proposals ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), batch_id UUID NOT NULL REFERENCES experiment_recipe_batches(id) ON DELETE CASCADE, @@ -442,6 +468,14 @@ CREATE TABLE IF NOT EXISTS experiment_recipe_proposals ( UNIQUE(batch_id, proposal_index) ); +CREATE TABLE IF NOT EXISTS retrain_run_credits ( + domain_id VARCHAR(100) NOT NULL, + credit_key VARCHAR(255) NOT NULL, + n_new INTEGER NOT NULL DEFAULT 1 CHECK (n_new > 0), + credited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (domain_id, credit_key) +); + CREATE INDEX IF NOT EXISTS idx_experiment_recipe_proposals_batch_id ON experiment_recipe_proposals(batch_id); diff --git a/api/tests/test_etcher_realtime_smoke.py b/api/tests/test_etcher_realtime_smoke.py index e8fad97..2a72ae9 100644 --- a/api/tests/test_etcher_realtime_smoke.py +++ b/api/tests/test_etcher_realtime_smoke.py @@ -31,7 +31,12 @@ } RUNS_HEADERS = { - "Authorization": "Bearer smoke-user", + "X-System-Token": os.environ["DT_SYSTEM_TOKEN"], + "X-User-Id": "smoke-user", + "X-User-Email": "smoke@example.com", + "X-User-Name": "Smoke User", + "X-User-Role": "researcher", + "X-User-Org": "Birck", } @@ -76,6 +81,9 @@ def fake_get_runs_list_pg( include_outliers=False, limit=200, offset=0, + is_admin=False, + project_id=None, + include_raw_payload=False, ): rows = synced_rows if include_outliers else [ row @@ -104,17 +112,56 @@ def fake_get_runs_list_pg( for row in sliced ] - def fake_pg_loader(): + def fake_pg_loader(*args, **kwargs): if not synced_rows: return pd.DataFrame() frame = pd.DataFrame(deepcopy(synced_rows)) frame.loc[:, "run_date"] = pd.to_datetime(frame["run_date"], utc=True) return frame - with patch.object(dataset_v2, "sync_runs_pg", side_effect=fake_sync_runs_pg), \ - patch.object(dataset_v2, "get_runs_list_pg", side_effect=fake_get_runs_list_pg), \ - patch.object(ml, "has_production_db", return_value=False), \ - patch.object(ml_engine, "_pg_loader", side_effect=fake_pg_loader): + with patch.dict( + os.environ, + { + "AUTO_RECIPE_PROPOSALS_ENABLED": "false", + "AUTO_RETRAIN_ENABLED": "false", + }, + clear=False, + ), patch.object( + dataset_v2, + "sync_runs_pg", + side_effect=fake_sync_runs_pg, + ), patch.object( + dataset_v2, + "enrich_glance_trace_outcomes_pg", + return_value={}, + ), patch.object( + dataset_v2, + "reconcile_ingested_runs_with_recipe_proposals_pg", + return_value={"matched": 0, "checked": 6}, + ), patch.object( + dataset_v2, + "get_runs_list_pg", + side_effect=fake_get_runs_list_pg, + ), patch( + "fair_archiver.archive_ingestion_batch", + return_value={"status": "skipped"}, + ), patch( + "ai_readiness.export_ml_matrix", + return_value={ + "snapshot_hash": "smoke-snapshot", + "num_rows": 6, + }, + ), patch( + "ai_readiness.record_dataset_snapshot", + ), patch.object( + ml, + "has_production_db", + return_value=False, + ), patch.object( + ml_engine, + "_pg_loader", + side_effect=fake_pg_loader, + ): sync_response = self.client.post( "/api/dataset/v2/runs/sync", headers=INGESTION_HEADERS, @@ -132,10 +179,13 @@ def fake_pg_loader(): self.assertEqual(len(runs_payload), 3) self.assertEqual(runs_payload[0]["lot_name"], "etch04/15/2026_smoke_0") - proposals_response = self.client.get("/api/ml/proposals") + proposals_response = self.client.get( + "/api/ml/proposals", + headers=RUNS_HEADERS, + ) self.assertEqual(proposals_response.status_code, 200) proposals_payload = proposals_response.json() - self.assertEqual(proposals_payload["source"], "local_gpr") + self.assertEqual(proposals_payload["source"], "legacy_nanohub") self.assertGreaterEqual(proposals_payload["total_iterations"], 1) self.assertGreaterEqual(len(proposals_payload["batches"]), 1) self.assertGreaterEqual( diff --git a/api/tests/test_glance_closed_loop.py b/api/tests/test_glance_closed_loop.py new file mode 100644 index 0000000..e8b72f1 --- /dev/null +++ b/api/tests/test_glance_closed_loop.py @@ -0,0 +1,3022 @@ +from __future__ import annotations + +import os +import sys +import unittest +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import numpy as np +import pandas as pd +from fastapi import FastAPI +from fastapi.testclient import TestClient + +API_ROOT = Path(__file__).resolve().parents[1] +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +from glance_closed_loop import ( # noqa: E402 + _candidate_rows, + choose_proposal_candidate, + derive_trace_feature_values, + enrich_glance_trace_outcomes_pg, + finish_glance_followup_claims_pg, + glance_followups_succeeded, + reconcile_glance_trace_runs_pg, + retrain_credit_key_for_run, + run_glance_closed_loop_followups, + training_values_for_run, +) +from data_loader_pg import ( # noqa: E402 + _merge_canonical_summary_into_trace_record, + equipment_training_revision_sha256, + get_run_detail_pg, + get_runs_list_pg, + get_training_df_pg, +) +from routers import dataset_v2 # noqa: E402 +from routers.dataset_v2 import router as dataset_v2_router # noqa: E402 +from metadata_pg import ( # noqa: E402 + auto_generate_recipe_proposals_for_projects_pg, + save_experiment_proposal_batches_pg, + update_recipe_proposal_status_pg, +) +from model_registry import credit_runs_counter_once # noqa: E402 +import ml_engine # noqa: E402 +from training import retrain # noqa: E402 + + +FEATURE_VALUES = { + "Etch_AvgO2Flow": 12.0, + "Etch_Avg_Rf1_Pow": 100.0, + "Etch_Avg_Rf2_Pow": 50.0, + "Etch_AvgPres": 25.0, + "Etch_Avgcf4Flow": 30.0, +} +PROPOSAL_ID = "11111111-1111-4111-8111-111111111111" +REQUEST_ID = "REQ-ABCDEF123456" + + +class GlanceClosedLoopSelectionTests(unittest.TestCase): + def test_summary_and_trace_share_same_retrain_credit_identity(self): + trace_key = retrain_credit_key_for_run( + { + "source_tool_id": 3, + "source_run_id": 161, + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "is_outlier": False, + "is_calibration_recipe": False, + } + ) + summary_key = retrain_credit_key_for_run( + { + "source_tool_id": 3, + "source_run_id": 161, + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "is_outlier": False, + "is_calibration_recipe": False, + } + ) + + self.assertEqual(trace_key, summary_key) + other_tool_key = retrain_credit_key_for_run( + { + "source_tool_id": 4, + "source_run_id": 161, + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "is_outlier": False, + "is_calibration_recipe": False, + } + ) + self.assertNotEqual(trace_key, other_tool_key) + + def test_trace_means_map_registered_names_to_training_features(self): + run = { + "inputs": {}, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "parameters": [ + { + "key": f"p{index}", + "registered_name": feature, + } + for index, feature in enumerate(FEATURE_VALUES, start=1) + ], + "samples": [ + { + "values": { + f"p{index}": value - 1 + for index, value in enumerate( + FEATURE_VALUES.values(), + start=1, + ) + } + }, + { + "values": { + f"p{index}": value + 1 + for index, value in enumerate( + FEATURE_VALUES.values(), + start=1, + ) + } + }, + ], + } + + derived = derive_trace_feature_values(run) + inputs, outputs, eligible = training_values_for_run(run) + + self.assertEqual(inputs, FEATURE_VALUES) + self.assertEqual(derived["Etch_AvgPres"], 25.0) + self.assertEqual(outputs["AvgEtchRate"], 101.5) + self.assertTrue(eligible) + + def test_ambiguous_parameter_match_is_not_guessed(self): + run = {"inputs": FEATURE_VALUES} + candidates = [ + { + "id": "one", + "status": "pending", + "proposal_json": {"parameters": FEATURE_VALUES}, + }, + { + "id": "two", + "status": "pending", + "proposal_json": {"parameters": FEATURE_VALUES}, + }, + ] + + selected, reason = choose_proposal_candidate(candidates, run) + + self.assertIsNone(selected) + self.assertEqual(reason, "ambiguous_parameter_match") + + def test_explicit_proposal_id_wins_without_fuzzy_matching(self): + run = { + "proposal_id": PROPOSAL_ID, + "inputs": {}, + "samples": [], + } + candidates = [ + { + "id": PROPOSAL_ID, + "status": "accepted", + "proposal_json": {"parameters": FEATURE_VALUES}, + } + ] + + selected, reason = choose_proposal_candidate(candidates, run) + + self.assertEqual(selected["id"], PROPOSAL_ID) + self.assertEqual(reason, "explicit_proposal_id") + + def test_explicit_identity_is_checked_before_wrong_legacy_link(self): + statements: list[str] = [] + correct = { + "id": PROPOSAL_ID, + "experiment_id": ( + "22222222-2222-4222-8222-222222222222" + ), + "execution_request_id": REQUEST_ID, + "status": "accepted", + "proposal_json": {"parameters": FEATURE_VALUES}, + "run_id": None, + "equipment_run_id": None, + } + + class Cursor: + def __init__(self): + self.rows = [] + + def execute(self, query, params=None): + text = str(query) + statements.append(text) + if "rp.id = %s" in text: + self.rows = [correct] + elif "rp.equipment_run_id = %s" in text: + self.rows = [] + else: + raise AssertionError( + "legacy run link was consulted before exact identity" + ) + + def fetchall(self): + return self.rows + + candidates = _candidate_rows( + Cursor(), + project_id="project-test", + run_record={ + "equipment_run_id": 7001, + "source_run_id": 161, + "proposal_id": PROPOSAL_ID, + "execution_request_id": REQUEST_ID, + }, + ) + + self.assertEqual(candidates, [correct]) + self.assertIn("rp.id = %s", statements[0]) + self.assertFalse( + any( + "ORDER BY rp.updated_at DESC" in statement + for statement in statements + ) + ) + + +class GlanceClosedLoopPersistenceTests(unittest.TestCase): + @staticmethod + def _run_record(): + return { + "equipment_run_id": 7001, + "source_tool_id": 3, + "source_run_id": 161, + "execution_request_id": REQUEST_ID, + "proposal_id": PROPOSAL_ID, + "lotname": REQUEST_ID, + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "parameters": [], + "samples": [{"values": {}}], + "is_outlier": False, + "is_calibration_recipe": False, + "closed_loop_revision_sha256": "a" * 64, + } + + def _connection( + self, + *, + linked: bool, + status: str | None = None, + legacy_run_id: int | None = None, + previous_revision: str | None = None, + completed_followup_revision: str | None = None, + attempted_followup_revision: str | None = None, + attempted_followup_kind: str = "", + ): + statements: list[str] = [] + revision = equipment_training_revision_sha256( + equipment_run_id=7001, + inputs=FEATURE_VALUES, + outputs={ + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + is_outlier=False, + is_calibration_recipe=False, + ) + candidate = { + "id": PROPOSAL_ID, + "experiment_id": "22222222-2222-4222-8222-222222222222", + "project_id": "project-test", + "execution_request_id": REQUEST_ID, + "status": status or ("completed" if linked else "accepted"), + "proposal_json": {"parameters": FEATURE_VALUES}, + "run_id": legacy_run_id, + "equipment_run_id": 7001 if linked else None, + } + + class Cursor: + def __init__(self): + self.rows = [] + self.rowcount = -1 + + def execute(self, query, params=None): + text = str(query) + statements.append(text) + self.rowcount = -1 + if ( + "FROM experiment_recipe_proposals rp" in text + and "rp.id = %s" in text + ): + self.rows = [candidate] + elif ( + "FROM experiment_recipe_proposals rp" in text + and "rp.equipment_run_id = %s" in text + ): + self.rows = [candidate] if linked else [] + elif ( + "FROM experiment_recipe_proposals rp" in text + and "rp.run_id = %s" in text + ): + self.rows = ( + [candidate] + if legacy_run_id is not None + else [] + ) + elif ( + "closed_loop_revision_sha256" in text + and "FROM equipment_runs" in text + ): + self.rows = [ + { + "closed_loop_revision_sha256": revision, + "closed_loop_followup_revision_sha256": ( + completed_followup_revision + if completed_followup_revision is not None + else (revision if linked else "") + ), + "closed_loop_followup_claim_revision_sha256": "", + "closed_loop_followup_claim_kind": ( + attempted_followup_kind + ), + "closed_loop_followup_claim_token": "", + "closed_loop_followup_claimed_at": None, + } + ] + if attempted_followup_revision is not None: + self.rows[0][ + "closed_loop_followup_claim_revision_sha256" + ] = attempted_followup_revision + if previous_revision is not None: + self.rows[0][ + "closed_loop_revision_sha256" + ] = previous_revision + else: + self.rows = [] + if "UPDATE experiment_recipe_proposals" in text: + self.rowcount = 1 + + def fetchall(self): + return self.rows + + def fetchone(self): + return self.rows[0] if self.rows else None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def __init__(self): + self.committed = False + self.rolled_back = False + + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + self.committed = True + + def rollback(self): + self.rolled_back = True + + def close(self): + pass + + return Connection(), statements + + def test_database_reconciliation_and_replay_are_idempotent(self): + first_connection, first_statements = self._connection(linked=False) + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=first_connection, + ): + first = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[self._run_record()], + ) + + self.assertEqual(first["newly_matched"], 1) + self.assertEqual(first["training_rows"], 1) + self.assertTrue(first_connection.committed) + self.assertTrue( + any( + "UPDATE experiment_recipe_proposals" in statement + for statement in first_statements + ) + ) + + replay_connection, _ = self._connection(linked=True) + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=replay_connection, + ): + replay = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[self._run_record()], + ) + + self.assertEqual(replay["replayed"], 1) + self.assertEqual(replay["training_rows"], 0) + self.assertTrue(replay_connection.committed) + + def test_replay_preserves_human_rejection(self): + connection, statements = self._connection( + linked=True, + status="rejected", + ) + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ): + result = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[self._run_record()], + ) + + self.assertEqual( + result["decisions"][0]["status"], + "rejected_preserved", + ) + self.assertFalse( + any( + "UPDATE experiment_recipe_proposals" in statement + for statement in statements + ) + ) + + def test_later_exact_identity_replaces_wrong_link_and_preserves_rejection( + self, + ): + correct_id = PROPOSAL_ID + wrong_id = "33333333-3333-4333-8333-333333333333" + proposals = { + wrong_id: { + "id": wrong_id, + "experiment_id": ( + "44444444-4444-4444-8444-444444444444" + ), + "project_id": "project-test", + "execution_request_id": "", + "status": "rejected", + "ingestion_status": "recipe_rejected", + "comment": "human rejection", + "proposal_json": {"parameters": FEATURE_VALUES}, + "run_id": None, + "equipment_run_id": 7001, + }, + correct_id: { + "id": correct_id, + "experiment_id": ( + "22222222-2222-4222-8222-222222222222" + ), + "project_id": "project-test", + "execution_request_id": REQUEST_ID, + "status": "accepted", + "ingestion_status": "waiting", + "comment": "", + "proposal_json": {"parameters": FEATURE_VALUES}, + "run_id": None, + "equipment_run_id": None, + }, + } + revision = equipment_training_revision_sha256( + equipment_run_id=7001, + inputs=FEATURE_VALUES, + outputs={ + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + is_outlier=False, + is_calibration_recipe=False, + ) + + class Cursor: + def __init__(self): + self.rows = [] + self.rowcount = -1 + + def execute(self, query, params=None): + text = str(query) + self.rows = [] + self.rowcount = -1 + if ( + "FROM experiment_recipe_proposals rp" in text + and "rp.id = %s" in text + ): + self.rows = [dict(proposals[correct_id])] + elif "WITH detached AS" in text: + wrong = proposals[wrong_id] + wrong["equipment_run_id"] = None + self.rowcount = 1 + elif ( + "closed_loop_revision_sha256" in text + and "FROM equipment_runs" in text + ): + self.rows = [ + { + "closed_loop_revision_sha256": "", + "closed_loop_followup_revision_sha256": "", + "closed_loop_followup_claim_revision_sha256": "", + "closed_loop_followup_claim_token": "", + "closed_loop_followup_claimed_at": None, + } + ] + elif ( + "UPDATE experiment_recipe_proposals" in text + and "status = 'completed'" in text + and "WHERE id = %s" in text + ): + correct = proposals[correct_id] + correct["status"] = "completed" + correct["ingestion_status"] = "approved" + correct["equipment_run_id"] = 7001 + self.rowcount = 1 + + def fetchall(self): + return self.rows + + def fetchone(self): + return self.rows[0] if self.rows else None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + pass + + def rollback(self): + pass + + def close(self): + pass + + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=Connection(), + ): + result = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[self._run_record()], + ) + + self.assertEqual(result["newly_matched"], 1) + self.assertEqual(proposals[correct_id]["equipment_run_id"], 7001) + self.assertEqual(proposals[correct_id]["status"], "completed") + self.assertIsNone(proposals[wrong_id]["equipment_run_id"]) + self.assertEqual(proposals[wrong_id]["status"], "rejected") + self.assertEqual( + proposals[wrong_id]["comment"], + "human rejection", + ) + + def test_summary_first_completed_proposal_moves_to_exact_trace_followup( + self, + ): + connection, statements = self._connection( + linked=False, + status="completed", + legacy_run_id=161, + ) + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ): + result = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[self._run_record()], + ) + + self.assertEqual(result["newly_matched"], 1) + self.assertEqual(result["training_rows"], 1) + self.assertTrue( + result["decisions"][0]["legacy_summary_link"] + ) + self.assertTrue( + any( + "run_id = NULL" in statement + for statement in statements + if "UPDATE experiment_recipe_proposals" in statement + ) + ) + conflict_repair = next( + statement + for statement in statements + if "ingestion_status = 'identity_conflict'" in statement + ) + self.assertIn("AND status = 'completed'", conflict_repair) + self.assertIn( + "AND ingestion_status = 'approved'", + conflict_repair, + ) + + def test_stale_followup_token_cannot_finalize_new_owner_claim(self): + class Cursor: + rowcount = 0 + + def execute(self, query, params=None): + self.rowcount = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def __init__(self): + self.committed = False + self.rolled_back = False + + def cursor(self): + return Cursor() + + def commit(self): + self.committed = True + + def rollback(self): + self.rolled_back = True + + def close(self): + pass + + connection = Connection() + with ( + patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ), + self.assertRaisesRegex( + RuntimeError, + "claim ownership was lost", + ), + ): + finish_glance_followup_claims_pg( + [ + { + "equipment_run_id": 7001, + "revision_sha256": "f" * 64, + "claim_token": ( + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + ), + } + ], + succeeded=True, + ) + + self.assertFalse(connection.committed) + self.assertTrue(connection.rolled_back) + + def test_eligible_run_corrected_to_outlier_forces_dataset_refresh(self): + previous_revision = "1" * 64 + connection, _ = self._connection( + linked=True, + previous_revision=previous_revision, + completed_followup_revision=previous_revision, + ) + corrected = { + **self._run_record(), + "is_outlier": True, + } + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ): + result = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[corrected], + ) + + self.assertEqual(result["training_rows"], 0) + self.assertEqual(result["dataset_change_rows"], 1) + self.assertEqual(len(result["followup_claims"]), 1) + self.assertIsNone( + result["followup_claims"][0]["retrain_credit_key"] + ) + + def test_failed_attempt_then_source_correction_forces_refresh(self): + attempted_revision = "8" * 64 + for correction in ( + {"is_outlier": True}, + {"outputs": {}}, + ): + with self.subTest(correction=correction): + connection, _ = self._connection( + linked=True, + previous_revision=attempted_revision, + completed_followup_revision="", + attempted_followup_revision=attempted_revision, + ) + corrected = { + **self._run_record(), + **correction, + } + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ): + result = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[corrected], + ) + + self.assertEqual(result["training_rows"], 0) + self.assertEqual(result["dataset_change_rows"], 1) + self.assertEqual( + result["followup_claims"][0]["followup_kind"], + "dataset_change", + ) + self.assertIsNone( + result["followup_claims"][0][ + "retrain_credit_key" + ] + ) + + def test_failed_dataset_correction_retries_same_revision(self): + corrections = ( + {"is_outlier": True}, + {"outputs": {}}, + ) + for correction in corrections: + with self.subTest(correction=correction): + corrected = { + **self._run_record(), + **correction, + } + revision = equipment_training_revision_sha256( + equipment_run_id=7001, + inputs=FEATURE_VALUES, + outputs=dict(corrected.get("outputs") or {}), + is_outlier=bool(corrected.get("is_outlier")), + is_calibration_recipe=False, + ) + connection, _ = self._connection( + linked=True, + previous_revision=revision, + completed_followup_revision="", + attempted_followup_revision=revision, + attempted_followup_kind="dataset_change", + ) + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ): + result = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[corrected], + ) + + self.assertEqual(result["training_rows"], 0) + self.assertEqual(result["dataset_change_rows"], 1) + self.assertEqual( + result["followup_claims"][0]["followup_kind"], + "dataset_change", + ) + + def test_first_outlier_result_wakes_optimizer_without_retrain_credit(self): + connection, _ = self._connection(linked=False) + outlier_record = { + **self._run_record(), + "is_outlier": True, + } + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ): + reconciliation = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[outlier_record], + ) + + self.assertEqual(reconciliation["training_rows"], 0) + self.assertEqual(reconciliation["dataset_change_rows"], 0) + self.assertEqual( + reconciliation["optimizer_resolution_rows"], + 1, + ) + self.assertEqual( + reconciliation["followup_claims"][0]["followup_kind"], + "optimizer_resolution", + ) + self.assertIsNone( + reconciliation["followup_claims"][0][ + "retrain_credit_key" + ] + ) + + with ( + patch( + "metadata_pg.auto_generate_recipe_proposals_for_projects_pg", + return_value={"status": "success", "generated": 1}, + ) as auto_generate, + patch( + "glance_closed_loop._maybe_retrain", + return_value={ + "status": "skipped", + "reason": "no_training_rows", + }, + ) as maybe_retrain, + ): + followup = run_glance_closed_loop_followups( + reconciliation + ) + + auto_generate.assert_called_once_with( + project_ids=["project-test"], + experiment_ids=[ + "22222222-2222-4222-8222-222222222222" + ], + reason="glance_ingestion", + ) + maybe_retrain.assert_called_once_with( + 0, + credit_keys=[], + force_dataset_refresh=False, + ) + self.assertEqual(followup["proposals"]["generated"], 1) + + def test_excluded_run_restored_to_eligible_forces_snapshot_refresh(self): + excluded_revision = equipment_training_revision_sha256( + equipment_run_id=7001, + inputs=FEATURE_VALUES, + outputs={ + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + is_outlier=True, + is_calibration_recipe=False, + ) + connection, _ = self._connection( + linked=True, + previous_revision=excluded_revision, + completed_followup_revision=excluded_revision, + ) + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ): + restored = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[self._run_record()], + ) + + self.assertEqual(restored["training_rows"], 0) + self.assertEqual(restored["dataset_change_rows"], 1) + self.assertEqual( + restored["followup_claims"][0]["followup_kind"], + "dataset_change", + ) + self.assertIsNone( + restored["followup_claims"][0]["retrain_credit_key"] + ) + + with patch( + "glance_closed_loop._maybe_retrain", + return_value={"status": "trained"}, + ) as retrain: + run_glance_closed_loop_followups(restored) + + retrain.assert_called_once_with( + 0, + credit_keys=[], + force_dataset_refresh=True, + ) + + def test_training_query_unions_complete_glance_runs(self): + captured: dict[str, object] = {} + + class Connection: + def close(self): + pass + + frame = pd.DataFrame( + [ + { + "idruns": 5_000_007_001, + "LOTNAME": REQUEST_ID, + "run_date": "2026-07-28", + "is_outlier": False, + "is_calibration_recipe": False, + **FEATURE_VALUES, + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + "Range_nm": 20.0, + } + ] + ) + + def read_sql(query, connection, params=None): + captured["query"] = query + captured["params"] = params + return frame + + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=Connection(), + ), + patch( + "data_loader_pg.pd.read_sql_query", + side_effect=read_sql, + ), + ): + result = get_training_df_pg( + project_id="project-test", + include_global_history=False, + ) + + self.assertEqual(len(result), 1) + self.assertIn("FROM equipment_runs e", captured["query"]) + self.assertIn("e.source_system = 'glance'", captured["query"]) + self.assertIn("5000000000 + e.id", captured["query"]) + self.assertIn( + "same_source.source_run_id = r.idruns", + captured["query"], + ) + self.assertIn( + "<> 'glance_summary'", + captured["query"], + ) + self.assertEqual(captured["params"], ["project-test"]) + + def test_successful_zero_row_training_query_returns_empty_frame(self): + class Connection: + def close(self): + pass + + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=Connection(), + ), + patch( + "data_loader_pg.pd.read_sql_query", + return_value=pd.DataFrame(), + ), + ): + result = get_training_df_pg() + + self.assertIsInstance(result, pd.DataFrame) + self.assertTrue(result.empty) + + def test_experiment_training_keeps_exact_dual_ingested_trace(self): + captured: dict[str, object] = {} + + class Connection: + def close(self): + pass + + def read_sql(query, connection, params=None): + captured["query"] = query + captured["params"] = params + trace_features = { + **FEATURE_VALUES, + "Etch_AvgPres": 99.0, + } + return pd.DataFrame( + [ + { + "idruns": 161, + "LOTNAME": REQUEST_ID, + "run_date": "2026-07-28", + "is_outlier": False, + "is_calibration_recipe": False, + **trace_features, + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + "Range_nm": 20.0, + } + ] + ) + + experiment_id = "22222222-2222-4222-8222-222222222222" + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=Connection(), + ), + patch( + "data_loader_pg.pd.read_sql_query", + side_effect=read_sql, + ), + ): + result = get_training_df_pg( + experiment_id=experiment_id, + ) + + self.assertEqual(len(result), 1) + self.assertEqual(result.iloc[0]["Etch_AvgPres"], 99.0) + self.assertEqual(result.iloc[0]["AvgEtchRate"], 101.5) + self.assertIn( + "rp.equipment_run_id =\n" + " r.equipment_run_id", + captured["query"], + ) + self.assertIn( + "linked_equipment.source_run_id = r.idruns", + captured["query"], + ) + self.assertIn( + "FROM legacy_training_runs\n" + " WHERE equipment_run_id IS NULL", + captured["query"], + ) + self.assertNotIn( + "FROM etcher_runs legacy", + captured["query"], + ) + self.assertEqual(captured["params"], [experiment_id]) + + def test_dual_ingestion_is_presented_as_one_trace_backed_run(self): + trace = { + "run_id": 5_000_007_001, + "source_run_id": 7001, + "source": "glance", + "features": {"Etch_AvgPres": 25.0}, + "outputs": {}, + "avg_etch_rate": None, + "range_etch_rate": None, + "range_nm": None, + "execution_request_id": REQUEST_ID, + "file_refs": [], + } + summary = { + "run_id": 7001, + "features": FEATURE_VALUES, + "avg_etch_rate": 101.5, + "range_etch_rate": 4.0, + "range_nm": 20.0, + "file_refs": [{"filename": "result.csv"}], + } + + merged = _merge_canonical_summary_into_trace_record(trace, summary) + + self.assertEqual(merged["run_id"], 5_000_007_001) + self.assertEqual(merged["outputs"]["AvgEtchRate"], 101.5) + self.assertEqual(merged["features"]["Etch_AvgPres"], 25.0) + self.assertEqual(merged["file_refs"][0]["filename"], "result.csv") + + def test_catalog_summary_merge_is_scoped_to_exact_tool(self): + canonical_row = { + "run_id": 161, + "lot_name": "summary", + "run_date": None, + "created_at": None, + "is_outlier": False, + "is_calibration": False, + "avg_etch_rate": 103.0, + "range_etch_rate": 4.0, + "range_nm": 20.0, + "etch_avgo2flow": 12.0, + "etch_avg_rf1_pow": 100.0, + "etch_avg_rf2_pow": 50.0, + "etch_avgpres": 25.0, + "etch_avgcf4flow": 30.0, + "source_tool_id": "3", + } + generic_rows = [ + { + "run_id": 5_000_007_003, + "lot_name": "tool-3", + "run_date": None, + "created_at": None, + "is_outlier": False, + "is_calibration": False, + "inputs_json": {"Etch_AvgPres": 33.0}, + "outputs_json": {}, + "source": "glance", + "source_run_id": "161", + "source_tool_id": 3, + "trace_sample_count": 2, + "equipment_id": "tool-3", + }, + { + "run_id": 5_000_007_004, + "lot_name": "tool-4", + "run_date": None, + "created_at": None, + "is_outlier": False, + "is_calibration": False, + "inputs_json": {"Etch_AvgPres": 44.0}, + "outputs_json": {}, + "source": "glance", + "source_run_id": "161", + "source_tool_id": 4, + "trace_sample_count": 2, + "equipment_id": "tool-4", + }, + ] + + class Cursor: + def __init__(self): + self.rows = [] + + def execute(self, query, params=None): + text = str(query) + if "FROM etcher_runs r" in text: + self.rows = [canonical_row] + elif "FROM equipment_runs r" in text: + self.rows = generic_rows + else: + self.rows = [] + + def fetchall(self): + return self.rows + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + with patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=Connection(), + ): + records = get_runs_list_pg( + is_admin=True, + include_outliers=True, + ) + + self.assertEqual(len(records), 2) + by_tool = { + record["source_tool_id"]: record + for record in records + } + self.assertEqual(by_tool[3]["outputs"]["AvgEtchRate"], 103.0) + self.assertNotIn("AvgEtchRate", by_tool[4]["outputs"]) + + def test_trace_detail_does_not_merge_other_tool_summary(self): + generic_row = { + "run_id": 5_000_007_004, + "lot_name": "tool-4", + "run_date": None, + "created_at": None, + "is_outlier": False, + "is_calibration": False, + "inputs_json": {"Etch_AvgPres": 44.0}, + "outputs_json": {}, + "source": "glance", + "source_run_id": "161", + "source_tool_id": 4, + "trace_sample_count": 2, + "equipment_id": "tool-4", + } + summary_row = { + "run_id": 161, + "lot_name": "tool-3-summary", + "run_date": None, + "created_at": None, + "is_outlier": False, + "is_calibration": False, + "avg_etch_rate": 103.0, + "range_etch_rate": 4.0, + "range_nm": 20.0, + "source_tool_id": "3", + } + + class Cursor: + def __init__(self): + self.row = None + + def execute(self, query, params=None): + text = str(query) + if "FROM equipment_runs r" in text: + self.row = generic_row + elif "FROM etcher_runs r" in text: + self.row = summary_row + else: + self.row = None + + def fetchone(self): + return self.row + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + with patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=Connection(), + ): + record = get_run_detail_pg( + 5_000_007_004, + is_admin=True, + ) + + self.assertEqual(record["source_tool_id"], 4) + self.assertNotIn("AvgEtchRate", record["outputs"]) + + def test_summary_measurements_enrich_matching_trace_without_duplication(self): + statements: list[str] = [] + executions: list[tuple[str, object]] = [] + + class Cursor: + def __init__(self): + self.rows = [] + + def execute(self, query, params=None): + text = str(query) + statements.append(text) + executions.append((text, params)) + if ( + "FROM equipment_runs" in text + and "source_run_id = %s" in text + ): + self.rows = [ + { + "id": 7001, + "project_id": "project-test", + "lotname": REQUEST_ID, + "is_outlier": False, + "is_calibration_recipe": False, + "execution_request_id": REQUEST_ID, + "inputs_json": { + "Etch_AvgPres": 99.0, + }, + "outputs_json": {}, + "raw_payload_json": { + "glance": {"proposal_id": PROPOSAL_ID} + }, + "source_tool_id": 3, + } + ] + else: + self.rows = [] + + def fetchall(self): + return self.rows + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def __init__(self): + self.committed = False + + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + self.committed = True + + def rollback(self): + pass + + def close(self): + pass + + connection = Connection() + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=connection, + ): + result = enrich_glance_trace_outcomes_pg( + [ + { + "idruns": 161, + **FEATURE_VALUES, + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + "is_outlier": True, + } + ] + ) + + record = result["project-test"][0] + self.assertEqual(record["equipment_run_id"], 7001) + self.assertEqual(record["proposal_id"], PROPOSAL_ID) + self.assertEqual(record["source_tool_id"], 3) + self.assertEqual(record["inputs"]["Etch_AvgPres"], 99.0) + self.assertEqual(record["inputs"]["Etch_AvgO2Flow"], 12.0) + self.assertEqual(record["outputs"]["AvgEtchRate"], 101.5) + self.assertTrue(record["is_outlier"]) + self.assertTrue(connection.committed) + self.assertTrue( + any( + "UPDATE equipment_runs" in statement + for statement in statements + ) + ) + self.assertTrue( + any( + "UPDATE etcher_runs" in statement + and params + and params[0] == "project-test" + and params[3] == 161 + for statement, params in executions + ) + ) + self.assertTrue( + any( + "pg_advisory_xact_lock" in statement + and params == ("glance_source_run|161",) + for statement, params in executions + ) + ) + stamped_summary = next( + (statement, params) + for statement, params in executions + if "UPDATE etcher_runs" in statement + ) + self.assertIn("jsonb_set", stamped_summary[0]) + self.assertEqual(stamped_summary[1][2], 3) + + previous_revision = "9" * 64 + reconcile_connection, _ = self._connection( + linked=True, + previous_revision=previous_revision, + completed_followup_revision=previous_revision, + ) + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=reconcile_connection, + ): + correction = reconcile_glance_trace_runs_pg( + project_id="project-test", + run_records=[record], + ) + self.assertEqual(correction["dataset_change_rows"], 1) + + def test_summary_bridge_isolates_same_run_number_by_tool(self): + class Cursor: + def __init__(self): + self.rows = [] + + def execute(self, query, params=None): + text = str(query) + if ( + "FROM equipment_runs" in text + and "source_run_id = %s" in text + ): + requested_tool = params[1] + rows = [ + { + "id": 7003, + "project_id": "project-tool-3", + "lotname": "tool-3", + "is_outlier": False, + "is_calibration_recipe": False, + "execution_request_id": "", + "inputs_json": { + "Etch_AvgPres": 33.0, + }, + "outputs_json": {}, + "raw_payload_json": {}, + "source_tool_id": 3, + }, + { + "id": 7004, + "project_id": "project-tool-4", + "lotname": "tool-4", + "is_outlier": False, + "is_calibration_recipe": False, + "execution_request_id": "", + "inputs_json": { + "Etch_AvgPres": 44.0, + }, + "outputs_json": {}, + "raw_payload_json": {}, + "source_tool_id": 4, + }, + ] + self.rows = ( + rows + if requested_tool is None + else [ + row + for row in rows + if row["source_tool_id"] == requested_tool + ] + ) + else: + self.rows = [] + + def fetchall(self): + return self.rows + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + pass + + def rollback(self): + pass + + def close(self): + pass + + with patch( + "glance_closed_loop.get_pg_superuser_connection", + return_value=Connection(), + ): + result = enrich_glance_trace_outcomes_pg( + [ + { + "idruns": 161, + "AvgEtchRate": 999.0, + }, + { + "idruns": 161, + "source_tool_id": 3, + "Etch_AvgPres": 3.0, + "AvgEtchRate": 103.0, + }, + { + "idruns": 161, + "source_tool_id": 4, + "Etch_AvgPres": 4.0, + "AvgEtchRate": 104.0, + }, + ] + ) + + tool_3 = result["project-tool-3"][0] + tool_4 = result["project-tool-4"][0] + self.assertEqual(tool_3["source_tool_id"], 3) + self.assertEqual(tool_4["source_tool_id"], 4) + self.assertEqual(tool_3["inputs"]["Etch_AvgPres"], 33.0) + self.assertEqual(tool_4["inputs"]["Etch_AvgPres"], 44.0) + self.assertEqual(tool_3["outputs"]["AvgEtchRate"], 103.0) + self.assertEqual(tool_4["outputs"]["AvgEtchRate"], 104.0) + + def test_retrain_snapshot_hash_includes_glance_training_rows(self): + class Cursor: + def execute(self, query, params=None): + pass + + def fetchone(self): + return (42,) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self): + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + frame = pd.DataFrame( + [ + { + "idruns": 5_000_007_001, + "is_outlier": False, + "is_calibration_recipe": False, + **FEATURE_VALUES, + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + } + ] + ) + with ( + patch( + "data_loader_pg.get_training_df_pg", + return_value=frame, + ), + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=Connection(), + ), + patch("ai_readiness.record_dataset_snapshot"), + patch("ai_readiness.export_ml_matrix") as legacy_export, + ): + snapshot_id, first_hash, num_rows = retrain._current_snapshot() + changed = frame.copy() + changed.loc[0, "AvgEtchRate"] = 102.5 + with patch( + "data_loader_pg.get_training_df_pg", + return_value=changed, + ): + _, changed_hash, _ = retrain._current_snapshot() + + self.assertEqual(snapshot_id, 42) + self.assertEqual(num_rows, 1) + self.assertNotEqual(first_hash, changed_hash) + legacy_export.assert_not_called() + + def test_duplicate_experiment_iteration_does_not_insert_proposals(self): + statements: list[str] = [] + + class Cursor: + def execute(self, query, params=None): + statements.append(str(query)) + + def fetchone(self): + # Simulate ON CONFLICT for (experiment_id, iteration). + return None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self): + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + with patch( + "metadata_pg.get_pg_connection", + return_value=Connection(), + ): + inserted = save_experiment_proposal_batches_pg( + experiment_id=( + "22222222-2222-4222-8222-222222222222" + ), + project_id="project-test", + execution_request_id=REQUEST_ID, + optimizer_result={ + "source": "test", + "batches": [ + { + "iteration": 1, + "proposals": [ + {"parameters": FEATURE_VALUES} + ], + } + ], + }, + ) + + self.assertEqual(inserted, 0) + self.assertTrue( + any( + "ON CONFLICT (experiment_id, iteration) DO NOTHING" + in statement + for statement in statements + ) + ) + self.assertFalse( + any( + "INSERT INTO experiment_recipe_proposals" in statement + for statement in statements + ) + ) + + def test_auto_proposal_freshness_uses_outcome_ingestion_time(self): + statements: list[str] = [] + + class Cursor: + def __init__(self): + self.last_query = "" + + def execute(self, query, params=None): + self.last_query = str(query) + statements.append(self.last_query) + + def fetchone(self): + if "SELECT project_id" in self.last_query: + return {"project_id": "project-test"} + return None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def close(self): + pass + + with patch( + "metadata_pg.get_pg_superuser_connection", + return_value=Connection(), + ): + result = auto_generate_recipe_proposals_for_projects_pg( + experiment_ids=[ + "22222222-2222-4222-8222-222222222222" + ], + reason="glance_ingestion", + ) + + proposal_query = next( + statement + for statement in statements + if "SELECT MAX(training_run.ingested_at)" in statement + ) + self.assertIn( + "equipment_run.ingested_at AS ingested_at", + proposal_query, + ) + self.assertNotIn( + "equipment_run.created_at AS ingested_at", + proposal_query, + ) + self.assertEqual(result["generated"], 0) + + def test_auto_proposal_uses_trusted_postgres_live_fit_context(self): + now = datetime.now(timezone.utc) + + class Cursor: + def __init__(self): + self.last_query = "" + + def execute(self, query, params=None): + self.last_query = str(query) + + def fetchone(self): + if "SELECT project_id" in self.last_query: + return {"project_id": "project-test"} + if "SELECT pg_advisory_xact_lock" in self.last_query: + return None + if "SELECT MAX(training_run.ingested_at)" in self.last_query: + return { + "experiment_id": ( + "22222222-2222-4222-8222-222222222222" + ), + "project_id": "project-test", + "type_id": "recipe_sweep", + "execution_request_id": REQUEST_ID, + "planned_parameters_json": [], + "input_snapshot_json": [], + "type_parameters_json": [], + "max_iteration": 1, + "latest_batch_at": now - timedelta(minutes=2), + "latest_open_proposals": 0, + "latest_run_at": now, + } + return None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def close(self): + pass + + optimizer_result = { + "source": "local_gpr", + "batches": [ + { + "iteration": 1, + "proposals": [{"parameters": FEATURE_VALUES}], + } + ], + } + with ( + patch( + "metadata_pg.get_pg_superuser_connection", + return_value=Connection(), + ), + patch( + "metadata_pg.update_experiment_proposal_generation_status_pg", + ), + patch( + "metadata_pg.save_experiment_proposal_batches_pg", + return_value=1, + ), + patch( + "ml_engine.compute_proposals", + return_value=optimizer_result, + ) as compute, + ): + result = auto_generate_recipe_proposals_for_projects_pg( + experiment_ids=[ + "22222222-2222-4222-8222-222222222222" + ], + reason="glance_ingestion", + ) + + context = compute.call_args.kwargs["optimization_context"] + self.assertTrue(context["is_admin"]) + self.assertTrue(context["include_global_history"]) + self.assertTrue(context["disable_csv_fallback"]) + self.assertTrue(context["force_live_fit"]) + self.assertEqual(result["generated"], 1) + + def test_next_batch_waits_for_all_completed_trace_outcomes(self): + now = datetime.now(timezone.utc) + unresolved_counts = iter([2, 1, 0]) + + class Cursor: + def __init__(self): + self.last_query = "" + + def execute(self, query, params=None): + self.last_query = str(query) + + def fetchone(self): + if "SELECT project_id" in self.last_query: + return {"project_id": "project-test"} + if "SELECT MAX(training_run.ingested_at)" in self.last_query: + return { + "experiment_id": ( + "22222222-2222-4222-8222-222222222222" + ), + "project_id": "project-test", + "type_id": "recipe_sweep", + "execution_request_id": REQUEST_ID, + "planned_parameters_json": [], + "input_snapshot_json": [], + "type_parameters_json": [], + "max_iteration": 1, + "latest_batch_at": now - timedelta(minutes=2), + "latest_open_proposals": 0, + "latest_unresolved_results": next( + unresolved_counts + ), + "latest_run_at": now, + } + return None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def close(self): + pass + + optimizer_result = { + "source": "local_gpr", + "batches": [ + { + "iteration": 1, + "proposals": [{"parameters": FEATURE_VALUES}], + } + ], + } + with ( + patch( + "metadata_pg.get_pg_superuser_connection", + side_effect=lambda: Connection(), + ), + patch( + "metadata_pg.update_experiment_proposal_generation_status_pg", + ), + patch( + "metadata_pg.save_experiment_proposal_batches_pg", + return_value=1, + ), + patch( + "ml_engine.compute_proposals", + return_value=optimizer_result, + ) as compute, + ): + results = [ + auto_generate_recipe_proposals_for_projects_pg( + experiment_ids=[ + "22222222-2222-4222-8222-222222222222" + ], + reason="glance_ingestion", + ) + for _ in range(3) + ] + + self.assertEqual( + results[0]["skipped"][0]["reason"], + "latest_batch_results_pending", + ) + self.assertEqual( + results[1]["skipped"][0]["reason"], + "latest_batch_results_pending", + ) + self.assertEqual(results[2]["generated"], 1) + compute.assert_called_once() + + def test_final_recipe_rejection_wakes_optimizer_readiness(self): + class Cursor: + def __init__(self): + self.row = None + + def execute(self, query, params=None): + text = str(query) + if "SELECT" in text and "experiment_recipe_proposals" in text: + self.row = { + "id": PROPOSAL_ID, + "experiment_id": ( + "22222222-2222-4222-8222-222222222222" + ), + "owner_id": "owner-1", + "project_id": "project-test", + } + elif "UPDATE experiment_recipe_proposals" in text: + self.row = { + "id": PROPOSAL_ID, + "status": "rejected", + "ingestion_status": ( + "recipe_rejected_followup_pending" + ), + "run_id": None, + "lotname": "", + "completed_at": None, + "rejected_at": datetime.now(timezone.utc), + "comment": "bad recipe", + } + else: + self.row = None + + def fetchone(self): + return self.row + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + with ( + patch( + "metadata_pg.get_pg_connection", + return_value=Connection(), + ), + patch( + "metadata_pg.drain_recipe_optimizer_wakeups_pg", + return_value={ + "status": "success", + "processed": 1, + "generated": 1, + }, + ) as drain_wakeup, + ): + result = update_recipe_proposal_status_pg( + proposal_id=PROPOSAL_ID, + status="rejected", + comment="bad recipe", + user=SimpleNamespace( + id="owner-1", + role="researcher", + ), + ) + + self.assertEqual( + result["ingestion_status"], + "recipe_rejected", + ) + drain_wakeup.assert_called_once_with( + proposal_ids=[PROPOSAL_ID], + ) + self.assertEqual( + result["optimizer_followup"]["generated"], + 1, + ) + + def test_rejection_returns_committed_status_when_wakeup_fails(self): + class Cursor: + def __init__(self): + self.row = None + + def execute(self, query, params=None): + text = str(query) + if "SELECT" in text: + self.row = { + "id": PROPOSAL_ID, + "experiment_id": ( + "22222222-2222-4222-8222-222222222222" + ), + "owner_id": "owner-1", + "project_id": "project-test", + } + else: + self.row = { + "id": PROPOSAL_ID, + "status": "rejected", + "ingestion_status": ( + "recipe_rejected_followup_pending" + ), + "run_id": None, + "lotname": "", + "completed_at": None, + "rejected_at": datetime.now(timezone.utc), + "comment": "bad recipe", + } + + def fetchone(self): + return self.row + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + with ( + patch( + "metadata_pg.get_pg_connection", + return_value=Connection(), + ), + patch( + "metadata_pg.drain_recipe_optimizer_wakeups_pg", + side_effect=RuntimeError("optimizer unavailable"), + ), + ): + result = update_recipe_proposal_status_pg( + proposal_id=PROPOSAL_ID, + status="rejected", + comment="bad recipe", + user=SimpleNamespace( + id="owner-1", + role="researcher", + ), + ) + + self.assertEqual(result["status"], "rejected") + self.assertEqual( + result["ingestion_status"], + "recipe_rejected_followup_pending", + ) + self.assertEqual( + result["optimizer_followup"]["status"], + "pending", + ) + + def test_force_live_fit_does_not_reuse_stale_registry_model(self): + feature_names = list(FEATURE_VALUES) + X = np.array( + [ + [value + index for value in FEATURE_VALUES.values()] + for index in range(6) + ], + dtype=float, + ) + ys = { + "AvgEtchRate": np.linspace(95.0, 105.0, 6), + "RangeEtchRate": np.linspace(3.0, 5.0, 6), + } + frame = pd.DataFrame(X, columns=feature_names) + frame["idruns"] = list(range(1, 7)) + frame["LOTNAME"] = [f"run-{index}" for index in range(6)] + frame["run_date"] = pd.date_range( + "2026-07-01", + periods=6, + freq="h", + ) + frame["AvgEtchRate"] = ys["AvgEtchRate"] + frame["RangeEtchRate"] = ys["RangeEtchRate"] + + def predict(X_train, y_train, X_pred, **kwargs): + return ( + np.full(len(X_pred), float(np.mean(y_train))), + np.ones(len(X_pred)), + ) + + with ( + patch.dict( + os.environ, + { + "NEXTJS_PROPOSAL_ENGINE": "ml", + "ML_PROPOSAL_CANDIDATES": "32", + }, + clear=False, + ), + patch( + "ml_engine._get_xy", + return_value=(X, ys, frame["LOTNAME"], frame), + ), + patch( + "ml_engine._registry_model_for", + return_value={ + "model_kind": "gpr", + "snapshot_hash": "stale", + "feature_columns": feature_names, + "estimator": object(), + }, + ) as registry_model, + patch( + "ml_engine._fit_predictive_model", + side_effect=predict, + ), + ): + result = ml_engine.compute_proposals( + n_batches=1, + batch_size=1, + optimization_context={"force_live_fit": True}, + ) + + self.assertEqual(result["source"], "local_gpr") + registry_model.assert_not_called() + + def test_retrain_revision_credit_is_idempotent(self): + state = {"keys": set(), "total": 0} + + class Cursor: + def __init__(self): + self.row = None + + def execute(self, query, params=None): + text = str(query) + if "INSERT INTO retrain_run_credits" in text: + credit_key = params[1] + if credit_key in state["keys"]: + self.row = None + else: + state["keys"].add(credit_key) + self.row = (params[2],) + elif "INSERT INTO retrain_state" in text: + state["total"] += int(params[1]) + self.row = (state["total"],) + elif "SELECT runs_since_last_train" in text: + self.row = (state["total"],) + + def fetchone(self): + return self.row + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self): + return Cursor() + + def commit(self): + pass + + def rollback(self): + pass + + def close(self): + pass + + with patch( + "data_loader_pg.get_pg_superuser_connection", + side_effect=lambda: Connection(), + ): + first = credit_runs_counter_once( + domain_id="etcher", + credits=[("glance:7001:revision-a", 1)], + ) + replay = credit_runs_counter_once( + domain_id="etcher", + credits=[("glance:7001:revision-a", 1)], + ) + + self.assertEqual(first["credited"], 1) + self.assertEqual(replay["credited"], 0) + self.assertEqual(replay["runs_since_last_train"], 1) + + +class GlanceClosedLoopEndToEndTests(unittest.TestCase): + def setUp(self) -> None: + app = FastAPI() + app.include_router(dataset_v2_router) + self.client = TestClient(app) + self.payload = { + "version": "1", + "source_system": "glance", + "batch_id": "experiment-batch-1", + "equipment_id": "equipment-test", + "source_tool_id": 3, + "source_tool_name": "VLN-11304 DSE", + "project_id": "project-test", + "previous_cursor": "160", + "proposed_cursor": "161", + "poll_started_at": "2026-07-28T10:00:00Z", + "runs": [ + { + "source_run_id": 161, + "execution_request_id": REQUEST_ID, + "proposal_id": PROPOSAL_ID, + "lotname": REQUEST_ID, + "run_start_time": "2026-07-28T10:00:00", + "run_end_time": "2026-07-28T10:01:00", + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "parameters": [], + "samples": [ + { + "source_sample_id": 1001, + "timestamp": "2026-07-28T10:00:30", + "values": {}, + } + ], + "events": [ + { + "source_event_id": 2001, + "timestamp": "2026-07-28T10:01:00", + "event_type": "Process Completed", + } + ], + } + ], + } + + def test_experiment_trace_to_next_proposal_and_retrain_is_idempotent(self): + state = { + "proposal_status": "accepted", + "experiment_status": "running", + "linked_equipment_run_id": None, + } + followups: list[dict] = [] + retrains: list[tuple[int, list[str]]] = [] + + run_record = { + "equipment_run_id": 7001, + "source_tool_id": 3, + "source_run_id": 161, + "execution_request_id": REQUEST_ID, + "proposal_id": PROPOSAL_ID, + "lotname": REQUEST_ID, + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "parameters": [], + "samples": [{"values": {}}], + "is_outlier": False, + "is_calibration_recipe": False, + "closed_loop_revision_sha256": "a" * 64, + } + + def reconcile(*, project_id, run_records): + self.assertEqual(project_id, "project-test") + self.assertEqual(run_records[0]["proposal_id"], PROPOSAL_ID) + if state["linked_equipment_run_id"] == 7001: + return { + "checked": 1, + "matched": 1, + "newly_matched": 0, + "replayed": 1, + "unmatched": 0, + "unresolved_identity": 0, + "training_rows": 0, + "training_project_ids": [], + "training_experiment_ids": [], + "followup_claims": [], + "followup_in_progress": 0, + "decisions": [ + { + "source_run_id": 161, + "status": "replayed", + "training_eligible": True, + "training_revision_changed": False, + } + ], + } + state["proposal_status"] = "completed" + state["experiment_status"] = "data_ingested" + state["linked_equipment_run_id"] = 7001 + return { + "checked": 1, + "matched": 1, + "newly_matched": 1, + "replayed": 0, + "unmatched": 0, + "unresolved_identity": 0, + "training_rows": 1, + "training_project_ids": ["project-test"], + "training_experiment_ids": [ + "22222222-2222-4222-8222-222222222222" + ], + "followup_claims": [ + { + "equipment_run_id": 7001, + "revision_sha256": "a" * 64, + } + ], + "followup_in_progress": 0, + "decisions": [ + { + "source_run_id": 161, + "proposal_id": PROPOSAL_ID, + "status": "matched", + "training_eligible": True, + "training_revision_changed": True, + } + ], + } + + def auto_propose(**kwargs): + followups.append(kwargs) + return {"status": "success", "generated": 1} + + def retrain( + n_new, + *, + credit_keys=None, + force_dataset_refresh=False, + ): + retrains.append((n_new, list(credit_keys or []))) + return {"status": "trained"} + + mapping = ( + '{"equipment-test":{"source_tool_id":3,' + '"source_tool_name":"VLN-11304 DSE",' + '"project_id":"project-test"}}' + ) + environment = { + "INGESTION_TOKEN": "test-ingestion-token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": mapping, + "AUTO_RECIPE_PROPOSALS_ENABLED": "true", + "AUTO_RETRAIN_ENABLED": "true", + } + persistence_result = { + "inserted": 1, + "updated": 0, + "run_count": 1, + "sample_count": 1, + "event_count": 1, + "run_records": [run_record], + } + with ( + patch.dict(os.environ, environment, clear=False), + patch( + "routers.dataset_v2.get_equipment_pg", + return_value={"parameters": [], "outputs": []}, + ), + patch( + "routers.dataset_v2.get_glance_ingestion_project_pg", + return_value={ + "id": "project-test", + "equipment_id": "equipment-test", + }, + ), + patch( + "routers.dataset_v2.sync_equipment_run_traces_pg", + return_value=persistence_result, + ), + patch( + "routers.dataset_v2.reconcile_glance_trace_runs_pg", + side_effect=reconcile, + ), + patch( + "metadata_pg.auto_generate_recipe_proposals_for_projects_pg", + side_effect=auto_propose, + ), + patch( + "glance_closed_loop._maybe_retrain", + side_effect=retrain, + ), + patch( + "routers.dataset_v2.finish_glance_followup_claims_pg", + ), + patch("routers.dataset_v2.record_glance_ingestion_audit_pg"), + ): + first = self.client.post( + "/dataset/v2/glance/traces/sync", + json=self.payload, + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + second = self.client.post( + "/dataset/v2/glance/traces/sync", + json=self.payload, + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + + self.assertEqual(first.status_code, 200) + self.assertTrue(first.json()["cursor_acknowledged"]) + self.assertEqual(first.json()["closed_loop"]["newly_matched"], 1) + self.assertEqual(second.status_code, 200) + self.assertEqual(second.json()["closed_loop"]["replayed"], 1) + self.assertEqual(state["proposal_status"], "completed") + self.assertEqual(state["experiment_status"], "data_ingested") + self.assertEqual(state["linked_equipment_run_id"], 7001) + self.assertEqual( + followups, + [ + { + "project_ids": ["project-test"], + "experiment_ids": [ + "22222222-2222-4222-8222-222222222222" + ], + "reason": "glance_ingestion", + } + ], + ) + self.assertEqual( + retrains, + [(1, [f"glance:7001:{'a' * 64}"])], + ) + + def test_connector_trace_waits_for_summary_outcomes_then_advances(self): + trace_payload = deepcopy(self.payload) + trace_payload["runs"][0]["inputs"] = {} + trace_payload["runs"][0]["outputs"] = {} + incomplete_record = { + "equipment_run_id": 7001, + "source_tool_id": 3, + "source_run_id": 161, + "execution_request_id": REQUEST_ID, + "proposal_id": PROPOSAL_ID, + "lotname": REQUEST_ID, + "inputs": {}, + "outputs": {}, + "parameters": [], + "samples": [{"values": {}}], + "is_outlier": False, + "is_calibration_recipe": False, + "closed_loop_revision_sha256": "b" * 64, + } + complete_record = { + **incomplete_record, + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "closed_loop_revision_sha256": "c" * 64, + } + state = {"trace_linked": False} + + def reconcile(*, project_id, run_records): + self.assertEqual(project_id, "project-test") + complete = bool(run_records[0].get("outputs")) + if not complete: + state["trace_linked"] = True + return { + "checked": 1, + "matched": 1, + "newly_matched": 1, + "replayed": 0, + "unmatched": 0, + "unresolved_identity": 0, + "training_rows": 0, + "training_project_ids": [], + "training_experiment_ids": [], + "followup_claims": [], + "followup_in_progress": 0, + "decisions": [], + } + self.assertTrue(state["trace_linked"]) + return { + "checked": 1, + "matched": 1, + "newly_matched": 0, + "replayed": 1, + "unmatched": 0, + "unresolved_identity": 0, + "training_rows": 1, + "training_project_ids": ["project-test"], + "training_experiment_ids": [ + "22222222-2222-4222-8222-222222222222" + ], + "followup_claims": [ + { + "equipment_run_id": 7001, + "revision_sha256": "c" * 64, + } + ], + "followup_in_progress": 0, + "decisions": [ + { + "source_run_id": 161, + "proposal_id": PROPOSAL_ID, + "status": "replayed", + "training_eligible": True, + "training_revision_changed": True, + } + ], + } + + exact_calls: list[dict] = [] + environment = { + "INGESTION_TOKEN": "test-ingestion-token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": ( + '{"equipment-test":{"source_tool_id":3,' + '"source_tool_name":"VLN-11304 DSE",' + '"project_id":"project-test"}}' + ), + "AUTO_RECIPE_PROPOSALS_ENABLED": "true", + "AUTO_RETRAIN_ENABLED": "true", + } + with ( + patch.dict(os.environ, environment, clear=False), + patch( + "routers.dataset_v2.get_equipment_pg", + return_value={"parameters": [], "outputs": []}, + ), + patch( + "routers.dataset_v2.get_glance_ingestion_project_pg", + return_value={ + "id": "project-test", + "equipment_id": "equipment-test", + }, + ), + patch( + "routers.dataset_v2.sync_equipment_run_traces_pg", + return_value={ + "inserted": 1, + "updated": 0, + "run_count": 1, + "sample_count": 1, + "event_count": 1, + "run_records": [incomplete_record], + }, + ), + patch( + "routers.dataset_v2.sync_runs_pg", + return_value=1, + ), + patch( + "routers.dataset_v2.enrich_glance_trace_outcomes_pg", + return_value={"project-test": [complete_record]}, + ), + patch( + "routers.dataset_v2.reconcile_glance_trace_runs_pg", + side_effect=reconcile, + ), + patch( + "routers.dataset_v2.reconcile_ingested_runs_with_recipe_proposals_pg", + return_value={"matched": 0, "checked": 0}, + ) as legacy_reconcile, + patch( + "metadata_pg.auto_generate_recipe_proposals_for_projects_pg", + side_effect=lambda **kwargs: ( + exact_calls.append(kwargs) + or {"status": "success", "generated": 1} + ), + ), + patch( + "routers.dataset_v2.auto_generate_recipe_proposals_for_projects_pg", + return_value={"status": "success", "generated": 0}, + ) as legacy_auto_propose, + patch( + "glance_closed_loop._maybe_retrain", + return_value={"status": "trained"}, + ) as retrain, + patch( + "model_registry.bump_runs_counter", + side_effect=AssertionError( + "legacy retrain must not count an exact-loop run" + ), + ), + patch( + "routers.dataset_v2.finish_glance_followup_claims_pg", + ) as finish_claims, + patch("routers.dataset_v2.record_glance_ingestion_audit_pg"), + ): + trace_response = self.client.post( + "/dataset/v2/glance/traces/sync", + json=trace_payload, + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + summary_response = self.client.post( + "/dataset/v2/runs/sync", + json=[ + { + "idruns": 161, + "project_id": "project-test", + "LOTNAME": REQUEST_ID, + **FEATURE_VALUES, + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + } + ], + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + + self.assertEqual(trace_response.status_code, 200) + self.assertEqual( + trace_response.json()["closed_loop"]["training_rows"], + 0, + ) + self.assertEqual(summary_response.status_code, 200) + self.assertEqual( + exact_calls[0]["experiment_ids"], + ["22222222-2222-4222-8222-222222222222"], + ) + retrain.assert_called_once_with( + 1, + credit_keys=[f"glance:7001:{'c' * 64}"], + force_dataset_refresh=False, + ) + legacy_reconcile.assert_called_once_with( + [], + project_resolver=dataset_v2._project_id_from_sync_row, + ) + legacy_auto_propose.assert_not_called() + finish_claims.assert_called_once_with( + [ + { + "equipment_run_id": 7001, + "revision_sha256": "c" * 64, + } + ], + succeeded=True, + ) + + def test_summary_first_then_trace_repairs_with_idempotent_followup(self): + complete_record = { + "equipment_run_id": 7001, + "source_tool_id": 3, + "source_run_id": 161, + "execution_request_id": REQUEST_ID, + "proposal_id": PROPOSAL_ID, + "lotname": REQUEST_ID, + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "parameters": [], + "samples": [{"values": {}}], + "is_outlier": False, + "is_calibration_recipe": False, + "closed_loop_revision_sha256": "5" * 64, + } + reconciliation = { + "checked": 1, + "matched": 1, + "newly_matched": 1, + "replayed": 0, + "unmatched": 0, + "unresolved_identity": 0, + "training_rows": 1, + "training_project_ids": ["project-test"], + "training_experiment_ids": [ + "22222222-2222-4222-8222-222222222222" + ], + "followup_claims": [ + { + "equipment_run_id": 7001, + "revision_sha256": "5" * 64, + "claim_token": ( + "55555555-5555-4555-8555-555555555555" + ), + } + ], + "followup_in_progress": 0, + "decisions": [ + { + "source_run_id": 161, + "proposal_id": PROPOSAL_ID, + "status": "matched", + "legacy_summary_link": True, + "training_eligible": True, + "training_revision_changed": True, + "training_followup_claimed": True, + } + ], + } + mapping = ( + '{"equipment-test":{"source_tool_id":3,' + '"source_tool_name":"VLN-11304 DSE",' + '"project_id":"project-test"}}' + ) + with ( + patch.dict( + os.environ, + { + "INGESTION_TOKEN": "test-ingestion-token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": mapping, + "AUTO_RECIPE_PROPOSALS_ENABLED": "false", + "AUTO_RETRAIN_ENABLED": "false", + }, + clear=False, + ), + patch( + "routers.dataset_v2.sync_runs_pg", + return_value=1, + ), + patch( + "routers.dataset_v2.enrich_glance_trace_outcomes_pg", + return_value={}, + ), + patch( + "routers.dataset_v2.reconcile_ingested_runs_with_recipe_proposals_pg", + return_value={"matched": 1, "checked": 1}, + ), + patch( + "routers.dataset_v2.get_equipment_pg", + return_value={"parameters": [], "outputs": []}, + ), + patch( + "routers.dataset_v2.get_glance_ingestion_project_pg", + return_value={ + "id": "project-test", + "equipment_id": "equipment-test", + }, + ), + patch( + "routers.dataset_v2.sync_equipment_run_traces_pg", + return_value={ + "inserted": 1, + "updated": 0, + "run_count": 1, + "sample_count": 1, + "event_count": 1, + "run_records": [complete_record], + }, + ), + patch( + "routers.dataset_v2.reconcile_glance_trace_runs_pg", + return_value=reconciliation, + ), + patch( + "routers.dataset_v2.run_glance_closed_loop_followups", + return_value={ + "proposals": { + "status": "success", + "generated": 0, + }, + "retrain": {"status": "debounced"}, + }, + ) as exact_followup, + patch( + "routers.dataset_v2.finish_glance_followup_claims_pg", + ) as finish_claim, + patch("routers.dataset_v2.record_glance_ingestion_audit_pg"), + ): + summary_response = self.client.post( + "/dataset/v2/runs/sync", + json=[ + { + "idruns": 161, + "project_id": "project-test", + "LOTNAME": REQUEST_ID, + **FEATURE_VALUES, + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + } + ], + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + trace_response = self.client.post( + "/dataset/v2/glance/traces/sync", + json=self.payload, + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + + self.assertEqual(summary_response.status_code, 200) + self.assertEqual(trace_response.status_code, 200) + self.assertTrue(trace_response.json()["cursor_acknowledged"]) + self.assertTrue( + trace_response.json()["closed_loop"]["decisions"][0][ + "legacy_summary_link" + ] + ) + exact_followup.assert_called_once_with(reconciliation) + finish_claim.assert_called_once_with( + reconciliation["followup_claims"], + succeeded=True, + ) + + def test_tagged_glance_summary_defers_approximate_legacy_matching(self): + with ( + patch.dict( + os.environ, + { + "INGESTION_TOKEN": "test-ingestion-token", + "AUTO_RECIPE_PROPOSALS_ENABLED": "true", + "AUTO_RETRAIN_ENABLED": "true", + }, + clear=False, + ), + patch( + "routers.dataset_v2.sync_runs_pg", + return_value=1, + ), + patch( + "routers.dataset_v2.enrich_glance_trace_outcomes_pg", + return_value={}, + ), + patch( + "routers.dataset_v2.reconcile_ingested_runs_with_recipe_proposals_pg", + return_value={"matched": 0, "checked": 0}, + ) as legacy_reconcile, + patch( + "routers.dataset_v2.auto_generate_recipe_proposals_for_projects_pg", + ) as legacy_auto, + patch( + "model_registry.credit_runs_counter_once", + side_effect=AssertionError( + "tagged summary must wait for exact trace identity" + ), + ), + ): + response = self.client.post( + "/dataset/v2/runs/sync", + json=[ + { + "idruns": 161, + "source_system": "glance_summary", + "LOTNAME": REQUEST_ID, + **FEATURE_VALUES, + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + } + ], + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + + self.assertEqual(response.status_code, 200) + legacy_reconcile.assert_called_once_with( + [], + project_resolver=dataset_v2._project_id_from_sync_row, + ) + legacy_auto.assert_not_called() + + def test_failed_followup_releases_claim_and_refuses_cursor_ack(self): + run_record = { + "equipment_run_id": 7001, + "source_tool_id": 3, + "source_run_id": 161, + "execution_request_id": REQUEST_ID, + "proposal_id": PROPOSAL_ID, + "lotname": REQUEST_ID, + "inputs": FEATURE_VALUES, + "outputs": { + "AvgEtchRate": 101.5, + "RangeEtchRate": 4.0, + }, + "parameters": [], + "samples": [{"values": {}}], + "is_outlier": False, + "is_calibration_recipe": False, + "closed_loop_revision_sha256": "d" * 64, + } + reconciliation = { + "checked": 1, + "matched": 1, + "newly_matched": 1, + "replayed": 0, + "unmatched": 0, + "unresolved_identity": 0, + "training_rows": 1, + "training_project_ids": ["project-test"], + "training_experiment_ids": [ + "22222222-2222-4222-8222-222222222222" + ], + "followup_claims": [ + { + "equipment_run_id": 7001, + "revision_sha256": "d" * 64, + } + ], + "followup_in_progress": 0, + "decisions": [], + } + mapping = ( + '{"equipment-test":{"source_tool_id":3,' + '"source_tool_name":"VLN-11304 DSE",' + '"project_id":"project-test"}}' + ) + with ( + patch.dict( + os.environ, + { + "INGESTION_TOKEN": "test-ingestion-token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": mapping, + }, + clear=False, + ), + patch( + "routers.dataset_v2.get_equipment_pg", + return_value={"parameters": [], "outputs": []}, + ), + patch( + "routers.dataset_v2.get_glance_ingestion_project_pg", + return_value={ + "id": "project-test", + "equipment_id": "equipment-test", + }, + ), + patch( + "routers.dataset_v2.sync_equipment_run_traces_pg", + return_value={ + "inserted": 1, + "updated": 0, + "run_count": 1, + "sample_count": 1, + "event_count": 1, + "run_records": [run_record], + }, + ), + patch( + "routers.dataset_v2.reconcile_glance_trace_runs_pg", + return_value=reconciliation, + ), + patch( + "routers.dataset_v2.run_glance_closed_loop_followups", + return_value={ + "proposals": {"status": "failed"}, + "retrain": {"status": "trained"}, + }, + ), + patch( + "routers.dataset_v2.finish_glance_followup_claims_pg", + ) as finish_claims, + patch("routers.dataset_v2.record_glance_ingestion_audit_pg"), + ): + response = self.client.post( + "/dataset/v2/glance/traces/sync", + json=self.payload, + headers={"X-Ingestion-Token": "test-ingestion-token"}, + ) + + self.assertEqual(response.status_code, 422) + self.assertFalse( + response.json()["detail"]["cursor_acknowledged"] + ) + finish_claims.assert_called_once_with( + reconciliation["followup_claims"], + succeeded=False, + ) + + +class GlanceClosedLoopFollowupTests(unittest.TestCase): + def test_incomplete_run_does_not_advance_optimizer_or_retrain(self): + reconciliation = { + "training_rows": 0, + "training_project_ids": [], + } + with patch( + "glance_closed_loop._maybe_retrain", + wraps=lambda value, **kwargs: { + "status": "skipped", + "reason": "no_training_rows", + }, + ) as retrain: + result = run_glance_closed_loop_followups(reconciliation) + + self.assertEqual( + result["proposals"]["reason"], + "no_complete_training_rows", + ) + retrain.assert_called_once_with( + 0, + credit_keys=[], + force_dataset_refresh=False, + ) + + def test_retrain_waits_when_next_proposal_generation_fails(self): + reconciliation = { + "training_rows": 1, + "training_project_ids": ["project-test"], + "training_experiment_ids": [ + "22222222-2222-4222-8222-222222222222" + ], + "followup_claims": [ + { + "equipment_run_id": 7001, + "revision_sha256": "e" * 64, + } + ], + } + with ( + patch.dict( + os.environ, + {"AUTO_RECIPE_PROPOSALS_ENABLED": "true"}, + clear=False, + ), + patch( + "metadata_pg.auto_generate_recipe_proposals_for_projects_pg", + side_effect=RuntimeError("optimizer unavailable"), + ), + patch( + "glance_closed_loop._maybe_retrain", + return_value={"status": "trained"}, + ) as retrain, + ): + result = run_glance_closed_loop_followups(reconciliation) + + self.assertEqual(result["proposals"]["status"], "failed") + self.assertEqual( + result["retrain"]["status"], + "skipped_dependency_failed", + ) + retrain.assert_not_called() + + def test_eligibility_removal_forces_snapshot_retraining(self): + reconciliation = { + "training_rows": 0, + "dataset_change_rows": 1, + "training_project_ids": ["project-test"], + "training_experiment_ids": [ + "22222222-2222-4222-8222-222222222222" + ], + "followup_claims": [ + { + "equipment_run_id": 7001, + "revision_sha256": "6" * 64, + "retrain_credit_key": None, + } + ], + } + with patch( + "glance_closed_loop._maybe_retrain", + return_value={"status": "trained"}, + ) as retrain: + result = run_glance_closed_loop_followups(reconciliation) + + self.assertEqual( + result["proposals"]["reason"], + "no_complete_training_rows", + ) + retrain.assert_called_once_with( + 0, + credit_keys=[], + force_dataset_refresh=True, + ) + + def test_disabled_retraining_cannot_complete_dataset_correction(self): + with patch.dict( + os.environ, + {"AUTO_RETRAIN_ENABLED": "false"}, + clear=False, + ): + result = run_glance_closed_loop_followups( + { + "training_rows": 0, + "dataset_change_rows": 1, + "followup_claims": [], + } + ) + + self.assertEqual(result["retrain"]["status"], "failed") + self.assertFalse( + glance_followups_succeeded(result) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_production_glance_ingestion.py b/api/tests/test_production_glance_ingestion.py index 64cf44b..8d7c1e1 100644 --- a/api/tests/test_production_glance_ingestion.py +++ b/api/tests/test_production_glance_ingestion.py @@ -1,5 +1,7 @@ from __future__ import annotations +import copy +import json import os import sys import unittest @@ -17,9 +19,10 @@ if path not in sys.path: sys.path.insert(0, path) -from data_loader_pg import sync_equipment_run_traces_pg +from data_loader_pg import sync_equipment_run_traces_pg, sync_runs_pg from glance_ingestion import ( equipment_parameter_registry, + load_production_mapping, prepare_live_run, ) from common.glance_connector import ( @@ -153,17 +156,149 @@ def test_missing_nullable_sample_timestamp_is_preserved(self): any("has no source timestamp" in warning for warning in warnings) ) + def test_duplicate_source_tool_mapping_is_rejected(self): + mapping = ( + '{"equipment-a":{"source_tool_id":3,' + '"project_id":"project-a"},' + '"equipment-b":{"source_tool_id":3,' + '"project_id":"project-b"}}' + ) + with ( + patch.dict( + os.environ, + {"GLANCE_PRODUCTION_MAPPINGS_JSON": mapping}, + clear=False, + ), + self.assertRaisesRegex( + RuntimeError, + "source_tool_id 3 is mapped to both", + ), + ): + load_production_mapping() + class ProductionGlancePersistenceTests(unittest.TestCase): + def test_scalar_summary_cannot_overwrite_same_run_from_other_tool(self): + state = {"tool_by_run": {}, "upserts": 0} + + class Cursor: + def __init__(self): + self.row = None + + def execute(self, query, params=None): + normalized = " ".join(str(query).split()) + self.row = None + if ( + "SELECT COALESCE(" in normalized + and "FROM etcher_runs" in normalized + ): + tool_id = state["tool_by_run"].get(int(params[0])) + self.row = (str(tool_id),) if tool_id else None + + def fetchone(self): + return self.row + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + def execute_values(cursor, query, values, **kwargs): + if "INSERT INTO etcher_runs" in str(query): + state["upserts"] += 1 + for row in values: + raw = row[17].adapted + state["tool_by_run"][int(row[0])] = int( + raw["source_tool_id"] + ) + + def summary(tool_id: int) -> dict: + return { + "idruns": 161, + "project_id": "project-test", + "LOTNAME": "etch7/30/2024", + "source_system": "glance_summary", + "source_tool_id": tool_id, + "Etch_AvgPres": float(tool_id), + } + + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + side_effect=lambda: Connection(), + ), + patch( + "psycopg2.extras.execute_values", + side_effect=execute_values, + ), + ): + self.assertEqual(sync_runs_pg([summary(3)]), 1) + unscoped = summary(3) + unscoped.pop("source_tool_id") + unscoped["source_system"] = "legacy" + with self.assertRaisesRegex( + ValueError, + "without an authoritative tool identity", + ): + sync_runs_pg([unscoped]) + with self.assertRaisesRegex( + ValueError, + "already assigned to GLANCE tool 3", + ): + sync_runs_pg([summary(4)]) + with self.assertRaisesRegex( + ValueError, + "duplicate run id 161", + ): + sync_runs_pg([summary(3), summary(3)]) + + self.assertEqual(state["upserts"], 1) + self.assertEqual(state["tool_by_run"][161], 3) + def test_live_upsert_uses_source_identity_and_returns_counts(self): queries: list[str] = [] + executions: list[tuple[str, object]] = [] value_queries: list[str] = [] class Cursor: + def __init__(self): + self.last_query = "" + def execute(self, query, params=None): - queries.append(str(query)) + self.last_query = str(query) + queries.append(self.last_query) + executions.append((self.last_query, params)) def fetchall(self): + if "FROM etcher_runs" in self.last_query: + return [ + ( + 101, + "REQ-ABCDEF123456", + 12.0, + 100.0, + 50.0, + 25.0, + 30.0, + 101.5, + 4.0, + 20.0, + False, + False, + {"source_tool_id": 5}, + ) + ] return [] def __enter__(self): @@ -213,6 +348,7 @@ def execute_values(cursor, query, values, **kwargs): runs=[ { "source_run_id": 101, + "inputs": {"Etch_AvgPres": 99.0}, "run_start_time": "2026-07-01T10:00:00", "parameters": [ { @@ -244,6 +380,41 @@ def execute_values(cursor, query, values, **kwargs): self.assertEqual(result["inserted"], 1) self.assertEqual(result["sample_count"], 1) self.assertEqual(result["event_count"], 1) + self.assertEqual( + result["run_records"][0]["inputs"]["Etch_AvgPres"], + 99.0, + ) + self.assertEqual( + result["run_records"][0]["outputs"]["AvgEtchRate"], + 101.5, + ) + self.assertEqual( + result["run_records"][0]["execution_request_id"], + "REQ-ABCDEF123456", + ) + self.assertTrue( + any( + "UPDATE etcher_runs" in query + and "project_id = %s" in query + and params[0] == "project-test" + for query, params in executions + if params + ) + ) + self.assertTrue( + any( + "pg_advisory_xact_lock" in query + and params == ("glance_source_run|101",) + for query, params in executions + ) + ) + self.assertTrue( + any( + "pg_advisory_xact_lock" in query + and params == ("glance_source_run|5|101",) + for query, params in executions + ) + ) self.assertTrue( any( "source_system, equipment_id, source_tool_id, source_run_id" @@ -251,9 +422,158 @@ def execute_values(cursor, query, values, **kwargs): for query in value_queries ) ) + self.assertTrue( + any( + "inputs_json = EXCLUDED.inputs_json" in query + and "outputs_json = EXCLUDED.outputs_json" in query + and "equipment_runs.inputs_json ||" not in query + and "equipment_runs.outputs_json ||" not in query + for query in value_queries + ) + ) self.assertTrue(connection.committed) self.assertFalse(connection.rolled_back) + def test_older_source_revision_cannot_replace_newer_trace(self): + state = { + "parent": None, + "samples": [], + "events": [], + "parent_upserts": 0, + } + + class Cursor: + def __init__(self): + self.rows = [] + + def execute(self, query, params=None): + normalized = " ".join(str(query).split()) + self.rows = [] + if ( + "SELECT id, source_run_id, source_updated_at" + in normalized + and state["parent"] + ): + parent = state["parent"] + self.rows = [ + ( + parent["id"], + parent["source_run_id"], + parent["source_updated_at"], + ) + ] + elif ( + "SELECT id, inputs_json, outputs_json" + in normalized + and state["parent"] + ): + parent = state["parent"] + self.rows = [ + ( + parent["id"], + parent["inputs"], + parent["outputs"], + ) + ] + elif "DELETE FROM equipment_run_trace_samples" in normalized: + state["samples"] = [] + elif "DELETE FROM equipment_run_trace_events" in normalized: + state["events"] = [] + + def fetchall(self): + return list(self.rows) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self, cursor_factory=None): + return Cursor() + + def commit(self): + pass + + def rollback(self): + pass + + def close(self): + pass + + def execute_values(cursor, query, values, **kwargs): + normalized = " ".join(str(query).split()) + if "INSERT INTO equipment_runs" in normalized: + state["parent_upserts"] += 1 + row = values[0] + state["parent"] = { + "id": 7001, + "source_run_id": row[13], + "inputs": dict(row[7].adapted), + "outputs": dict(row[8].adapted), + "source_updated_at": row[15], + } + return [(7001, row[13])] + if "INSERT INTO equipment_run_trace_samples" in normalized: + state["samples"] = list(values) + if "INSERT INTO equipment_run_trace_events" in normalized: + state["events"] = list(values) + return None + + def run_payload(*, revision: datetime, pressure: float, sample: float): + return sync_equipment_run_traces_pg( + equipment_id="equipment-test", + project_id="project-test", + source_system="glance", + source_tool_id=5, + return_stats=True, + runs=[ + { + "source_run_id": 101, + "source_updated_at": revision, + "run_start_time": "2026-07-01T10:00:00", + "inputs": {"Etch_AvgPres": pressure}, + "samples": [ + { + "source_sample_id": 1001, + "sample_index": 0, + "source_tool_id": 5, + "timestamp": "2026-07-01T10:00:00", + "values": {"p1": sample}, + } + ], + "events": [], + } + ], + ) + + newer = datetime(2026, 7, 2, tzinfo=timezone.utc) + older = datetime(2026, 7, 1, tzinfo=timezone.utc) + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=Connection(), + ), + patch( + "psycopg2.extras.execute_values", + side_effect=execute_values, + ), + ): + first = run_payload(revision=newer, pressure=50.0, sample=5.0) + delayed_retry = run_payload( + revision=older, + pressure=10.0, + sample=1.0, + ) + + self.assertEqual(first["updated"], 0) + self.assertEqual(delayed_retry["unchanged"], 1) + self.assertEqual(delayed_retry["run_records"], []) + self.assertEqual(state["parent_upserts"], 1) + self.assertEqual(state["parent"]["inputs"]["Etch_AvgPres"], 50.0) + self.assertEqual(state["samples"][0][6].adapted["p1"], 5.0) + class ProductionGlanceEndpointTests(unittest.TestCase): def setUp(self) -> None: @@ -338,6 +658,213 @@ def test_endpoint_requires_token_and_validates_authoritative_mapping(self): self.assertTrue(response.json()["cursor_acknowledged"]) self.assertEqual(response.json()["inserted"], 1) + def test_mixed_batch_rejects_nonfinite_and_oversized_ids_per_run(self): + mapping = ( + '{"equipment-test":{"source_tool_id":5,' + '"source_tool_name":"Tool 5","project_id":"project-test"}}' + ) + environment = { + "INGESTION_TOKEN": "test-ingestion-token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": mapping, + } + invalid_number = copy.deepcopy(self.payload["runs"][0]) + invalid_number["source_run_id"] = 102 + invalid_number["samples"][0]["source_sample_id"] = 1002 + invalid_number["samples"][0]["values"] = {"p1": float("nan")} + oversized_id = copy.deepcopy(self.payload["runs"][0]) + oversized_id["source_run_id"] = 103 + oversized_id["samples"][0]["source_sample_id"] = ( + 9_223_372_036_854_775_808 + ) + payload = copy.deepcopy(self.payload) + payload["runs"].extend([invalid_number, oversized_id]) + persisted_runs: list[dict] = [] + + def persist_valid_runs(**kwargs): + persisted_runs.extend(kwargs["runs"]) + return { + "inserted": len(kwargs["runs"]), + "updated": 0, + "run_count": len(kwargs["runs"]), + "sample_count": sum( + len(run["samples"]) for run in kwargs["runs"] + ), + "event_count": 0, + "run_records": [], + } + + with ( + patch.dict(os.environ, environment, clear=False), + patch( + "routers.dataset_v2.get_equipment_pg", + return_value={"parameters": [], "outputs": []}, + ), + patch( + "routers.dataset_v2.get_glance_ingestion_project_pg", + return_value={ + "id": "project-test", + "equipment_id": "equipment-test", + }, + ), + patch( + "routers.dataset_v2.sync_equipment_run_traces_pg", + side_effect=persist_valid_runs, + ), + patch( + "routers.dataset_v2.reconcile_glance_trace_runs_pg", + return_value={}, + ), + patch("routers.dataset_v2.record_glance_ingestion_audit_pg"), + ): + response = self.client.post( + "/dataset/v2/glance/traces/sync", + content=json.dumps(payload, allow_nan=True), + headers={ + "Content-Type": "application/json", + "X-Ingestion-Token": "test-ingestion-token", + }, + ) + + self.assertEqual(response.status_code, 207) + body = response.json() + self.assertEqual(body["status"], "partial") + self.assertFalse(body["cursor_acknowledged"]) + self.assertEqual(body["run_count"], 1) + self.assertEqual(body["rejected_count"], 2) + self.assertEqual( + [run["source_run_id"] for run in persisted_runs], + [101], + ) + errors = " ".join(item["error"] for item in body["rejected"]) + self.assertIn("non-finite number", errors) + self.assertIn("must be between 1 and", errors) + + def test_mixed_batch_isolates_every_duplicate_identity_class(self): + mapping = ( + '{"equipment-test":{"source_tool_id":5,' + '"source_tool_name":"Tool 5","project_id":"project-test"}}' + ) + environment = { + "INGESTION_TOKEN": "test-ingestion-token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": mapping, + } + + def invalid_run() -> dict: + run = copy.deepcopy(self.payload["runs"][0]) + run["source_run_id"] = 102 + return run + + duplicate_parameter_key = invalid_run() + duplicate_parameter_key["parameters"] = [ + {"source_parameter_id": 1, "key": "1"}, + {"source_parameter_id": 2, "key": "p1"}, + ] + duplicate_source_parameter_id = invalid_run() + duplicate_source_parameter_id["parameters"] = [ + {"source_parameter_id": 7, "key": "first"}, + {"source_parameter_id": "p7", "key": "second"}, + ] + duplicate_sample_id = invalid_run() + duplicate_sample_id["samples"].append( + { + "source_sample_id": 1001, + "timestamp": "2026-07-01T10:00:01", + "values": {}, + } + ) + duplicate_sample_index = invalid_run() + duplicate_sample_index["samples"][0]["sample_index"] = 4 + duplicate_sample_index["samples"].append( + { + "source_sample_id": 1002, + "sample_index": 4, + "timestamp": "2026-07-01T10:00:01", + "values": {}, + } + ) + duplicate_event_id = invalid_run() + duplicate_event_id["events"] = [ + { + "source_event_id": 9, + "timestamp": "2026-07-01T10:00:01", + }, + { + "source_event_id": 9, + "timestamp": "2026-07-01T10:00:01", + }, + ] + cases = { + "parameter key": duplicate_parameter_key, + "source parameter ID": duplicate_source_parameter_id, + "sample id": duplicate_sample_id, + "sample index": duplicate_sample_index, + "event id": duplicate_event_id, + } + + for expected_error, malformed in cases.items(): + with self.subTest(identity=expected_error): + payload = copy.deepcopy(self.payload) + payload["runs"].append(malformed) + persisted_runs: list[dict] = [] + + def persist_valid_runs(**kwargs): + persisted_runs.extend(kwargs["runs"]) + return { + "inserted": len(kwargs["runs"]), + "updated": 0, + "run_count": len(kwargs["runs"]), + "sample_count": sum( + len(run["samples"]) for run in kwargs["runs"] + ), + "event_count": 0, + "run_records": [], + } + + with ( + patch.dict(os.environ, environment, clear=False), + patch( + "routers.dataset_v2.get_equipment_pg", + return_value={"parameters": [], "outputs": []}, + ), + patch( + "routers.dataset_v2.get_glance_ingestion_project_pg", + return_value={ + "id": "project-test", + "equipment_id": "equipment-test", + }, + ), + patch( + "routers.dataset_v2.sync_equipment_run_traces_pg", + side_effect=persist_valid_runs, + ), + patch( + "routers.dataset_v2.reconcile_glance_trace_runs_pg", + return_value={}, + ), + patch( + "routers.dataset_v2.record_glance_ingestion_audit_pg" + ), + ): + response = self.client.post( + "/dataset/v2/glance/traces/sync", + json=payload, + headers={ + "X-Ingestion-Token": "test-ingestion-token" + }, + ) + + self.assertEqual(response.status_code, 207) + self.assertFalse(response.json()["cursor_acknowledged"]) + self.assertEqual(response.json()["run_count"], 1) + self.assertEqual( + [run["source_run_id"] for run in persisted_runs], + [101], + ) + self.assertIn( + expected_error, + response.json()["rejected"][0]["error"], + ) + class ProductionGlanceCursorTests(unittest.TestCase): def _settings(self) -> ConnectorSettings: diff --git a/api/tests/test_retrain.py b/api/tests/test_retrain.py index 488904e..ee12f3c 100644 --- a/api/tests/test_retrain.py +++ b/api/tests/test_retrain.py @@ -5,6 +5,7 @@ from unittest.mock import patch import numpy as np +import pandas as pd API_DIR = Path(__file__).resolve().parents[1] @@ -57,6 +58,217 @@ def test_negative_environment_limit_disables_loocv(self): with patch.dict(os.environ, {"RETRAIN_LOOCV_MAX_ROWS": "-1"}): self.assertEqual(retrain._configured_loocv_max_rows(), 0) + def test_small_authoritative_postgres_dataset_never_uses_csv_lineage(self): + pg_frame = pd.DataFrame( + [{"idruns": index} for index in range(5)] + ) + + class Cursor: + def execute(self, query, params=None): + self.query = str(query) + + def fetchone(self): + return (True,) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class Connection: + def cursor(self): + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + with ( + patch( + "data_loader_pg.get_training_df_pg", + return_value=pg_frame, + ), + patch("data_loader.get_clean_df") as csv_fallback, + ): + selected, source = retrain._load_training_dataset() + + self.assertEqual(source, "postgres") + self.assertEqual(len(selected), 5) + csv_fallback.assert_not_called() + + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=Connection(), + ), + patch.object( + retrain, + "_load_training_dataset", + return_value=(pg_frame, "postgres"), + ), + patch.object( + retrain, + "_current_snapshot", + return_value=(7, "a" * 64, 5), + ), + patch("model_registry.save_model") as save_model, + patch( + "model_registry.deactivate_active_models", + return_value=["AvgEtchRate"], + ) as deactivate_models, + patch( + "model_registry.update_retrain_state" + ) as update_state, + ): + result = retrain.run_retrain(reason="glance_ingestion") + + self.assertEqual(result["status"], "insufficient_data") + self.assertEqual(result["training_source"], "postgres") + self.assertEqual(result["deactivated_targets"], ["AvgEtchRate"]) + save_model.assert_not_called() + deactivate_models.assert_called_once() + update_state.assert_not_called() + + def test_empty_postgres_frame_is_authoritative(self): + empty_pg_frame = pd.DataFrame() + with ( + patch( + "data_loader_pg.get_training_df_pg", + return_value=empty_pg_frame, + ), + patch("data_loader.get_clean_df") as csv_fallback, + ): + selected, source = retrain._load_training_dataset() + + self.assertEqual(source, "postgres") + self.assertTrue(selected.empty) + csv_fallback.assert_not_called() + + def test_postgres_failure_does_not_silently_promote_csv(self): + with ( + patch( + "data_loader_pg.get_training_df_pg", + side_effect=RuntimeError("database unavailable"), + ), + patch("data_loader.get_clean_df") as csv_fallback, + self.assertRaisesRegex( + RuntimeError, + "Authoritative Postgres training load failed", + ), + ): + retrain._load_training_dataset() + + csv_fallback.assert_not_called() + + def test_target_below_minimum_is_deactivated_while_others_retrain(self): + frame = pd.DataFrame( + { + "feature": np.arange(10, dtype=float), + "primary": np.arange(10, dtype=float), + "secondary": [ + *np.arange(9, dtype=float), + np.nan, + ], + } + ) + trained = { + "model_kind": "gpr", + "metrics": {}, + "bundle": {}, + } + + class LockCursor: + def execute(self, query, params=None): + self.query = str(query) + + def fetchone(self): + return (True,) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + class LockConnection: + def cursor(self): + return LockCursor() + + def commit(self): + pass + + def close(self): + pass + + with ( + patch( + "data_loader_pg.get_pg_superuser_connection", + return_value=LockConnection(), + ), + patch.object( + retrain, + "_load_training_dataset", + return_value=(frame, "postgres"), + ), + patch.object( + retrain, + "_current_snapshot", + return_value=(7, "b" * 64, 10), + ), + patch.object( + retrain, + "_targets_and_features", + return_value=( + ["feature"], + ["primary", "secondary"], + ), + ), + patch.object( + retrain, + "_train_one_target", + return_value=trained, + ), + patch( + "model_registry.get_retrain_state", + return_value={}, + ), + patch( + "model_registry.save_model", + return_value=42, + ), + patch( + "model_registry.promote_model", + return_value={ + "promoted": True, + "reason": "forced", + }, + ) as promote_model, + patch( + "model_registry.deactivate_active_models", + return_value=["secondary"], + ) as deactivate_models, + patch("model_registry.update_retrain_state"), + ): + result = retrain.run_retrain( + reason="glance_dataset_correction", + force=True, + ) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["deactivated_targets"], ["secondary"]) + promote_model.assert_called_once_with(model_id=42, force=True) + deactivate_models.assert_called_once_with( + domain_id="etcher", + targets=["secondary"], + reason=( + "target has fewer than 10 usable rows in authoritative " + f"postgres snapshot {'b' * 64}" + ), + ) + if __name__ == "__main__": unittest.main() diff --git a/api/training/retrain.py b/api/training/retrain.py index 7b6e537..77412a7 100644 --- a/api/training/retrain.py +++ b/api/training/retrain.py @@ -28,6 +28,8 @@ from __future__ import annotations import argparse +import hashlib +import json import logging import os import sys @@ -120,23 +122,36 @@ def _loocv_scores( # ── Core training ──────────────────────────────────────────────────────────── -def _load_training_frame(): - """Pull the clean training frame from Postgres, fall back to CSV.""" +def _load_training_dataset( + *, + allow_csv_fallback: bool | None = None, +): + """Return one authoritative training frame and its provenance.""" + if allow_csv_fallback is None: + allow_csv_fallback = ( + os.getenv("RETRAIN_ALLOW_CSV_FALLBACK", "false").lower() + == "true" + ) try: from data_loader_pg import get_training_df_pg - except Exception: + except Exception as exc: + if not allow_csv_fallback: + raise RuntimeError( + "Authoritative Postgres training loader is unavailable" + ) from exc get_training_df_pg = None from data_loader import get_clean_df, RANGE_NM, SECONDARY_TARGET - df = None if get_training_df_pg is not None: try: pg_df = get_training_df_pg() - if pg_df is not None and not pg_df.empty: + if pg_df is not None: pg_df = pg_df.copy() if SECONDARY_TARGET in pg_df.columns and RANGE_NM not in pg_df.columns: - pg_df[RANGE_NM] = pg_df[SECONDARY_TARGET] * 5.0 + pg_df.loc[:, RANGE_NM] = ( + pg_df[SECONDARY_TARGET] * 5.0 + ) import pandas as pd mask = pd.Series([True] * len(pg_df), index=pg_df.index) if "is_outlier" in pg_df.columns: @@ -144,16 +159,33 @@ def _load_training_frame(): if "is_calibration_recipe" in pg_df.columns: mask &= pg_df["is_calibration_recipe"] != True # noqa: E712 pg_df = pg_df[mask].reset_index(drop=True) - if len(pg_df) >= 10: - df = pg_df - logger.info("Retrain: loaded %d rows from Postgres", len(df)) + logger.info( + "Retrain: loaded %d authoritative rows from Postgres", + len(pg_df), + ) + return pg_df, "postgres" + raise RuntimeError( + "Postgres training query returned no authoritative frame" + ) except Exception as exc: - logger.warning("Retrain: Postgres load failed (%s), using CSV", exc) + if not allow_csv_fallback: + raise RuntimeError( + "Authoritative Postgres training load failed" + ) from exc + logger.warning( + "Retrain: Postgres load failed (%s); explicit CSV " + "fallback is enabled", + exc, + ) + + df = get_clean_df() + logger.info("Retrain: loaded %d rows from CSV fallback", len(df)) + return df, "csv" + - if df is None: - df = get_clean_df() - logger.info("Retrain: loaded %d rows from CSV fallback", len(df)) - return df +def _load_training_frame(): + """Compatibility wrapper returning only the selected training frame.""" + return _load_training_dataset()[0] def _targets_and_features() -> tuple[list[str], list[str]]: @@ -280,22 +312,54 @@ def wrapped(*args, **kwargs): return wrapped -def _current_snapshot() -> tuple[Optional[int], Optional[str], int]: +def _current_snapshot( + training_df=None, + *, + source: str | None = None, +) -> tuple[Optional[int], Optional[str], int]: """Return (snapshot_id, snapshot_hash, num_rows) for the current dataset. Records a new snapshot row if the hash is new. """ - from ai_readiness import export_ml_matrix, record_dataset_snapshot + from ai_readiness import record_dataset_snapshot from data_loader_pg import get_pg_superuser_connection - result = export_ml_matrix( - include_outliers=False, include_metadata=False, is_admin=True, + if training_df is None: + training_df, source = _load_training_dataset() + snapshot_df = training_df.copy() + if "is_outlier" in snapshot_df.columns: + snapshot_df = snapshot_df[ + snapshot_df["is_outlier"] != True # noqa: E712 + ] + if "is_calibration_recipe" in snapshot_df.columns: + snapshot_df = snapshot_df[ + snapshot_df["is_calibration_recipe"] != True # noqa: E712 + ] + if "idruns" in snapshot_df.columns: + snapshot_df = snapshot_df.sort_values( + "idruns", + kind="mergesort", + ) + records = json.loads( + snapshot_df.to_json( + orient="records", + date_format="iso", + default_handler=str, + ) ) - snapshot_hash = result["snapshot_hash"] - num_rows = int(result["num_rows"]) + snapshot_hash = hashlib.sha256( + json.dumps( + records, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + ).hexdigest() + num_rows = int(len(snapshot_df)) record_dataset_snapshot( snapshot_hash=snapshot_hash, num_rows=num_rows, trigger="model_training", + metadata={"training_source": source or "unknown"}, ) conn = get_pg_superuser_connection() @@ -326,7 +390,11 @@ def run_retrain( ) -> dict[str, Any]: """Train a fresh model per target, save, promote, update state.""" from model_registry import ( - get_retrain_state, promote_model, save_model, update_retrain_state, + deactivate_active_models, + get_retrain_state, + promote_model, + save_model, + update_retrain_state, ) if loocv_max_rows is None: @@ -334,7 +402,38 @@ def run_retrain( else: loocv_max_rows = max(0, loocv_max_rows) - snapshot_id, snapshot_hash, num_rows = _current_snapshot() + df, training_source = _load_training_dataset( + # A forced correction must never promote a model from unrelated CSV + # lineage if the authoritative Postgres read is empty or fails. + allow_csv_fallback=False if force else None, + ) + snapshot_id, snapshot_hash, num_rows = _current_snapshot( + df, + source=training_source, + ) + features, targets = _targets_and_features() + if num_rows < 10: + logger.warning( + "Retrain skipped: authoritative %s dataset has only %d rows", + training_source, + num_rows, + ) + deactivated_targets = deactivate_active_models( + domain_id=domain_id, + targets=targets, + reason=( + f"authoritative {training_source} frame has {num_rows} " + f"rows at snapshot {snapshot_hash}" + ), + ) + return { + "status": "insufficient_data", + "snapshot_hash": snapshot_hash, + "num_rows": num_rows, + "training_source": training_source, + "reason": reason, + "deactivated_targets": deactivated_targets, + } state = get_retrain_state(domain_id) or {} if ( not force @@ -353,8 +452,6 @@ def run_retrain( "reason": reason, } - df = _load_training_frame() - features, targets = _targets_and_features() valid_features = [f for f in features if f in df.columns] if len(valid_features) < len(features): missing = set(features) - set(valid_features) @@ -362,10 +459,12 @@ def run_retrain( results: list[dict[str, Any]] = [] last_model_id: Optional[int] = None + insufficient_targets: list[str] = [] for target in targets: if target not in df.columns: logger.warning("Retrain: target '%s' not in dataframe, skipping", target) + insufficient_targets.append(target) continue mask = df[valid_features + [target]].notna().all(axis=1) sub = df[mask].reset_index(drop=True) @@ -373,6 +472,7 @@ def run_retrain( logger.warning( "Retrain[%s]: only %d usable rows, skipping.", target, len(sub), ) + insufficient_targets.append(target) continue X_raw = sub[valid_features].values.astype(float) @@ -399,6 +499,11 @@ def run_retrain( ) last_model_id = model_id promotion = promote_model(model_id=model_id, force=force) + if force and not promotion["promoted"]: + raise RuntimeError( + f"Forced correction model {model_id} for {target!r} " + f"was not promoted: {promotion['reason']}" + ) results.append({ "target": target, "model_id": model_id, @@ -408,6 +513,26 @@ def run_retrain( "promotion_reason": promotion["reason"], }) + deactivated_targets = deactivate_active_models( + domain_id=domain_id, + targets=insufficient_targets, + reason=( + f"target has fewer than 10 usable rows in authoritative " + f"{training_source} snapshot {snapshot_hash}" + ), + ) + if not results: + return { + "status": "insufficient_data", + "domain_id": domain_id, + "snapshot_hash": snapshot_hash, + "num_rows": num_rows, + "training_source": training_source, + "reason": reason, + "targets": [], + "deactivated_targets": deactivated_targets, + } + update_retrain_state( domain_id=domain_id, last_snapshot_hash=snapshot_hash, @@ -420,9 +545,11 @@ def run_retrain( "domain_id": domain_id, "snapshot_hash": snapshot_hash, "num_rows": num_rows, + "training_source": training_source, "trained_at": datetime.now(timezone.utc).isoformat(), "reason": reason, "targets": results, + "deactivated_targets": deactivated_targets, } logger.info( "Retrain done: domain=%s snapshot=%s num_rows=%d targets=%d", @@ -446,13 +573,30 @@ def run_retrain( def run_retrain_safe(**kwargs) -> dict[str, Any]: """Wrapper used by the ingestion BackgroundTask: never raise.""" try: - return run_retrain(**kwargs) + result = run_retrain(**kwargs) except _LockHeld as exc: logger.info("Retrain not run: %s", exc) - return {"status": "skipped_locked", "reason": str(exc)} + result = {"status": "skipped_locked", "reason": str(exc)} except Exception as exc: logger.error("Retrain failed: %s", exc, exc_info=True) - return {"status": "error", "error": str(exc)} + result = {"status": "error", "error": str(exc)} + try: + from metadata_pg import drain_recipe_optimizer_wakeups_pg + + result["optimizer_wakeups"] = ( + drain_recipe_optimizer_wakeups_pg() + ) + except Exception as exc: + logger.warning( + "Could not drain durable recipe optimizer wake-ups: %s", + exc, + exc_info=True, + ) + result["optimizer_wakeups"] = { + "status": "pending", + "error": str(exc), + } + return result def _parse_args(argv: list[str]) -> argparse.Namespace: @@ -489,7 +633,17 @@ def main(argv: Optional[list[str]] = None) -> int: # Summarise to stdout for CronJob logs. status = out.get("status", "error") logger.info("Retrain exit status: %s", status) - return 0 if status in {"ok", "skipped_no_data_change", "skipped_locked"} else 1 + return ( + 0 + if status + in { + "ok", + "insufficient_data", + "skipped_no_data_change", + "skipped_locked", + } + else 1 + ) if __name__ == "__main__": diff --git a/azure/.gitignore b/azure/.gitignore index 1b65495..1158962 100644 --- a/azure/.gitignore +++ b/azure/.gitignore @@ -193,3 +193,6 @@ process_etcher_data/OUTPUT_DATASET.csv # Timer-triggered variant (not used) process_etcher_data_timer/ + +# Production identity regression test. +!tests/test_legacy_summary_identity.py diff --git a/azure/common/glance_connector.py b/azure/common/glance_connector.py index fd5b847..f4811ba 100644 --- a/azure/common/glance_connector.py +++ b/azure/common/glance_connector.py @@ -25,6 +25,8 @@ logger = logging.getLogger("dt.azure.glance_connector") _T = TypeVar("_T") +_EXECUTION_REQUEST_ID_RE = re.compile(r"REQ-[A-F0-9]{12}", re.IGNORECASE) +_PG_BIGINT_MAX = 9_223_372_036_854_775_807 class ConnectorConfigurationError(RuntimeError): @@ -97,6 +99,7 @@ def from_env(cls) -> "ConnectorSettings": ) default_initial = os.getenv("GLANCE_INITIAL_RUN_ID", "").strip() mappings = [] + equipment_by_tool_id: dict[int, str] = {} for equipment_id, raw_mapping in mapping_json.items(): if not isinstance(raw_mapping, dict): raise ConnectorConfigurationError( @@ -111,14 +114,36 @@ def from_env(cls) -> "ConnectorSettings": f"Mapping for {equipment_id!r} requires numeric " "source_tool_id and initial_run_id" ) from exc + if not 1 <= tool_id <= _PG_BIGINT_MAX: + raise ConnectorConfigurationError( + f"Mapping for {equipment_id!r} has source_tool_id " + f"outside PostgreSQL BIGINT range" + ) + if not 0 <= initial_run_id <= _PG_BIGINT_MAX: + raise ConnectorConfigurationError( + f"Mapping for {equipment_id!r} has initial_run_id " + f"outside PostgreSQL BIGINT cursor range" + ) project_id = str(raw_mapping.get("project_id") or "").strip() if not project_id: raise ConnectorConfigurationError( f"Mapping for {equipment_id!r} requires project_id" ) + normalized_equipment_id = str(equipment_id) + previous_equipment_id = equipment_by_tool_id.get(tool_id) + if ( + previous_equipment_id is not None + and previous_equipment_id != normalized_equipment_id + ): + raise ConnectorConfigurationError( + f"GLANCE source_tool_id {tool_id} is mapped to both " + f"{previous_equipment_id!r} and " + f"{normalized_equipment_id!r}" + ) + equipment_by_tool_id[tool_id] = normalized_equipment_id mappings.append( ToolMapping( - equipment_id=str(equipment_id), + equipment_id=normalized_equipment_id, source_tool_id=tool_id, source_tool_name=str( raw_mapping.get("source_tool_name") or "" @@ -319,6 +344,33 @@ def _redacted_error(exc: Exception) -> str: return text[:1000] +def _source_experiment_identity( + run: dict[str, Any], + recipe: dict[str, Any], +) -> tuple[str, str]: + """Extract explicit DT identity, with exact standard-ID naming fallbacks.""" + request_id = str( + run.get("execution_request_id") + or recipe.get("execution_request_id") + or "" + ).strip() + if not request_id: + for candidate in ( + run.get("lotname"), + recipe.get("recipename"), + ): + value = str(candidate or "").strip() + if _EXECUTION_REQUEST_ID_RE.fullmatch(value): + request_id = value.upper() + break + proposal_id = str( + run.get("proposal_id") + or recipe.get("proposal_id") + or "" + ).strip() + return request_id, proposal_id + + class BlobCursorStore: """One JSON blob containing independent committed cursors per source tool.""" @@ -690,6 +742,7 @@ def complete_run( # identity, ordering, tool provenance, and absolute time. "select": "idsamplerecord,idruns,idtools,time", "idruns": f"eq.{run_id}", + "idtools": f"eq.{mapping.source_tool_id}", "order": "time.asc,idsamplerecord.asc", }, ) @@ -719,9 +772,12 @@ def complete_run( { "select": ( "idsamplerecord,idparameters,value," - "samplerecord!inner(idruns)" + "samplerecord!inner(idruns,idtools)" ), "samplerecord.idruns": f"eq.{run_id}", + "samplerecord.idtools": ( + f"eq.{mapping.source_tool_id}" + ), "and": ( f"(idsamplerecord.gte.{lower_sample_id}," f"idsamplerecord.lte.{upper_sample_id})" @@ -818,6 +874,10 @@ def complete_run( "file_size_bytes": recipe_file_size, "raw": recipe, } + execution_request_id, proposal_id = _source_experiment_identity( + run, + recipe, + ) samples = [] for index, sample in enumerate(sample_rows): @@ -848,6 +908,8 @@ def complete_run( "run_start_time": start_time, "run_end_time": end_time, "source_status": str(run.get("status") or ""), + "execution_request_id": execution_request_id, + "proposal_id": proposal_id, "recipe": recipe_provenance, "inputs": {}, "outputs": {}, @@ -1010,17 +1072,22 @@ def _bounded_values( def get_all(self, endpoint: str, params: dict[str, Any]) -> list[dict]: if endpoint == "/samplerecord": run_id = int(self._eq_value(params, "idruns")) + tool_id = int(self._eq_value(params, "idtools")) return self._query( """ SELECT idsamplerecord, idruns, idtools, time FROM public.samplerecord WHERE idruns = %s + AND idtools = %s ORDER BY time ASC NULLS LAST, idsamplerecord ASC """, - (run_id,), + (run_id, tool_id), ) if endpoint == "/data": run_id = int(self._eq_value(params, "samplerecord.idruns")) + tool_id = int( + self._eq_value(params, "samplerecord.idtools") + ) lower, upper = self._bounded_values(params, "idsamplerecord") return self._query( """ @@ -1029,11 +1096,12 @@ def get_all(self, endpoint: str, params: dict[str, Any]) -> list[dict]: JOIN public.samplerecord AS s ON s.idsamplerecord = d.idsamplerecord WHERE s.idruns = %s + AND s.idtools = %s AND d.idsamplerecord >= %s AND d.idsamplerecord <= %s ORDER BY d.idsamplerecord ASC, d.idparameters ASC """, - (run_id, int(lower), int(upper)), + (run_id, tool_id, int(lower), int(upper)), ) if endpoint == "/parameters": parameter_ids = self._in_values(params, "idparameters") diff --git a/azure/common/glance_identity.py b/azure/common/glance_identity.py new file mode 100644 index 0000000..299ce9a --- /dev/null +++ b/azure/common/glance_identity.py @@ -0,0 +1,27 @@ +"""Small identity guards shared by GLANCE ingestion paths.""" + +from __future__ import annotations + + +def filter_samples_to_selected_runs(samps, runs_df): + """Keep samples only when both their GLANCE tool and run were selected.""" + required = {"idtools", "idruns"} + if not required.issubset(samps.columns): + missing = sorted(required - set(samps.columns)) + raise RuntimeError( + "GLANCE sample records are missing identity columns: " + + ", ".join(missing) + ) + if not required.issubset(runs_df.columns): + missing = sorted(required - set(runs_df.columns)) + raise RuntimeError( + "Selected GLANCE runs are missing identity columns: " + + ", ".join(missing) + ) + selected_pairs = runs_df[["idtools", "idruns"]].drop_duplicates() + return samps.merge( + selected_pairs, + on=["idtools", "idruns"], + how="inner", + validate="many_to_one", + ) diff --git a/azure/process_etcher_data/__init__.py b/azure/process_etcher_data/__init__.py index 77ab7cc..3e44e24 100644 --- a/azure/process_etcher_data/__init__.py +++ b/azure/process_etcher_data/__init__.py @@ -16,6 +16,7 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from azure.identity import ClientSecretCredential +from common.glance_identity import filter_samples_to_selected_runs # ─── SHARED MODULE ───────────────────────────────────────────────────────────── # The common module provides reusable utilities shared across domain functions. @@ -77,8 +78,8 @@ # HTTP timeouts "TIMEOUT": (10, 120), # Which columns to pull - "SELECT_RUNS": "idruns,lotname,idrecipes", - "SELECT_SAMPLES": "idsamplerecord,time,idruns", + "SELECT_RUNS": "idruns,idtools,lotname,idrecipes", + "SELECT_SAMPLES": "idsamplerecord,time,idruns,idtools", "SELECT_DATA": "idsamplerecord,idparameters,value", } @@ -191,6 +192,11 @@ def _get_dt_ingestion_settings(): def _build_sync_payload(frame): """Serialize a DataFrame to JSON-safe records with ISO datetimes.""" sync_df = frame.copy() + # The API uses this marker to defer approximate legacy proposal matching + # until the authoritative full-trace identity is available. + sync_df["source_system"] = "glance_summary" + if "source_tool_id" not in sync_df.columns and "idtools" in sync_df.columns: + sync_df["source_tool_id"] = sync_df["idtools"] return json.loads(sync_df.to_json(orient="records", date_format="iso")) @@ -1731,7 +1737,21 @@ def _log(level, msg, *args): raise logging.info(f"Found {len(runs_etch)} etch runs and {len(runs_mm)} MM_DD_YYYY runs") - runs_df = pd.DataFrame(runs_etch + runs_mm).drop_duplicates("idruns") + runs_df = pd.DataFrame(runs_etch + runs_mm).drop_duplicates( + subset=["idtools", "idruns"] + ) + # The legacy scalar-summary table is keyed by idruns alone. Never merge + # two physical tools when a GLANCE installation reuses a run number; the + # complete-trace connector handles the durable composite identity. + duplicate_tool_counts = runs_df.groupby("idruns")["idtools"].nunique() + overlapping_run_ids = duplicate_tool_counts[ + duplicate_tool_counts > 1 + ].index.tolist() + if overlapping_run_ids: + raise RuntimeError( + "Legacy summary ingestion cannot safely represent GLANCE run IDs " + f"shared by multiple tools: {overlapping_run_ids[:10]}" + ) logging.info(f"Total unique runs after deduplication: {len(runs_df)}") # Check if we have any runs at all @@ -1861,6 +1881,19 @@ def fetch_samps(batch): send_no_data_notification() return func.HttpResponse(json.dumps({"message":"no new data"}), status_code=200) + samps = filter_samples_to_selected_runs(samps, runs_df) + if samps.empty: + logging.info( + "No sample records matched an exact selected tool/run pair" + ) + local_csv_path = get_local_csv_path() + save_local_copy_from_blob(cont, STEP_FILENAME, local_csv_path) + send_no_data_notification() + return func.HttpResponse( + json.dumps({"message": "no matching sample data"}), + status_code=200, + ) + logging.info(f"Created sample records DataFrame with {len(samps)} rows") samps["time"] = pd.to_datetime(samps["time"], utc=True, format='mixed') logging.info(f"Parsed timestamps for sample records") @@ -1906,8 +1939,17 @@ def fetch_data(batch): logging.info("Creating values DataFrame and merging data...") vals = ( pd.DataFrame(recs) - .merge(samps[["idsamplerecord","time","idruns"]], on="idsamplerecord") - .merge(runs_df[["idruns","lotname"]], on="idruns") + .merge( + samps[ + ["idsamplerecord", "time", "idruns", "idtools"] + ], + on="idsamplerecord", + ) + .merge( + runs_df[["idruns", "idtools", "lotname"]], + on=["idruns", "idtools"], + validate="many_to_one", + ) ) vals["time"] = pd.to_datetime(vals["time"], utc=True, format='mixed') logging.info(f"Merged data: {len(vals)} records across {len(vals['idruns'].unique())} runs") @@ -1961,6 +2003,15 @@ def fetch_data(batch): # Get runs_df with idrecipes for recipe setpoint extraction runs_with_recipes = runs_df[['idruns', 'lotname', 'idrecipes']].copy() if 'idrecipes' in runs_df.columns else None step_df = compute_all_step_averages(vals, pid_map, runs_df=runs_with_recipes) + if "idtools" in runs_df.columns and not step_df.empty: + step_df = step_df.merge( + runs_df[["idruns", "idtools"]].drop_duplicates( + subset=["idruns", "idtools"] + ), + on="idruns", + how="left", + validate="many_to_one", + ) logging.info(f"Computed step averages for {len(step_df)} runs") except Exception as e: logging.error(f"Failed to compute step averages: {str(e)}") diff --git a/azure/tests/test_glance_connector.py b/azure/tests/test_glance_connector.py index 54ac14d..89a03e3 100644 --- a/azure/tests/test_glance_connector.py +++ b/azure/tests/test_glance_connector.py @@ -20,6 +20,7 @@ from common.glance_connector import ( # noqa: E402 BlobCursorStore, + ConnectorConfigurationError, ConnectorSettings, FileFailureAuditStore, FileCursorStore, @@ -29,6 +30,7 @@ ToolMapping, _json_safe, _redacted_error, + _source_experiment_identity, ) @@ -55,6 +57,25 @@ class _ResourceNotFoundError(Exception): pass +class ExperimentIdentityTests(unittest.TestCase): + def test_exact_request_id_name_is_preserved(self): + request_id, proposal_id = _source_experiment_identity( + {"lotname": "req-abcdef123456"}, + {}, + ) + + self.assertEqual(request_id, "REQ-ABCDEF123456") + self.assertEqual(proposal_id, "") + + def test_request_id_is_not_inferred_from_free_text(self): + request_id, _ = _source_experiment_identity( + {"lotname": "sample REQ-ABCDEF123456 retry"}, + {}, + ) + + self.assertEqual(request_id, "") + + def _azure_core_modules() -> dict[str, ModuleType]: azure_module = ModuleType("azure") core_module = ModuleType("azure.core") @@ -335,6 +356,32 @@ def test_postgres_file_configuration_does_not_require_postgrest_or_azure(self): self.assertEqual(settings.postgrest_url, "") self.assertEqual(settings.storage_connection, "") + def test_duplicate_source_tool_mapping_is_rejected(self): + environment = { + "GLANCE_SOURCE_MODE": "postgres", + "GLANCE_DB_URI": "postgresql://readonly.invalid/logger", + "GLANCE_CURSOR_BACKEND": "file", + "GLANCE_CURSOR_FILE": "/state/cursors.json", + "GLANCE_INGESTION_API_URL": ( + "https://dt.invalid/api/dataset/v2/glance/traces/sync" + ), + "INGESTION_TOKEN": "token", + "GLANCE_PRODUCTION_MAPPINGS_JSON": ( + '{"equipment-a":{"source_tool_id":3,' + '"project_id":"project-a","initial_run_id":1},' + '"equipment-b":{"source_tool_id":3,' + '"project_id":"project-b","initial_run_id":1}}' + ), + } + with ( + patch.dict("os.environ", environment, clear=True), + self.assertRaisesRegex( + ConnectorConfigurationError, + "source_tool_id 3 is mapped to both", + ), + ): + ConnectorSettings.from_env() + def test_complete_run_uses_disjoint_run_scoped_ranges_and_preserves_values(self): client = _FixtureClient() payload = client.complete_run( @@ -359,7 +406,14 @@ def test_complete_run_uses_disjoint_run_scoped_ranges_and_preserves_values(self) ) for query in client.data_queries: self.assertEqual(query["samplerecord.idruns"], "eq.99") - self.assertIn("samplerecord!inner(idruns)", query["select"]) + self.assertEqual( + query["samplerecord.idtools"], + "eq.3", + ) + self.assertIn( + "samplerecord!inner(idruns,idtools)", + query["select"], + ) self.assertNotIn("idsamplerecord", { key for key in query if key != "select" }) @@ -456,6 +510,20 @@ def connect(*_args, **_kwargs): self.assertEqual(len(complete_run_queries), 1) self.assertIn("idruns = %s", complete_run_queries[0]) self.assertIn("idtools = %s", complete_run_queries[0]) + sample_query, sample_parameters = next( + execution + for execution in connections[1].cursor_instance.executions + if "FROM public.samplerecord" in execution[0] + ) + self.assertIn("idtools = %s", sample_query) + self.assertEqual(sample_parameters, (99, 3)) + data_query, data_parameters = next( + execution + for execution in connections[1].cursor_instance.executions + if "FROM public.data AS d" in execution[0] + ) + self.assertIn("s.idtools = %s", data_query) + self.assertEqual(data_parameters[:2], (99, 3)) def test_postgres_overlap_reads_actual_sparse_tool_rows(self): class SparseCursor(_SqlCursor): diff --git a/azure/tests/test_legacy_summary_identity.py b/azure/tests/test_legacy_summary_identity.py new file mode 100644 index 0000000..c5e408c --- /dev/null +++ b/azure/tests/test_legacy_summary_identity.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +import pandas as pd + + +AZURE_ROOT = Path(__file__).resolve().parents[1] +if str(AZURE_ROOT) not in sys.path: + sys.path.insert(0, str(AZURE_ROOT)) + +from common.glance_identity import ( # noqa: E402 + filter_samples_to_selected_runs, +) + + +class LegacySummaryIdentityTests(unittest.TestCase): + def test_hidden_other_tool_samples_cannot_contaminate_selected_run(self): + selected_runs = pd.DataFrame( + [ + { + "idtools": 3, + "idruns": 161, + "lotname": "etch7/30/2024", + } + ] + ) + fetched_samples = pd.DataFrame( + [ + { + "idsamplerecord": 301, + "idtools": 3, + "idruns": 161, + "time": "2026-07-01T00:00:00", + }, + { + "idsamplerecord": 401, + "idtools": 4, + "idruns": 161, + "time": "2026-07-01T00:00:00", + }, + ] + ) + + filtered = filter_samples_to_selected_runs( + fetched_samples, + selected_runs, + ) + + self.assertEqual(filtered["idsamplerecord"].tolist(), [301]) + self.assertEqual(filtered["idtools"].tolist(), [3]) + + +if __name__ == "__main__": + unittest.main() diff --git a/geddes/k8s/05-postgres.yaml b/geddes/k8s/05-postgres.yaml index 45a928d..7e67372 100644 --- a/geddes/k8s/05-postgres.yaml +++ b/geddes/k8s/05-postgres.yaml @@ -139,6 +139,13 @@ data: source_system VARCHAR(100) NOT NULL DEFAULT 'data_upload', source_tool_id BIGINT, source_run_id BIGINT, + execution_request_id VARCHAR(80) NOT NULL DEFAULT '', + closed_loop_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_claim_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_claim_kind VARCHAR(32) NOT NULL DEFAULT '', + closed_loop_followup_claim_token VARCHAR(36) NOT NULL DEFAULT '', + closed_loop_followup_claimed_at TIMESTAMP WITH TIME ZONE, source_updated_at TIMESTAMP WITH TIME ZONE, ingested_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP @@ -150,6 +157,9 @@ data: ON equipment_runs(project_id); CREATE INDEX IF NOT EXISTS idx_equipment_runs_upload_id ON equipment_runs(upload_id); + CREATE INDEX IF NOT EXISTS idx_equipment_runs_execution_request_id + ON equipment_runs(execution_request_id) + WHERE execution_request_id <> ''; CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row ON equipment_runs(upload_id, row_index) WHERE upload_id IS NOT NULL; @@ -424,6 +434,32 @@ data: CREATE INDEX IF NOT EXISTS idx_experiment_recipe_batches_experiment_id ON experiment_recipe_batches(experiment_id); + WITH duplicate_experiments AS ( + SELECT experiment_id + FROM experiment_recipe_batches + GROUP BY experiment_id + HAVING COUNT(*) > COUNT(DISTINCT iteration) + ), + renumbered AS ( + SELECT + b.id, + ROW_NUMBER() OVER ( + PARTITION BY b.experiment_id + ORDER BY b.iteration, b.created_at, b.id + ) AS normalized_iteration + FROM experiment_recipe_batches b + JOIN duplicate_experiments d + ON d.experiment_id = b.experiment_id + ) + UPDATE experiment_recipe_batches b + SET iteration = r.normalized_iteration + FROM renumbered r + WHERE b.id = r.id + AND b.iteration <> r.normalized_iteration; + + CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_batches_iteration + ON experiment_recipe_batches(experiment_id, iteration); + CREATE TABLE IF NOT EXISTS experiment_recipe_proposals ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), batch_id UUID NOT NULL REFERENCES experiment_recipe_batches(id) ON DELETE CASCADE, @@ -434,6 +470,7 @@ data: ingestion_status VARCHAR(50) NOT NULL DEFAULT 'waiting', proposal_json JSONB NOT NULL DEFAULT '{}'::jsonb, run_id INTEGER REFERENCES etcher_runs(idruns) ON DELETE SET NULL, + equipment_run_id BIGINT REFERENCES equipment_runs(id) ON DELETE SET NULL, lotname VARCHAR(255) DEFAULT '', completed_at TIMESTAMP WITH TIME ZONE, rejected_at TIMESTAMP WITH TIME ZONE, @@ -449,6 +486,18 @@ data: CREATE INDEX IF NOT EXISTS idx_experiment_recipe_proposals_project_status ON experiment_recipe_proposals(project_id, status); + CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_proposals_equipment_run_id + ON experiment_recipe_proposals(equipment_run_id) + WHERE equipment_run_id IS NOT NULL; + + CREATE TABLE IF NOT EXISTS retrain_run_credits ( + domain_id VARCHAR(100) NOT NULL, + credit_key VARCHAR(255) NOT NULL, + n_new INTEGER NOT NULL DEFAULT 1 CHECK (n_new > 0), + credited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (domain_id, credit_key) + ); + CREATE TABLE IF NOT EXISTS data_uploads ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), equipment_id VARCHAR(255) NOT NULL, diff --git a/geddes/k8s/postgres/schema_v2.sql b/geddes/k8s/postgres/schema_v2.sql index 32fb8cb..cf72628 100644 --- a/geddes/k8s/postgres/schema_v2.sql +++ b/geddes/k8s/postgres/schema_v2.sql @@ -114,6 +114,13 @@ CREATE TABLE IF NOT EXISTS equipment_runs ( source_system VARCHAR(100) NOT NULL DEFAULT 'data_upload', source_tool_id BIGINT, source_run_id BIGINT, + execution_request_id VARCHAR(80) NOT NULL DEFAULT '', + closed_loop_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_claim_revision_sha256 VARCHAR(64) NOT NULL DEFAULT '', + closed_loop_followup_claim_kind VARCHAR(32) NOT NULL DEFAULT '', + closed_loop_followup_claim_token VARCHAR(36) NOT NULL DEFAULT '', + closed_loop_followup_claimed_at TIMESTAMP WITH TIME ZONE, source_updated_at TIMESTAMP WITH TIME ZONE, ingested_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP @@ -122,6 +129,9 @@ CREATE TABLE IF NOT EXISTS equipment_runs ( CREATE INDEX IF NOT EXISTS idx_equipment_runs_equipment_id ON equipment_runs(equipment_id); CREATE INDEX IF NOT EXISTS idx_equipment_runs_project_id ON equipment_runs(project_id); CREATE INDEX IF NOT EXISTS idx_equipment_runs_upload_id ON equipment_runs(upload_id); +CREATE INDEX IF NOT EXISTS idx_equipment_runs_execution_request_id +ON equipment_runs(execution_request_id) +WHERE execution_request_id <> ''; -- Stable per-upload row identity so reprocessing an upload upserts in place -- (preserving each run's id / exposed run_id) instead of reallocating ids. CREATE UNIQUE INDEX IF NOT EXISTS uq_equipment_runs_upload_row @@ -396,6 +406,32 @@ CREATE TABLE IF NOT EXISTS experiment_recipe_batches ( CREATE INDEX IF NOT EXISTS idx_experiment_recipe_batches_experiment_id ON experiment_recipe_batches(experiment_id); +WITH duplicate_experiments AS ( + SELECT experiment_id + FROM experiment_recipe_batches + GROUP BY experiment_id + HAVING COUNT(*) > COUNT(DISTINCT iteration) +), +renumbered AS ( + SELECT + b.id, + ROW_NUMBER() OVER ( + PARTITION BY b.experiment_id + ORDER BY b.iteration, b.created_at, b.id + ) AS normalized_iteration + FROM experiment_recipe_batches b + JOIN duplicate_experiments d + ON d.experiment_id = b.experiment_id +) +UPDATE experiment_recipe_batches b +SET iteration = r.normalized_iteration +FROM renumbered r +WHERE b.id = r.id + AND b.iteration <> r.normalized_iteration; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_batches_iteration +ON experiment_recipe_batches(experiment_id, iteration); + CREATE TABLE IF NOT EXISTS experiment_recipe_proposals ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), batch_id UUID NOT NULL REFERENCES experiment_recipe_batches(id) ON DELETE CASCADE, @@ -406,6 +442,7 @@ CREATE TABLE IF NOT EXISTS experiment_recipe_proposals ( ingestion_status VARCHAR(50) NOT NULL DEFAULT 'waiting', proposal_json JSONB NOT NULL DEFAULT '{}'::jsonb, run_id INTEGER REFERENCES etcher_runs(idruns) ON DELETE SET NULL, + equipment_run_id BIGINT REFERENCES equipment_runs(id) ON DELETE SET NULL, lotname VARCHAR(255) DEFAULT '', completed_at TIMESTAMP WITH TIME ZONE, rejected_at TIMESTAMP WITH TIME ZONE, @@ -421,6 +458,18 @@ ON experiment_recipe_proposals(batch_id); CREATE INDEX IF NOT EXISTS idx_experiment_recipe_proposals_project_status ON experiment_recipe_proposals(project_id, status); +CREATE UNIQUE INDEX IF NOT EXISTS idx_experiment_recipe_proposals_equipment_run_id +ON experiment_recipe_proposals(equipment_run_id) +WHERE equipment_run_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS retrain_run_credits ( + domain_id VARCHAR(100) NOT NULL, + credit_key VARCHAR(255) NOT NULL, + n_new INTEGER NOT NULL DEFAULT 1 CHECK (n_new > 0), + credited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (domain_id, credit_key) +); + CREATE TABLE IF NOT EXISTS data_uploads ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), equipment_id VARCHAR(255) NOT NULL, diff --git a/web/lib/api-client.ts b/web/lib/api-client.ts index 6de8f81..9e2d943 100644 --- a/web/lib/api-client.ts +++ b/web/lib/api-client.ts @@ -1124,6 +1124,7 @@ export interface ExperimentRecipeProposal { status?: string; ingestion_status?: string; run_id?: number | null; + equipment_run_id?: number | null; lotname?: string | null; completed_at?: string | null; rejected_at?: string | null; From a14bfc3aa304e07e93ab08c97e9338df0c955b01 Mon Sep 17 00:00:00 2001 From: navidgh67 Date: Mon, 3 Aug 2026 12:39:46 -0400 Subject: [PATCH 3/3] Fix health check visibility of non-API-readable tables Co-authored-by: Cursor --- api/main.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/api/main.py b/api/main.py index 1d0bf9c..9a8bedc 100644 --- a/api/main.py +++ b/api/main.py @@ -100,11 +100,16 @@ def _postgres_health() -> dict[str, object]: with conn.cursor() as cur: cur.execute("SELECT current_database(), current_user") database_name, user_name = cur.fetchone() + # pg_tables lists tables regardless of the caller's privileges. + # information_schema.tables hides tables the API role cannot + # access (for example glance_ingestion_audit, which is + # deliberately not API-readable), which would report a healthy + # deployment as permanently degraded. cur.execute( """ - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' + SELECT tablename + FROM pg_catalog.pg_tables + WHERE schemaname = 'public' """ ) available_tables = {row[0] for row in cur.fetchall()}