diff --git a/.dockerignore b/.dockerignore index 72e2298..70522f3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,5 +26,10 @@ nanohub/config nanohub/data uploader/staging +# Generated uploads and local mock databases are runtime data, not image assets. +# Keep developer/test data out of API and worker builds. +api/data +api/local_data + .DS_Store *.log diff --git a/.gitignore b/.gitignore index 6b160ad..43c6823 100644 --- a/.gitignore +++ b/.gitignore @@ -60,9 +60,16 @@ debug_*.py !api/tests/test_e2e_platform.py !api/tests/test_glance_db3_trace_import.py !api/tests/test_retrain.py +!api/tests/test_monolayer.py +!api/tests/test_rcac_hardening.py !api/tests/test_production_glance_ingestion.py !api/tests/test_glance_closed_loop.py !api/tests/test_shadow_evaluation.py +!api/tests/test_experiment_run_endpoint.py +!api/tests/test_proposal_objectives.py +!api/tests/test_computation_cutoff_fidelity.py +!api/tests/test_experiment_values_postgres_e2e.py +!api/tests/test_pilot_review_permissions.py # Development files temp.py diff --git a/.serena/.gitignore b/.serena/.gitignore new file mode 100644 index 0000000..2e510af --- /dev/null +++ b/.serena/.gitignore @@ -0,0 +1,2 @@ +/cache +/project.local.yml diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 0000000..8d85dfd --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,169 @@ +# the name by which the project can be referenced within Serena/when chatting with the LLM. +project_name: "dt-computational-integration" + +# list of language servers to start when using the LSP backend; choose from: +# ada al angular ansible bash +# bsl clojure cpp cpp_ccls crystal +# csharp csharp_omnisharp cue dart deno +# elixir elm erlang fortran fsharp +# gdscript gleam go groovy haskell +# haxe hlsl html java json +# julia kotlin latex lean4 lua +# luau markdown matlab msl nextflow +# nix ocaml pascal perl php +# php_phpactor php_phpantom powershell python python_basedpyright +# python_jedi python_pyrefly python_ty qml r +# rego ruby ruby_solargraph rust scala +# scss solidity svelte swift systemverilog +# terraform toml typescript typescript_vts vue +# wolfram yaml zig +# (This list may be outdated; generated with scripts/print_language_list.py; +# For the current list, see values of the LanguageServerId enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py) +# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) +# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) +# - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH) +# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# Some language servers require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers +# When using multiple language servers, the first language server that supports a given file will be used for that file. +# The first language server is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +language_servers: +- python + +# the encoding used by text files in the project +# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings +encoding: "utf-8" + +# optional shell command to run before the language backend (LSP or JetBrains) is initialised. +# the command runs in the project root directory and is only executed if the project is trusted +# (see trusted_project_path_patterns in the global configuration). +# serena waits for the command to exit: a non-zero exit code is logged as an error but does not +# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety +# backstop for non-terminating commands; on expiry the process is killed and activation continues. +# example: activation_command: "npx nx run-many -t build" +activation_command: + +# maximum time in seconds to wait for activation_command to complete before killing it (default 180s). +# must be a positive number. +activation_command_timeout: 180.0 + +# line ending convention to use when writing source files. +# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default) +# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings. +line_ending: + +# The language backend to use for this project. +# If not set, the global setting from serena_config.yml is used. +# Valid values: LSP, JetBrains +# Note: the backend is fixed at startup. If a project with a different backend +# is activated post-init, an error will be returned. +language_backend: + +# whether to use project's .gitignore files to ignore files +ignore_all_files_in_gitignore: true + +# advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. +# The settings are considered only if the project is trusted (see global configuration to define trusted projects). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings +ls_specific_settings: {} + +# list of workspace folder paths (LSP backend only). +# These folders will be used to build up Serena's symbol index. +# Paths must be within the project root and should thus be relative to the project root. +# Furthermore, the paths should not be filtered by ignore settings. +# Default setting: The entire project root folder (".") is considered. +# In (large) monorepos, this can be used to index only subfolders of the project root, e.g. +# ls_workspace_folders: +# - "./subproject1" +# - "./subproject2" +ls_workspace_folders: +- "." + +# list of additional workspace folder paths for cross-package reference support. +# Paths can be absolute or relative to the project root. +# Each folder is registered as an LSP workspace folder, enabling language servers to discover +# symbols and references across package boundaries, but these folders are not indexed by Serena, +# i.e. the respective symbols will not be found using Serena's symbol search tools. +# Example: +# additional_workspace_folders: +# - ../sibling-package +# - ../shared-lib +ls_additional_workspace_folders: [] + +# list of additional paths to ignore in this project. +# Same syntax as gitignore, so you can use * and **. +# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases. +# Example: +# ignored_paths: +# - "examples/**" +# - ".worktrees/**" +# - "**/bin/**" +# - "**/obj/**" +# Note: global ignored_paths from serena_config.yml are also applied additively. +ignored_paths: [] + +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + +# list of tool names to exclude. +# This extends the existing exclusions (e.g. from the global configuration) +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +excluded_tools: [] + +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). +# This extends the existing inclusions (e.g. from the global configuration). +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +fixed_tools: [] + +# list of mode names that are to be activated by default, overriding the setting in the global configuration. +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply +# for this project. +# This setting can, in turn, be overridden by CLI parameters (--mode). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +default_modes: + +# list of mode names to be activated additionally for this project, e.g. ["query-projects"] +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +added_modes: + +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). +initial_prompt: "" + +# time budget (seconds) per tool call for the retrieval of additional symbol information +# such as docstrings or parameter information. +# This overrides the corresponding setting in the global configuration; see the documentation there. +# If null or missing, use the setting from the global configuration. +symbol_info_budget: + +# list of regex patterns which, when matched, mark a memory entry as read‑only. +# Extends the list from the global configuration, merging the two lists. +read_only_memory_patterns: [] + +# list of regex patterns for memories to completely ignore. +# Matching memories will not appear in list_memories or activate_project output +# and cannot be accessed via read_memory or write_memory. +# To access ignored memory files, use the read_file tool on the raw file path. +# Extends the list from the global configuration, merging the two lists. +# Example: ["_archive/.*", "_episodes/.*"] +ignored_memory_patterns: [] diff --git a/COMPUTATIONAL_PILOT_GUIDE.md b/COMPUTATIONAL_PILOT_GUIDE.md new file mode 100644 index 0000000..41dd48c --- /dev/null +++ b/COMPUTATIONAL_PILOT_GUIDE.md @@ -0,0 +1,119 @@ +# Computational experiments: student pilot + +Implementation branch: `feature/sim2l-rcac-integration`. +The existing production DT platform is unchanged. The instructions below apply +to an explicitly enabled pilot environment; the public feature is not deployed. +Start with the [Rushik/Justin handoff](COMPUTATIONAL_PILOT_HANDOFF.md), +[September 28 deployment validation](geddes/k8s/pilots/computations/PILOT_VALIDATION_2026-09-28.md), +and the [scientific review checklist](COMPUTATIONAL_SCIENTIFIC_REVIEW.md). +The isolated API, web, database, and worker are deployed with real nanoHUB SAML. +The validation report is the authority for current student access and readiness; +September 23 notes describe an earlier preparation stage. +See [live validation evidence](RCAC_E2E_VALIDATION_2026-09-22.md) and the +[administrator provisioning guide](RCAC_SIM2L_INTEGRATION.md). + +## What is available + +| Registered experiment | Scientific task | Supported input | +| --- | --- | --- | +| CellRelaxDFT, `r99-dt2` | Three-dimensional cell relaxation | Materials Project structure and DFT parameters | +| CellRelaxDFT 2D, `justin-dt1` | Extract and relax a monolayer with fixed vacuum | Justin's graphene example, `mp-48`; compatible layered structures subject to domain review | +| ElasticDFT, `r27-dt2` | Three-dimensional, second-order, stress/strain elastic tensor | A converged **3D** CellRelaxDFT run in the same project | + +Both Negishi (QE 7.1) and Gautschi (QE 7.3.1) are supported. A versioned +installation is registered as digital equipment; a scientific workflow is a unit +experiment linked to compatible installations. Each run records which actual +installation produced it. Existing physical-equipment routes are unchanged. + +This is not arbitrary Python/notebook/shell upload and execution. Only registered, +reviewed workflows are executable. There is no Sim2L-library import or required +Sim2L-repository merge. DT owns these adapters, scientific contracts, scheduling, +and persistence. Original notebooks remain reference implementations. + +## Student workflow + +1. Sign in normally and open a project where you have write access. +2. Find **Registered computational experiments**. Select a tool and cluster. +3. For graphene, select **CellRelaxDFT 2D** and `mp-48`. Defaults are 180 Ry, + an 8×8×1 k-grid, 20 Å vacuum, PBEsol/PAW, and zero external pressure. + Common parameters have form fields; other supported settings are in the + advanced JSON form. The worker resolves the Materials Project ID using the + server-held key, never a key supplied in the notebook or stored in results. +4. Submit once. The run panel shows its state, Slurm ID, cluster, requester, and + service-account execution identity. A cancel action is available for active + runs. A submission is not evidence of scientific convergence. +5. When results are ingested, expand the run and inspect convergence, warnings, + inputs, structured outputs, and provenance. Download the JSON export. +6. For **3D** elasticity, first complete a converged 3D CellRelaxDFT run, then + select it as the upstream result in ElasticDFT. Parameters and the relaxed + structure are inherited. A 2D relaxation is deliberately not offered as a + parent for the bulk elastic calculator. +7. If result transport failed after Slurm ran, use **Retry result retrieval**. + This rechecks the same job and retrieves its output; it does not submit a + replacement calculation. Scientific failures require investigation, not this + transport-recovery action. + +The live silicon tests used a deliberately coarse 30 Ry/2×2×2 setup and 12 strain +members to validate infrastructure cheaply. Those are **not recommended scientific +production settings**. Justin/Rushik should supply convergence-tested settings. + +## Execution, identity, and isolation + +The browser calls the DT backend through its authenticated proxy. DT checks +project permissions, validates registered inputs, and persists the request before +the worker invokes the allow-listed Globus gateway. The endpoint stages inputs +in scratch and submits `sbatch`; QE runs on allocated compute nodes, not login +nodes. Slurm polling is centralized and batched at approximately 30-second +intervals rather than performed by each browser. + +Jobs run as service account `dtwin0`, using `ncn / cpu / normal`. DT records the +authenticated requesting student and adds `--comment=dt-run:` itself. +This preserves attribution, **not individual-student Unix ownership or access to +their other queues**. No student password, Duo response, or private SSH key is +required by the runtime. Globus and Materials Project service secrets remain in +the isolated namespace. Geddes etcd is not encrypted at rest; namespace access +controls, secret rotation, and acceptance of that residual risk remain operational +responsibilities. Switching to a service account does not encrypt its secrets. + +## Data and scientific meaning + +- Portable pymatgen structure dictionaries, numeric arrays, explicit units, and + JSON provenance are stored in the DT database, not Python pickles. +- Runs retain inputs, upstream linkage, installation/modules, pseudopotential + hashes, job identity, and content hashes for archived artifacts. +- Relaxation results retain energies, stresses, forces, and available cell/atomic + coordinate histories. The independent history arrays are not automatically a + frame-aligned ML training dataset. +- Elastic results additionally retain each deformation's input structure and, + when present, final relaxed structure paired with final energy, forces, and + stress. `complete_structure_force_pair` is false when required information is + absent; missing coordinates are never silently replaced by unrelaxed ones. +- 2D results record the original bulk structure, extracted site indices, + orientation/rotation, initial monolayer, vacuum, and final relaxed cell. + Supercell pressure/stress and volume include the vacuum normalization: they + are **not 2D elastic moduli**. The pilot uses periodic vacuum, not Coulomb + truncation, and rejects unsupported/mixed-dimensional input rather than guessing. +- Raw selected QE inputs, output logs, scheduler logs, and the elastic input + archive are retained under `/depot/ncn/data/digital-twins/runs//`. + Hashes and sizes are in the database. The branch now supports opt-in, + project-authorized downloads up to 8 MiB with checksum verification, enabled + in the isolated pilot. An authorized maintainer can retrieve larger files. Large `qe_tmp` + restart/wavefunction files are not part of this selected-artifact archive. + +## Boundaries before wider release + +Scientific approval of Justin's 2D adapter and convergence settings is still +needed. The supplied reference CIF and metadata are not a full independently +validated reference-output suite. The infrastructure tests do not prove the +accuracy of an arbitrary material's result. + +2D elasticity, third-order tensors, an energy-fitting switch, automatic +multi-cluster scheduling, and arbitrary script uploads are not implemented. +Those need the actual scientific workflows and agreed contracts. No dummy tools +or zero-filled tensor components have been registered in their place. + +The isolated pilot uses independent session/SAML credentials, a fully migrated +private database, and the students' actual nanoHUB identities. Earlier temporary +tests used signed test sessions; September 28 real sign-ins and Navid's confirmed +logout/re-login cover the actual external login path. Production remains disabled +until its separate rollout is authorized. diff --git a/COMPUTATIONAL_PILOT_HANDOFF.md b/COMPUTATIONAL_PILOT_HANDOFF.md new file mode 100644 index 0000000..00449a9 --- /dev/null +++ b/COMPUTATIONAL_PILOT_HANDOFF.md @@ -0,0 +1,189 @@ +# Computational pilot: Rushik and Justin + +Ready for private scientific review as of September 28, 2026. Both verified +nanoHUB accounts have project-member access; refresh the project page if needed. + +September 29 UI update: convergence thresholds and maximum ionic steps are now +visible directly in the experiment form. Refresh the project page to load the +update. [UI checks and screenshots](geddes/k8s/pilots/computations/UI_VALIDATION_2026-09-29.md) +record the browser checks and verified deployment. + +**October 7 update — deployed and verified.** Resource controls, +per-calculation failure diagnostics and the ElasticDFT inherited-settings +panel are live on both clusters. Refresh the project page. See +[Computational resources and failure diagnostics](#computational-resources-and-failure-diagnostics) +and the [rollout verification](geddes/k8s/pilots/computations/ELASTICDFT_CLAUDE_REVIEW_2026-10-06.md). + +Open [the private pilot project](https://dt-sim2l-pilot.geddes.rcac.purdue.edu/projects/RCAC-PILOT) +from Purdue's network or VPN and sign in with your own nanoHUB account. +The project is **Computational Student Pilot** (`RCAC-PILOT`). Use this isolated +site for the review. Your existing production DT projects are separate. + +The platform now includes a searchable +[Computational Workflow FAQ](https://dt-sim2l-pilot.geddes.rcac.purdue.edu/help/computational-workflows?project_id=RCAC-PILOT). +Open **Help & FAQ** in the sidebar, or **Computational workflow FAQ** beside the +project's run form. It covers actual input values, the first 3D and 2D examples, +submission, status, downloads, elastic parents, and troubleshooting. Project links +return to the selected tool's input form. + +`ngholiza` has the highest platform role, **Administrator**, in this pilot. The +separate project role remains PI; Rushik and Justin remain project members with +their normal Researcher platform role. Refresh the page to reload your role. + +Deployment evidence and access status are recorded in the +[September 28 validation report](geddes/k8s/pilots/computations/PILOT_VALIDATION_2026-09-28.md). + +## Start here + +Actual computational input values are entered on the **project page**, in +**Registered computational experiments**. The **New RCAC Run** action beside a +computational process step opens that form with its tool selected. A unit +experiment definition describes its inputs; it does not submit a calculation. + +1. Open **Registered computational experiments** in the project. Expand a saved + run to inspect its inputs, convergence, results, requester, cluster, and Slurm + job ID. Download its JSON export and a small raw QE output. +2. Choose **Cell Relax DFT**, a cluster, and `mp-149` for a small 3D infrastructure + check. Set **Wavefunction cutoff (Ry)** to `30` and each **k-points** field + to `2`. Under **Convergence and iteration limits**, set **SCF convergence (Ry)** + and **Energy convergence (Ry)** to `1e-8`, **Force convergence (Ry/bohr)** to + `1e-4`, and **Pressure convergence (kbar)** to `1.0`. Keep **Maximum ionic steps** + at `50`. These fields are visible directly below the cutoff and k-points; + no JSON editing is needed. They correspond to `scf_conv`, `energy_conv`, + `force_conv`, `pressure_conv`, and `i_steps` in saved inputs. These coarse values + test the workflow; choose your own + convergence-tested settings for scientific comparisons. +3. For Justin's monolayer example, choose **Cell Relax DFT 2D** and `mp-48`. + Its defaults are 180 Ry, 8 × 8 × 1 k-points, and 20 Å vacuum. Review the + extracted layer, orientation, and final structure. Vacuum-normalized stress + is not a 2D elastic modulus. +4. For **Elastic DFT**, select a converged **3D** relaxation from this project. + The form uses its relaxed structure and submits the strain array. ElasticDFT + regenerates its k-point mesh from the relaxed lattice; the parent's k-points + are not reused. Inspect the recorded mesh when comparing results. + A 2D result cannot be used as a bulk elastic parent. + Every strain calculation inherits the parent's cutoff, SCF/energy/force + thresholds and electronic/ionic step limits unchanged. ElasticDFT has no + override, and the platform never loosens them. To use different values, run + a new 3D relaxation with them and select that run as the parent; whether + different thresholds are scientifically acceptable is your decision. +5. Submit once. Reopening the page preserves the run. If a capacity message + appears, wait for the occupied cluster to finish. Cancel is available for + active runs; cancellation is complete only after Slurm confirms it. +6. Use the [scientific review checklist](COMPUTATIONAL_SCIENTIFIC_REVIEW.md) to + report the run ID, reference inputs, expected values/tolerances, observed + values, and any usability problem. Rushik can review the 3D workflows; + Justin should review his monolayer method. + +Identical successful inputs on the same installation can reuse a saved result. +A cache hit has no new Slurm job; its provenance links to the source calculation. +For a genuinely new calculation, use the scientifically intended revised inputs. + +The September 29 run-value correction preserves fractional wavefunction cutoffs +(for example, 50.5 Ry) in the generated QE input. New fingerprints exclude results +from the older writer from cache reuse. A fractional-cutoff relaxation without +verifiable corrected-writer provenance must be rerun before it can become an +ElasticDFT parent. Unavailable parents are disabled in the form. New runs record +the worker's adapter version, which survives database number normalization. +Existing saved +results remain available for inspection. First-load and tool-switch defaults now +both come from the API catalog; the 3D defaults are 50 Ry, 1 × 1 × 1 k-points, and +100 electronic steps. The explicit settings in step 2 remain the pilot check. + +For ordinary equipment unit experiments, choose **New Run** on a project step, +or **Run** in the library and choose a project. Enter actual values under +**Run Values → Planned Values**, including any additional inputs, then choose +**Review & Save → Save Run in Project**. This saves a run plan. It does not start +physical equipment. Optimization campaigns use Fixed values and Variable ranges; +unsupported equipment receives an explicit error. + +[Implementation and verification details](geddes/k8s/pilots/computations/RUN_VALUES_FIXES_2026-09-29.md). + +## Scope and limits + +- Both Negishi (QE 7.1) and Gautschi (QE 7.3.1) are available. +- Supported tools: CellRelaxDFT `r99-dt2`, CellRelaxDFT 2D `justin-dt1`, and + second-order 3D ElasticDFT `r27-dt2`. +- At most two active requests per user and one active workflow per cluster. + Every job uses one node. You can choose 1–64 MPI ranks, memory per core, + walltime and 1–4 simultaneous elastic strain calculations within the limits + below. Blank fields retain the defaults: 64 ranks, cluster-default memory, + 24 hours per relaxation or 8 hours per strain calculation, and four + simultaneous elastic calculations. +- Runs execute as `dtwin0`, with the requesting nanoHUB user recorded separately. + No student password, Duo response, or private SSH key is needed. +- Small archived artifacts can be downloaded through the project (8 MiB/file). + Ask Navid for larger retained files. Wavefunction/restart directories are not + part of the selected-artifact archive. +- If transport fails, **Retry result retrieval** rechecks the same Slurm job; + it does not submit another simulation. Include the run ID when reporting it. + +This pilot is for scientific review. Numerical equivalence and production +convergence settings still require your review. 2D elasticity, third-order +elasticity, arbitrary script upload, and automatic multi-cluster scheduling +are outside this pilot's scope. + +## Computational resources and failure diagnostics + +**Verified October 7, 2026.** Both endpoints passed capability checks; the live +API offers the controls below and the updated pilot is healthy. PostgreSQL +preserved all existing runs and account roles through the restart. The accepted +Claude implementation passed 242 Python tests, five database integration tests, +and browser form checks. No new scientific simulation was submitted for this +rollout; students must still assess scientific convergence and validity. + +### Entering resources + +Below the scientific inputs, **Computational resources** shows each control, +its allowed range and its default for the selected cluster and tool. Leave a +field blank to use its default. + +| Control | How to enter it | Allowed | Default | +| --- | --- | --- | --- | +| Nodes | Shown only | 1 (fixed) | 1 | +| MPI ranks (CPU cores) per calculation or strain calculation | Whole number | 1–64 | 64 | +| Memory per CPU core (GB) | Number, e.g. `1.9` | 0.5 to about 1.96 GB (2010 MB, the Negishi and Gautschi CPU partitions' per-core maximum) | Blank: the cluster default, 1917 MB per core | +| Time limit per calculation or strain calculation | Whole hours and minutes | 10 minutes to 24 hours (relaxations) or 8 hours (each ElasticDFT strain calculation) | The maximum | +| Strain calculations at once (ElasticDFT) | Whole number | 1–4 | 4 | + +The form shows the resulting request, including memory per node and cores in +use at once. Values outside these limits are rejected before submission, and +other scheduler options cannot be added. Resources are not scientific inputs: +if the project already has a converged result with identical inputs on the same +installation, that saved result is reused, no new job runs, and the run says +that your resource choices were not used. + +More cores, nodes or memory do not fix an SCF or geometry-optimization +convergence failure. Lowering memory below the default can cause a calculation +to run out of memory; the memory maximum is only about 5% above the default. + +### ElasticDFT inherited settings + +After you select a parent relaxation, the ElasticDFT form lists the settings +every strain calculation will use: cutoff, SCF/energy/force thresholds, +electronic/ionic step limits, smearing/spin/Hubbard/van der Waals choices, +exchange-correlation, pseudopotentials and the regenerated k-point mesh. These +are read-only (see step 4 above for how to change them). + +### Failure diagnostics + +A failed or cancelled run shows **Failure diagnostics**: for each calculation +or strain calculation, the Slurm state and exit code, a finding, and the lines +of evidence behind it, with links to the QE output and Slurm error log. +**Download diagnostics (JSON)** saves the record for a report. A finding is +given only with evidence: out of memory requires Slurm's out-of-memory state or +a log message, and an exit code alone is shown as a hint. Slurm's MaxRSS is the +memory peak of a single MPI task, not a node total; QE's estimated RAM is a +prediction, not a measurement. + +Rushik's ElasticDFT run `47548c30-99b1-416c-b757-6931e047aece` (AlN): Codex's +review of all 24 archived QE outputs found that each of the 21 failed strain +calculations stopped with `bfgs failed … convergence not achieved` (exit +`3:0`) before meeting the inherited tolerances; the other three converged. It +has no evidence of a memory failure. On October 7, the run's diagnostics were +enriched from 72 checksum-verified archived files, without rerunning or +changing its status, inputs, results, artifact manifest or timestamps. The +original error is preserved, with one audit event. No retry was submitted; +any choice of different +thresholds is a scientific decision for Rushik, made through a new parent +relaxation. This diagnosis does not validate any elastic constants. diff --git a/COMPUTATIONAL_SCIENTIFIC_REVIEW.md b/COMPUTATIONAL_SCIENTIFIC_REVIEW.md new file mode 100644 index 0000000..3353ee5 --- /dev/null +++ b/COMPUTATIONAL_SCIENTIFIC_REVIEW.md @@ -0,0 +1,102 @@ +# Justin / Rushik: computational pilot review + +Status: scientific approval **pending**. This is an internal review package, +not a claim of publication-quality accuracy. Use the saved results and the +[pilot handoff](COMPUTATIONAL_PILOT_HANDOFF.md) for browser-review instructions +and current access status. Do not use the production DT site for these tests. + +## Supported scope + +| Tool | What to review | Not implemented | +| --- | --- | --- | +| CellRelaxDFT r99-dt2 | 3D cell relaxation and structured outputs | All original nanoHUB options are not promised | +| CellRelaxDFT 2D justin-dt1 | Justin's monolayer extraction/relaxation, graphene example | 2D elastic constants | +| ElasticDFT r27-dt2 | 3D second-order stress/strain fit from a converged 3D parent | Energy-fitting switch and third-order constants | + +These are DT adapters of the supplied workflows. Changes to extraction, +orientation, parsing, and normalization require scientific review; running to +completion is not proof of equivalence to the original notebooks. + +## Review saved results first + +The ZIP contains six runs (one of each supported tool on each cluster), scientific +inputs/outputs, upstream identifiers, pseudopotential and artifact checksums, +and Justin's supplied reference CIF/metadata. It omits requester identities, +credentials and deployment configuration. Negishi uses QE 7.1 and Gautschi uses +QE 7.3.1, so bit-for-bit equality is not expected. + +1. Confirm structure serialization, species/site ordering, lattice conventions, + energy normalization, force/stress signs and units. +2. Confirm the physics settings: XC/pseudopotentials, cutoffs, k-grid, smearing, + pressure, relaxation degrees of freedom and convergence tolerances. +3. Compare with your independently run reference using the **same** settings. + Supply the expected quantities and acceptable tolerances; none are assumed. +4. For elasticity, inspect every deformed configuration, the inherited parent + settings, strain/stress conventions, fitted tensor, stability interpretation, + and the paired final structures/energies/forces/stresses. +5. Tell Navid which intermediate states are worth retaining. Separate histories + must not be assumed to be frame-aligned training samples. + +Important differences to check: + +- Silicon infrastructure runs used 30/31 Ry and 2×2×2 for relaxation; the elastic + adapter generated a 7×7×7 mesh. This was an infrastructure check, not a + convergence study. Confirm that mesh selection and parameter inheritance are + scientifically appropriate. +- Relaxation energy fields and per-cell elastic configuration energies need + explicit normalization checks before combining them into a training dataset. +- The 2D adapter orients supported layers to XY, preserves extracted site indices + and the transformation, and keeps a fixed vacuum. It uses periodic vacuum, + **not** Coulomb truncation. Supercell stress/pressure includes vacuum and must + not be interpreted as a 2D elastic modulus. +- Graphene ran at 180 Ry, 8×8×1 and 20 Å vacuum. The Negishi lattice was about + 2.45676 Å; the supplied CIF is about 2.45947 Å. The supplied metadata and notebook + use different tolerances. This discrepancy needs review, not an automatic pass. +- The September 23 package predates runtime-observation hardening. The September + 28 package includes freshly captured QE executable hashes, observed modules, + Python/OS versions, and QE versions parsed from outputs. Missing observations + in older records have not been filled retrospectively. + +## Browser acceptance test (after the pilot is opened) + +1. Sign in with your own nanoHUB account. Open the private computational pilot + project; verify you cannot access another project's private results. +2. Select a cluster and supported tool in **Registered computational experiments**. + Review the input fields and advanced JSON for understandable names, units and + defaults. Do not supply API keys or upload private credentials. +3. Submit one small agreed example. Refresh/reopen the page; record the DT run ID + and Slurm ID. Check your requester identity and the separate `dtwin0` execution + identity. A run limit message means wait, not repeated clicking. +4. Inspect queued/running/collection states, convergence and warnings; export JSON + and, when enabled, download a small raw artifact. Verify inputs and outputs are + sufficient to understand the result without guessing hidden assumptions. +5. Chain a converged **3D** relaxation into ElasticDFT. Check the upstream ID and + scientific parameters. Confirm a 2D relaxation is rejected as a bulk parent. +6. With Navid present, request cancellation of one small run. Expect cancellation + to remain pending until Slurm confirms termination; inspect retained logs. +7. Report usability gaps and scientific problems separately. Do not use unreviewed + outputs as publication-quality values. + +Initial operational budget: at most two active requests per student, one active +workflow per cluster, at most 64 ranks per job and four elastic array members. +These are proposed pilot guards, not extra RCAC allocations. The normal Slurm +walltime ceilings are safety caps, not an invitation to launch long tests. + +## Feedback template + +Copy one entry per finding: + +```text +Reviewer / date: +Tool revision / cluster / DT run ID: +Input parameters and reference calculation: +Expected result and tolerance (with units): +Observed result: +Category: scientific correctness / missing data / UI / infrastructure +Reproduction steps: +Disposition: acceptable for pilot / changes required / not evaluated +``` + +Scientific sign-off must name the reviewed tool revision, parameter range and +materials tested. Justin owns review of his 2D method; Rushik can review the 3D +workflows and interoperability. Neither person's approval has been inferred. diff --git a/DIGITAL_ASSETS_SIM2L_README.md b/DIGITAL_ASSETS_SIM2L_README.md new file mode 100644 index 0000000..f24cc54 --- /dev/null +++ b/DIGITAL_ASSETS_SIM2L_README.md @@ -0,0 +1,233 @@ +# Digital computational assets on RCAC + +## Status + +This feature is implemented on the isolated `feature/sim2l-rcac-integration` +branch and remains disabled by default. It has not been deployed to production. Existing +Digital Twin physical-equipment, experiment, ingestion, and dashboard behavior +is unchanged unless `RCAC_COMPUTE_ENABLED=true` is deliberately configured. + +September 28: an isolated student pilot now has persistent web/API/database/worker +services and real nanoHUB SAML. Current access and validation are recorded in the +[pilot validation report](geddes/k8s/pilots/computations/PILOT_VALIDATION_2026-09-28.md). +Use the [student handoff](COMPUTATIONAL_PILOT_HANDOFF.md) for review instructions. +The [September 22 evidence](RCAC_E2E_VALIDATION_2026-09-22.md) remains historical +validation. Public production rollout still requires separate authorization. + +The production design follows RCAC's September 2026 guidance: + +- a dedicated, non-person RCAC service account owns the Slurm jobs; +- one single-user Globus Compute endpoint runs under that account on each + supported cluster; +- jobs use only the service account's `ncn` allocation; +- Digital Twin authenticates the requesting platform user and records the + mapping to an opaque Slurm comment, `dt-run:`; +- no Purdue password, Duo response, or student private SSH key is collected; +- one worker batches scheduler status checks per cluster every 30 seconds. + +RCAC does not currently provide the multi-user endpoint needed for real +per-student Unix execution. Platform attribution therefore requires Ale/team +approval before production rollout. + +## What is implemented + +Two existing nanoHUB workflow contracts and Justin's supplied 2D relaxation +example were adapted as registered Digital Twin computational experiments: + +| Workflow | Source snapshot | Current DT revision | Purpose | +|---|---|---|---| +| CellRelaxDFT | `cellrelaxdft-r99` | `r99-dt2` | Relax a structure with Quantum ESPRESSO | +| ElasticDFT | `elasticdft-r27` | `r27-dt2` | Fit a second-order 3D elastic tensor from strained QE calculations | +| CellRelaxDFT 2D | Justin's `cellrelaxdft_2d.ipynb` | `justin-dt1` | Extract a layered structure and relax a monolayer with fixed vacuum | + +These do not include Justin's 2D or third-order **elastic-constant** workflows. Those +require separate input/output contracts and scientific validation after Justin +provides the finalized code. + +## Digital Twin model + +The implementation maps simulations to the same concepts used for physical +assets without changing the physical-asset schema: + +```text +Digital equipment installation + Quantum ESPRESSO 7.1 on Negishi + Quantum ESPRESSO 7.3.1 on Gautschi + │ + └── supports + │ +Registered computational experiment + CellRelaxDFT r99-dt2 or ElasticDFT r27-dt2 + │ + └── computation run + inputs, requester, installation, scheduler job, + structured outputs, provenance, raw artifacts +``` + +One experiment may run on more than one compatible installation through +`computation_experiment_installations`. Each run records which installation +actually executed it. This preserves Ale's requirement that nominally similar +software installations are distinct digital equipment while the scientific +experiment remains reusable. + +## End-to-end flow + +1. An authenticated project member selects 3D/2D CellRelaxDFT or ElasticDFT and an + allowed cluster in the DT UI. +2. FastAPI validates the workflow's typed input contract and project write + permission, then creates a durable `computation_runs` row. +3. The central worker resolves the material structure, generates QE input files + with Pymatgen, and verifies every required pseudopotential against the + repository checksum manifest. +4. The worker invokes the fixed Globus Compute gateway function on the selected + cluster endpoint. It never sends an arbitrary command. +5. The gateway validates the cluster, module stack, scheduler settings, paths, + workflow, task count, walltime, pseudopotential hashes, and staged filenames. +6. It submits with `sbatch --parsable --comment=dt-run:` under the service + account. An on-cluster marker and file lock make submission idempotent. +7. One worker query checks up to 500 active jobs on a cluster with one `sacct` + call and, only for missing jobs, one `squeue` call. +8. When complete, the gateway parses compact scientific data and copies raw + artifacts from Scratch to `/depot/ncn/data/digital-twins/runs//`. +9. DT converts the parsed data into portable JSON/Pymatgen-compatible + structures and stores results plus full provenance in PostgreSQL. + +The implementation's conservative 7 MiB staging budget is respected: small generated inputs +and compact parsed results cross Globus Compute; raw QE output remains on RCAC +storage. + +## Scientific data and provenance + +CellRelaxDFT records the relaxed Pymatgen/MSONable structure, composition, +space groups, energy, pressure, stress, force, lattice, convergence status, +warnings, and useful ionic histories. + +ElasticDFT records every strain matrix, deformed structure, energy, stress, +member convergence status, intermediate histories, the fitted elastic tensor, +and derived bulk/shear/Young's/Poisson quantities. Each elastic member also keeps +its final relaxed structure paired with energy, forces, and stress where present. +Missing data is explicitly marked, not replaced with unrelaxed coordinates. + +Every run also records: + +- authenticated DT requester and project; +- selected digital installation and cluster; +- service-account execution identity; +- Globus endpoint/function/task identifiers; +- Slurm job ID and opaque attribution comment; +- QE/compiler/MPI module stack and launcher through installation metadata; +- workflow revision and normalized-input fingerprint; +- pseudopotential filenames and SHA-256 values; +- checksummed Depot artifact manifest; +- timestamps, lifecycle events, errors, and upstream run provenance. + +Raw text files are retained for debugging and reproducibility, but structured +scientific JSON is the primary data interface. A Python object alone is not +treated as a persistent standard. + +## Supported cluster installations + +| Setting | Negishi | Gautschi | +|---|---|---| +| Allocation / partition / QoS | `ncn / cpu / normal` | `ncn / cpu / normal` | +| QE | `quantum-espresso/7.1` | `quantum-espresso/7.3.1` | +| Prerequisites | `gcc/12.2.0`, `openmpi/4.1.4` | `gcc/11.4.1`, `openmpi/4.1.6` | +| Launcher | `srun pw.x` | `mpirun pw.x` | +| Maximum requested by these workflows | 64 tasks, one node | 64 tasks, one node | + +CellRelaxDFT requests 24 hours. ElasticDFT uses an array with at most four +concurrent members, each requesting 8 hours. Both are below the RCAC-provided +normal-QoS limits. + +## Storage + +```text +/depot/ncn/data/digital-twins/ +├── pseudopotentials/ +│ ├── pseudo_PAW_PBEsol/ +│ └── pseudo_USPP_PBEsol/ +└── runs/ + └── / +``` + +Generated inputs and active calculations use the service account's cluster +Scratch path. Pseudopotentials are read from Depot and verified before submit. +Completed raw artifacts are copied to Depot because Scratch purges inactive +files. The Geddes worker does not mount Depot. + +## Relationship to the Sim2L repository + +The production DT implementation does **not** import, build, or deploy the +external Sim2L Python repository. Its generic Slurm prototype helped establish +the initial behavior, but authentication, orchestration, provenance, and the +physical/digital catalog belong to Digital Twin. No Sim2L repository merge is +required for these registered workflows to run in DT. + +The original CellRelaxDFT/ElasticDFT directories remain scientific reference +snapshots. DT deliberately reuses their validated contracts and parsing logic, +not their nanoHUB runtime or database. + +## Security boundaries + +- Cluster definitions are operator allow-lists; HTTP users cannot provide + hosts, paths, accounts, partitions, QoS, modules, launchers, commands, or + resource counts. +- The remote gateway supports only `submit`, `status_many`, `cancel`, and + `collect` for the three registered workflows. +- Status, cancellation, and collection require an exact run UUID/job ID pair + matching the on-cluster marker. +- Pseudopotentials require approved relative paths and exact SHA-256 hashes. +- Elastic input archives reject traversal, links, and missing members. +- Only approved artifacts under the run directory can be retained. +- The Globus confidential-client secret is mounted read-only from one + namespace-scoped Kubernetes Secret. It is never committed or returned by the + API. Ryan confirmed this is acceptable for this pilot despite Geddes etcd not + being encrypted at rest; RBAC must remain restricted. +- The worker has no Kubernetes API token, no inbound service, no student key, + and no access to arbitrary DT Secrets. + +## Files + +| File | Responsibility | +|---|---| +| `api/computations/schemas.py` | Stable workflow input/output contracts and fingerprints | +| `api/computations/qe.py` | Pymatgen input generation and structured QE result conversion | +| `api/computations/monolayer.py` | Justin's 2D layer preparation, orientation, vacuum and result checks | +| `api/computations/slurm.py` | DT-owned validated Slurm contract and state normalization | +| `api/computations/globus_gateway.py` | Fixed remote gateway and Globus client | +| `api/computations/rcac.py` | Fixed cluster/install allow-list | +| `api/computations/store.py` | Authorization, catalog registration, run/event persistence | +| `api/computations/worker.py` | Durable centralized lifecycle and batched monitoring | +| `api/routers/computations.py` | Feature-flagged authenticated API | +| `web/components/project-computations.tsx` | Cluster/tool selection and run/provenance UI | +| `geddes/k8s/09-rcac-worker.example.yaml` | Review-only isolated worker Deployment | + +## Provisioning checklist + +The service account, endpoints, storage, and isolated live tests below have now +been exercised. Keep this checklist for reproducible provisioning of another +environment. The separate public rollout and scientific approval are still open. + +1. Ale/team approve platform-layer attribution instead of per-student Unix job + ownership for the pilot. +2. NCN selects or creates the dedicated service-account username and grants it + `ncn` access on Negishi and Gautschi. +3. NCN registers the Globus confidential client and stores its secret in the + isolated namespace. +4. Configure and start one single-user Globus Compute endpoint under that + service account on each cluster. +5. Register the source-serialized allow-listed gateway and record its function + UUID. +6. Copy the approved PAW/USPP PBEsol files to the documented Depot path and + verify the committed checksum manifest against those exact files. +7. Create the service-account Scratch and Depot run directories with reviewed + NCN group ACLs. +8. Populate the reviewed ConfigMap/Secret from + `api/computations.env.example` and build an immutable worker image. +9. Run low-cost CellRelaxDFT and ElasticDFT end-to-end tests on both clusters, + verify DB provenance and Depot artifacts, then obtain explicit deployment + approval. + +Do not enable or deploy the feature merely because the code and unit tests +pass; the external identities, endpoints, storage contents, and scientific +outputs must be verified on RCAC first. diff --git a/RCAC_E2E_EVIDENCE_2026-09-22.json b/RCAC_E2E_EVIDENCE_2026-09-22.json new file mode 100644 index 0000000..42443ee --- /dev/null +++ b/RCAC_E2E_EVIDENCE_2026-09-22.json @@ -0,0 +1,434 @@ +{ + "scope": "isolated nonproduction validation", + "worker_image": "sha256:13f977ced56cf95a52d3f78ff3a6c5d34cef83baff0abdce5bf6f14b16863f27", + "negishi_browser": { + "authentication": "locally signed test session; NanoHUB OAuth not exercised", + "checks": { + "anonymous_api_rejected": true, + "real_project_page_and_2d_defaults": true, + "graphene_real_rcac_ingestion": true, + "elastic_recovered_without_resubmission": true, + "portable_export": true, + "browser_errors": [] + }, + "runs": [ + { + "id": "c06d3739-926f-47f7-935c-cfad8a3961f7", + "tool": "cellrelaxdft2d", + "job": "43376192", + "status": "data_ingested", + "convergence": true, + "artifacts": 4 + } + ] + }, + "gautschi_browser": { + "cluster": "gautschi", + "authentication": "signed local test session; real BFF/API authorization", + "checks": { + "cell_submitted_through_ui": true, + "all_three_real_rcac_runs_ingested": true, + "provenance_and_export": true, + "two_dimensional_upstream_rejected_for_bulk_elastic": true, + "browser_errors": [] + }, + "runs": [ + { + "id": "8956cf41-00c1-45c7-9b9c-41c6244a7d96", + "tool": "cellrelaxdft", + "status": "data_ingested", + "job": "16619101", + "error": {} + }, + { + "id": "5b481af6-4f29-4a9a-abdb-423bcc45eef3", + "tool": "cellrelaxdft2d", + "status": "data_ingested", + "job": "16619102", + "error": {} + }, + { + "id": "1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e", + "tool": "elasticdft", + "status": "data_ingested", + "job": "16619126", + "error": {} + } + ] + }, + "independent_verification": { + "clusters": [ + { + "cluster": "negishi", + "runs": [ + { + "id": "c06d3739-926f-47f7-935c-cfad8a3961f7", + "job": "43376192", + "verified_artifacts": 4, + "slurm_members": 1, + "sacct": [ + [ + "43376192", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:c06d3739-926f-47f7-935c-cfad8a3961f7" + ] + ] + }, + { + "id": "b4caf247-e57f-464d-9069-276e3ac04bd2", + "job": "43375494", + "verified_artifacts": 37, + "slurm_members": 12, + "sacct": [ + [ + "43375495", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375496", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375497", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375498", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375512", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375513", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375514", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375515", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375518", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375519", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375520", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ], + [ + "43375494", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:b4caf247-e57f-464d-9069-276e3ac04bd2" + ] + ] + }, + { + "id": "44e09c05-6c10-4724-9fa2-638fd71859ea", + "job": "43375490", + "verified_artifacts": 4, + "slurm_members": 1, + "sacct": [ + [ + "43375490", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:44e09c05-6c10-4724-9fa2-638fd71859ea" + ] + ] + } + ] + }, + { + "cluster": "gautschi", + "runs": [ + { + "id": "1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e", + "job": "16619126", + "verified_artifacts": 37, + "slurm_members": 12, + "sacct": [ + [ + "16619127", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619128", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619129", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619130", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619131", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619132", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619133", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619134", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619135", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619136", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619137", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ], + [ + "16619126", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e" + ] + ] + }, + { + "id": "5b481af6-4f29-4a9a-abdb-423bcc45eef3", + "job": "16619102", + "verified_artifacts": 4, + "slurm_members": 1, + "sacct": [ + [ + "16619102", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:5b481af6-4f29-4a9a-abdb-423bcc45eef3" + ] + ] + }, + { + "id": "8956cf41-00c1-45c7-9b9c-41c6244a7d96", + "job": "16619101", + "verified_artifacts": 4, + "slurm_members": 1, + "sacct": [ + [ + "16619101", + "dtwin0", + "ncn", + "cpu", + "normal", + "COMPLETED", + "0:0", + "64", + "dt-run:8956cf41-00c1-45c7-9b9c-41c6244a7d96" + ] + ] + } + ] + } + ], + "cross_project_detail_and_export_forbidden": true + }, + "elastic_final_configuration_records": { + "negishi": 12, + "gautschi": 12 + } +} diff --git a/RCAC_E2E_VALIDATION_2026-09-16.md b/RCAC_E2E_VALIDATION_2026-09-16.md new file mode 100644 index 0000000..0ddb945 --- /dev/null +++ b/RCAC_E2E_VALIDATION_2026-09-16.md @@ -0,0 +1,122 @@ +# RCAC provisioning and validation record — 2026-09-16 + +This record distinguishes completed infrastructure/science validation from the +remaining service-account/Globus validation. No existing Digital Twin workload +or production database was changed. + +## Provisioned + +- Isolated namespace: `ncn-digitaltwins-zchen-sim2l` +- Credential-free Kubernetes ServiceAccount: `dt-computation-worker` +- Deny-ingress NetworkPolicy: `dt-computation-worker-deny-ingress` +- Isolated PostgreSQL database: `dt_rcac_e2e` +- Worker database Secret containing only app-role connection fields: + `dt-computation-database` +- Globus confidential-client UUID: + `2fea1cb8-521c-4f61-9b0b-5d7226389f66` +- Isolated Opaque Secret `dt-globus-confidential-client` with the expected + `client-secret` key (value not inspected or recorded) +- Isolated Opaque Secret `dt-computation-materials-project` with the expected + `api-key` key (value not inspected or recorded) +- Multi-architecture worker image: + `geddes-registry.rcac.purdue.edu/sdx/dt-computation-worker@sha256:3bab79be26ae4c4b8ed06e2d5e2a0f8f445825baed3943eed97e5ff9a490c481` +- NCN Depot roots: + - `/depot/ncn/data/digital-twins/pseudopotentials` + - `/depot/ncn/data/digital-twins/runs` +- 171 PAW/USPP PBEsol pseudopotentials (457,446,922 bytes), checked against + `api/computations/pseudopotentials.sha256` before and after transfer +- Named `ncn` ACL has read-only access to the pseudopotential hierarchy. + +The worker Deployment was intentionally not created because its service-account +identity and Globus identifiers/secret do not yet exist. + +## Live Negishi validation + +The test executed the exact source-serialized allow-listed gateway function, +but transported it through the already-consented pilot SSH identity because the +final Globus service-account endpoint is not yet available. This proves the +workflow, scheduler, QE, parser, artifact, and provenance behavior on Negishi; +it does not prove the final Globus authentication layer. + +Observed endpoint runtime requirement: + +- Negishi default `python3`: 3.6.8 (incompatible) +- Validated runtime module: `anaconda/2024.02-py311` +- Validated Python: 3.11.7 + +Observed Gautschi readiness (read-only login checks): + +- Personal pilot access to the `ncn` account: confirmed +- Purchased/free CPU capacity observed: 768 / 768 cores +- Scratch: `/scratch/gautschi/ngholiza` +- Quantum ESPRESSO: `quantum-espresso/7.3.1` +- Required modules: `gcc/11.4.1`, `openmpi/4.1.6` +- Default Python: 3.9.25 +- `anaconda/2024.02-py311`: unavailable; the registered gateway was made + Python-3.9-compatible instead + +### CellRelaxDFT + +- DT run UUID: `2d33d943-8b3a-46f5-9501-087cbf31f16e` +- Slurm job: `43237705` +- Owner: `ngholiza` (pilot identity only) +- Account / partition / QoS: `ncn` / `cpu` / `normal` +- Allocated CPUs: 64 +- Comment: `dt-run:2d33d943-8b3a-46f5-9501-087cbf31f16e` +- State / exit: `COMPLETED` / `0:0` +- Elapsed: 11 seconds +- QE stack: `gcc/12.2.0`, `openmpi/4.1.4`, + `quantum-espresso/7.1`, `srun pw.x` +- Structured result: process completed, scientifically converged, final energy + `-89.96870767 Ry`, final pressure `0.0 kbar` +- Intermediate structured data: 6 energy samples, 5 cell blocks, and 5 atomic + position blocks +- Retained artifacts: 4; every Depot copy independently SHA-256 verified +- The deliberately tiny 64-task smoke case reported two non-fatal warnings: + over-parallelization and one transient unconverged eigenvalue. + +### ElasticDFT + +- DT run UUID: `8a7e6261-85d8-4177-9365-4618f0a1b95f` +- Upstream run UUID: `2d33d943-8b3a-46f5-9501-087cbf31f16e` +- Slurm array parent: `43238114` +- Array: `0-11%4` (12 members, maximum four concurrent) +- Account / partition / QoS: `ncn` / `cpu` / `normal` +- Allocated CPUs per member: 64 +- Comment: `dt-run:8a7e6261-85d8-4177-9365-4618f0a1b95f` +- All 12 members completed with exit code zero and were scientifically + converged. +- Retained artifacts: 37 (489,219 bytes); every Depot copy independently + SHA-256 verified +- The platform's production tensor-fitting code generated the full 6×6 tensor. + For this small Si validation it reported approximately `C11=124.0 GPa`, + `C12=31.2 GPa`, `C44=66.3 GPa`, bulk modulus VRH `62.2 GPa`, shear modulus + VRH `57.45 GPa`, and Young's modulus `131.79 GPa`. + +These values validate execution and data handling; they are not a publication +dataset. Rushik/Justin still own scientific review of production inputs, +convergence settings, and interpretation. + +## Still required for the approved final E2E + +1. Dedicated DT service account `dtwin0` has been created and its `ncn` Slurm + access and SSH-key login have been verified on Negishi and Gautschi. +2. `dtwin0` must still be added to Unix group `ncn-data`; its existing `ncn` + membership cannot traverse the mode-`2770` `/depot/ncn/data` directory. + After that, verify write access to + `/depot/ncn/data/digital-twins/runs` and read access to the pseudopotentials. +3. The Globus confidential client and Kubernetes Secret are provisioned. +4. Globus Compute Endpoint/SDK 3.16.1, the `dt-negishi`/`dt-gautschi` + profiles, the hardened launcher, and protected confidential-client files + are installed under `dtwin0`; the profiles are currently `Initialized`. + They must be started after the Depot group is available, then both endpoint + UUIDs must be recorded. +5. The fixed gateway function is registered and its UUID is supplied. +6. The Materials Project API key is already stored in the isolated namespace + for MP-ID resolution. +7. The disabled worker is configured, started only in the isolated namespace, + and the same two workflows are repeated through Globus on both Negishi and + Gautschi. + +Until those inputs exist, `RCAC_COMPUTE_ENABLED` remains false and the current +Digital Twin platform remains unchanged. diff --git a/RCAC_E2E_VALIDATION_2026-09-22.md b/RCAC_E2E_VALIDATION_2026-09-22.md new file mode 100644 index 0000000..88cee58 --- /dev/null +++ b/RCAC_E2E_VALIDATION_2026-09-22.md @@ -0,0 +1,137 @@ +# RCAC computational pilot validation — September 22, 2026 + +Branch: `feature/sim2l-rcac-integration`. Production DT is unchanged. All writes +and tests used namespace `ncn-digitaltwins-zchen-sim2l`, database `dt_rcac_e2e`, +project `RCAC-E2E`, and ordinary non-superuser database role `api_client`. + +Machine-readable checks are preserved in +[the evidence file](RCAC_E2E_EVIDENCE_2026-09-22.json). + +## Real scientific executions + +These ran through DT → Globus Compute → Slurm → Quantum ESPRESSO → structured +result ingestion. They were not mocked calculations. + +| Cluster | Workflow | DT run | Slurm job | Result | +| --- | --- | --- | --- | --- | +| Negishi | 3D CellRelaxDFT | `44e09c05-6c10-4724-9fa2-638fd71859ea` | `43375490` | Ingested, converged | +| Negishi | 3D ElasticDFT | `b4caf247-e57f-464d-9069-276e3ac04bd2` | `43375494` | Ingested, 12 converged members | +| Negishi | Justin's 2D CellRelaxDFT | `c06d3739-926f-47f7-935c-cfad8a3961f7` | `43376192` | Ingested, converged graphene | +| Gautschi | 3D CellRelaxDFT | `8956cf41-00c1-45c7-9b9c-41c6244a7d96` | `16619101` | Ingested, converged | +| Gautschi | 3D ElasticDFT | `1dc35e8b-42ca-4ec0-9ecc-1869d3c30e3e` | `16619126` | Ingested, 12 converged members | +| Gautschi | Justin's 2D CellRelaxDFT | `5b481af6-4f29-4a9a-abdb-423bcc45eef3` | `16619102` | Ingested, converged graphene | + +Independent read-only checks of `sacct` confirmed all 28 allocations/array +members completed with exit `0:0`, owner `dtwin0`, account `ncn`, partition +`cpu`, QoS `normal`, and 64 cores. Every Slurm comment matched its DT run ID. +All **90 archived files** matched database sizes and SHA-256 hashes: +4 artifacts per relaxation, 37 per elastic workflow. They reside under +`/depot/ncn/data/digital-twins/runs//`. + +Both elastic outputs were re-collected through Globus using the final parser, +without rerunning Slurm. Each contains **12 complete paired final structure, +energy, force, and stress records**, its 6×6 elastic tensor, and upstream +provenance. Final coordinates are not replaced by unrelaxed input coordinates. + +## Coverage + +- Real API/database/worker/Globus/Slurm/QE lifecycle on both clusters. +- Actual browser submission of Negishi graphene and Gautschi silicon. Other + test submissions used the authenticated API/BFF. +- Recovery after a transport error, preserving the original Slurm job. +- Replay-safe Negishi submission, fresh-worker collection, and identical-input + cache reuse without another Slurm job. +- Student attribution separate from service-account execution identity. + Anonymous BFF access and cross-project detail/export reads are denied. +- API rejection of a 2D relaxation as the parent for bulk ElasticDFT. +- Structured JSON exports, warnings, hashes, inputs, results, and lineage. + +Browser tests use temporary, signed Auth.js test sessions and real BFF/API +authorization, not an authentication bypass. **External NanoHUB OAuth/login was +not tested.** Final production-build browser checks use localhost HTTPS and +secure cookies, with certificate verification relaxed only for that temporary +self-signed localhost certificate, never for RCAC or Globus. +Both final HTTPS browser suites passed with zero browser errors, including +structured export and 2D-to-bulk dependency rejection. + +## Infrastructure + +`dtwin0` can read pseudopotentials and write run archives on both clusters. +All 171 pseudopotentials matched the committed manifest on both. Negishi's +endpoint uses active group `ncn-data`; Gautschi file access worked before its +displayed supplementary group list caught up. No Depot ACL/chmod/chown changes +were made. + +- Negishi endpoint: `d7262b0a-0929-4eec-ae0d-76a360f9ea7e`, login03. +- Gautschi endpoint: `805f5a4a-f88a-465a-bf67-2af5c8e6cbcf`, login01. +- Allow-listed gateway: `07b8c5db-2372-464b-8949-a521c67d5a89`. +- Gateway source SHA-256: + `4d65fb6cd2164d6923089fc9a388580f41b51710f827c2660cb2238bdfa69e3c`. +- Final worker image: + `geddes-registry.rcac.purdue.edu/sdx/dt-computation-worker@sha256:13f977ced56cf95a52d3f78ff3a6c5d34cef83baff0abdce5bf6f14b16863f27`. + +The endpoint has one local control worker; QE runs only in Slurm allocations. +Pods have a read-only root filesystem, writable temporary volumes, +namespace-local Secrets, and no Kubernetes API token. Legacy `PG_SUPER_*` +environment names map to ordinary app-role credentials, not superuser access. +These are temporary validation Pods, not a durable/public DT deployment. +The worker deployment example remains disabled by default. +After validation the temporary Geddes Pod and disposable local PostgreSQL test +container were removed. The isolated DT database, all scientific Depot artifacts, +and both cluster endpoints were retained. No ongoing student-facing web server +was deployed by these tests. + +## Corrections discovered during testing + +1. Missing bootstrap/catalog migrations were applied only to the isolated + database. Failed initial setup attempts submitted no scientific jobs. +2. Globus's batch-result API repeatedly returned HTTP 500 `StorageException` + for ElasticDFT despite completed, archived cluster results. Its single-task + API retrieved the 34 KB response successfully. The gateway now uses that API, + bounded retries, and same-job collection recovery. This also avoids the SDK + batch path swallowing remote-task exceptions. No physics was rerun to recover. +3. Justin's notebook assumed XY relaxation after accepting other stacking + directions. The adapter normalizes supported layers to XY and preserves the + transform, original structure, and selected site indices. +4. An existing sidebar theme mismatch caused React hydration errors. The theme + icon now resolves the browser theme after hydration. +5. Non-converged ingested runs are excluded from the scientific cache and the + UI's upstream relaxation choices. + +## Regression verification + +- 50 focused computation/parser/worker/router tests passed. +- Disposable PostgreSQL API→worker→DB test passed, including catalog linkage, + cache reuse, same-job recovery, and rejection of non-converged cached data. +- TypeScript, changed-file ESLint, and production Next.js build passed. +- Broader regression selection: 83 passed, 5 failed. The exact same five failures + occur on unchanged baseline `42c607a`: older physical-project/experiment + fixtures expect obsolete helpers/validation and one misses a DB-helper mock. + These are not counted as passes; physical routes were not changed to fix them. + +## Scientific limits and rollout + +Silicon tests use 30 Ry and 2×2×2 sampling for cheap infrastructure validation, +not publication-quality convergence. Graphene uses Justin's notebook defaults: +180 Ry, 8×8×1, 20 Å vacuum. The Negishi lattice is about 2.45676 Å versus +2.45947 Å in the supplied reference CIF. Supplied metadata and notebook defaults +use different tolerances; full raw reference outputs were not supplied. +**Scientific equivalence has not been established.** Justin/Rushik must review +the adapter and convergence settings. + +No 2D elastic or third-order solver, energy-fitting switch, or arbitrary-script +upload was invented. Selected raw artifacts are archived but not web-downloadable; +large wavefunction/restart files are not archived. SDK runtime-version warnings +remain, although the self-contained gateway and portable data worked on both +endpoints. + +Gautschi's MPI stack also emitted an ignored OpenIB-plugin warning about missing +`libosmcomp.so.7`. The single-node jobs completed and converged; their original +stderr is archived. No module installation or MPI transport configuration was +changed to suppress this cluster-environment warning. Consult RCAC before +extrapolating these tests to multi-node performance. + +See [student pilot guide](COMPUTATIONAL_PILOT_GUIDE.md) and +[provisioning guide](RCAC_SIM2L_INTEGRATION.md). A student-facing pilot URL, +deployment, access grants, and actual NanoHUB login validation still need the +separate rollout approval. No Sim2L-repository merge is required. diff --git a/RCAC_HARDENING_STATUS_2026-09-23.md b/RCAC_HARDENING_STATUS_2026-09-23.md new file mode 100644 index 0000000..9bda07e --- /dev/null +++ b/RCAC_HARDENING_STATUS_2026-09-23.md @@ -0,0 +1,90 @@ +# Computational pilot preparation — September 23, 2026 + +Historical preparation record. For the later deployment and current access, +see [September 28 pilot validation](geddes/k8s/pilots/computations/PILOT_VALIDATION_2026-09-28.md). + +Branch: `feature/sim2l-rcac-integration`. No merge **into** `dev`, no push, no +production rollout. Latest `origin/dev` (`6a5e1d8`) was merged locally into this +branch for compatibility testing. No paper files are included. + +## Implemented locally + +- Scheduler accounting captures start/end/submit times, exit codes and attribution. + Active array siblings prevent premature completion/failure/archive. +- Cancellation waits for terminal accounting; uncertain submissions are reconciled + rather than falsely marked cancelled. Submission-intent markers prevent blindly + duplicating jobs after an ambiguous scheduler response. +- Available failed/cancelled logs are archived before parsing. Missing artifacts + and parse failures are explicit; incomplete output is not silently successful. +- Selected compute-node runtime observations and adapter/package versions are + recorded without dumping environment variables or credentials. +- Opt-in small-artifact downloads validate project/cache-source access, approved + paths, size and SHA-256. Large artifacts remain in Depot. +- Worker claims are refreshed before processing batch items and status queries. + Unknown status/transport retries are bounded. Single-worker operation is still + the supported pilot topology; multi-worker HA has not been certified. +- Opt-in project allowlist and atomic per-user/per-cluster active-run limits. +- Scientific review checklist and reproducible, sanitized six-run review ZIP. + +## Verification completed + +- 204 API tests and 19 subtests passed; the external-system test and disposable-DB + test module were excluded from that count. +- Two opt-in disposable-PostgreSQL tests passed: API→worker→DB/cache/retrieval, + capacity denial, claim ownership/refresh and cancellation before submission. +- Production Next.js build and changed computation-component ESLint passed. +- Read-only execution of the current gateway source against both RCAC clusters + confirmed 28 completed allocation/array records with start/end timestamps and + six checksum-verified archived output transfers. This used endpoint Python + environments over authenticated SSH, **not** a newly registered Globus function. +- The six original live DT→Globus→Slurm→QE runs remain documented in + `RCAC_E2E_VALIDATION_2026-09-22.md`. They predate today's hardening. No new + scientific Slurm jobs were submitted during this preparation. + +Four old physical-project/experiment tests were updated to match existing `dev` +behavior: project creation is equipment-independent, invisible projects are +rejected, planned parameters are required, and proposal-status writes need a unit +test mock. Physical routes were not changed to make those assertions pass. + +## Provisioned in the isolated namespace only + +- Service and private/VPN-only Ingress `dt-computation-pilot`. +- Host `dt-sim2l-pilot.geddes.rcac.purdue.edu` now resolves to the private Geddes + ingress. HTTPS certificate verification passed. HTTP 503 is expected because + there is no pilot application Deployment yet; **this is not a usable test URL**. +- Separate `dt-computation-pilot-saml` Secret containing a newly generated SP + keypair and public IdP certificate from nanoHUB metadata. No production signing + key or session credential was copied. Public metadata/certificate and a draft + registration email are outside Git in `Mxene/DT-computational-pilot-SAML-registration`. + +No live database, endpoint gateway registration, endpoint allowlist, production +Secret, production Ingress or production Deployment was changed by this work. +Existing evidence and archives were retained. Local generated `.next/dev` cache +from the removed auth route was moved to a temporary backup before rebuilding. + +## Not ready for independent student browser testing yet + +Needs external coordination: + +1. nanoHUB must register the separate pilot SAML SP; then real login/logout must + be tested. Do not use development/test sessions as student authentication. +2. Navid must confirm Justin/Rushik's canonical nanoHUB identities for private + pilot-project membership. Names alone are insufficient. + +Engineering still required after that: + +- Build/pin and deploy the isolated app/worker, verify the complete isolated DB + schema and role mapping, register/allowlist the new gateway, then perform real + submission/cancellation/download smoke tests through the deployed app. +- Review the dependency audit before release. `npm audit` reports two critical + package entries for **one** MapLibre advisory inherited through Plotly: + [GHSA-jrc7-96c5-q579](https://github.com/maplibre/maplibre-gl-js/security/advisories/GHSA-jrc7-96c5-q579). + The audited lock has Plotly 3.4.0 / MapLibre 4.7.1. The suggested Plotly upgrade + is major (4.1.1); no blanket major upgrade was applied to unrelated physical + dashboards. Current chart call sites do not configure maps, but that is not a + dependency-security sign-off. Resolve or explicitly assess this before opening + the full UI to testers. + +Justin/Rushik can review the six saved scientific records immediately using +`DT-computational-scientific-review-20260923.zip`. Scientific approval is pending; +2D elasticity and third-order solvers are not part of the supported pilot. diff --git a/RCAC_SIM2L_INTEGRATION.md b/RCAC_SIM2L_INTEGRATION.md new file mode 100644 index 0000000..b533846 --- /dev/null +++ b/RCAC_SIM2L_INTEGRATION.md @@ -0,0 +1,278 @@ +# RCAC registered-computation implementation guide + +This is the operational companion to +[`DIGITAL_ASSETS_SIM2L_README.md`](DIGITAL_ASSETS_SIM2L_README.md). The filename +is retained for existing links; the implementation itself has no Sim2L runtime +dependency. + +## Safety state + +- Branch: `feature/sim2l-rcac-integration` +- Feature flag default: off +- General Kubernetes manifest: example/review only; the isolated pilot uses + `geddes/k8s/pilots/computations/provision_compute.py`. +- Credential-free namespace bootstrap performed: yes +- Isolated API/web/worker/database Deployment: running. Current image digests, + gateway revision, access and tests are in the + [September 28 report](geddes/k8s/pilots/computations/PILOT_VALIDATION_2026-09-28.md). +- Real service-account validation: CellRelaxDFT, ElasticDFT, and Justin's + 2D graphene CellRelaxDFT reached the isolated database on both clusters. +- Service account, confidential client, and both Globus endpoints: provisioned. + The original validation used temporary Pods. See + [`RCAC_E2E_VALIDATION_2026-09-22.md`](RCAC_E2E_VALIDATION_2026-09-22.md). + +## Provisioning order + +### 1. RCAC service account + +Use a dedicated non-person account rather than `nano0` so DT activity remains +traceable. RCAC/NCN must add it to `ncn` on Negishi and Gautschi. Record the +exact username as `RCAC_SERVICE_ACCOUNT_USERNAME`. + +Slurm allocation access and Depot access are separate. The live `dtwin0` +preflight showed membership in Unix group `ncn`, while +`/depot/ncn/data` is `root:ncn-data` with mode `2770`. The service account must +also be added to `ncn-data` before it can traverse to the pseudopotentials or +persistent run archive. Reconnect the endpoint process after group membership +changes so its supplementary groups are refreshed. + +The NCN group-writable data area was selected after checking the actual Depot +permissions. The pseudopotential hierarchy is provisioned and verified at the +following path. Read/write probes and real result archival now pass as `dtwin0` +on both clusters; no additional ACL changes were needed: + +```text +/scratch/negishi//digital-twins/runs +/scratch/gautschi//digital-twins/runs +/depot/ncn/data/digital-twins/pseudopotentials +/depot/ncn/data/digital-twins/runs +``` + +Use NCN-managed Unix ACLs so approved maintainers can update +pseudopotentials, ordinary `ncn` users can read them, and only the service +account/maintainers can write run artifacts. + +### 2. Pseudopotentials + +Copy the exact contents of the reference bundles: + +```text +/depot/ncn/data/digital-twins/pseudopotentials/pseudo_PAW_PBEsol +/depot/ncn/data/digital-twins/pseudopotentials/pseudo_USPP_PBEsol +``` + +From the repository, compare every installed file with +`api/computations/pseudopotentials.sha256`. Do not update the manifest merely +to make a mismatch pass; first verify the scientific source/version. + +### 3. Globus confidential client + +NCN registers one confidential client. Put its client UUID in the computation +ConfigMap and its secret in only the isolated namespace: + +```text +Secret: dt-globus-confidential-client +Key: client-secret +``` + +The Deployment mounts that key at +`/var/run/secrets/globus/client-secret`. Do not put the secret directly in an +environment manifest, source file, Slack message, or support ticket. + +Store the Materials Project API key separately as +`dt-computation-materials-project/api-key`. The worker reads it to resolve the +submitted Materials Project ID; it is not needed by the API Pod. + +### 4. Single-user endpoints + +Under the RCAC service account, configure one Globus Compute endpoint on each +cluster. The endpoint must run on a supported modern Python. Negishi's default +`python3` is 3.6.8 and is not compatible with the serialized gateway or current +Globus Compute. The live validation used the directly loadable +`anaconda/2024.02-py311` module (Python 3.11.7); pin an equivalent reviewed +Python runtime in each endpoint configuration. + +Gautschi was verified on 2026-09-16 to provide Python 3.9.25 as its default +`python3`; the self-contained gateway is kept Python-3.9-compatible and Globus +Compute 3.16.1 supports Python 3.9. The +Negishi endpoint continues to use `anaconda/2024.02-py311` because Negishi's +default Python 3.6.8 is too old. + +The endpoint environment must contain: + +Negishi: + +```text +DT_CLUSTER_NAME=negishi +DT_WORK_ROOT=/scratch/negishi//digital-twins/runs +DT_PSEUDO_ROOT=/depot/ncn/data/digital-twins/pseudopotentials +DT_ARTIFACT_ROOT=/depot/ncn/data/digital-twins/runs +DT_SLURM_ACCOUNT=ncn +DT_SLURM_PARTITION=cpu +DT_SLURM_QOS=normal +DT_QE_MODULES_JSON=["gcc/12.2.0","openmpi/4.1.4","quantum-espresso/7.1"] +DT_MPI_LAUNCHER=srun +``` + +Gautschi: + +```text +DT_CLUSTER_NAME=gautschi +DT_WORK_ROOT=/scratch/gautschi//digital-twins/runs +DT_PSEUDO_ROOT=/depot/ncn/data/digital-twins/pseudopotentials +DT_ARTIFACT_ROOT=/depot/ncn/data/digital-twins/runs +DT_SLURM_ACCOUNT=ncn +DT_SLURM_PARTITION=cpu +DT_SLURM_QOS=normal +DT_QE_MODULES_JSON=["gcc/11.4.1","openmpi/4.1.6","quantum-espresso/7.3.1"] +DT_MPI_LAUNCHER=mpirun +``` + +Record both endpoint UUIDs. RCAC does not currently support a multi-user +endpoint for this use case. + +The reviewed launcher is `api/scripts/rcac_globus_endpoint.bash`. Install it as +`~/.local/bin/dt-globus-endpoint`, copy the two non-secret cluster templates +from `api/rcac-endpoints/` to `~/.config/dt-globus/.env`, create a +mode-`0600` `~/.config/dt-globus/client.env` from the client template, and +store only the confidential-client secret value in the mode-`0600` +`~/.config/dt-globus/client-secret` file. The launcher explicitly initializes +RCAC's login environment because +noninteractive SSH sessions do not provide Lmod or the scratch variables. It +also refuses to start if the service account cannot read pseudopotentials or +write the persistent artifact root. + +Register the gateway below before first startup and add its UUID to the +endpoint configuration's `allowed_functions` list. This confines the endpoint +to the reviewed gateway. If freshly granted supplementary groups have not yet +become effective for Depot, the verified Negishi administration command is +`sg ncn-data -c '$HOME/.local/bin/dt-globus-endpoint start negishi'`. +This activates an already-granted group; it does not change membership or ACLs. + +### 5. Register the gateway + +Install worker dependencies and set the client UUID/secret file locally in an +approved administrative environment. Then run: + +```bash +python -m computations.globus_gateway register +``` + +Registration deliberately uses `PureSourceTextInspect`: the endpoint receives +the full self-contained standard-library function and does not need the DT +repository installed. Save the returned function UUID as +`GLOBUS_COMPUTE_GATEWAY_FUNCTION_ID`. + +### 6. Configure DT + +Copy `api/computations.env.example` to the deployment configuration and replace +every placeholder. Both API and worker need the cluster allow-list and service +account name. Only the worker needs the Globus client secret. + +Build the worker independently of Sim2L: + +```bash +docker build -f api/Dockerfile.worker -t /dt-computation-worker: . +``` + +Pin the reviewed image by digest in +`geddes/k8s/09-rcac-worker.example.yaml`. The reviewed multi-architecture image +is already pinned there. Do not apply the Deployment until its external +identifiers and Secrets are available. + +### 7. Database migration + +`db_migrations.run_schema_migrations()` (run at API startup or with +`python -m db_migrations`) creates the computation tables from +`api/scripts/add_rcac_computations.sql` on every deployment, empty while +`RCAC_COMPUTE_ENABLED` is off. It then applies +`api/scripts/add_asset_kinds.sql`, which makes the physical/computational +split explicit: + +| Object | Meaning | +| --- | --- | +| `equipment_metadata.asset_kind` | `physical` (lab instrument) or `computational` (a registered QE installation) | +| `experiment_types.execution_kind` | Set by a trigger from the template's equipment; forks of a computational template stay computational | +| `all_runs` view | Every run with `run_kind`: `equipment_runs` and `etcher_runs` are physical, `computation_runs` computational. It uses `security_invoker`, so each table's row-level security still applies | + +The computation API and the RCAC worker connect as `api_worker`, not as the +table owner, and check project access in Python. Give the worker Deployment +`PG_WORKER_USER`/`PG_WORKER_PASS`; without them it falls back to the +migration credentials and logs a warning. The row-level security script grants +`api_worker` its policies on the three computation tables. + +Old per-student SSH profile tables from an earlier, never-production prototype +are not used or created by the current migration. If they exist in a disposable +test database, remove them separately only after confirming no required data is +present; this feature never drops them automatically. + +## Worker behavior + +The Deployment runs one replica with `Recreate` strategy. PostgreSQL leases +still protect each run, but one replica is intentional to minimize scheduler +RPCs. + +Each 30-second cycle: + +1. leases up to 500 active rows; +2. validates/prepares/submits new runs; +3. groups submitted/running rows by cluster; +4. sends one `status_many` request per cluster; +5. handles cancellation requests; +6. collects previously completed runs. + +The UI's eight-second refresh reads PostgreSQL only and does not query Slurm. + +## Gateway contract + +The registered function accepts exactly four actions: + +- `submit`: validates and stages known inputs, checks pseudopotential hashes, + renders a fixed script, and calls `sbatch` once under a file lock; +- `status_many`: validates each run/job marker, calls `sacct` once and `squeue` + at most once, and returns raw states; +- `cancel`: validates the run/job marker before `scancel`; +- `collect`: validates the marker and artifact allow-list, parses scientific + summaries, and copies raw files to Depot with SHA-256 values. + +It cannot execute a user-provided command or choose a user-provided account, +partition, QoS, module, launcher, working directory, or artifact path. + +## Validation before deployment + +Run locally: + +```bash +cd api +../.venv-computations/bin/python -m pytest tests -q + +cd ../web +npm test -- --runInBand +npm run lint +npm run build +``` + +Then, while the production feature remains disabled: + +1. invoke `status_many` against each endpoint with a controlled test marker; +2. submit one very small CellRelaxDFT case on Negishi; +3. verify `sacct` owner, account, partition, QoS, task count, comment, and + module/launcher provenance; +4. verify structured DB output and checksum-matching Depot artifacts; +5. repeat on Gautschi; +6. submit CellRelaxDFT followed by ElasticDFT and verify upstream provenance; +7. test cancel, endpoint retry/idempotency, failed QE convergence, and worker + restart recovery; +8. have Rushik/Justin validate scientific inputs and outputs; +9. obtain explicit deployment approval. + +The controlled Negishi evidence gathered on 2026-09-16 is recorded in +[`RCAC_E2E_VALIDATION_2026-09-16.md`](RCAC_E2E_VALIDATION_2026-09-16.md). + +## Rollback + +Set `RCAC_COMPUTE_ENABLED=false` on the API and worker, then scale the isolated +worker Deployment to zero. Existing physical functionality remains untouched. +Submitted RCAC jobs do not stop automatically; cancel any active jobs through +the controlled gateway before scaling down. Database rows and Depot artifacts +remain as provenance and should not be deleted as part of a routine rollback. diff --git a/README.md b/README.md index 580421f..6e9e469 100644 --- a/README.md +++ b/README.md @@ -99,5 +99,7 @@ Birck-Digital-Twin/ ## Documentation - **[Deployment Runbook](deployment_runbook.md)**: Step-by-step instructions for live setup. +- **[Digital Computational Assets](DIGITAL_ASSETS_SIM2L_README.md)**: Current physical/digital asset model, RCAC architecture, scientific data, security, and rollout prerequisites. +- **[RCAC Computation Integration](RCAC_SIM2L_INTEGRATION.md)**: Detailed service-account, Globus Compute, worker, gateway, and validation guide. - **[API V2 Docs](api/data_loader_pg.py)**: Python interface for Postgres data access. - **[Web Auth Docs](web/SAML_AUTHENTICATION.md)**: nanoHUB SAML integration and deployment details. diff --git a/api/Dockerfile.worker b/api/Dockerfile.worker new file mode 100644 index 0000000..9ee958e --- /dev/null +++ b/api/Dockerfile.worker @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1.7 +# Build from the Digital Twin repository root. No Sim2L source checkout or +# student SSH credential is part of this image. +# docker build -f api/Dockerfile.worker -t dt-computation-worker:dev . + +FROM python:3.12-slim +WORKDIR /app +ENV HOME=/tmp + +COPY api/requirements.txt api/requirements-computations.txt ./ +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates build-essential \ + && pip install --no-cache-dir -r requirements-computations.txt \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* + +COPY api/ ./ +COPY domains/ /app/domains/ + +RUN useradd --system --uid 10001 --home-dir /tmp --shell /usr/sbin/nologin dtworker \ + && chown -R 10001:10001 /app + +# The worker has no listening socket and must run separately from FastAPI. +USER 10001:10001 +CMD ["python", "-m", "computations.worker"] diff --git a/api/computations.env.example b/api/computations.env.example new file mode 100644 index 0000000..6761aa7 --- /dev/null +++ b/api/computations.env.example @@ -0,0 +1,40 @@ +# Off by default. Enabling the feature requires the API and exactly one +# centralized worker Deployment to use the same reviewed configuration. +RCAC_COMPUTE_ENABLED=false + +# Dedicated non-person RCAC account approved for the ncn allocation. Every +# Slurm job runs under this Unix identity; the DT database and Slurm's trusted +# --comment=dt-run: provide platform-user attribution. +RCAC_SERVICE_ACCOUNT_USERNAME=REPLACE_WITH_APPROVED_SERVICE_ACCOUNT + +# Register api/computations/globus_gateway.py once, then configure one +# single-user Globus Compute endpoint per cluster under the service account. +GLOBUS_COMPUTE_CLIENT_ID=REPLACE_WITH_CONFIDENTIAL_CLIENT_UUID +GLOBUS_COMPUTE_CLIENT_SECRET_FILE=/var/run/secrets/globus/client-secret +GLOBUS_COMPUTE_GATEWAY_FUNCTION_ID=REPLACE_WITH_REGISTERED_FUNCTION_UUID + +# Fixed cluster allow-list. UUIDs and service-account scratch paths must be +# replaced after provisioning; users cannot override any scheduler/module/path +# value through the API. +RCAC_CLUSTERS_JSON={"negishi":{"display_name":"Negishi","host":"negishi.rcac.purdue.edu","endpoint_id":"REPLACE_WITH_NEGISHI_ENDPOINT_UUID","qe_version":"7.1","module_dependencies":["gcc/12.2.0","openmpi/4.1.4"],"qe_module":"quantum-espresso/7.1","mpi_launcher":"srun","allocation":"ncn","partition":"cpu","qos":"normal","remote_root":"/scratch/negishi/REPLACE_WITH_SERVICE_ACCOUNT/digital-twins/runs","pseudo_root":"/depot/ncn/data/digital-twins/pseudopotentials","artifact_root":"/depot/ncn/data/digital-twins/runs"},"gautschi":{"display_name":"Gautschi","host":"gautschi.rcac.purdue.edu","endpoint_id":"REPLACE_WITH_GAUTSCHI_ENDPOINT_UUID","qe_version":"7.3.1","module_dependencies":["gcc/11.4.1","openmpi/4.1.6"],"qe_module":"quantum-espresso/7.3.1","mpi_launcher":"mpirun","allocation":"ncn","partition":"cpu","qos":"normal","remote_root":"/scratch/gautschi/REPLACE_WITH_SERVICE_ACCOUNT/digital-twins/runs","pseudo_root":"/depot/ncn/data/digital-twins/pseudopotentials","artifact_root":"/depot/ncn/data/digital-twins/runs"}} + +# Small generated inputs exist only until submission. Raw outputs are copied +# from cluster scratch to /depot/ncn/data/digital-twins/runs by the remote gateway. +RCAC_LOCAL_WORK_ROOT=/var/lib/dt-computation-worker/runs + +# Secret: used when a CellRelaxDFT request resolves its Materials Project ID. +# In Kubernetes, source this from dt-computation-materials-project/api-key; +# do not put the value in the ConfigMap. +MATERIALS_PROJECT_API_KEY=REPLACE_WITH_SECRET_VALUE + +# Endpoint environment (configure these on each Globus Compute endpoint, not +# in the Geddes Pod). Values must match that cluster's allow-list entry: +# DT_CLUSTER_NAME +# DT_WORK_ROOT +# DT_PSEUDO_ROOT=/depot/ncn/data/digital-twins/pseudopotentials +# DT_ARTIFACT_ROOT=/depot/ncn/data/digital-twins/runs +# DT_SLURM_ACCOUNT=ncn +# DT_SLURM_PARTITION=cpu +# DT_SLURM_QOS=normal +# DT_QE_MODULES_JSON=["compiler/...","openmpi/...","quantum-espresso/..."] +# DT_MPI_LAUNCHER=srun|mpirun diff --git a/api/computations/__init__.py b/api/computations/__init__.py new file mode 100644 index 0000000..996cdf9 --- /dev/null +++ b/api/computations/__init__.py @@ -0,0 +1,13 @@ +"""Feature-flagged registered-computation integration for Purdue RCAC.""" + +from .schemas import ( + TOOL_DEFINITIONS, + compute_input_fingerprint, + validate_tool_inputs, +) + +__all__ = [ + "TOOL_DEFINITIONS", + "compute_input_fingerprint", + "validate_tool_inputs", +] diff --git a/api/computations/artifacts.py b/api/computations/artifacts.py new file mode 100644 index 0000000..22584a0 --- /dev/null +++ b/api/computations/artifacts.py @@ -0,0 +1,188 @@ +"""Durable artifact stores for QE inputs, outputs, logs, and manifests.""" + +from __future__ import annotations + +import hashlib +import mimetypes +import os +import shutil +import base64 +import re +import tempfile +from threading import BoundedSemaphore +from collections.abc import Iterable +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlparse +from uuid import UUID + +MAX_DEPOT_DOWNLOAD_BYTES = 8 * 1024 * 1024 +_DOWNLOAD_SLOTS = BoundedSemaphore(2) + + +def depot_downloads_enabled() -> bool: + return os.getenv("RCAC_ARTIFACT_DOWNLOADS_ENABLED", "false").lower() in {"true", "1", "yes"} + + +def fetch_depot_artifact(run: dict[str, Any], artifact: dict[str, Any], cluster, gateway): + """Stage and verify a small, DB-authorized archive before emitting bytes. + + The caller must check project access, including a cache source. No path + supplied by the HTTP caller is forwarded to the endpoint. + """ + run_id = str(UUID(str(run["id"]))) + name = str(artifact.get("name", "")) + parts = Path(name) + if not name or parts.is_absolute() or ".." in parts.parts or any(ord(c) < 32 for c in name): + raise ValueError("Invalid artifact name") + expected_uri = "depot://" + str(Path(cluster.artifact_root) / run_id / name) + size = artifact.get("size_bytes") + expected_hash = str(artifact.get("sha256", "")) + if artifact.get("storage_uri") != expected_uri or not re.fullmatch(r"[a-f0-9]{64}", expected_hash): + raise ValueError("Artifact manifest does not match the run archive") + if not isinstance(size, int) or not 0 <= size <= MAX_DEPOT_DOWNLOAD_BYTES: + raise ValueError("Artifact exceeds the 8 MiB pilot download limit") + if not _DOWNLOAD_SLOTS.acquire(blocking=False): + raise BlockingIOError("Artifact download capacity is busy; retry later") + staged = tempfile.SpooledTemporaryFile(max_size=256 * 1024, mode="w+b") + try: + offset = 0 + digest = hashlib.sha256() + while True: + length = min(256 * 1024, max(1, size - offset)) + _, result = gateway.invoke( + endpoint_id=cluster.endpoint_id, function_id=cluster.gateway_function_id, + action="read_artifact", timeout_seconds=60, + payload={"cluster": cluster.name, "run_id": run_id, + "job_id": str(run["scheduler_job_id"]), "name": name, + "sha256": expected_hash, "size_bytes": size, + "offset": offset, "length": length}, + ) + if result.get("offset") != offset or result.get("size_bytes") != size or result.get("sha256") != expected_hash: + raise ValueError("Artifact response does not match the manifest") + encoded = result.get("data", "") + if not isinstance(encoded, str) or len(encoded) > 350000: + raise ValueError("Artifact response exceeds transfer bounds") + chunk = base64.b64decode(encoded, validate=True) + if len(chunk) != min(length, size - offset): + raise ValueError("Artifact response is truncated") + staged.write(chunk) + digest.update(chunk) + offset += len(chunk) + if offset == size: + break + if digest.hexdigest() != expected_hash: + raise ValueError("Artifact checksum verification failed") + staged.seek(0) + return staged + except Exception: + staged.close() + raise + finally: + _DOWNLOAD_SLOTS.release() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +class LocalArtifactStore: + """Filesystem store intended for development and fake-RCAC E2E tests.""" + + def __init__(self, root: Path): + self.root = Path(root).resolve() + self.root.mkdir(parents=True, exist_ok=True) + + def put_many( + self, run_id: str, base_dir: Path, paths: Iterable[Path] + ) -> list[dict[str, Any]]: + destination_root = self.root / run_id + destination_root.mkdir(parents=True, exist_ok=True) + base_dir = Path(base_dir).resolve() + manifest = [] + for path in paths: + path = Path(path).resolve() + relative = path.relative_to(base_dir) + destination = destination_root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, destination) + manifest.append(_artifact_record(path, relative, destination.as_uri())) + return manifest + + def resolve(self, uri: str) -> Path: + parsed = urlparse(uri) + if parsed.scheme != "file": + raise ValueError("artifact is not stored on the local filesystem") + path = Path(unquote(parsed.path)).resolve() + if self.root != path and self.root not in path.parents: + raise PermissionError("artifact path is outside the configured store") + if not path.is_file(): + raise FileNotFoundError(path.name) + return path + + +class AzureBlobArtifactStore: + """Azure-backed store using the platform's existing blob dependency.""" + + def __init__(self, connection_string: str, container: str, prefix: str = "qe-runs"): + from azure.storage.blob import BlobServiceClient + + self.container_name = container + self.prefix = prefix.strip("/") + client = BlobServiceClient.from_connection_string(connection_string) + self.container = client.get_container_client(container) + try: + self.container.create_container() + except Exception as exc: + # Existing containers raise ResourceExistsError. Avoid importing + # Azure exception classes during normal API startup. + if "ContainerAlreadyExists" not in str(exc) and "exists" not in str(exc): + raise + + def put_many( + self, run_id: str, base_dir: Path, paths: Iterable[Path] + ) -> list[dict[str, Any]]: + base_dir = Path(base_dir).resolve() + manifest = [] + for path in paths: + path = Path(path).resolve() + relative = path.relative_to(base_dir) + blob_name = f"{self.prefix}/{run_id}/{relative.as_posix()}" + with path.open("rb") as handle: + self.container.upload_blob(blob_name, handle, overwrite=True) + uri = f"azure://{self.container_name}/{blob_name}" + manifest.append(_artifact_record(path, relative, uri)) + return manifest + + def downloader(self, uri: str): + prefix = f"azure://{self.container_name}/" + if not uri.startswith(prefix): + raise ValueError("artifact is not in the configured Azure container") + blob_name = uri[len(prefix) :] + if not blob_name or ".." in Path(blob_name).parts: + raise ValueError("invalid Azure artifact path") + return self.container.download_blob(blob_name) + + +def _artifact_record(path: Path, relative: Path, uri: str) -> dict[str, Any]: + return { + "name": relative.as_posix(), + "uri": uri, + "sha256": _sha256(path), + "size_bytes": path.stat().st_size, + "content_type": mimetypes.guess_type(path.name)[0] + or "application/octet-stream", + } + + +def artifact_store_from_env(): + connection = os.getenv("RCAC_ARTIFACT_AZURE_CONNECTION_STRING", "").strip() + container = os.getenv("RCAC_ARTIFACT_AZURE_CONTAINER", "").strip() + if connection and container: + return AzureBlobArtifactStore(connection, container) + root = Path(os.getenv("RCAC_ARTIFACT_ROOT", "/tmp/dt-rcac-artifacts")) + return LocalArtifactStore(root) diff --git a/api/computations/backfill_diagnostics.py b/api/computations/backfill_diagnostics.py new file mode 100644 index 0000000..48d4653 --- /dev/null +++ b/api/computations/backfill_diagnostics.py @@ -0,0 +1,341 @@ +"""Operator-only: enrich one terminal run's diagnostics from its archive. + + python -m computations.backfill_diagnostics --run-id # preview + python -m computations.backfill_diagnostics --run-id --apply # write + +Evidence comes only from the run's checksum-verified archive copies through +the registered gateway's read-only ``archive_evidence`` action, summarized by +the same extractor and classifier as live collection. Nothing is resubmitted, +cancelled or re-collected from scratch, and no scheduler accounting is queried +or invented: Slurm records already stored with the run are reused, and a +member whose archive evidence is incomplete keeps its earlier evidence-based +diagnosis rather than losing it. +``--apply`` replaces only the stored diagnostics and the readable error +summary; the previous error is kept in the enrichment provenance and in one +audit event. Status, timestamps, inputs, results, fingerprint, artifacts and +scheduler records are untouched, and the write is refused if the run changed +since it was read. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import PurePosixPath +from typing import Any +from uuid import UUID + +from .diagnostics import ( + WEAK_CATEGORIES, + belongs_to_job, + build_diagnostics, + examined, + member_summary, + summary_message, +) +from .resources import RESOURCE_GATEWAY_PROTOCOL + +MAX_ARCHIVE_FILES = 400 +ENRICHMENT_EVENT = ( + "Diagnostics enriched from checksum-verified archived files; no job was rerun " + "and status, inputs and results are unchanged" +) + + +class EnrichmentError(RuntimeError): + """The run cannot be enriched; nothing was read remotely or written.""" + + +def _json(value: Any, default: Any) -> Any: + if isinstance(value, str): + try: + return json.loads(value) + except json.JSONDecodeError: + return default + return default if value is None else value + + +def _digest(value: Any) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def _evidence_names(prepared: dict[str, Any], job_id: str) -> dict[str, str]: + """Archive names that may carry evidence, mapped to their evidence kind.""" + + expected = [str(name) for name in prepared.get("expected_artifacts") or []] + names = {name: "qe_output" for name in expected} + names.update({str(PurePosixPath(name).parent / "CRASH"): "qe_crash" for name in expected}) + if prepared.get("job_array"): + logs = [f"slurm-{job_id}_{index}.{ext}" for index in range(len(expected)) for ext in ("err", "out")] + else: + logs = ["slurm.err", "slurm.out"] + names.update({name: "scheduler_log" for name in logs}) + return names + + +def _recorded_steps(current: dict[str, Any], job_id: str) -> list[dict[str, Any]]: + """Attributed Slurm records already stored with the run's diagnostics. + + These were recorded when the run was collected; enrichment reuses them and + never queries or invents accounting. + """ + + records = [] + for member in current.get("members") or []: + if not isinstance(member, dict): + continue + candidates = [member.get("allocation"), *(member.get("scheduler_steps") or [])] + records.extend( + copy.deepcopy(record) for record in candidates + if isinstance(record, dict) and belongs_to_job(record.get("job_id"), job_id) + ) + return records + + +def _archive_complete(member: dict[str, Any], files: dict[str, Any]) -> bool: + """Both the QE output and the stderr log of this member were read.""" + + artifacts = member["artifacts"] + return examined(files.get(artifacts["qe_output"])) and examined(files.get(artifacts["logs"][0])) + + +def _preserve_recorded_evidence( + diagnostics: dict[str, Any], current: dict[str, Any], external: dict[str, Any], files: dict[str, Any] +) -> list[Any]: + """Never let an incomplete archive reduce what was already known. + + Recorded per-member usage is kept. A member whose archive evidence is + incomplete and whose new category is weak keeps its earlier + evidence-based diagnosis unchanged. Returns the retained member ids. + """ + + previous = {member.get("array_task_id"): member for member in current.get("members") or [] + if isinstance(member, dict)} + usage = {item.get("array_task_id"): item for item in external.get("scheduler_resource_usage") or [] + if isinstance(item, dict)} + retained = [] + for position, member in enumerate(diagnostics["members"]): + index = member["array_task_id"] + old = previous.get(index) or {} + recorded = old.get("resource_usage") or usage.get(index) + if isinstance(recorded, dict): + member["resource_usage"] = copy.deepcopy(recorded) + if (old.get("category") and member["category"] in WEAK_CATEGORIES + and old["category"] not in WEAK_CATEGORIES | {"completed"} + and not _archive_complete(member, files)): + kept = copy.deepcopy(old) + kept["retained_from_previous_diagnostics"] = True + diagnostics["members"][position] = kept + retained.append(index) + if retained: + diagnostics.update(member_summary(diagnostics["members"])) + diagnostics["notes"].append( + f"The archive was incomplete for {len(retained)} member(s); their previously recorded " + "diagnosis was kept unchanged." + ) + return retained + + +def plan_enrichment(run: dict[str, Any], cluster: Any, gateway: Any) -> dict[str, Any]: + """Read archive evidence and build the proposed update; writes nothing.""" + + from .store import _public_run, run_version + + run_id = str(UUID(str(run["id"]))) + status = str(run.get("status") or "") + if status not in {"failed", "cancelled"}: + raise EnrichmentError(f"Only failed or cancelled runs can be enriched (status is {status!r})") + external = _json(run.get("external_workflow_ref_json"), {}) + if external.get("cache_hit"): + raise EnrichmentError("A reused (cached) result has no archive of its own") + job_id = str(run.get("scheduler_job_id") or "") + if not re.fullmatch(r"[0-9]+", job_id): + raise EnrichmentError("The run has no Slurm job, so it has no archived evidence") + if cluster.name != run.get("cluster"): + raise EnrichmentError("Cluster configuration does not match the run") + if cluster.gateway_protocol < RESOURCE_GATEWAY_PROTOCOL: + raise EnrichmentError( + f"{cluster.name} is configured for gateway protocol {cluster.gateway_protocol}; archive " + "evidence needs the registered protocol-2 gateway (see IMPLEMENTATION_NOTES.md)" + ) + prepared = _json(run.get("prepared_manifest_json"), {}) + if not prepared.get("expected_artifacts"): + raise EnrichmentError("The run's prepared manifest lists no QE outputs") + candidates = _evidence_names(prepared, job_id) + entries: list[dict[str, Any]] = [] + unmatched: dict[str, dict[str, str]] = {} + for artifact in _json(run.get("artifact_manifest_json"), []): + if not isinstance(artifact, dict) or str(artifact.get("name", "")) not in candidates: + continue + name = str(artifact["name"]) + uri = "depot://" + str(PurePosixPath(cluster.artifact_root) / run_id / name) + size, digest = artifact.get("size_bytes"), str(artifact.get("sha256", "")) + if artifact.get("storage_uri") != uri or type(size) is not int or not re.fullmatch(r"[a-f0-9]{64}", digest): + # Recorded entry does not describe this run's archive: unknown. + unmatched[name] = {"kind": candidates[name], "error": "manifest entry does not match the run archive"} + continue + entries.append({"name": name, "sha256": digest, "size_bytes": size}) + if not entries: + raise EnrichmentError("No archived QE output or Slurm log is recorded for this run") + if len(entries) > MAX_ARCHIVE_FILES: + raise EnrichmentError("Too many archived evidence files for one request") + entries.sort(key=lambda entry: entry["name"]) + + task_id, result = gateway.invoke( + endpoint_id=cluster.endpoint_id, function_id=cluster.gateway_function_id, + action="archive_evidence", timeout_seconds=300.0, + payload={"cluster": cluster.name, "run_id": run_id, "job_id": job_id, + "tool_name": run["tool_name"], "expected_artifacts": prepared["expected_artifacts"], + "artifacts": entries}, + ) + returned = (result.get("diagnostics") or {}).get("files") if isinstance(result, dict) else None + if not isinstance(returned, dict): + raise EnrichmentError("The gateway returned no archive evidence") + requested = {entry["name"] for entry in entries} + files = {name: record for name, record in returned.items() if name in requested and isinstance(record, dict)} + for name in requested - set(files): + files[name] = {"kind": candidates[name], "error": "no evidence was returned for this file"} + files.update(unmatched) + evidence_sha256 = _digest({"artifacts": entries, "files": files}) + + current = _public_run(run).get("diagnostics") or {} + previous_error = copy.deepcopy(_json(run.get("error_json"), {})) + protocol = external.get("gateway_protocol") + # Recorded collection observations plus archive evidence. Scheduler step + # records are only those already stored for this run; older runs without + # them stay "not collected" rather than gaining invented accounting. + collection: dict[str, Any] = { + "diagnostics": {"files": files}, + "missing_artifacts": external.get("missing_artifacts") or [], + "collection_errors": external.get("collection_errors") or [], + "scheduler_steps_error": (current.get("collection") or {}).get("scheduler_steps_error") or "", + } + recorded_steps = _recorded_steps(current, job_id) + if recorded_steps: + collection["scheduler_steps"] = recorded_steps + elif isinstance(external.get("scheduler_resource_usage"), list): + collection["scheduler_steps"] = [] # collected, but nothing attributable was recorded + diagnostics = build_diagnostics( + tool_name=str(run["tool_name"]), job_id=job_id, prepared_manifest=prepared, + accounting=external.get("scheduler_accounting"), collection=collection, + outcome=str(current.get("outcome") or status), parse_error=str(current.get("parse_error") or ""), + resources=external.get("submitted_resources") or prepared.get("resources") or {}, + pw_exit_logged=type(protocol) is int and protocol >= RESOURCE_GATEWAY_PROTOCOL, + ) + retained = _preserve_recorded_evidence(diagnostics, current, external, files) + unavailable = sorted(name for name, record in files.items() if record.get("error")) + diagnostics["enrichment"] = { + "source": "archived_artifacts", + "method": "read-only gateway archive_evidence with SHA-256 and size verification", + "enriched_at": datetime.now(timezone.utc).isoformat(), + "gateway_task_id": task_id, + "gateway_function_id": cluster.gateway_function_id, + "gateway_protocol": result.get("protocol"), + "files_requested": len(entries) + len(unmatched), + "files_unavailable": unavailable, + "recorded_scheduler_records_reused": len(recorded_steps), + "members_retained_from_previous_diagnostics": retained, + "evidence_sha256": evidence_sha256, + "previous_error": previous_error, + "previous_diagnostics": {key: current.get(key) for key in ("schema", "derived", "evidence_available", "category_counts")}, + } + diagnostics["notes"].insert(0, ( + "Evidence was extracted after the run ended from its checksum-verified archived files; " + "no job was rerun and no scheduler data was added." + )) + error_json = {**previous_error, "message": summary_message(diagnostics), + "category_counts": diagnostics["category_counts"], + "exit_code_counts": diagnostics["exit_code_counts"]} + already = (current.get("enrichment") or {}).get("evidence_sha256") == evidence_sha256 + return { + "run_id": run_id, + "version": run_version(run), + "already_enriched": already, + "diagnostics": diagnostics, + "error_json": error_json, + "current": {"message": previous_error.get("message"), "category_counts": current.get("category_counts")}, + "event_details": { + "kind": "diagnostics_enrichment", "evidence_sha256": evidence_sha256, + "gateway_task_id": task_id, "previous_error": previous_error, + "previous_category_counts": current.get("category_counts"), + "new_category_counts": diagnostics["category_counts"], "files_unavailable": unavailable, + }, + } + + +def preview(plan: dict[str, Any]) -> dict[str, Any]: + diagnostics = plan["diagnostics"] + return { + "run_id": plan["run_id"], + "mode": "preview", + "already_enriched": plan["already_enriched"], + "current": plan["current"], + "proposed": { + "message": plan["error_json"]["message"], + "category_counts": diagnostics["category_counts"], + "exit_code_counts": diagnostics["exit_code_counts"], + }, + "members": [ + {"array_task_id": member["array_task_id"], "label": member["label"], "state": member["state"], + "exit_code": member["exit_code"], "category": member["category"], + "evidence": (member["evidence"][0]["text"] if member["evidence"] else "")} + for member in diagnostics["members"] + ], + "files_unavailable": diagnostics["enrichment"]["files_unavailable"], + "recorded_scheduler_records_reused": diagnostics["enrichment"]["recorded_scheduler_records_reused"], + "members_retained_from_previous_diagnostics": + diagnostics["enrichment"]["members_retained_from_previous_diagnostics"], + "evidence_sha256": diagnostics["enrichment"]["evidence_sha256"], + } + + +def apply_plan(plan: dict[str, Any]) -> str: + from .store import enrich_run_diagnostics + + if plan["already_enriched"]: + return "unchanged" + return enrich_run_diagnostics( + plan["run_id"], expected_version=plan["version"], diagnostics=plan["diagnostics"], + error_json=plan["error_json"], event_message=ENRICHMENT_EVENT, event_details=plan["event_details"], + ) + + +def main(argv: list[str] | None = None, *, gateway: Any = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--run-id", required=True, help="exact computation run UUID") + parser.add_argument("--apply", action="store_true", help="write the enrichment (default: preview only)") + args = parser.parse_args(argv) + try: + run_id = str(UUID(args.run_id)) + except ValueError: + parser.error("--run-id must be a UUID") + from .globus_gateway import GlobusComputeGateway + from .rcac import get_cluster + from .store import EnrichmentConflict, get_run + + run = get_run(run_id, internal=True) + if not run: + raise SystemExit(f"Computation run {run_id} does not exist") + try: + plan = plan_enrichment(dict(run), get_cluster(run["cluster"]), gateway or GlobusComputeGateway()) + except EnrichmentError as exc: + raise SystemExit(f"Not enriched: {exc}") from exc + print(json.dumps(preview(plan), indent=2, ensure_ascii=False)) + if not args.apply: + print("Preview only; nothing was written. Re-run with --apply to store this diagnosis.") + return 0 + try: + outcome = apply_plan(plan) + except EnrichmentConflict as exc: + raise SystemExit(f"Not enriched: {exc}") from exc + print(json.dumps({"run_id": run_id, "mode": "apply", "outcome": outcome})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/api/computations/diagnostics.py b/api/computations/diagnostics.py new file mode 100644 index 0000000..007f0ec --- /dev/null +++ b/api/computations/diagnostics.py @@ -0,0 +1,581 @@ +"""Evidence-based diagnostics for failed registered computations. + +Categories come only from what was observed: Slurm states and exit codes, +step accounting, and bounded marker lines from QE output, QE CRASH files and +Slurm logs returned by the gateway. Nothing is labelled out-of-memory without +Slurm's OUT_OF_MEMORY state or an out-of-memory line in a log or QE output. An +exit code alone only produces a hint, and missing, unreadable or uncollected +evidence never proves a cause: only files the gateway actually examined count. +""" + +from __future__ import annotations + +import re +from collections import Counter +from datetime import datetime, timezone +from pathlib import PurePosixPath +from typing import Any + +SCHEMA = "dt-run-diagnostics/1" +MAX_MEMBERS = 1000 +MAX_EVIDENCE_LINES = 4 +MAX_TAILED_MEMBERS = 12 + +CATEGORY_LABELS = { + "completed": "Completed", + "cancelled": "Cancelled", + "out_of_memory": "Out of memory", + "walltime_exceeded": "Walltime limit reached", + "node_failure": "Node failure", + "qe_scf_not_converged": "SCF did not converge", + "qe_ionic_step_limit": "Ionic step limit reached", + "qe_bfgs_failed": "BFGS stopped before meeting the requested tolerances", + "qe_error": "Quantum ESPRESSO error", + "launch_or_environment": "Failed before pw.x finished", + "killed_by_signal": "Killed by a signal", + "nonzero_exit": "Non-zero exit, cause not identified", + "no_scheduler_record": "No scheduler record (not started or not reported)", + "unknown": "Unclassified", +} + +_NO_RESOURCE_FIX = "More cores, nodes or memory do not address this." +CATEGORY_GUIDANCE = { + "out_of_memory": ( + "Slurm or a log reported an out-of-memory event. Compare QE's estimated RAM (an estimate, " + "not a measured peak) with the memory per core, then request more memory where the cluster " + "offers it or reduce the calculation size." + ), + "walltime_exceeded": ( + "The job reached its walltime limit before QE finished. Use a longer walltime within the " + "limit, or reduce the cost of each calculation." + ), + "node_failure": ( + "The scheduler reported a node failure, which is not caused by the inputs. Report the run " + "ID; submitting again later is reasonable." + ), + "qe_scf_not_converged": ( + "pw.x stopped because an SCF cycle did not converge within e_steps. " + _NO_RESOURCE_FIX + + " Review e_steps, smearing/metal settings and the structure; ElasticDFT strain members " + "inherit these settings from the parent relaxation." + ), + "qe_ionic_step_limit": ( + "The relaxation used all ionic steps (i_steps) without meeting the energy and force " + "criteria. " + _NO_RESOURCE_FIX + " ElasticDFT strain members inherit i_steps and the " + "thresholds from the parent relaxation." + ), + "qe_bfgs_failed": ( + "The BFGS optimizer stopped (its history had already been reset) before the final energy " + "and gradient errors met the requested criteria shown here. " + _NO_RESOURCE_FIX + + " ElasticDFT strain members inherit scf_conv, energy_conv and force_conv from the " + "parent relaxation; the platform never loosens them. Whether different thresholds are " + "scientifically acceptable is your decision: to use them, run a new 3D relaxation with " + "those values and select it as the parent." + ), + "qe_error": ( + "pw.x stopped in its error routine; the evidence names the routine and message. Read it " + "before changing resources." + ), + "launch_or_environment": ( + "Positive evidence shows the job failed before pw.x finished normally (module setup, input " + "extraction or launch). This is an execution problem, not your inputs; report the run ID." + ), + "killed_by_signal": ( + "The process was killed by a signal. No out-of-memory evidence was found, so do not assume " + "memory; check the member's stderr log." + ), + "nonzero_exit": ( + "The job exited with a non-zero status and no examined log or QE marker identified the " + "cause. Inspect the member's QE output and stderr artifacts." + ), + "no_scheduler_record": "Slurm returned no accounting record and no examined file explains this member.", +} + +# Evidence tags shown for each category, most decisive first. +_CATEGORY_TAGS = { + "out_of_memory": ("out_of_memory", "memory"), + "walltime_exceeded": ("time_limit", "cpu_time"), + "cancelled": ("cancelled",), + "qe_scf_not_converged": ("scf_not_converged",), + "qe_ionic_step_limit": ("ionic_step_limit",), + "qe_error": ("qe_error",), + "qe_bfgs_failed": ("bfgs_failed", "bfgs_history_reset"), + "launch_or_environment": ("environment", "launcher", "signal"), + "killed_by_signal": ("signal", "launcher", "pw_exit"), + "nonzero_exit": ("pw_exit", "launcher", "signal"), +} + +_SIGNALS = {6: "SIGABRT", 7: "SIGBUS", 9: "SIGKILL", 11: "SIGSEGV", 15: "SIGTERM"} +_STATUS_HINTS = { + 1: "status 1: generic failure (QE error routine, MPI abort, or a failing shell command)", + 2: "status 2: pw.x uses 2 when an SCF cycle does not converge; confirm in the QE output", + 3: "status 3: pw.x uses 3 when the ionic optimization does not converge, either a BFGS " + "failure or the nstep limit; confirm in the QE output", + 127: "status 127: a command was not found", + 137: "status 137 = 128+9: killed by SIGKILL (memory is one possible reason, not a conclusion)", + 143: "status 143 = 128+15: terminated by SIGTERM", +} +_STEP_KEYS = ("job_id", "state", "exit_code", "derived_exit_code", "elapsed", "time_limit", + "max_rss", "req_mem", "alloc_cpus", "nnodes") +_AXES = "xyz" +_FAILED_STATES = {"FAILED", "OUT_OF_MEMORY", "TIMEOUT", "NODE_FAIL", "BOOT_FAIL", "DEADLINE"} + + +def _text(value: Any, limit: int = 64) -> str: + return str(value or "").strip()[:limit] + + +def _state(value: Any) -> str: + parts = str(value or "").strip().upper().split() + return parts[0].split("+", 1)[0] if parts else "" + + +def _exit_parts(exit_code: str) -> tuple[int, int] | None: + match = re.fullmatch(r"([0-9]{1,3}):([0-9]{1,3})", exit_code or "") + return (int(match.group(1)), int(match.group(2))) if match else None + + +def exit_code_hint(exit_code: str) -> str: + parts = _exit_parts(exit_code) + if not parts or parts == (0, 0): + return "" + status, signal = parts + if signal > 64: + return f"scheduler-reported termination code {signal}" + if signal: + name = _SIGNALS.get(signal) + return f"terminated by signal {signal}" + (f" ({name})" if name else "") + return _STATUS_HINTS.get(status, f"status {status}") + + +def _member_key(raw_id: str, job_id: str) -> tuple[int | None, str] | None: + match = re.fullmatch( + re.escape(job_id) + r"(?:_([0-9]{1,4}))?(?:\.([A-Za-z0-9_+-]+))?", raw_id or "" + ) + if not job_id or not match: + return None + return (int(match.group(1)) if match.group(1) is not None else None, match.group(2) or "") + + +def _strain_label(matrix: Any) -> str: + try: + size, row, column = max( + ((abs(float(matrix[i][j])), i, j) for i in range(3) for j in range(i, 3)), + key=lambda item: item[0], + ) + value = float(matrix[row][column]) + except (TypeError, ValueError, IndexError): + return "" + if size == 0: + return "unstrained" + return f"ε_{_AXES[row]}{_AXES[column]} = {value:+g}" + + +def _size_bytes(value: str) -> float: + match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)([KMGTP]?)", value or "") + if not match: + return -1.0 + return float(match.group(1)) * 1024 ** " KMGTP".index(match.group(2) or " ") + + +_REASON_TAGS = ( + ("scf_not_converged", "convergence not achieved after"), + ("ionic_step_limit", "maximum number of steps"), + ("bfgs_failed", "bfgs failed"), + ("qe_error", "error in routine"), +) + + +def _reasons_record(reasons: list[Any]) -> dict[str, Any]: + """Evidence from the gateway parser's failure lines (no file was scanned).""" + + counts: dict[str, int] = {} + markers = [] + for reason in reasons[:5]: + text = _text(reason, 240) + for tag, needle in _REASON_TAGS: + if needle in text.lower(): + counts[tag] = counts.get(tag, 0) + 1 + markers.append({"tag": tag, "line_number": None, "text": text}) + break + return {"kind": "qe_parser_failure_reasons", "counts": counts, "markers": markers} + + +def _examined(record: Any) -> bool: + """True only for a file the gateway actually read (not absent or unreadable).""" + + return isinstance(record, dict) and isinstance(record.get("size_bytes"), int) and not record.get("error") + + +def _skeleton(index: int | None, job_id: str, expected: list[str], matrices: list[Any]) -> dict[str, Any]: + if index is None: + qe_output = expected[0] if expected else "" + logs = ["slurm.err", "slurm.out"] + else: + qe_output = expected[index] if index < len(expected) else "" + logs = [f"slurm-{job_id}_{index}.err", f"slurm-{job_id}_{index}.out"] + crash = str(PurePosixPath(qe_output).parent / "CRASH") if qe_output else "CRASH" + label = _strain_label(matrices[index]) if index is not None and index < len(matrices) else "" + return { + "array_task_id": index, + "label": label, + "state": "", + "exit_code": "", + "artifacts": {"qe_output": qe_output, "crash": crash, "logs": logs}, + } + + +def _marker_lines(name: str, record: dict[str, Any], tags: tuple[str, ...]) -> list[dict[str, Any]]: + lines = [] + for tag in tags: + for entry in record.get("markers") or []: + if not isinstance(entry, dict) or entry.get("tag") != tag: + continue + line = {"source": name, "tag": tag, "line_number": entry.get("line_number"), + "text": _text(entry.get("text"), 240)} + if isinstance(record.get("scanned_from_byte"), int): + # Only the file's tail was scanned; line numbers count from here. + line["window_start_byte"] = record["scanned_from_byte"] + lines.append(line) + return lines + + +def _classify(member: dict[str, Any], files: dict[str, Any], missing: set[str], + evidence_available: bool, pw_exit_logged: bool) -> tuple[str, list[dict[str, Any]]]: + artifacts = member["artifacts"] + qe_name, crash_name = artifacts["qe_output"], artifacts["crash"] + qe = files.get(qe_name) or {} + crash = files.get(crash_name) or {} + logs = {name: files.get(name) or {} for name in artifacts["logs"]} + qe_tags = set(qe.get("counts") or {}) | set(crash.get("counts") or {}) + log_tags = set().union(*(set(record.get("counts") or {}) for record in logs.values())) + state = _state(member.get("state")) + step_states = {_state(step.get("state")) for step in member.get("scheduler_steps", [])} + parts = _exit_parts(member.get("exit_code", "")) + failed = state in _FAILED_STATES or bool(parts and parts != (0, 0)) + + def evidence(tags: tuple[str, ...]) -> list[dict[str, Any]]: + lines = _marker_lines(qe_name, qe, tags) + _marker_lines(crash_name, crash, tags) + for name, record in logs.items(): + lines += _marker_lines(name, record, tags) + return lines + + def observed(text: str, source: str = "slurm accounting") -> list[dict[str, Any]]: + return [{"source": source, "tag": "observation", "line_number": None, "text": text}] + + # Explicit scheduler states and examined file markers decide; absent + # accounting is recorded separately and never hides file evidence. + if "OUT_OF_MEMORY" in {state} | step_states: + oom_steps = [step["job_id"] for step in member.get("scheduler_steps", []) + if _state(step.get("state")) == "OUT_OF_MEMORY"] + return "out_of_memory", observed( + "OUT_OF_MEMORY" + (f" in {', '.join(oom_steps[:3])}" if oom_steps else "") + ) + evidence(_CATEGORY_TAGS["out_of_memory"]) + if "out_of_memory" in log_tags or "memory" in qe_tags: + return "out_of_memory", evidence(_CATEGORY_TAGS["out_of_memory"]) + if state in {"TIMEOUT", "DEADLINE"} or "TIMEOUT" in step_states: + return "walltime_exceeded", observed(state or "TIMEOUT") + evidence(_CATEGORY_TAGS["walltime_exceeded"]) + if "time_limit" in log_tags or "cpu_time" in qe_tags: + return "walltime_exceeded", evidence(_CATEGORY_TAGS["walltime_exceeded"]) + if state in {"NODE_FAIL", "BOOT_FAIL"} or step_states & {"NODE_FAIL", "BOOT_FAIL"}: + return "node_failure", observed(state or "NODE_FAIL") + if state in {"CANCELLED", "PREEMPTED", "REVOKED"}: + return "cancelled", observed(state) + evidence(_CATEGORY_TAGS["cancelled"]) + for tag, category in (("scf_not_converged", "qe_scf_not_converged"), + ("ionic_step_limit", "qe_ionic_step_limit")): + if tag in qe_tags: + return category, evidence(_CATEGORY_TAGS[category]) + if "qe_error" in qe_tags or "qe_error" in log_tags: + return "qe_error", evidence(_CATEGORY_TAGS["qe_error"]) + if "bfgs_failed" in qe_tags: + return "qe_bfgs_failed", evidence(_CATEGORY_TAGS["qe_bfgs_failed"]) + if state == "COMPLETED" and parts in {None, (0, 0)}: + return "completed", [] + if parts and (0 < parts[1] <= 64 or parts[0] in {137, 143}): + return "killed_by_signal", evidence(_CATEGORY_TAGS["killed_by_signal"]) + if failed and evidence_available: + # Only positive observations: an environment marker, an examined empty + # QE output, an output the gateway reported as never created, or an + # examined complete stderr lacking the runner's pw.x status line. + if "environment" in log_tags: + return "launch_or_environment", evidence(_CATEGORY_TAGS["launch_or_environment"]) + if _examined(qe) and qe["size_bytes"] == 0: + return "launch_or_environment", observed(f"{qe_name} is empty", "gateway") + if qe_name and qe_name in missing: + return "launch_or_environment", observed(f"{qe_name} was not created", "gateway") + stderr = logs.get(artifacts["logs"][0]) or {} + if (pw_exit_logged and _examined(stderr) and "scanned_from_byte" not in stderr + and "pw_exit" not in (stderr.get("counts") or {})): + return "launch_or_environment", observed( + f"{artifacts['logs'][0]} has no pw.x exit-status line", "gateway" + ) + evidence(_CATEGORY_TAGS["launch_or_environment"]) + if failed: + return "nonzero_exit", evidence(_CATEGORY_TAGS["nonzero_exit"]) + if not state and not step_states: + return "no_scheduler_record", [] + return "unknown", [] + + +def req_mem_scope(value: str) -> str: + """Slurm ReqMem suffix: ``c`` per CPU, ``n`` per node; newer Slurm may omit it.""" + + match = re.fullmatch(r"[0-9]+(?:\.[0-9]+)?[KMGTP]?([cn]?)", value or "") + if not match: + return "unknown" + return {"c": "per_cpu", "n": "per_node"}.get(match.group(1), "not_stated") + + +def belongs_to_job(raw_id: Any, job_id: str) -> bool: + """True if an accounting record id names this job or one of its members/steps.""" + + return _member_key(_text(raw_id), str(job_id or "")) is not None + + +def examined(record: Any) -> bool: + """Public form of the rule that only files actually read count as evidence.""" + + return _examined(record) + + +WEAK_CATEGORIES = frozenset({"nonzero_exit", "unknown", "no_scheduler_record"}) + + +def member_summary(members: list[dict[str, Any]]) -> dict[str, Any]: + """Counts, labels and guidance derived from the member list.""" + + categories = Counter(str(member.get("category") or "unknown") for member in members) + return { + "member_count": len(members), + "state_counts": dict(Counter(_state(m.get("state")) or "NO_RECORD" for m in members)), + "category_counts": dict(categories), + "exit_code_counts": dict(Counter( + m["exit_code"] for m in members if m.get("exit_code") and m["exit_code"] != "0:0" + )), + "category_labels": {name: CATEGORY_LABELS.get(name, name) for name in categories}, + "guidance": {name: CATEGORY_GUIDANCE[name] for name in categories if name in CATEGORY_GUIDANCE}, + } + + +def resource_usage(steps: Any, job_id: str) -> list[dict[str, Any]]: + """Compact per-member scheduler usage from step-level accounting. + + ``max_rss`` is Slurm's largest per-task resident-memory high-water mark + (one MPI task, not a node or job total) and is not comparable with + ``req_mem``, whose scope is given by ``req_mem_scope``. + """ + + members: dict[int | None, dict[str, Any]] = {} + for record in steps if isinstance(steps, list) else []: + if not isinstance(record, dict): + continue + key = _member_key(_text(record.get("job_id")), str(job_id)) + if key is None: + continue + index, step = key + if index not in members and len(members) >= MAX_MEMBERS: + continue + entry = members.setdefault(index, {"array_task_id": index}) + if not step: + for name in ("state", "exit_code", "elapsed", "time_limit", "req_mem", "alloc_cpus", "nnodes"): + entry[name] = _text(record.get(name)) + entry["req_mem_scope"] = req_mem_scope(entry["req_mem"]) + rss = _text(record.get("max_rss")) + if _size_bytes(rss) > _size_bytes(entry.get("max_rss", "")): + entry["max_rss"] = rss + return [members[key] for key in sorted(members, key=lambda key: -1 if key is None else key)] + + +def build_diagnostics( + *, + tool_name: str, + job_id: str, + prepared_manifest: dict[str, Any] | None, + accounting: Any = None, + collection: dict[str, Any] | None = None, + outcome: str, + parse_error: str = "", + resources: dict[str, Any] | None = None, + pw_exit_logged: bool = False, + derived: bool = False, +) -> dict[str, Any]: + job_id = str(job_id or "") + manifest = prepared_manifest if isinstance(prepared_manifest, dict) else {} + collection = collection if isinstance(collection, dict) else {} + expected = [str(name) for name in manifest.get("expected_artifacts") or []][:MAX_MEMBERS] + matrices = (manifest.get("metadata") or {}).get("strain_matrices") or [] + is_array = bool(manifest.get("job_array")) + members: dict[int | None, dict[str, Any]] = { + index: _skeleton(index, job_id, expected, matrices) + for index in (range(len(expected)) if is_array else [None]) + } + + def member_for(raw_id: Any) -> tuple[dict[str, Any] | None, str]: + key = _member_key(_text(raw_id), job_id) + if key is None: + return None, "" + index, step = key + if index not in members and len(members) < MAX_MEMBERS: + members[index] = _skeleton(index, job_id, expected, matrices) + return members.get(index), step + + for record in accounting if isinstance(accounting, list) else []: + if not isinstance(record, dict): + continue + member, step = member_for(record.get("job_id")) + if member is None or step: + continue + member.update( + state=_text(record.get("state")), exit_code=_text(record.get("exit_code"), 16), + started_at=record.get("started_at"), ended_at=record.get("ended_at"), + ) + steps = collection.get("scheduler_steps") + for record in steps if isinstance(steps, list) else []: + if not isinstance(record, dict): + continue + member, step = member_for(record.get("job_id")) + if member is None: + continue + entry = {key: _text(record.get(key)) for key in _STEP_KEYS} + if not step: + member["allocation"] = entry + if not member["state"]: + member.update(state=entry["state"], exit_code=entry["exit_code"][:16]) + elif len(member.setdefault("scheduler_steps", [])) < 8: + member["scheduler_steps"].append(entry) + usage = {item["array_task_id"]: item for item in resource_usage(steps, job_id)} + + evidence_block = collection.get("diagnostics") + evidence_available = isinstance(evidence_block, dict) + files = (evidence_block or {}).get("files") if evidence_available else {} + files = dict(files) if isinstance(files, dict) else {} + missing = {str(name) for name in collection.get("missing_artifacts") or []} + parsed = collection.get("parsed_outputs") if isinstance(collection.get("parsed_outputs"), dict) else {} + for name, summary in parsed.items(): + # Parser failure lines are evidence too when the file itself was not + # scanned; they are not "examined files", so they never imply launch. + if name not in files and isinstance(summary, dict) and summary.get("failure_reasons"): + files[name] = _reasons_record(list(summary["failure_reasons"])) + errors = { + str(item.get("artifact")): _text(item.get("error"), 200) + for item in collection.get("collection_errors") or [] if isinstance(item, dict) + } + + ordered = [members[key] for key in sorted(members, key=lambda key: -1 if key is None else key)] + tailed = 0 + for member in ordered: + member.setdefault("scheduler_steps", []) + member["scheduler_record"] = bool(member["state"] or member["scheduler_steps"]) + category, lines = _classify(member, files, missing, evidence_available, pw_exit_logged) + member["category"] = category + member["explanation"] = CATEGORY_GUIDANCE.get(category, "") + member["exit_code_hint"] = exit_code_hint(member.get("exit_code", "")) + member["evidence"] = lines[:MAX_EVIDENCE_LINES] + if member["array_task_id"] in usage: + member["resource_usage"] = usage[member["array_task_id"]] + qe_name = member["artifacts"]["qe_output"] + qe = files.get(qe_name) or {} + counts = qe.get("counts") or {} + member["qe"] = {key: counts[key] for key in + ("ionic_steps", "scf_cycles_converged", "eigenvalues_not_converged") if key in counts} + for key in ("context", "final"): + if isinstance(qe.get(key), dict): + member["qe"][key] = {name: _text(value, 240) for name, value in qe[key].items()} + if isinstance(qe.get("scanned_from_byte"), int): + member["qe"]["scanned_from_byte"] = qe["scanned_from_byte"] + stderr_name = member["artifacts"]["logs"][0] + pw_lines = _marker_lines(stderr_name, files.get(stderr_name) or {}, ("pw_exit",)) + status = re.search(r"exit status ([0-9]{1,3})", pw_lines[-1]["text"]) if pw_lines else None + if status: + member["pw_exit_status"] = int(status.group(1)) + summary = parsed.get(qe_name) if isinstance(parsed.get(qe_name), dict) else None + if summary: + member["scientific_status"] = _text(summary.get("scientific_status"), 32) + member["failure_reasons"] = [_text(item, 240) for item in (summary.get("failure_reasons") or [])[:3]] + if qe_name in errors: + member["collection_error"] = errors[qe_name] + if category not in {"completed", "cancelled"} and tailed < MAX_TAILED_MEMBERS and files: + tails = { + name: [_text(line, 240) for line in (files.get(name) or {}).get("tail") or []][-8:] + for name in [qe_name, member["artifacts"]["crash"], *member["artifacts"]["logs"]] + if (files.get(name) or {}).get("tail") + } + if tails: + tailed += 1 + member["tails"] = tails + + notes = [] + if derived: + notes.append("Derived from stored scheduler accounting for a run collected before failure " + "diagnostics existed.") + if not evidence_available: + notes.append("QE output and Slurm log evidence was not extracted for this run, so causes " + "rest on scheduler states and exit codes only. Download the retained QE " + "output and slurm-*.err artifacts to confirm.") + if not isinstance(steps, list): + notes.append("Slurm step accounting (per-task memory peaks, requested memory, step states) was " + "not collected for this run, so measured memory use is unknown. Out-of-memory can " + "only be confirmed by Slurm's state or a log message.") + if any(not member["scheduler_record"] for member in ordered): + notes.append("Some members have no scheduler accounting record; any category shown for " + "them rests on examined files only.") + if tailed >= MAX_TAILED_MEMBERS: + notes.append(f"Log tails are included for the first {MAX_TAILED_MEMBERS} unsuccessful members.") + return { + "schema": SCHEMA, + "generated_at": datetime.now(timezone.utc).isoformat(), + "tool_name": tool_name, + "scheduler_job_id": job_id, + "outcome": outcome, + "derived": derived, + "evidence_available": evidence_available, + **member_summary(ordered), + "members": ordered, + "collection": { + "missing_artifacts": [_text(n, 200) for n in (collection.get("missing_artifacts") or [])[:50]], + "collection_errors": [ + {"artifact": _text(e.get("artifact"), 200), "error": _text(e.get("error"), 200)} + for e in (collection.get("collection_errors") or [])[:50] if isinstance(e, dict) + ], + "scheduler_steps_error": _text(collection.get("scheduler_steps_error"), 200), + }, + "parse_error": _text(parse_error, 2000), + "resources": resources or {}, + "notes": notes, + } + + +def summary_message(diagnostics: dict[str, Any], *, next_step: str = "") -> str: + """One readable paragraph for error_json; details stay in the diagnostics.""" + + categories = diagnostics.get("category_counts") or {} + total = int(diagnostics.get("member_count") or 0) + completed = int(categories.get("completed", 0)) + elastic = diagnostics.get("tool_name") == "elasticdft" and total > 1 + unsuccessful = {name: count for name, count in categories.items() if name != "completed"} + labels = diagnostics.get("category_labels") or CATEGORY_LABELS + findings = "; ".join( + f"{labels.get(name, name)} ×{count}" + for name, count in sorted(unsuccessful.items(), key=lambda item: (-item[1], item[0])) + ) + parse_error = str(diagnostics.get("parse_error") or "") + if diagnostics.get("outcome") == "cancelled": + message = "Cancellation confirmed by Slurm." + if elastic and completed: + message += f" {completed} of {total} strain members had completed." + elif parse_error and not unsuccessful: + subject = f"all {total} strain members" if elastic else "the job" + message = f"Slurm completed {subject}, but the scientific result failed validation: {parse_error[:600]}" + else: + if elastic: + message = f"{total - completed} of {total} ElasticDFT strain members did not succeed" + else: + message = "The calculation did not succeed" + message += f": {findings}." if findings else "." + exits = sorted((diagnostics.get("exit_code_counts") or {}).items(), key=lambda item: -item[1]) + if exits: + message += " Slurm exit codes: " + ", ".join(f"{code} ×{count}" for code, count in exits[:4]) + "." + if completed: + message += f" {completed} completed." + if parse_error: + message += f" Result validation: {parse_error[:300]}" + message += " " + (next_step or "Open Failure diagnostics for per-member evidence.") + return message[:2000] diff --git a/api/computations/globus_gateway.py b/api/computations/globus_gateway.py new file mode 100644 index 0000000..6ca41a6 --- /dev/null +++ b/api/computations/globus_gateway.py @@ -0,0 +1,1322 @@ +"""Globus Compute transport for the Digital Twin RCAC service account. + +The registered function in this module is intentionally a narrow gateway. It +does not accept arbitrary commands: it can stage the two approved QE workflows, +submit them to Slurm, batch-query job states, cancel a job, and summarize/copy +their artifacts. The endpoint runs as the dedicated RCAC service account, so +Slurm ownership and filesystem access never depend on a student's credential. + +Protocol 2 adds bounded walltime/memory requests (``submit_v2``), a read-only +``capabilities`` action, bounded failure evidence from ``collect`` and the +read-only ``archive_evidence`` action for checksum-verified archives. Editing +this source does not change the registered function: re-register it and update +``gateway_function_id``/``gateway_protocol`` together (see IMPLEMENTATION_NOTES). +""" + +from __future__ import annotations + +import base64 +import json +import os +import time +from collections.abc import Iterable +from pathlib import Path, PurePosixPath +from typing import Any, Callable +from uuid import UUID + +from .slurm import SlurmJobSpec, SlurmState + + +class GlobusGatewayError(RuntimeError): + """A redacted Globus/endpoint failure safe to persist with a run.""" + + +class GlobusTransientError(GlobusGatewayError): + """A transport interruption; not evidence of a failed scientific job.""" + + +def _retryable_lookup_error(exc: Exception) -> bool: + """Only retry read-only transport failures, never authentication errors.""" + if getattr(exc, "http_status", None) in {429, 500, 502, 503, 504}: + return True + if isinstance(exc, (ConnectionError, TimeoutError)): + return True + try: + from globus_sdk.exc import NetworkError + except ImportError: + return False + return isinstance(exc, NetworkError) + + +def rcac_service_gateway(request: dict[str, Any]) -> dict[str, Any]: + """Execute one allow-listed operation on an RCAC single-user endpoint. + + This function uses only the Python standard library so it can be registered + once with Globus Compute and run on both cluster endpoints. Keep imports + inside the function: Globus serializes the registered function independently + of the Digital Twin source tree. + """ + + import base64 as _base64 + import fcntl + import hashlib + import json as _json + import mimetypes + import os as _os + import pwd + import re + import shlex + import shutil + import subprocess + import tarfile + from datetime import datetime as _datetime, timezone as _timezone + from pathlib import Path as _Path + from pathlib import PurePosixPath as _PurePosixPath + from uuid import UUID as _UUID + + def fail(message: str) -> None: + raise RuntimeError(message) + + # Resource contract, repeated from computations/resources.py because this + # function is serialized without the package (a test keeps them equal). + # Protocol 2 adds ``submit_v2`` (walltime/memory), ``capabilities`` and + # bounded failure evidence from ``collect``. + protocol = 2 + max_ntasks = 64 + max_concurrent_members = 4 + min_walltime_minutes = 10 + max_walltime_minutes = {"cellrelaxdft": 1440, "cellrelaxdft2d": 1440, "elasticdft": 480} + min_mem_per_cpu_mb = 512 + max_mem_per_cpu_mb = 4096 + + def safe_relative(value: str) -> str: + if not value or any(char in value for char in ("\x00", "\n", "\r")): + fail("unsafe relative path") + path = _PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or not path.parts: + fail("unsafe relative path") + return str(path) + + def safe_absolute(value: str) -> _Path: + path = _Path(value) + if not path.is_absolute() or ".." in path.parts: + fail("unsafe absolute path") + return path + + # Keep the registered function compatible with Python 3.9. Gautschi's + # supported default runtime is currently Python 3.9, where ``T | None`` + # annotations are evaluated as a bitwise operation unless postponed by a + # module-level future import. Globus serializes this function without the + # surrounding module, so avoid that annotation here. + def command(argv: list[str], *, cwd=None, okay=(0,)): + completed = subprocess.run( + argv, + cwd=str(cwd) if cwd else None, + check=False, + capture_output=True, + text=True, + timeout=120, + env={**_os.environ, "TZ": "UTC", "SLURM_TIME_FORMAT": "standard"}, + ) + if completed.returncode not in okay: + # Scheduler diagnostics can contain paths or credentials supplied + # by upstream services. Logs/artifacts, not exception strings, are + # the controlled diagnostic channel. + fail(f"{argv[0]} failed with exit code {completed.returncode}") + return completed + + def endpoint_setting(name: str) -> str: + value = _os.environ.get(name, "").strip() + if not value: + fail(f"endpoint setting is missing: {name}") + return value + + def scheduler_setting(name: str) -> str: + value = endpoint_setting(name) + if not re.fullmatch(r"[A-Za-z0-9_.:@,+%/=-]+", value): + fail(f"endpoint scheduler setting is invalid: {name}") + return value + + def memory_ceiling(): + # Explicit memory requests stay disabled unless the endpoint operator + # sets a per-CPU ceiling matching the allocation's node share. + value = _os.environ.get("DT_MAX_MEM_PER_CPU_MB", "").strip() + if not value: + return None + if not re.fullmatch(r"[0-9]{1,6}", value) or not ( + min_mem_per_cpu_mb <= int(value) <= max_mem_per_cpu_mb + ): + fail("endpoint memory ceiling is invalid") + return int(value) + + def check_cluster(payload: dict[str, Any]) -> None: + if payload.get("cluster") != endpoint_setting("DT_CLUSTER_NAME"): + fail("request was sent to the wrong cluster endpoint") + + def run_directory(root: _Path, run_id: str, label: str) -> _Path: + # ``root`` is already resolved, so an installation may keep its work + # or artifact root behind a symlink. The per-run directory itself is + # bound to this exact UUID: it may not be a symlink, and its resolved + # path must be / (never a sibling run's directory). + candidate = root / str(_UUID(run_id)) + if candidate.is_symlink(): + fail(f"invalid {label}: run directory is a symlink") + path = candidate.resolve() + if path.parent != root or path.name != candidate.name: + fail(f"invalid {label}") + return path + + def workdir_for(run_id: str) -> _Path: + root = safe_absolute(endpoint_setting("DT_WORK_ROOT")).resolve() + return run_directory(root, run_id, "run work directory") + + def owned_job(run_id: str, job_id: str) -> _Path: + if not re.fullmatch(r"[0-9]+", job_id): + fail("invalid Slurm job id") + workdir = workdir_for(run_id) + marker = workdir / ".dt-slurm-job-id" + if not marker.is_file() or marker.read_text(encoding="utf-8").strip() != job_id: + fail("Slurm job is not owned by this Digital Twin run") + return workdir + + def sha256(path: _Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def allowed_artifact(name: str, job_id: str) -> bool: + if name in {"vcrelax.in", "vcrelax.stdout", "slurm.out", "slurm.err", + "elastic-inputs.tar.gz", "dt-workflow.sbatch", "dt-request.json", + "execution-environment.json", "CRASH"}: + return True + if re.fullmatch(r"execution-environment-[0-9]{1,3}\.json", name): + return True + if re.fullmatch(rf"slurm-{re.escape(job_id)}_[0-9]{{1,3}}\.(out|err)", name): + return True + # QE writes CRASH beside its output when its error routine stops it. + match = re.fullmatch( + r"task_([0-9]{3})/elastic_inputs/strain_([0-9]{3})/(relax\.stdout|CRASH)", name + ) + return bool(match and match.group(1) == match.group(2)) + + def validate_pseudopotentials(entries: list[dict[str, str]]) -> None: + root = safe_absolute(endpoint_setting("DT_PSEUDO_ROOT")).resolve() + for entry in entries: + relative = safe_relative(str(entry.get("relative", ""))) + expected = str(entry.get("sha256", "")) + if not re.fullmatch(r"[0-9a-f]{64}", expected): + fail("invalid pseudopotential checksum") + path = (root / relative).resolve() + if root not in path.parents or not path.is_file(): + fail(f"approved pseudopotential is unavailable: {relative}") + if sha256(path) != expected: + fail(f"pseudopotential checksum mismatch: {relative}") + + def validate_archive(path: _Path, count: int) -> None: + allowed_prefix = _PurePosixPath("elastic_inputs") + with tarfile.open(path, "r:gz") as archive: + for member in archive.getmembers(): + member_path = _PurePosixPath(member.name) + if ( + member_path.is_absolute() + or ".." in member_path.parts + or not member_path.parts + or member_path.parts[0] != allowed_prefix.name + or member.issym() + or member.islnk() + ): + fail("elastic input archive contains an unsafe member") + required = { + f"elastic_inputs/strain_{index:03d}/relax.in" + for index in range(count) + } + names = {member.name.rstrip("/") for member in archive.getmembers()} + if not required.issubset(names) or "elastic_inputs/manifest.json" not in names: + fail("elastic input archive is incomplete") + + def render_script(spec: dict[str, Any], workdir: _Path, resource_request: bool = False): + """Return the gateway-rendered script and the resources it requests.""" + tool = str(spec.get("tool_name", "")) + if tool not in {"cellrelaxdft", "cellrelaxdft2d", "elasticdft"}: + fail("unsupported computation tool") + account = scheduler_setting("DT_SLURM_ACCOUNT") + partition = scheduler_setting("DT_SLURM_PARTITION") + qos = scheduler_setting("DT_SLURM_QOS") + try: + modules = _json.loads(endpoint_setting("DT_QE_MODULES_JSON")) + except (TypeError, ValueError) as exc: + fail(f"endpoint module stack is invalid: {exc}") + if ( + not isinstance(modules, list) + or not modules + or not all( + isinstance(module, str) + and re.fullmatch(r"[A-Za-z0-9_.+/-]+", module) + for module in modules + ) + ): + fail("endpoint module stack is invalid") + launcher = endpoint_setting("DT_MPI_LAUNCHER") + if launcher not in {"srun", "mpirun"}: + fail("endpoint MPI launcher is invalid") + if spec.get("account") != account or spec.get("partition") != partition: + fail("requested Slurm account or partition is not allowed") + if spec.get("qos") != qos or spec.get("modules") != modules: + fail("requested Slurm QoS or module stack is not allowed") + if spec.get("mpi_launcher") != launcher: + fail("requested MPI launcher is not allowed") + # Exact integer types only: never coerce 1.9, "64" or True into a + # directive. Every DT worker version sends JSON integers here. + ntasks = spec.get("ntasks") + if type(ntasks) is not int or not 1 <= ntasks <= max_ntasks: + fail(f"task count must be an integer between 1 and {max_ntasks}") + for key in ("nodes", "cpus_per_task"): + value = spec.get(key, 1) + if type(value) is not int or value != 1: + fail("only one node and one CPU per MPI rank are supported") + memory = spec.get("mem_per_cpu_mb") + if resource_request: + match = re.fullmatch(r"([0-9]{2}):([0-5][0-9]):00", str(spec.get("time_limit", ""))) + minutes = int(match.group(1)) * 60 + int(match.group(2)) if match else -1 + if not min_walltime_minutes <= minutes <= max_walltime_minutes[tool]: + fail("workflow walltime is not allowed") + time_limit = f"{minutes // 60:02d}:{minutes % 60:02d}:00" + if memory is not None: + ceiling = memory_ceiling() + if ceiling is None: + fail("explicit memory requests are not enabled on this endpoint") + if type(memory) is not int or not min_mem_per_cpu_mb <= memory <= ceiling: + fail("memory request is not allowed") + else: + # Original protocol-1 contract, kept for workers not yet upgraded. + time_limit = "08:00:00" if tool == "elasticdft" else "24:00:00" + if spec.get("time_limit") != time_limit: + fail("workflow walltime is not allowed") + if memory is not None: + fail("memory requests require the submit_v2 action") + array = str(spec.get("array", "")) + if tool in {"cellrelaxdft", "cellrelaxdft2d"} and array: + fail("CellRelaxDFT cannot use a job array") + if tool == "elasticdft" and not re.fullmatch( + rf"0-([0-9]{{1,3}})%([1-{max_concurrent_members}])", array + ): + fail("ElasticDFT array specification is not allowed") + output_file = "slurm-%A_%a.out" if array else "slurm.out" + error_file = "slurm-%A_%a.err" if array else "slurm.err" + run_id = str(_UUID(str(spec.get("run_id", "")))) + directives = [ + "#!/bin/bash -l", + f"#SBATCH --job-name=dt-{tool[:10]}-{run_id[:8]}", + f"#SBATCH --account={account}", + f"#SBATCH --partition={partition}", + f"#SBATCH --qos={qos}", + f"#SBATCH --time={time_limit}", + "#SBATCH --nodes=1", + f"#SBATCH --ntasks={ntasks}", + "#SBATCH --cpus-per-task=1", + f"#SBATCH --output={output_file}", + f"#SBATCH --error={error_file}", + ] + if memory is not None: + directives.append(f"#SBATCH --mem-per-cpu={memory}M") + if array: + directives.append(f"#SBATCH --array={array}") + body = ["set -euo pipefail", f"cd {shlex.quote(str(workdir))}", "module purge"] + body.extend(f"module load {shlex.quote(module)}" for module in modules) + body.append("export QE_DISABLE_GGA_PBE=0") + body.append("python3 dt-capture-environment.py") + if tool in {"cellrelaxdft", "cellrelaxdft2d"}: + body.extend([ + "mkdir -p qe_tmp", + "status=0", + f"{launcher} pw.x -in vcrelax.in > vcrelax.stdout || status=$?", + "echo \"dt: pw.x exit status ${status}\" >&2", + "exit \"${status}\"", + ]) + else: + body.append("bash run-elastic-task.sh") + resources = {"nodes": 1, "ntasks": ntasks, "cpus_per_task": 1, + "mem_per_cpu_mb": memory, "time_limit": time_limit, "array": array} + return "\n".join([*directives, "", *body]) + "\n", resources + + def parse_qe_summary(path: _Path, *, include_structure: bool) -> dict[str, Any]: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + if not lines: + fail(f"Quantum ESPRESSO output is empty: {path.name}") + energies: list[float] = [] + pressures: list[float] = [] + stresses: list[list[list[float]]] = [] + total_forces: list[float] = [] + atomic_forces: list[list[list[float]]] = [] + cells: list[dict[str, Any]] = [] + positions: list[dict[str, Any]] = [] + alat_angstrom = None + for index, line in enumerate(lines): + alat_match = re.search( + r"lattice parameter \(alat\)\s*=\s*([-+0-9.Ee]+)", line + ) + if alat_match: + alat_angstrom = float(alat_match.group(1)) * 0.529177210903 + if "!" in line and "total energy" in line: + match = re.search(r"=\s*([-+0-9.Ee]+)\s+Ry", line) + if match: + energies.append(float(match.group(1))) + if "P=" in line: + match = re.search(r"P=\s*([-+0-9.Ee]+)", line) + if match and index + 3 < len(lines): + matrix = [] + for row in lines[index + 1 : index + 4]: + values = row.split() + matrix.append([float(value) for value in values[3:6]]) + pressures.append(float(match.group(1))) + stresses.append(matrix) + if "Total force" in line: + match = re.search(r"Total force\s*=\s*([-+0-9.Ee]+)", line) + if match: + total_forces.append(float(match.group(1))) + if "Forces acting on atoms" in line: + forces: list[list[float]] = [] + cursor = index + 1 + while cursor < len(lines): + match = re.search( + r"force\s*=\s*([-+0-9.Ee]+)\s+([-+0-9.Ee]+)\s+([-+0-9.Ee]+)", + lines[cursor], + ) + if match: + forces.append([float(match.group(i)) for i in range(1, 4)]) + elif forces: + break + cursor += 1 + if forces: + atomic_forces.append(forces) + if include_structure and line.strip().startswith("CELL_PARAMETERS"): + unit = line.partition("(")[2].partition(")")[0].strip().lower() or "alat" + matrix = [ + [float(value) for value in row.split()[:3]] + for row in lines[index + 1 : index + 4] + ] + cells.append({"unit": unit, "matrix": matrix}) + if include_structure and line.strip().startswith("ATOMIC_POSITIONS"): + unit = line.partition("(")[2].partition(")")[0].strip().lower() or "alat" + species: list[str] = [] + coords: list[list[float]] = [] + cursor = index + 1 + while cursor < len(lines): + values = lines[cursor].split() + if len(values) < 4 or not re.fullmatch(r"[A-Za-z]{1,3}", values[0]): + break + try: + coordinate = [float(value) for value in values[1:4]] + except ValueError: + break + species.append(values[0]) + coords.append(coordinate) + cursor += 1 + if species: + positions.append({"unit": unit, "species": species, "coords": coords}) + if not energies or not pressures or not stresses: + fail(f"Quantum ESPRESSO output is incomplete: {path.name}") + failure_markers = ( + "convergence not achieved", + "bfgs failed", + "error in routine", + "maximum number of steps reached", + # QE's actual wording; it still prints "End of BFGS" afterwards. + "maximum number of steps has been reached", + ) + failures = [] + warnings = [] + for line in lines: + normalized = " ".join(line.split()) + lower = line.lower() + if normalized and any(marker in lower for marker in failure_markers): + if normalized not in failures: + failures.append(normalized) + if "warning:" in lower or ("c_bands:" in lower and "not converged" in lower): + if normalized and normalized not in warnings: + warnings.append(normalized) + completed = any("JOB DONE" in line for line in lines) + optimized = any("End of BFGS Geometry Optimization" in line for line in lines) + converged = completed and optimized and not failures + return { + "ionic_energies_ry": energies, + "pressures_kbar": pressures, + "stresses_kbar": stresses, + "total_forces_ry_bohr": total_forces, + "atomic_forces_ry_bohr": atomic_forces, + "cell_blocks": cells, + "position_blocks": positions, + "alat_angstrom": alat_angstrom, + "process_completed": completed, + "convergence": converged, + "scientific_status": "converged" if converged else "not_converged" if completed else "incomplete", + "failure_reasons": failures, + "warnings": warnings, + "warning_count": len(warnings), + } + + # Bounded evidence for failure diagnosis. Only marker lines, counters and a + # short tail leave the endpoint; whole files remain checksummed artifacts. + qe_markers = ( + ("scf_not_converged", ("convergence not achieved after",)), + ("ionic_step_limit", ("maximum number of steps has been reached", + "maximum number of steps reached")), + ("bfgs_failed", ("bfgs failed",)), + ("bfgs_history_reset", ("history already reset",)), + # stdout says "Error in routine X (N):"; CRASH says "from X : error # N". + ("qe_error", ("error in routine", ": error #")), + ("memory", ("cannot allocate", "out of memory", "allocation failed")), + ("cpu_time", ("maximum cpu time exceeded",)), + ("bfgs_converged", ("bfgs converged",)), + ("end_of_optimization", ("end of bfgs geometry optimization",)), + ("job_done", ("job done",)), + ) + # First occurrence: run setup and the thresholds QE actually read. + qe_context = ( + ("program", "program pwscf"), + ("parallelization", "running on"), + ("pools", "npool"), + ("k_points", "number of k points="), + ("estimated_ram_per_process", "estimated max dynamical ram per process"), + ("estimated_total_ram", "estimated total dynamical ram"), + ("scf_threshold", "scf convergence threshold"), + ("energy_threshold", "energy convergence thresh"), + ("force_threshold", "force convergence thresh"), + ) + # Last occurrence: the final optimizer state and the criteria it reports. + qe_final = ( + ("energy_error", "energy error"), + ("gradient_error", "gradient error"), + ("bfgs_criteria", "(criteria:"), + ("bfgs_result", "bfgs converged"), + ("bfgs_result", "bfgs failed"), + ) + log_markers = ( + ("pw_exit", ("dt: pw.x exit status",)), + ("out_of_memory", ("out of memory", "out_of_memory", "oom-kill", "oom_kill", + "oom killer", "oom-killer", "exceeded memory limit", + "exceeded job memory limit")), + ("time_limit", ("due to time limit", "time limit exceeded")), + ("cancelled", ("cancelled at",)), + ("qe_error", ("error in routine",)), + ("environment", ("command not found", "no such file or directory", "lmod has detected", + "unavailable after module initialization", "permission denied", + "traceback (most recent call last)")), + ("launcher", ("srun: error", "slurmstepd: error", "mpi_abort", "exited with exit code", + "mpirun noticed", "prterun", "primary job")), + ("signal", ("killed", "segmentation fault", "bus error", "core dumped")), + ) + + def bounded_line(text: str) -> str: + cleaned = "".join( + char if char == "\t" or (ord(char) >= 32 and not 127 <= ord(char) < 160) else "?" + for char in text.strip() + ) + return cleaned[:240] + + evidence_window = 16 * 1024 * 1024 + + def file_evidence(path: _Path, kind: str, budget: dict[str, int]) -> dict[str, Any]: + # Live collection: scan the last window of a file in the run directory. + size = path.stat().st_size + if size == 0: + return {"kind": kind, "size_bytes": 0} + with path.open("rb") as handle: + if size > evidence_window: + handle.seek(size - evidence_window) + data = handle.read(evidence_window) + return evidence_from_bytes(data, size, kind, budget) + + def verified_tail(path: _Path, size: int, expected_hash: str): + # One read pass hashes the whole file and keeps its last scan window, + # so the evidence comes from exactly the bytes that matched the + # recorded checksum. Returns None on any size or checksum mismatch. + digest = hashlib.sha256() + tail = bytearray() + total = 0 + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + total += len(chunk) + if total > size: + return None + digest.update(chunk) + tail += chunk + if len(tail) > evidence_window: + del tail[: len(tail) - evidence_window] + if total != size or digest.hexdigest() != expected_hash: + return None + return bytes(tail) + + def evidence_from_bytes(data: bytes, size: int, kind: str, budget: dict[str, int]) -> dict[str, Any]: + """Summarize the scanned tail ``data`` of a file of ``size`` bytes.""" + result: dict[str, Any] = {"kind": kind, "size_bytes": size} + if size == 0: + return result + if size > len(data): + # Line numbers below count from this byte, not the file start. + result["scanned_from_byte"] = size - len(data) + lines = data.decode("utf-8", errors="replace").splitlines() + markers = qe_markers if kind in {"qe_output", "qe_crash"} else log_markers + counts: dict[str, int] = {} + recent: dict[str, list[dict[str, Any]]] = {} + context: dict[str, str] = {} + final: dict[str, str] = {} + for index, line in enumerate(lines): + lower = line.lower() + if kind == "qe_output": + if line.lstrip().startswith("!") and "total energy" in lower: + counts["ionic_steps"] = counts.get("ionic_steps", 0) + 1 + if "convergence has been achieved in" in lower: + counts["scf_cycles_converged"] = counts.get("scf_cycles_converged", 0) + 1 + if "c_bands:" in lower and "not converged" in lower: + counts["eigenvalues_not_converged"] = counts.get("eigenvalues_not_converged", 0) + 1 + for key, needle in qe_context: + if key not in context and needle in lower: + context[key] = bounded_line(line) + if "nstep" not in context and lower.strip().startswith("nstep"): + context["nstep"] = bounded_line(line) + for key, needle in qe_final: + if needle in lower: + final[key] = bounded_line(line) + for tag, needles in markers: + if any(needle in lower for needle in needles): + counts[tag] = counts.get(tag, 0) + 1 + text = line + if tag == "qe_error" and index + 1 < len(lines): + # QE prints the error message on the following line. + text = " ".join(line.split()) + " | " + " ".join(lines[index + 1].split()) + entries = recent.setdefault(tag, []) + entries.append({"tag": tag, "line_number": index + 1, "text": bounded_line(text)}) + del entries[:-2] + break + result["counts"] = counts + if context: + result["context"] = context + if final: + result["final"] = final + lines_out = sorted((entry for entries in recent.values() for entry in entries), + key=lambda entry: entry["line_number"]) + tail = [bounded_line(line) for line in lines[-10:] if line.strip()] + cost = sum(len(entry["text"]) for entry in lines_out) + sum(len(line) for line in tail) + if budget["remaining"] < cost: + result["truncated"] = True + return result + budget["remaining"] -= cost + result["markers"] = lines_out + result["tail"] = tail + return result + + def scheduler_steps(run_id: str, job_id: str) -> dict[str, Any]: + # Same attribution rule as status_many: a job id alone is not trusted + # (ids can be reused, a marker could be stale). Allocation records must + # carry this run's comment and service-account owner; step records are + # kept only beneath such an attributed allocation. + keys = ("job_id", "state", "exit_code", "derived_exit_code", "elapsed", "time_limit", + "max_rss", "req_mem", "alloc_cpus", "nnodes") + accounting = command( + ["sacct", "-n", "-P", "--array", "-j", job_id, + "--format=JobID%64,State%32,ExitCode,DerivedExitCode,Elapsed,Timelimit," + "MaxRSS,ReqMem,AllocCPUS,NNodes,Comment%128,User%32"], + ) + expected_comment = f"dt-run:{run_id}" + owner = pwd.getpwuid(_os.getuid()).pw_name + rows, attributed, unattributed = [], set(), set() + for line in accounting.stdout.splitlines()[:2000]: + fields = [field.strip() for field in line.split("|")] + if len(fields) < len(keys) + 2 or not re.fullmatch( + re.escape(job_id) + r"(_[0-9]+)?(\.[A-Za-z0-9_+-]+)?", fields[0] + ): + continue + allocation = fields[0].split(".", 1)[0] + if "." not in fields[0]: + comment, user = fields[len(keys)], fields[len(keys) + 1] + if (comment and comment != expected_comment) or (user and user != owner): + fail("scheduler step attribution does not match the requested run") + (attributed if comment else unattributed).add(allocation) + rows.append((allocation, dict(zip(keys, (field[:64] for field in fields[: len(keys)]))))) + return { + "records": [record for allocation, record in rows if allocation in attributed], + "unattributed_allocations": len(unattributed - attributed), + } + + def persist_artifacts(run_id: str, workdir: _Path, names: list[str]): + root = safe_absolute(endpoint_setting("DT_ARTIFACT_ROOT")).resolve() + # Writes too: never archive one run's files into another run's directory. + destination_root = run_directory(root, run_id, "artifact directory") + manifest = [] + for name in names: + relative = safe_relative(name) + source = (workdir / relative).resolve() + if workdir.resolve() not in source.parents or not source.is_file(): + fail(f"expected artifact is missing: {relative}") + destination = (destination_root / relative).resolve() + if destination_root not in destination.parents: + fail("invalid artifact destination") + destination.parent.mkdir(parents=True, exist_ok=True) + # Atomic replacement prevents interrupted collection from leaving + # an apparently archived, truncated file. + import tempfile + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as handle: + temporary = _Path(handle.name) + try: + shutil.copy2(source, temporary) + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + manifest.append( + { + "name": relative, + "storage_uri": f"depot://{destination}", + "sha256": sha256(destination), + "size_bytes": destination.stat().st_size, + "content_type": mimetypes.guess_type(destination.name)[0] + or "application/octet-stream", + "downloadable": destination.stat().st_size <= 8 * 1024 * 1024, + } + ) + return manifest + + if not isinstance(request, dict): + fail("gateway request must be an object") + action = str(request.get("action", "")) + payload = request.get("payload") + if not isinstance(payload, dict): + fail("gateway payload must be an object") + check_cluster(payload) + + if action == "recover_submission": + run_id = str(_UUID(str(payload.get("run_id", "")))) + workdir = workdir_for(run_id) + marker = workdir / ".dt-slurm-job-id" + if not marker.is_file(): + # No submission or cancellation is attempted here. An older Globus + # task may still be queued or running: absence is not proof of no job. + return {"submission_outcome": "unknown"} + job_id = marker.read_text(encoding="utf-8").strip() + owned_job(run_id, job_id) + return {"submission_outcome": "submitted", "job_id": job_id, + "workdir": str(workdir), "comment": f"dt-run:{run_id}"} + + if action == "capabilities": + # Read-only rollout check; no scheduler or filesystem access. + return { + "protocol": protocol, "nodes": 1, "cpus_per_task": 1, "max_ntasks": max_ntasks, + "max_concurrent_members": max_concurrent_members, + "min_walltime_minutes": min_walltime_minutes, + "max_walltime_minutes": dict(max_walltime_minutes), + "min_mem_per_cpu_mb": min_mem_per_cpu_mb, "max_mem_per_cpu_mb": memory_ceiling(), + "diagnostics": True, "archive_evidence": True, + } + + if action in {"submit", "submit_v2"}: + # ``submit`` keeps the protocol-1 fixed walltime/default-memory + # contract; ``submit_v2`` accepts bounded walltime/memory requests. An + # older registered gateway rejects ``submit_v2`` before any side effect. + resource_request = action == "submit_v2" + run_id = str(_UUID(str(payload.get("run_id", "")))) + spec = payload.get("spec") + files = payload.get("files") + pseudos = payload.get("pseudopotentials", []) + if not isinstance(spec, dict) or not isinstance(files, dict) or not isinstance(pseudos, list): + fail("submission payload is malformed") + if spec.get("run_id") != run_id: + fail("run identifier mismatch") + workdir = workdir_for(run_id) + if spec.get("workdir") != str(workdir): + fail("work directory does not match endpoint configuration") + validate_pseudopotentials(pseudos) + tool = str(spec.get("tool_name", "")) + allowed_files = {"elastic-inputs.tar.gz"} if tool == "elasticdft" else {"vcrelax.in"} + if set(files) != allowed_files: + fail("submission contains unexpected staged files") + workdir.mkdir(parents=True, exist_ok=True) + marker = workdir / ".dt-slurm-job-id" + lock_path = workdir / ".dt-submit.lock" + with lock_path.open("a+", encoding="utf-8") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + if marker.is_file(): + job_id = marker.read_text(encoding="utf-8").strip() + if not re.fullmatch(r"[0-9]+", job_id): + fail("existing Slurm marker is invalid") + try: + recorded = _json.loads((workdir / "dt-request.json").read_text(encoding="utf-8")) + resources = recorded.get("resources") if isinstance(recorded, dict) else None + except (OSError, ValueError): + resources = None + else: + # A process can die after sbatch accepts a job but before the + # ID is persisted. Never blindly re-submit that uncertainty. + intent = workdir / ".dt-submit-intent" + if intent.exists(): + fail("submission outcome is uncertain; operator reconciliation required") + # Validate the complete scheduler request before staging. + script_text, resources = render_script(spec, workdir, resource_request) + for name, encoded in files.items(): + relative = safe_relative(name) + data = _base64.b64decode(str(encoded), validate=True) + if len(data) > 8 * 1024 * 1024: + fail("staged file exceeds the gateway limit") + (workdir / relative).write_bytes(data) + if tool == "elasticdft": + match = re.fullmatch( + r"0-([0-9]{1,3})%([1-4])", str(spec.get("array", "")) + ) + if not match: + fail("invalid elastic array") + count = int(match.group(1)) + 1 + validate_archive(workdir / "elastic-inputs.tar.gz", count) + launcher = endpoint_setting("DT_MPI_LAUNCHER") + runner = workdir / "run-elastic-task.sh" + # pw.x's own status is written to stderr so it cannot be + # confused with a failing tar/python step; the job's + # Slurm exit code is unchanged. + runner.write_text( + "#!/bin/bash\nset -euo pipefail\n" + "task=$(printf '%03d' \"${SLURM_ARRAY_TASK_ID}\")\n" + "task_root=\"task_${task}\"\n" + "mkdir -p \"${task_root}\"\n" + "tar -xzf elastic-inputs.tar.gz -C \"${task_root}\"\n" + "cd \"${task_root}/elastic_inputs/strain_${task}\"\n" + "mkdir -p qe_tmp\n" + "status=0\n" + f"{launcher} pw.x -in relax.in > relax.stdout || status=$?\n" + "echo \"dt: pw.x exit status ${status}\" >&2\n" + "exit \"${status}\"\n", + encoding="utf-8", + ) + runner.chmod(0o750) + script = workdir / "dt-workflow.sbatch" + script.write_text(script_text, encoding="utf-8") + request_record = { + "schema": "dt-execution-request/1", "run_id": run_id, + "cluster": payload["cluster"], "tool_name": tool, + "tool_revision": str(payload.get("tool_revision", "")), + "modules": spec["modules"], "mpi_launcher": spec["mpi_launcher"], + "pseudopotentials": pseudos, "ntasks": spec["ntasks"], + "resources": resources, "gateway_protocol": protocol, + "script_sha256": sha256(script), + } + (workdir / "dt-request.json").write_text(_json.dumps(request_record), encoding="utf-8") + # Standard-library-only compute-node observation. Deliberately + # do not capture the process environment or authorization data. + capture_source = '''import hashlib, json, os, pathlib, platform, shutil, subprocess, sys +from datetime import datetime, timezone +request = json.loads(pathlib.Path("dt-request.json").read_text()) +binary = shutil.which("pw.x") +if not binary: + raise RuntimeError("pw.x is unavailable after module initialization") +path = pathlib.Path(binary).resolve() +digest = hashlib.sha256() +with path.open("rb") as stream: + for block in iter(lambda: stream.read(1048576), b""): + digest.update(block) +record = {"schema": "dt-execution-environment/1", "observed_at": datetime.now(timezone.utc).isoformat(), + "run_id": request["run_id"], "tool_revision": request["tool_revision"], + "cluster": request["cluster"], "hostname": platform.node(), "os": platform.platform(), + "python_version": sys.version.split()[0], "loaded_modules": os.environ.get("LOADEDMODULES", "").split(":"), + "qe_executable": str(path), "qe_executable_sha256": digest.hexdigest(), + "mpi_launcher": request["mpi_launcher"], "scheduler_job_id": os.environ.get("SLURM_JOB_ID", ""), + "array_task_id": os.environ.get("SLURM_ARRAY_TASK_ID", ""), + "pseudopotentials": request["pseudopotentials"], "script_sha256": request["script_sha256"]} +try: + result = subprocess.run(["ldd", str(path)], capture_output=True, text=True, timeout=10) + record["linked_libraries"] = result.stdout[:32768] + record["linked_libraries_exit_code"] = result.returncode +except (OSError, subprocess.TimeoutExpired): + record["linked_libraries_status"] = "unavailable" +suffix = "-" + record["array_task_id"] if record["array_task_id"] else "" +pathlib.Path("execution-environment" + suffix + ".json").write_text(json.dumps(record, indent=2)) +''' + (workdir / "dt-capture-environment.py").write_text(capture_source, encoding="utf-8") + comment = f"dt-run:{run_id}" + intent.write_text(comment, encoding="utf-8") + completed = command( + ["sbatch", "--parsable", f"--comment={comment}", str(script)], + cwd=workdir, + ) + job_id = completed.stdout.strip().split(";", 1)[0] + if not re.fullmatch(r"[0-9]+", job_id): + fail("sbatch returned an invalid job id") + temporary_marker = workdir / ".dt-slurm-job-id.pending" + temporary_marker.write_text(job_id + "\n", encoding="utf-8") + temporary_marker.replace(marker) + return { + "job_id": job_id, + "workdir": str(workdir), + "comment": f"dt-run:{run_id}", + "protocol": protocol, + "resources": resources, + } + + if action == "status_many": + values = payload.get("jobs") + if not isinstance(values, list) or not 1 <= len(values) <= 500: + fail("status request must contain 1..500 jobs") + if not all(isinstance(value, dict) for value in values): + fail("status request is malformed") + job_ids = [] + for value in values: + run_id = str(_UUID(str(value.get("run_id", "")))) + job_id = str(value.get("job_id", "")) + owned_job(run_id, job_id) + job_ids.append(job_id) + if len(set(job_ids)) != len(job_ids) or any( + not re.fullmatch(r"[0-9]+", value) for value in job_ids + ): + fail("status request contains an invalid job id") + raw_states: dict[str, list[str]] = {job_id: [] for job_id in job_ids} + records = {job_id: [] for job_id in job_ids} + accounting = command( + ["sacct", "-n", "-P", "--array", "-X", "-j", ",".join(job_ids), + "--format=JobID%64,State%32,Start,End,ExitCode,Submit,Comment%128"], + ) + for line in accounting.stdout.splitlines(): + fields = line.split("|") + if len(fields) < 2: + continue + raw_id, state = fields[0].strip(), fields[1].strip() + for job_id in job_ids: + if raw_id == job_id or re.fullmatch(re.escape(job_id) + r"_[0-9]+", raw_id): + expected_comment = "dt-run:" + next( + str(v["run_id"]) for v in values if str(v["job_id"]) == job_id + ) + comment = fields[6].strip() if len(fields) >= 7 else "" + if comment and comment != expected_comment: + fail("scheduler attribution does not match the requested run") + if not comment: + # RCAC sacct leaves Comment empty for pending/running + # jobs and array members. Do not trust that record as a + # terminal result. The attributed live queue below can + # still establish an active state while accounting lags. + raw_states[job_id].append("UNKNOWN") + continue + raw_states[job_id].append(state) + record = {"job_id": raw_id, "state": state} + for index, key in ((2, "started_at"), (3, "ended_at"), (5, "submitted_at")): + value = fields[index].strip() if len(fields) > index else "" + try: + record[key] = _datetime.fromisoformat(value).replace(tzinfo=_timezone.utc).isoformat() + except ValueError: + record[key] = None + record["exit_code"] = fields[4].strip() if len(fields) > 4 else "" + records[job_id].append(record) + # Query the live queue once as well: an array's early completed member + # must not hide siblings that have not reached accounting yet. + if job_ids: + queued = command( + ["squeue", "-h", "-j", ",".join(job_ids), "-o", "%F|%T|%k|%u"], + okay=(0, 1), + ) + if queued.returncode and not any(raw_states.values()): + fail("scheduler state is unavailable") + for line in queued.stdout.splitlines(): + fields = [field.strip() for field in line.split("|")] + if fields[0] not in raw_states: + continue + job_id = fields[0] + expected_comment = "dt-run:" + next( + str(v["run_id"]) for v in values if str(v["job_id"]) == job_id + ) + if (len(fields) != 4 or fields[2] != expected_comment + or fields[3] != pwd.getpwuid(_os.getuid()).pw_name): + fail("live scheduler attribution does not match the requested run") + raw_states[job_id].append(fields[1]) + return {"states": raw_states, "accounting": records} + + if action == "cancel": + run_id = str(_UUID(str(payload.get("run_id", "")))) + job_id = str(payload.get("job_id", "")) + owned_job(run_id, job_id) + command(["scancel", job_id]) + return {"job_id": job_id, "cancelled": True} + + if action == "read_artifact": + # Only manifest-selected files in the run's permanent archive; never + # accept an absolute path or depend on scratch surviving its purge. + run_id = str(_UUID(str(payload.get("run_id", "")))) + job_id = str(payload.get("job_id", "")) + name = safe_relative(str(payload.get("name", ""))) + if not re.fullmatch(r"[0-9]+", job_id) or not allowed_artifact(name, job_id): + fail("artifact is not an approved run output") + root = safe_absolute(endpoint_setting("DT_ARTIFACT_ROOT")).resolve() + archive = run_directory(root, run_id, "artifact directory") + path = (archive / name).resolve() + if archive not in path.parents or not path.is_file(): + fail("artifact is unavailable") + size = int(payload.get("size_bytes", -1)) + offset = int(payload.get("offset", -1)) + length = int(payload.get("length", 0)) + expected_hash = str(payload.get("sha256", "")) + if not 0 <= size <= 8 * 1024 * 1024 or not 0 <= offset <= size or not 1 <= length <= 256 * 1024: + fail("artifact exceeds transfer bounds") + if not re.fullmatch(r"[a-f0-9]{64}", expected_hash): + fail("artifact checksum is invalid") + if path.stat().st_size != size or sha256(path) != expected_hash: + fail("archived artifact does not match its recorded checksum") + with path.open("rb") as handle: + handle.seek(offset) + data = handle.read(min(length, size - offset)) + return {"offset": offset, "size_bytes": size, "sha256": expected_hash, + "data": _base64.b64encode(data).decode("ascii")} + + if action == "archive_evidence": + # Read-only diagnosis of an already collected run from its permanent + # archive (never mutable scratch): approved evidence files only, each + # verified against the size and SHA-256 recorded in the DT manifest, + # then summarized by the same rules as ``collect`` from the very bytes + # that were verified. Request, ownership and path errors reject the + # whole request; a single unavailable or unreadable file stays unknown. + run_id = str(_UUID(str(payload.get("run_id", "")))) + job_id = str(payload.get("job_id", "")) + tool = str(payload.get("tool_name", "")) + expected = payload.get("expected_artifacts") + entries = payload.get("artifacts") + if not re.fullmatch(r"[0-9]+", job_id) or tool not in {"cellrelaxdft", "cellrelaxdft2d", "elasticdft"}: + fail("archive evidence request is malformed") + if (not isinstance(expected, list) or not isinstance(entries, list) + or not 1 <= len(entries) <= 400 or len(expected) > 1000): + fail("archive evidence request is malformed") + expected_names = {safe_relative(str(name)) for name in expected} + if not all(allowed_artifact(name, job_id) for name in expected_names): + fail("archive evidence request contains an unapproved path") + if tool in {"cellrelaxdft", "cellrelaxdft2d"} and expected_names - {"vcrelax.stdout"}: + fail("relaxation outputs do not match the registered workflow") + if tool == "elasticdft" and not all(name.endswith("/relax.stdout") for name in expected_names): + fail("elastic outputs do not match the registered workflow") + root = safe_absolute(endpoint_setting("DT_ARTIFACT_ROOT")).resolve() + archive = run_directory(root, run_id, "artifact directory") + log_name = re.compile(rf"slurm(-{re.escape(job_id)}_[0-9]{{1,3}})?\.(out|err)") + budget = {"remaining": 600000} + total, seen, evidence = 0, set(), {} + for entry in entries: + if not isinstance(entry, dict): + fail("archive evidence request is malformed") + name = safe_relative(str(entry.get("name", ""))) + if name in seen or not allowed_artifact(name, job_id): + fail("archive evidence request contains an unapproved path") + seen.add(name) + if name in expected_names: + kind = "qe_output" + elif _PurePosixPath(name).name == "CRASH": + kind = "qe_crash" + elif log_name.fullmatch(name): + kind = "scheduler_log" + else: + fail("archive evidence request names a file that is not evidence") + size = entry.get("size_bytes") + expected_hash = str(entry.get("sha256", "")) + if type(size) is not int or not 0 <= size <= 64 * 1024 * 1024 or not re.fullmatch(r"[a-f0-9]{64}", expected_hash): + fail("archive evidence entry exceeds bounds or lacks a checksum") + total += size + if total > 512 * 1024 * 1024: + fail("archive evidence request exceeds the total size bound") + path = (archive / name).resolve() + # Escapes stay unknown without being opened. + if archive not in path.parents: + evidence[name] = {"kind": kind, "error": "archived file is unavailable"} + continue + try: + # File I/O failures (permissions, vanished files) affect only + # this file: it stays unknown and the others are still read. + if not path.is_file(): + evidence[name] = {"kind": kind, "error": "archived file is unavailable"} + continue + if path.stat().st_size != size: + evidence[name] = {"kind": kind, "error": "archived file does not match its recorded checksum"} + continue + verified = verified_tail(path, size, expected_hash) + except OSError: + evidence[name] = {"kind": kind, "error": "archived file could not be read"} + continue + if verified is None: + evidence[name] = {"kind": kind, "error": "archived file does not match its recorded checksum"} + continue + evidence[name] = evidence_from_bytes(verified, size, kind, budget) + return {"protocol": protocol, "diagnostics": {"files": evidence, "source": "archive"}} + + if action == "collect": + run_id = str(_UUID(str(payload.get("run_id", "")))) + job_id = str(payload.get("job_id", "")) + tool = str(payload.get("tool_name", "")) + expected = payload.get("expected_artifacts") + retained = payload.get("retained_artifacts") + if tool not in {"cellrelaxdft", "cellrelaxdft2d", "elasticdft"}: + fail("unsupported computation tool") + if not isinstance(expected, list) or not isinstance(retained, list): + fail("artifact request is malformed") + workdir = owned_job(run_id, job_id) + expected_names = [safe_relative(str(name)) for name in expected] + retained_names = [safe_relative(str(name)) for name in retained] + if len(expected_names) > 1000 or len(retained_names) > 4000 or not all( + allowed_artifact(name, job_id) for name in expected_names + retained_names + ): + fail("artifact request contains an unapproved path") + if tool in {"cellrelaxdft", "cellrelaxdft2d"} and expected_names != ["vcrelax.stdout"]: + fail("relaxation outputs do not match the registered workflow") + if tool == "elasticdft" and not all(name.endswith("/relax.stdout") for name in expected_names): + fail("elastic outputs do not match the registered workflow") + metadata_names = ["dt-workflow.sbatch", "dt-request.json"] + if tool == "elasticdft": + # One observation per array member, not a nonexistent parent record. + indices = {int(name.split("/")[0].removeprefix("task_")) for name in expected_names} + metadata_names.extend(f"execution-environment-{index}.json" for index in sorted(indices)) + else: + metadata_names.append("execution-environment.json") + names = list(dict.fromkeys(retained_names + expected_names + metadata_names)) + available, missing = [], [] + for name in names: + path = (workdir / name).resolve() + if workdir not in path.parents: + fail("artifact path escapes the run directory") + if path.is_file(): + available.append(name) + elif _PurePosixPath(name).name != "CRASH": + # QE writes CRASH only when it fails; absence is normal. + missing.append(name) + # Preserve evidence before parsing. Empty/truncated QE outputs must not + # discard the only useful explanation of a failed calculation. + manifest = persist_artifacts(run_id, workdir, available) + parsed: dict[str, Any] = {} + errors = [] + for relative in expected_names: + source = (workdir / relative).resolve() + if payload.get("diagnostics_only") or relative not in available: + continue + try: + parsed[relative] = parse_qe_summary(source, include_structure=True) + version = re.search(r"Program\s+PWSCF\s+v\.?\s*([^\s]+)", source.read_text(errors="replace")) + parsed[relative]["qe_version"] = version.group(1) if version else None + except (RuntimeError, ValueError, IndexError): + errors.append({"artifact": relative, "error": "QE output is incomplete or unparsable"}) + environments = [] + for name in available: + if not name.startswith("execution-environment"): + continue + source = workdir / name + try: + if source.stat().st_size > 128 * 1024: + raise ValueError("oversized environment record") + record = _json.loads(source.read_text(encoding="utf-8")) + if record.get("run_id") != run_id: + raise ValueError("wrong run") + environments.append(record) + except (ValueError, TypeError, AttributeError): + errors.append({"artifact": name, "error": "Invalid execution environment record"}) + result = { + "workdir": str(workdir), + "parsed_outputs": parsed, + "artifact_manifest": manifest, + "missing_artifacts": missing, + "collection_errors": errors, + "execution_environments": environments, + "protocol": protocol, + } + if payload.get("include_diagnostics"): + budget = {"remaining": 600000} + evidence = {} + log_name = re.compile(rf"slurm(-{re.escape(job_id)}_[0-9]{{1,3}})?\.(out|err)") + for name in available: + if name in expected_names: + kind = "qe_output" + elif _PurePosixPath(name).name == "CRASH": + kind = "qe_crash" + elif log_name.fullmatch(name): + kind = "scheduler_log" + else: + continue + try: + evidence[name] = file_evidence(workdir / name, kind, budget) + except OSError: + evidence[name] = {"kind": kind, "error": "unreadable"} + result["diagnostics"] = {"files": evidence} + if payload.get("include_scheduler_steps"): + # Never let optional accounting block archived evidence. + try: + steps = scheduler_steps(run_id, job_id) + result["scheduler_steps"] = steps["records"] + if steps["unattributed_allocations"]: + result["scheduler_steps_error"] = ( + f"{steps['unattributed_allocations']} allocation records lacked this " + "run's scheduler comment and were omitted" + ) + except RuntimeError as exc: + # Messages raised here are the gateway's own fixed strings. + result["scheduler_steps_error"] = str(exc)[:200] + except subprocess.TimeoutExpired: + result["scheduler_steps_error"] = "scheduler step accounting timed out" + return result + + fail("unsupported gateway action") + return {} + + +def _load_client_secret() -> None: + """Load a mounted secret without requiring it in a Pod environment spec.""" + + if os.getenv("GLOBUS_COMPUTE_CLIENT_SECRET", "").strip(): + return + path_value = os.getenv("GLOBUS_COMPUTE_CLIENT_SECRET_FILE", "").strip() + if not path_value: + raise GlobusGatewayError("GLOBUS_COMPUTE_CLIENT_SECRET_FILE is not configured") + path = Path(path_value) + try: + secret = path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise GlobusGatewayError("Globus client secret is unavailable") from exc + if not secret: + raise GlobusGatewayError("Globus client secret is empty") + os.environ["GLOBUS_COMPUTE_CLIENT_SECRET"] = secret + + +class GlobusComputeGateway: + """Invoke the registered service-account gateway on configured endpoints.""" + + def __init__( + self, + client: Any | None = None, + *, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self._client = client + self._sleep = sleep + + @property + def client(self): + if self._client is None: + if not os.getenv("GLOBUS_COMPUTE_CLIENT_ID", "").strip(): + raise GlobusGatewayError("GLOBUS_COMPUTE_CLIENT_ID is not configured") + _load_client_secret() + try: + from globus_compute_sdk import Client + except ImportError as exc: + raise GlobusGatewayError("globus-compute-sdk is not installed") from exc + self._client = Client() + return self._client + + def invoke( + self, + *, + endpoint_id: str, + function_id: str, + action: str, + payload: dict[str, Any], + timeout_seconds: float = 180.0, + ) -> tuple[str, dict[str, Any]]: + try: + UUID(endpoint_id) + UUID(function_id) + except ValueError as exc: + raise GlobusGatewayError("Globus endpoint/function identifier is invalid") from exc + try: + task_id = self.client.run( + {"action": action, "payload": payload}, + endpoint_id=endpoint_id, + function_id=function_id, + ) + except Exception as exc: # SDK exception hierarchy changes across versions. + raise GlobusGatewayError("Globus Compute task submission failed") from exc + deadline = time.monotonic() + timeout_seconds + lookup_failures = 0 + while True: + try: + # This invocation waits for exactly one gateway task. The + # single-task API also propagates remote execution errors; + # SDK get_batch_result suppresses deserialization/task errors + # and, in live validation, returned StorageException for an + # ElasticDFT result that get_task retrieved successfully. + task = self.client.get_task(task_id) + except Exception as exc: + # A task may already have submitted a Slurm job or archived + # its results. Retry lookup of that same task, not execution. + lookup_failures += 1 + delay = min(2 ** lookup_failures, 8) + if ( + _retryable_lookup_error(exc) + and lookup_failures <= 3 + and time.monotonic() + delay < deadline + ): + self._sleep(delay) + continue + error_type = GlobusTransientError if _retryable_lookup_error(exc) else GlobusGatewayError + raise error_type("Globus Compute task lookup failed") from exc + lookup_failures = 0 + if task and not task.get("pending", True): + if task.get("status") != "success": + raise GlobusGatewayError("RCAC Globus gateway task failed") + result = task.get("result") + if not isinstance(result, dict): + raise GlobusGatewayError("RCAC Globus gateway returned invalid data") + return str(task_id), result + if time.monotonic() >= deadline: + raise GlobusTransientError("RCAC Globus gateway task timed out") + self._sleep(1.0) + + @staticmethod + def encoded_files(paths: Iterable[Path]) -> dict[str, str]: + encoded: dict[str, str] = {} + total = 0 + for path in paths: + path = Path(path) + if not path.is_file(): + raise FileNotFoundError(path) + data = path.read_bytes() + total += len(data) + if total > 7 * 1024 * 1024: + raise ValueError("staged workflow inputs exceed the Globus payload budget") + encoded[path.name] = base64.b64encode(data).decode("ascii") + return encoded + + +def register_gateway_function() -> str: + """Register the fixed gateway once and return its Globus function UUID.""" + + _load_client_secret() + try: + from globus_compute_sdk import Client + from globus_compute_sdk.serialize import PureSourceTextInspect + except ImportError as exc: + raise GlobusGatewayError("globus-compute-sdk is not installed") from exc + # Serialize the full source. The RCAC endpoint does not and should not + # contain the Digital Twin Python package, so reference/pickle-based + # serialization would fail remotely. + client = Client(code_serialization_strategy=PureSourceTextInspect()) + function_id = client.register_function( + rcac_service_gateway, + description="Birck Digital Twin allow-listed RCAC Slurm gateway", + ) + return str(function_id) + + +def main() -> None: + import argparse + + parser = argparse.ArgumentParser(description="Manage the DT Globus gateway") + parser.add_argument("command", choices=["register", "capabilities"]) + parser.add_argument("--cluster", help="configured RCAC cluster to check (capabilities)") + args = parser.parse_args() + if args.command == "register": + print(json.dumps({"function_id": register_gateway_function()})) + return + # Read-only rollout check: the configured function must accept at least + # the resource policy the DT catalog offers for this cluster. + from .rcac import get_cluster + from .resources import gateway_policy_differences + + if not args.cluster: + parser.error("capabilities requires --cluster") + cluster = get_cluster(args.cluster) + _task_id, capabilities = GlobusComputeGateway().invoke( + endpoint_id=cluster.endpoint_id, function_id=cluster.gateway_function_id, + action="capabilities", payload={"cluster": cluster.name}, timeout_seconds=120.0, + ) + problems = gateway_policy_differences(cluster.resource_policy, capabilities) + print(json.dumps({"cluster": cluster.name, "capabilities": capabilities, + "consistent": not problems, "problems": problems}, indent=2)) + if problems: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/api/computations/materials.py b/api/computations/materials.py new file mode 100644 index 0000000..0ed29b2 --- /dev/null +++ b/api/computations/materials.py @@ -0,0 +1,35 @@ +"""Materials Project resolution kept outside request handlers.""" + +from __future__ import annotations + +import os +from typing import Any + + +def resolve_material_inputs(inputs: dict[str, Any]) -> dict[str, Any]: + """Fill structure and metallicity when the caller supplied only an MP id.""" + + if inputs.get("structure_dict"): + return inputs + api_key = os.getenv("MATERIALS_PROJECT_API_KEY", "").strip() + if not api_key: + raise RuntimeError( + "MATERIALS_PROJECT_API_KEY is required when structure_dict is omitted" + ) + try: + from mp_api.client import MPRester + except ImportError as exc: + raise RuntimeError("mp-api is required by the computation worker") from exc + + with MPRester(api_key) as rester: + documents = rester.materials.summary.search( + material_ids=[inputs["mp_id"]], + fields=["material_id", "structure", "is_metal"], + ) + if not documents: + raise ValueError(f"Materials Project record not found: {inputs['mp_id']}") + document = documents[0] + resolved = dict(inputs) + resolved["structure_dict"] = document.structure.as_dict() + resolved["is_metal"] = bool(document.is_metal) + return resolved diff --git a/api/computations/monolayer.py b/api/computations/monolayer.py new file mode 100644 index 0000000..c2c0d0d --- /dev/null +++ b/api/computations/monolayer.py @@ -0,0 +1,117 @@ +"""Portable monolayer preparation adapted from Justin's CellRelaxDFT 2D. + +No credentials, scheduler calls, or filesystem conventions live here. The +source structure and every transformation are retained in the run manifest. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import numpy as np + +from .qe import ( + PreparedQeRun, QePreparationError, QeParseError, _structure_from_dict, + _write_pw_input, _pseudo_directory, required_pseudopotentials, + cellrelax_outputs_from_parsed, +) + + +def extract_monolayer(source: dict[str, Any], vacuum_angstrom: float): + from pymatgen.analysis.dimensionality import get_dimensionality_larsen, get_structure_components + from pymatgen.analysis.local_env import CrystalNN + from pymatgen.core import Lattice, Structure + + bulk = _structure_from_dict(source) + if not bulk.is_ordered: + raise QePreparationError("Disordered/partial-occupancy structures are not supported") + bonded = CrystalNN().get_bonded_structure(bulk) + if get_dimensionality_larsen(bonded) != 2: + raise QePreparationError("2D relaxation requires a bonded layered (dimensionality 2) structure") + components = get_structure_components(bonded, inc_orientation=True, inc_site_ids=True) + if not components or any(c["dimensionality"] != 2 for c in components): + raise QePreparationError("Mixed-dimensional structures require explicit scientific review") + # Do not silently discard a chemically different layer of a heterostructure. + signatures = { + tuple(sorted(str(bulk[i].specie) for i in c["site_ids"])) for c in components + } + if len(signatures) != 1: + raise QePreparationError("Inequivalent layer compositions require explicit layer selection") + component = min(components, key=lambda c: tuple(sorted(c["site_ids"]))) + ids = sorted(component["site_ids"]) + orientation = [int(x) for x in component["orientation"]] + axes = [i for i, value in enumerate(orientation) if value] + if len(axes) != 1: + raise QePreparationError("Stacking must align with one lattice axis; supply a conventional layered cell") + stack = axes[0] + plane = [i for i in range(3) if i != stack] + lattice = np.array(bulk.lattice.matrix) + a, b = lattice[plane] + normal = np.cross(a, b) + normal /= np.linalg.norm(normal) + if np.dot(normal, lattice[stack]) < 0: + plane.reverse() + a, b = lattice[plane] + normal = -normal + x = a / np.linalg.norm(a) + y = np.cross(normal, x) + rotation = np.stack([x, y, normal], axis=1) + frac = np.array(bulk.frac_coords)[ids] + unwrapped = frac[0, stack] + (frac[:, stack] - frac[0, stack] + 0.5) % 1.0 - 0.5 + frac[:, stack] = unwrapped + cart = frac @ lattice @ rotation + thickness = float(np.ptp(cart[:, 2])) + height = thickness + vacuum_angstrom + cart[:, 2] += height / 2 - (cart[:, 2].max() + cart[:, 2].min()) / 2 + matrix = np.array([a @ rotation, b @ rotation, [0, 0, height]]) + matrix[:2, 2] = 0.0 + layer = Structure(Lattice(matrix), [bulk[i].specie for i in ids], cart, + coords_are_cartesian=True, to_unit_cell=True) + metadata = { + "method": "CrystalNN/Larsen bonded layer, normalized to XY", + "source_structure_sha256": hashlib.sha256(json.dumps(source, sort_keys=True).encode()).hexdigest(), + "source_site_indices": ids, "source_stacking_orientation": orientation, + "source_in_plane_axes": plane, "cartesian_rotation_matrix": rotation.tolist(), + "layer_thickness_angstrom": thickness, "vacuum_angstrom": vacuum_angstrom, + "cell_height_angstrom": height, "boundary_condition": "3D periodic supercell with vacuum; no Coulomb truncation", + "cell_dofree": "2Dxy", "kpoints_z": 1, + "source_structure": source, "initial_monolayer_structure": layer.as_dict(), + } + return layer, metadata + + +def prepare_cellrelax2d(inputs: dict[str, Any], workdir: Path, pseudo_root: str) -> PreparedQeRun: + layer, metadata = extract_monolayer(inputs["structure_dict"], inputs["vacuum_angstrom"]) + workdir.mkdir(parents=True, exist_ok=True) + path = workdir / "vcrelax.in" + _write_pw_input(path, layer, inputs, calculation="vc-relax", + pseudo_dir=_pseudo_directory(pseudo_root, inputs["pseudo"], inputs["exchange_correlation"])) + return PreparedQeRun( + tool_name="cellrelaxdft2d", local_workdir=workdir, stage_files=[path], + expected_artifacts=["vcrelax.stdout"], + pseudopotentials=required_pseudopotentials(layer, pseudo_root, inputs["pseudo"], inputs["exchange_correlation"]), + ntasks=64, time_limit="24:00:00", metadata=metadata, + ) + + +def cellrelax2d_outputs(inputs, parsed, preparation): + initial = preparation["initial_monolayer_structure"] + result = cellrelax_outputs_from_parsed({**inputs, "structure_dict": initial}, parsed) + final = _structure_from_dict(result["structure_dict"]) + height = preparation["cell_height_angstrom"] + if not np.allclose(final.lattice.matrix[2], [0, 0, height], atol=1e-5): + raise QeParseError("2D relaxation unexpectedly changed the fixed vacuum vector") + if not np.allclose(final.lattice.matrix[:2, 2], [0, 0], atol=1e-5): + raise QeParseError("2D relaxed lattice is no longer in the XY plane") + if len(final) != len(initial["sites"]): + raise QeParseError("2D relaxation changed the atom count") + result.update( + dimensionality=2, preparation=preparation, + units={"energy": "eV per reduced formula unit", "atomic_forces": "eV/angstrom", + "stress": "GPa; QE supercell stress, compression positive, includes vacuum normalization", + "pressure": "GPa; supercell hydrostatic pressure, not a 2D in-plane pressure", + "lattice_vectors": "angstrom", "volume": "angstrom^3 per reduced formula unit; includes vacuum"}, + ) + return result diff --git a/api/computations/pseudopotentials.sha256 b/api/computations/pseudopotentials.sha256 new file mode 100644 index 0000000..c32f444 --- /dev/null +++ b/api/computations/pseudopotentials.sha256 @@ -0,0 +1,171 @@ +b72accf6f3c9cc0fd028cad42dc2fcbb01c71ad6f6bd522e4c234600ea8db9df pseudo_PAW_PBEsol/Ag.upf +bbf459847b29cbc32468e1a2cddeb4b1c958ca215023cc66cd40a92d2e0ed11c pseudo_PAW_PBEsol/Al.upf +6a18441a4314ab62f305745db9c09b02a63ef16023a7cfb8f7d0130037a961f3 pseudo_PAW_PBEsol/Ar.upf +fddbebe9c62aaa45c4b77abd5b90844d86b23cdd58c54c67bd2ff1b3ed62be00 pseudo_PAW_PBEsol/As.upf +07deffb7dc25da1691b821ae94a3d30fd3dfa8d5e482fc8ae422ddfdb5a373a7 pseudo_PAW_PBEsol/At.upf +c960e1484b4598715bfa33994161bdd6b2d39c83e6b6af59a2429389f1fd908a pseudo_PAW_PBEsol/Au.upf +0a9b322243b9ff02014bb87f574f5f8ab7037d9f1ed5257832f129c3c7333538 pseudo_PAW_PBEsol/B.upf +89c29de33e932c922217cdbb525a453b9ff47fc8c34161676c51e9f9b068ba8f pseudo_PAW_PBEsol/Ba.upf +db78275c1d71cc95520e02e842036ad4b95a58271ed1e997d333921984cd312f pseudo_PAW_PBEsol/Be.upf +827c550c44757b93e79cd870bf040b486a0e95e8b7864fc05b013fbe26fabec6 pseudo_PAW_PBEsol/Bi.upf +961d12e487e0460945e5d7670ada9dfe822620eb82e94fcc9666f463c0960563 pseudo_PAW_PBEsol/Br.upf +3fca0f06ef8dac54134ee149f0f17ce1990b24e9fcd0c73251b15efd19d1227a pseudo_PAW_PBEsol/C.upf +e080a39f0a6bad615296971c79f136665d63cdb4e7f854f057b854b4c2b5b12a pseudo_PAW_PBEsol/Ca.upf +8acf61688626d2db885c17e2dc07693276adabfea308b5e3e9389fac01b715d0 pseudo_PAW_PBEsol/Cd.upf +806759fbfc1af6ce93a0913a78c62eaa7530695f6ffadc41d1788eec21c89acc pseudo_PAW_PBEsol/Ce.upf +2665dbcc2c0b10b0a299617294d7248cc477ad7664ef0d3dba89daf42926fafc pseudo_PAW_PBEsol/Cl.upf +6cfc3c8e2a8140367d5442c3e03c2173f7cb6797d17f10ec5c0710c706b0258b pseudo_PAW_PBEsol/Co.upf +289d484b561678e08415b34c5c8cd627d8f634036e51e2b3f27c0eb4305227fd pseudo_PAW_PBEsol/Cr.upf +5c787d8716cd6e1a40725e014c01e9e0afec197633fb63b5ab3fddf914ddbca6 pseudo_PAW_PBEsol/Cs.upf +0e441d790ce4dd4d184c07968ae4b489814fb2d507c42bf74ae496989e333149 pseudo_PAW_PBEsol/Cu.upf +4053428d648adf04e3c6bd49033d5936c4928cf8995099313b4b838061780463 pseudo_PAW_PBEsol/Dy.upf +7f18c0d4bf647b98515858f49d30868df36c4aaa7acc673d081ef2e11364e3ae pseudo_PAW_PBEsol/Er.upf +214550cc73c804bd7056b7f39fb4a24b5809f680e3ea38a069e7e6a36569c65f pseudo_PAW_PBEsol/Eu.upf +f4759a69517310a2c93e3327cc4a511f347387e4dd3a27ebdde884c8bc3275c1 pseudo_PAW_PBEsol/F.upf +73feaa7a689b0298b4498cc13c3e9f53f4db1fde9afb569b933152ecdd5a96b7 pseudo_PAW_PBEsol/Fe.upf +9ddec49b4c381ffa1581189ff170d42febbc76c3d75efcdbc1de39dab68f87ec pseudo_PAW_PBEsol/Ga.upf +6f481a958b089fac08909ff07fc5554fe7c72a79c07ba3641035d4e6a1d48ef9 pseudo_PAW_PBEsol/Gd.upf +dedef770ac40368d94916ff9a52b3a8f184d4afae0ff470d9783dbf008f0c4ca pseudo_PAW_PBEsol/Ge.upf +8feb30fdbfad1b27e2f63f3643d876315e13827bb067214403a06ca19b06c720 pseudo_PAW_PBEsol/H.upf +cfcc03856855b7e82ab4e3d02902bdcdf6694823de273b8024b3bea6f2cc9ed2 pseudo_PAW_PBEsol/He.upf +f9506f1ea87d0b3a537977e91a19ae2268493a70c06df7d5af5c4bf1764a3481 pseudo_PAW_PBEsol/Hf.upf +19abecf9cba6ddbd95400d6d26bf059c3c9ad83c4bc8b868bc1219a84e10af66 pseudo_PAW_PBEsol/Hg.upf +62885e5b26236f2134471346d3df19f58e5682df0a24cf537aa2773397190060 pseudo_PAW_PBEsol/Ho.upf +9b235bb8a2dfcb312bfcb0e1d211beba7e44fc6ddb109a389b71e78e4576da69 pseudo_PAW_PBEsol/I.upf +c84ce1b32d8c839f81db252424bc4383ba023fcb30144c59ef801e0759f055ce pseudo_PAW_PBEsol/In.upf +18edc80f846dac7cd1d7c1541eab2069cdcdeeb8bf6808998d2bdb4ea7d64c2b pseudo_PAW_PBEsol/Ir.upf +cdf778dc665c388ef86927c05ec3f5fcd6fb55b7f2b2569a7ddbb84d68257785 pseudo_PAW_PBEsol/K.upf +b61cdcf86c1cb8bcf670746cc2b0c8402a428072f2d8780ee3d2866c98a25f46 pseudo_PAW_PBEsol/Kr.upf +0eac0fd619038c7b0fc2d1995f27702f753de246967a8b1bc9b89b601a0370fa pseudo_PAW_PBEsol/La.upf +6bb4c6c29493e25fb31ecc18d15e231a85cd8ff113965da7f87b351e3ed90e0a pseudo_PAW_PBEsol/Li.upf +525cac65df4bf0328c6476e1414ba6b80ba0958687da3cfc5b2d17e387f39326 pseudo_PAW_PBEsol/Lu.upf +0bf2bef727ce060ccbcfb36078743b5f537a9fbc4cb5df88c0ab91e343438736 pseudo_PAW_PBEsol/Mg.upf +d4cc7956efb17f03f8e89d076969952423c0fc7dd7e0f0acef0bc49c5b2aaa56 pseudo_PAW_PBEsol/Mn.upf +ac708da3dfdb552eb1ccaf0a8c1d200607d9e747516c3c6b6f9d379bfe0f1df6 pseudo_PAW_PBEsol/Mo.upf +098f4c428ee562d5eb6f5fdbb036e849c135d9c4c7ed0a8f70a8e2263df967ff pseudo_PAW_PBEsol/N.upf +f5541371f3823de7e8893534d9e05313c8a5fb17caf5c042006c9f60372f5267 pseudo_PAW_PBEsol/Na.upf +f4f3bb463efac6babaae59afa65d63ea7e11090ff754b871bb5e26c752a06803 pseudo_PAW_PBEsol/Nb.upf +7dd44b310f3eb1dcaf330574ef962921a8c14da02d49f645418db2a3924c4e20 pseudo_PAW_PBEsol/Nd.upf +0a51e12b6a3c2dd5ad7c7bdb68d27486ef93ae9c1d01fa53e8d127d244aea8bf pseudo_PAW_PBEsol/Ne.upf +28002e40e36bf4e2258f7c8c6717d391bceb23170451bd15f0a6dd905e2a2115 pseudo_PAW_PBEsol/Ni.upf +3cc26539673d44cf887ed311e96383164a1af08a44dc2d1dcad0ed77b269ca09 pseudo_PAW_PBEsol/O.upf +c45dcdc3b481bc7a393172d7dc0bfd0a30c9363ad5786dfea9d1101aac61c0eb pseudo_PAW_PBEsol/Os.upf +df6d996620772fbf473811d1616b1bc73ad5d40a856f6429d7dd9371840f7043 pseudo_PAW_PBEsol/P.upf +277d05a1ba44c0ac0a455f583f18050e0a1aeff814814592de0fedd3590a6f3f pseudo_PAW_PBEsol/Pb.upf +e34db5212290c7976b92106b6e0da102a4f9e7d4b675932b4c69c53e9f951a96 pseudo_PAW_PBEsol/Pd.upf +bfe73d26e6cce998bfe62c866b4926307fb22c88d5e8c759014d53f35b9b960d pseudo_PAW_PBEsol/Pm.upf +03ece3d0953391a41b6b2f78959bcab3b1762078d3e6db9373eee09772380ee0 pseudo_PAW_PBEsol/Po.upf +fabb6c4b175b26857b9b584e580083f0a3d13821df96801e485c2a35a6307959 pseudo_PAW_PBEsol/Pr.upf +3086672fc9640fd2bf58f53443a3e344fd0d16247e4d27f9256f2eab504ef552 pseudo_PAW_PBEsol/Pt.upf +ba043c0d342eae708ba0b7c14993d8b315f828bbccf136c28192dbc11784f688 pseudo_PAW_PBEsol/Rb.upf +9c36aaf2b179e7cf5684eb2461916b6e105553fe8fa4e76e5db137fb58950fac pseudo_PAW_PBEsol/Re.upf +5929ce9bcae3bcd9e466fee58197a7d96adcdbd79b7eb4d823cbf82a357e7817 pseudo_PAW_PBEsol/Rh.upf +bbd7611b5352504e4141eef52f8dc3c31e4b507630f0ef20aef70047fff5a702 pseudo_PAW_PBEsol/Rn.upf +5aae4d11c694a5d011742254203af06cddd881e67d9f83e54f6e413bb1f51eee pseudo_PAW_PBEsol/Ru.upf +eb356a2831fbb254e9086992f862714f844649b6849b23f0cacc2a3d9450af2b pseudo_PAW_PBEsol/S.upf +84584d07d4a6cfac3631799e2262dde986164df9ccb46ebd24847915cc0ed2dc pseudo_PAW_PBEsol/Sb.upf +65e271a90d96083809b2cd269eada462e8d2dd86c37b9823316bde1ed09d402e pseudo_PAW_PBEsol/Sc.upf +6bf49a97d32e78fa51fa39da1fc6cefb04763b15501ce72f672691927e61d41a pseudo_PAW_PBEsol/Se.upf +d2f59acaf5ed0d0c5bca36c97c99595c206b292acf70dd21c6665a1ea448c403 pseudo_PAW_PBEsol/Si.upf +d04cac215b1b345fa8fdfc17384634532ecb49b3085b74796168d88fa7f005cb pseudo_PAW_PBEsol/Sm.upf +5b18dceb51f8d5ae0a80e0e61234ad5299d2bc75f276b9fa700efab749463f56 pseudo_PAW_PBEsol/Sn.upf +bc0edb842a875086240976ef6b40a143df40d7b6ab26be969490a73c82e50643 pseudo_PAW_PBEsol/Sr.upf +ede9ce1e4f6c2aa045a5e5a41d79fd858de7307881a88a370a155fadfd60a179 pseudo_PAW_PBEsol/Ta.upf +34ea31919e7e5b57870601668f6b0ab23c0fc8f70732869933b2c5dd40f5dc7f pseudo_PAW_PBEsol/Tb.upf +0bf0cc94fe40d006f0019e65763a2295fd1383cbf6c957a1bcd33966376aae4d pseudo_PAW_PBEsol/Tc.upf +644d5d8686bb53a4d3089db27d06eb47ea850df1ddc7dd966827927b5210e195 pseudo_PAW_PBEsol/Te.upf +6c4983638781c1801f6b190030e2c38b36f4f2b6b1fd6e3b723e169044d3d500 pseudo_PAW_PBEsol/Ti.upf +60dbb0b7480bc30fdb97e1f77b9f17230cc9bc71bc9bd8825bcea752e73a402d pseudo_PAW_PBEsol/Tl.upf +f9a9811a36137aeb49d737739b04d96cd68f8760971a1c44952f8923f801d062 pseudo_PAW_PBEsol/Tm.upf +74766837b0a6dbbbc26ea9c26c7433793fb08b47c684e0cf0345af4290555ffe pseudo_PAW_PBEsol/V.upf +4404012b4ddf96cc5cb2e872f07a7328f0f1abd0d7d56f4429fe7e1de3cdcf1f pseudo_PAW_PBEsol/W.upf +8c9dd42e8e8cb9a90058c1e11e66da8b99df048905483ec192f78332126e4450 pseudo_PAW_PBEsol/Xe.upf +19817ece8cecf4d012dd1a0fee3f29538be8db75b1b88e3d47cb79f70e29a84b pseudo_PAW_PBEsol/Y.upf +416c54d3d20504cf2cf54773ff7618851840f93ebee586f31ae38b59d577c582 pseudo_PAW_PBEsol/Yb.upf +d7326e4d018653e735b266ffc023e9e3180ae9cd9efdc49571fc4bf0f8cea9d4 pseudo_PAW_PBEsol/Zn.upf +f282fba4f7fb4803678f168541ac28b9fde950734ca16c5b74193115ea901e97 pseudo_PAW_PBEsol/Zr.upf +7647cf9273cc207495764cc7f806e6e29341ff18c48b2fde7cb54f4726178c64 pseudo_USPP_PBEsol/Ag.upf +801a04431015c4326c8891f7f7b31e06ab51e5c0ad32493ee7436d28e82bd7a1 pseudo_USPP_PBEsol/Al.upf +2af5b207d0923b80542a4195082db45121d59b7f6e977616ead2d33465ae54f8 pseudo_USPP_PBEsol/Ar.upf +cdf99423e6aebe17641e1ee30c7ec68da9099903000394d47a6f55bd51dadecf pseudo_USPP_PBEsol/As.upf +af2872fcda537ac5dec334bd2759adbd1685d8b5dd9c06c58fa72c11288afbc5 pseudo_USPP_PBEsol/Au.upf +87ce207ef55d64e2f16efa9c368a1914c1dbaf1414bd204e2d72410f4bbd2641 pseudo_USPP_PBEsol/B.upf +b8114a98a94bfc0b9570acf1b4e583a75584675112ecb1972d7e521f92374ddf pseudo_USPP_PBEsol/Ba.upf +c23946724cd76b989e52863a7c4d10e40fa194a393e51d6bc847d28666aa400b pseudo_USPP_PBEsol/Be.upf +d2eea7a660cf839d1332de73d8c2845bad0d301d67d2d791f0374c37ace8ae9e pseudo_USPP_PBEsol/Bi.upf +cb6795b97049c40ca66443da5d725f9214ded6ca41a79c4eb3d2e7457713adc8 pseudo_USPP_PBEsol/Br.upf +d379b4cd2f440c357b310c3772554838c13124d7967e59e045429002efe7055b pseudo_USPP_PBEsol/C.upf +1875414aeb16344d59fef9e7959a5894899fddc21b034b239cc30a66da508e44 pseudo_USPP_PBEsol/Ca.upf +5d4602fbc2340e69c98516cb7a3f6102657778c30b687b296174f887b115d003 pseudo_USPP_PBEsol/Cd.upf +a26a6c021fa35d204c7a05491b69dc4208050a7457cbe2b3957bb90b4845f93a pseudo_USPP_PBEsol/Ce.upf +e302a65c1c989cbde50de95a81297e9f4401e0ef76d9a4178281c3abe7440441 pseudo_USPP_PBEsol/Cl.upf +05d65a6dbaba75cb4b9ad552d0113d25bb7cc91bf15ff16d3735f65d16e9aaa1 pseudo_USPP_PBEsol/Co.upf +afb3bea4a7b436f20ee6cae05e0ed6d266f6bac12c9c513de34f6f8b4ad0bda2 pseudo_USPP_PBEsol/Cr.upf +4bd6d33a2c99f681515df646712c10235d1f6ffc5809728d5af8215ae4dc4840 pseudo_USPP_PBEsol/Cs.upf +4aaa1d2269f0696ea232b3d0b8a901d66ebf58cc9ccf9cb81c383676b8dbee6e pseudo_USPP_PBEsol/Cu.upf +741b589abd977b578d36de1b984e6599dc48168ad3af2e54f5d76dded2d35d50 pseudo_USPP_PBEsol/Dy.upf +fc625713ca5c63de7084d45f664e01d4931d35af1100a94a77a1b13ef91f9a91 pseudo_USPP_PBEsol/Er.upf +26323dbc87d6fc71c48ef12dc2d7612bd1e87b20c12588cd793b1973873946bd pseudo_USPP_PBEsol/Eu.upf +3c23050eabbb40ad23379b2eb86337248eac6c6f443b74aace290bc7349c1242 pseudo_USPP_PBEsol/F.upf +c6a5cf098f72dcd22d1c4a74bbe56953619aadd64753026cb085d73c93598f01 pseudo_USPP_PBEsol/Fe.upf +129b768a4e97e04a5a33fd5ebedc858e30bd761d25562231ad0a7d3f54114042 pseudo_USPP_PBEsol/Ga.upf +5836ee0896d1fa88eb50063e3d48fb1a1d430e65e8e291c86ebc517d6765411c pseudo_USPP_PBEsol/Gd.upf +659ce92c49ab47a83ace5623561c7d4e557d34e1954565cbd53ba31406f288f8 pseudo_USPP_PBEsol/Ge.upf +5a07198cd1f0f3011bcac332163860abdd0eb2024a6b514a4f6dd12eb58c541d pseudo_USPP_PBEsol/H.upf +c93139553032cd2dacdf2b806486e92e07f643caa3bfcaee8028574f58b7a14a pseudo_USPP_PBEsol/He.upf +eb426d5a9a9a7cc8509fa5528a6bbc795b90fa83ee3ed9ce253c45d464781a9e pseudo_USPP_PBEsol/Hf.upf +08a6ba99461d7b5dd7b62800083a7dad1c51a20647b823cf85d8ac4ac0134664 pseudo_USPP_PBEsol/Hg.upf +2a56aa3df1427a6622ec295bc06fbbf3c03e48f54c9e257b9d0cb6f5b2359b64 pseudo_USPP_PBEsol/Ho.upf +4e1d523d29dc3435abc661df11e7eb71a6247c280c21f585a6cbc7de68791019 pseudo_USPP_PBEsol/I.upf +90ff08232379a881f5870dbc9e5a330f36fc45b941978907cccb2d0a963d6f7b pseudo_USPP_PBEsol/In.upf +7da93d6e375c4aa35c7fe3b47b23c1beb9164bb0e81f3f5e3fd9b78275b4c835 pseudo_USPP_PBEsol/Ir.upf +99db5c40ea33d74b0f20648b9cd61ea51cf30575a389f799266cf39d06063286 pseudo_USPP_PBEsol/K.upf +b43d9a0141156365062a5aea61fffae2f7d7bcc10e753f084d0d44ba69ad2d8f pseudo_USPP_PBEsol/Kr.upf +e3832e95b8d5e039120ef09ac7a948717c7d6ebe6b3cd17e3eed5a2df2a3c895 pseudo_USPP_PBEsol/La.upf +6cea3cb688b62fd4cbe137527d3df2efe8bebc6ef9c3bd62254b4c52a5ab749e pseudo_USPP_PBEsol/Li.upf +1f95ef6e6e1a7364f29d9515f78ff6b589999ce0b4604128c48495b55d472004 pseudo_USPP_PBEsol/Lu.upf +baa13b102c840c64e358eb64c38768b5b76765ceaef4ff3c22a54c2d5f41e791 pseudo_USPP_PBEsol/Mg.upf +74217ac6e49375a3e98397b18f98a4dc13263f0b2292818a1823495432b393bc pseudo_USPP_PBEsol/Mn.upf +eda027d30a7a8444461a2f1f5414fad4b4c4a3bb1908b0ab52b2df0ebdc750e5 pseudo_USPP_PBEsol/Mo.upf +2eea521c1eccac913da8c0c5ffc0d0ab86819c975a5dec7458e6a4fc07120354 pseudo_USPP_PBEsol/N.upf +62e0a0eb5bff3bd89e3039005a257a5e2f823b4fb5d39d4c5ff280b62114a202 pseudo_USPP_PBEsol/Na.upf +6910088685586744219051b12125da12ee78e9c70694295c3c59195ec75f4842 pseudo_USPP_PBEsol/Nb.upf +57b8177768ef07db25e4c549be370f04fc9660d7cc24636aa262bc749cc84497 pseudo_USPP_PBEsol/Nd.upf +002153d883e86f43428a0b67e09c7f6e43f0b468c620ff38c47d2c873f08d927 pseudo_USPP_PBEsol/Ne.upf +e293ca8dba4c646066dd63411b93d0fc07b1c2c3a8277a4b41f0f8966205b4cf pseudo_USPP_PBEsol/Ni.upf +fab367c8e8ebb07f7071a5256040af7f38e850cb0c326cac53b0123ebe1d1a1b pseudo_USPP_PBEsol/O.upf +c5b41389ed973dec21db2b9ff9891313c8e4876284a363279392d0fa36448418 pseudo_USPP_PBEsol/Os.upf +336bf7086784d62a8d415edafff2ba22fd62824035d3a7c593e74b0a7bf672e9 pseudo_USPP_PBEsol/P.upf +29bbae4cb95b640f7d62971ca84c96abe308886b9a8bdc910b27c2907a5bb808 pseudo_USPP_PBEsol/Pb.upf +3276e38ae2886a0bc2b8b595bae0a32bab639107047d3e4bdecebb2b4dfa9afc pseudo_USPP_PBEsol/Pd.upf +8a8d0a5060fcb83fb1dcfd0413ad8e9dc23e41314727939e525e757644ebfd02 pseudo_USPP_PBEsol/Pm.upf +0775358c854012e4945517d73304c9a775746cb7bebc483dbd59728d45f0a884 pseudo_USPP_PBEsol/Po.upf +fd6548b4810fff270fe673a679384cc62a5ca86c4e29afffe358261b8de2403f pseudo_USPP_PBEsol/Pr.upf +fa53f2ea9be66744abb5bbf503e2f863eb8bffcfb271ebbcf66b072b402292b8 pseudo_USPP_PBEsol/Pt.upf +00a0ffcccc4252b9cb2f494e061f29fc2b786bb0c2079096b30f47957c080ed0 pseudo_USPP_PBEsol/Rb.upf +bdd681aa1072daff4d2c11fcf041dbf6837d9317a5590f1106450489a9f56854 pseudo_USPP_PBEsol/Re.upf +11be90bd38aa55c771b969de4edfaf910e1f0457bdaf439c35d563e8cf424cbf pseudo_USPP_PBEsol/Rh.upf +ac145055ef8a9a358681e68a010344dceda18603698c05b023e433cb704bbacf pseudo_USPP_PBEsol/Rn.upf +4f1346e9f322c15f9ee1d7106d6efe6122ccbed6c1e574e5d1da84a83d6ea84f pseudo_USPP_PBEsol/Ru.upf +5785e57e7320fbd0292aa2f26938c4a4d595d4313fe1494b2d0e49072ac80438 pseudo_USPP_PBEsol/S.upf +50043644d41f233fd86caf966daa3d88cb1f482077dfceb1076d93f0ca9c4637 pseudo_USPP_PBEsol/Sb.upf +223e62c5acc55044f5f064778bc28ffac9ccd3be01490e8b0776c6686e08f723 pseudo_USPP_PBEsol/Sc.upf +f6f121d83feb9750517cd80b40911229a7749d3334ed89f1516da2f5417c0f4f pseudo_USPP_PBEsol/Se.upf +82e6d35f71dfa816fb8fcc7530464af3e724703d176d885ce5b35cc341ab3e4a pseudo_USPP_PBEsol/Si.upf +05f448a4e6575cf13e4f8a49f16f59dbfee12a416513f70ba26dcb0c3180f1a4 pseudo_USPP_PBEsol/Sm.upf +5fe98cd4c11ae4ae5dfd2827cb744148e3582660e9f69730b5eeb9c964c23cb1 pseudo_USPP_PBEsol/Sn.upf +c7c6fa80a86d360b51ed43e4f5154a6f1625a5d90b7215c95ef2591ebd366db8 pseudo_USPP_PBEsol/Sr.upf +34fb4d2ca944250bcdbec370146045115d5a495752b22b899777d8ba3ccd6b8d pseudo_USPP_PBEsol/Ta.upf +bb2f139cd9421de9dfc781f179180834a4758d8e0823eea9d8e24e9a2f4b2c91 pseudo_USPP_PBEsol/Tb.upf +ed06469c707087ed7e0a780671a8e01b8eba100f3c3c43030bda8d6589ef6821 pseudo_USPP_PBEsol/Tc.upf +a4fca9a03053725dd5b7e83ff29974dda64d614fb9cd8c1d25edea08fc5efd6b pseudo_USPP_PBEsol/Te.upf +9f50adf8d3f6a1b98981d5fb94205e53a523f405883dcb060c06b5eb93805f69 pseudo_USPP_PBEsol/Ti.upf +3451e912db7c2b7423f86b3a7e041cdd27de07a381d03bde0df21174b3330682 pseudo_USPP_PBEsol/Tl.upf +5ea437d39aabc01718c4a7a8638bb97b215ecad63810749623f86afbcf727694 pseudo_USPP_PBEsol/Tm.upf +b8ad92b3ca6dd6a2cf5fbc53eba1d30bd52bf311c9a9037491bb49893857689c pseudo_USPP_PBEsol/V.upf +df0c771d305ca3ace086e9eb4114e72c7e7ec8731157c866400a04feced89729 pseudo_USPP_PBEsol/W.upf +c40bc560b5a17e41c38b5c756a0054865088179a72c2d5475606672d1d131ed6 pseudo_USPP_PBEsol/Xe.upf +a8800467a40fcbbab313965aeb4869bc4c8d71446abd5c006192f984fc325e00 pseudo_USPP_PBEsol/Y.upf +4228a0dc23d687b3414efe3c844d8474d60f053152f7d8642cedd09dab353859 pseudo_USPP_PBEsol/Yb.upf +43c605383db2aa50e263a35c6a574bb2996af3ba35d8ee0abff474d18571f3aa pseudo_USPP_PBEsol/Zn.upf +6402fbcfbb6970779153fa49d988c8cfedf1b4245b295c89af1ed6b89e82164a pseudo_USPP_PBEsol/Zr.upf diff --git a/api/computations/qe.py b/api/computations/qe.py new file mode 100644 index 0000000..51cec9b --- /dev/null +++ b/api/computations/qe.py @@ -0,0 +1,868 @@ +"""Quantum ESPRESSO preparation/parsing for two registered DT workflows. + +Heavy scientific dependencies are imported lazily so the existing API can +start normally while the RCAC feature is disabled. +""" + +from __future__ import annotations + +import json +import math +import re +import tarfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +RY_TO_EV = 13.605684958731 +RY_BOHR_TO_EV_ANGSTROM = 25.71104309541616 +BOHR_TO_ANGSTROM = 0.529177210903 + + +class QePreparationError(ValueError): + pass + + +class QeParseError(ValueError): + pass + + +@dataclass +class PreparedQeRun: + tool_name: str + local_workdir: Path + stage_files: list[Path] + expected_artifacts: list[str] + pseudopotentials: list[dict[str, str]] + job_array: str = "" + ntasks: int = 1 + time_limit: str = "24:00:00" + metadata: dict[str, Any] | None = None + + +def _pymatgen(): + try: + from pymatgen.analysis.elasticity.elastic import ElasticTensor + from pymatgen.analysis.elasticity.strain import Strain + from pymatgen.analysis.elasticity.stress import Stress + from pymatgen.core import Lattice, Structure + from pymatgen.io.pwscf import PWInput + from pymatgen.symmetry.analyzer import SpacegroupAnalyzer + except ImportError as exc: + raise QePreparationError( + "pymatgen is required by the RCAC computation worker" + ) from exc + return { + "ElasticTensor": ElasticTensor, + "Strain": Strain, + "Stress": Stress, + "Lattice": Lattice, + "Structure": Structure, + "PWInput": PWInput, + "SpacegroupAnalyzer": SpacegroupAnalyzer, + } + + +def _structure_from_dict(value: dict[str, Any]): + if not value: + raise QePreparationError( + "structure_dict is required after Materials Project resolution" + ) + try: + return _pymatgen()["Structure"].from_dict(value) + except Exception as exc: + raise QePreparationError( + "structure_dict is not a valid pymatgen structure" + ) from exc + + +def _elements(structure) -> list[str]: + return sorted( + { + element.symbol + for site in structure.sites + for element in site.species.elements + } + ) + + +def _pseudo_directory(pseudo_root: str, pseudo: str, xc: str) -> str: + # The source snapshots contain PBEsol pseudopotentials only. Explicitly + # reject combinations that would otherwise silently use the wrong set. + if xc != "PBEsol": + raise QePreparationError( + "The downloaded source contains only PBEsol pseudopotentials; " + f"cannot run exchange_correlation={xc!r}" + ) + root = pseudo_root.rstrip("/") + if not root.startswith("/") or "/../" in f"{root}/": + raise QePreparationError("pseudopotential root must be a safe absolute path") + return f"{root}/pseudo_{pseudo}_PBEsol" + + +def _pseudopotential_manifest() -> dict[str, str]: + manifest_path = Path(__file__).with_name("pseudopotentials.sha256") + if not manifest_path.is_file(): + raise QePreparationError("pseudopotential checksum manifest is missing") + values: dict[str, str] = {} + for line in manifest_path.read_text(encoding="utf-8").splitlines(): + digest, separator, relative = line.partition(" ") + if separator: + values[relative] = digest + return values + + +def required_pseudopotentials( + structure, pseudo_root: str, pseudo: str, xc: str +) -> list[dict[str, str]]: + directory = _pseudo_directory(pseudo_root, pseudo, xc) + directory_name = PurePosixPath(directory).name + manifest = _pseudopotential_manifest() + records = [] + for element in _elements(structure): + relative = f"{directory_name}/{element}.upf" + digest = manifest.get(relative) + if digest is None: + raise QePreparationError( + f"pseudopotential is not approved by the manifest: {relative}" + ) + records.append( + { + "element": element, + "relative": relative, + "path": f"{directory}/{element}.upf", + "sha256": digest, + } + ) + return records + + +def _qe_cutoff(value: Any) -> int | float: + """Return the validated wavefunction cutoff without truncation. + + Integral cutoffs keep their historical integer rendering; fractional + cutoffs are passed through exactly as validated (QE accepts reals). + """ + cutoff = float(value) + if not math.isfinite(cutoff) or cutoff <= 0: + raise QePreparationError("KE_cutoff must be a positive finite number") + return int(cutoff) if cutoff.is_integer() else cutoff + + +def _qe_dictionaries(inputs: dict[str, Any], structure, *, calculation: str): + electrons = { + "electron_maxstep": inputs["e_steps"], + "conv_thr": inputs["scf_conv"], + } + system: dict[str, Any] = { + "ecutwfc": _qe_cutoff(inputs["KE_cutoff"]), + "input_dft": inputs["exchange_correlation"], + "vdw_corr": inputs["vdw_corr"], + } + if inputs["is_metal"]: + system.update(occupations="smearing", smearing="gauss", degauss=inputs.get("degauss_ry", 0.02)) + if inputs["spin_polar"]: + electrons["mixing_beta"] = 0.3 + system["nspin"] = 2 + for index in range(len(structure.symbol_set)): + system[f"starting_magnetization({index + 1})"] = 0.6 + if not inputs["is_metal"]: + system.update(occupations="smearing", smearing="gauss", degauss=inputs.get("degauss_ry", 0.02)) + if inputs["hubbard_U_on"]: + electrons["mixing_beta"] = 0.3 + if calculation == "vc-relax" and inputs["ibrav"] != 0: + system.update( + { + "ibrav": inputs["ibrav"], + "a": structure.lattice.a, + "b": structure.lattice.b, + "c": structure.lattice.c, + # Preserve the r99 notebook's angle mapping for parity. + "cosAB": math.cos(math.radians(structure.lattice.angles[0])), + "cosAC": math.cos(math.radians(structure.lattice.angles[1])), + "cosBC": math.cos(math.radians(structure.lattice.angles[2])), + } + ) + if inputs["is_metal"]: + electrons["mixing_beta"] = 0.3 + + control = { + "pseudo_dir": "./", + "calculation": calculation, + "outdir": "./qe_tmp", + "tstress": True, + "tprnfor": True, + "nstep": inputs["i_steps"], + "etot_conv_thr": inputs["energy_conv"], + "forc_conv_thr": inputs["force_conv"], + "disk_io": "nowf", + } + return control, system, electrons + + +def _append_hubbard(path: Path, structure, inputs: dict[str, Any]) -> None: + if not inputs["hubbard_U_on"]: + return + lines = [f"HUBBARD {{{inputs['hubbard_projector']}}}\n"] + for element in _elements(structure): + try: + value = inputs["hubbard_U_values"][element] + orbital = inputs["hubbard_U_orbitals"][element] + except KeyError as exc: + raise QePreparationError( + f"missing Hubbard U data for element {element}" + ) from exc + lines.append(f"U {element}-{orbital} {value}\n") + with path.open("a", encoding="utf-8") as handle: + handle.writelines(lines) + + +def _write_pw_input( + path: Path, + structure, + inputs: dict[str, Any], + *, + calculation: str, + pseudo_dir: str = "./", +) -> None: + pmg = _pymatgen() + control, system, electrons = _qe_dictionaries( + inputs, structure, calculation=calculation + ) + control["pseudo_dir"] = pseudo_dir + if calculation == "vc-relax": + cell = { + "press": inputs["target_pressure"], + "press_conv_thr": inputs["pressure_conv"], + } + if inputs["ibrav"] != 0: + cell["cell_dofree"] = "ibrav" + if "vacuum_angstrom" in inputs: + cell.update(cell_dynamics="bfgs", cell_dofree="2Dxy", press=0.0) + else: + cell = None + pseudo = {element: f"{element}.upf" for element in _elements(structure)} + writer = pmg["PWInput"]( + structure=structure, + pseudo=pseudo, + control=control, + system=system, + electrons=electrons, + cell=cell, + kpoints_grid=[inputs["k_x"], inputs["k_y"], inputs["k_z"]], + ) + writer.write_file(path) + if calculation == "vc-relax" and inputs["ibrav"] != 0: + _remove_cell_parameters(path) + _append_hubbard(path, structure, inputs) + + +def _remove_cell_parameters(path: Path) -> None: + """Remove the explicit cell card when QE derives it from nonzero ibrav.""" + + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + for index, line in enumerate(lines): + if line.strip().startswith("CELL_PARAMETERS"): + del lines[index : index + 4] + path.write_text("".join(lines), encoding="utf-8") + return + raise QePreparationError("pymatgen did not emit the expected CELL_PARAMETERS card") + + +def prepare_cellrelax( + inputs: dict[str, Any], workdir: Path, pseudo_root: str +) -> PreparedQeRun: + workdir = Path(workdir) + workdir.mkdir(parents=True, exist_ok=True) + structure = _structure_from_dict(inputs["structure_dict"]) + if inputs["target_pressure"] > 0: + # Preserve CellRelaxDFT r99's Murnaghan-style initial compression. + strain = ((1 + 4 * inputs["target_pressure"] / inputs["bulk_m"]) ** -0.25) ** ( + 1 / 3 + ) - 1 + structure.apply_strain(strain) + + input_path = workdir / "vcrelax.in" + pseudos = required_pseudopotentials( + structure, pseudo_root, inputs["pseudo"], inputs["exchange_correlation"] + ) + _write_pw_input( + input_path, + structure, + inputs, + calculation="vc-relax", + pseudo_dir=_pseudo_directory( + pseudo_root, inputs["pseudo"], inputs["exchange_correlation"] + ), + ) + + return PreparedQeRun( + tool_name="cellrelaxdft", + local_workdir=workdir, + stage_files=[input_path], + expected_artifacts=["vcrelax.stdout"], + pseudopotentials=pseudos, + ntasks=64, + time_limit="24:00:00", + metadata={"num_atoms": len(structure.sites), "elements": _elements(structure)}, + ) + + +def _parse_matrix( + lines: Sequence[str], start: int, columns: slice +) -> list[list[float]]: + try: + return [ + [float(value) for value in lines[index].split()[columns]] + for index in range(start, start + 3) + ] + except (IndexError, ValueError) as exc: + raise QeParseError("malformed matrix in Quantum ESPRESSO output") from exc + + +def _unique_messages(messages: Sequence[str]) -> list[str]: + """Return normalized messages in source order without duplicates.""" + + unique: list[str] = [] + seen: set[str] = set() + for message in messages: + normalized = " ".join(message.split()) + if normalized and normalized not in seen: + seen.add(normalized) + unique.append(normalized) + return unique + + +def parse_qe_output( + path: Path, *, natoms: int | None = None, require_stress: bool = False +) -> dict[str, Any]: + lines = Path(path).read_text(encoding="utf-8", errors="replace").splitlines() + if not lines: + raise QeParseError("Quantum ESPRESSO output is empty") + + ionic_energies: list[float] = [] + pressures: list[float] = [] + stresses: list[list[list[float]]] = [] + total_forces: list[float] = [] + atomic_forces: list[list[list[float]]] = [] + cell_blocks: list[tuple[str, list[list[float]]]] = [] + position_blocks: list[tuple[str, list[str], list[list[float]]]] = [] + + for index, line in enumerate(lines): + if "!" in line and "total energy" in line: + match = re.search(r"=\s*([-+0-9.Ee]+)\s+Ry", line) + if match: + ionic_energies.append(float(match.group(1))) + if "P=" in line: + match = re.search(r"P=\s*([-+0-9.Ee]+)", line) + if match: + pressures.append(float(match.group(1))) + stresses.append(_parse_matrix(lines, index + 1, slice(3, 6))) + if "Total force" in line: + match = re.search(r"Total force\s*=\s*([-+0-9.Ee]+)", line) + if match: + total_forces.append(float(match.group(1))) + if "Forces acting on atoms" in line and natoms: + forces = [] + cursor = index + 1 + while cursor < len(lines) and len(forces) < natoms: + match = re.search( + r"force\s*=\s*([-+0-9.Ee]+)\s+([-+0-9.Ee]+)\s+([-+0-9.Ee]+)", + lines[cursor], + ) + if match: + forces.append([float(match.group(i)) for i in range(1, 4)]) + cursor += 1 + if len(forces) == natoms: + atomic_forces.append(forces) + if line.strip().startswith("CELL_PARAMETERS"): + unit = line.partition("(")[2].partition(")")[0].strip().lower() or "alat" + cell_blocks.append((unit, _parse_matrix(lines, index + 1, slice(0, 3)))) + if line.strip().startswith("ATOMIC_POSITIONS"): + unit = line.partition("(")[2].partition(")")[0].strip().lower() or "alat" + species: list[str] = [] + positions: list[list[float]] = [] + cursor = index + 1 + while cursor < len(lines): + parts = lines[cursor].split() + if len(parts) < 4 or not re.fullmatch(r"[A-Za-z]{1,3}", parts[0]): + break + try: + position = [float(value) for value in parts[1:4]] + except ValueError: + break + species.append(parts[0]) + positions.append(position) + cursor += 1 + if species: + position_blocks.append((unit, species, positions)) + + if not ionic_energies or (require_stress and (not pressures or not stresses)): + tail = "\n".join(lines[-5:]) + raise QeParseError(f"Quantum ESPRESSO output is incomplete. Tail:\n{tail}") + + process_completed = any("JOB DONE" in line for line in lines) + geometry_optimization_ended = any( + "End of BFGS Geometry Optimization" in line for line in lines + ) + failure_markers = ( + "convergence not achieved", + "bfgs failed", + "error in routine", + "maximum number of steps reached", + # QE's actual wording; it still prints "End of BFGS" afterwards. + "maximum number of steps has been reached", + ) + failure_reasons = _unique_messages( + line + for line in lines + if any(marker in line.lower() for marker in failure_markers) + ) + warning_lines = [ + line + for line in lines + if "warning:" in line.lower() + or ("c_bands:" in line.lower() and "not converged" in line.lower()) + ] + convergence = ( + process_completed and geometry_optimization_ended and not failure_reasons + ) + + result: dict[str, Any] = { + "ionic_energies_ry": ionic_energies, + "pressures_kbar": pressures, + "stresses_kbar": stresses, + "total_forces_ry_bohr": total_forces, + "atomic_forces_ry_bohr": atomic_forces, + # QE may print JOB DONE after a BFGS optimization failed. Keep process + # completion separate from scientific convergence so callers cannot + # accidentally promote an unconverged structure. + "process_completed": process_completed, + "convergence": convergence, + "scientific_status": ( + "converged" + if convergence + else "not_converged" + if process_completed + else "incomplete" + ), + "failure_reasons": failure_reasons, + "warnings": _unique_messages(warning_lines), + "warning_count": len(warning_lines), + "cell_blocks": [ + {"unit": unit, "matrix": matrix} for unit, matrix in cell_blocks + ], + "position_blocks": [ + {"unit": unit, "species": species, "coords": positions} + for unit, species, positions in position_blocks + ], + } + + try: + result["alat_angstrom"] = _last_alat_angstrom(lines) + except QeParseError: + result["alat_angstrom"] = None + + if cell_blocks and position_blocks: + unit, lattice = cell_blocks[-1] + if unit.startswith("bohr"): + lattice = [[value * BOHR_TO_ANGSTROM for value in row] for row in lattice] + elif unit.startswith("alat"): + alat = _last_alat_angstrom(lines) + lattice = [[value * alat for value in row] for row in lattice] + pos_unit, species, positions = position_blocks[-1] + coords_are_cartesian = pos_unit.startswith(("angstrom", "bohr")) + if pos_unit.startswith("bohr"): + positions = [ + [value * BOHR_TO_ANGSTROM for value in position] + for position in positions + ] + structure = _pymatgen()["Structure"]( + lattice, + species, + positions, + coords_are_cartesian=coords_are_cartesian, + ) + result["structure"] = structure + return result + + +def _last_alat_angstrom(lines: Sequence[str]) -> float: + for line in reversed(lines): + match = re.search(r"lattice parameter \(alat\)\s*=\s*([-+0-9.Ee]+)", line) + if match: + return float(match.group(1)) * BOHR_TO_ANGSTROM + raise QeParseError("CELL_PARAMETERS uses alat but no lattice parameter was found") + + +def cellrelax_outputs(inputs: dict[str, Any], output_path: Path) -> dict[str, Any]: + initial = _structure_from_dict(inputs["structure_dict"]) + parsed = parse_qe_output( + output_path, natoms=len(initial.sites), require_stress=True + ) + return cellrelax_outputs_from_parsed(inputs, parsed) + + +def _structure_from_parsed(parsed: dict[str, Any]): + if parsed.get("structure") is not None: + return parsed["structure"] + cells = parsed.get("cell_blocks") or [] + positions = parsed.get("position_blocks") or [] + if not cells or not positions: + return None + cell = cells[-1] + position = positions[-1] + lattice = cell["matrix"] + cell_unit = str(cell.get("unit") or "alat").lower() + if cell_unit.startswith("bohr"): + lattice = [[value * BOHR_TO_ANGSTROM for value in row] for row in lattice] + elif cell_unit.startswith("alat"): + alat = parsed.get("alat_angstrom") + if not alat: + raise QeParseError("CELL_PARAMETERS uses alat but no lattice parameter was found") + lattice = [[value * float(alat) for value in row] for row in lattice] + coords = position["coords"] + position_unit = str(position.get("unit") or "alat").lower() + cartesian = position_unit.startswith(("angstrom", "bohr", "alat")) + if position_unit.startswith("bohr"): + coords = [[value * BOHR_TO_ANGSTROM for value in row] for row in coords] + elif position_unit.startswith("alat"): + alat = parsed.get("alat_angstrom") + if not alat: + raise QeParseError("ATOMIC_POSITIONS uses alat but no lattice parameter was found") + coords = [[value * float(alat) for value in row] for row in coords] + return _pymatgen()["Structure"]( + lattice, + position["species"], + coords, + coords_are_cartesian=cartesian, + ) + + +def cellrelax_outputs_from_parsed( + inputs: dict[str, Any], parsed: dict[str, Any] +) -> dict[str, Any]: + """Build the portable scientific result returned by the RCAC gateway.""" + + initial = _structure_from_dict(inputs["structure_dict"]) + final = _structure_from_parsed(parsed) + if final is None: + raise QeParseError("final relaxed structure is missing") + pmg = _pymatgen() + n_units = final.composition.get_integer_formula_and_factor()[1] + stress = parsed["stresses_kbar"][-1] + forces = ( + parsed["atomic_forces_ry_bohr"][-1] if parsed["atomic_forces_ry_bohr"] else [] + ) + structure_dict = final.as_dict() + structure_dict.get("lattice", {}).pop("pbc", None) + return { + "num_atoms": len(initial.sites), + "chemical_system": initial.chemical_system, + "composition": initial.composition.as_dict(), + "formula_unit": initial.composition.reduced_formula, + "mp_space_group": pmg["SpacegroupAnalyzer"](initial).get_space_group_number(), + "energy": parsed["ionic_energies_ry"][-1] * RY_TO_EV / n_units, + "atomic_forces": [ + [value * RY_BOHR_TO_EV_ANGSTROM for value in row] for row in forces + ], + "qe_space_group": final.get_space_group_info()[1], + "pressure": parsed["pressures_kbar"][-1] / 10.0, + "stress": [[value / 10.0 for value in row] for row in stress], + "volume": final.volume / n_units, + "structure_dict": structure_dict, + "lattice_vectors": final.lattice.matrix.tolist(), + "lattice_parameters": list(final.lattice.abc), + "angles": list(final.lattice.angles), + "process_completed": parsed["process_completed"], + "convergence": parsed["convergence"], + "scientific_status": parsed["scientific_status"], + "failure_reasons": parsed["failure_reasons"], + "warnings": parsed["warnings"], + "warning_count": parsed["warning_count"], + "intermediate_data": { + "energies_ry": parsed.get("ionic_energies_ry", []), + "pressures_kbar": parsed.get("pressures_kbar", []), + "stresses_kbar": parsed.get("stresses_kbar", []), + "total_forces_ry_bohr": parsed.get("total_forces_ry_bohr", []), + "atomic_forces_ry_bohr": parsed.get("atomic_forces_ry_bohr", []), + "cell_blocks": parsed.get("cell_blocks", []), + "position_blocks": parsed.get("position_blocks", []), + }, + } + + +def inherited_elastic_parameters( + elastic_inputs: dict[str, Any], + upstream_inputs: dict[str, Any], + upstream_outputs: dict[str, Any], +) -> dict[str, Any]: + if elastic_inputs["mp_id"] != upstream_inputs["mp_id"]: + raise QePreparationError("ElasticDFT mp_id does not match its CellRelaxDFT run") + if not upstream_outputs.get("convergence") or not upstream_outputs.get( + "structure_dict" + ): + raise QePreparationError("CellRelaxDFT upstream run is not converged") + merged = dict(upstream_inputs) + merged["structure_dict"] = upstream_outputs["structure_dict"] + return merged + + +ELASTIC_KPOINT_SPACING = {True: 0.16, False: 0.22} # 1/angstrom, by is_metal + +# Parent settings an ElasticDFT strain member reuses unchanged. ElasticDFT has +# no override for them: different values require a new parent relaxation. +ELASTIC_INHERITED_KEYS = ( + "KE_cutoff", "scf_conv", "energy_conv", "force_conv", "e_steps", "i_steps", + "is_metal", "spin_polar", "hubbard_U_on", "vdw_corr", "exchange_correlation", "pseudo", +) + + +def elastic_kmesh(lattice_abc: Sequence[float], is_metal: bool) -> list[int]: + """K-mesh ElasticDFT regenerates from the relaxed parent lattice.""" + + spacing = ELASTIC_KPOINT_SPACING[bool(is_metal)] + return [max(round((2 * math.pi / float(length)) / spacing), 1) for length in lattice_abc] + + +def elastic_inherited_settings( + upstream_inputs: dict[str, Any], upstream_outputs: dict[str, Any] +) -> dict[str, Any]: + """Public summary of what strain members inherit; no structure data.""" + + settings = {key: upstream_inputs[key] for key in ELASTIC_INHERITED_KEYS if key in upstream_inputs} + abc = upstream_outputs.get("lattice_parameters") + if isinstance(abc, list) and len(abc) == 3 and "is_metal" in upstream_inputs: + try: + settings["kmesh"] = elastic_kmesh(abc, upstream_inputs["is_metal"]) + settings["kpoint_spacing_inverse_angstrom"] = ELASTIC_KPOINT_SPACING[bool(upstream_inputs["is_metal"])] + except (TypeError, ValueError, ZeroDivisionError): + pass + return settings + + +def _elastic_strains(inputs: dict[str, Any]): + Strain = _pymatgen()["Strain"] + strains = [] + for index in [(0, 0), (1, 1), (2, 2)]: + strains.extend( + Strain.from_index_amount(index, amount) + for amount in inputs["diagonal_strain_magnitudes"] + ) + for index in [(0, 1), (0, 2), (1, 2)]: + strains.extend( + Strain.from_index_amount(index, amount) + for amount in inputs["offdiagonal_strain_magnitudes"] + ) + return strains + + +def prepare_elastic( + elastic_inputs: dict[str, Any], + upstream_inputs: dict[str, Any], + upstream_outputs: dict[str, Any], + workdir: Path, + pseudo_root: str, +) -> PreparedQeRun: + workdir = Path(workdir) + workdir.mkdir(parents=True, exist_ok=True) + inherited = inherited_elastic_parameters( + elastic_inputs, upstream_inputs, upstream_outputs + ) + structure = _structure_from_dict(inherited["structure_dict"]) + k_x, k_y, k_z = elastic_kmesh(structure.lattice.abc, inherited["is_metal"]) + inherited.update({"k_x": k_x, "k_y": k_y, "k_z": k_z}) + + package_root = workdir / "elastic_inputs" + package_root.mkdir(parents=True) + pseudos = required_pseudopotentials( + structure, + pseudo_root, + inherited["pseudo"], + inherited["exchange_correlation"], + ) + pseudo_directory = _pseudo_directory( + pseudo_root, inherited["pseudo"], inherited["exchange_correlation"] + ) + + strains = _elastic_strains(elastic_inputs) + deformed_structures = [] + for index, strain in enumerate(strains): + deformed = strain.get_deformation_matrix().apply_to_structure(structure) + deformed_structures.append(deformed.as_dict()) + directory = package_root / f"strain_{index:03d}" + directory.mkdir() + _write_pw_input( + directory / "relax.in", + deformed, + inherited, + calculation="relax", + pseudo_dir=pseudo_directory, + ) + + manifest = { + "strain_matrices": [strain.tolist() for strain in strains], + "deformed_structures": deformed_structures, + "count": len(strains), + "inherited_inputs": inherited, + "external_pressure": upstream_outputs["pressure"], + } + (package_root / "manifest.json").write_text( + json.dumps(manifest, sort_keys=True), encoding="utf-8" + ) + archive = workdir / "elastic-inputs.tar.gz" + with tarfile.open(archive, "w:gz") as handle: + handle.add(package_root, arcname="elastic_inputs") + + concurrency = min(4, len(strains)) + expected = [ + f"task_{index:03d}/elastic_inputs/strain_{index:03d}/relax.stdout" + for index in range(len(strains)) + ] + return PreparedQeRun( + tool_name="elasticdft", + local_workdir=workdir, + stage_files=[archive], + expected_artifacts=expected, + pseudopotentials=pseudos, + job_array=f"0-{len(strains) - 1}%{concurrency}", + ntasks=64, + time_limit="08:00:00", + metadata=manifest, + ) + + +def elastic_outputs(prepared: PreparedQeRun) -> dict[str, Any]: + parsed_outputs = { + relative: parse_qe_output( + prepared.local_workdir / relative, require_stress=True + ) + for relative in prepared.expected_artifacts + } + return elastic_outputs_from_parsed(prepared, parsed_outputs) + + +def elastic_outputs_from_parsed( + prepared: PreparedQeRun, parsed_outputs: dict[str, dict[str, Any]] +) -> dict[str, Any]: + metadata = prepared.metadata or {} + strain_matrices = metadata["strain_matrices"] + structures = metadata["deformed_structures"] + inherited = metadata["inherited_inputs"] + original = _structure_from_dict(inherited["structure_dict"]) + stresses = [] + energies = [] + member_statuses = [] + configurations = [] + pmg = _pymatgen() + for index, relative in enumerate(prepared.expected_artifacts): + try: + parsed = parsed_outputs[relative] + except KeyError as exc: + raise QeParseError(f"ElasticDFT output is missing: {relative}") from exc + member_statuses.append( + { + "artifact": relative, + "process_completed": parsed["process_completed"], + "convergence": parsed["convergence"], + "scientific_status": parsed["scientific_status"], + "failure_reasons": parsed["failure_reasons"], + "warnings": parsed["warnings"], + "warning_count": parsed["warning_count"], + "energy_history_ry": parsed.get("ionic_energies_ry", []), + "stress_history_kbar": parsed.get("stresses_kbar", []), + "atomic_force_history_ry_bohr": parsed.get( + "atomic_forces_ry_bohr", [] + ), + "cell_blocks": parsed.get("cell_blocks", []), + "position_blocks": parsed.get("position_blocks", []), + } + ) + if not parsed["convergence"]: + reasons = "; ".join(parsed["failure_reasons"]) or "no convergence marker" + raise QeParseError( + f"ElasticDFT member did not converge: {relative}: {reasons}" + ) + stresses.append( + pmg["Stress"]( + [ + [-value / 10.0 for value in row] + for row in parsed["stresses_kbar"][-1] + ] + ) + ) + n_units = original.composition.get_integer_formula_and_factor()[1] + energies.append(parsed["ionic_energies_ry"][-1] * RY_TO_EV / n_units) + # Fixed-cell relax normally prints final positions but no cell card. + # Pair its final energy/forces with those positions and the exact + # submitted strained lattice, not the unrelaxed input coordinates. + deformed = _structure_from_dict(structures[index]) + final = _structure_from_parsed({ + **parsed, + "cell_blocks": parsed.get("cell_blocks") or [{ + "unit": "angstrom", "matrix": deformed.lattice.matrix.tolist(), + }], + }) + forces = parsed.get("atomic_forces_ry_bohr") or [] + final_forces = forces[-1] if forces else [] + configurations.append({ + "artifact": relative, + "strain_matrix": strain_matrices[index], + "input_structure": structures[index], + "relaxed_structure": final.as_dict() if final is not None else None, + "energy_ev_per_cell": parsed["ionic_energies_ry"][-1] * RY_TO_EV, + "forces_ev_per_angstrom": [[v * RY_BOHR_TO_EV_ANGSTROM for v in row] for row in final_forces], + "stress_gpa_tension_positive": stresses[-1].tolist(), + "complete_structure_force_pair": final is not None and len(final_forces) == len(final), + }) + + strain_objects = [pmg["Strain"](matrix) for matrix in strain_matrices] + tensor = pmg["ElasticTensor"].from_pseudoinverse( + strains=strain_objects, stresses=stresses + ) + voigt = tensor.voigt.tolist() + result: dict[str, Any] = { + "external_pressure": metadata["external_pressure"], + "strain_matrices": strain_matrices, + "stress_tensors": [stress.tolist() for stress in stresses], + "deformed_structures": structures, + "energies": energies, + "elastic_tensor": voigt, + "bulk_modulus_reuss": float(tensor.k_reuss), + "bulk_modulus_voigt": float(tensor.k_voigt), + "bulk_modulus_vrh": float(tensor.k_vrh), + "shear_modulus_reuss": float(tensor.g_reuss), + "shear_modulus_voigt": float(tensor.g_voigt), + "shear_modulus_vrh": float(tensor.g_vrh), + "homogeneous_poisson": float(tensor.homogeneous_poisson), + "youngs_modulus": float(tensor.y_mod * 1e-9), + "calc_params": { + "pseudopotential": inherited["pseudo"], + "exchange_correlation": inherited["exchange_correlation"], + "vdw_correction": inherited["vdw_corr"], + "kmesh": [inherited["k_x"], inherited["k_y"], inherited["k_z"]], + "kinetic_energy_cutoff": inherited["KE_cutoff"], + }, + "chemical_system": original.chemical_system, + "composition": original.composition.as_dict(), + "formula_unit": original.composition.reduced_formula, + "process_completed": all( + member["process_completed"] for member in member_statuses + ), + "convergence": all(member["convergence"] for member in member_statuses), + "member_statuses": member_statuses, + "configurations": configurations, + "units": { + "elastic_tensor": "GPa; Voigt order xx, yy, zz, yz, xz, xy; engineering shear strain", + "stress_tensors": "GPa, tension positive (sign-reversed from QE)", + "strain_matrices": "dimensionless Green-Lagrange strain", + "energies": "eV per reduced formula unit", + "external_pressure": "GPa, compression positive", + "bulk_moduli_shear_moduli_youngs_modulus": "GPa", + }, + } + for i in range(6): + for j in range(i, 6): + result[f"C{i + 1}{j + 1}"] = float(voigt[i][j]) + return result diff --git a/api/computations/rcac.py b/api/computations/rcac.py new file mode 100644 index 0000000..d57f5dc --- /dev/null +++ b/api/computations/rcac.py @@ -0,0 +1,198 @@ +"""Validated RCAC/Globus Compute execution-backend configuration. + +The application never accepts arbitrary hosts, module names, paths, accounts, +or scheduler settings from an HTTP request. Operators configure a small +allow-list of service-account endpoints and the API exposes only their public +scientific/runtime metadata. +""" + +from __future__ import annotations + +import json +import os +import re +from dataclasses import dataclass, field +from pathlib import PurePosixPath +from typing import Any +from uuid import UUID + +from .resources import ResourcePolicy +from .schemas import TOOL_DEFINITIONS +from .slurm import SlurmJobSpec, checked_slurm_value, validate_remote_path + + +_SAFE_CLUSTER_NAME = re.compile(r"^[A-Za-z0-9_-]+$") +_SAFE_MODULE = re.compile(r"^[A-Za-z0-9_.+/-]+$") + + +@dataclass(frozen=True) +class ClusterConfig: + name: str + display_name: str + host: str + endpoint_id: str + gateway_function_id: str + qe_version: str + qe_module: str + remote_root: str + pseudo_root: str + artifact_root: str + allocation: str = "ncn" + partition: str = "cpu" + qos: str = "normal" + mpi_launcher: str = "srun" + module_dependencies: tuple[str, ...] = () + resource_policy: ResourcePolicy = field(default_factory=ResourcePolicy) + + @property + def modules(self) -> tuple[str, ...]: + return (*self.module_dependencies, self.qe_module) + + @property + def gateway_protocol(self) -> int: + # Set together with gateway_function_id when a new gateway is + # registered; protocol 1 keeps the original fixed-resource contract. + return self.resource_policy.gateway_protocol + + @property + def installation_id(self) -> str: + return f"digital_qe_{self.qe_version.replace('.', '_')}_{self.name}" + + def workdir(self, run_id: str) -> str: + UUID(run_id) + return validate_remote_path(str(PurePosixPath(self.remote_root) / run_id)) + + def public_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "display_name": self.display_name, + "host": self.host, + "qe_version": self.qe_version, + "qe_module": self.qe_module, + "module_dependencies": list(self.module_dependencies), + "allocation": self.allocation, + "partition": self.partition, + "qos": self.qos, + "mpi_launcher": self.mpi_launcher, + "installation_id": self.installation_id, + "execution_backend": "globus_compute_service_account", + "gateway_protocol": self.gateway_protocol, + "resource_policy": { + name: self.resource_policy.public(name) for name in TOOL_DEFINITIONS + }, + } + + def job_spec( + self, prepared: Any, run_id: str, resources: dict[str, Any] | None = None + ) -> SlurmJobSpec: + """Ranks, walltime and array come from ``prepared`` (see apply_resources).""" + + return SlurmJobSpec( + run_id=run_id, + tool_name=prepared.tool_name, + workdir=self.workdir(run_id), + account=self.allocation, + partition=self.partition, + qos=self.qos, + time_limit=prepared.time_limit, + ntasks=prepared.ntasks, + modules=self.modules, + mpi_launcher=self.mpi_launcher, + array=prepared.job_array, + environment={"QE_DISABLE_GGA_PBE": "0"}, + mem_per_cpu_mb=(resources or {}).get("mem_per_cpu_mb"), + ) + + +def _uuid(label: str, value: Any) -> str: + try: + return str(UUID(str(value))) + except (TypeError, ValueError, AttributeError) as exc: + raise RuntimeError(f"{label} must be a UUID") from exc + + +def _absolute_path(label: str, value: Any) -> str: + try: + return validate_remote_path(str(value)) + except ValueError as exc: + raise RuntimeError(f"{label} must be a safe absolute path") from exc + + +def configured_clusters() -> dict[str, ClusterConfig]: + raw = os.getenv("RCAC_CLUSTERS_JSON", "").strip() + if not raw: + return {} + try: + values = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError("RCAC_CLUSTERS_JSON is invalid JSON") from exc + if not isinstance(values, dict): + raise TypeError("RCAC_CLUSTERS_JSON must be an object") + + default_function_id = os.getenv( + "GLOBUS_COMPUTE_GATEWAY_FUNCTION_ID", "" + ).strip() + clusters: dict[str, ClusterConfig] = {} + for name, value in values.items(): + if not _SAFE_CLUSTER_NAME.fullmatch(str(name)) or not isinstance(value, dict): + raise RuntimeError("RCAC cluster names must be safe identifiers") + try: + dependencies = value.get("module_dependencies", []) + if not isinstance(dependencies, list) or not all( + isinstance(module, str) and _SAFE_MODULE.fullmatch(module) + for module in dependencies + ): + raise TypeError("module_dependencies must be a list of module names") + qe_module = str(value["qe_module"]) + if not _SAFE_MODULE.fullmatch(qe_module): + raise ValueError("unsafe qe_module") + cluster = ClusterConfig( + name=str(name), + display_name=str(value.get("display_name", str(name).title())), + host=str(value["host"]), + endpoint_id=_uuid( + f"RCAC cluster {name!r} endpoint_id", value["endpoint_id"] + ), + gateway_function_id=_uuid( + f"RCAC cluster {name!r} gateway_function_id", + value.get("gateway_function_id") or default_function_id, + ), + qe_version=str(value["qe_version"]), + qe_module=qe_module, + remote_root=_absolute_path( + f"RCAC cluster {name!r} remote_root", value["remote_root"] + ), + pseudo_root=_absolute_path( + f"RCAC cluster {name!r} pseudo_root", value["pseudo_root"] + ), + artifact_root=_absolute_path( + f"RCAC cluster {name!r} artifact_root", value["artifact_root"] + ), + allocation=checked_slurm_value( + "account", str(value.get("allocation", "ncn")) + ), + partition=checked_slurm_value( + "partition", str(value.get("partition", "cpu")) + ), + qos=checked_slurm_value("QoS", str(value.get("qos", "normal"))), + mpi_launcher=str(value.get("mpi_launcher", "srun")), + module_dependencies=tuple(dependencies), + resource_policy=ResourcePolicy.from_config( + value.get("gateway_protocol", 1), value.get("resources") + ), + ) + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError(f"RCAC cluster {name!r} is incomplete or invalid") from exc + if cluster.mpi_launcher not in {"srun", "mpirun"}: + raise RuntimeError(f"RCAC cluster {name!r} has an unsupported launcher") + if not cluster.host or any(character.isspace() for character in cluster.host): + raise RuntimeError(f"RCAC cluster {name!r} has an invalid host") + clusters[cluster.name] = cluster + return clusters + + +def get_cluster(name: str) -> ClusterConfig: + try: + return configured_clusters()[name] + except KeyError as exc: + raise ValueError(f"RCAC cluster is not configured: {name}") from exc diff --git a/api/computations/resources.py b/api/computations/resources.py new file mode 100644 index 0000000..7b5219a --- /dev/null +++ b/api/computations/resources.py @@ -0,0 +1,307 @@ +"""Bounded execution-resource requests for the registered QE workflows. + +Resources are execution settings, not scientific inputs: they never enter the +input fingerprint, so identical scientific inputs can reuse a cached result +whatever resources were requested. A request is validated against the +cluster's configured policy by the API and again by the worker, by +``SlurmJobSpec``, and finally by the registered RCAC gateway. The gateway is +serialized without this package, so it repeats the hard ceilings below; a test +keeps the two copies equal. + +No free-form scheduler text is accepted anywhere: every field is a bounded +integer, and account, partition, QoS and modules stay endpoint-fixed. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, replace +from typing import Any + +from pydantic import BaseModel, ConfigDict, StrictInt + +# Hard ceilings. Raising any of them needs an allocation review and a +# coordinated gateway re-registration (the gateway repeats these values). +NODES = 1 +CPUS_PER_TASK = 1 +MAX_NTASKS = 64 +MAX_CONCURRENT_MEMBERS = 4 +MIN_WALLTIME_MINUTES = 10 +# The protocol-1 gateway accepted exactly these walltimes. +MAX_WALLTIME_MINUTES = {"cellrelaxdft": 1440, "cellrelaxdft2d": 1440, "elasticdft": 480} +MIN_MEM_PER_CPU_MB = 512 +MAX_MEM_PER_CPU_MB = 4096 + +# First gateway protocol accepting walltime/memory requests (action +# ``submit_v2``) and returning failure evidence from ``collect``. Ranks and +# elastic array concurrency were already validated by protocol 1 (1..64 tasks, +# ``%1-4``), so they are adjustable on either protocol. +RESOURCE_GATEWAY_PROTOCOL = 2 +SUPPORTED_GATEWAY_PROTOCOLS = (1, 2) + +RESOURCE_REQUEST_SCHEMA = "dt-resource-request/1" + + +class ResourcePolicyError(ValueError): + """A resource request is outside the configured pilot policy.""" + + +class ResourceRequest(BaseModel): + """User-entered resources. Omitted fields take the cluster default.""" + + model_config = ConfigDict(extra="forbid") + + ntasks: StrictInt | None = None + nodes: StrictInt | None = None + mem_per_cpu_mb: StrictInt | None = None + time_limit_minutes: StrictInt | None = None + max_concurrent_members: StrictInt | None = None + + +def format_time_limit(minutes: int) -> str: + return f"{minutes // 60:02d}:{minutes % 60:02d}:00" + + +def time_limit_minutes(value: str) -> int: + match = re.fullmatch(r"([0-9]{2}):([0-5][0-9]):00", str(value)) + if not match: + raise ValueError("Slurm time limit must be HH:MM:00") + return int(match.group(1)) * 60 + int(match.group(2)) + + +def _bounded_int(label: str, value: Any, low: int, high: int) -> int: + if type(value) is not int or not low <= value <= high: + raise ValueError(f"{label} must be an integer between {low} and {high}") + return value + + +@dataclass(frozen=True) +class ResourcePolicy: + """Operator-configured per-cluster limits, never above the hard ceilings.""" + + gateway_protocol: int = 1 + max_ntasks: int = MAX_NTASKS + # 0 means explicit memory requests are not offered on this cluster; jobs + # then use the partition's default memory per CPU, as before. + max_mem_per_cpu_mb: int = 0 + max_concurrent_members: int = MAX_CONCURRENT_MEMBERS + # Tuples keep the frozen dataclass hashable. + max_walltime: tuple[tuple[str, int], ...] = tuple(MAX_WALLTIME_MINUTES.items()) + + @classmethod + def from_config(cls, gateway_protocol: Any = 1, value: Any = None) -> ResourcePolicy: + protocol = gateway_protocol + if type(protocol) is not int or protocol not in SUPPORTED_GATEWAY_PROTOCOLS: + raise ValueError("gateway_protocol must be 1 or 2") + value = {} if value is None else value + if not isinstance(value, dict): + raise TypeError("resources must be an object") + unknown = set(value) - { + "max_ntasks", "max_mem_per_cpu_mb", "max_concurrent_members", "max_walltime_minutes", + } + if unknown: + raise ValueError(f"unsupported resource settings: {sorted(unknown)}") + max_ntasks = _bounded_int("max_ntasks", value.get("max_ntasks", MAX_NTASKS), 1, MAX_NTASKS) + memory = value.get("max_mem_per_cpu_mb", 0) + if memory != 0: + memory = _bounded_int("max_mem_per_cpu_mb", memory, MIN_MEM_PER_CPU_MB, MAX_MEM_PER_CPU_MB) + if protocol < RESOURCE_GATEWAY_PROTOCOL: + raise ValueError("max_mem_per_cpu_mb requires gateway_protocol 2") + concurrency = _bounded_int( + "max_concurrent_members", value.get("max_concurrent_members", MAX_CONCURRENT_MEMBERS), + 1, MAX_CONCURRENT_MEMBERS, + ) + walltime = dict(MAX_WALLTIME_MINUTES) + configured = value.get("max_walltime_minutes", {}) + if not isinstance(configured, dict) or set(configured) - set(walltime): + raise ValueError("max_walltime_minutes must map supported tools to minutes") + for tool, minutes in configured.items(): + walltime[tool] = _bounded_int( + f"max_walltime_minutes.{tool}", minutes, MIN_WALLTIME_MINUTES, MAX_WALLTIME_MINUTES[tool] + ) + if protocol < RESOURCE_GATEWAY_PROTOCOL and walltime != MAX_WALLTIME_MINUTES: + # The protocol-1 gateway accepts only the fixed walltimes. + raise ValueError("lower walltime ceilings require gateway_protocol 2") + return cls( + gateway_protocol=protocol, max_ntasks=max_ntasks, max_mem_per_cpu_mb=memory, + max_concurrent_members=concurrency, max_walltime=tuple(walltime.items()), + ) + + @property + def walltime_adjustable(self) -> bool: + return self.gateway_protocol >= RESOURCE_GATEWAY_PROTOCOL + + @property + def memory_adjustable(self) -> bool: + return self.gateway_protocol >= RESOURCE_GATEWAY_PROTOCOL and self.max_mem_per_cpu_mb > 0 + + def max_walltime_minutes(self, tool_name: str) -> int: + try: + return dict(self.max_walltime)[tool_name] + except KeyError as exc: + raise ResourcePolicyError("unsupported computation tool") from exc + + def resolve(self, tool_name: str, request: Any = None) -> dict[str, Any]: + """Return the complete, validated request with defaults filled in.""" + + if isinstance(request, ResourceRequest): + model = request + else: + try: + model = ResourceRequest.model_validate(request or {}) + except ValueError as exc: + raise ResourcePolicyError("Resource request is malformed") from exc + values = model.model_dump(exclude_none=True) + max_time = self.max_walltime_minutes(tool_name) + nodes = values.get("nodes", NODES) + if nodes != NODES: + raise ResourcePolicyError( + "This pilot runs each job on exactly one node; multi-node requests are not supported" + ) + ntasks = values.get("ntasks", self.max_ntasks) + if not 1 <= ntasks <= self.max_ntasks: + raise ResourcePolicyError(f"MPI ranks must be between 1 and {self.max_ntasks}") + minutes = values.get("time_limit_minutes", max_time) + if minutes != max_time and not self.walltime_adjustable: + raise ResourcePolicyError( + "Walltime selection is not enabled on this cluster yet; the fixed limit applies" + ) + if not MIN_WALLTIME_MINUTES <= minutes <= max_time: + raise ResourcePolicyError( + f"Walltime must be between {MIN_WALLTIME_MINUTES} and {max_time} minutes" + ) + memory = values.get("mem_per_cpu_mb") + if memory is not None: + if not self.memory_adjustable: + raise ResourcePolicyError( + "Memory requests are not enabled on this cluster; the partition default applies" + ) + if not MIN_MEM_PER_CPU_MB <= memory <= self.max_mem_per_cpu_mb: + raise ResourcePolicyError( + f"Memory per CPU core must be between {MIN_MEM_PER_CPU_MB} and " + f"{self.max_mem_per_cpu_mb} MB" + ) + resolved: dict[str, Any] = { + "nodes": NODES, + "ntasks": ntasks, + "cpus_per_task": CPUS_PER_TASK, + "mem_per_cpu_mb": memory, + "time_limit_minutes": minutes, + } + concurrency = values.get("max_concurrent_members") + if tool_name == "elasticdft": + concurrency = self.max_concurrent_members if concurrency is None else concurrency + if not 1 <= concurrency <= self.max_concurrent_members: + raise ResourcePolicyError( + f"Concurrent strain members must be between 1 and {self.max_concurrent_members}" + ) + resolved["max_concurrent_members"] = concurrency + elif concurrency is not None: + raise ResourcePolicyError("Concurrent strain members apply only to ElasticDFT") + return resolved + + def record(self, tool_name: str, request: ResourceRequest | dict[str, Any] | None) -> dict[str, Any]: + """Stored API request: what the user set and what it resolved to.""" + + model = request if isinstance(request, ResourceRequest) else ResourceRequest.model_validate(request or {}) + return { + "schema": RESOURCE_REQUEST_SCHEMA, + "requested": model.model_dump(exclude_none=True), + "resolved": self.resolve(tool_name, model), + } + + def public(self, tool_name: str) -> dict[str, Any]: + defaults = self.resolve(tool_name) + per = "per strain member" if tool_name == "elasticdft" else "per job" + # Student-facing text; protocol details stay in operator notes. + walltime_reason = "" if self.walltime_adjustable else ( + "This cluster does not offer a different time limit yet." + ) + if self.memory_adjustable: + memory_reason = "Leave blank to use the cluster's default memory per core." + else: + memory_reason = "Uses the cluster's default memory per core; this cluster does not offer a memory choice." + subject = "Each strain calculation" if tool_name == "elasticdft" else "Each calculation" + policy: dict[str, Any] = { + "gateway_protocol": self.gateway_protocol, + "cpus_per_task": CPUS_PER_TASK, + "nodes": { + "min": NODES, "max": NODES, "default": NODES, "adjustable": False, + "unit": "node", "reason": f"{subject} runs on one node in this pilot.", + }, + "ntasks": { + "min": 1, "max": self.max_ntasks, "default": defaults["ntasks"], "adjustable": True, + "unit": f"MPI ranks {per}, one CPU core each", "reason": "", + }, + "mem_per_cpu_mb": { + "min": MIN_MEM_PER_CPU_MB if self.memory_adjustable else None, + "max": self.max_mem_per_cpu_mb if self.memory_adjustable else None, + "default": None, "adjustable": self.memory_adjustable, + "unit": "MB per CPU core", "reason": memory_reason, + }, + "time_limit_minutes": { + "min": MIN_WALLTIME_MINUTES if self.walltime_adjustable else defaults["time_limit_minutes"], + "max": defaults["time_limit_minutes"], "default": defaults["time_limit_minutes"], + "adjustable": self.walltime_adjustable, "unit": f"minutes {per}", "reason": walltime_reason, + }, + } + if tool_name == "elasticdft": + policy["max_concurrent_members"] = { + "min": 1, "max": self.max_concurrent_members, + "default": defaults["max_concurrent_members"], "adjustable": True, + "unit": "strain members running at once", "reason": "", + } + return policy + + +def stored_request(record: Any) -> dict[str, Any]: + """User-set fields from a stored record; legacy rows mean all defaults.""" + + if not isinstance(record, dict): + return {} + requested = record.get("requested", {}) + return requested if isinstance(requested, dict) else {} + + +def apply_resources(prepared: Any, resources: dict[str, Any]) -> Any: + """Return the prepared run with the validated execution request applied.""" + + values: dict[str, Any] = { + "ntasks": resources["ntasks"], + "time_limit": format_time_limit(resources["time_limit_minutes"]), + } + if prepared.job_array: + count = int((prepared.metadata or {})["count"]) + concurrency = min(int(resources.get("max_concurrent_members", MAX_CONCURRENT_MEMBERS)), count) + values["job_array"] = f"0-{count - 1}%{concurrency}" + return replace(prepared, **values) + + +def gateway_policy_differences(policy: ResourcePolicy, capabilities: dict[str, Any]) -> list[str]: + """Rollout check: the DT policy must fit inside what the endpoint accepts.""" + + problems = [] + protocol = capabilities.get("protocol") + if type(protocol) is not int or protocol < policy.gateway_protocol: + problems.append(f"gateway protocol {protocol!r} is older than configured {policy.gateway_protocol}") + if capabilities.get("nodes") != NODES or capabilities.get("cpus_per_task") != CPUS_PER_TASK: + problems.append("gateway node/CPU-per-rank contract differs") + if not isinstance(capabilities.get("max_ntasks"), int) or capabilities["max_ntasks"] < policy.max_ntasks: + problems.append("gateway accepts fewer MPI ranks than the DT policy offers") + if not isinstance(capabilities.get("max_concurrent_members"), int) or ( + capabilities["max_concurrent_members"] < policy.max_concurrent_members + ): + problems.append("gateway accepts fewer concurrent strain members than the DT policy offers") + gateway_walltime = capabilities.get("max_walltime_minutes") or {} + for tool, minutes in policy.max_walltime: + if not isinstance(gateway_walltime.get(tool), int) or gateway_walltime[tool] < minutes: + problems.append(f"gateway walltime ceiling for {tool} is below the DT policy") + if policy.walltime_adjustable and capabilities.get("min_walltime_minutes") != MIN_WALLTIME_MINUTES: + problems.append("gateway minimum walltime differs") + if policy.memory_adjustable: + ceiling = capabilities.get("max_mem_per_cpu_mb") + if not isinstance(ceiling, int) or ceiling < policy.max_mem_per_cpu_mb: + problems.append("endpoint DT_MAX_MEM_PER_CPU_MB is unset or below the DT memory ceiling") + if capabilities.get("min_mem_per_cpu_mb") != MIN_MEM_PER_CPU_MB: + problems.append("gateway minimum memory per CPU differs") + return problems diff --git a/api/computations/schemas.py b/api/computations/schemas.py new file mode 100644 index 0000000..6feca6f --- /dev/null +++ b/api/computations/schemas.py @@ -0,0 +1,316 @@ +"""Stable input contracts derived from CellRelaxDFT r99 and ElasticDFT r27.""" + +from __future__ import annotations + +import codecs +import hashlib +import json +import math +from typing import Any, Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class CellRelaxInputs(BaseModel): + """Inputs retained from the canonical CellRelaxDFT r99 workflow.""" + + model_config = ConfigDict(extra="forbid") + + mp_id: str = Field(pattern=r"^mp-[A-Za-z0-9-]+$") + structure_dict: dict[str, Any] = Field(default_factory=dict) + is_metal: bool = True + bulk_m: float = Field(default=2000.0, ge=1, le=6000) + KE_cutoff: float = Field(default=50.0, ge=10, le=300) + k_x: int = Field(default=1, ge=1, le=30) + k_y: int = Field(default=1, ge=1, le=30) + k_z: int = Field(default=1, ge=1, le=30) + i_steps: int = Field(default=50, ge=1, le=1000) + e_steps: int = Field(default=100, ge=100, le=1000) + scf_conv: float = Field(default=1e-10, gt=0) + energy_conv: float = Field(default=1e-10, ge=1e-15, le=1e-4) + force_conv: float = Field(default=1e-5, ge=1e-8, le=1e-3) + pressure_conv: float = Field(default=0.5, ge=0.01, le=10) + target_pressure: float = Field(default=0.0, ge=-100, le=10000) + vdw_corr: Literal["grimme-d2", "grimme-d3", "none"] = "none" + spin_polar: bool = False + hubbard_U_on: bool = False + hubbard_projector: Literal[ + "atomic", "ortho-atomic", "norm-atomic", "wf", "pseudo" + ] = "ortho-atomic" + hubbard_U_values: dict[str, float] = Field(default_factory=dict) + hubbard_U_orbitals: dict[str, str] = Field(default_factory=dict) + ibrav: int = 0 + exchange_correlation: Literal["LDA", "PBE", "PBEsol"] = "PBEsol" + # The source offers NC, but the r27 bundle contains only PAW and USPP. + pseudo: Literal["PAW", "USPP"] = "PAW" + + @field_validator("hubbard_U_orbitals") + @classmethod + def orbitals_are_safe(cls, value: dict[str, str]) -> dict[str, str]: + for orbital in value.values(): + if not orbital or not orbital.replace("-", "").isalnum(): + raise ValueError( + "Hubbard orbitals must contain letters, digits, or '-'" + ) + return value + + def validate_cross_fields(self) -> CellRelaxInputs: + if self.hubbard_U_on: + if not self.hubbard_U_values or not self.hubbard_U_orbitals: + raise ValueError( + "Hubbard U values and orbitals are required when hubbard_U_on is true" + ) + if set(self.hubbard_U_values) != set(self.hubbard_U_orbitals): + raise ValueError("Hubbard U value and orbital elements must match") + return self + + +class CellRelax2DInputs(CellRelaxInputs): + """Justin's monolayer workflow; distinct from bulk relaxation/elasticity.""" + + vacuum_angstrom: float = Field(default=20.0, ge=10, le=100, + description="Empty space between periodic monolayer images, in angstrom.") + degauss_ry: float = Field(default=0.002, gt=0, le=0.1) + k_x: int = Field(default=8, ge=1, le=30) + k_y: int = Field(default=8, ge=1, le=30) + k_z: Literal[1] = 1 + ibrav: Literal[0] = 0 + target_pressure: Literal[0] = 0 + KE_cutoff: float = Field(default=180, ge=10, le=300) + i_steps: int = Field(default=100, ge=1, le=1000) + e_steps: int = Field(default=200, ge=100, le=1000) + scf_conv: float = Field(default=1e-8, gt=0) + energy_conv: float = Field(default=1e-6, ge=1e-15, le=1e-4) + force_conv: float = Field(default=1e-4, ge=1e-8, le=1e-3) + pressure_conv: float = Field(default=5.0, ge=0.01, le=10, + description="QE pressure threshold in kbar; 5 kbar = 0.5 GPa.") + exchange_correlation: Literal["PBEsol"] = "PBEsol" + pseudo: Literal["PAW"] = "PAW" + + +class ElasticInputs(BaseModel): + """Inputs for ElasticDFT with a DT run id replacing nanoHUB squid lookup.""" + + model_config = ConfigDict(extra="forbid") + + mp_id: str = Field(pattern=r"^mp-[A-Za-z0-9-]+$") + upstream_execution_id: UUID + diagonal_strain_magnitudes: list[float] = Field( + default_factory=lambda: [-0.01, -0.005, 0.005, 0.01], + min_length=2, + max_length=12, + ) + offdiagonal_strain_magnitudes: list[float] = Field( + default_factory=lambda: [-0.01, -0.005, 0.005, 0.01], + min_length=2, + max_length=12, + ) + + @field_validator("diagonal_strain_magnitudes", "offdiagonal_strain_magnitudes") + @classmethod + def strains_are_small_and_distinct(cls, values: list[float]) -> list[float]: + values = [float(value) for value in values] + if any(value == 0 or abs(value) > 0.05 for value in values): + raise ValueError( + "strain magnitudes must be non-zero and no larger than 0.05" + ) + if len(set(values)) != len(values): + raise ValueError("strain magnitudes must be unique") + if not any(value < 0 for value in values) or not any( + value > 0 for value in values + ): + raise ValueError("strain magnitudes must include compression and tension") + return values + + +class ToolDefinition(BaseModel): + name: str + title: str + revision: str + description: str + input_model: Any + output_names: list[str] + + model_config = ConfigDict(arbitrary_types_allowed=True) + + def public_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "title": self.title, + "revision": self.revision, + "description": self.description, + "input_schema": self.input_model.model_json_schema(), + "output_names": self.output_names, + } + + +CELLRELAX_OUTPUTS = [ + "num_atoms", + "chemical_system", + "composition", + "formula_unit", + "mp_space_group", + "energy", + "atomic_forces", + "qe_space_group", + "pressure", + "stress", + "volume", + "structure_dict", + "lattice_vectors", + "lattice_parameters", + "angles", + "process_completed", + "convergence", + "scientific_status", + "failure_reasons", + "warnings", + "warning_count", + "intermediate_data", +] + +ELASTIC_OUTPUTS = [ + "external_pressure", + "strain_matrices", + "stress_tensors", + "deformed_structures", + "energies", + "elastic_tensor", + *[f"C{i}{j}" for i in range(1, 7) for j in range(i, 7)], + "bulk_modulus_reuss", + "bulk_modulus_voigt", + "bulk_modulus_vrh", + "shear_modulus_reuss", + "shear_modulus_voigt", + "shear_modulus_vrh", + "homogeneous_poisson", + "youngs_modulus", + "calc_params", + "chemical_system", + "composition", + "formula_unit", + "process_completed", + "convergence", + "member_statuses", + "configurations", + "units", +] + +TOOL_DEFINITIONS = { + "cellrelaxdft": ToolDefinition( + name="cellrelaxdft", + title="Cell Relax DFT", + revision="r99-dt2", + description=( + "Relax a crystal cell with a registered Quantum ESPRESSO installation " + "on Purdue RCAC and preserve structured provenance." + ), + input_model=CellRelaxInputs, + output_names=CELLRELAX_OUTPUTS, + ), + "elasticdft": ToolDefinition( + name="elasticdft", + title="Elastic DFT", + revision="r27-dt2", + description=( + "Fit an elastic tensor from registered, provenance-linked strained " + "Quantum ESPRESSO calculations." + ), + input_model=ElasticInputs, + output_names=ELASTIC_OUTPUTS, + ), + "cellrelaxdft2d": ToolDefinition( + name="cellrelaxdft2d", + title="Cell Relax DFT — 2D monolayer", + revision="justin-dt1", + description=( + "Extract an equivalent monolayer from a layered Materials Project structure, " + "orient its plane in XY, and relax atoms and in-plane cell at zero pressure " + "with fixed vacuum. PBEsol/PAW; not a 2D elastic tensor calculator." + ), + input_model=CellRelax2DInputs, + output_names=[*CELLRELAX_OUTPUTS, "dimensionality", "preparation", "units"], + ), +} + + +def validate_tool_inputs(tool_name: str, inputs: dict[str, Any]) -> dict[str, Any]: + try: + definition = TOOL_DEFINITIONS[tool_name] + except KeyError as exc: + raise ValueError(f"Unknown computation tool: {tool_name}") from exc + model = definition.input_model.model_validate(inputs) + if isinstance(model, CellRelaxInputs): + model.validate_cross_fields() + return model.model_dump(mode="json") + + +# Version of the DT's input -> Quantum ESPRESSO translation. It is part of +# every fingerprint so results produced by an older translation are never +# reused as cache hits. Version 2: KE_cutoff is no longer truncated to an +# integer when writing ``ecutwfc``. Bump it whenever the same validated +# inputs would produce different QE input files. +QE_INPUT_ADAPTER_VERSION = 2 + + +def compute_input_fingerprint(tool_name: str, inputs: dict[str, Any]) -> str: + """Generate a stable workflow/revision/input fingerprint for result caching.""" + + return _input_fingerprint(tool_name, inputs, QE_INPUT_ADAPTER_VERSION) + + +def _input_fingerprint( + tool_name: str, inputs: dict[str, Any], adapter_version: int | None, +) -> str: + + definition = TOOL_DEFINITIONS[tool_name] + normalized = validate_tool_inputs(tool_name, inputs) + normalized = json.loads(json.dumps(normalized, sort_keys=True)) + prologue = [f"{key} {normalized[key]}" for key in sorted(normalized)] + prologue.extend( + [ + f"simtoolName {definition.name}", + f"simtoolRevision {definition.revision}", + ] + ) + if adapter_version is not None: + prologue.append(f"dtQeInputAdapter {adapter_version}") + digest = hashlib.sha1( + memoryview(codecs.encode("\n".join(prologue), "utf-8")) + ).hexdigest() + return f"{definition.name}/{definition.revision}/{digest}" + + +def upstream_cutoff_was_truncated( + tool_name: str, + inputs: dict[str, Any], + stored_fingerprint: str, + adapter_version: Any = None, +) -> bool: + """Reject fractional upstreams without evidence of a corrected adapter. + + Runs fingerprinted before QE_INPUT_ADAPTER_VERSION 2 wrote + ``int(KE_cutoff)``, so a fractional recorded cutoff differs from what QE + actually used. Integral cutoffs are unaffected. Unknown fingerprints are + conservatively rejected. Adapter versions from 2 onward preserve cutoff + fractions, so a future bump does not invalidate a verified version-2 parent. + """ + try: + cutoff = float(inputs.get("KE_cutoff")) + except (TypeError, ValueError): + return False + if not math.isfinite(cutoff) or cutoff.is_integer(): + return False + # Worker-authored provenance survives JSONB number normalization (e.g. + # -0.0 becomes 0.0). Never take this version from user input parameters. + if type(adapter_version) is int and adapter_version >= 2: + return False + try: + corrected = { + _input_fingerprint(tool_name, inputs, version) + for version in range(2, QE_INPUT_ADAPTER_VERSION + 1) + } + except (KeyError, ValueError): + return True + return str(stored_fingerprint or "") not in corrected diff --git a/api/computations/slurm.py b/api/computations/slurm.py new file mode 100644 index 0000000..da80dcc --- /dev/null +++ b/api/computations/slurm.py @@ -0,0 +1,249 @@ +"""Digital Twin-owned Slurm contracts and state normalization. + +This module deliberately contains no transport implementation. The production +transport is Globus Compute; keeping the scheduler contract here avoids a +runtime dependency on the external Sim2L repository. +""" + +from __future__ import annotations + +import re +import shlex +from dataclasses import dataclass, field +from enum import Enum +from pathlib import PurePosixPath +from typing import Any, Iterable, Mapping, Sequence + +from .resources import ( + CPUS_PER_TASK, + MAX_CONCURRENT_MEMBERS, + MAX_MEM_PER_CPU_MB, + MAX_NTASKS, + MAX_WALLTIME_MINUTES, + MIN_MEM_PER_CPU_MB, + MIN_WALLTIME_MINUTES, + NODES, + time_limit_minutes, +) + + +_SAFE_SLURM_VALUE = re.compile(r"^[A-Za-z0-9_.:@,+%/=-]+$") +_RUN_COMMENT = re.compile( + r"^dt-run:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" +) + + +class SlurmState(str, Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + UNKNOWN = "unknown" + + +def checked_slurm_value(name: str, value: str) -> str: + value = str(value) + if not value or not _SAFE_SLURM_VALUE.fullmatch(value): + raise ValueError(f"invalid Slurm {name}") + return value + + +def validate_remote_path(path: str) -> str: + if not path or any(character in path for character in ("\x00", "\n", "\r")): + raise ValueError("remote path is empty or contains control characters") + parsed = PurePosixPath(path) + if not parsed.is_absolute() or ".." in parsed.parts: + raise ValueError("remote paths must be absolute and cannot contain '..'") + return str(parsed) + + +def normalize_state(state: str) -> SlurmState: + value = str(state).strip().upper().split("+", 1)[0].split() + value = value[0] if value else "" + if value in {"PENDING", "CONFIGURING", "REQUEUED", "RESIZING", "SUSPENDED"}: + return SlurmState.PENDING + if value in {"RUNNING", "COMPLETING", "STAGE_OUT"}: + return SlurmState.RUNNING + if value == "COMPLETED": + return SlurmState.COMPLETED + if value in {"CANCELLED", "PREEMPTED"}: + return SlurmState.CANCELLED + if value in { + "BOOT_FAIL", + "DEADLINE", + "FAILED", + "NODE_FAIL", + "OUT_OF_MEMORY", + "REVOKED", + "SPECIAL_EXIT", + "TIMEOUT", + }: + return SlurmState.FAILED + return SlurmState.UNKNOWN + + +def aggregate_states(states: Iterable[str | SlurmState]) -> SlurmState: + normalized = [ + state if isinstance(state, SlurmState) else normalize_state(state) + for state in states + ] + for state in ( + SlurmState.RUNNING, + SlurmState.PENDING, + SlurmState.UNKNOWN, + SlurmState.FAILED, + SlurmState.CANCELLED, + ): + if state in normalized: + return state + if normalized and all(state == SlurmState.COMPLETED for state in normalized): + return SlurmState.COMPLETED + return SlurmState.UNKNOWN + + +@dataclass(frozen=True) +class SlurmJobSpec: + """Validated description of one supported scientific Slurm job.""" + + run_id: str + tool_name: str + workdir: str + account: str + partition: str + qos: str + time_limit: str + ntasks: int + modules: Sequence[str] + mpi_launcher: str + array: str = "" + environment: Mapping[str, str] = field(default_factory=dict) + output_file: str = "slurm.out" + error_file: str = "slurm.err" + nodes: int = NODES + # None keeps the partition's default memory per CPU (no directive). + mem_per_cpu_mb: int | None = None + + def __post_init__(self) -> None: + import uuid + + uuid.UUID(self.run_id) + if self.tool_name not in {"cellrelaxdft", "cellrelaxdft2d", "elasticdft"}: + raise ValueError("unsupported computation tool") + validate_remote_path(self.workdir) + checked_slurm_value("account", self.account) + checked_slurm_value("partition", self.partition) + checked_slurm_value("QoS", self.qos) + checked_slurm_value("time limit", self.time_limit) + minutes = time_limit_minutes(self.time_limit) + if not MIN_WALLTIME_MINUTES <= minutes <= MAX_WALLTIME_MINUTES[self.tool_name]: + raise ValueError("Digital Twin QE walltime is outside the pilot limits") + if self.tool_name == "elasticdft": + match = re.fullmatch(r"0-([0-9]{1,3})%([0-9]+)", self.array) + if not match or not 1 <= int(match.group(2)) <= MAX_CONCURRENT_MEMBERS: + raise ValueError("ElasticDFT array specification is not allowed") + elif self.array: + raise ValueError("relaxation jobs cannot use a job array") + # Exact integers only (bool is an int subclass): a coerced value must + # never be validated while the original is rendered into a directive. + if type(self.ntasks) is not int or not 1 <= self.ntasks <= MAX_NTASKS: + raise ValueError(f"Digital Twin QE jobs must request 1..{MAX_NTASKS} tasks") + if type(self.nodes) is not int or self.nodes != NODES: + raise ValueError("Digital Twin QE jobs run on exactly one node") + if self.mem_per_cpu_mb is not None and ( + type(self.mem_per_cpu_mb) is not int + or not MIN_MEM_PER_CPU_MB <= self.mem_per_cpu_mb <= MAX_MEM_PER_CPU_MB + ): + raise ValueError("memory per CPU is outside the pilot limits") + if self.mpi_launcher not in {"srun", "mpirun"}: + raise ValueError("unsupported MPI launcher") + for module in self.modules: + checked_slurm_value("module", module) + for key in self.environment: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise ValueError(f"invalid environment variable name: {key!r}") + + @property + def comment(self) -> str: + value = f"dt-run:{self.run_id}" + if not _RUN_COMMENT.fullmatch(value): + raise ValueError("invalid Digital Twin Slurm comment") + return value + + @property + def job_name(self) -> str: + return f"dt-{self.tool_name[:10]}-{self.run_id[:8]}" + + def command(self) -> str: + if self.tool_name in {"cellrelaxdft", "cellrelaxdft2d"}: + return ( + "mkdir -p qe_tmp && " + f"{self.mpi_launcher} pw.x -in vcrelax.in > vcrelax.stdout" + ) + return "bash run-elastic-task.sh" + + def render(self) -> str: + directives = [ + "#!/bin/bash -l", + f"#SBATCH --job-name={checked_slurm_value('job name', self.job_name)}", + f"#SBATCH --account={self.account}", + f"#SBATCH --partition={self.partition}", + f"#SBATCH --qos={self.qos}", + f"#SBATCH --time={self.time_limit}", + f"#SBATCH --nodes={self.nodes}", + f"#SBATCH --ntasks={self.ntasks}", + f"#SBATCH --cpus-per-task={CPUS_PER_TASK}", + f"#SBATCH --output={checked_slurm_value('output file', self.output_file)}", + f"#SBATCH --error={checked_slurm_value('error file', self.error_file)}", + ] + if self.mem_per_cpu_mb is not None: + directives.append(f"#SBATCH --mem-per-cpu={self.mem_per_cpu_mb}M") + if self.array: + directives.append(f"#SBATCH --array={self.array}") + body = [ + "set -euo pipefail", + f"cd {shlex.quote(validate_remote_path(self.workdir))}", + "module purge", + ] + body.extend(f"module load {shlex.quote(module)}" for module in self.modules) + body.extend( + f"export {key}={shlex.quote(str(value))}" + for key, value in sorted(self.environment.items()) + ) + body.append(self.command()) + return "\n".join([*directives, "", *body]) + "\n" + + def rendered_resources(self) -> dict[str, Any]: + """Resource directives the gateway is expected to render and echo.""" + + return { + "nodes": self.nodes, + "ntasks": self.ntasks, + "cpus_per_task": CPUS_PER_TASK, + "mem_per_cpu_mb": self.mem_per_cpu_mb, + "time_limit": self.time_limit, + "array": self.array, + } + + def as_payload(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "tool_name": self.tool_name, + "workdir": self.workdir, + "account": self.account, + "partition": self.partition, + "qos": self.qos, + "time_limit": self.time_limit, + "ntasks": self.ntasks, + "nodes": self.nodes, + "cpus_per_task": CPUS_PER_TASK, + "mem_per_cpu_mb": self.mem_per_cpu_mb, + "modules": list(self.modules), + "mpi_launcher": self.mpi_launcher, + "array": self.array, + "environment": dict(self.environment), + "output_file": self.output_file, + "error_file": self.error_file, + "comment": self.comment, + "script": self.render(), + } diff --git a/api/computations/store.py b/api/computations/store.py new file mode 100644 index 0000000..b37edef --- /dev/null +++ b/api/computations/store.py @@ -0,0 +1,950 @@ +"""PostgreSQL persistence and project authorization for computation runs.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import os +from datetime import datetime +from pathlib import Path +from typing import Any + +from data_loader_pg import get_pg_migration_connection, get_pg_worker_connection +from metadata_pg import _ensure_project_access, _ensure_project_write_access +from psycopg2.extras import Json, RealDictCursor +from security import PlatformUser + +from computations.rcac import ClusterConfig +from computations.schemas import TOOL_DEFINITIONS, upstream_cutoff_was_truncated +from computations.artifacts import depot_downloads_enabled, MAX_DEPOT_DOWNLOAD_BYTES +from computations.diagnostics import build_diagnostics +from computations.qe import ELASTIC_INHERITED_KEYS, elastic_inherited_settings + +TRUNCATED_UPSTREAM_MESSAGE = ( + "The selected CellRelaxDFT run has a fractional KE_cutoff without a " + "verified corrected input adapter. Older adapters truncated this value; " + "rerun CellRelaxDFT before ElasticDFT" +) + +ACTIVE_STATUSES = { + "requested", + "validating", + "staging", + "submitted", + "running", + "collecting", + "cancelling", +} +TERMINAL_STATUSES = {"data_ingested", "cancelled", "failed"} +ALL_STATUSES = ACTIVE_STATUSES | TERMINAL_STATUSES + + +class SubmissionCapacityError(RuntimeError): + """The configured pilot submission budget is already occupied.""" + + +def _enforce_submission_budget(cur, *, user_id: str, project_id: str, cluster_name: str) -> None: + allowed = {value.strip() for value in os.getenv("RCAC_ALLOWED_PROJECT_IDS", "").split(",") if value.strip()} + if allowed and project_id not in allowed: + raise PermissionError("Computations are restricted to the approved pilot project") + per_user = int(os.getenv("RCAC_MAX_ACTIVE_RUNS_PER_USER", "0")) + per_cluster = int(os.getenv("RCAC_MAX_ACTIVE_RUNS_PER_CLUSTER", "0")) + if per_user < 0 or per_cluster < 0: + raise RuntimeError("Invalid computation capacity configuration") + if not (per_user or per_cluster): + return + # Transaction lock makes count+insert atomic across simultaneous browser + # requests. It affects only this database's simulation submission budget. + cur.execute("SELECT pg_advisory_xact_lock(783412, 1)") + cur.execute( + """SELECT COUNT(*) FILTER (WHERE requested_by = %s) AS user_count, + COUNT(*) FILTER (WHERE cluster = %s) AS cluster_count + FROM computation_runs WHERE status = ANY(%s)""", + (user_id, cluster_name, list(ACTIVE_STATUSES)), + ) + counts = cur.fetchone() + if (per_user and counts["user_count"] >= per_user) or (per_cluster and counts["cluster_count"] >= per_cluster): + raise SubmissionCapacityError("Pilot run limit reached; wait for an active run to finish") + + +def feature_enabled() -> bool: + return os.getenv("RCAC_COMPUTE_ENABLED", "false").strip().lower() in { + "1", + "true", + "yes", + "on", + } + + +def ensure_tables() -> None: + path = Path(__file__).resolve().parents[1] / "scripts" / "add_rcac_computations.sql" + sql = path.read_text(encoding="utf-8") + conn = get_pg_migration_connection() + try: + with conn.cursor() as cur: + cur.execute(sql) + conn.commit() + finally: + conn.close() + + +def _json(value: Any, default: Any) -> Any: + if value is None: + return default + if isinstance(value, str): + try: + return json.loads(value) + except json.JSONDecodeError: + return default + return value + + +def _iso(value: Any) -> str: + if isinstance(value, datetime): + return value.isoformat() + return str(value or "") + + +def elastic_parent_eligibility(row: dict[str, Any]) -> tuple[bool, str]: + """One parent check shared by the public catalog, submission and worker.""" + if row.get("tool_name") != "cellrelaxdft" or row.get("status") != "data_ingested": + return False, "ElasticDFT requires a completed 3D CellRelaxDFT run" + external = _json(row.get("external_workflow_ref_json"), {}) + if upstream_cutoff_was_truncated( + row["tool_name"], _json(row.get("input_json"), {}), + row.get("input_fingerprint") or "", external.get("qe_input_adapter_version"), + ): + return False, TRUNCATED_UPSTREAM_MESSAGE + if _json(row.get("output_json"), {}).get("convergence") is not True: + return False, "ElasticDFT requires a converged 3D CellRelaxDFT run" + return True, "" + + +def _run_diagnostics(value: dict[str, Any]) -> dict[str, Any] | None: + """Stored diagnostics, or per-member states/exit codes for older runs. + + ``value`` shares nested JSON with the caller's row, so nothing here may + mutate it: the public provenance gets a copy without the duplicate key. + """ + + external = value["external_workflow_ref"] + stored = external.get("diagnostics") + if "diagnostics" in external: + value["external_workflow_ref"] = {key: item for key, item in external.items() if key != "diagnostics"} + if isinstance(stored, dict): + return copy.deepcopy(stored) + accounting = external.get("scheduler_accounting") + if value.get("status") not in {"failed", "cancelled"} or not isinstance(accounting, list) or not accounting: + return None + try: + return build_diagnostics( + tool_name=str(value.get("tool_name") or ""), + job_id=str(value.get("scheduler_job_id") or ""), + prepared_manifest=value["prepared_manifest"], + accounting=accounting, + # Only the gateway's own lists; no file evidence was extracted. + collection={"missing_artifacts": external.get("missing_artifacts") or [], + "collection_errors": external.get("collection_errors") or []}, + outcome=str(value.get("status")), + derived=True, + ) + except Exception: # noqa: BLE001 - never break run listing + return None + + +def _inherited_settings(value: dict[str, Any]) -> dict[str, Any] | None: + """Convergence settings ElasticDFT strain members take from a parent.""" + + if value.get("tool_name") == "cellrelaxdft" and value.get("status") == "data_ingested": + return elastic_inherited_settings(value["input"], value["output"]) + if value.get("tool_name") == "elasticdft": + inherited = (value["prepared_manifest"].get("metadata") or {}).get("inherited_inputs") or {} + if not isinstance(inherited, dict) or not inherited: + return None + settings = {key: inherited[key] for key in ELASTIC_INHERITED_KEYS if key in inherited} + if all(key in inherited for key in ("k_x", "k_y", "k_z")): + settings["kmesh"] = [inherited["k_x"], inherited["k_y"], inherited["k_z"]] + return settings + return None + + +def _public_run(row: dict[str, Any], *, include_events: bool = False) -> dict[str, Any]: + value = dict(row) + value["elastic_parent_eligible"], value["elastic_parent_ineligibility_reason"] = elastic_parent_eligibility(row) + for key in ( + "input_json", + "output_json", + "artifact_manifest_json", + "external_workflow_ref_json", + "error_json", + "prepared_manifest_json", + "resource_request_json", + ): + public_key = key.removesuffix("_json") + default: Any = [] if key == "artifact_manifest_json" else {} + value[public_key] = _json(value.pop(key, None), default) + value["diagnostics"] = _run_diagnostics(value) + value["elastic_inherited_settings"] = _inherited_settings(value) + for key in ( + "created_at", + "updated_at", + "submitted_at", + "started_at", + "completed_at", + "worker_lease_expires_at", + ): + value[key] = _iso(value.get(key)) + value["id"] = str(value.get("id") or "") + value["upstream_run_id"] = str(value.get("upstream_run_id") or "") + value["artifact_manifest"] = [ + { + **artifact, + **( + { + "uri": f"/api/dt/computations/runs/{value['id']}/artifacts/{index}" + } + if ((artifact.get("downloadable", True) and "storage_uri" not in artifact) + or (depot_downloads_enabled() and str(artifact.get("storage_uri", "")).startswith("depot://") + and type(artifact.get("size_bytes")) is int + and 0 <= artifact["size_bytes"] <= MAX_DEPOT_DOWNLOAD_BYTES)) + else {"uri": ""} + ), + } + for index, artifact in enumerate(value.get("artifact_manifest", [])) + ] + value["input_fingerprint"] = value.get("input_fingerprint") or value.get( + "sim2l_squid_id", "" + ) + value.pop("worker_lease_owner", None) + if not include_events: + value.pop("events", None) + return value + + +def create_run( + *, + user: PlatformUser, + project_id: str, + tool_name: str, + tool_revision: str, + input_fingerprint: str, + inputs: dict[str, Any], + cluster: ClusterConfig, + execution_identity: str, + upstream_run_id: str = "", + resource_request: dict[str, Any] | None = None, +) -> dict[str, Any]: + """``resource_request`` is stored apart from the fingerprinted inputs.""" + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + _ensure_project_write_access(cur, project_id=project_id, user=user) + _enforce_submission_budget(cur, user_id=user.id, project_id=project_id, cluster_name=cluster.name) + _ensure_tool_catalog( + cur, + user=user, + project_id=project_id, + tool_name=tool_name, + cluster=cluster, + ) + if upstream_run_id: + cur.execute( + """ + SELECT project_id, tool_name, status, input_json, input_fingerprint, + output_json, external_workflow_ref_json + FROM computation_runs WHERE id = %s + """, + (upstream_run_id,), + ) + upstream = cur.fetchone() + if not upstream or upstream["project_id"] != project_id: + raise ValueError( + "upstream computation was not found in this project" + ) + eligible, reason = elastic_parent_eligibility(dict(upstream)) + if not eligible: + raise ValueError(reason) + cur.execute( + """ + INSERT INTO computation_runs ( + project_id, requested_by, tool_name, tool_revision, + sim2l_squid_id, input_fingerprint, upstream_run_id, cluster, partition, + qos, allocation, input_json, digital_installation_id, + execution_backend, execution_identity, slurm_comment, + resource_request_json + ) VALUES ( + %s, %s, %s, %s, %s, %s, NULLIF(%s, '')::uuid, + %s, %s, %s, %s, %s, %s, + 'globus_compute_service_account', %s, '', %s + ) + RETURNING * + """, + ( + project_id, + user.id, + tool_name, + tool_revision, + input_fingerprint, + input_fingerprint, + upstream_run_id, + cluster.name, + cluster.partition, + cluster.qos, + cluster.allocation, + Json(inputs), + cluster.installation_id, + execution_identity, + Json(resource_request or {}), + ), + ) + row = dict(cur.fetchone()) + _insert_event(cur, str(row["id"]), "requested", "Execution requested") + conn.commit() + return _public_run(row) + finally: + conn.close() + + +def list_runs(project_id: str, user: PlatformUser) -> list[dict[str, Any]]: + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + _ensure_project_access(cur, project_id=project_id, user=user) + cur.execute( + "SELECT * FROM computation_runs WHERE project_id = %s ORDER BY created_at DESC", + (project_id,), + ) + return [_public_run(dict(row)) for row in cur.fetchall()] + finally: + conn.close() + + +def get_run(run_id: str, user: PlatformUser | None = None, *, internal: bool = False): + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute("SELECT * FROM computation_runs WHERE id = %s", (run_id,)) + row = cur.fetchone() + if not row: + return None + if user is not None: + _ensure_project_access(cur, project_id=row["project_id"], user=user) + if internal: + return dict(row) + cur.execute( + """ + SELECT status, message, details_json, created_at + FROM computation_run_events WHERE run_id = %s ORDER BY created_at + """, + (run_id,), + ) + value = dict(row) + value["events"] = [ + { + "status": event["status"], + "message": event["message"], + "details": _json(event["details_json"], {}), + "created_at": _iso(event["created_at"]), + } + for event in cur.fetchall() + ] + return _public_run(value, include_events=True) + finally: + conn.close() + + +def request_cancel(run_id: str, user: PlatformUser) -> dict[str, Any] | None: + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + "SELECT * FROM computation_runs WHERE id = %s FOR UPDATE", (run_id,) + ) + row = cur.fetchone() + if not row: + return None + _ensure_project_write_access(cur, project_id=row["project_id"], user=user) + # Collection starts only after the scheduler is terminal. Repeated + # cancel clicks must not move it back to cancelling and starve + # diagnostic/result collection. Already requested cancellation is + # likewise idempotent while the worker waits for Slurm. + if (row["status"] in TERMINAL_STATUSES | {"collecting", "cancelling"} + or row.get("cancel_requested")): + return _public_run(dict(row)) + cur.execute( + """ + UPDATE computation_runs + SET cancel_requested = true, status = 'cancelling', updated_at = CURRENT_TIMESTAMP + WHERE id = %s RETURNING * + """, + (run_id,), + ) + updated = dict(cur.fetchone()) + _insert_event(cur, run_id, "cancelling", "Cancellation requested") + conn.commit() + return _public_run(updated) + finally: + conn.close() + + +def retry_collection(run_id: str, user: PlatformUser) -> dict[str, Any] | None: + """Retry result retrieval for an existing job; never submit new physics.""" + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute("SELECT * FROM computation_runs WHERE id=%s FOR UPDATE", (run_id,)) + row = cur.fetchone() + if not row: + return None + _ensure_project_write_access(cur, project_id=row["project_id"], user=user) + error = _json(row.get("error_json"), {}) + if (row["status"] != "failed" or not row["scheduler_job_id"] + or not _json(row["prepared_manifest_json"], {}) + or error.get("type") not in {"GlobusGatewayError", "GlobusTransientError"}): + raise ValueError("Only transport-failed existing jobs can retry collection") + cur.execute("""UPDATE computation_runs SET status='submitted', error_json='{}', + completed_at=NULL, worker_lease_owner='', worker_lease_expires_at=NULL, + external_workflow_ref_json=external_workflow_ref_json - 'transport_retry_count', + updated_at=CURRENT_TIMESTAMP WHERE id=%s RETURNING *""", (run_id,)) + result = dict(cur.fetchone()) + _insert_event(cur, run_id, "submitted", "Rechecking existing Slurm job to retry result retrieval; no resubmission") + conn.commit() + return _public_run(result) + finally: + conn.close() + + +def claim_next_run(worker_id: str, lease_seconds: int = 120) -> dict[str, Any] | None: + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + WITH candidate AS ( + SELECT id FROM computation_runs + WHERE status IN ( + 'requested', 'validating', 'staging', 'submitted', 'running', + 'collecting', 'cancelling' + ) + AND (worker_lease_expires_at IS NULL OR worker_lease_expires_at < CURRENT_TIMESTAMP) + ORDER BY created_at + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + UPDATE computation_runs r + SET worker_lease_owner = %s, + worker_lease_expires_at = CURRENT_TIMESTAMP + (%s * INTERVAL '1 second'), + updated_at = CURRENT_TIMESTAMP + FROM candidate + WHERE r.id = candidate.id + RETURNING r.* + """, + (worker_id, lease_seconds), + ) + row = cur.fetchone() + conn.commit() + return dict(row) if row else None + finally: + conn.close() + + +def claim_active_runs( + worker_id: str, *, lease_seconds: int = 120, limit: int = 500 +) -> list[dict[str, Any]]: + """Lease a bounded batch so each cluster needs one scheduler status query.""" + + limit = max(1, min(int(limit), 500)) + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + WITH candidates AS ( + SELECT id FROM computation_runs + WHERE status = ANY(%s) + AND (worker_lease_expires_at IS NULL + OR worker_lease_expires_at < CURRENT_TIMESTAMP) + ORDER BY created_at + FOR UPDATE SKIP LOCKED + LIMIT %s + ) + UPDATE computation_runs r + SET worker_lease_owner = %s, + worker_lease_expires_at = CURRENT_TIMESTAMP + + (%s * INTERVAL '1 second'), + updated_at = CURRENT_TIMESTAMP + FROM candidates + WHERE r.id = candidates.id + RETURNING r.* + """, + ( + list(ACTIVE_STATUSES), + limit, + worker_id, + lease_seconds, + ), + ) + rows = [dict(row) for row in cur.fetchall()] + conn.commit() + return rows + finally: + conn.close() + + +def claim_run( + run_id: str, worker_id: str, lease_seconds: int = 120 +) -> dict[str, Any] | None: + """Lease exactly one run for a credential-isolated Kubernetes worker.""" + + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + UPDATE computation_runs + SET worker_lease_owner = %s, + worker_lease_expires_at = CURRENT_TIMESTAMP + (%s * INTERVAL '1 second'), + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + AND status = ANY(%s) + AND ( + worker_lease_expires_at IS NULL + OR worker_lease_expires_at < CURRENT_TIMESTAMP + OR worker_lease_owner = %s + ) + RETURNING * + """, + (worker_id, lease_seconds, run_id, list(ACTIVE_STATUSES), worker_id), + ) + row = cur.fetchone() + conn.commit() + return dict(row) if row else None + finally: + conn.close() + + +def renew_run_claim(run_id: str, worker_id: str, lease_seconds: int) -> dict[str, Any] | None: + """Refresh a queued batch item only if this worker still owns it. + + Also reload cancellation requests received since the batch was claimed. + A stolen/finished item must never be executed from a stale batch snapshot. + """ + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """UPDATE computation_runs + SET worker_lease_expires_at = CURRENT_TIMESTAMP + (%s * INTERVAL '1 second') + WHERE id = %s AND worker_lease_owner = %s AND status = ANY(%s) + RETURNING *""", + (lease_seconds, run_id, worker_id, list(ACTIVE_STATUSES)), + ) + row = cur.fetchone() + conn.commit() + return dict(row) if row else None + finally: + conn.close() + + +def find_cache_hit(run: dict[str, Any]) -> dict[str, Any] | None: + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + SELECT * FROM computation_runs + WHERE tool_name = %s AND input_fingerprint = %s + AND project_id = %s + AND digital_installation_id = %s + AND status = 'data_ingested' AND id <> %s + AND output_json->>'convergence' = 'true' + -- Reuse the run that executed, never another reuse, so + -- artifact lineage stays one hop however often it is reused. + AND COALESCE((external_workflow_ref_json->>'cache_hit')::boolean, false) = false + ORDER BY completed_at DESC NULLS LAST LIMIT 1 + """, + ( + run["tool_name"], + run["input_fingerprint"], + run["project_id"], + run.get("digital_installation_id", ""), + run["id"], + ), + ) + row = cur.fetchone() + return dict(row) if row else None + finally: + conn.close() + + +def update_run( + run_id: str, + *, + status: str | None = None, + message: str = "", + values: dict[str, Any] | None = None, + event_details: dict[str, Any] | None = None, + release_lease: bool = True, +) -> dict[str, Any]: + if status is not None and status not in ALL_STATUSES: + raise ValueError(f"invalid computation status: {status}") + allowed = { + "scheduler_job_id", + "remote_workdir", + "input_fingerprint", + "input_json", + "output_json", + "artifact_manifest_json", + "external_workflow_ref_json", + "error_json", + "prepared_manifest_json", + "cancel_requested", + "attempt_count", + "globus_task_id", + "slurm_comment", + "execution_identity", + "digital_installation_id", + "submitted_at", + "started_at", + "completed_at", + } + values = values or {} + unknown = set(values) - allowed + if unknown: + raise ValueError(f"unsupported computation fields: {sorted(unknown)}") + assignments = ["updated_at = CURRENT_TIMESTAMP"] + if release_lease: + assignments.extend( + ["worker_lease_expires_at = NULL", "worker_lease_owner = ''"] + ) + params: list[Any] = [] + if status is not None: + assignments.append("status = %s") + params.append(status) + for key, value in values.items(): + assignments.append(f"{key} = %s") + if key.endswith("_json"): + value = Json(value) + params.append(value) + params.append(run_id) + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + f"UPDATE computation_runs SET {', '.join(assignments)} WHERE id = %s RETURNING *", + params, + ) + row = cur.fetchone() + if not row: + raise LookupError("computation run not found") + if status: + _insert_event(cur, run_id, status, message, event_details) + conn.commit() + return dict(row) + finally: + conn.close() + + +class EnrichmentConflict(RuntimeError): + """The run changed after the enrichment preview; nothing was written.""" + + +def run_version(row: dict[str, Any]) -> str: + """Fingerprint of everything an enrichment must find unchanged.""" + + fields = { + key: _json(row.get(key), None) if key.endswith("_json") else _iso(row.get(key)) + for key in ( + "status", "scheduler_job_id", "input_fingerprint", "input_json", "output_json", + "artifact_manifest_json", "external_workflow_ref_json", "error_json", + "prepared_manifest_json", "submitted_at", "started_at", "completed_at", "updated_at", + ) + } + encoded = json.dumps(fields, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def enrich_run_diagnostics( + run_id: str, + *, + expected_version: str, + diagnostics: dict[str, Any], + error_json: dict[str, Any], + event_message: str, + event_details: dict[str, Any], +) -> str: + """Replace only stored diagnostics and the readable error summary. + + Called after evidence was fetched (no gateway call happens while the row + lock is held). Status, timestamps (including ``updated_at``), inputs, + outputs, fingerprint, artifacts and scheduler records are not written. A + repeat with the same evidence digest is a no-op without a new event. + """ + + digest = (diagnostics.get("enrichment") or {}).get("evidence_sha256") + if not digest: + raise ValueError("enriched diagnostics must carry an evidence digest") + conn = get_pg_worker_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute("SELECT * FROM computation_runs WHERE id = %s FOR UPDATE", (run_id,)) + row = cur.fetchone() + if not row: + conn.rollback() + raise LookupError("computation run not found") + row = dict(row) + current = (_json(row.get("external_workflow_ref_json"), {}).get("diagnostics") or {}) + if (current.get("enrichment") or {}).get("evidence_sha256") == digest: + conn.rollback() + return "unchanged" + if row["status"] not in {"failed", "cancelled"} or run_version(row) != expected_version: + conn.rollback() + raise EnrichmentConflict("The run changed after the preview; nothing was written") + cur.execute( + """ + UPDATE computation_runs + SET external_workflow_ref_json = jsonb_set( + external_workflow_ref_json, '{diagnostics}', %s::jsonb, true), + error_json = %s::jsonb + WHERE id = %s + """, + (Json(diagnostics), Json(error_json), run_id), + ) + _insert_event(cur, run_id, row["status"], event_message, event_details) + conn.commit() + return "updated" + finally: + conn.close() + + +def _insert_event( + cur, + run_id: str, + status: str, + message: str, + details: dict[str, Any] | None = None, +) -> None: + cur.execute( + """ + INSERT INTO computation_run_events (run_id, status, message, details_json) + VALUES (%s, %s, %s, %s) + """, + (run_id, status, message, Json(details or {})), + ) + + +def _ensure_tool_catalog( + cur, + *, + user: PlatformUser, + project_id: str, + tool_name: str, + cluster: ClusterConfig, +) -> None: + """Register computational equipment/type when the full DT schema exists. + + The computation tables can also be tested against a minimal database, so + this integration is deliberately conditional on the platform catalog + tables being present. + """ + + cur.execute( + "SELECT to_regclass('public.equipment_metadata') AS equipment_table, " + "to_regclass('public.experiment_types') AS type_table, " + "to_regclass('public.project_experiments') AS project_table" + ) + tables = cur.fetchone() + equipment_table = tables["equipment_table"] + type_table = tables["type_table"] + project_table = tables["project_table"] + if not equipment_table or not type_table: + return + definition = TOOL_DEFINITIONS[tool_name] + # A versioned executable installation is digital equipment. The same + # scientific experiment can be compatible with several installations. + equipment_id = cluster.installation_id + type_id = f"computation_{tool_name}_{definition.revision.replace('-', '_')}" + schema = definition.input_model.model_json_schema() + required = set(schema.get("required", [])) + parameters = [] + for name, field in schema.get("properties", {}).items(): + parameters.append( + { + "name": name, + "display_name": field.get("title", name.replace("_", " ").title()), + "type": field.get("type", "object"), + "required": name in required, + "default": field.get("default"), + "description": field.get("description", ""), + "source": "computational_workflow", + } + ) + outputs = [ + { + "name": name, + "display_name": name.replace("_", " ").title(), + "type": "object", + "source": "computational_workflow", + } + for name in definition.output_names + ] + equipment_parameters = [ + { + "name": "executable", + "display_name": "Executable", + "type": "string", + "required": True, + "default": "pw.x", + "description": "Quantum ESPRESSO plane-wave executable.", + "source": "digital_asset", + }, + { + "name": "software_version", + "display_name": "Software Version", + "type": "string", + "required": True, + "default": cluster.qe_version, + "description": "Version of Quantum ESPRESSO in this RCAC installation.", + "source": "digital_asset", + }, + ] + equipment_config = { + "asset_kind": "digital", + "digital_asset_type": "scientific_executable", + "execution_backend": "globus_compute_service_account", + "scheduler": "slurm", + "cluster": cluster.name, + "endpoint_id": cluster.endpoint_id, + "module": cluster.qe_module, + "module_dependencies": list(cluster.module_dependencies), + "mpi_launcher": cluster.mpi_launcher, + "allocation": cluster.allocation, + "partition": cluster.partition, + "qos": cluster.qos, + "executable": "pw.x", + "feature_flag": "RCAC_COMPUTE_ENABLED", + } + cur.execute( + """ + INSERT INTO equipment_metadata ( + domain_id, equipment_name, equipment_type, asset_kind, manufacturer, model, + location, description, source_type, dt_mode, outlier_method, + owner_id, owner_org, status, parameters_json, features_json, + config_json, approved_at, reviewed_by, updated_at + ) VALUES ( + %s, %s, 'Computational Software', 'computational', 'Quantum ESPRESSO', %s, + 'Purdue RCAC', %s, 'rcac_slurm', 'dashboard', 'none', + %s, %s, 'approved', %s, '[]'::jsonb, %s, + CURRENT_TIMESTAMP, %s, CURRENT_TIMESTAMP + ) + ON CONFLICT (domain_id) DO UPDATE SET + equipment_name = EXCLUDED.equipment_name, + asset_kind = EXCLUDED.asset_kind, + model = EXCLUDED.model, + description = EXCLUDED.description, + parameters_json = EXCLUDED.parameters_json, + config_json = EXCLUDED.config_json, + updated_at = CURRENT_TIMESTAMP + """, + ( + equipment_id, + f"Quantum ESPRESSO {cluster.qe_version} on {cluster.display_name}", + cluster.qe_version, + f"Versioned Quantum ESPRESSO executable on Purdue RCAC {cluster.display_name}.", + user.id, + user.organization, + Json(equipment_parameters), + Json(equipment_config), + user.id, + ), + ) + cur.execute( + """ + INSERT INTO experiment_types ( + id, name, description, scientific_objective, equipment_id, + type_parameters_json, outputs_json, additional_inputs_json, + visibility, version, author_name, author_email, owner_id, owner_org + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, '[]'::jsonb, + 'shared', 1, %s, %s, %s, %s + ) + ON CONFLICT (id) DO UPDATE SET + name = EXCLUDED.name, + description = EXCLUDED.description, + type_parameters_json = EXCLUDED.type_parameters_json, + outputs_json = EXCLUDED.outputs_json, + updated_at = CURRENT_TIMESTAMP + """, + ( + type_id, + definition.title, + definition.description, + definition.description, + equipment_id, + Json(parameters), + Json(outputs), + user.name, + user.email, + user.id, + user.organization, + ), + ) + cur.execute( + """ + INSERT INTO computation_experiment_installations ( + experiment_type_id, equipment_id, cluster, enabled, updated_at + ) VALUES (%s, %s, %s, true, CURRENT_TIMESTAMP) + ON CONFLICT (experiment_type_id, equipment_id) DO UPDATE SET + cluster = EXCLUDED.cluster, + enabled = true, + updated_at = CURRENT_TIMESTAMP + """, + (type_id, equipment_id, cluster.name), + ) + if not project_table: + return + upstream_definition = TOOL_DEFINITIONS["cellrelaxdft"] + upstream_type = ( + "computation_cellrelaxdft_" + + upstream_definition.revision.replace("-", "_") + if tool_name == "elasticdft" + else "" + ) + cur.execute( + """ + INSERT INTO project_experiments ( + project_id, type_id, added_by, status, sequence_index, + upstream_type_id, handoff_type, handoff_description, + output_mapping_json + ) + SELECT %s, %s, %s, 'active', + COALESCE(MAX(sequence_index), 0) + 1, %s, + 'automated_computational_job', + 'Registered computation on Purdue RCAC through Globus Compute and Slurm.', %s + FROM project_experiments WHERE project_id = %s + ON CONFLICT (project_id, type_id) DO UPDATE SET + status = 'active', + upstream_type_id = EXCLUDED.upstream_type_id, + handoff_type = EXCLUDED.handoff_type, + handoff_description = EXCLUDED.handoff_description, + output_mapping_json = EXCLUDED.output_mapping_json, + updated_at = CURRENT_TIMESTAMP + """, + ( + project_id, + type_id, + user.id, + upstream_type, + Json( + { + "execution_backend": "globus_compute_service_account", + "upstream_output": "structure_dict" if upstream_type else "", + "downstream_input": "upstream_execution_id" + if upstream_type + else "", + } + ), + project_id, + ), + ) diff --git a/api/computations/worker.py b/api/computations/worker.py new file mode 100644 index 0000000..f157b59 --- /dev/null +++ b/api/computations/worker.py @@ -0,0 +1,772 @@ +"""Durable, centralized Globus Compute/Slurm computation worker. + +One worker cycle submits new runs, issues at most one status request per RCAC +cluster, and then collects completed runs. This follows RCAC's requirement to +avoid per-run polling of the shared Slurm controller token bucket. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import shutil +import socket +import time +import uuid +import hashlib +import importlib.metadata +import platform +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any + +from .diagnostics import build_diagnostics, resource_usage, summary_message +from .globus_gateway import GlobusComputeGateway, GlobusTransientError +from .materials import resolve_material_inputs +from .monolayer import prepare_cellrelax2d, cellrelax2d_outputs +from .qe import ( + PreparedQeRun, + QeParseError, + QePreparationError, + cellrelax_outputs_from_parsed, + elastic_outputs_from_parsed, + prepare_cellrelax, + prepare_elastic, +) +from .rcac import ClusterConfig, get_cluster +from .resources import ( + RESOURCE_GATEWAY_PROTOCOL, + ResourcePolicyError, + apply_resources, + stored_request, +) +from .schemas import ( + QE_INPUT_ADAPTER_VERSION, + compute_input_fingerprint, + validate_tool_inputs, +) +from .slurm import SlurmState, aggregate_states +from .store import ( + TERMINAL_STATUSES, + elastic_parent_eligibility, + claim_active_runs, + claim_next_run, + claim_run, + renew_run_claim, + feature_enabled, + find_cache_hit, + get_run, + update_run, +) + +logger = logging.getLogger("dt.computations.worker") +LEASE_SECONDS = 900 # Exceeds the longest bounded gateway wait (300 seconds). + + +def _scheduler_datetime(value: Any) -> datetime | None: + if not isinstance(value, str) or not value: + return None + try: + result = datetime.fromisoformat(value) + except ValueError: + return None + return result.astimezone(timezone.utc) if result.tzinfo else None + + +def _json(value: Any, default: Any) -> Any: + if isinstance(value, str): + try: + return json.loads(value) + except json.JSONDecodeError: + return default + return value if value is not None else default + + +def _prepared_dict( + prepared: PreparedQeRun, resources: dict[str, Any] | None = None +) -> dict[str, Any]: + versions = {} + for name in ("pymatgen", "numpy", "scipy", "spglib", "ase"): + try: + versions[name] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + versions[name] = "not-installed" + return { + "tool_name": prepared.tool_name, + "stage_files": [path.name for path in prepared.stage_files], + "expected_artifacts": list(prepared.expected_artifacts), + "pseudopotentials": list(prepared.pseudopotentials), + "job_array": prepared.job_array, + "ntasks": prepared.ntasks, + "time_limit": prepared.time_limit, + # Validated execution request; not part of the input fingerprint. + "resources": resources or {}, + "metadata": prepared.metadata or {}, + "platform_environment": { + "python_version": platform.python_version(), "packages": versions, + "adapter_sha256": {name: hashlib.sha256(Path(__file__).with_name(name).read_bytes()).hexdigest() + for name in ("qe.py", "monolayer.py", "schemas.py")}, + }, + } + + +def _restore_prepared(run: dict[str, Any], local_workdir: Path) -> PreparedQeRun: + manifest = _json(run.get("prepared_manifest_json"), {}) + if not manifest: + raise RuntimeError("prepared run manifest is missing") + return PreparedQeRun( + tool_name=manifest["tool_name"], + local_workdir=local_workdir, + stage_files=[local_workdir / name for name in manifest.get("stage_files", [])], + expected_artifacts=list(manifest["expected_artifacts"]), + pseudopotentials=list(manifest.get("pseudopotentials", [])), + job_array=manifest.get("job_array", ""), + ntasks=int(manifest["ntasks"]), + time_limit=manifest["time_limit"], + metadata=manifest.get("metadata", {}), + ) + + +def _safe_diagnostics(**kwargs: Any) -> dict[str, Any]: + """Diagnostics must never block a terminal state or archived evidence.""" + + try: + return build_diagnostics(**kwargs) + except Exception: # noqa: BLE001 + logger.exception("Failure diagnostics could not be built") + return {"schema": "dt-run-diagnostics/1", "outcome": kwargs.get("outcome", ""), + "error": "Diagnostics could not be built; retained artifacts are unaffected", + "members": [], "category_counts": {}, "exit_code_counts": {}, "state_counts": {}} + + +class ComputationWorker: + def __init__( + self, + worker_id: str | None = None, + gateway: GlobusComputeGateway | None = None, + ) -> None: + self.worker_id = worker_id or f"{socket.gethostname()}-{uuid.uuid4().hex[:8]}" + self.work_root = Path( + os.getenv("RCAC_LOCAL_WORK_ROOT", "/tmp/dt-computation-worker") + ).resolve() + self.work_root.mkdir(parents=True, exist_ok=True) + self.gateway = gateway or GlobusComputeGateway() + self.execution_identity = os.getenv( + "RCAC_SERVICE_ACCOUNT_USERNAME", "" + ).strip() + + def run_cycle(self) -> bool: + """Process one bounded batch and aggregate all Slurm status queries.""" + + if not feature_enabled(): + return False + runs = claim_active_runs(self.worker_id, lease_seconds=LEASE_SECONDS, limit=100) + if not runs: + return False + + status_groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for run in runs: + run = renew_run_claim(str(run["id"]), self.worker_id, LEASE_SECONDS) + if not run: + continue + try: + if (run.get("cancel_requested") or run["status"] == "cancelling") and _json(run.get("external_workflow_ref_json"), {}).get("cancellation_sent"): + status_groups[run["cluster"]].append(run) + elif run.get("cancel_requested") or run["status"] == "cancelling": + self._cancel(run) + elif run["status"] in {"requested", "validating", "staging"}: + self._submit(run) + elif run["status"] == "collecting": + self._collect(run) + elif run["status"] in {"submitted", "running"}: + status_groups[run["cluster"]].append(run) + else: + update_run(str(run["id"]), values={}) + except Exception as exc: # noqa: BLE001 + self._handle_error(run, exc) + + for cluster_name, cluster_runs in status_groups.items(): + renewed = [fresh for run in cluster_runs + if (fresh := renew_run_claim(str(run["id"]), self.worker_id, LEASE_SECONDS))] + if renewed: + self._advance_cluster(cluster_name, renewed) + return True + + def run_once(self, run_id: str | None = None) -> bool: + """Compatibility/testing helper; production uses :meth:`run_cycle`.""" + + if not feature_enabled(): + return False + run = ( + claim_run(run_id, self.worker_id, lease_seconds=LEASE_SECONDS) + if run_id + else claim_next_run(self.worker_id, lease_seconds=LEASE_SECONDS) + ) + if not run: + return False + try: + if run.get("cancel_requested") or run["status"] == "cancelling": + self._cancel(run) + elif run["status"] in {"requested", "validating", "staging"}: + self._submit(run) + elif run["status"] == "collecting": + self._collect(run) + else: + self._advance_cluster(run["cluster"], [run]) + except Exception as exc: # noqa: BLE001 + self._handle_error(run, exc) + return True + + def _invoke( + self, + cluster: ClusterConfig, + action: str, + payload: dict[str, Any], + *, + timeout_seconds: float = 180.0, + ) -> tuple[str, dict[str, Any]]: + return self.gateway.invoke( + endpoint_id=cluster.endpoint_id, + function_id=cluster.gateway_function_id, + action=action, + payload={"cluster": cluster.name, **payload}, + timeout_seconds=timeout_seconds, + ) + + def _submit(self, run: dict[str, Any]) -> None: + run_id = str(run["id"]) + if not self.execution_identity: + raise RuntimeError("RCAC_SERVICE_ACCOUNT_USERNAME is not configured") + if run["status"] != "validating": + update_run( + run_id, + status="validating", + message="Validating registered computation inputs", + release_lease=False, + ) + inputs = validate_tool_inputs(run["tool_name"], _json(run["input_json"], {})) + if run["tool_name"] in {"cellrelaxdft", "cellrelaxdft2d"}: + inputs = resolve_material_inputs(inputs) + # An MP-only request is resolved to an exact portable structure before + # cache lookup. This prevents a later Materials Project record update + # from silently reusing a result for an older structure. + input_fingerprint = compute_input_fingerprint(run["tool_name"], inputs) + run["input_fingerprint"] = input_fingerprint + external = _json(run.get("external_workflow_ref_json"), {}) + cached = None if external.get("submission_requested") else find_cache_hit(run) + if cached: + update_run( + run_id, + status="data_ingested", + message="Reused a matching registered computation result", + values={ + "input_json": inputs, + "input_fingerprint": input_fingerprint, + "output_json": _json(cached.get("output_json"), {}), + "artifact_manifest_json": _json( + cached.get("artifact_manifest_json"), [] + ), + "prepared_manifest_json": _json(cached.get("prepared_manifest_json"), {}), + "external_workflow_ref_json": { + "source_provenance": _json(cached.get("external_workflow_ref_json"), {}), + # Cache lookup requires the exact current-adapter + # fingerprint, including for pre-marker cached runs. + "qe_input_adapter_version": QE_INPUT_ADAPTER_VERSION, + "cache_hit": True, + "source_run_id": str(cached["id"]), + # Resources are not part of the fingerprint. No job + # ran, so this run's resource request was not used. + "resources_used": False, + }, + "execution_identity": self.execution_identity, + "completed_at": datetime.now(timezone.utc), + }, + ) + return + update_run( + run_id, + status="staging", + message="Preparing standardized Quantum ESPRESSO inputs", + values={ + "input_json": inputs, + "input_fingerprint": input_fingerprint, + }, + release_lease=False, + ) + + cluster = get_cluster(run["cluster"]) + # Re-check the stored request against the current cluster policy and + # resolve defaults now; the result is recorded with the prepared run. + resources = cluster.resource_policy.resolve( + run["tool_name"], stored_request(_json(run.get("resource_request_json"), {})) + ) + local_workdir = self.work_root / run_id + local_workdir.mkdir(parents=True, exist_ok=True) + if run["tool_name"] == "cellrelaxdft": + prepared = prepare_cellrelax(inputs, local_workdir, cluster.pseudo_root) + elif run["tool_name"] == "cellrelaxdft2d": + prepared = prepare_cellrelax2d(inputs, local_workdir, cluster.pseudo_root) + elif run["tool_name"] == "elasticdft": + upstream_id = str(run.get("upstream_run_id") or "") + upstream = get_run(upstream_id, internal=True) + if not upstream: + raise RuntimeError("completed CellRelaxDFT upstream run is unavailable") + eligible, reason = elastic_parent_eligibility(upstream) + if not eligible: + raise RuntimeError(reason) + prepared = prepare_elastic( + inputs, + _json(upstream["input_json"], {}), + _json(upstream["output_json"], {}), + local_workdir, + cluster.pseudo_root, + ) + else: + raise ValueError(f"unsupported computation tool: {run['tool_name']}") + + prepared = apply_resources(prepared, resources) + spec = cluster.job_spec(prepared, run_id, resources) + # Protocol-1 gateways know only the fixed-walltime ``submit``; they + # reject ``submit_v2`` before any side effect. + action = "submit_v2" if cluster.gateway_protocol >= RESOURCE_GATEWAY_PROTOCOL else "submit" + external["qe_input_adapter_version"] = QE_INPUT_ADAPTER_VERSION + external["submission_requested"] = True + update_run(run_id, values={"prepared_manifest_json": _prepared_dict(prepared, resources), + "external_workflow_ref_json": external}, release_lease=False) + task_id, result = self._invoke( + cluster, + action, + { + "run_id": run_id, + "tool_revision": run["tool_revision"], + "spec": spec.as_payload(), + "files": self.gateway.encoded_files(prepared.stage_files), + "pseudopotentials": prepared.pseudopotentials, + }, + ) + job_id = str(result.get("job_id", "")) + remote_workdir = str(result.get("workdir", "")) + comment = str(result.get("comment", "")) + if not job_id.isdigit() or remote_workdir != cluster.workdir(run_id): + raise RuntimeError("RCAC gateway returned inconsistent submission data") + if comment != spec.comment: + raise RuntimeError("RCAC gateway returned an inconsistent attribution comment") + # Directives submitted to Slurm. The actual allocation (AllocCPUS, + # ReqMem, MaxRSS) is recorded separately from accounting at collection. + expected_resources = spec.rendered_resources() + if action == "submit_v2": + submitted_resources = result.get("resources") + confirmation = "confirmed_by_gateway" if submitted_resources == expected_resources else "mismatch" + if confirmation == "mismatch": + # The job exists; keep tracking it and expose the discrepancy. + logger.error("Run %s: gateway-rendered resources differ from the request", run_id) + else: + # A protocol-1 gateway renders exactly this fixed contract but + # does not echo it back. + submitted_resources = expected_resources + confirmation = "protocol_1_fixed_contract" + protocol = result.get("protocol") + + update_run( + run_id, + status="submitted", + message=f"Submitted Slurm job {job_id} on {cluster.display_name}", + values={ + "scheduler_job_id": job_id, + "remote_workdir": remote_workdir, + "prepared_manifest_json": _prepared_dict(prepared, resources), + "globus_task_id": task_id, + "slurm_comment": comment, + "execution_identity": self.execution_identity, + "digital_installation_id": cluster.installation_id, + "external_workflow_ref_json": { + "qe_input_adapter_version": QE_INPUT_ADAPTER_VERSION, + "manager": "globus-compute", + "execution_backend": "globus_compute_service_account", + "endpoint_id": cluster.endpoint_id, + "gateway_function_id": cluster.gateway_function_id, + "globus_task_id": task_id, + "scheduler": "slurm", + "cluster": cluster.name, + "scheduler_job_id": job_id, + "slurm_comment": comment, + "requested_by": run["requested_by"], + "execution_identity": self.execution_identity, + "digital_installation_id": cluster.installation_id, + "cache_hit": False, + "gateway_action": action, + "gateway_protocol": protocol if type(protocol) is int else 1, + "resource_request": resources, + "submitted_resources": submitted_resources, + "resource_confirmation": confirmation, + }, + "attempt_count": int(run.get("attempt_count") or 0) + 1, + "submitted_at": datetime.now(timezone.utc), + }, + ) + + def _advance_cluster( + self, cluster_name: str, runs: list[dict[str, Any]] + ) -> None: + cluster = get_cluster(cluster_name) + jobs = [ + { + "run_id": str(run["id"]), + "job_id": str(run["scheduler_job_id"]), + } + for run in runs + ] + try: + task_id, result = self._invoke( + cluster, + "status_many", + {"jobs": jobs}, + timeout_seconds=120.0, + ) + states = result.get("states") + if not isinstance(states, dict): + raise RuntimeError("RCAC gateway returned an invalid status response") + except Exception as exc: # noqa: BLE001 + for run in runs: + self._handle_error(run, exc) + return + + for run in runs: + run_id = str(run["id"]) + job_id = str(run["scheduler_job_id"]) + try: + raw_states = states.get(job_id) or [] + if not raw_states: + # Accounting lag is expected, but not forever. Persist a + # bounded retry count and leave the job identity intact. + raise GlobusTransientError("Scheduler record is not yet available") + state = aggregate_states(raw_states) + if state == SlurmState.UNKNOWN: + raise GlobusTransientError("Scheduler state is not yet available") + external = _json(run.get("external_workflow_ref_json"), {}) + external.pop("transport_retry_count", None) + records = result.get("accounting", {}).get(job_id, []) + external["scheduler_accounting"] = records + values: dict[str, Any] = { + "globus_task_id": task_id, "external_workflow_ref_json": external, + } + starts = [dt for item in records if (dt := _scheduler_datetime(item.get("started_at")))] + ends = [dt for item in records if (dt := _scheduler_datetime(item.get("ended_at")))] + if starts: + values["started_at"] = min(starts) + if ends and state in {SlurmState.COMPLETED, SlurmState.FAILED, SlurmState.CANCELLED}: + external["scheduler_ended_at"] = max(ends).isoformat() + if state in {SlurmState.COMPLETED, SlurmState.FAILED, SlurmState.CANCELLED}: + values["cancel_requested"] = False + if state == SlurmState.PENDING: + update_run(run_id, values=values) + elif state == SlurmState.RUNNING: + update_run( + run_id, + status="running" if run["status"] != "running" else None, + message=f"Slurm job {job_id} is running", + values=values, + ) + elif state == SlurmState.COMPLETED: + update_run( + run_id, + status="collecting", + message="Slurm job completed; collecting structured results", + values=values, + ) + else: + terminal_status = "cancelled" if state == SlurmState.CANCELLED else "failed" + external["terminal_outcome"] = terminal_status + values["cancel_requested"] = False + # Per-member states and exit codes now; collection adds + # QE/log evidence. No cause is inferred from states alone. + diagnostics = _safe_diagnostics( + tool_name=str(run.get("tool_name") or ""), job_id=job_id, + prepared_manifest=_json(run.get("prepared_manifest_json"), {}), + accounting=records, outcome=terminal_status, + ) + external["diagnostics"] = diagnostics + evidence = cluster.gateway_protocol >= RESOURCE_GATEWAY_PROTOCOL + values["error_json"] = { + "type": "SchedulerFailure" if terminal_status == "failed" else "Cancelled", + "message": summary_message(diagnostics, next_step=( + "Archiving logs and QE output and extracting per-member evidence." + if evidence else "Archiving the available logs and QE output." + )), + "scheduler_state_counts": dict(Counter(str(item)[:32] for item in raw_states)), + "exit_code_counts": diagnostics.get("exit_code_counts", {}), + } + update_run( + run_id, + status="collecting", + message=f"Slurm job {job_id} is {terminal_status}; preserving available diagnostics", + values=values, + ) + except Exception as exc: # noqa: BLE001 + self._handle_error(run, exc) + + def _cancel(self, run: dict[str, Any]) -> None: + run_id = str(run["id"]) + job_id = str(run.get("scheduler_job_id") or "") + external = _json(run.get("external_workflow_ref_json"), {}) + if not job_id and external.get("submission_requested"): + cluster = get_cluster(run["cluster"]) + task_id, recovered = self._invoke(cluster, "recover_submission", {"run_id": run_id}) + job_id = str(recovered.get("job_id") or "") + if not job_id.isdigit(): + raise GlobusTransientError("Submission outcome is unknown; cancellation needs reconciliation") + if recovered.get("workdir") != cluster.workdir(run_id) or recovered.get("comment") != f"dt-run:{run_id}": + raise RuntimeError("Recovered submission identity is inconsistent") + update_run(run_id, release_lease=False, values={ + "scheduler_job_id": job_id, "remote_workdir": recovered["workdir"], + "slurm_comment": recovered["comment"], "globus_task_id": task_id, + }) + run["scheduler_job_id"] = job_id + if job_id: + cluster = get_cluster(run["cluster"]) + external = _json(run.get("external_workflow_ref_json"), {}) + if not external.get("cancellation_sent"): + task_id, _result = self._invoke( + cluster, "cancel", {"run_id": run_id, "job_id": job_id} + ) + external["cancellation_sent"] = True + update_run(run_id, values={"globus_task_id": task_id, + "external_workflow_ref_json": external}) + # scancel acknowledges a request, not job termination. Poll on the + # next normal worker cycle; only terminal accounting permits archive. + else: + self._advance_cluster(run["cluster"], [run]) + return + else: + values = {} + values.update( + cancel_requested=False, + completed_at=datetime.now(timezone.utc), + ) + update_run( + run_id, + status="cancelled", + message="Computation cancelled", + values=values, + ) + + @staticmethod + def _retained_artifacts( + prepared: PreparedQeRun, scheduler_job_id: str, include_crash: bool = False + ) -> list[str]: + names = [path.name for path in prepared.stage_files] + names.extend(prepared.expected_artifacts) + if prepared.job_array: + count = int((prepared.metadata or {})["count"]) + names.extend( + f"slurm-{scheduler_job_id}_{index}.out" for index in range(count) + ) + names.extend( + f"slurm-{scheduler_job_id}_{index}.err" for index in range(count) + ) + else: + names.extend(["slurm.out", "slurm.err"]) + if include_crash: + # Protocol-1 gateways reject these names; QE writes CRASH only on + # failure and protocol-2 gateways never report it as missing. + names.extend( + str(PurePosixPath(name).parent / "CRASH") for name in prepared.expected_artifacts + ) + return list(dict.fromkeys(names)) + + def _collect(self, run: dict[str, Any]) -> None: + run_id = str(run["id"]) + local_workdir = self.work_root / run_id + prepared = _restore_prepared(run, local_workdir) + cluster = get_cluster(run["cluster"]) + external = _json(run.get("external_workflow_ref_json"), {}) + terminal = external.get("terminal_outcome") + job_id = str(run["scheduler_job_id"]) + # Evidence options are sent only to a protocol-2 gateway; protocol 1 + # rejects the CRASH names and ignores the flags. + evidence = cluster.gateway_protocol >= RESOURCE_GATEWAY_PROTOCOL + payload: dict[str, Any] = { + "run_id": run_id, + "job_id": job_id, + "tool_name": prepared.tool_name, + "diagnostics_only": terminal in {"failed", "cancelled"}, + "expected_artifacts": prepared.expected_artifacts, + "retained_artifacts": self._retained_artifacts(prepared, job_id, include_crash=evidence), + } + if evidence: + payload.update(include_diagnostics=True, include_scheduler_steps=True) + task_id, result = self._invoke(cluster, "collect", payload, timeout_seconds=300.0) + parsed = result.get("parsed_outputs") + manifest = result.get("artifact_manifest") + if not isinstance(parsed, dict) or not isinstance(manifest, list): + raise RuntimeError("RCAC gateway returned an invalid collection response") + external.update( + collection_task_id=task_id, artifact_storage="ncn_depot", + execution_environments=result.get("execution_environments", []), + missing_artifacts=result.get("missing_artifacts", []), + collection_errors=result.get("collection_errors", []), + collection_status="complete" if not result.get("missing_artifacts") and not result.get("collection_errors") else "partial", + qe_versions_from_outputs=sorted({str(p["qe_version"]) for p in parsed.values() if p.get("qe_version")}), + results_ingested_at=datetime.now(timezone.utc).isoformat(), + ) + external.pop("transport_retry_count", None) + if isinstance(result.get("scheduler_steps"), list): + # Scheduler-observed allocation and peak RSS for this run's own job. + external["scheduler_resource_usage"] = resource_usage(result["scheduler_steps"], job_id) + common = {"artifact_manifest_json": manifest, + "external_workflow_ref_json": external, "globus_task_id": task_id} + # Retain diagnostics even when scientific parsing below cannot succeed. + update_run(run_id, values=common, release_lease=False) + prepared_manifest = _json(run.get("prepared_manifest_json"), {}) + + def diagnose(outcome: str, parse_error: str = "") -> dict[str, Any]: + return _safe_diagnostics( + tool_name=prepared.tool_name, job_id=job_id, prepared_manifest=prepared_manifest, + accounting=external.get("scheduler_accounting"), collection=result, + outcome=outcome, parse_error=parse_error, + resources=external.get("submitted_resources") or prepared_manifest.get("resources") or {}, + # Only protocol-2 job scripts print pw.x's own exit status. + pw_exit_logged=type(external.get("gateway_protocol")) is int + and external["gateway_protocol"] >= RESOURCE_GATEWAY_PROTOCOL, + ) + + if terminal in {"failed", "cancelled"}: + diagnostics = diagnose(terminal) + external["diagnostics"] = diagnostics + error = _json(run.get("error_json"), {}) or { + "type": "SchedulerFailure" if terminal == "failed" else "Cancelled", + } + error.update( + message=summary_message(diagnostics), + category_counts=diagnostics.get("category_counts", {}), + exit_code_counts=diagnostics.get("exit_code_counts", {}), + ) + update_run(run_id, status=terminal, + message="Available diagnostics archived; scientific results are not marked successful", + values={**common, "error_json": error, + "completed_at": datetime.now(timezone.utc), "cancel_requested": False}) + return + inputs = _json(run["input_json"], {}) + try: + if prepared.tool_name in {"cellrelaxdft", "cellrelaxdft2d"}: + summary = parsed.get("vcrelax.stdout") + if summary is None: + raise QeParseError("CellRelaxDFT structured output is missing or unparsable") + outputs = ( + cellrelax2d_outputs(inputs, summary, prepared.metadata) + if prepared.tool_name == "cellrelaxdft2d" + else cellrelax_outputs_from_parsed(inputs, summary) + ) + else: + outputs = elastic_outputs_from_parsed(prepared, parsed) + except QeParseError as exc: + # Slurm completed, but the scientific result is not valid (for + # example, a member that did not converge). Keep it precise. + diagnostics = diagnose("result_validation_failed", str(exc)) + external["diagnostics"] = diagnostics + update_run( + run_id, + status="failed", + message="Slurm completed, but the scientific result failed validation; diagnostics archived", + values={ + **common, + "error_json": { + "type": "QeParseError", + "message": summary_message(diagnostics), + "category_counts": diagnostics.get("category_counts", {}), + }, + "completed_at": datetime.now(timezone.utc), + }, + ) + return + + update_run( + run_id, + status="data_ingested", + message="Structured Quantum ESPRESSO results ingested", + values={ + "output_json": outputs, + "artifact_manifest_json": manifest, + "external_workflow_ref_json": external, + "globus_task_id": task_id, + "completed_at": datetime.now(timezone.utc), + "error_json": {}, + }, + ) + shutil.rmtree(local_workdir, ignore_errors=True) + + def _handle_error(self, run: dict[str, Any], exc: Exception) -> None: + if isinstance(exc, GlobusTransientError): + # Persist bounded retries across Pod restarts. The next cycle uses + # the same run id; submission is guarded by the remote marker/lock. + current = get_run(str(run["id"]), internal=True) + external = _json((current or run).get("external_workflow_ref_json"), {}) + retries = int(external.get("transport_retry_count", 0)) + 1 + if retries <= 3: + external["transport_retry_count"] = retries + update_run(str(run["id"]), values={"external_workflow_ref_json": external}) + logger.warning("Transport retry %s scheduled for run %s", retries, run["id"]) + return + self._fail(run, exc) + + @staticmethod + def _fail(run: dict[str, Any], exc: Exception) -> None: + run_id = str(run["id"]) + logger.error("Computation run %s failed (%s)", run_id, type(exc).__name__) + # Unknown exception messages may carry credentials or service replies. + # These types carry only messages written by the DT adapters/policy. + safe = (GlobusTransientError, QeParseError, QePreparationError, ResourcePolicyError) + detail = str(exc)[:2000] if isinstance(exc, safe) else "Execution or collection failed; inspect retained diagnostics and operator logs" + update_run( + run_id, + status="failed", + message="Computation execution failed", + values={ + "error_json": { + "type": type(exc).__name__, + "message": detail, + }, + "completed_at": datetime.now(timezone.utc), + }, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Birck DT RCAC computation worker") + parser.add_argument("--once", action="store_true", help="process one worker cycle") + parser.add_argument("--run-id", help="process only this run until terminal") + parser.add_argument("--poll-seconds", type=float, default=30.0) + args = parser.parse_args() + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + if not feature_enabled(): + raise SystemExit( + "RCAC computation worker is disabled; set RCAC_COMPUTE_ENABLED=true" + ) + worker = ComputationWorker() + poll_seconds = max(20.0, args.poll_seconds) + if args.run_id: + try: + uuid.UUID(args.run_id) + except ValueError as exc: + raise SystemExit("--run-id must be a UUID") from exc + while True: + run = get_run(args.run_id, internal=True) + if not run: + raise SystemExit("requested computation run does not exist") + if run["status"] in TERMINAL_STATUSES: + return + worker.run_once(args.run_id) + time.sleep(poll_seconds) + if args.once: + worker.run_cycle() + return + while True: + worker.run_cycle() + time.sleep(poll_seconds) + + +if __name__ == "__main__": + main() diff --git a/api/db_migrations.py b/api/db_migrations.py index b1b9a8b..0b2f503 100644 --- a/api/db_migrations.py +++ b/api/db_migrations.py @@ -25,6 +25,8 @@ STARTUP_MIGRATION_LOCK_ID = 6420260506 RLS_SCRIPT_PATH = Path(__file__).resolve().parent / "scripts" / "add_row_level_security.sql" RLS_MIGRATION_NAME = "row_level_security" +ASSET_KINDS_SCRIPT_PATH = Path(__file__).resolve().parent / "scripts" / "add_asset_kinds.sql" +ASSET_KINDS_MIGRATION_NAME = "asset_kinds" # Tables whose row-level security is switched by DB_RLS_ENFORCEMENT. The six # tables that already enforced RLS before this migration (projects and the run @@ -154,6 +156,53 @@ def ensure_row_level_security_pg() -> dict[str, Any]: } +def ensure_asset_kinds_pg() -> bool: + """ + Apply scripts/add_asset_kinds.sql (physical vs computational columns and + the all_runs view) once per script revision, like the RLS script, so + routine restarts take no table locks. Returns whether it ran. + """ + from data_loader_pg import get_pg_migration_connection + + script = ASSET_KINDS_SCRIPT_PATH.read_text() + checksum = hashlib.sha256(script.encode("utf-8")).hexdigest() + conn = get_pg_migration_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE IF NOT EXISTS app_schema_migrations ( + name TEXT PRIMARY KEY, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + cur.execute( + "SELECT checksum FROM app_schema_migrations WHERE name = %s", + (ASSET_KINDS_MIGRATION_NAME,), + ) + row = cur.fetchone() + applied = False + if not row or row[0] != checksum: + cur.execute(script) + cur.execute( + """ + INSERT INTO app_schema_migrations (name, checksum) + VALUES (%s, %s) + ON CONFLICT (name) DO UPDATE + SET checksum = EXCLUDED.checksum, applied_at = CURRENT_TIMESTAMP + """, + (ASSET_KINDS_MIGRATION_NAME, checksum), + ) + applied = True + conn.commit() + finally: + conn.close() + logger.info("Asset kind migration: script_applied=%s", applied) + return applied + + def run_schema_migrations() -> None: """ Run idempotent schema migrations under a Postgres advisory lock. @@ -190,6 +239,7 @@ def run_schema_migrations() -> None: ensure_sample_and_publication_tables_pg, ensure_user_role_audit_table_pg, ) + from computations.store import ensure_tables as ensure_computation_tables from ml_response_cache import ensure_ml_endpoint_cache_table_pg from platform_notifications import ensure_platform_notifications_table_pg from role_requests import ensure_role_change_requests_table_pg @@ -227,6 +277,12 @@ def run_schema_migrations() -> None: ensure_ml_endpoint_cache_table_pg() ensure_platform_notifications_table_pg() ensure_role_change_requests_table_pg() + # Always created (empty while RCAC_COMPUTE_ENABLED is off) because the + # row-level security script below grants and policies on them. + ensure_computation_tables() + # After equipment, templates and both run tables exist (the view + # reads all three run tables). + ensure_asset_kinds_pg() # Last: the policies reference tables and columns created above. ensure_row_level_security_pg() finally: diff --git a/api/main.py b/api/main.py index de5b08b..4c4dcca 100644 --- a/api/main.py +++ b/api/main.py @@ -28,7 +28,7 @@ from middleware.error_handler import global_exception_handler, insufficient_privilege_handler from runtime_config import MissingRuntimeConfiguration, validate_runtime_configuration -from routers import ml, dataset, dataset_v2, equipment, upload, admin, processing +from routers import ml, dataset, dataset_v2, equipment, upload, admin, processing, computations # ── Structured logging ────────────────────────────────────────────────────── logging.basicConfig( @@ -253,6 +253,7 @@ async def lifespan(app: FastAPI): app.include_router(upload.router, prefix="/api") app.include_router(admin.router, prefix="/api") app.include_router(processing.router, prefix="/api") +app.include_router(computations.router, prefix="/api") # ── Health check ───────────────────────────────────────────────────────────── diff --git a/api/metadata_pg.py b/api/metadata_pg.py index acbeb4c..c81a62c 100644 --- a/api/metadata_pg.py +++ b/api/metadata_pg.py @@ -2,10 +2,12 @@ import hashlib import json import logging +import math import os import re import time import uuid +from decimal import Decimal from datetime import date, datetime from threading import Thread from pathlib import Path @@ -1211,6 +1213,8 @@ def _serialize_equipment( "owner_id": record.get("owner_id", ""), "owner_org": record.get("owner_org", ""), "equipment_type": record.get("equipment_type", ""), + # physical | computational (scripts/add_asset_kinds.sql) + "asset_kind": record.get("asset_kind") or "physical", "manufacturer": record.get("manufacturer", ""), "model": record.get("model", ""), "location": record.get("location", ""), @@ -1306,6 +1310,7 @@ def list_equipment_pg(user: PlatformUser) -> list[dict[str, Any]]: domain_id, equipment_name, equipment_type, + asset_kind, manufacturer, model, location, @@ -1369,6 +1374,7 @@ def get_equipment_pg(domain_id: str) -> dict[str, Any] | None: domain_id, equipment_name, equipment_type, + asset_kind, manufacturer, model, location, @@ -2369,6 +2375,8 @@ def _experiment_type_row_to_dict(row: dict) -> dict[str, Any]: "description": row.get("description", ""), "scientific_objective": row.get("scientific_objective", ""), "equipment_id": row.get("equipment_id", ""), + # physical | computational, the kind of the template's equipment + "execution_kind": row.get("execution_kind") or "physical", "visibility": row.get("visibility", "shared"), "version": row.get("version", 1), "pid": row.get("pid", ""), @@ -2396,7 +2404,7 @@ def _experiment_type_row_to_dict(row: dict) -> dict[str, Any]: visibility, version, pid, parent_id, forked_from, author_name, author_email, primary_target, primary_objective, secondary_target, secondary_objective, - owner_id, owner_org, created_at, updated_at + owner_id, owner_org, execution_kind, created_at, updated_at """ _EXPERIMENT_TYPE_SELECT_COLS_T = """ @@ -2405,7 +2413,7 @@ def _experiment_type_row_to_dict(row: dict) -> dict[str, Any]: t.visibility, t.version, t.pid, t.parent_id, t.forked_from, t.author_name, t.author_email, t.primary_target, t.primary_objective, t.secondary_target, t.secondary_objective, - t.owner_id, t.owner_org, t.created_at, t.updated_at + t.owner_id, t.owner_org, t.execution_kind, t.created_at, t.updated_at """ @@ -2449,12 +2457,23 @@ def _visible_experiment_type_filter( ) +def _user_template_slug(name: str) -> str: + """Slug for a user-created template id. + + ``computation_`` ids are reserved for the registered computation catalog + (computations.store._ensure_tool_catalog), so a template named, say, + "Computation calibration" is not mistaken for one. + """ + slug = _slugify(name, "experiment_type") + return f"unit_{slug}" if slug.startswith("computation") else slug + + def create_experiment_type_pg( *, payload: dict[str, Any], user: PlatformUser, ) -> dict[str, Any]: - type_slug = _slugify(payload.get("type_name", ""), "experiment_type") + type_slug = _user_template_slug(payload.get("type_name", "")) type_id = f"{type_slug}_{uuid.uuid4().hex[:8]}" pid = _generate_pid() conn = get_pg_connection(user) @@ -2853,7 +2872,7 @@ def fork_experiment_type_pg( if not source: return {"status": "not_found"} - new_slug = _slugify(source.get("name", ""), "experiment_type") + new_slug = _user_template_slug(source.get("name", "")) new_id = f"{new_slug}_fork_{uuid.uuid4().hex[:8]}" new_pid = _generate_pid() @@ -3495,53 +3514,682 @@ def update_publication_request_status_pg( conn.close() +class RegisteredComputationTemplateError(ValueError): + """A registered computational workflow was sent to the generic run path.""" + + def __init__(self, tool_name: str = "") -> None: + self.tool_name = tool_name + tool_hint = f" (tool: {tool_name})" if tool_name else "" + super().__init__( + "This unit experiment is a registered computational experiment" + f"{tool_hint}. Generic project runs cannot execute it; submit it " + "from the project's Registered computational experiments form." + ) + + +class UnsupportedOptimizationError(ValueError): + """The selected inputs/targets cannot be optimized by the proposal engine.""" + + +_COMPUTATION_TEMPLATE_PREFIX = "computation_" +_COMPUTATION_WORKFLOW_SOURCE = "computational_workflow" + + +def registered_computation_tool_for_template( + type_id: str, + type_parameters: list[Any] | None = None, + execution_kind: str = "", +) -> str | None: + """Return the tool for a registered computation template, else None. + + ``execution_kind`` (experiment_types.execution_kind) is authoritative when + the caller has it. Registered templates also use versioned ids + ``computation__`` (see + ``computations.store._ensure_tool_catalog``) and mark every input with + ``source="computational_workflow"``. An empty string means the template is + computational but its tool is not recognized. + """ + type_id = str(type_id or "").strip() + declared_kind = str(execution_kind or "").strip().lower() + if declared_kind == "physical": + return None + computational = declared_kind == "computational" + if computational or type_id.startswith(_COMPUTATION_TEMPLATE_PREFIX): + if not type_id.startswith(_COMPUTATION_TEMPLATE_PREFIX): + return "" + remainder = type_id[len(_COMPUTATION_TEMPLATE_PREFIX):] + try: + from computations.schemas import TOOL_DEFINITIONS + + tool_names = sorted(TOOL_DEFINITIONS, key=len, reverse=True) + except ImportError: # pragma: no cover - computations package absent + tool_names = ["cellrelaxdft2d", "cellrelaxdft", "elasticdft"] + for tool_name in tool_names: + if remainder == tool_name or remainder.startswith(f"{tool_name}_"): + return tool_name + return "" + if any( + isinstance(param, dict) + and param.get("source") == _COMPUTATION_WORKFLOW_SOURCE + for param in (type_parameters or []) + ): + return "" + return None + + +_VALUE_TYPE_ALIASES = { + "float": "float", + "number": "float", + "numeric": "float", + "double": "float", + "real": "float", + "decimal": "float", + "int": "integer", + "integer": "integer", + "bool": "boolean", + "boolean": "boolean", + "string": "string", + "str": "string", + "text": "string", + "enum": "string", + "categorical": "string", + "datetime": "datetime", + "date": "datetime", + "timestamp": "datetime", + "object": "object", + "map": "object", + "dict": "object", + "json": "object", + "array": "array", + "list": "array", +} +_STRICT_NUMBER = re.compile(r"^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$") +_NONFINITE_WORDS = {"nan", "+nan", "-nan", "inf", "+inf", "-inf", "infinity", "+infinity", "-infinity"} + + +def _input_mode(param: dict[str, Any]) -> str: + mode = str(param.get("mode") or "").strip().lower() + if mode in {"not_used", "fixed", "variable"}: + return mode + return "variable" if param.get("is_variable") is True else "fixed" + + +def _declared_value_type(param: dict[str, Any], default: str) -> str: + raw = str(param.get("type") or "").strip().lower() + if not raw: + return default + kind = _VALUE_TYPE_ALIASES.get(raw) + if kind is None: + name = str(param.get("name") or "").strip() + raise ValueError(f"Input '{name}' declares unsupported value type '{raw}'") + return kind + + +def _finite_bound(param: dict[str, Any], *keys: str) -> float | None: + for key in keys: + value = param.get(key) + if value in (None, "") or isinstance(value, bool): + continue + number = _to_float(value) + if number is not None and math.isfinite(number): + return number + return None + + +def _reject_json_constant(value: str) -> Any: + raise ValueError(f"non-finite JSON number {value}") + + +def _contains_nonfinite_number(value: Any) -> bool: + """True when parsed JSON holds inf/NaN anywhere (e.g. overflowed 1e999).""" + if isinstance(value, float): + return not math.isfinite(value) + if isinstance(value, dict): + return any(_contains_nonfinite_number(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_nonfinite_number(item) for item in value) + return False + + +def _json_values_equal(left: Any, right: Any) -> bool: + """JSON equality: 50 equals 50.0, but true never equals 1.""" + if isinstance(left, bool) or isinstance(right, bool): + return isinstance(left, bool) and isinstance(right, bool) and left == right + if isinstance(left, (int, float)) and isinstance(right, (int, float)): + # Python compares int and float exactly: no rounding, no overflow. + return left == right + if isinstance(left, dict) and isinstance(right, dict): + return left.keys() == right.keys() and all( + _json_values_equal(left[key], right[key]) for key in left + ) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all( + _json_values_equal(a, b) for a, b in zip(left, right) + ) + return type(left) is type(right) and left == right + + +def _normalize_planned_value( + name: str, + text: str, + kind: str, + param: dict[str, Any], +) -> str: + """Validate a wire-encoded value against its declared type and limits.""" + normalized: str + if kind in {"float", "integer"}: + if not _STRICT_NUMBER.match(text): + if text.lower() in _NONFINITE_WORDS: + raise ValueError(f"Planned parameter '{name}' must be a finite number") + raise ValueError(f"Planned parameter '{name}' must be numeric") + number = float(text) + if not math.isfinite(number): + raise ValueError(f"Planned parameter '{name}' must be a finite number") + if kind == "integer": + exact_number = Decimal(text) + if exact_number != exact_number.to_integral_value(): + raise ValueError(f"Planned parameter '{name}' must be a whole number") + normalized = str(int(exact_number)) + else: + normalized = text + min_value = _finite_bound(param, "min_value", "minimum") + max_value = _finite_bound(param, "max_value", "maximum") + if min_value is not None and number < min_value: + raise ValueError( + f"Planned parameter '{name}' must be greater than or equal to {min_value:g}" + ) + if max_value is not None and number > max_value: + raise ValueError( + f"Planned parameter '{name}' must be less than or equal to {max_value:g}" + ) + elif kind == "boolean": + lowered = text.lower() + if lowered not in {"true", "false"}: + raise ValueError(f"Planned parameter '{name}' must be true or false") + normalized = lowered + elif kind == "datetime": + try: + datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError( + f"Planned parameter '{name}' must be an ISO 8601 date or date-time" + ) from exc + normalized = text + elif kind in {"object", "array"}: + try: + value = json.loads(text, parse_constant=_reject_json_constant) + except ValueError as exc: + raise ValueError(f"Planned parameter '{name}' must be valid JSON") from exc + expected = dict if kind == "object" else list + if not isinstance(value, expected): + label = "a JSON object" if kind == "object" else "a JSON array" + raise ValueError(f"Planned parameter '{name}' must be {label}") + # parse_constant only sees NaN/Infinity tokens; numeric literals + # such as 1e999 overflow to inf silently, so check the parsed tree. + if _contains_nonfinite_number(value): + raise ValueError( + f"Planned parameter '{name}' must contain only finite numbers" + ) + normalized = json.dumps(value, separators=(",", ":"), sort_keys=True) + else: + # Strings are kept verbatim (wire contract), including whitespace. + normalized = text + + allowed = next( + ( + param.get(key) + for key in ("allowed_values", "enum", "options") + if isinstance(param.get(key), list) and param.get(key) + ), + None, + ) + if allowed is not None: + def matches(option: Any) -> bool: + if kind in {"float", "integer"}: + option_number = _to_float(option) + return option_number is not None and option_number == float(normalized) + if isinstance(option, bool): + return normalized == str(option).lower() + if kind in {"object", "array"}: + # Compare parsed JSON, not Python's str() of a dict or list. + if isinstance(option, str): + try: + option = json.loads(option) + except ValueError: + return False + return _json_values_equal(json.loads(normalized), option) + return normalized == str(option) + + if not any(matches(option) for option in allowed): + choices = ", ".join(str(option) for option in allowed) + raise ValueError(f"Planned parameter '{name}' must be one of: {choices}") + return normalized + + def _validate_planned_parameters_against_experiment_inputs( planned_parameters: list[Any], input_snapshot: list[Any], -) -> None: - selected_inputs: dict[str, dict[str, Any]] = {} - for param in input_snapshot: + additional_inputs_snapshot: list[Any] | None = None, + *, + optimization: bool = False, +) -> list[dict[str, str]]: + """Validate per-run values against the definition snapshots. + + Returns normalized ``{name, planned_value, unit}`` rows for storage in + ``planned_parameters_json``. Units are definition-owned: an omitted unit + is filled from the definition and a conflicting one is rejected. Blank + values count as omitted. Ordinary runs need a concrete value for every + selected equipment input; optimization campaigns need values only for + Fixed inputs because Variable inputs are searched over their ranges. + Additional inputs are optional unless their definition says required. + """ + equipment_inputs: dict[str, dict[str, Any]] = {} + for param in input_snapshot or []: if not isinstance(param, dict): continue name = str(param.get("name") or "").strip() - mode = str(param.get("mode") or "").strip().lower() - if name and mode != "not_used": - selected_inputs[name] = param - - seen: set[str] = set() - for param in planned_parameters: + if name and _input_mode(param) != "not_used": + equipment_inputs[name] = param + additional_inputs: dict[str, dict[str, Any]] = {} + for param in additional_inputs_snapshot or []: if not isinstance(param, dict): continue name = str(param.get("name") or "").strip() + if name and name not in equipment_inputs: + additional_inputs[name] = param + if not equipment_inputs and not additional_inputs: + # A run records what was done; a definition that selects no inputs + # cannot capture that, so it must be edited before it is run. + raise ValueError( + "This unit experiment selects no inputs. Edit its definition to " + "select at least one equipment or additional input before running it." + ) + + seen: set[str] = set() + provided: set[str] = set() + normalized: list[dict[str, str]] = [] + for item in planned_parameters or []: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").strip() if not name: continue if name in seen: raise ValueError(f"Planned parameter '{name}' is duplicated") seen.add(name) - selected_input = selected_inputs.get(name) - if not selected_input: + is_additional = name not in equipment_inputs + definition = equipment_inputs.get(name) or additional_inputs.get(name) + if definition is None: raise ValueError( f"Planned parameter '{name}' is not a selected input for this experiment" ) - raw_value = param.get("planned_value") - if raw_value in (None, ""): - raise ValueError(f"Planned parameter '{name}' requires a value") - value = _to_float(raw_value) - if value is None: - raise ValueError(f"Planned parameter '{name}' must be numeric") - - min_value = _to_float(selected_input.get("min_value")) - max_value = _to_float(selected_input.get("max_value")) - if min_value is not None and value < min_value: + raw_value = item.get("planned_value") + raw_text = "" if raw_value is None else str(raw_value) + # Trimming only detects blanks; string values stay verbatim. + if not raw_text.strip(): + continue + if optimization and not is_additional and _input_mode(definition) == "variable": raise ValueError( - f"Planned parameter '{name}' must be greater than or equal to {min_value:g}" + f"Planned parameter '{name}' is a Variable input. Optimization " + "campaigns search its defined range, so do not submit a point value." ) - if max_value is not None and value > max_value: + + kind = _declared_value_type(definition, "string" if is_additional else "float") + text = raw_text if kind == "string" else raw_text.strip() + value = _normalize_planned_value(name, text, kind, definition) + + definition_unit = str(definition.get("unit") or "").strip() + submitted_unit = str(item.get("unit") or "").strip() + if submitted_unit and submitted_unit != definition_unit: + expected = f"'{definition_unit}'" if definition_unit else "no unit" raise ValueError( - f"Planned parameter '{name}' must be less than or equal to {max_value:g}" + f"Planned parameter '{name}' unit '{submitted_unit}' conflicts with " + f"the unit experiment definition ({expected})" + ) + provided.add(name) + normalized.append({"name": name, "planned_value": value, "unit": definition_unit}) + + missing = [ + name + for name, param in equipment_inputs.items() + if name not in provided + and (not optimization or _input_mode(param) == "fixed") + ] + missing.extend( + name + for name, param in additional_inputs.items() + if name not in provided and param.get("required") is True + ) + if missing: + raise ValueError( + "Planned parameter values are required for: " + ", ".join(missing) + ) + return normalized + + +# The proposal engine trains only on etch-rate history (``data_loader`` and +# ``data_loader_pg.get_training_df_pg``); these mirror its feature/target +# schema so unsupported equipment fails before any recipe is generated. +ETCHER_OPTIMIZER_FEATURES = ( + "Etch_AvgO2Flow", + "Etch_Avg_Rf1_Pow", + "Etch_Avg_Rf2_Pow", + "Etch_AvgPres", + "Etch_Avgcf4Flow", +) +ETCHER_OPTIMIZER_TARGETS = ("AvgEtchRate", "RangeEtchRate") + + +def _objective_direction(value: Any, default: str, label: str) -> str: + text = str(value or "").strip().lower() + if not text: + return default + if text not in {"maximize", "minimize"}: + raise UnsupportedOptimizationError( + f"{label} objective must be 'maximize' or 'minimize'" + ) + return text + + +# The proposal engine's training data is keyed to this domain config. +PROPOSAL_ENGINE_DOMAIN_ID = "etcher" +# Operators may map additional registered equipment ids onto the etch-rate +# model explicitly; nothing is inferred from parameter names. +PROPOSAL_EQUIPMENT_IDS_ENV = "RECIPE_PROPOSAL_ETCHER_EQUIPMENT_IDS" + + +def load_optimization_domain(equipment_id: str) -> dict[str, Any]: + """Return the validated proposal-domain config for the selected equipment. + + The canonical etcher, or equipment explicitly mapped to its training model + by the operator, is supported. Use the selected equipment's registered + configuration, falling back to its own legacy file only when no record + exists. Other equipment is rejected, even with matching parameter names. + Only feature, target and limit metadata is returned. + """ + from domain_configs import load_domain_config + + equipment_id = str(equipment_id or "").strip() + if not equipment_id: + raise UnsupportedOptimizationError( + "Optimization requires a selected equipment" + ) + allowed_ids = {PROPOSAL_ENGINE_DOMAIN_ID} | { + value.strip() + for value in os.getenv(PROPOSAL_EQUIPMENT_IDS_ENV, "").split(",") + if value.strip() + } + if equipment_id not in allowed_ids: + raise UnsupportedOptimizationError( + f"Recipe proposals are not available for equipment '{equipment_id}'. " + "The proposal engine currently supports only explicitly configured " + "equipment using the etch-rate model." + ) + record = get_equipment_pg(equipment_id) + config = (record.get("config_json") or {}) if record else (load_domain_config(equipment_id) or {}) + domain = config.get("domain") if isinstance(config.get("domain"), dict) else {} + if str(domain.get("id") or "").strip() != equipment_id: + raise UnsupportedOptimizationError( + f"The selected equipment '{equipment_id}' has no matching proposal configuration" + ) + display_name = str(domain.get("display_name") or equipment_id) + + limits: dict[str, list[float]] = {} + feature_display: dict[str, str] = {} + feature_units: dict[str, str] = {} + for feature in config.get("features") or []: + if not isinstance(feature, dict): + continue + feature_id = str(feature.get("id") or "").strip() + low = _to_float(feature.get("min")) + high = _to_float(feature.get("max")) + if ( + not feature_id + or low is None + or high is None + or not math.isfinite(low) + or not math.isfinite(high) + or low > high + ): + raise UnsupportedOptimizationError( + f"The {display_name} configuration has invalid limits for '{feature_id}'" + ) + limits[feature_id] = [low, high] + feature_display[feature_id] = str(feature.get("display_name") or feature_id) + feature_units[feature_id] = str(feature.get("unit") or "") + if set(limits) != set(ETCHER_OPTIMIZER_FEATURES): + raise UnsupportedOptimizationError( + f"The {display_name} configuration does not match the proposal model inputs" + ) + + targets = config.get("targets") if isinstance(config.get("targets"), dict) else {} + default_objectives: list[tuple[str, str]] = [] + for key, fallback in (("primary", "maximize"), ("secondary", "minimize")): + target = targets.get(key) if isinstance(targets.get(key), dict) else {} + target_id = str(target.get("id") or "").strip() + if target_id: + default_objectives.append( + ( + target_id, + _objective_direction(target.get("objective"), fallback, key.title()), + ) ) + target_ids = [target_id for target_id, _direction in default_objectives] + if not target_ids or not set(target_ids) <= set(ETCHER_OPTIMIZER_TARGETS): + raise UnsupportedOptimizationError( + f"The {display_name} configuration does not match the proposal model targets" + ) + + features = [feature for feature in ETCHER_OPTIMIZER_FEATURES] + return { + "domain_id": PROPOSAL_ENGINE_DOMAIN_ID, + "equipment_id": equipment_id, + "display_name": display_name, + "features": features, + "targets": target_ids, + "feature_limits": limits, + "feature_display": feature_display, + "feature_units": feature_units, + "default_objectives": default_objectives, + } + + +def _declared_range_bound(param: dict[str, Any], keys: tuple[str, ...], name: str) -> float | None: + """A declared bound, or None when absent; malformed bounds are rejected.""" + for key in keys: + value = param.get(key) + if value in (None, "") or isinstance(value, bool): + continue + number = _to_float(value) + if number is None or not math.isfinite(number): + raise UnsupportedOptimizationError( + f"Input '{name}' has a non-finite or non-numeric range bound" + ) + return number + return None + + +def resolve_optimization_request( + *, + equipment_id: str, + input_snapshot: list[Any], + planned_parameters: list[Any], + optimization_context: dict[str, Any] | None, + output_snapshot: list[Any] | None = None, + strict: bool = False, +) -> dict[str, Any]: + """Build proposal-engine arguments from the selected equipment's config + and the run's own snapshots. + + - The selected equipment must map to a supported proposal domain + (``load_optimization_domain``); otherwise the request is rejected. + - Every model input must be selected (Fixed or Variable): a Not Used + input cannot silently become an unconstrained recipe feature. + - Variable inputs are searched over their complete declared range. Ranges must + be finite, ordered and inside those limits. + - Fixed inputs are pinned to their planned value, which must lie inside + the configured limits. + - Additional and sample-sourced inputs are context, never constraints. + - Targets must be outputs of the definition and supported by the domain. + ``strict`` (run creation) requires a primary target. Legacy rows + without one use the domain's configured default objectives. + + Returns ``{"domain_id", "constraints", "domain_config", "objectives"}``. + ``objectives`` is None when the domain's default objectives apply, which + keeps the established etcher proposal path. + """ + domain = load_optimization_domain(equipment_id) + context = optimization_context or {} + primary = str(context.get("primary_target") or "").strip() + secondary = str(context.get("secondary_target") or "").strip() + + if primary: + objectives = [ + (primary, _objective_direction(context.get("primary_objective"), "maximize", "Primary")) + ] + if secondary: + if secondary == primary: + raise UnsupportedOptimizationError( + "Secondary optimization target must differ from the primary target" + ) + objectives.append( + ( + secondary, + _objective_direction( + context.get("secondary_objective"), "minimize", "Secondary" + ), + ) + ) + output_names = { + str(output.get("name") or "").strip() + for output in (output_snapshot or []) + if isinstance(output, dict) and not output.get("artifact") + } + if not output_names: + raise UnsupportedOptimizationError( + "This unit experiment defines no outputs, so it has no optimization target" + ) + for target, _direction in objectives: + if target not in output_names: + raise UnsupportedOptimizationError( + f"Optimization target '{target}' is not an output of this unit experiment" + ) + if target not in domain["targets"]: + raise UnsupportedOptimizationError( + f"Optimization target '{target}' is not supported for " + f"{domain['display_name']} (supported targets: " + f"{', '.join(domain['targets'])})." + ) + elif strict: + raise UnsupportedOptimizationError( + "Primary optimization target is required for optimization runs" + ) + else: + objectives = list(domain["default_objectives"]) + + selected: dict[str, dict[str, Any]] = {} + unsupported_inputs: list[str] = [] + for param in input_snapshot or []: + if not isinstance(param, dict): + continue + name = str(param.get("name") or "").strip() + if not name or _input_mode(param) == "not_used" or param.get("source") == "sample": + continue + if name in domain["feature_limits"]: + selected[name] = param + else: + unsupported_inputs.append(name) + if unsupported_inputs: + raise UnsupportedOptimizationError( + f"These equipment inputs are not inputs of the {domain['display_name']} " + f"proposal model: {', '.join(unsupported_inputs)}." + ) + not_selected = [name for name in domain["features"] if name not in selected] + if not_selected: + raise UnsupportedOptimizationError( + "Optimization needs every proposal-model input to be Fixed or Variable; " + f"not selected: {', '.join(not_selected)}." + ) + + planned_values: dict[str, float] = {} + for item in planned_parameters or []: + if not isinstance(item, dict): + continue + number = _to_float(item.get("planned_value")) + if number is not None and math.isfinite(number): + planned_values[str(item.get("name") or "").strip()] = number + + constraints: dict[str, list[float]] = {} + for name in domain["features"]: + param = selected[name] + limit_low, limit_high = domain["feature_limits"][name] + unit = domain["feature_units"].get(name, "") + limits_text = f"{limit_low:g}–{limit_high:g}{(' ' + unit) if unit else ''}" + if _input_mode(param) == "variable": + declared_low = _declared_range_bound(param, ("min_value", "minimum"), name) + declared_high = _declared_range_bound(param, ("max_value", "maximum"), name) + low = limit_low if declared_low is None else declared_low + high = limit_high if declared_high is None else declared_high + if declared_low is None or declared_high is None: + raise UnsupportedOptimizationError( + f"Variable input '{name}' needs a complete minimum and maximum range" + ) + if low > high: + raise UnsupportedOptimizationError( + f"Variable input '{name}' has an inverted range ({low:g} > {high:g})" + ) + if low < limit_low or high > limit_high: + raise UnsupportedOptimizationError( + f"Variable input '{name}' range {low:g}–{high:g} is outside the " + f"equipment limits {limits_text}" + ) + constraints[name] = [low, high] + else: + if name not in planned_values: + raise UnsupportedOptimizationError( + f"Fixed input '{name}' needs a planned value for optimization" + ) + value = planned_values[name] + if value < limit_low or value > limit_high: + raise UnsupportedOptimizationError( + f"Fixed input '{name}' value {value:g} is outside the equipment " + f"limits {limits_text}" + ) + constraints[name] = [value, value] + + domain_config = { + "features": list(domain["features"]), + "targets": [target for target, _direction in objectives], + "feature_display": dict(domain["feature_display"]), + "feature_units": dict(domain["feature_units"]), + "feature_limits": {name: list(bounds) for name, bounds in domain["feature_limits"].items()}, + "ml_model": "gpr", + } + # The legacy engine hard-codes these directions. A changed equipment + # default must still steer ranking and therefore needs explicit objectives. + explicit = tuple(objectives) != (("AvgEtchRate", "maximize"), ("RangeEtchRate", "minimize")) + return { + "domain_id": domain["domain_id"], + "constraints": constraints, + "domain_config": domain_config, + "objectives": ( + [{"target": target, "direction": direction} for target, direction in objectives] + if explicit + else None + ), + } + + +def _proposal_engine_kwargs(request: dict[str, Any]) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "constraints": request["constraints"], + "domain_config": request["domain_config"], + } + if request.get("objectives") is not None: + kwargs["objectives"] = request["objectives"] + return kwargs def _project_experiment_row_to_dict(row: dict[str, Any]) -> dict[str, Any]: @@ -4827,12 +5475,19 @@ def create_experiment_definition_pg( input_snapshot: list[Any] = [] output_snapshot: list[Any] = [] additional_inputs_snapshot: list[Any] = [] + planned_parameters = payload.get("planned_parameters", []) + optimization_context = payload.get("optimization_context") or {} + is_optimization = ( + str(optimization_context.get("mode") or "none").strip().lower() + == "optimization" + ) if type_id: visibility_sql, visibility_params = _visible_experiment_type_filter(user) cur.execute( f""" SELECT equipment_id, + execution_kind, type_parameters_json, outputs_json, additional_inputs_json @@ -4855,10 +5510,28 @@ def create_experiment_definition_pg( type_row.get("additional_inputs_json"), [], ) - _validate_planned_parameters_against_experiment_inputs( - payload.get("planned_parameters", []), + computation_tool = registered_computation_tool_for_template( + type_id, input_snapshot, + str(type_row.get("execution_kind") or ""), ) + if computation_tool is not None: + raise RegisteredComputationTemplateError(computation_tool) + planned_parameters = _validate_planned_parameters_against_experiment_inputs( + planned_parameters, + input_snapshot, + additional_inputs_snapshot, + optimization=is_optimization, + ) + if is_optimization: + resolve_optimization_request( + equipment_id=selected_equipment or template_equipment_id, + input_snapshot=input_snapshot, + planned_parameters=planned_parameters, + optimization_context=optimization_context, + output_snapshot=output_snapshot, + strict=True, + ) cur.execute( """ @@ -4901,8 +5574,8 @@ def create_experiment_definition_pg( execution_request_id, payload.get("experiment_name", ""), payload.get("planned_date") or None, - Json(payload.get("planned_parameters", [])), - Json(payload.get("optimization_context") or {}), + Json(planned_parameters), + Json(optimization_context), Json(input_snapshot), Json(output_snapshot), Json(additional_inputs_snapshot), @@ -4989,77 +5662,6 @@ def create_experiment_definition_pg( conn.close() -def get_experiment_type_constraints_pg(type_id: str) -> dict[str, list[float | None]]: - if not type_id: - return {} - - conn = get_pg_worker_connection() - try: - with conn.cursor(cursor_factory=RealDictCursor) as cur: - cur.execute( - "SELECT type_parameters_json FROM experiment_types WHERE id = %s", - (type_id,), - ) - row = cur.fetchone() - finally: - conn.close() - - params = _coerce_json(row["type_parameters_json"], []) if row else [] - constraints: dict[str, list[float | None]] = {} - for param in params: - if not isinstance(param, dict): - continue - mode = str(param.get("mode") or "").strip().lower() - if mode == "not_used" or not param.get("is_variable", True): - continue - name = str(param.get("name") or "").strip() - if not name: - continue - min_v = _to_float(param.get("min_value")) - max_v = _to_float(param.get("max_value")) - if min_v is not None or max_v is not None: - constraints[name] = [min_v, max_v] - return constraints - - -def _constraints_from_type_parameters(params: list[Any]) -> dict[str, list[float | None]]: - constraints: dict[str, list[float | None]] = {} - for param in params: - if not isinstance(param, dict): - continue - mode = str(param.get("mode") or "").strip().lower() - if mode == "not_used" or not param.get("is_variable", True): - continue - name = str(param.get("name") or "").strip() - if not name: - continue - min_v = _to_float(param.get("min_value")) - max_v = _to_float(param.get("max_value")) - if min_v is not None or max_v is not None: - constraints[name] = [min_v, max_v] - return constraints - - -def _merge_planned_parameters_into_constraints( - constraints: dict[str, list[float | None]], - planned_parameters: list[Any], -) -> dict[str, list[float | None]]: - merged = dict(constraints) - for param in planned_parameters: - if not isinstance(param, dict): - continue - name = str(param.get("name") or "").strip() - planned_value = param.get("planned_value") - if not name or planned_value in (None, ""): - continue - try: - value = float(planned_value) - except (TypeError, ValueError): - continue - merged[name] = [value, value] - return merged - - def _proposal_feature_distance( proposal: dict[str, Any], run_features: dict[str, float], @@ -5353,8 +5955,11 @@ def claim_experiment_proposal_job_pg(*, experiment_id: str = "") -> dict[str, An e.project_id, e.type_id, e.execution_request_id, + e.equipment_id, e.planned_parameters_json, e.input_snapshot_json, + e.output_snapshot_json, + e.optimization_context_json, e.proposal_generation_attempts, e.proposal_generation_claim_id::text AS claim_id """, @@ -5399,23 +6004,23 @@ def run_experiment_proposal_job_pg(job: dict[str, Any]) -> dict[str, Any]: experiment_id = str(job.get("experiment_id") or "") claim_id = str(job.get("claim_id") or "") project_id = str(job.get("project_id") or "") - type_id = str(job.get("type_id") or "") execution_request_id = str(job.get("execution_request_id") or "") - planned_parameters = _coerce_json(job.get("planned_parameters_json"), []) - input_snapshot = _coerce_json(job.get("input_snapshot_json"), []) try: - constraints = _merge_planned_parameters_into_constraints( - _constraints_from_type_parameters(input_snapshot) - if input_snapshot - else get_experiment_type_constraints_pg(type_id), - planned_parameters, + # The claimed row's stored values are the validated, normalized + # values for this run, and its objectives come from the definition. + request = resolve_optimization_request( + equipment_id=str(job.get("equipment_id") or ""), + input_snapshot=_coerce_json(job.get("input_snapshot_json"), []), + planned_parameters=_coerce_json(job.get("planned_parameters_json"), []), + optimization_context=_coerce_json(job.get("optimization_context_json"), {}), + output_snapshot=_coerce_json(job.get("output_snapshot_json"), []), ) from ml_engine import compute_proposals optimizer_result = compute_proposals( - constraints=constraints, + **_proposal_engine_kwargs(request), n_batches=1, batch_size=3, optimization_context={ @@ -5810,9 +6415,12 @@ def auto_generate_recipe_proposals_for_projects_pg( e.id::text AS experiment_id, e.project_id, e.type_id, + e.equipment_id, e.execution_request_id, e.planned_parameters_json, e.input_snapshot_json, + e.output_snapshot_json, + e.optimization_context_json, t.type_parameters_json, COALESCE(( SELECT MAX(b.iteration) @@ -6012,17 +6620,43 @@ def auto_generate_recipe_proposals_for_projects_pg( skipped.append({"project_id": project_id, "reason": "already_current"}) continue - type_params = _coerce_json( - row.get("input_snapshot_json"), - [], - ) or _coerce_json(row.get("type_parameters_json"), []) + # The run's own snapshot is authoritative; the live template + # may have changed since the run was created. + input_snapshot = _coerce_json(row.get("input_snapshot_json"), []) planned_params = _coerce_json(row.get("planned_parameters_json"), []) - constraints = _merge_planned_parameters_into_constraints( - _constraints_from_type_parameters(type_params), - planned_params, - ) - experiment_id = str(row["experiment_id"]) + try: + request = resolve_optimization_request( + equipment_id=str(row.get("equipment_id") or ""), + input_snapshot=input_snapshot, + planned_parameters=planned_params, + optimization_context=_coerce_json( + row.get("optimization_context_json"), + {}, + ), + output_snapshot=_coerce_json( + row.get("output_snapshot_json"), + [], + ), + ) + except UnsupportedOptimizationError as exc: + # Retrying cannot fix an unsupported configuration, so + # this is recorded as handled rather than generation_failed. + update_experiment_proposal_generation_status_pg( + experiment_id=experiment_id, + status="failed", + error=str(exc)[:2000], + ) + skipped.append( + { + "project_id": project_id, + "experiment_id": experiment_id, + "reason": "unsupported_optimization", + "error": str(exc), + } + ) + continue + update_experiment_proposal_generation_status_pg( experiment_id=experiment_id, status="generating", @@ -6031,7 +6665,7 @@ def auto_generate_recipe_proposals_for_projects_pg( from ml_engine import compute_proposals optimizer_result = compute_proposals( - constraints=constraints, + **_proposal_engine_kwargs(request), n_batches=1, batch_size=3, training_iteration=max(1, int(row.get("max_iteration") or 1)), diff --git a/api/ml_engine.py b/api/ml_engine.py index f74a1db..b97566c 100644 --- a/api/ml_engine.py +++ b/api/ml_engine.py @@ -109,6 +109,7 @@ def _resolve_config(domain_config: dict | None) -> dict: "targets": domain_config.get("targets", [ETCHER_PRIMARY_TARGET]), "feature_display": domain_config.get("feature_display", {}), "feature_units": domain_config.get("feature_units", {}), + "feature_limits": domain_config.get("feature_limits", {}), "ml_model": domain_config.get("ml_model", "gpr"), } @@ -390,17 +391,35 @@ def _feature_bounds( df: pd.DataFrame, features: list[str], constraints: dict[str, list] | None, + limits: dict[str, list] | None = None, ) -> tuple[np.ndarray, np.ndarray]: + """Candidate bounds per feature. + + With configured ``limits`` (selected equipment config) the bounds are + strict: every feature needs finite limits, constraints must be finite and + lie inside them, and an empty intersection raises instead of being + swapped, so proposals never leave the declared limits. Without limits the + established etcher behavior is unchanged. + """ + strict = bool(limits) lower = [] upper = [] for feature in features: - default_min, default_max = ETCHER_DEFAULT_RANGES.get( - feature, - ( - float(df[feature].min()) if feature in df.columns else 0.0, - float(df[feature].max()) if feature in df.columns else 1.0, - ), - ) + if strict: + try: + default_min, default_max = (float(value) for value in limits[feature]) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"No configured limits for feature '{feature}'") from exc + if not (np.isfinite(default_min) and np.isfinite(default_max)) or default_min > default_max: + raise ValueError(f"Configured limits for feature '{feature}' are invalid") + else: + default_min, default_max = ETCHER_DEFAULT_RANGES.get( + feature, + ( + float(df[feature].min()) if feature in df.columns else 0.0, + float(df[feature].max()) if feature in df.columns else 1.0, + ), + ) min_v = default_min max_v = default_max bounds = (constraints or {}).get(feature) @@ -409,9 +428,19 @@ def _feature_bounds( cmin = float(bounds[0]) if len(bounds) > 0 and bounds[0] not in (None, "") else None cmax = float(bounds[1]) if len(bounds) > 1 and bounds[1] not in (None, "") else None except (TypeError, ValueError): + if strict: + raise ValueError(f"Constraint for feature '{feature}' is not numeric") cmin = None cmax = None + if strict and any( + value is not None and not np.isfinite(value) for value in (cmin, cmax) + ): + raise ValueError(f"Constraint for feature '{feature}' is not finite") if cmin is not None and cmax is not None and cmin == cmax: + if strict and not default_min <= cmin <= default_max: + raise ValueError( + f"Fixed value for feature '{feature}' is outside its configured limits" + ) min_v = max_v = cmin else: if cmin is not None: @@ -419,6 +448,10 @@ def _feature_bounds( if cmax is not None: max_v = min(max_v, cmax) if min_v > max_v: + if strict: + raise ValueError( + f"Constraint for feature '{feature}' does not intersect its configured limits" + ) min_v, max_v = max_v, min_v lower.append(min_v) upper.append(max_v) @@ -498,16 +531,24 @@ def _pareto_improvement_scores( candidate_range: np.ndarray, observed_rate: np.ndarray, observed_range: np.ndarray, + primary_sign: float = 1.0, + secondary_sign: float = -1.0, ) -> tuple[np.ndarray, np.ndarray]: + """Score candidates against the observed Pareto front. + + Signs map each objective into maximize space: +1 maximizes and -1 + minimizes. The defaults are the etcher objectives (maximize rate, + minimize range). + """ rate_span = max(float(np.ptp(observed_rate)), 1.0) range_span = max(float(np.ptp(observed_range)), 1.0) observed = np.column_stack([ - observed_rate / rate_span, - -observed_range / range_span, + primary_sign * observed_rate / rate_span, + secondary_sign * observed_range / range_span, ]) candidates = np.column_stack([ - candidate_rate / rate_span, - -candidate_range / range_span, + primary_sign * candidate_rate / rate_span, + secondary_sign * candidate_range / range_span, ]) front = observed[_pareto_mask_maximize(observed)] if len(front) == 0: @@ -767,6 +808,51 @@ def _constraint_mask(df: pd.DataFrame, constraints: dict | None, features: list[ return mask +def _objective_signs(objectives: list[dict] | None) -> list[tuple[str, float]] | None: + """Validate explicit objectives into ``[(target, sign)]`` (max 2).""" + if objectives is None: + return None + if not isinstance(objectives, list) or not 1 <= len(objectives) <= 2: + raise ValueError("Optimization needs one or two objectives") + specs: list[tuple[str, float]] = [] + for objective in objectives: + target = str((objective or {}).get("target") or "").strip() + direction = str((objective or {}).get("direction") or "").strip().lower() + if not target or direction not in {"maximize", "minimize"}: + raise ValueError( + "Each objective needs a target and a 'maximize' or 'minimize' direction" + ) + if any(target == existing for existing, _sign in specs): + raise ValueError(f"Optimization target '{target}' is repeated") + specs.append((target, 1.0 if direction == "maximize" else -1.0)) + return specs + + +def _explicit_prediction_fields( + pidx: int, + means: dict[str, np.ndarray], + stds: dict[str, np.ndarray], +) -> dict: + """Per-target predictions keyed by name, plus compat fields by target. + + The legacy ``predicted_etch_rate``/``predicted_range_nm`` fields are + populated from the matching target (never positionally) so a proposal + for a different target order cannot display the wrong quantity. + """ + rate = means.get(ETCHER_PRIMARY_TARGET) + spread = means.get(ETCHER_SECONDARY_TARGET) + rate_std = stds.get(ETCHER_PRIMARY_TARGET) + spread_std = stds.get(ETCHER_SECONDARY_TARGET) + return { + "predicted_values": {target: float(values[pidx]) for target, values in means.items()}, + "predicted_uncertainties": {target: float(values[pidx]) for target, values in stds.items()}, + "predicted_etch_rate": float(rate[pidx]) if rate is not None else 0.0, + "predicted_range_nm": float(spread[pidx] * 5.0) if spread is not None else 0.0, + "uncertainty_rate": float(rate_std[pidx]) if rate_std is not None else 0.0, + "uncertainty_range": float(spread_std[pidx] * 5.0) if spread_std is not None else 0.0, + } + + def compute_proposals( domain_config: dict | None = None, constraints: dict | None = None, @@ -774,6 +860,7 @@ def compute_proposals( batch_size: int = 3, training_iteration: int | None = None, optimization_context: dict | None = None, + objectives: list[dict] | None = None, ) -> dict: """ Bayesian Optimization proposals using GPR acquisition (Expected Improvement). @@ -785,8 +872,18 @@ def compute_proposals( - selection_reason: "exploitation" | "exploration" | "diversity" When RF is active, proposals are sampled randomly from feature space. + + ``objectives`` (``[{"target", "direction"}]``, one or two entries) + selects the targets and their maximize/minimize directions explicitly. + Candidates are then ranked by those directions, including on the RF + path, and a target missing from the training data raises ValueError. + Without it, the established etcher behavior (maximize the primary, + minimize the secondary target) is unchanged. """ + objective_specs = _objective_signs(objectives) cfg = _resolve_config(domain_config) + if objective_specs is not None: + cfg = {**cfg, "targets": [target for target, _sign in objective_specs]} scoped_context = dict(optimization_context or {}) if training_iteration is not None: scoped_context.setdefault("training_iteration", training_iteration) @@ -795,12 +892,23 @@ def compute_proposals( if len(X) == 0: return {"source": "scoped_empty", "total_iterations": 0, "batches": []} + if objective_specs is not None: + missing_targets = [target for target, _sign in objective_specs if target not in ys] + if missing_targets: + raise ValueError( + "Optimization target(s) not available in the training data: " + + ", ".join(missing_targets) + ) if primary_target not in ys: return {"source": "simulated", "total_iterations": 0, "batches": []} y = ys[primary_target] secondary_target = cfg["targets"][1] if len(cfg["targets"]) > 1 else None y_secondary = ys.get(secondary_target) if secondary_target else None + primary_sign = objective_specs[0][1] if objective_specs else 1.0 + secondary_sign = ( + objective_specs[1][1] if objective_specs and len(objective_specs) > 1 else -1.0 + ) features = cfg["features"] valid_features = [f for f in features if f in df.columns] snapshot_metadata = _training_snapshot_metadata( @@ -814,6 +922,21 @@ def compute_proposals( valid_features, cfg.get("feature_display", {}), ) + strict_limits = cfg.get("feature_limits") or None + strict_bounds = None + if strict_limits: + # A selected equipment config is authoritative: every configured + # feature must be trainable and every constraint must be applied. + missing_features = [f for f in features if f not in valid_features] + if missing_features: + raise ValueError( + "Training data lacks configured features: " + ", ".join(missing_features) + ) + if ignored_constraints: + raise ValueError( + "Constraints reference unknown features: " + ", ".join(ignored_constraints) + ) + strict_bounds = _feature_bounds(df, valid_features, canonical_constraints, strict_limits) proposal_engine = os.getenv("NEXTJS_PROPOSAL_ENGINE", "legacy").strip().lower() legacy_feature_set = { @@ -823,8 +946,11 @@ def compute_proposals( "Etch_AvgPres", "Etch_Avgcf4Flow", } + # The legacy engine hard-codes maximize-rate/minimize-range, so it only + # serves the implicit etcher default, never explicit objectives. if ( proposal_engine == "legacy" + and objective_specs is None and set(valid_features) == legacy_feature_set and y_secondary is not None ): @@ -841,6 +967,11 @@ def compute_proposals( batch_size=batch_size, training_iteration=training_iteration, ) + if strict_bounds is not None: + # The legacy engine intersects with its own parameter-limit + # sets and may swap an empty intersection; never accept a + # proposal outside the selected limits (falls back below). + _assert_proposals_within_bounds(result, valid_features, *strict_bounds) result["constraints_applied"] = canonical_constraints result["constraints_ignored"] = ignored_constraints result.update(snapshot_metadata) @@ -882,7 +1013,7 @@ def compute_proposals( batch_size * 20, int(os.getenv("ML_PROPOSAL_CANDIDATES", "4096")), ) - lower, upper = _feature_bounds(df, valid_features, canonical_constraints) + lower, upper = strict_bounds or _feature_bounds(df, valid_features, canonical_constraints) X_candidates = _sample_candidate_space(lower, upper, valid_features, n_candidates) if "Etch_Avg_Rf1_Pow" in valid_features and "Etch_Avg_Rf2_Pow" in valid_features: @@ -930,11 +1061,13 @@ def compute_proposals( mu_secondary, y[idx_train], y_secondary[idx_train], + primary_sign, + secondary_sign, ) combined_sigma = np.sqrt((sigma_primary / rate_span) ** 2 + (sigma_secondary / range_span) ** 2) else: - candidate_objective = mu_primary - observed_objective = y[idx_train] + candidate_objective = primary_sign * mu_primary + observed_objective = primary_sign * y[idx_train] combined_sigma = sigma_primary from scipy.stats import norm as _norm @@ -986,8 +1119,34 @@ def compute_proposals( else: mu_secondary = np.zeros(len(X_candidates), dtype=float) reasons = ["exploitation", "exploration", "diversity"] - for j in range(batch_size): - pidx = int(rng.randint(0, len(X_candidates))) + if objective_specs is not None: + # Explicit objectives must steer selection, so rank by the + # directed prediction instead of sampling at random. + if y_secondary is not None: + rf_scores, _observed = _pareto_improvement_scores( + mu_primary, + mu_secondary, + y[idx_train], + y_secondary[idx_train], + primary_sign, + secondary_sign, + ) + else: + rf_scores = primary_sign * mu_primary + selected_idxs: list[int] = [] + for ridx in np.argsort(rf_scores)[::-1]: + if len(selected_idxs) >= batch_size: + break + if all( + np.linalg.norm(X_candidates_scaled[ridx] - X_candidates_scaled[s]) > 0.3 + for s in selected_idxs + ): + selected_idxs.append(int(ridx)) + else: + selected_idxs = [ + int(rng.randint(0, len(X_candidates))) for _ in range(batch_size) + ] + for j, pidx in enumerate(selected_idxs): param_values = { feat: float(X_candidates[pidx, feature_idx]) for feature_idx, feat in enumerate(valid_features) @@ -1006,6 +1165,18 @@ def compute_proposals( "uncertainty_range": 0.0, }) + if objective_specs is not None: + zeros = np.zeros(len(X_candidates), dtype=float) + means = {primary_target: mu_primary} + stds = {primary_target: sigma_primary if use_gpr else zeros} + if y_secondary is not None: + means[secondary_target] = mu_secondary + stds[secondary_target] = sigma_secondary if use_gpr else zeros + for proposal in proposals: + proposal.update( + _explicit_prediction_fields(int(proposal["point_index"]), means, stds) + ) + batches.append({ "iteration": it, "n_train": int(len(idx_train)), @@ -1019,7 +1190,7 @@ def compute_proposals( source = "registry_gpr" else: source = "local_gpr" if use_gpr else "local_rf" - return { + result = { "source": source, "total_iterations": len(batches), "batches": batches, @@ -1030,3 +1201,9 @@ def compute_proposals( "model_trained_at": registry_bundle.get("trained_at") if registry_bundle else None, "model_snapshot_hash": registry_bundle.get("snapshot_hash") if registry_bundle else None, } + if objective_specs is not None: + result["objectives"] = [ + {"target": target, "direction": "maximize" if sign > 0 else "minimize"} + for target, sign in objective_specs + ] + return result diff --git a/api/rcac-endpoints/client.env.example b/api/rcac-endpoints/client.env.example new file mode 100644 index 0000000..d033108 --- /dev/null +++ b/api/rcac-endpoints/client.env.example @@ -0,0 +1,6 @@ +# Store the real file as ~/.config/dt-globus/client.env with mode 0600. +# Store only the secret value in client-secret with mode 0600. The launcher +# reads it without placing the value in a command argument or shell source file. +# Never commit or transmit the secret through chat, email, or a support ticket. +GLOBUS_COMPUTE_CLIENT_ID=2fea1cb8-521c-4f61-9b0b-5d7226389f66 +GLOBUS_COMPUTE_CLIENT_SECRET_FILE=$HOME/.config/dt-globus/client-secret diff --git a/api/rcac-endpoints/gautschi.env.example b/api/rcac-endpoints/gautschi.env.example new file mode 100644 index 0000000..cbb9822 --- /dev/null +++ b/api/rcac-endpoints/gautschi.env.example @@ -0,0 +1,9 @@ +DT_CLUSTER_NAME=gautschi +DT_WORK_ROOT=/scratch/gautschi/dtwin0/digital-twins/runs +DT_PSEUDO_ROOT=/depot/ncn/data/digital-twins/pseudopotentials +DT_ARTIFACT_ROOT=/depot/ncn/data/digital-twins/runs +DT_SLURM_ACCOUNT=ncn +DT_SLURM_PARTITION=cpu +DT_SLURM_QOS=normal +DT_QE_MODULES_JSON='["gcc/11.4.1","openmpi/4.1.6","quantum-espresso/7.3.1"]' +DT_MPI_LAUNCHER=mpirun diff --git a/api/rcac-endpoints/negishi.env.example b/api/rcac-endpoints/negishi.env.example new file mode 100644 index 0000000..4ca128a --- /dev/null +++ b/api/rcac-endpoints/negishi.env.example @@ -0,0 +1,9 @@ +DT_CLUSTER_NAME=negishi +DT_WORK_ROOT=/scratch/negishi/dtwin0/digital-twins/runs +DT_PSEUDO_ROOT=/depot/ncn/data/digital-twins/pseudopotentials +DT_ARTIFACT_ROOT=/depot/ncn/data/digital-twins/runs +DT_SLURM_ACCOUNT=ncn +DT_SLURM_PARTITION=cpu +DT_SLURM_QOS=normal +DT_QE_MODULES_JSON='["gcc/12.2.0","openmpi/4.1.4","quantum-espresso/7.1"]' +DT_MPI_LAUNCHER=srun diff --git a/api/requirements-computations.txt b/api/requirements-computations.txt new file mode 100644 index 0000000..4adbdd5 --- /dev/null +++ b/api/requirements-computations.txt @@ -0,0 +1,8 @@ +-r requirements.txt + +# Scientific dependencies are isolated in the worker image so the existing +# FastAPI image remains unchanged. +pymatgen>=2025.6,<2027 +mp-api>=0.45,<0.47 +# The registered gateway uses PureSourceTextInspect, verified against 3.16.1. +globus-compute-sdk>=3.16.1,<4 diff --git a/api/requirements.txt b/api/requirements.txt index 99a48fb..cbf5a5b 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -7,6 +7,7 @@ psycopg2-binary>=2.9.0 python-multipart>=0.0.9 email-validator>=2.2.0 requests>=2.32.0 +cryptography>=43.0.0 azure-functions>=1.20.0 azure-storage-blob>=12.20.0 azure-identity>=1.17.0 diff --git a/api/routers/computations.py b/api/routers/computations.py new file mode 100644 index 0000000..cbcd21d --- /dev/null +++ b/api/routers/computations.py @@ -0,0 +1,309 @@ +"""Feature-flagged API for registered RCAC computational experiments.""" + +from __future__ import annotations + +import json +import os +from urllib.parse import quote +from typing import Annotated, Any + +from computations.artifacts import ( + AzureBlobArtifactStore, + LocalArtifactStore, + artifact_store_from_env, + depot_downloads_enabled, + fetch_depot_artifact, +) +from computations.globus_gateway import GlobusComputeGateway, GlobusGatewayError +from computations.rcac import configured_clusters, get_cluster +from computations.resources import ResourceRequest +from computations.schemas import ( + TOOL_DEFINITIONS, + compute_input_fingerprint, + validate_tool_inputs, +) +from computations.store import ( + SubmissionCapacityError, + create_run, + feature_enabled, + get_run, + list_runs, + request_cancel, + retry_collection, +) +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import FileResponse, StreamingResponse, Response +from starlette.background import BackgroundTask +from pydantic import BaseModel, ConfigDict, Field +from security import PlatformUser, get_platform_user + +router = APIRouter(prefix="/computations", tags=["computations"]) +CurrentUser = Annotated[PlatformUser, Depends(get_platform_user)] + + +def _require_enabled() -> None: + if not feature_enabled(): + raise HTTPException(status_code=404, detail="RCAC computations are not enabled") + + +class RunPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + cluster: str = Field(min_length=1, max_length=100, pattern=r"^[A-Za-z0-9_-]+$") + inputs: dict[str, Any] + # Bounded integers only, validated against the cluster's published + # policy; never part of ``inputs`` or the scientific fingerprint. + resources: ResourceRequest | None = None + + +@router.get("/tools") +def tools(user: CurrentUser): + _require_enabled() + del user + return { + "tools": [definition.public_dict() for definition in TOOL_DEFINITIONS.values()], + "clusters": [ + cluster.public_dict() for cluster in configured_clusters().values() + ], + "attribution": { + "model": "platform", + "scheduler_owner": "rcac_service_account", + "mapping": "authenticated_user -> computation_run -> Slurm comment", + }, + } + + +@router.post("/projects/{project_id}/runs/{tool_name}", status_code=202) +def submit_run( + project_id: str, + tool_name: str, + payload: RunPayload, + user: CurrentUser, +): + _require_enabled() + try: + definition = TOOL_DEFINITIONS[tool_name] + except KeyError as exc: + raise HTTPException( + status_code=404, detail="Computation tool not found" + ) from exc + try: + cluster = get_cluster(payload.cluster) + execution_identity = os.getenv("RCAC_SERVICE_ACCOUNT_USERNAME", "").strip() + if not execution_identity: + raise ValueError("RCAC service-account execution is not configured") + inputs = validate_tool_inputs(tool_name, payload.inputs) + input_fingerprint = compute_input_fingerprint(tool_name, inputs) + resource_request = cluster.resource_policy.record(tool_name, payload.resources) + upstream = inputs.get("upstream_execution_id", "") + return create_run( + user=user, + project_id=project_id, + tool_name=tool_name, + tool_revision=definition.revision, + input_fingerprint=input_fingerprint, + inputs=inputs, + cluster=cluster, + execution_identity=execution_identity, + upstream_run_id=str(upstream), + resource_request=resource_request, + ) + except SubmissionCapacityError as exc: + raise HTTPException(status_code=429, detail=str(exc), headers={"Retry-After": "30"}) from exc + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + except (RuntimeError, TypeError, ValueError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/projects/{project_id}/runs") +def project_runs(project_id: str, user: CurrentUser): + _require_enabled() + try: + return list_runs(project_id, user) + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + + +@router.get("/runs/{run_id}") +def run_detail(run_id: str, user: CurrentUser): + _require_enabled() + try: + value = get_run(run_id, user) + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + if not value: + raise HTTPException(status_code=404, detail="Computation run not found") + return value + + +@router.post("/runs/{run_id}/cancel", status_code=202) +def cancel_run(run_id: str, user: CurrentUser): + _require_enabled() + try: + value = request_cancel(run_id, user) + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + if not value: + raise HTTPException(status_code=404, detail="Computation run not found") + return value + + +@router.get("/runs/{run_id}/artifacts/{artifact_index}") +def download_artifact( + run_id: str, + artifact_index: int, + user: CurrentUser, +): + """Project-authorized, bounded, integrity-checked artifact download.""" + + _require_enabled() + try: + run = get_run(run_id, user, internal=True) + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + if not run: + raise HTTPException(status_code=404, detail="Computation run not found") + manifest = run.get("artifact_manifest_json") or [] + if isinstance(manifest, str): + manifest = json.loads(manifest) + if artifact_index < 0 or artifact_index >= len(manifest): + raise HTTPException(status_code=404, detail="Artifact not found") + artifact = manifest[artifact_index] + if "storage_uri" in artifact and depot_downloads_enabled(): + source = run + visited = {str(run["id"])} + try: + # Cached results point to the original archive. Check authorization + # again at every hop rather than trusting a caller-visible URI. + for _ in range(10): + external = source.get("external_workflow_ref_json") or {} + if isinstance(external, str): + external = json.loads(external) + if not external.get("cache_hit"): + break + source_id = str(external.get("source_run_id", "")) + if not source_id or source_id in visited: + raise ValueError("Invalid cache lineage") + visited.add(source_id) + source = get_run(source_id, user, internal=True) + if not source or source["project_id"] != run["project_id"]: + raise PermissionError("Archive is outside the project") + else: + raise ValueError("Cache lineage is too deep") + staged = fetch_depot_artifact(source, artifact, get_cluster(source["cluster"]), GlobusComputeGateway()) + except PermissionError as exc: + raise HTTPException(status_code=403, detail="Artifact access denied") from exc + except BlockingIOError as exc: + raise HTTPException(status_code=429, detail=str(exc), headers={"Retry-After": "30"}) from exc + except (ValueError, TypeError, KeyError) as exc: + raise HTTPException(status_code=409, detail="Artifact manifest or integrity check failed, or file exceeds 8 MiB") from exc + except GlobusGatewayError as exc: + raise HTTPException(status_code=503, detail="Archive endpoint is unavailable; retry later") from exc + + def chunks(): + try: + while chunk := staged.read(64 * 1024): + yield chunk + finally: + staged.close() + + filename = quote(str(artifact.get("name", "artifact")).rsplit("/", 1)[-1], safe="") + return StreamingResponse(chunks(), media_type="application/octet-stream", + background=BackgroundTask(staged.close), headers={ + "Content-Disposition": f"attachment; filename*=UTF-8''{filename}", + "Cache-Control": "private, no-store", "X-Content-Type-Options": "nosniff", + "Content-Length": str(artifact["size_bytes"]), + }) + if artifact.get("downloadable") is False or "storage_uri" in artifact: + raise HTTPException( + status_code=409, + detail="Raw artifact is retained in the ncn Depot; pilot downloads are disabled", + ) + artifact_name = str(artifact.get("name") or "artifact") + store = artifact_store_from_env() + try: + if isinstance(store, LocalArtifactStore): + return FileResponse( + store.resolve(artifact["uri"]), + media_type=artifact.get("content_type") or "application/octet-stream", + filename=artifact_name.rsplit("/", 1)[-1], + ) + if isinstance(store, AzureBlobArtifactStore): + downloader = store.downloader(artifact["uri"]) + return StreamingResponse( + downloader.chunks(), + media_type=artifact.get("content_type") or "application/octet-stream", + headers={ + "Content-Disposition": ( + 'attachment; filename="' + + artifact_name.rsplit("/", 1)[-1].replace('"', "") + + '"' + ) + }, + ) + except (FileNotFoundError, ValueError, PermissionError) as exc: + raise HTTPException(status_code=404, detail="Artifact unavailable") from exc + raise HTTPException(status_code=500, detail="Artifact store is unsupported") + + +@router.post("/runs/{run_id}/retry-collection", status_code=202) +def retry_results(run_id: str, user: CurrentUser): + _require_enabled() + try: + result = retry_collection(run_id, user) + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + if not result: + raise HTTPException(status_code=404, detail="Computation run not found") + return result + + +@router.get("/runs/{run_id}/diagnostics") +def export_diagnostics(run_id: str, user: CurrentUser): + """Failure diagnostics, resources and inherited settings as one JSON file.""" + run = run_detail(run_id, user) + prepared = run.get("prepared_manifest") or {} + external = run.get("external_workflow_ref") or {} + document = { + "schema": "dt-computation-diagnostics-export/1", + "run_id": run["id"], + "tool_name": run.get("tool_name"), + "tool_revision": run.get("tool_revision"), + "status": run.get("status"), + "cluster": run.get("cluster"), + "scheduler_job_id": run.get("scheduler_job_id"), + "upstream_run_id": run.get("upstream_run_id"), + "error": run.get("error"), + "diagnostics": run.get("diagnostics"), + "inherited_settings": run.get("elastic_inherited_settings"), + "resource_request": run.get("resource_request"), + "submitted_resources": external.get("submitted_resources"), + "resource_confirmation": external.get("resource_confirmation"), + "scheduler_resource_usage": external.get("scheduler_resource_usage"), + "scheduler_accounting": external.get("scheduler_accounting"), + "cache_hit": bool(external.get("cache_hit")), + "prepared": {key: prepared.get(key) for key in ("ntasks", "time_limit", "job_array", "resources")}, + "artifacts": [ + {key: item.get(key) for key in ("name", "sha256", "size_bytes", "uri")} + for item in run.get("artifact_manifest") or [] + ], + } + return Response( + json.dumps(document, indent=2, allow_nan=False), + media_type="application/json", + headers={"Content-Disposition": f'attachment; filename="computation-{run["id"]}-diagnostics.json"'}, + ) + + +@router.get("/runs/{run_id}/export") +def export_result(run_id: str, user: CurrentUser): + """Portable inputs/results/provenance; no credentials or Python pickles.""" + run = run_detail(run_id, user) + return Response( + json.dumps({"schema": "dt-computation-export/1", "run": run}, indent=2, allow_nan=False), + media_type="application/json", + headers={"Content-Disposition": f'attachment; filename="computation-{run["id"]}.json"'}, + ) diff --git a/api/routers/dataset_v2.py b/api/routers/dataset_v2.py index 19c9a9d..4436b32 100644 --- a/api/routers/dataset_v2.py +++ b/api/routers/dataset_v2.py @@ -82,6 +82,7 @@ from security import ( PlatformUser, get_platform_user, + require_admin, require_ingestion_token, resolve_request_user, ) @@ -390,7 +391,7 @@ def instantiate_project_process( @router.post("/projects/{project_id}/workflow-examples/etcher-profilometer") def create_etcher_profilometer_workflow_example( project_id: str, - user: PlatformUser = Depends(get_platform_user), + user: PlatformUser = Depends(require_admin), ): try: return create_etcher_profilometer_workflow_example_pg( @@ -407,7 +408,7 @@ def create_etcher_profilometer_workflow_example( def create_rcac_closed_loop_demo( project_id: str, payload: RcacClosedLoopDemoPayload, - user: PlatformUser = Depends(get_platform_user), + user: PlatformUser = Depends(require_admin), ): try: return create_rcac_closed_loop_demo_pg( diff --git a/api/routers/equipment.py b/api/routers/equipment.py index 4c77122..4ae8147 100644 --- a/api/routers/equipment.py +++ b/api/routers/equipment.py @@ -5,7 +5,7 @@ from their own organization, while owners/admins can also see pending items. """ import logging -from typing import List, Optional +from typing import Any, List, Optional from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field @@ -19,6 +19,7 @@ ) from metadata_pg import ( ExperimentRequestConflictError, + RegisteredComputationTemplateError, approved_equipment_available_to_user, create_experiment_definition_pg, create_experiment_type_pg, @@ -35,6 +36,7 @@ list_public_experiment_types_pg, list_equipment_pg, register_equipment_pg, + registered_computation_tool_for_template, revoke_equipment_access_pg, search_users_pg, start_experiment_proposal_job, @@ -146,6 +148,17 @@ class ExperimentTypeParamPayload(BaseModel): description: str = "" equipment_parameter_id: str = "" recipe_settable: bool = True + # Marks an additional input whose per-run value must be provided. + required: bool = False + # Declared value metadata. It is retained on create/update so run + # validation can enforce it (see metadata_pg._normalize_planned_value). + display_name: str = "" + options: List[Any] = [] + allowed_values: List[Any] = [] + enum: List[Any] = [] + minimum: Optional[float] = None + maximum: Optional[float] = None + default: Any = None class ExperimentTypePayload(BaseModel): @@ -225,14 +238,6 @@ def _can_manage_maintainers( return any(m.get("user_id") == user.id for m in maintainers) -def _has_planned_parameter_values(params: List[PlannedParamPayload]) -> bool: - return any( - param.name.strip() - and str(param.planned_value or "").strip() - for param in params - ) - - def _legacy_item_visible(item: dict, user: PlatformUser) -> bool: if user.role == "admin": return True @@ -565,6 +570,12 @@ def create_experiment_definition( ) equipment_record = get_equipment_pg(payload.selected_equipment) + if equipment_record and equipment_record.get("asset_kind") == "computational": + # Computational tools run only through /api/computations. + raise HTTPException( + status_code=409, + detail=str(RegisteredComputationTemplateError("")), + ) if equipment_record: if equipment_record.get("status") != "approved": raise HTTPException( @@ -587,14 +598,9 @@ def create_experiment_definition( project_id = payload.project_id.strip() if not payload.type_id.strip(): raise HTTPException(status_code=400, detail="Experiment type is required") - if project_id and not _has_planned_parameter_values(payload.planned_parameters): - raise HTTPException( - status_code=400, - detail=( - "At least one planned parameter value is required before running " - "an experiment and generating recipe proposals." - ), - ) + # Which values are required depends on the definition (selected inputs, + # Fixed/Variable roles, required additional inputs), so it is enforced + # against the snapshot in create_experiment_definition_pg, not here. if project_id: visible_projects = get_projects_list_pg( nanohub_user_id=user.id, @@ -623,7 +629,7 @@ def create_experiment_definition( }, user=user, ) - except ExperimentRequestConflictError as exc: + except (ExperimentRequestConflictError, RegisteredComputationTemplateError) as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc diff --git a/api/scripts/add_asset_kinds.sql b/api/scripts/add_asset_kinds.sql new file mode 100644 index 0000000..4e1ae41 --- /dev/null +++ b/api/scripts/add_asset_kinds.sql @@ -0,0 +1,118 @@ +-- Physical equipment and computational tools, told apart by explicit columns. +-- +-- Before this, a Quantum ESPRESSO installation and its workflows sat in the +-- same catalog tables as the furnaces and were recognized only by conventions +-- (equipment_type text, config_json.asset_kind, the computation_ id prefix). +-- +-- equipment_metadata.asset_kind physical | computational +-- experiment_types.execution_kind physical | computational, always the +-- kind of the template's equipment +-- all_runs every run with run_kind physical | +-- computational, under the caller's RLS +-- +-- Applied by db_migrations with the migration connection; idempotent. + +ALTER TABLE equipment_metadata + ADD COLUMN IF NOT EXISTS asset_kind VARCHAR(20) NOT NULL DEFAULT 'physical'; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'equipment_metadata_asset_kind_check' + ) THEN + ALTER TABLE equipment_metadata + ADD CONSTRAINT equipment_metadata_asset_kind_check + CHECK (asset_kind IN ('physical', 'computational')); + END IF; +END +$$; + +-- Registered RCAC installations written before this column existed. +UPDATE equipment_metadata +SET asset_kind = 'computational' +WHERE asset_kind <> 'computational' + AND (config_json ->> 'asset_kind' = 'digital' OR source_type = 'rcac_slurm'); + +ALTER TABLE experiment_types + ADD COLUMN IF NOT EXISTS execution_kind VARCHAR(20) NOT NULL DEFAULT 'physical'; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'experiment_types_execution_kind_check' + ) THEN + ALTER TABLE experiment_types + ADD CONSTRAINT experiment_types_execution_kind_check + CHECK (execution_kind IN ('physical', 'computational')); + END IF; +END +$$; + +-- A template runs the way its equipment does, whichever path created it +-- (the definition form, a fork, or the computation catalog). SECURITY DEFINER +-- so the lookup sees the equipment row regardless of the caller's policies. +CREATE OR REPLACE FUNCTION app_set_execution_kind() RETURNS trigger + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path = public, pg_temp +AS $$ +BEGIN + NEW.execution_kind := COALESCE( + (SELECT asset_kind FROM equipment_metadata WHERE domain_id = NEW.equipment_id), + 'physical' + ); + RETURN NEW; +END +$$; +REVOKE ALL ON FUNCTION app_set_execution_kind() FROM PUBLIC; + +DROP TRIGGER IF EXISTS dt_experiment_types_execution_kind ON experiment_types; +CREATE TRIGGER dt_experiment_types_execution_kind + BEFORE INSERT OR UPDATE OF equipment_id, execution_kind ON experiment_types + FOR EACH ROW EXECUTE FUNCTION app_set_execution_kind(); + +UPDATE experiment_types t +SET execution_kind = 'computational' +FROM equipment_metadata e +WHERE e.domain_id = t.equipment_id + AND e.asset_kind = 'computational' + AND t.execution_kind <> 'computational'; + +-- One list of every run. security_invoker keeps each base table's row-level +-- security in force for whoever queries the view. +CREATE OR REPLACE VIEW all_runs WITH (security_invoker = true) AS +SELECT + 'physical'::text AS run_kind, + er.id::text AS run_id, + er.project_id, + er.equipment_id, + er.experiment_type_id, + ''::text AS status, + COALESCE(er.run_date, er.created_at) AS run_at, + er.created_at +FROM equipment_runs er +UNION ALL +SELECT + 'physical'::text, + r.idruns::text, + r.project_id, + 'etcher'::text, + ''::text, + ''::text, + COALESCE(r.run_date, r.created_at), + r.created_at +FROM etcher_runs r +UNION ALL +SELECT + 'computational'::text, + cr.id::text, + cr.project_id, + cr.digital_installation_id, + -- Registered templates use computation__ ids + -- (computations.store._ensure_tool_catalog). + 'computation_' || cr.tool_name || '_' || replace(cr.tool_revision, '-', '_'), + cr.status, + COALESCE(cr.completed_at, cr.created_at), + cr.created_at +FROM computation_runs cr; + +COMMENT ON VIEW all_runs IS + 'Every run, physical (equipment_runs, etcher_runs) or computational (computation_runs).'; diff --git a/api/scripts/add_rcac_computations.sql b/api/scripts/add_rcac_computations.sql new file mode 100644 index 0000000..c037bf3 --- /dev/null +++ b/api/scripts/add_rcac_computations.sql @@ -0,0 +1,105 @@ +-- Additive schema for feature-flagged Digital Twin/RCAC computations. +-- Contains no changes to existing experiment or execution tables. + +CREATE TABLE IF NOT EXISTS computation_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id VARCHAR(50) NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + requested_by VARCHAR(255) NOT NULL REFERENCES users(id), + tool_name VARCHAR(100) NOT NULL, + tool_revision VARCHAR(100) NOT NULL, + -- Kept for additive compatibility with the earlier pilot schema. + sim2l_squid_id VARCHAR(255) NOT NULL, + input_fingerprint VARCHAR(255) NOT NULL DEFAULT '', + upstream_run_id UUID REFERENCES computation_runs(id) ON DELETE RESTRICT, + status VARCHAR(50) NOT NULL DEFAULT 'requested' + CHECK (status IN ( + 'requested', 'validating', 'staging', 'submitted', 'running', + 'collecting', 'data_ingested', 'cancelling', 'cancelled', 'failed' + )), + cluster VARCHAR(100) NOT NULL, + partition VARCHAR(255) NOT NULL, + qos VARCHAR(255) NOT NULL DEFAULT 'normal', + allocation VARCHAR(255) NOT NULL, + scheduler_job_id VARCHAR(100) NOT NULL DEFAULT '', + remote_workdir TEXT NOT NULL DEFAULT '', + input_json JSONB NOT NULL DEFAULT '{}'::jsonb, + output_json JSONB NOT NULL DEFAULT '{}'::jsonb, + artifact_manifest_json JSONB NOT NULL DEFAULT '[]'::jsonb, + external_workflow_ref_json JSONB NOT NULL DEFAULT '{}'::jsonb, + error_json JSONB NOT NULL DEFAULT '{}'::jsonb, + prepared_manifest_json JSONB NOT NULL DEFAULT '{}'::jsonb, + cancel_requested BOOLEAN NOT NULL DEFAULT false, + attempt_count INTEGER NOT NULL DEFAULT 0, + worker_lease_owner VARCHAR(255) NOT NULL DEFAULT '', + worker_lease_expires_at TIMESTAMP WITH TIME ZONE, + submitted_at TIMESTAMP WITH TIME ZONE, + started_at TIMESTAMP WITH TIME ZONE, + completed_at TIMESTAMP WITH TIME ZONE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +ALTER TABLE computation_runs +ADD COLUMN IF NOT EXISTS qos VARCHAR(255) NOT NULL DEFAULT 'normal'; +ALTER TABLE computation_runs +ADD COLUMN IF NOT EXISTS input_fingerprint VARCHAR(255) NOT NULL DEFAULT ''; +UPDATE computation_runs +SET input_fingerprint = sim2l_squid_id +WHERE input_fingerprint = ''; +ALTER TABLE computation_runs +ADD COLUMN IF NOT EXISTS digital_installation_id VARCHAR(255) NOT NULL DEFAULT ''; +ALTER TABLE computation_runs +ADD COLUMN IF NOT EXISTS execution_backend VARCHAR(100) NOT NULL + DEFAULT 'globus_compute_service_account'; +ALTER TABLE computation_runs +ADD COLUMN IF NOT EXISTS execution_identity VARCHAR(255) NOT NULL DEFAULT ''; +ALTER TABLE computation_runs +ADD COLUMN IF NOT EXISTS globus_task_id VARCHAR(255) NOT NULL DEFAULT ''; +ALTER TABLE computation_runs +ADD COLUMN IF NOT EXISTS slurm_comment VARCHAR(255) NOT NULL DEFAULT ''; +-- Execution resources (ranks, walltime, memory) requested by the user. Kept +-- apart from input_json so they never enter the scientific fingerprint; '{}' +-- (all earlier runs) means the original fixed pilot defaults. +ALTER TABLE computation_runs +ADD COLUMN IF NOT EXISTS resource_request_json JSONB NOT NULL DEFAULT '{}'::jsonb; + +-- One scientific experiment/workflow can run on multiple versioned digital +-- equipment installations. Strings are used deliberately so this migration +-- remains safe in minimal test databases where the catalog tables are absent. +CREATE TABLE IF NOT EXISTS computation_experiment_installations ( + experiment_type_id VARCHAR(255) NOT NULL, + equipment_id VARCHAR(255) NOT NULL, + cluster VARCHAR(100) NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (experiment_type_id, equipment_id) +); + +CREATE INDEX IF NOT EXISTS idx_computation_runs_project_created +ON computation_runs(project_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_computation_runs_worker_queue +ON computation_runs(status, worker_lease_expires_at, created_at) +WHERE status NOT IN ('data_ingested', 'cancelled', 'failed'); + +CREATE INDEX IF NOT EXISTS idx_computation_runs_fingerprint +ON computation_runs(tool_name, input_fingerprint, status); + +CREATE TABLE IF NOT EXISTS computation_run_events ( + id BIGSERIAL PRIMARY KEY, + run_id UUID NOT NULL REFERENCES computation_runs(id) ON DELETE CASCADE, + status VARCHAR(50) NOT NULL, + message TEXT NOT NULL DEFAULT '', + details_json JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_computation_run_events_run +ON computation_run_events(run_id, created_at); + +-- Do not grant mutation access to PostgREST's api_client role. FastAPI owns +-- authorization for this feature and uses its existing privileged connection. +ALTER TABLE computation_runs ENABLE ROW LEVEL SECURITY; +ALTER TABLE computation_run_events ENABLE ROW LEVEL SECURITY; +ALTER TABLE computation_experiment_installations ENABLE ROW LEVEL SECURITY; diff --git a/api/scripts/add_row_level_security.sql b/api/scripts/add_row_level_security.sql index b32416f..2dfa990 100644 --- a/api/scripts/add_row_level_security.sql +++ b/api/scripts/add_row_level_security.sql @@ -565,6 +565,22 @@ DROP POLICY IF EXISTS dt_trace_events_worker ON equipment_run_trace_events; CREATE POLICY dt_trace_events_worker ON equipment_run_trace_events FOR ALL TO api_worker USING (true) WITH CHECK (true); +-- ── Policies: registered computations ────────────────────────────────────── +-- The computation API and its RCAC worker connect as api_worker and check +-- project access in Python (computations.store). Submission budgets count +-- runs across projects, so these tables are not read with user-scoped +-- api_client connections. RLS is enabled by add_rcac_computations.sql. + +DROP POLICY IF EXISTS dt_computation_runs_worker ON computation_runs; +CREATE POLICY dt_computation_runs_worker ON computation_runs + FOR ALL TO api_worker USING (true) WITH CHECK (true); +DROP POLICY IF EXISTS dt_computation_run_events_worker ON computation_run_events; +CREATE POLICY dt_computation_run_events_worker ON computation_run_events + FOR ALL TO api_worker USING (true) WITH CHECK (true); +DROP POLICY IF EXISTS dt_computation_installations_worker ON computation_experiment_installations; +CREATE POLICY dt_computation_installations_worker ON computation_experiment_installations + FOR ALL TO api_worker USING (true) WITH CHECK (true); + -- ── Policies: project membership ─────────────────────────────────────────── -- Enforced immediately (see the header). Anyone who can read a project can -- read its membership, matching what api_client could read before, so diff --git a/api/scripts/bootstrap_rcac_live_e2e.sql b/api/scripts/bootstrap_rcac_live_e2e.sql new file mode 100644 index 0000000..ac4317b --- /dev/null +++ b/api/scripts/bootstrap_rcac_live_e2e.sql @@ -0,0 +1,110 @@ +-- Minimal, disposable Digital Twin catalog used only by an approved RCAC E2E +-- harness. Apply this to a dedicated test database, never the production +-- digital_twin database. + +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +CREATE TABLE IF NOT EXISTS users ( + id VARCHAR(255) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS projects ( + id VARCHAR(50) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + access_mode VARCHAR(20) DEFAULT 'private' +); + +CREATE TABLE IF NOT EXISTS project_members ( + project_id VARCHAR(50) REFERENCES projects(id) ON DELETE CASCADE, + nanohub_user_id VARCHAR(100) NOT NULL, + role VARCHAR(20) DEFAULT 'member', + PRIMARY KEY (project_id, nanohub_user_id) +); + +CREATE TABLE IF NOT EXISTS equipment_metadata ( + domain_id VARCHAR(255) PRIMARY KEY, + equipment_name VARCHAR(255) NOT NULL, + equipment_type VARCHAR(100) DEFAULT '', + asset_kind VARCHAR(20) NOT NULL DEFAULT 'physical', + manufacturer VARCHAR(255) DEFAULT '', + model VARCHAR(255) DEFAULT '', + location VARCHAR(255) DEFAULT '', + serial_number VARCHAR(255) DEFAULT '', + description TEXT DEFAULT '', + source_type VARCHAR(100) NOT NULL DEFAULT 'postgresql', + connection_host VARCHAR(255) DEFAULT '', + connection_port VARCHAR(50) DEFAULT '5432', + connection_database VARCHAR(255) DEFAULT '', + dt_mode VARCHAR(50) NOT NULL DEFAULT 'dashboard', + outlier_method VARCHAR(100) DEFAULT 'zscore', + outlier_threshold DOUBLE PRECISION DEFAULT 3.0, + owner_id VARCHAR(255) REFERENCES users(id), + owner_org VARCHAR(255) DEFAULT '', + status VARCHAR(50) NOT NULL DEFAULT 'pending', + parameters_json JSONB NOT NULL DEFAULT '[]'::jsonb, + columns_json JSONB NOT NULL DEFAULT '[]'::jsonb, + features_json JSONB NOT NULL DEFAULT '[]'::jsonb, + primary_target_json JSONB NOT NULL DEFAULT '{}'::jsonb, + secondary_target_json JSONB NOT NULL DEFAULT '{}'::jsonb, + config_json JSONB NOT NULL DEFAULT '{}'::jsonb, + registered_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + approved_at TIMESTAMP WITH TIME ZONE, + reviewed_by VARCHAR(255) REFERENCES users(id), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS experiment_types ( + id VARCHAR(255) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT DEFAULT '', + scientific_objective TEXT DEFAULT '', + equipment_id VARCHAR(255) DEFAULT '', + execution_kind VARCHAR(20) NOT NULL DEFAULT 'physical', + type_parameters_json JSONB NOT NULL DEFAULT '[]'::jsonb, + outputs_json JSONB NOT NULL DEFAULT '[]'::jsonb, + additional_inputs_json JSONB NOT NULL DEFAULT '[]'::jsonb, + visibility VARCHAR(50) NOT NULL DEFAULT 'shared', + version INT NOT NULL DEFAULT 1, + pid VARCHAR(50) DEFAULT '', + parent_id VARCHAR(255) DEFAULT '', + forked_from VARCHAR(255) DEFAULT '', + author_name VARCHAR(255) DEFAULT '', + author_email VARCHAR(255) DEFAULT '', + primary_target VARCHAR(255) DEFAULT '', + primary_objective VARCHAR(50) DEFAULT '', + secondary_target VARCHAR(255) DEFAULT '', + secondary_objective VARCHAR(50) DEFAULT '', + owner_id VARCHAR(255) REFERENCES users(id), + owner_org VARCHAR(255) DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS project_experiments ( + project_id VARCHAR(50) NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + type_id VARCHAR(255) NOT NULL REFERENCES experiment_types(id) ON DELETE CASCADE, + added_by VARCHAR(255), + status VARCHAR(50) NOT NULL DEFAULT 'active', + sequence_index INT NOT NULL DEFAULT 1, + upstream_type_id VARCHAR(255) DEFAULT '', + handoff_type VARCHAR(100) NOT NULL DEFAULT 'manual', + handoff_description TEXT DEFAULT '', + output_mapping_json JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (project_id, type_id) +); + +INSERT INTO users (id, name, email) +VALUES ('rcac-e2e-ngholiza', 'RCAC E2E', 'ngholiza@purdue.edu') +ON CONFLICT (id) DO NOTHING; + +INSERT INTO projects (id, name, access_mode) +VALUES ('RCAC-E2E', 'RCAC Registered Computation E2E', 'private') +ON CONFLICT (id) DO NOTHING; + +INSERT INTO project_members (project_id, nanohub_user_id, role) +VALUES ('RCAC-E2E', 'rcac-e2e-ngholiza', 'pi') +ON CONFLICT (project_id, nanohub_user_id) DO NOTHING; diff --git a/api/scripts/build_computation_review.py b/api/scripts/build_computation_review.py new file mode 100644 index 0000000..67c3ced --- /dev/null +++ b/api/scripts/build_computation_review.py @@ -0,0 +1,92 @@ +"""Build an internal scientific-review ZIP from already exported DT runs. + +No database/network access. Does not copy credentials, requester identities, +Globus identifiers, deployment files, or arbitrary execution metadata. +""" +import argparse +import hashlib +import json +from pathlib import Path +import zipfile + + +def scientific_reference(value): + """Keep scientific settings without nested local paths or account metadata.""" + def allowed(key, item): + return (not any(word in key.lower() for word in + ("path", "folder", "directory", "queue", "user", "password", "secret", "token")) + and not (isinstance(item, str) and item.startswith(("/", "~/", "$HOME")))) + if isinstance(value, dict): + return {key: scientific_reference(item) for key, item in value.items() if allowed(key, item)} + if isinstance(value, list): + return [scientific_reference(item) for item in value if allowed("", item)] + return value + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--exports", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--reference-dir", type=Path, required=True) + args = parser.parse_args() + repo = Path(__file__).resolve().parents[2] + if args.output.exists(): + raise SystemExit("Choose a new output path; an existing review package will not be overwritten") + files = sorted(args.exports.glob("export-*.json")) + if not files: + raise SystemExit("No exported runs found") + with zipfile.ZipFile(args.output, "x", compression=zipfile.ZIP_DEFLATED) as archive: + archive.write(repo / "COMPUTATIONAL_SCIENTIFIC_REVIEW.md", "START_HERE.md") + manifest = [] + for path in files: + document = json.loads(path.read_text()) + run = document.get("run", document) + record = {key: run.get(key) for key in ( + "id", "tool_name", "tool_revision", "cluster", "status", "input", "output", + "upstream_run_id", "input_fingerprint", "digital_installation_id", + )} + record["artifacts"] = [{key: entry.get(key) for key in ("name", "sha256", "size_bytes")} + for entry in run.get("artifact_manifest", [])] + prepared = run.get("prepared_manifest", {}) + record["execution_request"] = {key: prepared.get(key) for key in ( + "ntasks", "time_limit", "job_array", "platform_environment", + )} + record["pseudopotentials"] = [{key: entry.get(key) for key in ("element", "relative", "sha256")} + for entry in prepared.get("pseudopotentials", [])] + execution = run.get("external_workflow_ref", {}) + record["qe_versions_from_outputs"] = execution.get("qe_versions_from_outputs", []) + record["runtime_observations"] = [ + {key: entry.get(key) for key in ( + "schema", "observed_at", "tool_revision", "cluster", "os", + "python_version", "loaded_modules", "qe_executable_sha256", + "script_sha256", "array_task_id", + )} + for entry in execution.get("execution_environments", []) + ] + record["review_status"] = "pending_scientific_review" + encoded = json.dumps(record, indent=2, allow_nan=False).encode() + name = f"runs/{run['id']}.json" + archive.writestr(name, encoded) + manifest.append({"file": name, "sha256": hashlib.sha256(encoded).hexdigest()}) + reference = args.reference_dir / "mp-48_relaxed.cif" + archive.write(reference, "reference/justin-mp-48-relaxed.cif") + metadata = json.loads((args.reference_dir / "metadata.json").read_text()) + # Preserve scientific reference settings, not local user/scratch paths. + scientific_metadata = scientific_reference({ + key: metadata[key] for key in ( + "name", "timestamp", "converged", "dft_parameters", "electron_dict", + "system_dict", "cell_namelist", + ) if key in metadata + }) + archive.writestr("reference/justin-metadata.json", json.dumps(scientific_metadata, indent=2)) + sources = ["qe.py", "monolayer.py", "schemas.py", "globus_gateway.py", "worker.py"] + archive.writestr("adapter-source-checksums.json", json.dumps({ + name: hashlib.sha256((repo / "api" / "computations" / name).read_bytes()).hexdigest() + for name in sources}, indent=2)) + archive.writestr("manifest.json", json.dumps(manifest, indent=2)) + print(json.dumps({"package": str(args.output), "runs": len(files), + "sha256": hashlib.sha256(args.output.read_bytes()).hexdigest()})) + + +if __name__ == "__main__": + main() diff --git a/api/scripts/rcac_globus_endpoint.bash b/api/scripts/rcac_globus_endpoint.bash new file mode 100755 index 0000000..7ef220c --- /dev/null +++ b/api/scripts/rcac_globus_endpoint.bash @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + echo "usage: $0 {start|stop|list} {negishi|gautschi}" >&2 + exit 2 +} + +action="${1:-}" +cluster="${2:-}" +case "$action" in + start|stop|list) ;; + *) usage ;; +esac + +case "$cluster" in + negishi) + endpoint_name="dt-negishi" + venv="$HOME/.venvs/dt-globus-negishi-3.16.1" + python_module="anaconda/2024.02-py311" + ;; + gautschi) + endpoint_name="dt-gautschi" + venv="$HOME/.venvs/dt-globus-gautschi-3.16.1" + python_module="" + ;; + *) usage ;; +esac + +if ! type module >/dev/null 2>&1; then + # RCAC's noninteractive SSH shell does not initialize Lmod or scratch + # variables. Endpoint administration must always use the login profile. + # The RCAC profile references interactive-shell variables which may be + # unset, so temporarily disable nounset while it initializes the shell. + # shellcheck disable=SC1091 + set +u + source /etc/profile + set -u +fi + +if [[ -n "$python_module" ]]; then + module load "$python_module" +fi + +client_env="$HOME/.config/dt-globus/client.env" +cluster_env="$HOME/.config/dt-globus/${cluster}.env" +for required_file in "$client_env" "$cluster_env"; do + if [[ ! -r "$required_file" ]]; then + echo "required endpoint configuration is unavailable: $required_file" >&2 + exit 1 + fi +done + +set -a +# shellcheck disable=SC1090 +source "$client_env" +# shellcheck disable=SC1090 +source "$cluster_env" +set +a + +if [[ -z "${GLOBUS_COMPUTE_CLIENT_SECRET:-}" ]]; then + secret_file="${GLOBUS_COMPUTE_CLIENT_SECRET_FILE:-}" + if [[ -z "$secret_file" || ! -r "$secret_file" ]]; then + echo "Globus confidential-client secret is unavailable" >&2 + exit 1 + fi + GLOBUS_COMPUTE_CLIENT_SECRET="$(<"$secret_file")" + export GLOBUS_COMPUTE_CLIENT_SECRET +fi + +for name in \ + GLOBUS_COMPUTE_CLIENT_ID \ + GLOBUS_COMPUTE_CLIENT_SECRET \ + DT_CLUSTER_NAME \ + DT_WORK_ROOT \ + DT_PSEUDO_ROOT \ + DT_ARTIFACT_ROOT \ + DT_SLURM_ACCOUNT \ + DT_SLURM_PARTITION \ + DT_SLURM_QOS \ + DT_QE_MODULES_JSON \ + DT_MPI_LAUNCHER; do + if [[ -z "${!name:-}" ]]; then + echo "required endpoint setting is empty: $name" >&2 + exit 1 + fi +done + +if [[ "$DT_CLUSTER_NAME" != "$cluster" ]]; then + echo "endpoint configuration does not match requested cluster" >&2 + exit 1 +fi + +if [[ ! -x "$venv/bin/globus-compute-endpoint" ]]; then + echo "Globus Compute endpoint virtual environment is unavailable: $venv" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +source "$venv/bin/activate" + +installed_version="$(python -c 'import importlib.metadata as m; print(m.version("globus-compute-endpoint"))')" +if [[ "$installed_version" != "3.16.1" ]]; then + echo "unexpected Globus Compute endpoint version: $installed_version" >&2 + exit 1 +fi + +case "$action" in + start) + mkdir -p "$DT_WORK_ROOT" + if [[ ! -r "$DT_PSEUDO_ROOT" ]]; then + echo "pseudopotential directory is not readable: $DT_PSEUDO_ROOT" >&2 + exit 1 + fi + if [[ ! -w "$DT_ARTIFACT_ROOT" ]]; then + echo "artifact directory is not writable: $DT_ARTIFACT_ROOT" >&2 + exit 1 + fi + globus-compute-endpoint start "$endpoint_name" + ;; + stop) + globus-compute-endpoint stop "$endpoint_name" + ;; + list) + globus-compute-endpoint list + ;; +esac diff --git a/api/scripts/serve_rcac_validation.py b/api/scripts/serve_rcac_validation.py new file mode 100644 index 0000000..2b9a1e3 --- /dev/null +++ b/api/scripts/serve_rcac_validation.py @@ -0,0 +1,40 @@ +"""Local-port-forward-only pilot server with the real API and worker. + +Explicitly limited to the disposable dt_rcac_e2e database. Authentication is +not bypassed. Read /tmp/dt-validation-system-token through an authorized local +administrative channel; never publish it or use this runner for production. +""" +import os +from pathlib import Path +import secrets +import sys +import threading +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +if os.environ.get('PG_DB') != 'dt_rcac_e2e': + raise SystemExit('Only the isolated dt_rcac_e2e database is permitted') +os.umask(0o077) +os.environ['PG_SUPER_USER'] = os.environ['PG_USER'] +os.environ['PG_SUPER_PASS'] = os.environ['PG_PASS'] +os.environ['DT_SYSTEM_TOKEN'] = secrets.token_urlsafe(32) +os.environ['INGESTION_TOKEN'] = secrets.token_urlsafe(32) +Path('/tmp/dt-validation-system-token').write_text(os.environ['DT_SYSTEM_TOKEN']) +from computations.worker import ComputationWorker +from main import app +import uvicorn + + +def work(): + worker = ComputationWorker('isolated-validation-worker') + while True: + try: + worker.run_cycle() + except Exception: + import logging + logging.getLogger(__name__).exception('Isolated worker cycle interrupted') + time.sleep(30) + + +threading.Thread(target=work, daemon=True).start() +uvicorn.run(app, host='127.0.0.1', port=8800, log_level='warning') diff --git a/api/scripts/validate_rcac_live.py b/api/scripts/validate_rcac_live.py new file mode 100644 index 0000000..0ea2db5 --- /dev/null +++ b/api/scripts/validate_rcac_live.py @@ -0,0 +1,214 @@ +"""Opt-in real API/DB/Globus/Slurm validation in the isolated pilot database. + +This submits a small Si relaxation and a 12-member elastic array (64 tasks per +member, four concurrent). Run only inside the approved isolated test environment: + + python scripts/validate_rcac_live.py --cluster negishi --submit + +The real FastAPI auth dependency is exercised with a temporary BFF system token. +Browser login is outside this harness. No physics, parser, storage, or transport +implementation is mocked. Scientific accuracy still requires domain review. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import secrets +import socket +import sys +import threading +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cluster', required=True, choices=['negishi', 'gautschi']) + parser.add_argument('--submit', action='store_true', required=True) + parser.add_argument('--timeout-seconds', type=int, default=1800) + args = parser.parse_args() + if os.environ.get('PG_DB') != 'dt_rcac_e2e': + raise SystemExit('This harness requires the isolated dt_rcac_e2e database') + if os.environ.get('RCAC_SERVICE_ACCOUNT_USERNAME') != 'dtwin0': + raise SystemExit('This pilot requires service-account identity dtwin0') + + # The legacy connection helper name does not require an elevated DB role. + os.environ['PG_SUPER_USER'] = os.environ['PG_USER'] + os.environ['PG_SUPER_PASS'] = os.environ['PG_PASS'] + os.environ['DT_SYSTEM_TOKEN'] = secrets.token_urlsafe(32) + os.environ['RCAC_COMPUTE_ENABLED'] = 'true' + from fastapi import FastAPI + import requests + import uvicorn + from routers.computations import router + from computations.rcac import get_cluster + from computations.store import TERMINAL_STATUSES, get_run + from computations.worker import ComputationWorker, _restore_prepared + from data_loader_pg import get_pg_superuser_connection + + connection = get_pg_superuser_connection() + with connection, connection.cursor() as cursor: + cursor.execute('SELECT current_database(), rolsuper FROM pg_roles WHERE rolname=current_user') + database, superuser = cursor.fetchone() + assert database == 'dt_rcac_e2e' and not superuser + cursor.execute("SELECT count(*) FROM computation_runs WHERE status NOT IN ('data_ingested','failed','cancelled')") + assert cursor.fetchone()[0] == 0, 'Finish existing isolated runs before starting validation' + connection.close() + + # The dedicated database was bootstrapped from the base schema. Apply the + # same additive catalog migrations as normal FastAPI startup, scoped by + # the database assertion above; the existing DT database is never used. + from metadata_pg import ( + ensure_experiment_type_reuse_columns_pg, + ensure_experiment_type_versioning_columns_pg, + ensure_project_experiments_table_pg, + ) + ensure_experiment_type_reuse_columns_pg() + ensure_experiment_type_versioning_columns_pg() + ensure_project_experiments_table_pg() + + project = 'RCAC-E2E' + user_id = 'rcac-e2e-ngholiza' + headers = {'X-System-Token':os.environ['DT_SYSTEM_TOKEN'], 'X-User-Id':user_id, + 'X-User-Email':'ngholiza@purdue.edu', 'X-User-Name':'RCAC E2E', + 'X-User-Role':'researcher', 'X-User-Org':'Purdue University'} + app = FastAPI() + app.include_router(router, prefix='/api') + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + api_base = f'http://127.0.0.1:{listener.getsockname()[1]}' + server = uvicorn.Server(uvicorn.Config(app, log_level='warning')) + threading.Thread(target=server.run, kwargs={'sockets':[listener]}, daemon=True).start() + startup_deadline = time.monotonic() + 20 + while not server.started: + if time.monotonic() >= startup_deadline: + raise RuntimeError('Local validation API did not start') + time.sleep(0.1) + client = requests.Session() + client.headers.update(headers) + cluster = get_cluster(args.cluster) + report_path = Path(os.environ['RCAC_LOCAL_WORK_ROOT']) / f'live-{args.cluster}.json' + report = {'cluster':args.cluster, 'checks':{}, 'runs':[]} + report_path.parent.mkdir(parents=True, exist_ok=True) + + def record(): + report_path.write_text(json.dumps(report, indent=2, default=str) + '\n') + + def emit(**fields): + print(json.dumps(fields, default=str), flush=True) + + def request(method, path, **kwargs): + response = client.request(method, api_base + '/api/computations' + path, timeout=60, **kwargs) + assert response.ok, f'API {method} {path} returned {response.status_code}: {response.text}' + return response.json() + + def submit(tool, inputs): + row = request('POST', f'/projects/{project}/runs/{tool}', + json={'cluster':args.cluster, 'inputs':inputs}) + report['runs'].append({'id':row['id'], 'tool':tool}) + record() + emit(event='requested', run_id=row['id'], tool=tool, cluster=args.cluster) + return row['id'] + + def advance(run_id, *, replay=False): + deadline = time.monotonic() + args.timeout_seconds + previous = None + replayed = False + restart_checked = False + while True: + # A fresh worker exercises recovery from persisted run state. + worker = ComputationWorker(worker_id=f'live-{args.cluster}') + worker.run_once(run_id) + row = request('GET', f'/runs/{run_id}') + if previous != row['status']: + emit(event='state', run_id=run_id, status=row['status'], + slurm_job_id=row.get('scheduler_job_id')) + previous = row['status'] + if replay and not replayed and row['status'] == 'submitted': + stored = get_run(run_id, internal=True) + prepared = _restore_prepared(stored, worker.work_root / run_id) + _, result = worker._invoke(cluster, 'submit', { + 'run_id':run_id, 'spec':cluster.job_spec(prepared, run_id).as_payload(), + 'files':worker.gateway.encoded_files(prepared.stage_files), + 'pseudopotentials':prepared.pseudopotentials, + }) + assert result['job_id'] == row['scheduler_job_id'] + replayed = True + report['checks']['submission_idempotency'] = 'passed' + record() + if row['status'] == 'collecting' and not restart_checked: + # Collection must work on a replacement Pod with no local inputs. + worker = ComputationWorker(worker_id=f'live-{args.cluster}-recovery') + worker.work_root = worker.work_root / 'recovery' + worker.work_root.mkdir(exist_ok=True) + worker.run_once(run_id) + row = request('GET', f'/runs/{run_id}') + restart_checked = True + if row['status'] in TERMINAL_STATUSES: + assert row['status'] == 'data_ingested', row.get('error') + assert row['output']['convergence'] is True, 'Scientific result did not converge' + assert row['requested_by'] == user_id + assert row['execution_identity'] == 'dtwin0' + assert row['slurm_comment'] == f'dt-run:{run_id}' + assert row['external_workflow_ref']['endpoint_id'] == cluster.endpoint_id + assert row['artifact_manifest'] + for artifact in row['artifact_manifest']: + assert artifact['storage_uri'].startswith(f'depot://{cluster.artifact_root}/{run_id}/') + assert len(artifact['sha256']) == 64 + report['checks']['restart_collection'] = 'passed' if restart_checked else 'not exercised' + entry = next(item for item in report['runs'] if item['id'] == run_id) + entry.update(status=row['status'], scheduler_job_id=row['scheduler_job_id'], + artifact_count=len(row['artifact_manifest']), + globus_task_id=row['globus_task_id'], + scientific_convergence=row['output']['convergence']) + record() + return row + if time.monotonic() >= deadline: + request('POST', f'/runs/{run_id}/cancel') + worker.run_once(run_id) + raise TimeoutError(f'Cancelled run {run_id} after the test deadline') + time.sleep(30) + + tools = request('GET', '/tools') + assert any(item['name'] == args.cluster for item in tools['clusters']) + inputs = {'mp_id':'mp-149', 'KE_cutoff':30, 'k_x':2, 'k_y':2, 'k_z':2, + 'scf_conv':1e-8, 'energy_conv':1e-8, 'force_conv':1e-4, + 'pressure_conv':1.0, 'i_steps':50} + cell_id = submit('cellrelaxdft', inputs) + cell = advance(cell_id, replay=True) + assert len(cell['output']['structure_dict']['sites']) == 2 + assert cell['output']['intermediate_data'] + forbidden = client.get(f'{api_base}/api/computations/runs/{cell_id}', timeout=60, + headers={**headers, 'X-User-Id':'rcac-e2e-outsider'}) + assert forbidden.status_code == 403 + report['checks']['project_read_isolation'] = 'passed' + + cached_id = submit('cellrelaxdft', inputs) + ComputationWorker(worker_id='live-cache').run_once(cached_id) + cached = request('GET', f'/runs/{cached_id}') + assert cached['status'] == 'data_ingested' + assert cached['external_workflow_ref']['source_run_id'] == cell_id + assert not cached['scheduler_job_id'] + report['checks']['cache_reuse_without_slurm_submission'] = 'passed' + + elastic_id = submit('elasticdft', {'mp_id':'mp-149', 'upstream_execution_id':cell_id, + 'diagonal_strain_magnitudes':[-0.005,0.005], + 'offdiagonal_strain_magnitudes':[-0.005,0.005]}) + elastic = advance(elastic_id) + assert elastic['upstream_run_id'] == cell_id + assert len(elastic['output']['elastic_tensor']) == 6 + assert all(len(row) == 6 for row in elastic['output']['elastic_tensor']) + assert len(elastic['output']['member_statuses']) == 12 + report['checks']['upstream_provenance'] = 'passed' + report['checks']['elastic_tensor_shape'] = 'passed' + record() + emit(event='validation_passed', report=report) + server.should_exit = True + + +if __name__ == '__main__': + main() diff --git a/api/tests/test_computation_cutoff_fidelity.py b/api/tests/test_computation_cutoff_fidelity.py new file mode 100644 index 0000000..4c23ba4 --- /dev/null +++ b/api/tests/test_computation_cutoff_fidelity.py @@ -0,0 +1,103 @@ +"""Regression tests: validated KE_cutoff reaches QE unchanged, and results +written by the old truncating adapter can never be reused.""" + +import tempfile +import hashlib +import json +import unittest +from pathlib import Path +from unittest.mock import patch + +from computations import schemas +from computations.qe import prepare_cellrelax +from computations.schemas import ( + compute_input_fingerprint, + upstream_cutoff_was_truncated, + validate_tool_inputs, +) +from pymatgen.core import Lattice, Structure + +PSEUDO_ROOT = "/depot/ncn/data/digital-twins/pseudopotentials" + + +def _silicon_inputs(**overrides): + structure = Structure(Lattice.cubic(5.43), ["Si"], [[0.0, 0.0, 0.0]]) + return validate_tool_inputs( + "cellrelaxdft", + {"mp_id": "mp-149", "structure_dict": structure.as_dict(), **overrides}, + ) + + +def _qe_input(inputs) -> str: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepare_cellrelax(inputs, root / "run", PSEUDO_ROOT) + return (root / "run" / "vcrelax.in").read_text(encoding="utf-8") + + +def _legacy_fingerprint(inputs) -> str: + # Historical writer used no adapter-version line at all. + normalized = json.loads(json.dumps(inputs, sort_keys=True)) + lines = [f"{key} {normalized[key]}" for key in sorted(normalized)] + lines.extend(["simtoolName cellrelaxdft", "simtoolRevision r99-dt2"]) + return "cellrelaxdft/r99-dt2/" + hashlib.sha1("\n".join(lines).encode()).hexdigest() + + +class CutoffFidelityTests(unittest.TestCase): + def test_fractional_cutoff_is_written_exactly(self): + inputs = _silicon_inputs(KE_cutoff=50.5) + self.assertEqual(inputs["KE_cutoff"], 50.5) + qe_input = _qe_input(inputs) + self.assertIn("ecutwfc = 50.5", qe_input) + self.assertNotIn("ecutwfc = 50,", qe_input) + + def test_integral_cutoff_keeps_integer_rendering(self): + qe_input = _qe_input(_silicon_inputs(KE_cutoff=50.0)) + self.assertIn("ecutwfc = 50,", qe_input) + + def test_adapter_version_changes_fingerprint_so_old_cache_is_not_reused(self): + inputs = _silicon_inputs(KE_cutoff=50.5) + current = compute_input_fingerprint("cellrelaxdft", inputs) + # find_cache_hit matches on exact input_fingerprint equality. + self.assertNotEqual(current, _legacy_fingerprint(inputs)) + self.assertTrue(current.startswith("cellrelaxdft/r99-dt2/")) + self.assertEqual(current, compute_input_fingerprint("cellrelaxdft", inputs)) + + def test_future_adapter_bump_keeps_corrected_fractional_parent_valid(self): + inputs = _silicon_inputs(KE_cutoff=50.5) + version2 = compute_input_fingerprint("cellrelaxdft", inputs) + with patch.object(schemas, "QE_INPUT_ADAPTER_VERSION", 3): + self.assertFalse(upstream_cutoff_was_truncated("cellrelaxdft", inputs, version2)) + + def test_truncated_upstream_detection(self): + fractional = _silicon_inputs(KE_cutoff=50.5) + integral = _silicon_inputs(KE_cutoff=50.0) + self.assertTrue( + upstream_cutoff_was_truncated( + "cellrelaxdft", fractional, _legacy_fingerprint(fractional) + ) + ) + self.assertFalse( + upstream_cutoff_was_truncated( + "cellrelaxdft", + fractional, + compute_input_fingerprint("cellrelaxdft", fractional), + ) + ) + # Integral cutoffs ran exactly as recorded under either adapter. + self.assertFalse( + upstream_cutoff_was_truncated( + "cellrelaxdft", integral, _legacy_fingerprint(integral) + ) + ) + + def test_trusted_adapter_provenance_does_not_depend_on_numeric_encoding(self): + inputs = _silicon_inputs(KE_cutoff=50.5) + self.assertFalse(upstream_cutoff_was_truncated("cellrelaxdft", inputs, "normalized-jsonb", adapter_version=2)) + for invalid in [None, 1, True, "2", 2.0]: + with self.subTest(adapter_version=invalid): + self.assertTrue(upstream_cutoff_was_truncated("cellrelaxdft", inputs, "unknown", adapter_version=invalid)) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_computation_diagnostics.py b/api/tests/test_computation_diagnostics.py new file mode 100644 index 0000000..38e15e5 --- /dev/null +++ b/api/tests/test_computation_diagnostics.py @@ -0,0 +1,466 @@ +"""Failure diagnostics: evidence rules, gateway extraction, worker and API. + +The BFGS excerpt reproduces the decisive lines of the checksum-verified +ElasticDFT output from run 47548c30 (strain 0, AlN, QE 7.1); see +ACTUAL_FAILURE_FINDINGS.md. No real cluster, endpoint or database is used. +""" +import json +import os +import pwd +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from computations.diagnostics import build_diagnostics, exit_code_hint, summary_message +from computations.globus_gateway import rcac_service_gateway +from computations.qe import elastic_kmesh, parse_qe_output +from computations.store import _public_run +from computations.worker import ComputationWorker +from test_computation_resources import _clusters_v2 +from test_computation_worker import _clusters + +ELASTIC = "task_000/elastic_inputs/strain_000/relax.stdout" +ELASTIC_1 = "task_001/elastic_inputs/strain_001/relax.stdout" + +BFGS_FAILED_OUTPUT = """ + Program PWSCF v.7.1 starts on 6Oct2026 at 18:56: 0 + Parallel version (MPI & OpenMP), running on 64 processor cores + K-points division: npool = 4 + lattice parameter (alat) = 5.8295 a.u. + scf convergence threshold = 1.0E-10 + energy convergence thresh.= 1.0E-10 + force convergence thresh. = 1.0E-05 + nstep = 100 + number of k points= 100 + Estimated max dynamical RAM per process > 3.45 MB + Estimated total dynamical RAM > 197.10 MB +! total energy = -133.06553012 Ry + convergence has been achieved in 9 iterations + total stress (Ry/bohr**3) (kbar) P= 21.33 + 0.00026520 -0.00000000 0.00000000 39.01 -0.00 0.00 + -0.00000000 0.00009519 0.00000000 -0.00 14.00 0.00 + 0.00000000 0.00000000 0.00007469 0.00 0.00 10.99 + Energy error = 1.3E-08 Ry + Gradient error = 2.7E-05 Ry/Bohr + Message from routine bfgs :: compute_trust_radius: + history already reset at previous step: stopping +! total energy = -133.06553014 Ry + convergence has been achieved in 3 iterations + total stress (Ry/bohr**3) (kbar) P= 21.33 + 0.00026522 -0.00000000 0.00000000 39.02 -0.00 0.00 + -0.00000000 0.00009535 0.00000000 -0.00 14.03 0.00 + 0.00000000 0.00000000 0.00007450 0.00 0.00 10.96 + Message from routine bfgs: + history already reset at previous step: exiting + Energy error = 2.7E-09 Ry + Gradient error = 1.3E-05 Ry/Bohr + + bfgs failed after 10 scf cycles and 9 bfgs steps, convergence not achieved + (criteria: energy < 1.0E-10 Ry, force < 1.0E-05 Ry/Bohr) + + End of BFGS Geometry Optimization + JOB DONE. +""" + +STDERR_EXIT_3 = """Note: The following floating-point exceptions are signalling: IEEE_DENORMAL +STOP 3 +srun: error: a123: tasks 0-63: Exited with exit code 3 +dt: pw.x exit status 3 +""" + + +def _elastic_manifest(count=2): + names = [f"task_{i:03d}/elastic_inputs/strain_{i:03d}/relax.stdout" for i in range(count)] + matrices = [[[-0.01, 0, 0], [0, 0, 0], [0, 0, 0]], [[0, 0.005, 0], [0.005, 0, 0], [0, 0, 0]]] + return {"tool_name": "elasticdft", "expected_artifacts": names, "job_array": f"0-{count - 1}%2", + "stage_files": ["elastic-inputs.tar.gz"], "ntasks": 64, "time_limit": "08:00:00", + "metadata": {"count": count, "strain_matrices": matrices[:count]}} + + +def _record(**counts_and_markers): + markers = counts_and_markers.pop("markers", []) + return {"kind": "scheduler_log", "size_bytes": 100, "counts": counts_and_markers, "markers": markers} + + +# --- evidence rules ----------------------------------------------------------- + +def test_r1_file_evidence_is_used_when_accounting_is_missing(): + oom = _record(out_of_memory=1, markers=[{"tag": "out_of_memory", "line_number": 3, + "text": "slurmstepd: error: Detected 1 oom_kill event in StepId=42.0."}]) + result = build_diagnostics( + tool_name="cellrelaxdft", job_id="42", + prepared_manifest={"expected_artifacts": ["vcrelax.stdout"]}, accounting=[], + collection={"diagnostics": {"files": {"slurm.err": oom}}}, outcome="failed", + ) + member = result["members"][0] + assert member["category"] == "out_of_memory" + assert member["scheduler_record"] is False + assert "oom_kill" in member["evidence"][0]["text"] + assert any("no scheduler accounting" in note for note in result["notes"]) + + +@pytest.mark.parametrize("files", [{}, {"vcrelax.stdout": {"kind": "qe_output", "error": "unreadable"}}]) +def test_r2_unavailable_evidence_never_proves_a_launch_failure(files): + result = build_diagnostics( + tool_name="cellrelaxdft", job_id="42", prepared_manifest={"expected_artifacts": ["vcrelax.stdout"]}, + accounting=[{"job_id": "42", "state": "FAILED", "exit_code": "2:0"}], + collection={"diagnostics": {"files": files}}, outcome="failed", pw_exit_logged=True, + ) + member = result["members"][0] + assert member["category"] == "nonzero_exit" + assert "SCF" in member["exit_code_hint"] + + +def test_r2_truncated_or_unread_stderr_is_not_evidence_of_missing_pw_status(): + truncated = {**_record(), "scanned_from_byte": 1024} + for stderr in (truncated, {"kind": "scheduler_log", "error": "unreadable"}): + result = build_diagnostics( + tool_name="cellrelaxdft", job_id="42", prepared_manifest={"expected_artifacts": ["vcrelax.stdout"]}, + accounting=[{"job_id": "42", "state": "FAILED", "exit_code": "1:0"}], + collection={"diagnostics": {"files": {"slurm.err": stderr, + "vcrelax.stdout": {"kind": "qe_output", "size_bytes": 10, "counts": {}}}}}, + outcome="failed", pw_exit_logged=True, + ) + assert result["members"][0]["category"] == "nonzero_exit" + + +@pytest.mark.parametrize("files,missing", [ + ({"slurm.err": _record(environment=1, markers=[{"tag": "environment", "line_number": 1, + "text": "Lmod has detected the following error"}])}, []), + ({"vcrelax.stdout": {"kind": "qe_output", "size_bytes": 0}}, []), + ({}, ["vcrelax.stdout"]), + ({"slurm.err": _record(), "vcrelax.stdout": {"kind": "qe_output", "size_bytes": 10, "counts": {}}}, []), +]) +def test_positive_observations_identify_launch_failures(files, missing): + result = build_diagnostics( + tool_name="cellrelaxdft", job_id="42", prepared_manifest={"expected_artifacts": ["vcrelax.stdout"]}, + accounting=[{"job_id": "42", "state": "FAILED", "exit_code": "1:0"}], + collection={"diagnostics": {"files": files}, "missing_artifacts": missing}, + outcome="failed", pw_exit_logged=True, + ) + assert result["members"][0]["category"] == "launch_or_environment" + + +def test_exit_codes_alone_are_hints_never_memory_findings(): + result = build_diagnostics( + tool_name="cellrelaxdft", job_id="42", prepared_manifest={"expected_artifacts": ["vcrelax.stdout"]}, + accounting=[{"job_id": "42", "state": "FAILED", "exit_code": "0:9"}], outcome="failed", + ) + member = result["members"][0] + assert member["category"] == "killed_by_signal" + assert "SIGKILL" in member["exit_code_hint"] + assert "BFGS" in exit_code_hint("3:0") and "nstep" in exit_code_hint("3:0") + assert result["evidence_available"] is False + + +def test_scan_window_qualifies_line_numbers(): + qe = {"kind": "qe_output", "size_bytes": 20_000_000, "scanned_from_byte": 3_222_784, + "counts": {"bfgs_failed": 1}, + "markers": [{"tag": "bfgs_failed", "line_number": 12, "text": "bfgs failed after 3 scf cycles"}]} + result = build_diagnostics( + tool_name="cellrelaxdft", job_id="42", prepared_manifest={"expected_artifacts": ["vcrelax.stdout"]}, + accounting=[{"job_id": "42", "state": "FAILED", "exit_code": "3:0"}], + collection={"diagnostics": {"files": {"vcrelax.stdout": qe}}}, outcome="failed", + ) + member = result["members"][0] + assert member["evidence"][0]["window_start_byte"] == 3_222_784 + assert member["qe"]["scanned_from_byte"] == 3_222_784 + + +# --- gateway evidence on the actual failure mode ---------------------------- + +@pytest.fixture +def remote(tmp_path, monkeypatch): + run_id = str(uuid4()) + root = tmp_path.resolve() + work = root / "scratch" / run_id + (work / "task_000/elastic_inputs/strain_000").mkdir(parents=True) + (work / "task_001/elastic_inputs/strain_001").mkdir(parents=True) + (work / ".dt-slurm-job-id").write_text("42") + (work / ELASTIC).write_text(BFGS_FAILED_OUTPUT) + (work / ELASTIC_1).write_text(BFGS_FAILED_OUTPUT.replace( + "bfgs failed after 10 scf cycles and 9 bfgs steps, convergence not achieved", + "bfgs converged in 5 scf cycles and 4 bfgs steps")) + (work / "slurm-42_0.err").write_text(STDERR_EXIT_3) + (work / "slurm-42_1.err").write_text("dt: pw.x exit status 0\n") + for key, value in {"DT_CLUSTER_NAME": "negishi", "DT_WORK_ROOT": str(root / "scratch"), + "DT_ARTIFACT_ROOT": str(root / "archive")}.items(): + monkeypatch.setenv(key, value) + return run_id, work + + +def _sacct(run_id, comment=None, owner=None): + comment = f"dt-run:{run_id}" if comment is None else comment + owner = owner or pwd.getpwuid(os.getuid()).pw_name + lines = [ + f"42_0|FAILED|3:0|0:0|00:00:27|08:00:00||128000M|64|1|{comment}|{owner}", + "42_0.batch|FAILED|3:0||00:00:27||9000K||64|1||", + "42_0.0|FAILED|3:0||00:00:20||250000K||64|1||", + f"42_1|COMPLETED|0:0|0:0|00:00:13|08:00:00||128000M|64|1|dt-run:{run_id}|{owner}", + "42_1.0|COMPLETED|0:0||00:00:12||260000K||64|1||", + ] + return "\n".join(lines) + "\n" + + +def _collect(run_id, **extra): + names = [ELASTIC, ELASTIC_1] + retained = names + ["slurm-42_0.err", "slurm-42_1.err", "task_000/elastic_inputs/strain_000/CRASH"] + return rcac_service_gateway({"action": "collect", "payload": { + "cluster": "negishi", "run_id": run_id, "job_id": "42", "tool_name": "elasticdft", + "expected_artifacts": names, "retained_artifacts": retained, "diagnostics_only": True, + "include_diagnostics": True, **extra, + }}) + + +def test_actual_bfgs_failure_is_classified_from_qe_output_with_criteria(remote): + run_id, _ = remote + collection = _collect(run_id) + assert "task_000/elastic_inputs/strain_000/CRASH" not in collection["missing_artifacts"] + evidence = collection["diagnostics"]["files"][ELASTIC] + assert evidence["final"]["bfgs_criteria"].startswith("(criteria: energy < 1.0E-10 Ry") + assert "2.7E-09" in evidence["final"]["energy_error"] + assert "1.3E-05" in evidence["final"]["gradient_error"] + assert evidence["context"]["force_threshold"].startswith("force convergence thresh.") + assert evidence["context"]["nstep"].startswith("nstep") + assert evidence["counts"]["bfgs_history_reset"] == 2 + accounting = [{"job_id": "42_0", "state": "FAILED", "exit_code": "3:0"}, + {"job_id": "42_1", "state": "COMPLETED", "exit_code": "0:0"}] + result = build_diagnostics(tool_name="elasticdft", job_id="42", prepared_manifest=_elastic_manifest(), + accounting=accounting, collection=collection, outcome="failed", + pw_exit_logged=True) + failed, completed = result["members"] + assert failed["category"] == "qe_bfgs_failed" and completed["category"] == "completed" + assert failed["label"] == "ε_xx = -0.01" and completed["label"] == "ε_xy = +0.005" + assert failed["evidence"][0]["text"].startswith("bfgs failed after") + assert failed["pw_exit_status"] == 3 + assert "Estimated total dynamical RAM" in failed["qe"]["context"]["estimated_total_ram"] + message = summary_message(result) + assert "1 of 2 ElasticDFT strain members did not succeed" in message + assert "BFGS stopped before meeting the requested tolerances" in message + assert "memory" not in message.lower() + assert "never loosens" in result["guidance"]["qe_bfgs_failed"] + + +def test_parser_rejects_the_actual_failure_and_maximum_step_wording(tmp_path): + path = tmp_path / "relax.stdout" + path.write_text(BFGS_FAILED_OUTPUT) + parsed = parse_qe_output(path, require_stress=True) + assert parsed["process_completed"] is True and parsed["convergence"] is False + path.write_text(BFGS_FAILED_OUTPUT.replace( + "bfgs failed after 10 scf cycles and 9 bfgs steps, convergence not achieved", + "The maximum number of steps has been reached.")) + assert parse_qe_output(path, require_stress=True)["convergence"] is False + + +def test_step_accounting_is_attributed_to_this_run(remote): + run_id, _ = remote + with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout=_sacct(run_id), stderr="")): + collection = _collect(run_id, include_scheduler_steps=True) + steps = collection["scheduler_steps"] + assert {step["job_id"] for step in steps} == {"42_0", "42_0.batch", "42_0.0", "42_1", "42_1.0"} + assert "scheduler_steps_error" not in collection + for comment, owner in (("dt-run:another-run", None), (None, "someone-else")): + output = _sacct(run_id, comment=comment, owner=owner) + with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout=output, stderr="")): + rejected = _collect(run_id, include_scheduler_steps=True) + assert "scheduler_steps" not in rejected and "attribution" in rejected["scheduler_steps_error"] + assert rejected["artifact_manifest"] # archived evidence is unaffected + unattributed = _sacct(run_id).replace(f"|dt-run:{run_id}|", "||", 1) + with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout=unattributed, stderr="")): + partial = _collect(run_id, include_scheduler_steps=True) + assert {step["job_id"] for step in partial["scheduler_steps"]} == {"42_1", "42_1.0"} + assert "omitted" in partial["scheduler_steps_error"] + + +def test_scheduler_memory_state_requires_scheduler_evidence(): + steps = [{"job_id": "42_0", "state": "FAILED", "exit_code": "0:125"}, + {"job_id": "42_0.0", "state": "OUT_OF_MEMORY", "exit_code": "0:125", "max_rss": "250000K"}] + result = build_diagnostics(tool_name="elasticdft", job_id="42", prepared_manifest=_elastic_manifest(1), + accounting=[{"job_id": "42_0", "state": "FAILED", "exit_code": "0:125"}], + collection={"scheduler_steps": steps, "diagnostics": {"files": {}}}, + outcome="failed") + member = result["members"][0] + assert member["category"] == "out_of_memory" + assert "42_0.0" in member["evidence"][0]["text"] + assert member["resource_usage"]["max_rss"] == "250000K" + + +# --- worker collection -------------------------------------------------------- + +class CollectGateway: + def __init__(self, result): + self.result, self.calls = result, [] + + def invoke(self, **kwargs): + self.calls.append(kwargs) + return "task-collect", self.result + + +def _failed_run(protocol=2): + return { + "id": str(uuid4()), "cluster": "negishi", "status": "collecting", "tool_name": "elasticdft", + "scheduler_job_id": "42", "input_json": {}, "prepared_manifest_json": _elastic_manifest(), + "error_json": {"type": "SchedulerFailure", "message": "pending"}, + "external_workflow_ref_json": { + "terminal_outcome": "failed", "gateway_protocol": protocol, + "scheduler_accounting": [{"job_id": "42_0", "state": "FAILED", "exit_code": "3:0"}, + {"job_id": "42_1", "state": "COMPLETED", "exit_code": "0:0"}], + }, + } + + +def _worker_collect(run, clusters, result, tmp_path): + def update(_id, status=None, values=None, **_kwargs): + if status: + run["status"] = status + run.update(values or {}) + + gateway = CollectGateway(result) + with patch.dict(os.environ, {"RCAC_CLUSTERS_JSON": clusters}), \ + patch("computations.worker.update_run", side_effect=update): + worker = ComputationWorker("diagnostics", gateway=gateway) + worker.work_root = tmp_path + worker._collect(run) + return run, gateway.calls[0]["payload"] + + +def test_worker_collection_merges_evidence_on_protocol_2(remote, tmp_path): + run_id, _ = remote + collection = _collect(run_id) + run, payload = _worker_collect(_failed_run(), _clusters_v2(), collection, tmp_path) + assert payload["include_diagnostics"] is True and payload["include_scheduler_steps"] is True + assert "task_000/elastic_inputs/strain_000/CRASH" in payload["retained_artifacts"] + assert run["status"] == "failed" + assert run["artifact_manifest_json"] == collection["artifact_manifest"] + diagnostics = run["external_workflow_ref_json"]["diagnostics"] + assert diagnostics["evidence_available"] is True + assert diagnostics["category_counts"] == {"qe_bfgs_failed": 1, "completed": 1} + assert "BFGS stopped" in run["error_json"]["message"] + assert run["error_json"]["type"] == "SchedulerFailure" + + +def test_worker_collection_stays_compatible_with_protocol_1(tmp_path): + result = {"parsed_outputs": {}, "artifact_manifest": [{"name": "slurm-42_0.err"}], + "missing_artifacts": [], "collection_errors": []} + run, payload = _worker_collect(_failed_run(protocol=1), _clusters(), result, tmp_path) + assert "include_diagnostics" not in payload and "include_scheduler_steps" not in payload + assert not any(name.endswith("CRASH") for name in payload["retained_artifacts"]) + diagnostics = run["external_workflow_ref_json"]["diagnostics"] + assert diagnostics["evidence_available"] is False + assert diagnostics["category_counts"] == {"nonzero_exit": 1, "completed": 1} + assert run["status"] == "failed" + + +def test_cancelled_collection_keeps_cancellation_semantics(tmp_path): + run = _failed_run(protocol=1) + run["external_workflow_ref_json"].update(terminal_outcome="cancelled", scheduler_accounting=[ + {"job_id": "42_0", "state": "CANCELLED by 1", "exit_code": "0:15"}, + {"job_id": "42_1", "state": "COMPLETED", "exit_code": "0:0"}]) + run["error_json"] = {"type": "Cancelled", "message": "pending"} + run, _ = _worker_collect(run, _clusters(), {"parsed_outputs": {}, "artifact_manifest": []}, tmp_path) + assert run["status"] == "cancelled" and run["cancel_requested"] is False + assert run["error_json"]["type"] == "Cancelled" + assert run["error_json"]["message"].startswith("Cancellation confirmed by Slurm.") + + +def test_completed_job_with_invalid_science_fails_precisely(tmp_path): + run = _failed_run(protocol=1) + run["external_workflow_ref_json"].pop("terminal_outcome") + run["external_workflow_ref_json"]["scheduler_accounting"] = [ + {"job_id": "42_0", "state": "COMPLETED", "exit_code": "0:0"}, + {"job_id": "42_1", "state": "COMPLETED", "exit_code": "0:0"}] + reason = "bfgs failed after 5 scf cycles and 4 bfgs steps, convergence not achieved" + result = {"parsed_outputs": {ELASTIC: {"scientific_status": "not_converged", "failure_reasons": [reason]}, + ELASTIC_1: {"scientific_status": "converged", "failure_reasons": []}}, + "artifact_manifest": [{"name": ELASTIC}]} + from computations.qe import QeParseError + with patch("computations.worker.elastic_outputs_from_parsed", + side_effect=QeParseError(f"ElasticDFT member did not converge: {ELASTIC}: {reason}")): + run, _ = _worker_collect(run, _clusters(), result, tmp_path) + assert run["status"] == "failed" and run["error_json"]["type"] == "QeParseError" + assert "did not converge" in run["error_json"]["message"] + diagnostics = run["external_workflow_ref_json"]["diagnostics"] + assert diagnostics["outcome"] == "result_validation_failed" + assert diagnostics["category_counts"] == {"qe_bfgs_failed": 1, "completed": 1} + assert diagnostics["members"][0]["scientific_status"] == "not_converged" + assert "memory" not in run["error_json"]["message"].lower() + + +def test_scheduler_terminal_message_is_not_a_flat_state_list(monkeypatch): + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters()) + run = _failed_run(protocol=1) + run.update(status="running", external_workflow_ref_json={}) + states = ["FAILED"] * 21 + ["COMPLETED"] * 3 + accounting = [{"job_id": f"42_{i}", "state": state, "exit_code": "3:0" if state == "FAILED" else "0:0"} + for i, state in enumerate(states)] + with patch.object(ComputationWorker, "_invoke", return_value=("task", {"states": {"42": states}, + "accounting": {"42": accounting}})), \ + patch("computations.worker.update_run") as update: + ComputationWorker()._advance_cluster("negishi", [run]) + error = update.call_args.kwargs["values"]["error_json"] + assert "FAILED, FAILED" not in error["message"] + assert "21 of 24 ElasticDFT strain members did not succeed" in error["message"] + assert "3:0 ×21" in error["message"] and "memory" not in error["message"].lower() + assert error["scheduler_state_counts"] == {"FAILED": 21, "COMPLETED": 3} + assert update.call_args.kwargs["status"] == "collecting" + + +# --- API view ---------------------------------------------------------------- + +def test_older_failed_run_gets_derived_per_member_diagnostics(): + row = {"id": str(uuid4()), "tool_name": "elasticdft", "status": "failed", "scheduler_job_id": "42", + "prepared_manifest_json": _elastic_manifest(), + "external_workflow_ref_json": {"scheduler_accounting": [ + {"job_id": "42_0", "state": "FAILED", "exit_code": "3:0"}, + {"job_id": "42_1", "state": "COMPLETED", "exit_code": "0:0"}]}} + public = _public_run(row) + diagnostics = public["diagnostics"] + assert diagnostics["derived"] is True and diagnostics["evidence_available"] is False + assert diagnostics["category_counts"] == {"nonzero_exit": 1, "completed": 1} + assert "BFGS" in diagnostics["members"][0]["exit_code_hint"] + assert public["resource_request"] == {} + + +def test_stored_diagnostics_are_not_duplicated_in_provenance(): + stored = {"schema": "dt-run-diagnostics/1", "members": []} + public = _public_run({"id": str(uuid4()), "status": "failed", + "external_workflow_ref_json": {"diagnostics": stored, "cluster": "negishi"}}) + assert public["diagnostics"] == stored and "diagnostics" not in public["external_workflow_ref"] + + +def test_inherited_settings_are_shown_without_changes(): + parent = {"id": str(uuid4()), "tool_name": "cellrelaxdft", "status": "data_ingested", + "input_json": {"KE_cutoff": 90, "scf_conv": 1e-10, "energy_conv": 1e-10, "force_conv": 1e-05, + "e_steps": 100, "i_steps": 100, "is_metal": False, "structure_dict": {"big": 1}}, + "output_json": {"convergence": True, "lattice_parameters": [3.116151483, 3.1161517566381574, 4.987618695]}} + settings = _public_run(parent)["elastic_inherited_settings"] + assert settings["energy_conv"] == 1e-10 and settings["force_conv"] == 1e-05 and settings["scf_conv"] == 1e-10 + assert settings["kmesh"] == [9, 9, 6] # the AlN strain-0 mesh in the verified run + assert "structure_dict" not in settings + assert elastic_kmesh([3.116151483, 3.116151483, 4.987618695], False) == [9, 9, 6] + elastic = {"id": str(uuid4()), "tool_name": "elasticdft", "status": "failed", + "prepared_manifest_json": {"metadata": {"inherited_inputs": { + **parent["input_json"], "k_x": 9, "k_y": 9, "k_z": 6}}}} + used = _public_run(elastic)["elastic_inherited_settings"] + assert used["kmesh"] == [9, 9, 6] and used["energy_conv"] == 1e-10 + + +def test_diagnostics_export_endpoint_is_authorized_json(): + from test_computations_router import ComputationRouterSafetyTests + run = {"id": "123", "tool_name": "elasticdft", "status": "failed", "error": {"type": "SchedulerFailure"}, + "diagnostics": {"members": []}, "external_workflow_ref": {"submitted_resources": {"ntasks": 64}}, + "prepared_manifest": {"ntasks": 64}, "artifact_manifest": [{"name": ELASTIC, "uri": "/x", "sha256": "a"}]} + client = ComputationRouterSafetyTests.client() + with patch("routers.computations.feature_enabled", return_value=True), \ + patch("routers.computations.get_run", return_value=run): + response = client.get("/api/computations/runs/123/diagnostics") + assert response.status_code == 200 + body = response.json() + assert body["schema"] == "dt-computation-diagnostics-export/1" + assert body["submitted_resources"] == {"ntasks": 64} and body["artifacts"][0]["name"] == ELASTIC + assert "attachment" in response.headers["content-disposition"] + with patch("routers.computations.feature_enabled", return_value=True), \ + patch("routers.computations.get_run", side_effect=PermissionError("denied")): + assert client.get("/api/computations/runs/123/diagnostics").status_code == 403 diff --git a/api/tests/test_computation_enrichment.py b/api/tests/test_computation_enrichment.py new file mode 100644 index 0000000..0d2ede1 --- /dev/null +++ b/api/tests/test_computation_enrichment.py @@ -0,0 +1,543 @@ +"""Archive-based diagnostic enrichment and non-mutating serialization. + +No real endpoint, cluster or database: the registered gateway function runs +locally against a temporary archive, and the database layer is mocked. +""" +import hashlib +import io +import json +import os +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +import pytest +from computations.backfill_diagnostics import ( + ENRICHMENT_EVENT, + EnrichmentError, + apply_plan, + main, + plan_enrichment, +) +from computations.diagnostics import req_mem_scope, resource_usage +from computations.globus_gateway import rcac_service_gateway +from computations.rcac import configured_clusters +from computations.store import EnrichmentConflict, _public_run, enrich_run_diagnostics, run_version +from test_computation_diagnostics import BFGS_FAILED_OUTPUT, STDERR_EXIT_3, _elastic_manifest +from test_computation_resources import _clusters_v2 +from test_computation_worker import _clusters + +ARTIFACT_ROOT = "/depot/ncn/data/digital-twins/runs" # from _clusters() +QE_0 = "task_000/elastic_inputs/strain_000/relax.stdout" +QE_1 = "task_001/elastic_inputs/strain_001/relax.stdout" + + +# --- R5: serialization never mutates its source ----------------------------- + +def test_public_serialization_keeps_stored_diagnostics_in_the_source(): + stored = {"schema": "dt-run-diagnostics/1", "category_counts": {"qe_bfgs_failed": 21, "completed": 3}, + "members": [{"category": "qe_bfgs_failed"}]} + row = {"id": str(uuid4()), "tool_name": "elasticdft", "status": "failed", + "external_workflow_ref_json": {"cluster": "negishi", "diagnostics": stored}} + original = deepcopy(row) + first, second = _public_run(row), _public_run(row) + assert row == original + assert first["diagnostics"] == second["diagnostics"] == original["external_workflow_ref_json"]["diagnostics"] + assert "diagnostics" not in first["external_workflow_ref"] + first["diagnostics"]["members"].clear() # a caller editing the copy must not reach the row + assert row == original + + +def test_req_mem_scope_and_usage_keep_per_task_and_requested_memory_apart(): + assert [req_mem_scope(v) for v in ("2000Mc", "125Gn", "125G", "", "?")] == [ + "per_cpu", "per_node", "not_stated", "unknown", "unknown"] + usage = resource_usage([{"job_id": "42", "req_mem": "2000Mc"}, {"job_id": "42.0", "max_rss": "900K"}], "42") + assert usage == [{"array_task_id": None, "state": "", "exit_code": "", "elapsed": "", "time_limit": "", + "req_mem": "2000Mc", "alloc_cpus": "", "nnodes": "", "req_mem_scope": "per_cpu", + "max_rss": "900K"}] + + +# --- archive fixture ------------------------------------------------------------- + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +@pytest.fixture +def archived(tmp_path, monkeypatch): + """A failed two-member ElasticDFT run with its checksummed archive.""" + run_id = str(uuid4()) + root = (tmp_path / "archive").resolve() + files = { + QE_0: BFGS_FAILED_OUTPUT.encode(), + QE_1: BFGS_FAILED_OUTPUT.replace( + "bfgs failed after 10 scf cycles and 9 bfgs steps, convergence not achieved", + "bfgs converged in 5 scf cycles and 4 bfgs steps").encode(), + "slurm-42_0.err": STDERR_EXIT_3.encode(), + "slurm-42_1.err": b"", + "elastic-inputs.tar.gz": b"not evidence", + } + manifest = [] + for name, data in files.items(): + path = root / run_id / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + manifest.append({"name": name, "storage_uri": f"depot://{ARTIFACT_ROOT}/{run_id}/{name}", + "sha256": _sha(data), "size_bytes": len(data), "downloadable": True}) + monkeypatch.setenv("DT_CLUSTER_NAME", "negishi") + monkeypatch.setenv("DT_ARTIFACT_ROOT", str(root)) + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters_v2()) + stamp = datetime(2026, 10, 6, 18, 59, 45, tzinfo=timezone.utc) + run = { + "id": run_id, "tool_name": "elasticdft", "status": "failed", "cluster": "negishi", + "scheduler_job_id": "42", "input_fingerprint": "elasticdft/r27-dt2/abc", + "input_json": {"mp_id": "mp-661"}, "output_json": {}, + "prepared_manifest_json": _elastic_manifest(), "artifact_manifest_json": manifest, + "error_json": {"type": "SchedulerFailure", "message": "Scheduler reported FAILED, COMPLETED"}, + "external_workflow_ref_json": { + "scheduler_accounting": [{"job_id": "42_0", "state": "FAILED", "exit_code": "3:0"}, + {"job_id": "42_1", "state": "COMPLETED", "exit_code": "0:0"}], + "missing_artifacts": [], "collection_errors": [], + }, + "submitted_at": stamp, "started_at": stamp, "completed_at": stamp, "updated_at": stamp, + } + return run, root / run_id + + +class LocalGateway: + """Runs the serialized gateway function in-process against the archive.""" + + def __init__(self): + self.calls = [] + + def invoke(self, **kwargs): + self.calls.append(kwargs) + return "task-archive", rcac_service_gateway({"action": kwargs["action"], "payload": kwargs["payload"]}) + + +def _cluster(): + return configured_clusters()["negishi"] + + +# --- gateway archive_evidence ------------------------------------------------ + +def _archive_request(run, **overrides): + entries = [{key: item[key] for key in ("name", "sha256", "size_bytes")} + for item in run["artifact_manifest_json"] if item["name"] != "elastic-inputs.tar.gz"] + payload = {"cluster": "negishi", "run_id": run["id"], "job_id": "42", "tool_name": "elasticdft", + "expected_artifacts": [QE_0, QE_1], "artifacts": entries} + payload.update(overrides) + return {"action": "archive_evidence", "payload": payload} + + +def test_gateway_archive_evidence_uses_the_collection_extractor(archived): + run, _ = archived + files = rcac_service_gateway(_archive_request(run))["diagnostics"]["files"] + assert set(files) == {QE_0, QE_1, "slurm-42_0.err", "slurm-42_1.err"} + assert files[QE_0]["counts"]["bfgs_failed"] == 1 + assert "2.7E-09" in files[QE_0]["final"]["energy_error"] + assert files["slurm-42_1.err"] == {"kind": "scheduler_log", "size_bytes": 0} + + +def test_gateway_archive_evidence_keeps_mismatched_or_missing_files_unknown(archived): + run, directory = archived + request = _archive_request(run) + request["payload"]["artifacts"][0]["sha256"] = "0" * 64 + (directory / "slurm-42_0.err").unlink() + outside = directory.parent / "outside.txt" + outside.write_text("secret") + (directory / "slurm-42_1.err").unlink() + (directory / "slurm-42_1.err").symlink_to(outside) + files = rcac_service_gateway(request)["diagnostics"]["files"] + assert files[request["payload"]["artifacts"][0]["name"]]["error"] == "archived file does not match its recorded checksum" + assert files["slurm-42_0.err"]["error"] == "archived file is unavailable" + assert files["slurm-42_1.err"]["error"] == "archived file is unavailable" + assert all("markers" not in record for record in files.values() if "error" in record) + + +@pytest.mark.parametrize("entry,message", [ + ({"name": "../other/relax.stdout"}, "unsafe relative path"), + ({"name": "/etc/passwd"}, "unsafe relative path"), + ({"name": "task_000/elastic_inputs/strain_001/relax.stdout"}, "unapproved"), + ({"name": "slurm-43_0.err"}, "unapproved"), + ({"name": "dt-request.json"}, "not evidence"), + ({"size_bytes": 64 * 1024 * 1024 + 1}, "bounds"), + ({"size_bytes": True}, "bounds"), + ({"sha256": "not-a-digest"}, "bounds"), +]) +def test_gateway_archive_evidence_rejects_unsafe_requests(archived, entry, message): + run, _ = archived + request = _archive_request(run) + request["payload"]["artifacts"][0].update(entry) + with pytest.raises(RuntimeError, match=message): + rcac_service_gateway(request) + + +def test_gateway_archive_evidence_enforces_count_and_shape(archived): + run, _ = archived + too_many = [{"name": QE_0, "sha256": "a" * 64, "size_bytes": 1}] * 401 + for overrides in ({"artifacts": too_many}, {"artifacts": []}, {"job_id": "42;rm"}, + {"tool_name": "shell"}, {"expected_artifacts": ["vcrelax.stdout"]}): + with pytest.raises(RuntimeError): + rcac_service_gateway(_archive_request(run, **overrides)) + duplicate = _archive_request(run) + duplicate["payload"]["artifacts"].append(dict(duplicate["payload"]["artifacts"][0])) + with pytest.raises(RuntimeError, match="unapproved"): + rcac_service_gateway(duplicate) + + +# --- planning ------------------------------------------------------------------- + +def test_plan_reproduces_the_confirmed_bfgs_diagnosis_without_writing(archived): + run, _ = archived + original = deepcopy(run) + gateway = LocalGateway() + plan = plan_enrichment(run, _cluster(), gateway) + assert run == original + sent = gateway.calls[0]["payload"]["artifacts"] + assert {entry["name"] for entry in sent} == {QE_0, QE_1, "slurm-42_0.err", "slurm-42_1.err"} + diagnostics = plan["diagnostics"] + assert diagnostics["category_counts"] == {"qe_bfgs_failed": 1, "completed": 1} + failed = diagnostics["members"][0] + assert failed["evidence"][0]["text"].startswith("bfgs failed after") + assert "resource_usage" not in failed and "scheduler_resource_usage" not in json.dumps(diagnostics) + assert any("no job was rerun" in note for note in diagnostics["notes"]) + enrichment = diagnostics["enrichment"] + assert enrichment["previous_error"]["message"] == "Scheduler reported FAILED, COMPLETED" + assert enrichment["previous_diagnostics"]["derived"] is True + assert plan["error_json"]["type"] == "SchedulerFailure" + assert "BFGS stopped before meeting the requested tolerances" in plan["error_json"]["message"] + assert "memory" not in plan["error_json"]["message"].lower() + assert plan["version"] == run_version(original) and plan["already_enriched"] is False + + +def test_unmatched_or_unavailable_archive_evidence_stays_unknown(archived): + run, directory = archived + run["artifact_manifest_json"][0]["storage_uri"] = f"depot://{ARTIFACT_ROOT}/{uuid4()}/{QE_0}" + (directory / "slurm-42_0.err").unlink() + gateway = LocalGateway() + plan = plan_enrichment(run, _cluster(), gateway) + assert QE_0 not in {entry["name"] for entry in gateway.calls[0]["payload"]["artifacts"]} + failed = plan["diagnostics"]["members"][0] + assert failed["category"] == "nonzero_exit" # not inferred from absent files + assert set(plan["diagnostics"]["enrichment"]["files_unavailable"]) == {QE_0, "slurm-42_0.err"} + + +@pytest.mark.parametrize("change,message", [ + ({"status": "running"}, "Only failed or cancelled"), + ({"status": "data_ingested"}, "Only failed or cancelled"), + ({"scheduler_job_id": ""}, "no Slurm job"), + ({"external_workflow_ref_json": {"cache_hit": True}}, "cached"), + ({"artifact_manifest_json": []}, "No archived"), +]) +def test_ineligible_runs_are_refused_before_any_remote_read(archived, change, message): + run, _ = archived + run.update(change) + gateway = LocalGateway() + with pytest.raises(EnrichmentError, match=message): + plan_enrichment(run, _cluster(), gateway) + assert gateway.calls == [] + + +def test_legacy_protocol_cluster_is_refused_without_contacting_the_gateway(archived, monkeypatch): + run, _ = archived + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters()) + gateway = LocalGateway() + with pytest.raises(EnrichmentError, match="protocol-2 gateway"): + plan_enrichment(run, _cluster(), gateway) + assert gateway.calls == [] + + +def test_repeated_enrichment_with_the_same_evidence_is_a_no_op(archived): + run, _ = archived + plan = plan_enrichment(run, _cluster(), LocalGateway()) + run["external_workflow_ref_json"]["diagnostics"] = plan["diagnostics"] + run["error_json"] = plan["error_json"] + again = plan_enrichment(run, _cluster(), LocalGateway()) + assert again["already_enriched"] is True + assert again["diagnostics"]["enrichment"]["evidence_sha256"] == plan["diagnostics"]["enrichment"]["evidence_sha256"] + with patch("computations.store.enrich_run_diagnostics") as write: + assert apply_plan(again) == "unchanged" + write.assert_not_called() + + +# --- guarded write --------------------------------------------------------------- + +def _connection(row): + conn = MagicMock() + cursor = conn.cursor.return_value.__enter__.return_value + cursor.fetchone.return_value = row + return conn, cursor + + +def _write(row, plan, **overrides): + values = dict(expected_version=plan["version"], diagnostics=plan["diagnostics"], + error_json=plan["error_json"], event_message=ENRICHMENT_EVENT, + event_details=plan["event_details"]) + values.update(overrides) + return enrich_run_diagnostics(row["id"], **values) + + +def test_store_writes_only_diagnostics_and_summary_once(archived): + run, _ = archived + plan = plan_enrichment(run, _cluster(), LocalGateway()) + conn, cursor = _connection(deepcopy(run)) + with patch("computations.store.get_pg_worker_connection", return_value=conn), \ + patch("computations.store._insert_event") as event: + assert _write(run, plan) == "updated" + sql, params = cursor.execute.call_args_list[-1].args + assert "jsonb_set" in sql and "error_json" in sql + for untouched in ("status", "updated_at", "completed_at", "output_json", "input_json", + "artifact_manifest_json", "input_fingerprint", "scheduler_job_id"): + assert f"{untouched} =" not in sql + assert params[0].adapted == plan["diagnostics"] and params[1].adapted == plan["error_json"] + event.assert_called_once() + assert event.call_args.args[2:4] == ("failed", ENRICHMENT_EVENT) + assert event.call_args.args[4]["previous_error"]["message"] == "Scheduler reported FAILED, COMPLETED" + conn.commit.assert_called_once() + + +@pytest.mark.parametrize("change", [{"status": "submitted"}, {"error_json": {"type": "Other"}}, + {"artifact_manifest_json": []}, + {"updated_at": datetime(2026, 10, 7, tzinfo=timezone.utc)}]) +def test_store_refuses_when_the_run_changed_after_preview(archived, change): + run, _ = archived + plan = plan_enrichment(run, _cluster(), LocalGateway()) + conn, cursor = _connection({**deepcopy(run), **change}) + with patch("computations.store.get_pg_worker_connection", return_value=conn), \ + patch("computations.store._insert_event") as event: + with pytest.raises(EnrichmentConflict): + _write(run, plan) + assert cursor.execute.call_count == 1 # the locked read only + event.assert_not_called() + conn.commit.assert_not_called() + + +def test_store_is_idempotent_for_an_already_stored_digest(archived): + run, _ = archived + plan = plan_enrichment(run, _cluster(), LocalGateway()) + stored = deepcopy(run) + stored["external_workflow_ref_json"]["diagnostics"] = plan["diagnostics"] + conn, cursor = _connection(stored) + with patch("computations.store.get_pg_worker_connection", return_value=conn), \ + patch("computations.store._insert_event") as event: + assert _write(run, plan, expected_version="stale") == "unchanged" + assert cursor.execute.call_count == 1 + event.assert_not_called() + + +# --- operator command ---------------------------------------------------------- + +def test_command_previews_by_default_and_writes_only_with_apply(archived, capsys): + run, _ = archived + with patch("computations.store.get_run", return_value=deepcopy(run)), \ + patch("computations.store.enrich_run_diagnostics", return_value="updated") as write: + assert main(["--run-id", run["id"]], gateway=LocalGateway()) == 0 + write.assert_not_called() + preview = capsys.readouterr().out + assert "Preview only; nothing was written" in preview and '"qe_bfgs_failed": 1' in preview + assert main(["--run-id", run["id"], "--apply"], gateway=LocalGateway()) == 0 + write.assert_called_once() + assert write.call_args.args[0] == run["id"] + assert '"outcome": "updated"' in capsys.readouterr().out + + +def test_command_requires_an_exact_run_id(): + with pytest.raises(SystemExit): + main(["--run-id", "not-a-uuid"]) + with pytest.raises(SystemExit): + main([]) + + +def test_command_reports_refusals_without_writing(archived): + run, _ = archived + with patch("computations.store.get_run", return_value={**deepcopy(run), "status": "running"}), \ + patch("computations.store.enrich_run_diagnostics") as write: + with pytest.raises(SystemExit, match="Only failed or cancelled"): + main(["--run-id", run["id"], "--apply"], gateway=LocalGateway()) + write.assert_not_called() + + +# --- round 3: enrichment never reduces recorded information -------------------- + +def _stored_member(index, category, **fields): + return {"array_task_id": index, "label": "", "state": "FAILED", "exit_code": "3:0", "category": category, + "artifacts": {"qe_output": QE_0, "crash": "task_000/elastic_inputs/strain_000/CRASH", + "logs": ["slurm-42_0.err", "slurm-42_0.out"]}, + "scheduler_steps": [], "evidence": [], **fields} + + +def test_enrichment_reuses_recorded_attributed_steps_and_usage(archived): + run, _ = archived + allocation = {"job_id": "42_0", "state": "FAILED", "exit_code": "3:0", "req_mem": "2000Mc"} + oom_step = {"job_id": "42_0.0", "state": "OUT_OF_MEMORY", "exit_code": "0:9", "max_rss": "1000K"} + usage = {"array_task_id": 0, "max_rss": "1500K", "req_mem": "2000Mc", "req_mem_scope": "per_cpu"} + run["external_workflow_ref_json"]["diagnostics"] = { + "schema": "dt-run-diagnostics/1", "outcome": "failed", + "collection": {"scheduler_steps_error": ""}, + "members": [_stored_member(0, "out_of_memory", allocation=allocation, resource_usage=usage, + scheduler_steps=[oom_step, {"job_id": "99_0.0", "state": "OUT_OF_MEMORY"}])], + } + plan = plan_enrichment(run, _cluster(), LocalGateway()) + member = plan["diagnostics"]["members"][0] + assert member["category"] == "out_of_memory" + assert [step["job_id"] for step in member["scheduler_steps"]] == ["42_0.0"] # foreign job ignored + assert member["allocation"]["req_mem"] == "2000Mc" + assert member["resource_usage"] == usage # recorded observation, not recomputed or invented + enrichment = plan["diagnostics"]["enrichment"] + assert enrichment["recorded_scheduler_records_reused"] == 2 + assert not any("not collected" in note for note in plan["diagnostics"]["notes"]) + + +def test_run_without_recorded_accounting_stays_unknown(archived): + run, _ = archived + plan = plan_enrichment(run, _cluster(), LocalGateway()) + assert plan["diagnostics"]["enrichment"]["recorded_scheduler_records_reused"] == 0 + assert all("resource_usage" not in member and not member["scheduler_steps"] + for member in plan["diagnostics"]["members"]) + assert any("not collected" in note for note in plan["diagnostics"]["notes"]) + + +def test_incomplete_archive_keeps_the_previous_evidence_based_diagnosis(archived): + run, directory = archived + previous = _stored_member(0, "qe_bfgs_failed", evidence=[ + {"source": QE_0, "tag": "bfgs_failed", "line_number": 1450, "text": "bfgs failed after 10 scf cycles"}], + qe={"final": {"bfgs_result": "bfgs failed after 10 scf cycles"}}) + run["external_workflow_ref_json"]["diagnostics"] = {"schema": "dt-run-diagnostics/1", "outcome": "failed", + "members": [previous]} + (directory / QE_0).unlink() + plan = plan_enrichment(run, _cluster(), LocalGateway()) + kept = plan["diagnostics"]["members"][0] + assert kept == {**previous, "retained_from_previous_diagnostics": True} + assert plan["diagnostics"]["category_counts"] == {"qe_bfgs_failed": 1, "completed": 1} + assert plan["diagnostics"]["enrichment"]["members_retained_from_previous_diagnostics"] == [0] + assert QE_0 in plan["diagnostics"]["enrichment"]["files_unavailable"] + assert any("previously recorded diagnosis was kept" in note for note in plan["diagnostics"]["notes"]) + + +def test_complete_archive_evidence_replaces_an_earlier_diagnosis(archived): + run, directory = archived + plain = b" JOB DONE.\n" + (directory / QE_0).write_bytes(plain) + entry = next(item for item in run["artifact_manifest_json"] if item["name"] == QE_0) + entry.update(sha256=_sha(plain), size_bytes=len(plain)) + run["external_workflow_ref_json"]["diagnostics"] = { + "schema": "dt-run-diagnostics/1", "outcome": "failed", + "members": [_stored_member(0, "launch_or_environment")]} + plan = plan_enrichment(run, _cluster(), LocalGateway()) + # Both files were examined, so the archive evidence decides; nothing kept. + assert plan["diagnostics"]["members"][0]["category"] == "nonzero_exit" + assert plan["diagnostics"]["enrichment"]["members_retained_from_previous_diagnostics"] == [] + + +# --- round 3: run-directory isolation --------------------------------------------- + +def _read_artifact(run, name=QE_0): + entry = next(item for item in run["artifact_manifest_json"] if item["name"] == name) + return {"action": "read_artifact", "payload": { + "cluster": "negishi", "run_id": run["id"], "job_id": "42", "name": name, + "size_bytes": entry["size_bytes"], "sha256": entry["sha256"], "offset": 0, "length": 1024}} + + +def _symlink_to_sibling(directory): + sibling = directory.with_name(str(uuid4())) + directory.rename(sibling) + directory.symlink_to(sibling, target_is_directory=True) + return sibling + + +def test_run_directory_symlink_to_a_sibling_run_is_rejected_for_reads(archived): + run, directory = archived + assert rcac_service_gateway(_read_artifact(run))["offset"] == 0 # control + _symlink_to_sibling(directory) + with pytest.raises(RuntimeError, match="symlink"): + rcac_service_gateway(_archive_request(run)) + with pytest.raises(RuntimeError, match="symlink"): + rcac_service_gateway(_read_artifact(run)) + + +def test_collection_never_archives_into_a_sibling_run(archived, tmp_path, monkeypatch): + run, directory = archived + sibling = _symlink_to_sibling(directory) + before = (sibling / "slurm-42_0.err").read_bytes() + work = (tmp_path / "scratch").resolve() / run["id"] + work.mkdir(parents=True) + (work / ".dt-slurm-job-id").write_text("42") + (work / "slurm-42_0.err").write_text("this run's log") + monkeypatch.setenv("DT_WORK_ROOT", str(work.parent)) + with pytest.raises(RuntimeError, match="symlink"): + rcac_service_gateway({"action": "collect", "payload": { + "cluster": "negishi", "run_id": run["id"], "job_id": "42", "tool_name": "elasticdft", + "expected_artifacts": [QE_0], "retained_artifacts": ["slurm-42_0.err"]}}) + assert (sibling / "slurm-42_0.err").read_bytes() == before + + +def test_work_directory_symlink_to_a_sibling_run_is_rejected(tmp_path, monkeypatch): + root = (tmp_path / "scratch").resolve() + run_id = str(uuid4()) + sibling = root / str(uuid4()) + sibling.mkdir(parents=True) + (sibling / ".dt-slurm-job-id").write_text("42") + (root / run_id).symlink_to(sibling, target_is_directory=True) + monkeypatch.setenv("DT_CLUSTER_NAME", "negishi") + monkeypatch.setenv("DT_WORK_ROOT", str(root)) + with pytest.raises(RuntimeError, match="symlink"): + rcac_service_gateway({"action": "recover_submission", "payload": {"cluster": "negishi", "run_id": run_id}}) + + +def test_symlinked_artifact_root_itself_is_still_allowed(archived, tmp_path, monkeypatch): + run, directory = archived + link = tmp_path / "archive-link" + link.symlink_to(directory.parent, target_is_directory=True) + monkeypatch.setenv("DT_ARTIFACT_ROOT", str(link)) + files = rcac_service_gateway(_archive_request(run))["diagnostics"]["files"] + assert files[QE_0]["counts"]["bfgs_failed"] == 1 + + +# --- round 3: one unreadable file stays unknown ---------------------------------- + +@pytest.mark.parametrize("method", ["open", "stat"]) +def test_unreadable_archive_file_does_not_discard_other_evidence(archived, method): + run, directory = archived + blocked = directory / QE_0 + original = getattr(Path, method) + + def guarded(self, *args, **kwargs): + if self == blocked: + raise PermissionError("fixture: unreadable") + return original(self, *args, **kwargs) + + with patch.object(Path, method, guarded): + files = rcac_service_gateway(_archive_request(run))["diagnostics"]["files"] + assert files[QE_0] == {"kind": "qe_output", "error": "archived file could not be read"} + assert files[QE_1]["counts"]["bfgs_converged"] == 1 + assert "counts" in files["slurm-42_0.err"] + + +def test_evidence_comes_only_from_the_bytes_that_were_verified(archived): + run, directory = archived + blocked = directory / QE_0 + size = blocked.stat().st_size + original = Path.open + + def substituted(self, *args, **kwargs): + # Same size as recorded, different content: must not be trusted. + if self == blocked: + return io.BytesIO(b"bfgs failed after 1 scf cycles\n".ljust(size, b" ")) + return original(self, *args, **kwargs) + + with patch.object(Path, "open", substituted): + files = rcac_service_gateway(_archive_request(run))["diagnostics"]["files"] + assert files[QE_0] == {"kind": "qe_output", "error": "archived file does not match its recorded checksum"} + + +def test_malformed_requests_are_still_rejected_not_treated_as_unknown(archived): + run, _ = archived + request = _archive_request(run) + request["payload"]["artifacts"][0]["name"] = "../escape/relax.stdout" + # A path error is a rejected request, never an "unreadable file". + with patch.object(Path, "open", side_effect=PermissionError("must not be reached")): + with pytest.raises(RuntimeError, match="unsafe relative path"): + rcac_service_gateway(request) diff --git a/api/tests/test_computation_resources.py b/api/tests/test_computation_resources.py new file mode 100644 index 0000000..1344223 --- /dev/null +++ b/api/tests/test_computation_resources.py @@ -0,0 +1,430 @@ +"""Execution-resource contract: policy, job spec, gateway, API and worker. + +No real cluster, endpoint or database is contacted. +""" +import base64 +import hashlib +import json +import os +from pathlib import Path +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from computations import resources as policy_module +from computations.globus_gateway import rcac_service_gateway +from computations.qe import PreparedQeRun +from computations.rcac import configured_clusters +from computations.resources import ( + ResourcePolicy, + ResourcePolicyError, + apply_resources, + gateway_policy_differences, +) +from computations.slurm import SlurmJobSpec +from computations.worker import ComputationWorker +from test_computation_worker import ENDPOINT_ID, FUNCTION_ID, _clusters +from test_computations_router import ComputationRouterSafetyTests + +V2 = ResourcePolicy.from_config(2, {"max_mem_per_cpu_mb": 2000}) + + +def _clusters_v2(**resources) -> str: + config = json.loads(_clusters()) + config["negishi"]["gateway_protocol"] = 2 + if resources: + config["negishi"]["resources"] = resources + return json.dumps(config) + + +def _elastic_prepared(tmp_path: Path, count: int = 24) -> PreparedQeRun: + return PreparedQeRun( + tool_name="elasticdft", local_workdir=tmp_path, stage_files=[tmp_path / "elastic-inputs.tar.gz"], + expected_artifacts=[f"task_{i:03d}/elastic_inputs/strain_{i:03d}/relax.stdout" for i in range(count)], + pseudopotentials=[], job_array=f"0-{count - 1}%4", ntasks=64, time_limit="08:00:00", + metadata={"count": count}, + ) + + +def _spec(**overrides): + values = dict(run_id=str(uuid4()), tool_name="cellrelaxdft", workdir="/scratch/dt/run", + account="ncn", partition="cpu", qos="normal", time_limit="24:00:00", ntasks=64, + modules=["quantum-espresso/7.1"], mpi_launcher="srun") + values.update(overrides) + return SlurmJobSpec(**values) + + +# --- shared policy ----------------------------------------------------------- + +def test_defaults_reproduce_the_original_fixed_pilot_contract(tmp_path): + policy = ResourcePolicy() + relax = policy.resolve("cellrelaxdft") + elastic = policy.resolve("elasticdft") + assert relax == {"nodes": 1, "ntasks": 64, "cpus_per_task": 1, "mem_per_cpu_mb": None, + "time_limit_minutes": 1440} + assert elastic["time_limit_minutes"] == 480 and elastic["max_concurrent_members"] == 4 + prepared = apply_resources(_elastic_prepared(tmp_path), elastic) + assert (prepared.ntasks, prepared.time_limit, prepared.job_array) == (64, "08:00:00", "0-23%4") + script = _spec(tool_name="elasticdft", time_limit=prepared.time_limit, array=prepared.job_array).render() + assert "--mem-per-cpu" not in script and "#SBATCH --nodes=1" in script + + +@pytest.mark.parametrize("request_value,message", [ + ({"ntasks": 0}, "MPI ranks"), ({"ntasks": 65}, "MPI ranks"), ({"nodes": 2}, "one node"), + ({"ntasks": "64"}, "malformed"), ({"ntasks": 1.9}, "malformed"), ({"ntasks": True}, "malformed"), + ({"nodes": True}, "malformed"), ({"partition": "standby"}, "malformed"), + ({"max_concurrent_members": 2}, "only to ElasticDFT"), +]) +def test_resource_requests_are_bounded_integers(request_value, message): + with pytest.raises(ResourcePolicyError, match=message): + ResourcePolicy().resolve("cellrelaxdft", request_value) + + +def test_walltime_and_memory_need_protocol_2_and_a_configured_ceiling(): + legacy = ResourcePolicy() + with pytest.raises(ResourcePolicyError, match="Walltime selection is not enabled"): + legacy.resolve("elasticdft", {"time_limit_minutes": 120}) + with pytest.raises(ResourcePolicyError, match="Memory requests are not enabled"): + legacy.resolve("elasticdft", {"mem_per_cpu_mb": 1000}) + no_memory = ResourcePolicy.from_config(2, {}) + with pytest.raises(ResourcePolicyError, match="Memory requests are not enabled"): + no_memory.resolve("elasticdft", {"mem_per_cpu_mb": 1000}) + assert V2.resolve("elasticdft", {"time_limit_minutes": 120, "mem_per_cpu_mb": 1500, + "max_concurrent_members": 2, "ntasks": 16}) == { + "nodes": 1, "ntasks": 16, "cpus_per_task": 1, "mem_per_cpu_mb": 1500, + "time_limit_minutes": 120, "max_concurrent_members": 2, + } + for bad in ({"time_limit_minutes": 481}, {"time_limit_minutes": 5}, {"mem_per_cpu_mb": 2001}, + {"mem_per_cpu_mb": 100}, {"max_concurrent_members": 5}): + with pytest.raises(ResourcePolicyError): + V2.resolve("elasticdft", bad) + + +@pytest.mark.parametrize("protocol,resources", [ + (2, {"max_ntasks": 65}), (2, {"max_mem_per_cpu_mb": 8192}), (1, {"max_mem_per_cpu_mb": 1000}), + (1, {"max_walltime_minutes": {"elasticdft": 60}}), (3, {}), (2, {"partition": "standby"}), + (2, {"max_walltime_minutes": {"elasticdft": 600}}), (2, {"max_ntasks": True}), +]) +def test_cluster_policy_cannot_exceed_hard_ceilings(monkeypatch, protocol, resources): + config = json.loads(_clusters()) + config["negishi"].update(gateway_protocol=protocol, resources=resources) + monkeypatch.setenv("RCAC_CLUSTERS_JSON", json.dumps(config)) + with pytest.raises(RuntimeError, match="incomplete or invalid"): + configured_clusters() + + +def test_catalog_policy_is_honest_about_fixed_values(monkeypatch): + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters()) + public = configured_clusters()["negishi"].public_dict() + elastic = public["resource_policy"]["elasticdft"] + assert public["gateway_protocol"] == 1 + assert elastic["nodes"] == {**elastic["nodes"], "min": 1, "max": 1, "adjustable": False} + assert elastic["ntasks"]["max"] == 64 and elastic["ntasks"]["adjustable"] is True + assert elastic["time_limit_minutes"]["adjustable"] is False + assert elastic["time_limit_minutes"]["default"] == 480 + assert elastic["mem_per_cpu_mb"]["adjustable"] is False and elastic["mem_per_cpu_mb"]["max"] is None + assert "max_concurrent_members" not in public["resource_policy"]["cellrelaxdft"] + v2 = V2.public("elasticdft") + assert v2["mem_per_cpu_mb"]["max"] == 2000 and v2["time_limit_minutes"]["min"] == 10 + + +# --- SlurmJobSpec boundary (R3) ---------------------------------------------- + +@pytest.mark.parametrize("overrides", [ + {"ntasks": 1.9}, {"ntasks": True}, {"ntasks": "64"}, {"nodes": True}, {"nodes": 2}, + {"mem_per_cpu_mb": True}, {"mem_per_cpu_mb": 1500.0}, {"mem_per_cpu_mb": 100}, + {"time_limit": "24:00:01"}, {"time_limit": "25:00:00"}, {"time_limit": "00:05:00"}, + {"array": "0-3%2"}, +]) +def test_job_spec_rejects_non_integer_or_out_of_policy_values(overrides): + with pytest.raises(ValueError): + _spec(**overrides) + + +def test_job_spec_renders_only_validated_integers(): + spec = _spec(tool_name="elasticdft", time_limit="02:00:00", ntasks=16, array="0-23%2", + mem_per_cpu_mb=1500) + script = spec.render() + assert "#SBATCH --ntasks=16" in script and "#SBATCH --mem-per-cpu=1500M" in script + assert spec.rendered_resources() == {"nodes": 1, "ntasks": 16, "cpus_per_task": 1, + "mem_per_cpu_mb": 1500, "time_limit": "02:00:00", + "array": "0-23%2"} + with pytest.raises(ValueError, match="array"): + _spec(tool_name="elasticdft", time_limit="08:00:00", array="0-23%5") + + +# --- remote gateway ---------------------------------------------------------- + +@pytest.fixture +def endpoint(tmp_path, monkeypatch): + root = tmp_path.resolve() + binary = root / "bin" + binary.mkdir() + sbatch = binary / "sbatch" + sbatch.write_text("#!/bin/sh\necho 9100\n", encoding="utf-8") + sbatch.chmod(0o755) + pseudo = root / "pseudos" / "pseudo_PAW_PBEsol" / "Si.upf" + pseudo.parent.mkdir(parents=True) + pseudo.write_text("approved pseudo", encoding="utf-8") + modules = ["gcc/12.2.0", "openmpi/4.1.4", "quantum-espresso/7.1"] + for key, value in { + "PATH": f"{binary}:{os.environ['PATH']}", "DT_CLUSTER_NAME": "negishi", + "DT_WORK_ROOT": str(root / "scratch"), "DT_PSEUDO_ROOT": str(root / "pseudos"), + "DT_ARTIFACT_ROOT": str(root / "archive"), "DT_SLURM_ACCOUNT": "ncn", + "DT_SLURM_PARTITION": "cpu", "DT_SLURM_QOS": "normal", + "DT_QE_MODULES_JSON": json.dumps(modules), "DT_MPI_LAUNCHER": "srun", + }.items(): + monkeypatch.setenv(key, value) + monkeypatch.delenv("DT_MAX_MEM_PER_CPU_MB", raising=False) + return {"root": root, "modules": modules, "pseudos": [{ + "relative": "pseudo_PAW_PBEsol/Si.upf", + "sha256": hashlib.sha256(pseudo.read_bytes()).hexdigest(), + }]} + + +def _submit_request(endpoint, action="submit_v2", **spec_overrides): + run_id = str(uuid4()) + spec = _spec(run_id=run_id, workdir=str(endpoint["root"] / "scratch" / run_id), + modules=endpoint["modules"], time_limit="02:30:00", ntasks=32).as_payload() + spec.update(spec_overrides) + return run_id, {"action": action, "payload": { + "cluster": "negishi", "run_id": run_id, "spec": spec, + "files": {"vcrelax.in": base64.b64encode(b"&CONTROL\n/").decode("ascii")}, + "pseudopotentials": endpoint["pseudos"], + }} + + +def test_gateway_submit_v2_renders_and_echoes_bounded_resources(endpoint, monkeypatch): + monkeypatch.setenv("DT_MAX_MEM_PER_CPU_MB", "2000") + run_id, request = _submit_request(endpoint, mem_per_cpu_mb=2000) + result = rcac_service_gateway(request) + workdir = endpoint["root"] / "scratch" / run_id + script = (workdir / "dt-workflow.sbatch").read_text(encoding="utf-8") + assert "#SBATCH --mem-per-cpu=2000M" in script and "#SBATCH --time=02:30:00" in script + assert "#SBATCH --ntasks=32" in script and "#SBATCH --nodes=1" in script + expected = {"nodes": 1, "ntasks": 32, "cpus_per_task": 1, "mem_per_cpu_mb": 2000, + "time_limit": "02:30:00", "array": ""} + assert result["protocol"] == 2 and result["resources"] == expected + assert json.loads((workdir / "dt-request.json").read_text())["resources"] == expected + # pw.x's own status is logged without changing the job's exit code. + assert "dt: pw.x exit status" in script and 'exit "${status}"' in script + assert rcac_service_gateway(request)["resources"] == expected # idempotent replay + + +@pytest.mark.parametrize("action,overrides,env_memory,message", [ + ("submit_v2", {"mem_per_cpu_mb": 1000}, None, "not enabled on this endpoint"), + ("submit_v2", {"mem_per_cpu_mb": 2048}, "2000", "memory request is not allowed"), + ("submit_v2", {"mem_per_cpu_mb": True}, "2000", "memory request is not allowed"), + ("submit_v2", {"time_limit": "24:00:01"}, None, "walltime"), + ("submit_v2", {"time_limit": "00:05:00"}, None, "walltime"), + ("submit_v2", {"ntasks": 1.9}, None, "task count"), + ("submit_v2", {"ntasks": True}, None, "task count"), + ("submit_v2", {"ntasks": "32"}, None, "task count"), + ("submit_v2", {"nodes": True}, None, "one node"), + ("submit_v2", {"nodes": 2}, None, "one node"), + ("submit", {"time_limit": "02:30:00"}, None, "walltime"), + ("submit", {"time_limit": "24:00:00", "mem_per_cpu_mb": 1000}, "2000", "submit_v2"), + ("submit", {"time_limit": "24:00:00", "ntasks": 1.9}, None, "task count"), +]) +def test_gateway_rejects_out_of_policy_requests_before_staging( + endpoint, monkeypatch, action, overrides, env_memory, message +): + if env_memory: + monkeypatch.setenv("DT_MAX_MEM_PER_CPU_MB", env_memory) + run_id, request = _submit_request(endpoint, action, **overrides) + with pytest.raises(RuntimeError, match=message): + rcac_service_gateway(request) + workdir = endpoint["root"] / "scratch" / run_id + assert not (workdir / "vcrelax.in").exists() and not (workdir / ".dt-submit-intent").exists() + + +def test_legacy_submit_keeps_protocol_1_contract(endpoint): + _run_id, request = _submit_request(endpoint, "submit", time_limit="24:00:00") + result = rcac_service_gateway(request) + assert result["resources"]["time_limit"] == "24:00:00" + assert result["resources"]["mem_per_cpu_mb"] is None + + +def test_gateway_ceilings_match_the_shared_policy(endpoint, monkeypatch): + monkeypatch.setenv("DT_MAX_MEM_PER_CPU_MB", "2000") + capabilities = rcac_service_gateway({"action": "capabilities", "payload": {"cluster": "negishi"}}) + assert capabilities == { + "protocol": policy_module.RESOURCE_GATEWAY_PROTOCOL, "nodes": policy_module.NODES, + "cpus_per_task": policy_module.CPUS_PER_TASK, "max_ntasks": policy_module.MAX_NTASKS, + "max_concurrent_members": policy_module.MAX_CONCURRENT_MEMBERS, + "min_walltime_minutes": policy_module.MIN_WALLTIME_MINUTES, + "max_walltime_minutes": policy_module.MAX_WALLTIME_MINUTES, + "min_mem_per_cpu_mb": policy_module.MIN_MEM_PER_CPU_MB, "max_mem_per_cpu_mb": 2000, + "diagnostics": True, "archive_evidence": True, + } + assert gateway_policy_differences(V2, capabilities) == [] + too_large = ResourcePolicy.from_config(2, {"max_mem_per_cpu_mb": 3000}) + assert any("DT_MAX_MEM_PER_CPU_MB" in item for item in gateway_policy_differences(too_large, capabilities)) + monkeypatch.setenv("DT_MAX_MEM_PER_CPU_MB", "999999") + with pytest.raises(RuntimeError, match="memory ceiling is invalid"): + rcac_service_gateway({"action": "capabilities", "payload": {"cluster": "negishi"}}) + + +def test_protocol_1_capabilities_failure_is_reported_as_a_rollout_problem(): + assert gateway_policy_differences(V2, {"protocol": 1}) != [] + + +# --- API --------------------------------------------------------------------- + +def _post(payload, **patches): + created = {"id": "33333333-3333-4333-8333-333333333333", "status": "requested"} + with ( + patch("routers.computations.feature_enabled", return_value=True), + patch.dict(os.environ, {"RCAC_CLUSTERS_JSON": patches.get("clusters", _clusters()), + "RCAC_SERVICE_ACCOUNT_USERNAME": "dtservice"}, clear=False), + patch("routers.computations.create_run", return_value=created) as create, + ): + response = ComputationRouterSafetyTests.client().post( + "/api/computations/projects/P-1/runs/cellrelaxdft", json=payload) + return response, create + + +def test_api_stores_resources_apart_from_fingerprinted_inputs(): + inputs = {"mp_id": "mp-149", "structure_dict": {}} + first, create = _post({"cluster": "negishi", "inputs": inputs, "resources": {"ntasks": 16}}) + assert first.status_code == 202, first.text + kwargs = create.call_args.kwargs + assert kwargs["resource_request"]["requested"] == {"ntasks": 16} + assert kwargs["resource_request"]["resolved"]["ntasks"] == 16 + assert "ntasks" not in kwargs["inputs"] + second, create_default = _post({"cluster": "negishi", "inputs": inputs}) + assert second.status_code == 202 + # Resources never change the scientific fingerprint (cache identity). + assert create_default.call_args.kwargs["input_fingerprint"] == kwargs["input_fingerprint"] + assert create_default.call_args.kwargs["resource_request"]["requested"] == {} + + +@pytest.mark.parametrize("payload,status", [ + ({"resources": {"ntasks": 65}}, 400), + ({"resources": {"nodes": 2}}, 400), + ({"resources": {"time_limit_minutes": 60}}, 400), # protocol-1 cluster + ({"resources": {"mem_per_cpu_mb": 1000}}, 400), + ({"resources": {"ntasks": "16"}}, 422), + ({"resources": {"ntasks": 1.5}}, 422), + ({"resources": {"ntasks": True}}, 422), + ({"resources": {"sbatch": "--partition=standby"}}, 422), + ({"inputs": {"mp_id": "mp-149", "structure_dict": {}, "ntasks": 4}}, 400), +]) +def test_api_rejects_invalid_or_unsupported_resources(payload, status): + body = {"cluster": "negishi", "inputs": {"mp_id": "mp-149", "structure_dict": {}}, **payload} + response, create = _post(body) + assert response.status_code == status, response.text + create.assert_not_called() + + +# --- worker ------------------------------------------------------------------ + +class SubmitGateway: + def __init__(self, echo=None): + self.calls, self.echo = [], echo + + @staticmethod + def encoded_files(paths): + return {Path(path).name: base64.b64encode(Path(path).read_bytes()).decode("ascii") for path in paths} + + def invoke(self, **kwargs): + self.calls.append(kwargs) + spec = kwargs["payload"]["spec"] + result = {"job_id": "4242", "workdir": spec["workdir"], "comment": spec["comment"]} + if kwargs["action"] == "submit_v2": + rendered = {key: spec[key] for key in ("nodes", "ntasks", "cpus_per_task", "mem_per_cpu_mb")} + rendered.update(time_limit=spec["time_limit"], array=spec["array"]) + result.update(protocol=2, resources=self.echo or rendered) + return "task-submit", result + + +def _relax_run(requested): + return {"id": str(uuid4()), "project_id": "P-1", "requested_by": "student", "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", "status": "requested", "cluster": "negishi", "scheduler_job_id": "", + "input_json": {"mp_id": "mp-149", "structure_dict": {}}, "attempt_count": 0, + "resource_request_json": {"schema": "dt-resource-request/1", "requested": requested}} + + +def _submit(run, clusters, gateway, tmp_path, cached=None): + def update(_id, status=None, values=None, **_kwargs): + if status: + run["status"] = status + run.update(values or {}) + + def prepared(_inputs, workdir, _root): + path = workdir / "vcrelax.in" + path.write_text("input", encoding="utf-8") + return PreparedQeRun(tool_name="cellrelaxdft", local_workdir=workdir, stage_files=[path], + expected_artifacts=["vcrelax.stdout"], pseudopotentials=[], ntasks=64) + + with ( + patch.dict(os.environ, {"RCAC_CLUSTERS_JSON": clusters, "RCAC_SERVICE_ACCOUNT_USERNAME": "dtservice"}), + patch("computations.worker.feature_enabled", return_value=True), + patch("computations.worker.claim_run", return_value=run), + patch("computations.worker.update_run", side_effect=update), + patch("computations.worker.find_cache_hit", return_value=cached), + patch("computations.worker.validate_tool_inputs", side_effect=lambda _tool, value: value), + patch("computations.worker.resolve_material_inputs", side_effect=lambda value: value), + patch("computations.worker.prepare_cellrelax", side_effect=prepared), + ): + worker = ComputationWorker("resources", gateway=gateway) + worker.work_root = tmp_path + worker.run_once(run["id"]) + return run + + +def test_worker_submits_v2_and_records_requested_and_submitted_resources(tmp_path): + gateway = SubmitGateway() + run = _submit(_relax_run({"ntasks": 16, "time_limit_minutes": 120}), _clusters_v2(), gateway, tmp_path) + assert run["status"] == "submitted" + call = gateway.calls[0] + assert call["action"] == "submit_v2" + assert call["payload"]["spec"]["ntasks"] == 16 and call["payload"]["spec"]["time_limit"] == "02:00:00" + external = run["external_workflow_ref_json"] + assert external["resource_confirmation"] == "confirmed_by_gateway" + assert external["submitted_resources"]["ntasks"] == 16 + assert external["resource_request"]["time_limit_minutes"] == 120 + assert run["prepared_manifest_json"]["resources"]["ntasks"] == 16 + + +def test_worker_keeps_tracking_a_job_when_gateway_echo_differs(tmp_path): + gateway = SubmitGateway(echo={"nodes": 1, "ntasks": 64}) + run = _submit(_relax_run({"ntasks": 16}), _clusters_v2(), gateway, tmp_path) + assert run["status"] == "submitted" and run["scheduler_job_id"] == "4242" + assert run["external_workflow_ref_json"]["resource_confirmation"] == "mismatch" + + +def test_worker_uses_legacy_submit_on_protocol_1(tmp_path): + gateway = SubmitGateway() + run = _submit(_relax_run({"ntasks": 32}), _clusters(), gateway, tmp_path) + assert gateway.calls[0]["action"] == "submit" + assert run["external_workflow_ref_json"]["resource_confirmation"] == "protocol_1_fixed_contract" + assert gateway.calls[0]["payload"]["spec"]["time_limit"] == "24:00:00" + + +def test_worker_rejects_stored_request_outside_current_policy_without_submitting(tmp_path): + gateway = SubmitGateway() + run = _relax_run({"time_limit_minutes": 120}) + with patch("computations.worker.get_run", return_value=run): + run = _submit(run, _clusters(), gateway, tmp_path) + assert gateway.calls == [] + assert run["status"] == "failed" + assert "Walltime selection is not enabled" in run["error_json"]["message"] + + +def test_cache_hit_does_not_fabricate_used_resources(tmp_path): + cached = {"id": str(uuid4()), "output_json": {"convergence": True}, "artifact_manifest_json": [], + "prepared_manifest_json": {"resources": {"ntasks": 64}}, + "external_workflow_ref_json": {"submitted_resources": {"ntasks": 64}}} + gateway = SubmitGateway() + run = _submit(_relax_run({"ntasks": 8}), _clusters(), gateway, tmp_path, cached=cached) + external = run["external_workflow_ref_json"] + assert run["status"] == "data_ingested" and gateway.calls == [] + assert external["cache_hit"] is True and external["resources_used"] is False + assert "submitted_resources" not in external + + +def test_endpoint_ids_are_not_published_in_resource_policy(monkeypatch): + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters_v2(max_mem_per_cpu_mb=2000)) + text = json.dumps(configured_clusters()["negishi"].public_dict()) + assert ENDPOINT_ID not in text and FUNCTION_ID not in text diff --git a/api/tests/test_computation_schemas.py b/api/tests/test_computation_schemas.py new file mode 100644 index 0000000..0094757 --- /dev/null +++ b/api/tests/test_computation_schemas.py @@ -0,0 +1,61 @@ +import unittest +from uuid import uuid4 + +from computations.schemas import compute_input_fingerprint, validate_tool_inputs +from pydantic import ValidationError + + +class ComputationSchemaTests(unittest.TestCase): + def test_cellrelax_defaults_and_squid_are_stable(self): + inputs = validate_tool_inputs( + "cellrelaxdft", {"mp_id": "mp-1265", "structure_dict": {}} + ) + self.assertEqual(inputs["pseudo"], "PAW") + self.assertEqual(inputs["exchange_correlation"], "PBEsol") + first = compute_input_fingerprint("cellrelaxdft", inputs) + second = compute_input_fingerprint( + "cellrelaxdft", dict(reversed(list(inputs.items()))) + ) + self.assertEqual(first, second) + self.assertRegex(first, r"^cellrelaxdft/r99-dt2/[0-9a-f]{40}$") + + def test_missing_nc_pseudopotentials_are_not_offered(self): + with self.assertRaises(ValidationError): + validate_tool_inputs( + "cellrelaxdft", + {"mp_id": "mp-1265", "structure_dict": {}, "pseudo": "NC"}, + ) + + def test_electronic_steps_match_r99_bounds(self): + with self.assertRaises(ValidationError): + validate_tool_inputs( + "cellrelaxdft", + {"mp_id": "mp-1265", "structure_dict": {}, "e_steps": 99}, + ) + + def test_hubbard_cross_field_validation(self): + with self.assertRaisesRegex(ValueError, "Hubbard U values"): + validate_tool_inputs( + "cellrelaxdft", + { + "mp_id": "mp-1265", + "structure_dict": {}, + "hubbard_U_on": True, + }, + ) + + def test_elastic_strains_require_compression_and_tension(self): + with self.assertRaises(ValidationError): + validate_tool_inputs( + "elasticdft", + { + "mp_id": "mp-1265", + "upstream_execution_id": str(uuid4()), + "diagonal_strain_magnitudes": [0.005, 0.01], + "offdiagonal_strain_magnitudes": [-0.01, 0.01], + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_computation_worker.py b/api/tests/test_computation_worker.py new file mode 100644 index 0000000..0da1d4d --- /dev/null +++ b/api/tests/test_computation_worker.py @@ -0,0 +1,249 @@ +import base64 +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from uuid import uuid4 + +from computations.qe import PreparedQeRun +from computations.rcac import configured_clusters +from computations.worker import ComputationWorker +from computations.globus_gateway import GlobusTransientError + + +ENDPOINT_ID = "11111111-1111-4111-8111-111111111111" +FUNCTION_ID = "22222222-2222-4222-8222-222222222222" + + +def _clusters() -> str: + return ( + '{"negishi":{"host":"negishi.rcac.purdue.edu",' + f'"endpoint_id":"{ENDPOINT_ID}","gateway_function_id":"{FUNCTION_ID}",' + '"qe_version":"7.1",' + '"module_dependencies":["gcc/12.2.0","openmpi/4.1.4"],' + '"qe_module":"quantum-espresso/7.1","mpi_launcher":"srun",' + '"remote_root":"/scratch/negishi/dtservice/digital-twins/runs",' + '"pseudo_root":"/depot/ncn/data/digital-twins/pseudopotentials",' + '"artifact_root":"/depot/ncn/data/digital-twins/runs"}}' + ) + + +class FakeGateway: + def __init__(self): + self.calls = [] + + @staticmethod + def encoded_files(paths): + return { + Path(path).name: base64.b64encode(Path(path).read_bytes()).decode("ascii") + for path in paths + } + + def invoke(self, **kwargs): + self.calls.append(kwargs) + action = kwargs["action"] + payload = kwargs["payload"] + if action == "submit": + return "task-submit", { + "job_id": "4242", + "workdir": payload["spec"]["workdir"], + "comment": payload["spec"]["comment"], + } + if action == "status_many": + return "task-status", { + "states": { + job["job_id"]: ["COMPLETED"] for job in payload["jobs"] + } + } + if action == "collect": + return "task-collect", { + "parsed_outputs": {"vcrelax.stdout": {"parsed": True}}, + "artifact_manifest": [ + { + "name": "vcrelax.stdout", + "storage_uri": "depot:///depot/ncn/run/vcrelax.stdout", + "sha256": "a" * 64, + "size_bytes": 100, + "downloadable": False, + } + ], + } + if action == "cancel": + return "task-cancel", {"cancelled": True} + raise AssertionError(action) + + +class WorkerLifecycleTests(unittest.TestCase): + def test_transport_retries_are_persisted_and_bounded(self): + run = {"id":str(uuid4()), "external_workflow_ref_json":{}} + def update(_id, values=None, **_kwargs): + run.update(values or {}) + with ( + patch("computations.worker.get_run", side_effect=lambda *a, **kw: run), + patch("computations.worker.update_run", side_effect=update), + patch.object(ComputationWorker, "_fail") as fail, + ): + for attempt in range(1, 4): + # New workers simulate Pod replacement between attempts. + ComputationWorker()._handle_error(run, GlobusTransientError("lookup failed")) + self.assertEqual(run['external_workflow_ref_json']['transport_retry_count'], attempt) + fail.assert_not_called() + ComputationWorker()._handle_error(run, GlobusTransientError("lookup failed")) + fail.assert_called_once() + + def test_negishi_configuration_is_fixed_and_public_metadata_is_safe(self): + with patch.dict(os.environ, {"RCAC_CLUSTERS_JSON": _clusters()}, clear=False): + cluster = configured_clusters()["negishi"] + + self.assertEqual( + cluster.modules, + ("gcc/12.2.0", "openmpi/4.1.4", "quantum-espresso/7.1"), + ) + self.assertEqual(cluster.allocation, "ncn") + self.assertEqual(cluster.partition, "cpu") + self.assertEqual(cluster.qos, "normal") + self.assertEqual(cluster.installation_id, "digital_qe_7_1_negishi") + self.assertNotIn("endpoint_id", cluster.public_dict()) + + def test_disabled_worker_does_not_claim_a_run(self): + with ( + patch("computations.worker.feature_enabled", return_value=False), + patch("computations.worker.claim_next_run") as claim, + ): + self.assertFalse(ComputationWorker("disabled-worker").run_once()) + claim.assert_not_called() + + def test_submit_status_and_collect_preserve_attribution(self): + run_id = str(uuid4()) + run = { + "id": run_id, + "project_id": "P-1", + "requested_by": "student-1", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "sim2l_squid_id": "cellrelaxdft/r99-dt2/hash", + "status": "requested", + "cluster": "negishi", + "scheduler_job_id": "", + "remote_workdir": "", + "input_json": {"mp_id": "mp-1265", "structure_dict": {}}, + "attempt_count": 0, + } + updates = [] + + def update(_run_id, status=None, values=None, **_kwargs): + if status: + run["status"] = status + run.update(values or {}) + updates.append(status) + return run + + def prepared(_inputs, workdir, _pseudo_root): + path = workdir / "vcrelax.in" + path.write_text("input", encoding="utf-8") + return PreparedQeRun( + tool_name="cellrelaxdft", + local_workdir=workdir, + stage_files=[path], + expected_artifacts=["vcrelax.stdout"], + pseudopotentials=[ + { + "element": "C", + "relative": "pseudo_PAW_PBEsol/C.upf", + "path": "/depot/ncn/data/digital-twins/pseudopotentials/pseudo_PAW_PBEsol/C.upf", + "sha256": "a" * 64, + } + ], + ntasks=64, + metadata={"num_atoms": 1}, + ) + + gateway = FakeGateway() + with tempfile.TemporaryDirectory() as directory: + with ( + patch.dict( + os.environ, + { + "RCAC_CLUSTERS_JSON": _clusters(), + "RCAC_SERVICE_ACCOUNT_USERNAME": "dtservice", + }, + clear=False, + ), + patch("computations.worker.claim_next_run", return_value=run), + patch("computations.worker.feature_enabled", return_value=True), + patch("computations.worker.update_run", side_effect=update), + patch("computations.worker.find_cache_hit", return_value=None), + patch( + "computations.worker.validate_tool_inputs", + side_effect=lambda _tool, value: value, + ), + patch( + "computations.worker.resolve_material_inputs", + side_effect=lambda value: {**value, "structure_dict": {"resolved": True}}, + ), + patch("computations.worker.prepare_cellrelax", side_effect=prepared), + patch( + "computations.worker.cellrelax_outputs_from_parsed", + return_value={"convergence": True, "energy": -1.0}, + ), + ): + worker = ComputationWorker("test-worker", gateway=gateway) + worker.work_root = Path(directory) + self.assertTrue(worker.run_once()) + self.assertEqual(run["status"], "submitted") + self.assertEqual(run["scheduler_job_id"], "4242") + self.assertEqual(run["execution_identity"], "dtservice") + self.assertEqual(run["slurm_comment"], f"dt-run:{run_id}") + + self.assertTrue(worker.run_once()) + self.assertEqual(run["status"], "collecting") + self.assertTrue(worker.run_once()) + self.assertEqual(run["status"], "data_ingested") + self.assertEqual(run["output_json"]["energy"], -1.0) + self.assertFalse(run["artifact_manifest_json"][0]["downloadable"]) + + self.assertIn("validating", updates) + self.assertIn("collecting", updates) + self.assertIn("data_ingested", updates) + + def test_one_status_request_covers_all_runs_on_a_cluster(self): + runs = [ + { + "id": str(uuid4()), + "status": "submitted", + "cluster": "negishi", + "scheduler_job_id": str(5000 + index), + "cancel_requested": False, + } + for index in range(2) + ] + gateway = FakeGateway() + with ( + patch.dict( + os.environ, + { + "RCAC_CLUSTERS_JSON": _clusters(), + "RCAC_SERVICE_ACCOUNT_USERNAME": "dtservice", + }, + clear=False, + ), + patch("computations.worker.feature_enabled", return_value=True), + patch("computations.worker.claim_active_runs", return_value=runs), + patch("computations.worker.renew_run_claim", side_effect=lambda run_id, *_: next(r for r in runs if r["id"] == run_id)), + patch("computations.worker.update_run"), + ): + self.assertTrue(ComputationWorker("batch", gateway=gateway).run_cycle()) + status_calls = [call for call in gateway.calls if call["action"] == "status_many"] + self.assertEqual(len(status_calls), 1) + self.assertEqual( + status_calls[0]["payload"]["jobs"], + [ + {"run_id": runs[0]["id"], "job_id": "5000"}, + {"run_id": runs[1]["id"], "job_id": "5001"}, + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_computations_postgres_e2e.py b/api/tests/test_computations_postgres_e2e.py new file mode 100644 index 0000000..9b01b42 --- /dev/null +++ b/api/tests/test_computations_postgres_e2e.py @@ -0,0 +1,587 @@ +"""Opt-in API -> PostgreSQL -> centralized worker -> API lifecycle test. + +Set ``DT_COMPUTATIONS_E2E=1`` and point the standard PG variables at a +disposable PostgreSQL database. This test uses a fake Globus gateway and never +contacts RCAC. +""" + +import base64 +import hashlib +import json +import os +import tempfile +from pathlib import Path +from unittest.mock import patch + +import psycopg2 +import pytest +from computations.backfill_diagnostics import ENRICHMENT_EVENT, apply_plan, plan_enrichment +from computations.globus_gateway import rcac_service_gateway +from computations.qe import PreparedQeRun +from computations.rcac import get_cluster +from computations.store import ensure_tables +from computations.worker import ComputationWorker +from fastapi import FastAPI +from fastapi.testclient import TestClient +from routers import computations as computation_router +from security import PlatformUser, get_platform_user + +pytestmark = pytest.mark.skipif( + os.getenv("DT_COMPUTATIONS_E2E") != "1", + reason="requires an explicitly configured disposable PostgreSQL database", +) + +ENDPOINT_ID = "11111111-1111-4111-8111-111111111111" +FUNCTION_ID = "22222222-2222-4222-8222-222222222222" + + +class FakeGateway: + def __init__(self): + self.actions = [] + + @staticmethod + def encoded_files(paths): + return { + Path(path).name: base64.b64encode(Path(path).read_bytes()).decode("ascii") + for path in paths + } + + def invoke(self, **kwargs): + action = kwargs["action"] + self.actions.append(action) + payload = kwargs["payload"] + if action == "submit": + return "task-submit", { + "job_id": "9001", + "workdir": payload["spec"]["workdir"], + "comment": payload["spec"]["comment"], + } + if action == "status_many": + return "task-status", { + "states": { + job["job_id"]: ["COMPLETED"] for job in payload["jobs"] + } + } + if action == "collect": + return "task-collect", { + "parsed_outputs": {"vcrelax.stdout": {"parsed": True}}, + "artifact_manifest": [ + { + "name": "vcrelax.stdout", + "storage_uri": "depot:///depot/ncn/data/digital-twins/runs/test/vcrelax.stdout", + "sha256": "a" * 64, + "size_bytes": 16, + "content_type": "text/plain", + "downloadable": False, + } + ], + } + raise AssertionError(action) + + +def _connection(): + return psycopg2.connect( + host=os.getenv("PG_HOST", "127.0.0.1"), + port=os.getenv("PG_PORT", "5432"), + dbname=os.getenv("PG_DB", "digital_twin"), + user=os.getenv("PG_SUPER_USER", "postgres"), + password=os.environ["PG_SUPER_PASS"], + ) + + +@pytest.fixture(scope="module") +def configured_database(): + os.environ.update( + { + "RCAC_COMPUTE_ENABLED": "true", + "RCAC_SERVICE_ACCOUNT_USERNAME": "dtservice", + "RCAC_CLUSTERS_JSON": json.dumps( + { + "fake": { + "host": "localhost", + "endpoint_id": ENDPOINT_ID, + "gateway_function_id": FUNCTION_ID, + "qe_version": "7.1", + "module_dependencies": ["gcc/12.2.0", "openmpi/4.1.4"], + "qe_module": "quantum-espresso/7.1", + "mpi_launcher": "srun", + "remote_root": "/scratch/fake/dtservice/digital-twins/runs", + "pseudo_root": "/depot/ncn/data/digital-twins/pseudopotentials", + "artifact_root": "/depot/ncn/data/digital-twins/runs", + } + } + ), + } + ) + conn = _connection() + try: + with conn.cursor() as cur: + cur.execute('CREATE EXTENSION IF NOT EXISTS "pgcrypto"') + cur.execute( + """ + CREATE TABLE IF NOT EXISTS users ( + id VARCHAR(255) PRIMARY KEY, name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL + ); + CREATE TABLE IF NOT EXISTS projects ( + id VARCHAR(50) PRIMARY KEY, name VARCHAR(255) NOT NULL, + access_mode VARCHAR(20) DEFAULT 'private' + ); + CREATE TABLE IF NOT EXISTS project_members ( + project_id VARCHAR(50) REFERENCES projects(id) ON DELETE CASCADE, + nanohub_user_id VARCHAR(100) NOT NULL, + role VARCHAR(20) DEFAULT 'member', + PRIMARY KEY (project_id, nanohub_user_id) + ); + CREATE TABLE IF NOT EXISTS equipment_metadata ( + domain_id VARCHAR(255) PRIMARY KEY, + equipment_name VARCHAR(255) NOT NULL, + equipment_type VARCHAR(100) DEFAULT '', + asset_kind VARCHAR(20) NOT NULL DEFAULT 'physical', + manufacturer VARCHAR(255) DEFAULT '', model VARCHAR(255) DEFAULT '', + location VARCHAR(255) DEFAULT '', description TEXT DEFAULT '', + source_type VARCHAR(100) NOT NULL DEFAULT 'postgresql', + dt_mode VARCHAR(50) NOT NULL DEFAULT 'dashboard', + outlier_method VARCHAR(100) DEFAULT 'zscore', + owner_id VARCHAR(255) REFERENCES users(id), owner_org VARCHAR(255) DEFAULT '', + status VARCHAR(50) NOT NULL DEFAULT 'pending', + parameters_json JSONB NOT NULL DEFAULT '[]'::jsonb, + features_json JSONB NOT NULL DEFAULT '[]'::jsonb, + config_json JSONB NOT NULL DEFAULT '{}'::jsonb, + approved_at TIMESTAMP WITH TIME ZONE, + reviewed_by VARCHAR(255) REFERENCES users(id), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE IF NOT EXISTS experiment_types ( + id VARCHAR(255) PRIMARY KEY, name VARCHAR(255) NOT NULL, + description TEXT DEFAULT '', scientific_objective TEXT DEFAULT '', + equipment_id VARCHAR(255) DEFAULT '', + type_parameters_json JSONB NOT NULL DEFAULT '[]'::jsonb, + outputs_json JSONB NOT NULL DEFAULT '[]'::jsonb, + additional_inputs_json JSONB NOT NULL DEFAULT '[]'::jsonb, + visibility VARCHAR(50) NOT NULL DEFAULT 'shared', version INT NOT NULL DEFAULT 1, + author_name VARCHAR(255) DEFAULT '', author_email VARCHAR(255) DEFAULT '', + owner_id VARCHAR(255) REFERENCES users(id), owner_org VARCHAR(255) DEFAULT '', + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE IF NOT EXISTS project_experiments ( + project_id VARCHAR(50) NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + type_id VARCHAR(255) NOT NULL REFERENCES experiment_types(id) ON DELETE CASCADE, + added_by VARCHAR(255), status VARCHAR(50) NOT NULL DEFAULT 'active', + sequence_index INT NOT NULL DEFAULT 1, upstream_type_id VARCHAR(255) DEFAULT '', + handoff_type VARCHAR(100) NOT NULL DEFAULT 'manual', + handoff_description TEXT DEFAULT '', + output_mapping_json JSONB NOT NULL DEFAULT '{}'::jsonb, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (project_id, type_id) + ); + INSERT INTO users (id, name, email) VALUES ('e2e-user', 'E2E', 'e2e@purdue.edu') + ON CONFLICT (id) DO NOTHING; + INSERT INTO projects (id, name, access_mode) VALUES ('E2E-PROJECT', 'E2E', 'private') + ON CONFLICT (id) DO NOTHING; + INSERT INTO project_members (project_id, nanohub_user_id, role) + VALUES ('E2E-PROJECT', 'e2e-user', 'pi') + ON CONFLICT (project_id, nanohub_user_id) DO NOTHING; + """ + ) + conn.commit() + finally: + conn.close() + ensure_tables() + yield + + +def test_pilot_budget_and_claim_renewal(client, monkeypatch): + from computations.store import claim_run, renew_run_claim + monkeypatch.setenv("RCAC_ALLOWED_PROJECT_IDS", "E2E-PROJECT") + monkeypatch.setenv("RCAC_MAX_ACTIVE_RUNS_PER_USER", "1") + monkeypatch.setenv("RCAC_MAX_ACTIVE_RUNS_PER_CLUSTER", "1") + payload = {"cluster": "fake", "inputs": {"mp_id": "mp-1265", "structure_dict": {}}} + first = client.post("/api/computations/projects/E2E-PROJECT/runs/cellrelaxdft", json=payload) + assert first.status_code == 202, first.text + run_id = first.json()["id"] + second = client.post("/api/computations/projects/E2E-PROJECT/runs/cellrelaxdft", json=payload) + assert second.status_code == 429, second.text + assert second.headers["Retry-After"] == "30" + assert claim_run(run_id, "owner", lease_seconds=900) + assert renew_run_claim(run_id, "different-worker", 900) is None + assert client.post(f"/api/computations/runs/{run_id}/cancel").status_code == 202 + fresh = renew_run_claim(run_id, "owner", 900) + assert fresh["cancel_requested"] and fresh["status"] == "cancelling" + worker = ComputationWorker("owner", gateway=FakeGateway()) + assert worker.run_once(run_id) + assert client.get(f"/api/computations/runs/{run_id}").json()["status"] == "cancelled" + + +@pytest.fixture +def client(configured_database): + app = FastAPI() + app.include_router(computation_router.router, prefix="/api") + app.dependency_overrides[get_platform_user] = lambda: PlatformUser( + id="e2e-user", + email="e2e@purdue.edu", + name="E2E", + role="researcher", + organization="Purdue University", + ) + return TestClient(app) + + +def test_api_database_worker_and_cache_lifecycle(client): + payload = { + "cluster": "fake", + "inputs": {"mp_id": "mp-1265", "structure_dict": {}}, + } + submitted = client.post( + "/api/computations/projects/E2E-PROJECT/runs/cellrelaxdft", json=payload + ) + assert submitted.status_code == 202, submitted.text + run_id = submitted.json()["id"] + + def prepare(_inputs, workdir, _pseudo_root): + input_path = workdir / "vcrelax.in" + input_path.write_text("QE input", encoding="utf-8") + return PreparedQeRun( + tool_name="cellrelaxdft", + local_workdir=workdir, + stage_files=[input_path], + expected_artifacts=["vcrelax.stdout"], + pseudopotentials=[], + ntasks=1, + metadata={"num_atoms": 1}, + ) + + with ( + tempfile.TemporaryDirectory() as directory, + patch( + "computations.worker.resolve_material_inputs", + side_effect=lambda value: {**value, "structure_dict": {"resolved": True}}, + ), + patch("computations.worker.prepare_cellrelax", side_effect=prepare), + patch( + "computations.worker.cellrelax_outputs_from_parsed", + return_value={"convergence": True, "energy": -10.5}, + ), + ): + worker = ComputationWorker("postgres-e2e", gateway=FakeGateway()) + worker.work_root = Path(directory) + assert worker.run_once() + assert worker.run_once() + assert worker.run_once() + + detail = client.get(f"/api/computations/runs/{run_id}") + assert detail.status_code == 200, detail.text + body = detail.json() + assert body["status"] == "data_ingested" + assert body["output"]["energy"] == -10.5 + assert body["execution_identity"] == "dtservice" + assert body["slurm_comment"] == f"dt-run:{run_id}" + assert body["artifact_manifest"][0]["uri"] == "" + + cached = client.post( + "/api/computations/projects/E2E-PROJECT/runs/cellrelaxdft", json=payload + ) + assert cached.status_code == 202, cached.text + assert worker.run_once() + cached_detail = client.get( + f"/api/computations/runs/{cached.json()['id']}" + ).json() + assert cached_detail["status"] == "data_ingested" + assert cached_detail["external_workflow_ref"]["cache_hit"] is True + assert cached_detail["external_workflow_ref"]["qe_input_adapter_version"] == 2 + + # Reusing again points at the run that executed, not at the reuse, + # so artifact lineage stays one hop. + again = client.post( + "/api/computations/projects/E2E-PROJECT/runs/cellrelaxdft", json=payload + ) + assert again.status_code == 202, again.text + assert worker.run_once() + again_ref = client.get(f"/api/computations/runs/{again.json()['id']}").json()["external_workflow_ref"] + assert again_ref["cache_hit"] is True and again_ref["source_run_id"] == run_id + + # A result-retrieval retry must preserve the scientific job and never + # turn a transport error into a fresh Slurm submission. + conn = _connection() + with conn, conn.cursor() as cur: + cur.execute("UPDATE computation_runs SET status='failed', error_json=%s::jsonb WHERE id=%s", + (json.dumps({'type':'GlobusTransientError','message':'lookup failed'}),run_id)) + conn.close() + worker.gateway.actions.clear() + retry = client.post(f"/api/computations/runs/{run_id}/retry-collection") + assert retry.status_code == 202, retry.text + assert retry.json()['scheduler_job_id'] == '9001' + assert worker.run_once(run_id) + assert worker.run_once(run_id) + assert worker.gateway.actions == ['status_many','collect'] + recovered = client.get(f"/api/computations/runs/{run_id}").json() + assert recovered['status'] == 'data_ingested' and not recovered['error'] + assert client.post(f"/api/computations/runs/{run_id}/retry-collection").status_code == 409 + + from computations.store import get_run, find_cache_hit + conn = _connection() + with conn, conn.cursor() as cur: + cur.execute("UPDATE computation_runs SET output_json=jsonb_set(output_json,'{convergence}','false') WHERE project_id='E2E-PROJECT'") + conn.close() + assert find_cache_hit(get_run(run_id, internal=True)) is None + + conn = _connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + SELECT equipment_type, asset_kind FROM equipment_metadata + WHERE domain_id = 'digital_qe_7_1_fake' + """ + ) + assert cur.fetchone() == ("Computational Software", "computational") + # A fully migrated database (scripts/add_asset_kinds.sql) also + # classifies the template and lists the run as computational. + cur.execute("SELECT to_regclass('public.all_runs') IS NOT NULL") + if cur.fetchone()[0]: + cur.execute( + "SELECT execution_kind FROM experiment_types " + "WHERE id = 'computation_cellrelaxdft_r99_dt2'" + ) + assert cur.fetchone() == ("computational",) + cur.execute( + "SELECT DISTINCT run_kind, experiment_type_id FROM all_runs " + "WHERE project_id = 'E2E-PROJECT'" + ) + assert cur.fetchall() == [("computational", "computation_cellrelaxdft_r99_dt2")] + cur.execute( + """ + SELECT equipment_id FROM computation_experiment_installations + WHERE experiment_type_id = 'computation_cellrelaxdft_r99_dt2' + """ + ) + assert cur.fetchone() == ("digital_qe_7_1_fake",) + finally: + conn.close() + + +def test_fractional_parent_survives_jsonb_normalization_and_cache(client): + from computations.schemas import compute_input_fingerprint + from pymatgen.core import Lattice, Structure + + structure = Structure(Lattice.cubic(5.43), ["Si"], [[0, 0, 0]]).as_dict() + structure["lattice"]["matrix"][0][1] = -0.0 + payload = {"cluster": "fake", "inputs": {"mp_id": "mp-149", "KE_cutoff": 50.5}} + url = "/api/computations/projects/E2E-PROJECT/runs/" + submitted = client.post(url + "cellrelaxdft", json=payload) + assert submitted.status_code == 202, submitted.text + run_id = submitted.json()["id"] + with ( + tempfile.TemporaryDirectory() as directory, + patch("computations.worker.resolve_material_inputs", side_effect=lambda inputs: {**inputs, "structure_dict": structure}), + patch("computations.worker.cellrelax_outputs_from_parsed", return_value={"convergence": True, "energy": -10.5}), + ): + worker = ComputationWorker("fractional-parent", gateway=FakeGateway()) + worker.work_root = Path(directory) + assert worker.run_once(run_id) + assert "ecutwfc = 50.5" in (Path(directory) / run_id / "vcrelax.in").read_text() + assert worker.run_once(run_id) + assert worker.run_once(run_id) + parent = client.get(f"/api/computations/runs/{run_id}").json() + assert parent["status"] == "data_ingested" + assert parent["input_fingerprint"] != compute_input_fingerprint("cellrelaxdft", parent["input"]) + assert parent["external_workflow_ref"]["qe_input_adapter_version"] == 2 + assert parent["elastic_parent_eligible"] is True + + cached = client.post(url + "cellrelaxdft", json=payload) + assert cached.status_code == 202, cached.text + cached_id = cached.json()["id"] + assert worker.run_once(cached_id) + cached_parent = client.get(f"/api/computations/runs/{cached_id}").json() + assert cached_parent["external_workflow_ref"]["cache_hit"] is True + assert cached_parent["external_workflow_ref"]["qe_input_adapter_version"] == 2 + assert cached_parent["elastic_parent_eligible"] is True + + for parent_id in [run_id, cached_id]: + elastic = client.post(url + "elasticdft", json={"cluster": "fake", "inputs": {"mp_id": "mp-149", "upstream_execution_id": parent_id}}) + assert elastic.status_code == 202, elastic.text + client.post(f"/api/computations/runs/{elastic.json()['id']}/cancel") + + # Without either trusted adapter provenance or a matching corrected + # fingerprint, historical fractional runs must remain ineligible. + conn = _connection() + with conn, conn.cursor() as cur: + cur.execute("UPDATE computation_runs SET external_workflow_ref_json='{}'::jsonb WHERE id=%s", (run_id,)) + conn.close() + historical = client.get(f"/api/computations/runs/{run_id}").json() + assert historical["elastic_parent_eligible"] is False + assert "rerun CellRelaxDFT" in historical["elastic_parent_ineligibility_reason"] + rejected = client.post(url + "elasticdft", json={"cluster": "fake", "inputs": {"mp_id": "mp-149", "upstream_execution_id": run_id}}) + assert rejected.status_code == 400 + assert "rerun CellRelaxDFT" in rejected.json()["detail"] + + +# --- resources and diagnostics persistence --------------------------------------- +# Distinct mp_id values and resolved structures keep these runs out of the +# cache lineage of the tests above. + +RELAX_URL = "/api/computations/projects/E2E-PROJECT/runs/cellrelaxdft" + + +class RecordingGateway(FakeGateway): + def __init__(self): + super().__init__() + self.payloads = {} + + def invoke(self, **kwargs): + self.payloads[kwargs["action"]] = kwargs["payload"] + return super().invoke(**kwargs) + + +class FailedJobGateway(RecordingGateway): + """Slurm reports exit 3:0; the archive holds the run's real-format evidence.""" + + def __init__(self, archive_root): + super().__init__() + self.archive_root = archive_root + self.manifest = [] + + def invoke(self, **kwargs): + action, payload = kwargs["action"], kwargs["payload"] + self.payloads[action] = payload + if action == "status_many": + job_id = payload["jobs"][0]["job_id"] + return "task-status", {"states": {job_id: ["FAILED"]}, "accounting": {job_id: [ + {"job_id": job_id, "state": "FAILED", "exit_code": "3:0", "started_at": None, "ended_at": None}]}} + if action == "collect": + return "task-collect", {"parsed_outputs": {}, "artifact_manifest": self.manifest, + "missing_artifacts": [], "collection_errors": []} + if action == "archive_evidence": + with patch.dict(os.environ, {"DT_CLUSTER_NAME": "fake", "DT_ARTIFACT_ROOT": str(self.archive_root)}): + return "task-archive", rcac_service_gateway({"action": action, "payload": payload}) + return FakeGateway.invoke(self, **kwargs) + + +def _prepare_relaxation(_inputs, workdir, _pseudo_root): + input_path = workdir / "vcrelax.in" + input_path.write_text("QE input", encoding="utf-8") + return PreparedQeRun(tool_name="cellrelaxdft", local_workdir=workdir, stage_files=[input_path], + expected_artifacts=["vcrelax.stdout"], pseudopotentials=[], ntasks=1, + metadata={"num_atoms": 1}) + + +def _snapshot(client, run_id): + response = client.get(f"/api/computations/runs/{run_id}") + assert response.status_code == 200, response.text + return response.json() + + +def _worker_patches(marker): + return ( + patch("computations.worker.resolve_material_inputs", + side_effect=lambda value: {**value, "structure_dict": {"resolved": marker}}), + patch("computations.worker.prepare_cellrelax", side_effect=_prepare_relaxation), + patch("computations.worker.cellrelax_outputs_from_parsed", + return_value={"convergence": True, "energy": -7.0}), + ) + + +def test_explicit_resources_survive_storage_submission_and_cache_reuse(client): + inputs = {"mp_id": "mp-2534", "structure_dict": {}} + first = client.post(RELAX_URL, json={"cluster": "fake", "inputs": inputs, "resources": {"ntasks": 16}}) + assert first.status_code == 202, first.text + run_id = first.json()["id"] + stored = _snapshot(client, run_id) + assert stored["resource_request"]["requested"] == {"ntasks": 16} + assert stored["resource_request"]["resolved"]["ntasks"] == 16 + assert "ntasks" not in stored["input"] + gateway = RecordingGateway() + resolve, prepare, outputs = _worker_patches("mp-2534") + with tempfile.TemporaryDirectory() as directory, resolve, prepare, outputs: + worker = ComputationWorker("resources-e2e", gateway=gateway) + worker.work_root = Path(directory) + assert worker.run_once(run_id) + spec = gateway.payloads["submit"]["spec"] + assert (spec["ntasks"], spec["nodes"], spec["time_limit"]) == (16, 1, "24:00:00") + submitted = _snapshot(client, run_id) + assert submitted["prepared_manifest"]["resources"]["ntasks"] == 16 + assert submitted["external_workflow_ref"]["submitted_resources"]["ntasks"] == 16 + assert submitted["external_workflow_ref"]["resource_confirmation"] == "protocol_1_fixed_contract" + assert worker.run_once(run_id) and worker.run_once(run_id) + source = _snapshot(client, run_id) + assert source["status"] == "data_ingested" + + reused = client.post(RELAX_URL, json={"cluster": "fake", "inputs": inputs, "resources": {"ntasks": 8}}) + assert reused.status_code == 202, reused.text + reused_id = reused.json()["id"] + gateway.payloads.clear() + assert worker.run_once(reused_id) + body = _snapshot(client, reused_id) + external = body["external_workflow_ref"] + assert body["status"] == "data_ingested" and not body["scheduler_job_id"] + assert external["cache_hit"] is True and external["source_run_id"] == run_id + assert external["resources_used"] is False + # No allocation use is attributed to a request that ran no job. + assert "submitted_resources" not in external and "scheduler_resource_usage" not in external + assert body["resource_request"]["requested"] == {"ntasks": 8} + assert body["input_fingerprint"] == source["input_fingerprint"] + assert gateway.payloads == {} + + +def test_archive_enrichment_persists_only_diagnostics(client, monkeypatch): + from computations.store import get_run + from test_computation_diagnostics import BFGS_FAILED_OUTPUT + + created = client.post(RELAX_URL, json={"cluster": "fake", "inputs": {"mp_id": "mp-2999", "structure_dict": {}}}) + assert created.status_code == 202, created.text + run_id = created.json()["id"] + clusters = json.loads(os.environ["RCAC_CLUSTERS_JSON"]) + resolve, prepare, outputs = _worker_patches("mp-2999") + with tempfile.TemporaryDirectory() as directory, resolve, prepare, outputs: + root = Path(directory).resolve() + gateway = FailedJobGateway(root / "archive") + for name, data in {"vcrelax.stdout": BFGS_FAILED_OUTPUT.encode(), + "slurm.err": b"STOP 3\nsrun: error: n1: task 0: Exited with exit code 3\n"}.items(): + path = root / "archive" / run_id / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + gateway.manifest.append({ + "name": name, "storage_uri": f"depot://{clusters['fake']['artifact_root']}/{run_id}/{name}", + "sha256": hashlib.sha256(data).hexdigest(), "size_bytes": len(data), + "content_type": "text/plain", "downloadable": True, + }) + worker = ComputationWorker("enrichment-e2e", gateway=gateway) + worker.work_root = root / "work" + worker.work_root.mkdir() + assert worker.run_once(run_id) # submit + assert worker.run_once(run_id) # Slurm FAILED 3:0 -> collecting + assert worker.run_once(run_id) # collect (protocol 1: no file evidence) + before = _snapshot(client, run_id) + assert before["status"] == "failed" + assert before["diagnostics"]["category_counts"] == {"nonzero_exit": 1} + + clusters["fake"]["gateway_protocol"] = 2 + monkeypatch.setenv("RCAC_CLUSTERS_JSON", json.dumps(clusters)) + plan = plan_enrichment(get_run(run_id, internal=True), get_cluster("fake"), gateway) + assert plan["diagnostics"]["category_counts"] == {"qe_bfgs_failed": 1} + assert _snapshot(client, run_id) == before # preview writes nothing + assert apply_plan(plan) == "updated" + + first, second = _snapshot(client, run_id), _snapshot(client, run_id) + exported = client.get(f"/api/computations/runs/{run_id}/diagnostics") + assert exported.status_code == 200, exported.text + assert first["diagnostics"] == second["diagnostics"] == exported.json()["diagnostics"] + assert first["diagnostics"]["category_counts"] == {"qe_bfgs_failed": 1} + assert first["diagnostics"]["enrichment"]["previous_error"] == before["error"] + for key in ("status", "input", "output", "input_fingerprint", "artifact_manifest", "scheduler_job_id", + "prepared_manifest", "resource_request", "submitted_at", "started_at", "completed_at", + "updated_at", "upstream_run_id", "requested_by"): + assert first[key] == before[key], key + # Provenance other than the (separately exposed) diagnostics is unchanged. + assert first["external_workflow_ref"] == before["external_workflow_ref"] + assert first["error"]["type"] == before["error"]["type"] + assert "BFGS stopped before meeting the requested tolerances" in first["error"]["message"] + + again = plan_enrichment(get_run(run_id, internal=True), get_cluster("fake"), gateway) + assert again["already_enriched"] is True and apply_plan(again) == "unchanged" + conn = _connection() + with conn, conn.cursor() as cur: + cur.execute("SELECT count(*) FROM computation_run_events WHERE run_id=%s AND message=%s", + (run_id, ENRICHMENT_EVENT)) + assert cur.fetchone()[0] == 1 + conn.close() diff --git a/api/tests/test_computations_router.py b/api/tests/test_computations_router.py new file mode 100644 index 0000000..05b8514 --- /dev/null +++ b/api/tests/test_computations_router.py @@ -0,0 +1,155 @@ +import json +import os +import unittest +from unittest.mock import patch + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from routers import computations +from security import PlatformUser, get_platform_user + + +ENDPOINT_ID = "11111111-1111-4111-8111-111111111111" +FUNCTION_ID = "22222222-2222-4222-8222-222222222222" +CLUSTERS = json.dumps( + { + "negishi": { + "host": "negishi.rcac.purdue.edu", + "endpoint_id": ENDPOINT_ID, + "gateway_function_id": FUNCTION_ID, + "qe_version": "7.1", + "module_dependencies": ["gcc/12.2.0", "openmpi/4.1.4"], + "qe_module": "quantum-espresso/7.1", + "mpi_launcher": "srun", + "remote_root": "/scratch/negishi/dtservice/digital-twins/runs", + "pseudo_root": "/depot/ncn/data/digital-twins/pseudopotentials", + "artifact_root": "/depot/ncn/data/digital-twins/runs", + } + } +) + + +class ComputationRouterSafetyTests(unittest.TestCase): + @staticmethod + def client(): + app = FastAPI() + app.include_router(computations.router, prefix="/api") + app.dependency_overrides[get_platform_user] = lambda: PlatformUser( + id="test-user", + email="test@purdue.edu", + name="Test User", + role="researcher", + organization="Purdue University", + ) + return TestClient(app) + + def test_feature_disabled_returns_not_found_without_database_access(self): + with patch("routers.computations.feature_enabled", return_value=False): + response = self.client().get("/api/computations/tools") + self.assertEqual(response.status_code, 404) + self.assertEqual(response.json()["detail"], "RCAC computations are not enabled") + + def test_catalog_exposes_runtime_metadata_but_not_endpoint_credentials(self): + with ( + patch("routers.computations.feature_enabled", return_value=True), + patch.dict(os.environ, {"RCAC_CLUSTERS_JSON": CLUSTERS}, clear=False), + ): + response = self.client().get("/api/computations/tools") + self.assertEqual(response.status_code, 200) + cluster = response.json()["clusters"][0] + self.assertEqual(cluster["qe_version"], "7.1") + self.assertEqual(cluster["execution_backend"], "globus_compute_service_account") + self.assertNotIn("endpoint_id", cluster) + self.assertNotIn(ENDPOINT_ID, response.text) + + def test_submit_uses_fixed_cluster_and_authenticated_platform_user(self): + created = { + "id": "33333333-3333-4333-8333-333333333333", + "status": "requested", + } + with ( + patch("routers.computations.feature_enabled", return_value=True), + patch.dict( + os.environ, + { + "RCAC_CLUSTERS_JSON": CLUSTERS, + "RCAC_SERVICE_ACCOUNT_USERNAME": "dtservice", + }, + clear=False, + ), + patch( + "routers.computations.validate_tool_inputs", + return_value={"mp_id": "mp-149", "structure_dict": {}}, + ), + patch( + "routers.computations.compute_input_fingerprint", + return_value="fingerprint", + ), + patch("routers.computations.create_run", return_value=created) as create, + ): + response = self.client().post( + "/api/computations/projects/P-1/runs/cellrelaxdft", + json={ + "cluster": "negishi", + "inputs": {"mp_id": "mp-149", "structure_dict": {}}, + }, + ) + self.assertEqual(response.status_code, 202) + self.assertEqual(response.json(), created) + kwargs = create.call_args.kwargs + self.assertEqual(kwargs["user"].id, "test-user") + self.assertEqual(kwargs["execution_identity"], "dtservice") + self.assertEqual(kwargs["cluster"].allocation, "ncn") + self.assertEqual(kwargs["cluster"].partition, "cpu") + + def test_per_student_ssh_credential_routes_do_not_exist(self): + with patch("routers.computations.feature_enabled", return_value=True): + response = self.client().post( + "/api/computations/profiles/negishi/credential" + ) + self.assertEqual(response.status_code, 404) + + def test_export_checks_project_access_and_feature_flag(self): + with ( + patch("routers.computations.feature_enabled", return_value=True), + patch("routers.computations.get_run", side_effect=PermissionError("Access denied")), + ): + self.assertEqual(self.client().get("/api/computations/runs/123/export").status_code, 403) + with ( + patch("routers.computations.feature_enabled", return_value=False), + patch("routers.computations.get_run") as read, + ): + self.assertEqual(self.client().get("/api/computations/runs/123/export").status_code, 404) + read.assert_not_called() + + def test_export_is_portable_json_from_authorized_public_run(self): + run = {"id":"123", "inputs":{"mp_id":"mp-48"}, "output":{"convergence":True}} + with ( + patch("routers.computations.feature_enabled", return_value=True), + patch("routers.computations.get_run", return_value=run) as read, + ): + response = self.client().get("/api/computations/runs/123/export") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), {"schema":"dt-computation-export/1", "run":run}) + self.assertIn('attachment;', response.headers['content-disposition']) + self.assertEqual(read.call_args.args[1].id, "test-user") + + def test_retry_propagates_authenticated_user_and_rejects_ineligible_runs(self): + with ( + patch("routers.computations.feature_enabled", return_value=True), + patch("routers.computations.retry_collection", return_value={"scheduler_job_id":"42"}) as retry, + ): + response = self.client().post("/api/computations/runs/123/retry-collection") + self.assertEqual(response.status_code, 202) + self.assertEqual(response.json()['scheduler_job_id'], '42') + self.assertEqual(retry.call_args.args[1].id, "test-user") + for error, expected in [(PermissionError('Access denied'),403), (ValueError('Not retryable'),409)]: + with ( + patch("routers.computations.feature_enabled", return_value=True), + patch("routers.computations.retry_collection", side_effect=error), + ): + self.assertEqual(self.client().post("/api/computations/runs/123/retry-collection").status_code, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_experiment_run_endpoint.py b/api/tests/test_experiment_run_endpoint.py new file mode 100644 index 0000000..e53bad9 --- /dev/null +++ b/api/tests/test_experiment_run_endpoint.py @@ -0,0 +1,173 @@ +"""Endpoint regression tests for generic project-run creation.""" + +import os +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +os.environ.setdefault("DT_SYSTEM_TOKEN", "test-system-token") +os.environ.setdefault("INGESTION_TOKEN", "test-ingestion-token") +os.environ.setdefault("PG_PASS", "test-api-password") +os.environ.setdefault("PG_SUPER_PASS", "test-super-password") + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +from fastapi import FastAPI # noqa: E402 +from fastapi.testclient import TestClient # noqa: E402 + +from metadata_pg import RegisteredComputationTemplateError # noqa: E402 +from routers import equipment # noqa: E402 +from security import PlatformUser, get_platform_user # noqa: E402 + +USER = PlatformUser(id="owner-1", email="o@example.org", name="Owner", + role="researcher", organization="Birck") +APPROVED = {"domain_id": "etcher_01", "owner_id": "owner-1", + "owner_org": "Birck", "status": "approved"} +PROJECTS = [{"id": "project-1", "name": "Project"}] + + +def _payload(**overrides): + payload = { + "experiment_name": "Run", + "selected_equipment": "etcher_01", + "project_id": "project-1", + "type_id": "recipe_sweep", + "planned_parameters": [], + "optimization_context": {"mode": "none"}, + } + payload.update(overrides) + return payload + + +class ExperimentRunEndpointTests(unittest.TestCase): + def setUp(self) -> None: + app = FastAPI() + app.include_router(equipment.router) + app.dependency_overrides[get_platform_user] = lambda: USER + self.client = TestClient(app) + + def test_computation_template_on_computational_equipment_is_rejected(self): + # The router decides from the equipment's asset_kind, not the id; a + # template's own execution_kind is checked in the database layer. + computational = {**APPROVED, "asset_kind": "computational"} + with patch.object(equipment, "get_equipment_pg", return_value=computational), patch.object( + equipment, "create_experiment_definition_pg" + ) as create: + response = self.client.post( + "/equipment/experiment", + json=_payload( + type_id="computation_cellrelaxdft_r99_dt2", + planned_parameters=[{"name": "mp_id", "planned_value": "mp-149", "unit": ""}], + ), + ) + self.assertEqual(response.status_code, 409) + self.assertIn("Registered computational experiments", response.json()["detail"]) + create.assert_not_called() + + def test_explicitly_physical_template_is_not_treated_as_computational(self): + from metadata_pg import registered_computation_tool_for_template + + self.assertIsNone( + registered_computation_tool_for_template("computation_calibration_1a2b3c4d", [], "physical") + ) + self.assertEqual( + registered_computation_tool_for_template("computation_elasticdft_r27_dt2", [], "computational"), + "elasticdft", + ) + + def test_json_choices_compare_as_json(self): + from metadata_pg import _normalize_planned_value + + param = {"allowed_values": [{"mode": "safe"}, [1, 2]]} + self.assertEqual( + _normalize_planned_value("Settings", '{ "mode": "safe" }', "object", param), + '{"mode":"safe"}', + ) + self.assertEqual(_normalize_planned_value("Layers", "[1,2]", "array", param), "[1,2]") + with self.assertRaises(ValueError): + _normalize_planned_value("Settings", '{"mode": "fast"}', "object", param) + + numeric = {"allowed_values": [{"cutoff": 50.0}, [1.0, 2.0], {"on": True}]} + self.assertEqual(_normalize_planned_value("C", '{"cutoff": 50}', "object", numeric), '{"cutoff":50}') + self.assertEqual(_normalize_planned_value("L", "[1, 2]", "array", numeric), "[1,2]") + with self.assertRaises(ValueError): + _normalize_planned_value("O", '{"on": 1}', "object", numeric) + seeds = {"allowed_values": [{"seed": 9007199254740992}]} + with self.assertRaises(ValueError): + _normalize_planned_value("S", '{"seed": 9007199254740993}', "object", seeds) + + def test_user_template_ids_never_use_the_catalog_prefix(self): + from metadata_pg import _user_template_slug + + self.assertTrue(_user_template_slug("Computation calibration").startswith("unit_computation")) + self.assertEqual(_user_template_slug("Furnace anneal"), "furnace_anneal") + + def test_metadata_detected_computation_template_maps_to_409(self): + with patch.object(equipment, "get_equipment_pg", return_value=APPROVED), patch.object( + equipment, "get_projects_list_pg", return_value=PROJECTS + ), patch.object( + equipment, + "create_experiment_definition_pg", + side_effect=RegisteredComputationTemplateError(""), + ): + response = self.client.post( + "/equipment/experiment", + json=_payload( + planned_parameters=[{"name": "KE_cutoff", "planned_value": "50", "unit": ""}], + ), + ) + self.assertEqual(response.status_code, 409) + + def test_all_variable_optimization_needs_no_point_value(self): + with patch.object(equipment, "get_equipment_pg", return_value=APPROVED), patch.object( + equipment, "get_projects_list_pg", return_value=PROJECTS + ), patch.object( + equipment, + "create_experiment_definition_pg", + return_value={"status": "created", "experiment_id": ""}, + ) as create: + response = self.client.post( + "/equipment/experiment", + json=_payload( + optimization_context={ + "mode": "optimization", + "primary_target": "AvgEtchRate", + "primary_objective": "maximize", + }, + ), + ) + self.assertEqual(response.status_code, 200) + create.assert_called_once() + + def test_ordinary_run_uses_definition_required_rules(self): + with patch.object(equipment, "get_equipment_pg", return_value=APPROVED), patch.object( + equipment, "get_projects_list_pg", return_value=PROJECTS + ), patch.object(equipment, "create_experiment_definition_pg", + return_value={"status": "created", "experiment_id": ""}) as create: + response = self.client.post("/equipment/experiment", json=_payload()) + self.assertEqual(response.status_code, 200) + create.assert_called_once() + + def test_validation_error_detail_is_returned(self): + with patch.object(equipment, "get_equipment_pg", return_value=APPROVED), patch.object( + equipment, "get_projects_list_pg", return_value=PROJECTS + ), patch.object( + equipment, + "create_experiment_definition_pg", + side_effect=ValueError("Planned parameter 'k_x' must be a whole number"), + ): + response = self.client.post( + "/equipment/experiment", + json=_payload( + planned_parameters=[{"name": "k_x", "planned_value": "2.5", "unit": ""}], + ), + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()["detail"], "Planned parameter 'k_x' must be a whole number") + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_experiment_run_validation.py b/api/tests/test_experiment_run_validation.py index 07b754c..24aa171 100644 --- a/api/tests/test_experiment_run_validation.py +++ b/api/tests/test_experiment_run_validation.py @@ -2,63 +2,513 @@ import unittest from pathlib import Path +from psycopg2.extras import Json API_DIR = Path(__file__).resolve().parents[1] if str(API_DIR) not in sys.path: sys.path.insert(0, str(API_DIR)) -from metadata_pg import _validate_planned_parameters_against_experiment_inputs +from unittest.mock import patch # noqa: E402 +import metadata_pg # noqa: E402 +from metadata_pg import ( # noqa: E402 + ETCHER_OPTIMIZER_FEATURES, + ETCHER_OPTIMIZER_TARGETS, + RegisteredComputationTemplateError, + UnsupportedOptimizationError, + _validate_planned_parameters_against_experiment_inputs as validate, + registered_computation_tool_for_template, + resolve_optimization_request, +) +from security import PlatformUser # noqa: E402 +from domain_configs import load_domain_config -class ExperimentRunValidationTests(unittest.TestCase): + +def _variable_inputs(): + return [{"name": f["id"], "mode": "variable", "min_value": str(f["min"]), "max_value": str(f["max"])} + for f in load_domain_config("etcher")["features"]] + + + +def _row(name, value, unit=""): + return {"name": name, "planned_value": value, "unit": unit} + + +class PlannedValueValidationTests(unittest.TestCase): def setUp(self) -> None: self.inputs = [ - { - "name": "O2_Flow", - "mode": "fixed", - "min_value": "5", - "max_value": "20", - }, - { - "name": "CF4_Flow", - "mode": "variable", - "min_value": "40", - "max_value": "120", - }, - { - "name": "Unused_Input", - "mode": "not_used", - "min_value": "0", - "max_value": "10", - }, + {"name": "O2_Flow", "type": "float", "unit": "sccm", "mode": "fixed", + "min_value": "5", "max_value": "20"}, + {"name": "CF4_Flow", "type": "float", "unit": "sccm", "mode": "variable", + "min_value": "40", "max_value": "120"}, + {"name": "Unused_Input", "mode": "not_used", "min_value": "0", "max_value": "10"}, ] + self.ordinary = [_row("O2_Flow", "12"), _row("CF4_Flow", "60")] - def test_accepts_selected_input_inside_range(self) -> None: - _validate_planned_parameters_against_experiment_inputs( - [{"name": "O2_Flow", "planned_value": "12", "unit": "sccm"}], - self.inputs, + def test_accepts_values_and_fills_definition_units(self) -> None: + rows = validate(self.ordinary, self.inputs) + self.assertEqual( + rows, + [_row("O2_Flow", "12", "sccm"), _row("CF4_Flow", "60", "sccm")], ) def test_rejects_unselected_input(self) -> None: with self.assertRaisesRegex(ValueError, "not a selected input"): - _validate_planned_parameters_against_experiment_inputs( - [{"name": "Unused_Input", "planned_value": "3"}], - self.inputs, - ) + validate([*self.ordinary, _row("Unused_Input", "3")], self.inputs) def test_rejects_value_outside_range(self) -> None: with self.assertRaisesRegex(ValueError, "less than or equal to 20"): - _validate_planned_parameters_against_experiment_inputs( - [{"name": "O2_Flow", "planned_value": "25"}], - self.inputs, - ) + validate([_row("O2_Flow", "25"), _row("CF4_Flow", "60")], self.inputs) def test_rejects_non_numeric_value(self) -> None: with self.assertRaisesRegex(ValueError, "must be numeric"): - _validate_planned_parameters_against_experiment_inputs( - [{"name": "O2_Flow", "planned_value": "high"}], - self.inputs, + validate([_row("O2_Flow", "high"), _row("CF4_Flow", "60")], self.inputs) + + def test_rejects_nonfinite_values_even_with_ranges(self) -> None: + for value in ("NaN", "inf", "-Infinity", "1e999"): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "finite number"): + validate([_row("O2_Flow", value), _row("CF4_Flow", "60")], self.inputs) + + def test_rejects_python_only_numeric_spellings(self) -> None: + with self.assertRaisesRegex(ValueError, "must be numeric"): + validate([_row("O2_Flow", "1_0"), _row("CF4_Flow", "60")], self.inputs) + + def test_rejects_conflicting_unit(self) -> None: + with self.assertRaisesRegex(ValueError, "conflicts with the unit experiment definition"): + validate([_row("O2_Flow", "12", "slm"), _row("CF4_Flow", "60")], self.inputs) + + def test_rejects_unit_when_definition_has_none(self) -> None: + inputs = [{"name": "Count", "type": "integer", "mode": "fixed"}] + with self.assertRaisesRegex(ValueError, r"\(no unit\)"): + validate([_row("Count", "3", "pcs")], inputs) + + def test_ordinary_run_requires_every_selected_input(self) -> None: + with self.assertRaisesRegex(ValueError, "required for: CF4_Flow"): + validate([_row("O2_Flow", "12"), _row("CF4_Flow", " ")], self.inputs) + + def test_rejects_duplicates(self) -> None: + with self.assertRaisesRegex(ValueError, "duplicated"): + validate([*self.ordinary, _row("O2_Flow", "13")], self.inputs) + + def test_integer_type_rejects_fractions_and_normalizes(self) -> None: + inputs = [{"name": "k_x", "type": "integer", "mode": "fixed", "min_value": "1"}] + with self.assertRaisesRegex(ValueError, "whole number"): + validate([_row("k_x", "2.5")], inputs) + self.assertEqual(validate([_row("k_x", "8.0")], inputs)[0]["planned_value"], "8") + with self.assertRaisesRegex(ValueError, "whole number"): + validate([_row("k_x", "1.0000000000000000001")], inputs) + self.assertEqual(validate([_row("k_x", "100000000000000001")], inputs)[0]["planned_value"], "100000000000000001") + + def test_typed_boolean_string_datetime_json_values(self) -> None: + inputs = [ + {"name": "is_metal", "type": "boolean", "mode": "fixed"}, + {"name": "recipe", "type": "string", "mode": "fixed"}, + {"name": "started", "type": "datetime", "mode": "fixed"}, + {"name": "settings", "type": "map", "mode": "fixed"}, + {"name": "strains", "type": "array", "mode": "fixed"}, + ] + rows = validate( + [ + _row("is_metal", "TRUE"), + _row("recipe", "Recipe A"), + _row("started", "2026-09-29T10:00:00Z"), + _row("settings", '{"b": 1, "a": 2}'), + _row("strains", "[-0.01, 0.01]"), + ], + inputs, + ) + values = {row["name"]: row["planned_value"] for row in rows} + self.assertEqual(values["is_metal"], "true") + self.assertEqual(values["recipe"], "Recipe A") + self.assertEqual(values["settings"], '{"a":2,"b":1}') + self.assertEqual(values["strains"], "[-0.01,0.01]") + + bad_cases = [ + ("is_metal", "yes", "true or false"), + ("started", "yesterday", "ISO 8601"), + ("settings", "[1]", "JSON object"), + ("settings", '{"a": NaN}', "valid JSON"), + ("strains", "{}", "JSON array"), + ] + for name, value, message in bad_cases: + with self.subTest(name=name, value=value): + good = [row for row in rows if row["name"] != name] + with self.assertRaisesRegex(ValueError, message): + validate([*good, _row(name, value)], inputs) + + def test_allowed_values_are_enforced(self) -> None: + inputs = [{"name": "gas", "type": "string", "mode": "fixed", + "allowed_values": ["CF4", "SF6"]}] + self.assertEqual(validate([_row("gas", "SF6")], inputs)[0]["planned_value"], "SF6") + with self.assertRaisesRegex(ValueError, "one of: CF4, SF6"): + validate([_row("gas", "O2")], inputs) + + def test_unknown_declared_type_is_rejected(self) -> None: + inputs = [{"name": "blob", "type": "file", "mode": "fixed"}] + with self.assertRaisesRegex(ValueError, "unsupported value type 'file'"): + validate([_row("blob", "x")], inputs) + + def test_additional_inputs_are_accepted_typed_and_persisted(self) -> None: + additional = [ + {"name": "MaterialBatch", "type": "string", "unit": ""}, + {"name": "Thickness", "type": "float", "unit": "nm", "required": True}, + ] + rows = validate( + [*self.ordinary, _row("MaterialBatch", "B-42"), _row("Thickness", "12.5")], + self.inputs, + additional, + ) + self.assertIn(_row("MaterialBatch", "B-42", ""), rows) + self.assertIn(_row("Thickness", "12.5", "nm"), rows) + with self.assertRaisesRegex(ValueError, "must be numeric"): + validate([*self.ordinary, _row("Thickness", "thin")], self.inputs, additional) + + def test_required_additional_input_must_be_provided(self) -> None: + additional = [ + {"name": "MaterialBatch", "type": "string"}, + {"name": "Thickness", "type": "float", "unit": "nm", "required": True}, + ] + with self.assertRaisesRegex(ValueError, "required for: Thickness"): + validate(self.ordinary, self.inputs, additional) + # Optional additional inputs may be omitted. + rows = validate([*self.ordinary, _row("Thickness", "3")], self.inputs, additional) + self.assertNotIn("MaterialBatch", {row["name"] for row in rows}) + + def test_untyped_additional_input_defaults_to_string(self) -> None: + rows = validate([*self.ordinary, _row("Note", "operator A")], self.inputs, + [{"name": "Note"}]) + self.assertIn(_row("Note", "operator A"), rows) + + def test_optimization_needs_only_fixed_values_and_rejects_variable_points(self) -> None: + rows = validate([_row("O2_Flow", "12")], self.inputs, optimization=True) + self.assertEqual(rows, [_row("O2_Flow", "12", "sccm")]) + with self.assertRaisesRegex(ValueError, "Variable input"): + validate(self.ordinary, self.inputs, optimization=True) + with self.assertRaisesRegex(ValueError, "required for: O2_Flow"): + validate([], self.inputs, optimization=True) + + def test_all_variable_optimization_needs_no_point_value(self) -> None: + inputs = [{"name": "CF4_Flow", "mode": "variable", "min_value": "40", "max_value": "120"}] + self.assertEqual(validate([], inputs, optimization=True), []) + + def test_legacy_parameters_without_mode_use_is_variable(self) -> None: + inputs = [ + {"name": "A", "is_variable": True}, + {"name": "B", "is_variable": False}, + ] + self.assertEqual( + validate([_row("B", "1")], inputs, optimization=True), + [_row("B", "1")], + ) + + +class ComputationTemplateDetectionTests(unittest.TestCase): + def test_versioned_ids_map_to_tools(self) -> None: + self.assertEqual( + registered_computation_tool_for_template("computation_cellrelaxdft_r99_dt2"), + "cellrelaxdft", + ) + self.assertEqual( + registered_computation_tool_for_template("computation_cellrelaxdft2d_justin_dt1"), + "cellrelaxdft2d", + ) + self.assertEqual( + registered_computation_tool_for_template("computation_elasticdft_r27_dt2"), + "elasticdft", + ) + self.assertEqual(registered_computation_tool_for_template("computation_future_r1"), "") + + def test_source_metadata_marks_template_computational(self) -> None: + self.assertEqual( + registered_computation_tool_for_template( + "renamed-template", + [{"name": "mp_id", "source": "computational_workflow"}], + ), + "", + ) + self.assertIsNone( + registered_computation_tool_for_template( + "recipe_sweep", [{"name": "pressure", "source": "equipment"}] ) + ) + + def test_error_message_points_to_dedicated_form(self) -> None: + message = str(RegisteredComputationTemplateError("cellrelaxdft")) + self.assertIn("Registered computational experiments", message) + self.assertIn("tool: cellrelaxdft", message) + + +class _FakeCursor: + def __init__(self, type_row): + self.type_row = type_row + self.statements: list[tuple[str, object]] = [] + self._last = "" + + def execute(self, query, params=None): + self._last = str(query) + self.statements.append((self._last, params)) + + def fetchone(self): + if "FROM experiment_types" in self._last: + return self.type_row + if "INSERT INTO experiment_definitions" in self._last: + return {"id": "exp-1", "execution_request_id": "REQ-TEST"} + return None + + def fetchall(self): + return [] + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +class _FakeConnection: + def __init__(self, cursor): + self._cursor = cursor + self.committed = False + + def cursor(self, cursor_factory=None): + return self._cursor + + def commit(self): + self.committed = True + + def close(self): + pass + + +USER = PlatformUser(id="owner-1", name="Owner", email="o@example.org", + role="researcher", organization="Birck") + + +def _create(type_row, planned, context=None): + cursor = _FakeCursor(type_row) + connection = _FakeConnection(cursor) + with patch.object(metadata_pg, "get_pg_connection", return_value=connection), patch.object(metadata_pg, "get_equipment_pg", return_value=None): + result = metadata_pg.create_experiment_definition_pg( + payload={ + "experiment_name": "Run", + "selected_equipment": "etcher", + "project_id": "project-1", + "type_id": type_row["id"], + "planned_parameters": planned, + "optimization_context": context or {"mode": "none"}, + "sample_ids": [], + }, + user=USER, + ) + return result, cursor, connection + + +class CreateExperimentDefinitionTests(unittest.TestCase): + def _type_row(self, **overrides): + row = { + "id": "recipe_sweep", + "equipment_id": "etcher", + "type_parameters_json": [ + {"name": "Etch_AvgO2Flow", "type": "float", "unit": "sccm", "mode": "fixed"}, + {"name": "Etch_AvgPres", "type": "float", "unit": "mTorr", "mode": "variable", + "min_value": "10", "max_value": "60"}, + ], + "outputs_json": [{"name": "AvgEtchRate"}, {"name": "RangeEtchRate"}], + "additional_inputs_json": [{"name": "MaterialBatch", "type": "string"}], + } + row.update(overrides) + return row + + def test_persists_normalized_equipment_and_additional_values(self) -> None: + _result, cursor, connection = _create( + self._type_row(), + [ + _row("Etch_AvgO2Flow", " 40 "), + _row("Etch_AvgPres", "25", "mTorr"), + _row("MaterialBatch", "B-7"), + ], + ) + insert_params = next(p for q, p in cursor.statements + if "INSERT INTO experiment_definitions" in q) + planned = insert_params[6] + self.assertIsInstance(planned, Json) + self.assertEqual( + planned.adapted, + [ + _row("Etch_AvgO2Flow", "40", "sccm"), + _row("Etch_AvgPres", "25", "mTorr"), + _row("MaterialBatch", "B-7", ""), + ], + ) + self.assertTrue(connection.committed) + + def test_rejects_registered_computation_template_before_insert(self) -> None: + row = self._type_row( + id="computation_cellrelaxdft_r99_dt2", + type_parameters_json=[{"name": "mp_id", "type": "string", + "source": "computational_workflow"}], + ) + with self.assertRaises(RegisteredComputationTemplateError) as caught: + _create(row, [_row("mp_id", "mp-149")]) + self.assertEqual(caught.exception.tool_name, "cellrelaxdft") + + def test_rejects_template_identified_only_by_source_metadata(self) -> None: + row = self._type_row( + id="legacy-dft", + type_parameters_json=[{"name": "KE_cutoff", "type": "number", + "source": "computational_workflow"}], + ) + with self.assertRaises(RegisteredComputationTemplateError): + _create(row, [_row("KE_cutoff", "50")]) + + def test_optimization_validates_target_against_output_snapshot(self) -> None: + with self.assertRaisesRegex(UnsupportedOptimizationError, "not an output"): + _create( + self._type_row(outputs_json=[{"name": "AvgEtchRate"}]), + [_row("Etch_AvgO2Flow", "40")], + {"mode": "optimization", "primary_target": "RangeEtchRate", + "primary_objective": "minimize"}, + ) + + def test_all_variable_optimization_is_accepted(self) -> None: + row = self._type_row(type_parameters_json=_variable_inputs()) + result, _cursor, _connection = _create( + row, [], {"mode": "optimization", "primary_target": "AvgEtchRate", + "primary_objective": "maximize"}, + ) + self.assertEqual(result["experiment_id"], "exp-1") + + +class OptimizationRequestTests(unittest.TestCase): + def setUp(self) -> None: + self.equipment_patch = patch.object(metadata_pg, "get_equipment_pg", return_value=None) + self.equipment_patch.start() + self.addCleanup(self.equipment_patch.stop) + self.inputs = [ + {"name": "Etch_AvgO2Flow", "mode": "fixed", "min_value": "10", "max_value": "90"}, + {"name": "Etch_AvgPres", "mode": "variable", "min_value": "20", "max_value": "60"}, + *[p for p in _variable_inputs() if p["name"] not in {"Etch_AvgO2Flow", "Etch_AvgPres"}], + {"name": "Wafer_Lot", "mode": "fixed", "source": "sample", "type": "string"}, + ] + self.outputs = [{"name": "AvgEtchRate"}, {"name": "RangeEtchRate"}] + + def _resolve(self, context, planned=None, **kwargs): + return resolve_optimization_request( + equipment_id=kwargs.get("equipment_id", "etcher"), + input_snapshot=kwargs.get("inputs", self.inputs), + planned_parameters=planned if planned is not None else [ + _row("Etch_AvgO2Flow", "40"), + _row("Etch_AvgPres", "50"), # legacy row value for a Variable input + _row("Wafer_Lot", "L1"), + ], + optimization_context=context, + output_snapshot=kwargs.get("outputs", self.outputs), + strict=kwargs.get("strict", False), + ) + + def test_variable_keeps_range_and_only_fixed_is_pinned(self) -> None: + request = self._resolve({"primary_target": "AvgEtchRate", "primary_objective": "maximize", + "secondary_target": "RangeEtchRate", + "secondary_objective": "minimize"}) + self.assertEqual( + request["constraints"], + {"Etch_AvgO2Flow": [40.0, 40.0], "Etch_AvgPres": [20.0, 60.0], + "Etch_Avg_Rf1_Pow": [0.0, 100.0], "Etch_Avg_Rf2_Pow": [50.0, 700.0], + "Etch_Avgcf4Flow": [10.0, 90.0]}, + ) + self.assertNotIn("Wafer_Lot", request["constraints"]) + + def test_etcher_default_objectives_keep_established_path(self) -> None: + request = self._resolve({"primary_target": "AvgEtchRate", "primary_objective": "maximize", + "secondary_target": "RangeEtchRate", + "secondary_objective": "minimize"}) + self.assertIsNone(request["objectives"]) + self.assertEqual(request["domain_config"]["features"], list(ETCHER_OPTIMIZER_FEATURES)) + + def test_legacy_row_without_target_uses_etcher_default(self) -> None: + request = self._resolve({"mode": "optimization"}) + self.assertIsNone(request["objectives"]) + with self.assertRaisesRegex(UnsupportedOptimizationError, "Primary optimization target"): + self._resolve({"mode": "optimization"}, strict=True) + + def test_explicit_targets_and_directions_are_propagated(self) -> None: + request = self._resolve({"primary_target": "RangeEtchRate", + "primary_objective": "minimize"}) + self.assertEqual(request["objectives"], [{"target": "RangeEtchRate", + "direction": "minimize"}]) + self.assertEqual(request["domain_config"]["targets"], ["RangeEtchRate"]) + self.assertEqual(request["domain_config"]["features"], list(ETCHER_OPTIMIZER_FEATURES)) + + flipped = self._resolve({"primary_target": "AvgEtchRate", "primary_objective": "minimize"}) + self.assertEqual(flipped["objectives"], [{"target": "AvgEtchRate", + "direction": "minimize"}]) + + def test_unsupported_target_fails_clearly(self) -> None: + with self.assertRaisesRegex(UnsupportedOptimizationError, "not supported"): + self._resolve({"primary_target": "Thickness"}, + outputs=[{"name": "Thickness"}]) + + def test_unsupported_equipment_inputs_fail_clearly(self) -> None: + inputs = [{"name": "Anneal_Temp", "mode": "variable", "min_value": "300", + "max_value": "600"}] + with self.assertRaisesRegex(UnsupportedOptimizationError, "Anneal_Temp"): + self._resolve({"primary_target": "AvgEtchRate"}, planned=[], inputs=inputs) + + def test_invalid_direction_and_duplicate_target(self) -> None: + with self.assertRaisesRegex(UnsupportedOptimizationError, "maximize"): + self._resolve({"primary_target": "AvgEtchRate", "primary_objective": "up"}) + with self.assertRaisesRegex(UnsupportedOptimizationError, "differ"): + self._resolve({"primary_target": "AvgEtchRate", "secondary_target": "AvgEtchRate"}) + + def test_strict_mode_requires_fixed_values(self) -> None: + with self.assertRaisesRegex(UnsupportedOptimizationError, "Etch_AvgO2Flow"): + self._resolve({"primary_target": "AvgEtchRate"}, planned=[], strict=True) + + def test_targets_mirror_training_schema(self) -> None: + import data_loader + + self.assertEqual(list(ETCHER_OPTIMIZER_FEATURES), list(data_loader.FEATURES)) + self.assertEqual( + ETCHER_OPTIMIZER_TARGETS, + (data_loader.PRIMARY_TARGET, data_loader.SECONDARY_TARGET), + ) + + def test_unknown_equipment_cannot_reuse_familiar_etcher_names(self): + with self.assertRaisesRegex(UnsupportedOptimizationError, "not available"): + self._resolve({"primary_target": "AvgEtchRate"}, equipment_id="unrelated-device") + + def test_selected_registered_configuration_is_authoritative(self): + config = load_domain_config("etcher") + pressure = next(f for f in config["features"] if f["id"] == "Etch_AvgPres") + pressure["max"] = 45 + with patch.object(metadata_pg, "get_equipment_pg", return_value={"config_json": config}): + with self.assertRaisesRegex(UnsupportedOptimizationError, "equipment limits"): + self._resolve({"primary_target": "AvgEtchRate"}) + self.inputs[1]["max_value"] = "40" + request = self._resolve({"primary_target": "AvgEtchRate"}) + self.assertEqual(request["domain_config"]["feature_limits"]["Etch_AvgPres"], [4, 45]) + + def test_changed_config_default_direction_is_explicit(self): + config = load_domain_config("etcher") + config["targets"]["primary"]["objective"] = "minimize" + with patch.object(metadata_pg, "get_equipment_pg", return_value={"config_json": config}): + result = self._resolve({}) + self.assertEqual(result["objectives"][0], {"target": "AvgEtchRate", "direction": "minimize"}) + + def test_missing_outputs_and_not_used_features_reject(self): + with self.assertRaisesRegex(UnsupportedOptimizationError, "no outputs"): + self._resolve({"primary_target": "AvgEtchRate"}, outputs=[]) + self.inputs[1]["mode"] = "not_used" + with self.assertRaisesRegex(UnsupportedOptimizationError, "not selected: Etch_AvgPres"): + self._resolve({"primary_target": "AvgEtchRate"}) + + def test_invalid_ranges_reject_without_expanding_search(self): + for low, high in [("60", "20"), ("1e999", "60"), ("", "60"), ("20", "120")]: + with self.subTest(low=low, high=high): + self.inputs[1].update(min_value=low, max_value=high) + with self.assertRaises(UnsupportedOptimizationError): + self._resolve({"primary_target": "AvgEtchRate"}) if __name__ == "__main__": diff --git a/api/tests/test_experiment_values_postgres_e2e.py b/api/tests/test_experiment_values_postgres_e2e.py new file mode 100644 index 0000000..d22eff6 --- /dev/null +++ b/api/tests/test_experiment_values_postgres_e2e.py @@ -0,0 +1,194 @@ +"""Opt-in HTTP-to-Postgres test for typed per-run values. + +Run with DT_COMPUTATIONS_E2E=1 and PG_* set to a disposable database. +Each test uses its own schema. No production database or compute gateway is used. +""" + +import os +from contextlib import ExitStack +from uuid import uuid4 +from unittest.mock import patch + +import psycopg2 +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from psycopg2.extras import Json + +import metadata_pg +from routers import equipment +from security import PlatformUser, get_platform_user + +pytestmark = pytest.mark.skipif( + os.getenv("DT_COMPUTATIONS_E2E") != "1", + reason="requires an explicitly configured disposable PostgreSQL database", +) + + +@pytest.fixture() +def isolated_run_database(): + schema = "run_values_" + uuid4().hex + kwargs = dict(host=os.environ["PG_HOST"], port=os.environ["PG_PORT"], + dbname=os.environ["PG_DB"], user=os.environ["PG_SUPER_USER"], + password=os.environ["PG_SUPER_PASS"]) + admin = psycopg2.connect(**kwargs) + admin.autocommit = True + with admin.cursor() as cur: + cur.execute(f'CREATE SCHEMA "{schema}"') + + def connect(*_args, **_kwargs): + # Stands in for the user-scoped and worker connections alike. + return psycopg2.connect(**kwargs, options=f"-c search_path={schema},public") + + try: + conn = connect() + with conn, conn.cursor() as cur: + cur.execute(""" + CREATE TABLE experiment_types ( + id TEXT PRIMARY KEY, equipment_id TEXT, + type_parameters_json JSONB, outputs_json JSONB, + additional_inputs_json JSONB, owner_id TEXT, owner_org TEXT, + visibility TEXT DEFAULT 'shared', execution_kind TEXT DEFAULT 'physical' + ); + CREATE TABLE experiment_definitions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id TEXT, equipment_id TEXT, type_id TEXT, + execution_request_id TEXT, name TEXT, planned_date DATE, + planned_parameters_json JSONB, optimization_context_json JSONB, + input_snapshot_json JSONB, output_snapshot_json JSONB, + additional_inputs_snapshot_json JSONB, owner_id TEXT, owner_org TEXT, + client_request_id TEXT NOT NULL DEFAULT '', + client_request_fingerprint TEXT, + proposal_generation_status TEXT, proposal_generation_error TEXT, + proposal_generation_completed_at TIMESTAMPTZ, + proposal_generation_updated_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT now() + ); + CREATE UNIQUE INDEX ON experiment_definitions (owner_id, client_request_id) + WHERE client_request_id <> ''; + CREATE TABLE project_experiments ( + project_id TEXT, type_id TEXT, added_by TEXT, status TEXT, + sequence_index INTEGER, updated_at TIMESTAMPTZ, + PRIMARY KEY (project_id, type_id) + ); + """) + cur.execute("INSERT INTO experiment_types (id, equipment_id, type_parameters_json, outputs_json, additional_inputs_json) VALUES (%s,%s,%s,%s,%s)", ( + "typed-values", "review-equipment", + Json([{"name": "Count", "type": "integer", "mode": "fixed", "unit": "cycles"}]), + Json([]), Json([ + {"name": "Batch", "type": "string", "required": True}, + {"name": "Enabled", "type": "boolean", "required": True}, + {"name": "Settings", "type": "object", "required": True}, + {"name": "Layers", "type": "array", "required": True}, + {"name": "Optional", "type": "string", "required": False}, + ]), + )) + conn.close() + user = PlatformUser(id="reviewer", name="Review", email="review@example.org", + role="researcher", organization="Review") + app = FastAPI() + app.include_router(equipment.router) + app.dependency_overrides[get_platform_user] = lambda: user + with ExitStack() as stack: + stack.enter_context(patch.object(metadata_pg, "get_pg_connection", side_effect=connect)) + stack.enter_context(patch.object(metadata_pg, "get_pg_worker_connection", side_effect=connect)) + stack.enter_context(patch.object(equipment, "get_equipment_pg", return_value={ + "domain_id": "review-equipment", "owner_id": user.id, "status": "approved", + })) + stack.enter_context(patch.object(equipment, "get_projects_list_pg", return_value=[{"id": "review-project"}])) + stack.enter_context(patch.object(equipment, "start_experiment_proposal_job")) + yield TestClient(app), connect + finally: + with admin.cursor() as cur: + cur.execute(f'DROP SCHEMA "{schema}" CASCADE') + admin.close() + + +def _payload(): + return { + "experiment_name": "Typed values", "selected_equipment": "review-equipment", + "project_id": "review-project", "type_id": "typed-values", + "optimization_context": {"mode": "none"}, + "planned_parameters": [ + {"name": "Count", "planned_value": "5e1", "unit": "cycles"}, + {"name": "Batch", "planned_value": " batch-A ", "unit": ""}, + {"name": "Enabled", "planned_value": "false", "unit": ""}, + {"name": "Settings", "planned_value": '{"level": 2}', "unit": ""}, + {"name": "Layers", "planned_value": "[3, 4]", "unit": ""}, + ], + } + + +def test_actual_values_and_definition_snapshots_survive_storage(isolated_run_database): + client, connect = isolated_run_database + response = client.post("/equipment/experiment", json=_payload()) + assert response.status_code == 200, response.text + conn = connect() + try: + with conn.cursor() as cur: + cur.execute("SELECT planned_parameters_json, additional_inputs_snapshot_json FROM experiment_definitions WHERE id=%s", (response.json()["experiment_id"],)) + values, snapshot = cur.fetchone() + assert {v["name"]: v["planned_value"] for v in values} == { + "Count": "50", "Batch": " batch-A ", "Enabled": "false", + "Settings": '{"level":2}', "Layers": "[3,4]", + } + assert values[0]["unit"] == "cycles" + assert snapshot[0]["name"] == "Batch" + assert snapshot[0]["required"] is True + finally: + conn.close() + + +def test_invalid_values_leave_no_partial_run(isolated_run_database): + client, connect = isolated_run_database + for name, value, detail in [("Count", "2.5", "whole number"), + ("Settings", '{"x":1e999}', "finite"), + ("Batch", "", "required")]: + payload = _payload() + next(v for v in payload["planned_parameters"] if v["name"] == name)["planned_value"] = value + response = client.post("/equipment/experiment", json=payload) + assert response.status_code == 400, response.text + assert detail in response.json()["detail"], response.text + conn = connect() + try: + with conn.cursor() as cur: + cur.execute("SELECT count(*) FROM experiment_definitions") + assert cur.fetchone()[0] == 0 + cur.execute("SELECT count(*) FROM project_experiments") + assert cur.fetchone()[0] == 0 + finally: + conn.close() + + +def test_mixed_optimization_saves_only_fixed_values(isolated_run_database, monkeypatch): + from copy import deepcopy + from domain_configs import load_domain_config + + client, connect = isolated_run_database + config = deepcopy(load_domain_config("etcher")) + config["domain"]["id"] = "review-equipment" + monkeypatch.setenv("RECIPE_PROPOSAL_ETCHER_EQUIPMENT_IDS", "review-equipment") + inputs = [{ + "name": feature["id"], "type": "float", "unit": feature.get("unit", ""), + "mode": "fixed" if index == 0 else "variable", + "min_value": str(feature["min"]), "max_value": str(feature["max"]), + } for index, feature in enumerate(config["features"])] + target = config["targets"]["primary"]["id"] + fixed = inputs[0] + value = str((float(fixed["min_value"]) + float(fixed["max_value"])) / 2) + conn = connect() + with conn, conn.cursor() as cur: + cur.execute("UPDATE experiment_types SET type_parameters_json=%s, outputs_json=%s, additional_inputs_json='[]'::jsonb WHERE id='typed-values'", (Json(inputs), Json([{"name": target}]))) + conn.close() + payload = {**_payload(), "planned_parameters": [{"name": fixed["name"], "planned_value": value, "unit": fixed["unit"]}], "optimization_context": {"mode": "optimization", "primary_target": target, "primary_objective": "maximize"}} + with patch.object(metadata_pg, "get_equipment_pg", return_value={"config_json": config}): + response = client.post("/equipment/experiment", json=payload) + assert response.status_code == 200, response.text + assert response.json()["optimization_status"] == "queued" + conn = connect() + with conn, conn.cursor() as cur: + cur.execute("SELECT planned_parameters_json FROM experiment_definitions WHERE id=%s", (response.json()["experiment_id"],)) + saved = cur.fetchone()[0] + conn.close() + assert [item["name"] for item in saved] == [fixed["name"]] + assert float(saved[0]["planned_value"]) == float(value) diff --git a/api/tests/test_glance_closed_loop.py b/api/tests/test_glance_closed_loop.py index e822710..a665f9e 100644 --- a/api/tests/test_glance_closed_loop.py +++ b/api/tests/test_glance_closed_loop.py @@ -1671,6 +1671,9 @@ def close(self): self.assertEqual(result["generated"], 0) def test_auto_proposal_uses_trusted_postgres_live_fit_context(self): + from domain_configs import load_domain_config + inputs = [{"name": f["id"], "mode": "variable", "min_value": str(f["min"]), "max_value": str(f["max"])} + for f in load_domain_config("etcher")["features"]] now = datetime.now(timezone.utc) class Cursor: @@ -1694,7 +1697,8 @@ def fetchone(self): "type_id": "recipe_sweep", "execution_request_id": REQUEST_ID, "planned_parameters_json": [], - "input_snapshot_json": [], + "input_snapshot_json": inputs, + "equipment_id": "etcher", "type_parameters_json": [], "max_iteration": 1, "latest_batch_at": now - timedelta(minutes=2), @@ -1726,6 +1730,7 @@ def close(self): ], } with ( + patch("metadata_pg.get_equipment_pg", return_value=None), patch( "metadata_pg.get_pg_worker_connection", return_value=Connection(), @@ -1757,6 +1762,9 @@ def close(self): self.assertEqual(result["generated"], 1) def test_next_batch_waits_for_all_completed_trace_outcomes(self): + from domain_configs import load_domain_config + inputs = [{"name": f["id"], "mode": "variable", "min_value": str(f["min"]), "max_value": str(f["max"])} + for f in load_domain_config("etcher")["features"]] now = datetime.now(timezone.utc) unresolved_counts = iter([2, 1, 0]) @@ -1779,7 +1787,8 @@ def fetchone(self): "type_id": "recipe_sweep", "execution_request_id": REQUEST_ID, "planned_parameters_json": [], - "input_snapshot_json": [], + "input_snapshot_json": inputs, + "equipment_id": "etcher", "type_parameters_json": [], "max_iteration": 1, "latest_batch_at": now - timedelta(minutes=2), @@ -1814,6 +1823,7 @@ def close(self): ], } with ( + patch("metadata_pg.get_equipment_pg", return_value=None), patch( "metadata_pg.get_pg_worker_connection", side_effect=lambda: Connection(), diff --git a/api/tests/test_monolayer.py b/api/tests/test_monolayer.py new file mode 100644 index 0000000..a2bd35a --- /dev/null +++ b/api/tests/test_monolayer.py @@ -0,0 +1,75 @@ +import json +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pytest +from pymatgen.core import Lattice, Structure + +from computations.monolayer import extract_monolayer, prepare_cellrelax2d, cellrelax2d_outputs +from computations.qe import QePreparationError, QeParseError +from computations.schemas import validate_tool_inputs +from computations.slurm import SlurmJobSpec + + +def graphene(): + return Structure(Lattice.hexagonal(2.46, 20), ["C", "C"], [[0, 0, .5], [1/3, 2/3, .5]]) + + +@pytest.mark.parametrize("axis", [0, 1, 2]) +def test_layer_normalized_to_xy_for_every_stacking_axis(axis): + original = graphene() + order = [0, 1] + order.insert(axis, 2) + rotated = Structure(original.lattice.matrix[order], ["C", "C"], original.frac_coords[:, order]) + layer, metadata = extract_monolayer(rotated.as_dict(), 20) + assert len(layer) == 2 + assert np.allclose(layer.lattice.matrix[2], [0, 0, 20]) + assert np.allclose(layer.lattice.matrix[:2, 2], 0) + assert np.allclose(layer.frac_coords[:, 2], .5) + assert np.isclose(layer.get_distance(0, 1), original.get_distance(0, 1)) + assert metadata["source_stacking_orientation"][axis] != 0 + assert json.loads(json.dumps(metadata))["source_structure"] + + +def test_graphite_extracts_one_layer_and_preserves_source(): + bulk = Structure(Lattice.hexagonal(2.46, 6.7), ["C"]*4, + [[0,0,0], [1/3,2/3,0], [0,0,.5], [2/3,1/3,.5]]) + layer, metadata = extract_monolayer(bulk.as_dict(), 20) + assert len(layer) == 2 + assert len(metadata["source_structure"]["sites"]) == 4 + assert len(metadata["source_site_indices"]) == 2 + + +def test_rejects_bulk_structure(): + with pytest.raises(QePreparationError, match="dimensionality 2"): + extract_monolayer(Structure(Lattice.cubic(3.6), ["Cu"], [[0,0,0]]).as_dict(), 20) + + +@pytest.mark.parametrize("invalid", [{"k_z":2}, {"target_pressure":1}, {"vacuum_angstrom":2}, {"ibrav":2}]) +def test_2d_contract_rejects_bulk_settings(invalid): + with pytest.raises(ValueError): + validate_tool_inputs("cellrelaxdft2d", {"mp_id":"mp-48", **invalid}) + + +def test_qe_input_and_scheduler_use_2d_contract(tmp_path): + inputs = validate_tool_inputs("cellrelaxdft2d", {"mp_id":"mp-48", "structure_dict":graphene().as_dict(), "is_metal":True}) + prepared = prepare_cellrelax2d(inputs, tmp_path, "/depot/ncn/data/digital-twins/pseudopotentials") + text = (tmp_path / "vcrelax.in").read_text() + for expected in ["cell_dofree = '2Dxy'", "press_conv_thr = 5.0", "tprnfor = .TRUE.", "ecutwfc = 180", "8 8 1", "degauss = 0.002"]: + assert expected in text + assert prepared.tool_name == "cellrelaxdft2d" + assert prepared.pseudopotentials[0]["sha256"] == "3fca0f06ef8dac54134ee149f0f17ce1990b24e9fcd0c73251b15efd19d1227a" + spec = SlurmJobSpec(run_id="11111111-1111-4111-8111-111111111111", tool_name=prepared.tool_name, + workdir="/scratch/test", account="ncn", partition="cpu", qos="normal", time_limit=prepared.time_limit, + ntasks=64, modules=["quantum-espresso/7.1"], mpi_launcher="srun") + assert "srun pw.x -in vcrelax.in" in spec.render() + + +def test_collection_rejects_changed_vacuum(): + initial = graphene() + final = initial.copy() + final.scale_lattice(final.volume * 1.1) + with patch("computations.monolayer.cellrelax_outputs_from_parsed", return_value={"structure_dict":final.as_dict()}): + with pytest.raises(QeParseError, match="vacuum"): + cellrelax2d_outputs({}, {}, {"initial_monolayer_structure":initial.as_dict(), "cell_height_angstrom":20}) diff --git a/api/tests/test_phase3_security.py b/api/tests/test_phase3_security.py index 3ba64b4..fd39fe9 100644 --- a/api/tests/test_phase3_security.py +++ b/api/tests/test_phase3_security.py @@ -635,9 +635,7 @@ def test_experiment_creation_links_visible_matching_project(self): "project-1", ) - def test_project_run_requires_planned_parameter_values(self): - # Projects may span several tools, so a project on other equipment is - # accepted; a project run still needs at least one planned value. + def test_experiment_creation_rejects_invisible_project(self): with patch.object( equipment, "get_equipment_pg", @@ -650,13 +648,7 @@ def test_project_run_requires_planned_parameter_values(self): ), patch.object( equipment, "get_projects_list_pg", - return_value=[ - { - "id": "project-1", - "name": "Other Project", - "equipment_id": "other_tool", - } - ], + return_value=[], ), patch.object( equipment, "create_experiment_definition_pg", @@ -670,12 +662,15 @@ def test_project_run_requires_planned_parameter_values(self): "project_id": "project-1", "type_id": "recipe_sweep", "planned_date": "2026-04-15", - "planned_parameters": [], + "planned_parameters": [{"name": "pressure", "planned_value": "55", "unit": "mTorr"}], }, ) - self.assertEqual(response.status_code, 400) - self.assertIn("At least one planned parameter value is required", response.json()["detail"]) + self.assertEqual(response.status_code, 404) + self.assertEqual( + response.json()["detail"], + "Selected project was not found", + ) self.assertEqual(create_experiment_definition_pg.call_count, 0) def test_experiment_creation_requires_approved_equipment(self): @@ -711,6 +706,36 @@ def test_experiment_creation_requires_approved_equipment(self): ) self.assertEqual(create_experiment_definition_pg.call_count, 0) + def test_experiment_run_rejects_computational_equipment(self): + with patch.object( + equipment, + "get_equipment_pg", + return_value={ + "domain_id": "digital_qe_7_1_negishi", + "owner_id": "owner-1", + "owner_org": "Birck", + "status": "approved", + "asset_kind": "computational", + }, + ), patch.object( + equipment, + "create_experiment_definition_pg", + ) as create_experiment_definition_pg: + response = self.client.post( + "/api/equipment/experiment", + headers=OWNER_HEADERS, + json={ + "experiment_name": "Relax Si", + "selected_equipment": "digital_qe_7_1_negishi", + "type_id": "forked_relax_1234abcd", + "planned_parameters": [], + }, + ) + + self.assertEqual(response.status_code, 409) + self.assertIn("Registered computational experiments", response.json()["detail"]) + self.assertEqual(create_experiment_definition_pg.call_count, 0) + def test_upload_records_equipment_association(self): with patch.object( upload, diff --git a/api/tests/test_pilot_review_permissions.py b/api/tests/test_pilot_review_permissions.py new file mode 100644 index 0000000..27aa070 --- /dev/null +++ b/api/tests/test_pilot_review_permissions.py @@ -0,0 +1,28 @@ +"""Synthetic demonstration actions are separate from student calculations.""" +from unittest.mock import patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from routers import dataset_v2 +from security import PlatformUser, get_platform_user + + +@pytest.mark.parametrize("role", ["researcher", "pi", "equipment_owner", "admin"]) +@pytest.mark.parametrize("path,handler", [ + ("etcher-profilometer", "create_etcher_profilometer_workflow_example_pg"), + ("rcac-closed-loop", "create_rcac_closed_loop_demo_pg"), +]) +def test_only_admin_can_create_synthetic_workflow_examples(role, path, handler): + app = FastAPI() + app.include_router(dataset_v2.router, prefix="/api") + app.dependency_overrides[get_platform_user] = lambda: PlatformUser( + id="review-user", email="review@example.test", name="Review", + role=role, organization="Review", + ) + with patch.object(dataset_v2, handler, return_value={"status": "created"}) as create: + response = TestClient(app).post( + f"/api/dataset/v2/projects/REVIEW/workflow-examples/{path}", json={}, + ) + assert response.status_code == (200 if role == "admin" else 403), response.text + assert create.called is (role == "admin") diff --git a/api/tests/test_proposal_objectives.py b/api/tests/test_proposal_objectives.py new file mode 100644 index 0000000..99f1e30 --- /dev/null +++ b/api/tests/test_proposal_objectives.py @@ -0,0 +1,188 @@ +"""Regression tests: explicit objectives steer proposal ranking, selected +ranges are honored, and the implicit etcher default path is unchanged.""" + +import os +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pandas as pd + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +import ml_engine # noqa: E402 +import metadata_pg +from domain_configs import load_domain_config +from metadata_pg import resolve_optimization_request # noqa: E402 + +FEATURES = list(ml_engine.ETCHER_FEATURES) +PRESSURE = FEATURES.index("Etch_AvgPres") + + +def _training_frame(n: int = 40): + rng = np.random.RandomState(7) + lows = np.array([ml_engine.ETCHER_DEFAULT_RANGES[f][0] for f in FEATURES]) + highs = np.array([ml_engine.ETCHER_DEFAULT_RANGES[f][1] for f in FEATURES]) + X = lows + rng.uniform(size=(n, len(FEATURES))) * (highs - lows) + rate = 3.0 * X[:, PRESSURE] + rng.normal(scale=0.5, size=n) + spread = 0.5 * X[:, PRESSURE] + rng.normal(scale=0.2, size=n) + frame = pd.DataFrame(X, columns=FEATURES) + frame["LOTNAME"] = [f"run-{i}" for i in range(n)] + frame["AvgEtchRate"] = rate + frame["RangeEtchRate"] = spread + return frame + + +def _fake_get_xy(cfg, _context=None): + frame = _training_frame() + features = [f for f in cfg["features"] if f in frame.columns] + X = frame[features].to_numpy() + ys = {t: frame[t].to_numpy() for t in cfg["targets"] if t in frame.columns} + return X, ys, frame["LOTNAME"].tolist(), frame + + +def _pressure_predictor(X_train, y_train, X_pred, **kwargs): + # Prediction increases with (scaled) pressure. With a constant sigma, + # expected improvement is strictly increasing in the directed mean, so + # EI ranking equals the directed-mean ranking regardless of scale. + return X_pred[:, PRESSURE].astype(float), np.full(len(X_pred), 100.0) + + +def _proposals(model: str, **kwargs): + with ( + patch.dict( + os.environ, + {"ML_PROPOSAL_MODEL": model, "ML_PROPOSAL_CANDIDATES": "256", + "NEXTJS_PROPOSAL_ENGINE": "ml"}, + clear=False, + ), + patch.object(ml_engine, "_get_xy", side_effect=_fake_get_xy), + patch.object(ml_engine, "_registry_model_for", return_value=None), + patch.object(ml_engine, "_fit_predictive_model", side_effect=_pressure_predictor), + ): + result = ml_engine.compute_proposals( + n_batches=1, + batch_size=3, + optimization_context={"force_live_fit": True}, + **kwargs, + ) + return result, [p["parameters"] for p in result["batches"][0]["proposals"]] + + +class ProposalObjectiveTests(unittest.TestCase): + def setUp(self): + self.equipment_patch = patch.object(metadata_pg, "get_equipment_pg", return_value=None) + self.equipment_patch.start() + self.addCleanup(self.equipment_patch.stop) + + def _mean_pressure(self, params): + return float(np.mean([p["Etch_AvgPres"] for p in params])) + + def test_direction_changes_ranking_on_gpr_path(self): + maximize, high = _proposals( + "gpr", + domain_config={"features": FEATURES, "targets": ["AvgEtchRate"]}, + objectives=[{"target": "AvgEtchRate", "direction": "maximize"}], + ) + _minimize, low = _proposals( + "gpr", + domain_config={"features": FEATURES, "targets": ["AvgEtchRate"]}, + objectives=[{"target": "AvgEtchRate", "direction": "minimize"}], + ) + self.assertGreater(self._mean_pressure(high), 70.0) + self.assertLess(self._mean_pressure(low), 35.0) + self.assertEqual( + maximize["objectives"], [{"target": "AvgEtchRate", "direction": "maximize"}] + ) + + def test_direction_changes_ranking_on_rf_path(self): + _maximize, high = _proposals( + "rf", objectives=[{"target": "AvgEtchRate", "direction": "maximize"}] + ) + _minimize, low = _proposals( + "rf", objectives=[{"target": "AvgEtchRate", "direction": "minimize"}] + ) + self.assertGreater(self._mean_pressure(high), self._mean_pressure(low) + 20.0) + + def test_fixed_input_pinned_and_variable_range_respected(self): + request = resolve_optimization_request( + equipment_id="etcher", + input_snapshot=[ + *[{"name": f["id"], "mode": "variable", "min_value": str(f["min"]), "max_value": str(f["max"])} + for f in load_domain_config("etcher")["features"] if f["id"] not in {"Etch_AvgO2Flow", "Etch_AvgPres"}], + {"name": "Etch_AvgO2Flow", "mode": "fixed"}, + {"name": "Etch_AvgPres", "mode": "variable", "min_value": "20", + "max_value": "40"}, + ], + planned_parameters=[{"name": "Etch_AvgO2Flow", "planned_value": "40", "unit": ""}], + optimization_context={"primary_target": "AvgEtchRate", + "primary_objective": "maximize"}, + output_snapshot=[{"name": "AvgEtchRate"}], + strict=True, + ) + result, params = _proposals( + "gpr", + constraints=request["constraints"], + domain_config=request["domain_config"], + objectives=request["objectives"], + ) + self.assertEqual(result["constraints_applied"]["Etch_AvgPres"], [20.0, 40.0]) + for proposal in params: + self.assertEqual(proposal["Etch_AvgO2Flow"], 40.0) + self.assertGreaterEqual(proposal["Etch_AvgPres"], 20.0) + self.assertLessEqual(proposal["Etch_AvgPres"], 40.0) + # Maximizing a pressure-increasing target pushes to the top of the range. + self.assertGreater(self._mean_pressure(params), 35.0) + + def test_prediction_fields_follow_target_names(self): + result, _params = _proposals( + "gpr", + objectives=[ + {"target": "RangeEtchRate", "direction": "minimize"}, + {"target": "AvgEtchRate", "direction": "maximize"}, + ], + ) + proposal = result["batches"][0]["proposals"][0] + values = proposal["predicted_values"] + self.assertEqual(set(values), {"RangeEtchRate", "AvgEtchRate"}) + self.assertEqual(proposal["predicted_etch_rate"], values["AvgEtchRate"]) + self.assertAlmostEqual(proposal["predicted_range_nm"], values["RangeEtchRate"] * 5.0) + + def test_missing_target_raises_instead_of_simulating(self): + with self.assertRaisesRegex(ValueError, "Thickness"): + _proposals("gpr", objectives=[{"target": "Thickness", "direction": "maximize"}]) + + def test_invalid_objectives_raise(self): + with self.assertRaisesRegex(ValueError, "direction"): + ml_engine._objective_signs([{"target": "AvgEtchRate", "direction": "up"}]) + with self.assertRaisesRegex(ValueError, "repeated"): + ml_engine._objective_signs([ + {"target": "AvgEtchRate", "direction": "maximize"}, + {"target": "AvgEtchRate", "direction": "minimize"}, + ]) + + def test_default_pareto_signs_are_the_etcher_objectives(self): + rng = np.random.RandomState(3) + rate, spread = rng.rand(20), rng.rand(20) + candidates_rate, candidates_spread = rng.rand(8), rng.rand(8) + default = ml_engine._pareto_improvement_scores( + candidates_rate, candidates_spread, rate, spread + ) + explicit = ml_engine._pareto_improvement_scores( + candidates_rate, candidates_spread, rate, spread, 1.0, -1.0 + ) + np.testing.assert_allclose(default[0], explicit[0]) + np.testing.assert_allclose(default[1], explicit[1]) + + def test_implicit_default_does_not_report_objectives(self): + result, _params = _proposals("gpr") + self.assertNotIn("objectives", result) + self.assertNotIn("predicted_values", result["batches"][0]["proposals"][0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_qe_parser.py b/api/tests/test_qe_parser.py new file mode 100644 index 0000000..6c9f3f0 --- /dev/null +++ b/api/tests/test_qe_parser.py @@ -0,0 +1,92 @@ +import tempfile +import unittest +from pathlib import Path + +from computations.qe import QeParseError, parse_qe_output + +COMPLETE_OUTPUT = """ + ! total energy = -15.25000000 Ry + Forces acting on atoms (cartesian axes, Ry/au): + + atom 1 type 1 force = 0.00100000 -0.00200000 0.00300000 + Total force = 0.003741657 Ry/au + total stress (Ry/bohr**3) (kbar) P= 5.50 + 0.0000 0.0000 0.0000 10.0 1.0 2.0 + 0.0000 0.0000 0.0000 1.0 20.0 3.0 + 0.0000 0.0000 0.0000 2.0 3.0 30.0 + End of BFGS Geometry Optimization + JOB DONE. +""" + +FIXED_CELL_RELAX_OUTPUT = """ + ! total energy = -31.50000000 Ry + Total force = 0.000000100 Ry/au + End of BFGS Geometry Optimization + JOB DONE. +""" + +UNCONVERGED_VC_RELAX_OUTPUT = """ + ! total energy = -73.80371995 Ry + total stress (Ry/bohr**3) (kbar) P= 0.37 + 0.0000 0.0000 0.0000 0.37 0.0 0.0 + 0.0000 0.0000 0.0000 0.00 0.1 0.0 + 0.0000 0.0000 0.0000 0.00 0.0 0.2 + c_bands: 1 eigenvalues not converged + bfgs failed after 20 scf cycles and 17 bfgs steps, convergence not achieved + End of BFGS Geometry Optimization + JOB DONE. +""" + + +class QeParserTests(unittest.TestCase): + def test_extracts_energy_pressure_stress_and_forces(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "qe.out" + path.write_text(COMPLETE_OUTPUT, encoding="utf-8") + result = parse_qe_output(path, natoms=1) + self.assertEqual(result["ionic_energies_ry"], [-15.25]) + self.assertEqual(result["pressures_kbar"], [5.5]) + self.assertEqual(result["stresses_kbar"][0][1][1], 20.0) + self.assertEqual(result["atomic_forces_ry_bohr"][0][0][2], 0.003) + self.assertTrue(result["process_completed"]) + self.assertTrue(result["convergence"]) + self.assertEqual(result["scientific_status"], "converged") + + def test_fixed_cell_relax_does_not_require_stress(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "qe.out" + path.write_text(FIXED_CELL_RELAX_OUTPUT, encoding="utf-8") + result = parse_qe_output(path) + self.assertTrue(result["process_completed"]) + self.assertTrue(result["convergence"]) + self.assertEqual(result["pressures_kbar"], []) + + def test_stress_can_be_required_by_the_workflow(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "qe.out" + path.write_text(FIXED_CELL_RELAX_OUTPUT, encoding="utf-8") + with self.assertRaises(QeParseError): + parse_qe_output(path, require_stress=True) + + def test_job_done_does_not_override_failed_bfgs(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "qe.out" + path.write_text(UNCONVERGED_VC_RELAX_OUTPUT, encoding="utf-8") + result = parse_qe_output(path, require_stress=True) + self.assertTrue(result["process_completed"]) + self.assertFalse(result["convergence"]) + self.assertEqual(result["scientific_status"], "not_converged") + self.assertIn("convergence not achieved", result["failure_reasons"][0]) + self.assertEqual(result["warning_count"], 1) + self.assertIn("eigenvalues not converged", result["warnings"][0]) + + def test_rejects_truncated_output(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "qe.out" + path.write_text("JOB DONE.\n", encoding="utf-8") + with self.assertRaises(QeParseError): + parse_qe_output(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_qe_workflows.py b/api/tests/test_qe_workflows.py new file mode 100644 index 0000000..37a2f83 --- /dev/null +++ b/api/tests/test_qe_workflows.py @@ -0,0 +1,159 @@ +import tempfile +import unittest +from pathlib import Path +from uuid import uuid4 + +import numpy as np +from computations.qe import elastic_outputs, prepare_cellrelax, prepare_elastic +from computations.schemas import validate_tool_inputs +from pymatgen.analysis.elasticity.strain import Strain +from pymatgen.analysis.elasticity.stress import Stress +from pymatgen.core import Lattice, Structure + + +def _silicon_structure() -> Structure: + return Structure(Lattice.cubic(5.43), ["Si"], [[0.0, 0.0, 0.0]]) + + +def _pseudo_root(_root: Path) -> str: + return "/depot/ncn/data/digital-twins/pseudopotentials" + + +def _qe_relax_output(stress_gpa: np.ndarray) -> str: + # QE reports the opposite stress sign from pymatgen's convention and the + # source Sim2L converts kbar to GPa by dividing by ten. + qe_stress_kbar = -10.0 * np.asarray(stress_gpa) + rows = "\n".join( + " 0.0 0.0 0.0 " + " ".join(f"{value:.12f}" for value in row) + for row in qe_stress_kbar + ) + return f""" + ! total energy = -10.00000000 Ry + total stress (Ry/bohr**3) (kbar) P= 0.0 +{rows} + End of BFGS Geometry Optimization + JOB DONE. +""" + + +class QeWorkflowTests(unittest.TestCase): + def test_cellrelax_preparation_preserves_r99_contract(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inputs = validate_tool_inputs( + "cellrelaxdft", + { + "mp_id": "mp-149", + "structure_dict": _silicon_structure().as_dict(), + }, + ) + prepared = prepare_cellrelax(inputs, root / "run", _pseudo_root(root)) + + qe_input = (root / "run" / "vcrelax.in").read_text(encoding="utf-8") + self.assertIn("calculation = 'vc-relax'", qe_input) + self.assertIn("press = 0.0", qe_input) + self.assertIn("press_conv_thr = 0.5", qe_input) + self.assertIn("disk_io = 'nowf'", qe_input) + self.assertIn( + "pseudo_dir = '/depot/ncn/data/digital-twins/pseudopotentials/pseudo_PAW_PBEsol'", + qe_input, + ) + self.assertEqual(prepared.ntasks, 64) + self.assertEqual([path.name for path in prepared.stage_files], ["vcrelax.in"]) + self.assertEqual(prepared.pseudopotentials[0]["element"], "Si") + self.assertEqual(len(prepared.pseudopotentials[0]["sha256"]), 64) + + def test_elastic_default_has_24_independent_strain_members(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + structure = _silicon_structure() + upstream_inputs = validate_tool_inputs( + "cellrelaxdft", + {"mp_id": "mp-149", "structure_dict": structure.as_dict()}, + ) + elastic_inputs = validate_tool_inputs( + "elasticdft", + { + "mp_id": "mp-149", + "upstream_execution_id": str(uuid4()), + }, + ) + upstream_outputs = { + "convergence": True, + "structure_dict": structure.as_dict(), + "pressure": 0.0, + } + prepared = prepare_elastic( + elastic_inputs, + upstream_inputs, + upstream_outputs, + root / "run", + _pseudo_root(root), + ) + + self.assertEqual(prepared.metadata["count"], 24) + self.assertEqual(len(prepared.metadata["strain_matrices"]), 24) + self.assertEqual(prepared.job_array, "0-23%4") + self.assertEqual(len(prepared.expected_artifacts), 24) + self.assertEqual([path.name for path in prepared.stage_files], ["elastic-inputs.tar.gz"]) + + def test_elastic_fitting_recovers_known_cubic_tensor(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + structure = _silicon_structure() + upstream_inputs = validate_tool_inputs( + "cellrelaxdft", + {"mp_id": "mp-149", "structure_dict": structure.as_dict()}, + ) + elastic_inputs = validate_tool_inputs( + "elasticdft", + { + "mp_id": "mp-149", + "upstream_execution_id": str(uuid4()), + }, + ) + upstream_outputs = { + "convergence": True, + "structure_dict": structure.as_dict(), + "pressure": 0.0, + } + prepared = prepare_elastic( + elastic_inputs, + upstream_inputs, + upstream_outputs, + root / "run", + _pseudo_root(root), + ) + + expected = np.zeros((6, 6)) + expected[:3, :3] = 100.0 + np.fill_diagonal(expected[:3, :3], 200.0) + np.fill_diagonal(expected[3:, 3:], 50.0) + for matrix, relative in zip( + prepared.metadata["strain_matrices"], + prepared.expected_artifacts, + strict=True, + ): + strain = Strain(matrix) + stress = Stress.from_voigt(expected @ strain.voigt) + output_path = prepared.local_workdir / relative + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + _qe_relax_output(np.asarray(stress)), encoding="utf-8" + ) + + result = elastic_outputs(prepared) + np.testing.assert_allclose(result["elastic_tensor"], expected, atol=1e-7) + self.assertTrue(result["process_completed"]) + self.assertTrue(result["convergence"]) + self.assertEqual(len(result["member_statuses"]), 24) + self.assertEqual(len(result["configurations"]), 24) + # This synthetic fixture has no final atomic coordinates/forces; + # it must never masquerade as a complete training configuration. + self.assertIsNone(result["configurations"][0]["relaxed_structure"]) + self.assertFalse(result["configurations"][0]["complete_structure_force_pair"]) + self.assertIn("tension positive", result["units"]["stress_tensors"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_rcac_gateway.py b/api/tests/test_rcac_gateway.py new file mode 100644 index 0000000..53dc23e --- /dev/null +++ b/api/tests/test_rcac_gateway.py @@ -0,0 +1,476 @@ +import tempfile +import hashlib +import json +import os +import base64 +import tarfile +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from computations.globus_gateway import ( + GlobusComputeGateway, + GlobusGatewayError, + rcac_service_gateway, +) +from computations.slurm import SlurmJobSpec, aggregate_states + + +class FakeClient: + def __init__(self, result): + self.result = result + self.submissions = [] + + def run(self, request, *, endpoint_id, function_id): + self.submissions.append((request, endpoint_id, function_id)) + return "33333333-3333-4333-8333-333333333333" + + def get_task(self, task_id): + return { + "pending": False, + "status": "success", + "result": self.result, + } + + +def _spec(**overrides): + values = { + "run_id": str(uuid4()), + "tool_name": "cellrelaxdft", + "workdir": "/scratch/negishi/dtservice/runs/test", + "account": "ncn", + "partition": "cpu", + "qos": "normal", + "time_limit": "24:00:00", + "ntasks": 64, + "modules": ["gcc/12.2.0", "openmpi/4.1.4", "quantum-espresso/7.1"], + "mpi_launcher": "srun", + } + values.update(overrides) + return SlurmJobSpec(**values) + + +def test_slurm_contract_has_trusted_comment_and_fixed_runtime(): + spec = _spec() + payload = spec.as_payload() + assert payload["comment"] == f"dt-run:{spec.run_id}" + assert "#SBATCH --account=ncn" in payload["script"] + assert "module load quantum-espresso/7.1" in payload["script"] + assert "srun pw.x -in vcrelax.in" in payload["script"] + + +def test_slurm_contract_rejects_arbitrary_commands_and_oversized_jobs(): + with pytest.raises(ValueError, match="unsupported computation tool"): + _spec(tool_name="shell") + with pytest.raises(ValueError, match="1..64"): + _spec(ntasks=65) + + +def test_array_state_aggregation_waits_for_live_members_before_terminal_failure(): + assert aggregate_states(["COMPLETED", "FAILED", "COMPLETED"]).value == "failed" + assert aggregate_states(["COMPLETED", "RUNNING"]).value == "running" + assert aggregate_states(["FAILED", "RUNNING"]).value == "running" + assert aggregate_states(["CANCELLED", "PENDING"]).value == "pending" + assert aggregate_states([""]).value == "unknown" + assert aggregate_states(["COMPLETED", "COMPLETED"]).value == "completed" + + +def test_globus_client_sends_only_requested_endpoint_and_function(): + endpoint_id = "11111111-1111-4111-8111-111111111111" + function_id = "22222222-2222-4222-8222-222222222222" + client = FakeClient({"job_id": "42"}) + gateway = GlobusComputeGateway(client=client, sleep=lambda _seconds: None) + task_id, result = gateway.invoke( + endpoint_id=endpoint_id, + function_id=function_id, + action="status_many", + payload={ + "cluster": "negishi", + "jobs": [{"run_id": str(uuid4()), "job_id": "42"}], + }, + ) + assert task_id == "33333333-3333-4333-8333-333333333333" + assert result == {"job_id": "42"} + assert client.submissions[0][1:] == (endpoint_id, function_id) + + +def test_globus_client_rejects_invalid_identifiers_before_submission(): + client = FakeClient({}) + gateway = GlobusComputeGateway(client=client) + with pytest.raises(GlobusGatewayError, match="identifier"): + gateway.invoke( + endpoint_id="not-a-uuid", + function_id=str(uuid4()), + action="status_many", + payload={}, + ) + assert client.submissions == [] + + +def test_globus_lookup_retries_server_failure_without_resubmitting_task(): + class StorageFailure(Exception): + http_status = 500 + + client = FakeClient({"artifact_manifest": []}) + successful_result = client.get_task("task") + sleeps = [] + with patch.object(client, "get_task", side_effect=[ + StorageFailure("StorageException"), StorageFailure("StorageException"), + successful_result, + ]) as lookup: + gateway = GlobusComputeGateway(client=client, sleep=sleeps.append) + task_id, result = gateway.invoke(endpoint_id=str(uuid4()), function_id=str(uuid4()), + action="collect", payload={}) + assert result == {"artifact_manifest": []} + assert len(client.submissions) == 1 + assert lookup.call_count == 3 + assert all(call.args[0] == task_id + for call in lookup.call_args_list) + assert sleeps == [2, 4] + + +@pytest.mark.parametrize("http_status,expected_lookups", [(403, 1), (500, 4)]) +def test_globus_lookup_retries_are_bounded_and_do_not_retry_auth(http_status, expected_lookups): + class ApiFailure(Exception): + pass + + failure = ApiFailure("sensitive diagnostic") + failure.http_status = http_status + client = FakeClient({}) + with patch.object(client, "get_task", side_effect=failure) as lookup: + gateway = GlobusComputeGateway(client=client, sleep=lambda _: None) + with pytest.raises(GlobusGatewayError, match="task lookup failed") as caught: + gateway.invoke(endpoint_id=str(uuid4()), function_id=str(uuid4()), + action="collect", payload={}) + assert lookup.call_count == expected_lookups + assert len(client.submissions) == 1 + assert "sensitive" not in str(caught.value) + + +def test_staged_payload_budget_is_enforced(): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "too-large.bin" + path.write_bytes(b"0" * (7 * 1024 * 1024 + 1)) + with pytest.raises(ValueError, match="payload budget"): + GlobusComputeGateway.encoded_files([path]) + + +def test_gateway_is_source_serializable_without_the_dt_package(): + from globus_compute_sdk.serialize import ComputeSerializer, PureSourceTextInspect + + serializer = ComputeSerializer(strategy_code=PureSourceTextInspect()) + packed = serializer.serialize(rcac_service_gateway) + restored = serializer.deserialize(packed) + assert restored.__name__ == "rcac_service_gateway" + assert "computations.globus_gateway" not in packed + + +def test_serialized_gateway_avoids_python_310_only_union_annotations(): + """The registered source must load under Gautschi's Python 3.9 runtime.""" + + import inspect + + source = inspect.getsource(rcac_service_gateway) + assert "_Path | None" not in source + + +@pytest.mark.parametrize("relax_tool", ["cellrelaxdft", "cellrelaxdft2d"]) +def test_remote_gateway_submit_status_and_collect_are_idempotent_and_scoped(relax_tool): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + binary = root / "bin" + work_root = root / "scratch" + pseudo_root = root / "pseudos" + artifact_root = root / "depot-runs" + binary.mkdir() + (pseudo_root / "pseudo_PAW_PBEsol").mkdir(parents=True) + pseudo = pseudo_root / "pseudo_PAW_PBEsol" / "C.upf" + pseudo.write_text("approved pseudo", encoding="utf-8") + digest = hashlib.sha256(pseudo.read_bytes()).hexdigest() + submit_log = root / "submits.log" + + commands = { + "pw.x": "#!/bin/sh\nexit 0\n", + "sbatch": f"#!/bin/sh\necho submit >> '{submit_log}'\necho 9001\n", + "sacct": "#!/bin/sh\necho '9001|COMPLETED'\n", + "squeue": "#!/bin/sh\nexit 0\n", + "scancel": "#!/bin/sh\nexit 0\n", + } + for name, source in commands.items(): + path = binary / name + path.write_text(source, encoding="utf-8") + path.chmod(0o755) + + run_id = str(uuid4()) + (binary / "sacct").write_text( + f"#!/bin/sh\necho '9001|COMPLETED|Unknown|Unknown|0:0|Unknown|dt-run:{run_id}'\n", + encoding="utf-8", + ) + workdir = work_root / run_id + spec = SlurmJobSpec( + run_id=run_id, + tool_name=relax_tool, + workdir=str(workdir), + account="ncn", + partition="cpu", + qos="normal", + time_limit="24:00:00", + ntasks=64, + modules=["gcc/12.2.0", "openmpi/4.1.4", "quantum-espresso/7.1"], + mpi_launcher="srun", + environment={"QE_DISABLE_GGA_PBE": "0"}, + ) + endpoint_env = { + "PATH": f"{binary}:{os.environ['PATH']}", + "DT_CLUSTER_NAME": "negishi", + "DT_WORK_ROOT": str(work_root), + "DT_PSEUDO_ROOT": str(pseudo_root), + "DT_ARTIFACT_ROOT": str(artifact_root), + "DT_SLURM_ACCOUNT": "ncn", + "DT_SLURM_PARTITION": "cpu", + "DT_SLURM_QOS": "normal", + "DT_QE_MODULES_JSON": json.dumps(list(spec.modules)), + "DT_MPI_LAUNCHER": "srun", + } + request = { + "action": "submit", + "payload": { + "cluster": "negishi", + "run_id": run_id, + "spec": spec.as_payload(), + "files": { + "vcrelax.in": base64.b64encode(b"&CONTROL\n/").decode("ascii") + }, + "pseudopotentials": [ + { + "relative": "pseudo_PAW_PBEsol/C.upf", + "sha256": digest, + } + ], + }, + } + with patch.dict(os.environ, endpoint_env, clear=False): + with patch.dict( + os.environ, {"DT_SLURM_ACCOUNT": "ncn\n#SBATCH --qos=standby"} + ): + with pytest.raises(RuntimeError, match="scheduler setting is invalid"): + rcac_service_gateway(request) + first = rcac_service_gateway(request) + second = rcac_service_gateway(request) + marker = workdir / ".dt-slurm-job-id" + marker.rename(workdir / ".saved-marker") + with pytest.raises(RuntimeError, match="outcome is uncertain"): + rcac_service_gateway(request) + (workdir / ".saved-marker").rename(marker) + subprocess.run([sys.executable, "dt-capture-environment.py"], cwd=workdir, + env={**os.environ, "PRIVATE_SENTINEL": "not-for-provenance"}, check=True) + observation = json.loads((workdir / "execution-environment.json").read_text()) + assert observation["qe_executable_sha256"] == hashlib.sha256((binary / "pw.x").read_bytes()).hexdigest() + assert observation["run_id"] == run_id + assert "not-for-provenance" not in json.dumps(observation) + states = rcac_service_gateway( + { + "action": "status_many", + "payload": { + "cluster": "negishi", + "jobs": [{"run_id": run_id, "job_id": "9001"}], + }, + } + ) + + assert first == second + assert submit_log.read_text(encoding="utf-8").splitlines() == ["submit"] + assert states["states"] == {"9001": ["COMPLETED"]} + assert states["accounting"]["9001"][0]["started_at"] is None + assert f"--comment=dt-run:{run_id}" not in ( + workdir / "dt-workflow.sbatch" + ).read_text(encoding="utf-8") + + (workdir / "vcrelax.stdout").write_text( + """ + lattice parameter (alat) = 10.000000 a.u. + ! total energy = -10.0 Ry + total stress (Ry/bohr**3) (kbar) P= 0.0 + 0 0 0 1 0 0 + 0 0 0 0 1 0 + 0 0 0 0 0 1 + CELL_PARAMETERS (alat) + 1 0 0 + 0 1 0 + 0 0 1 + ATOMIC_POSITIONS (crystal) + C 0 0 0 + End of BFGS Geometry Optimization + JOB DONE. + """, + encoding="utf-8", + ) + (workdir / "slurm.out").write_text("", encoding="utf-8") + (workdir / "slurm.err").write_text("", encoding="utf-8") + collected = rcac_service_gateway( + { + "action": "collect", + "payload": { + "cluster": "negishi", + "run_id": run_id, + "job_id": "9001", + "tool_name": "cellrelaxdft", + "expected_artifacts": ["vcrelax.stdout"], + "retained_artifacts": [ + "vcrelax.in", + "vcrelax.stdout", + "slurm.out", + "slurm.err", + ], + }, + } + ) + assert collected["parsed_outputs"]["vcrelax.stdout"]["convergence"] is True + assert len(collected["artifact_manifest"]) == 7 + assert all(item["downloadable"] is True for item in collected["artifact_manifest"]) + + wrong_run = str(uuid4()) + with patch.dict(os.environ, endpoint_env, clear=False), pytest.raises( + RuntimeError, match="not owned" + ): + rcac_service_gateway( + { + "action": "cancel", + "payload": { + "cluster": "negishi", + "run_id": wrong_run, + "job_id": "9001", + }, + } + ) + + +def test_remote_gateway_runs_elastic_array_with_cluster_launcher(): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + binary = root / "bin" + work_root = root / "scratch" + pseudo_root = root / "pseudos" + artifact_root = root / "depot-runs" + package_root = root / "elastic_inputs" + binary.mkdir() + (pseudo_root / "pseudo_PAW_PBEsol").mkdir(parents=True) + pseudo = pseudo_root / "pseudo_PAW_PBEsol" / "Si.upf" + pseudo.write_text("approved pseudo", encoding="utf-8") + digest = hashlib.sha256(pseudo.read_bytes()).hexdigest() + for index in range(2): + strain = package_root / f"strain_{index:03d}" + strain.mkdir(parents=True) + (strain / "relax.in").write_text("&CONTROL\n/", encoding="utf-8") + (package_root / "manifest.json").write_text( + json.dumps({"count": 2}), encoding="utf-8" + ) + archive = root / "elastic-inputs.tar.gz" + with tarfile.open(archive, "w:gz") as handle: + handle.add(package_root, arcname="elastic_inputs") + + sbatch = binary / "sbatch" + sbatch.write_text("#!/bin/sh\necho 9002\n", encoding="utf-8") + sbatch.chmod(0o755) + run_id = str(uuid4()) + workdir = work_root / run_id + modules = ["gcc/11.4.1", "openmpi/4.1.6", "quantum-espresso/7.3.1"] + spec = SlurmJobSpec( + run_id=run_id, + tool_name="elasticdft", + workdir=str(workdir), + account="ncn", + partition="cpu", + qos="normal", + time_limit="08:00:00", + ntasks=64, + modules=modules, + mpi_launcher="mpirun", + array="0-1%2", + ) + endpoint_env = { + "PATH": f"{binary}:{os.environ['PATH']}", + "DT_CLUSTER_NAME": "gautschi", + "DT_WORK_ROOT": str(work_root), + "DT_PSEUDO_ROOT": str(pseudo_root), + "DT_ARTIFACT_ROOT": str(artifact_root), + "DT_SLURM_ACCOUNT": "ncn", + "DT_SLURM_PARTITION": "cpu", + "DT_SLURM_QOS": "normal", + "DT_QE_MODULES_JSON": json.dumps(modules), + "DT_MPI_LAUNCHER": "mpirun", + } + request = { + "action": "submit", + "payload": { + "cluster": "gautschi", + "run_id": run_id, + "spec": spec.as_payload(), + "files": { + "elastic-inputs.tar.gz": base64.b64encode( + archive.read_bytes() + ).decode("ascii") + }, + "pseudopotentials": [ + { + "relative": "pseudo_PAW_PBEsol/Si.upf", + "sha256": digest, + } + ], + }, + } + expected = [ + f"task_{index:03d}/elastic_inputs/strain_{index:03d}/relax.stdout" + for index in range(2) + ] + qe_output = """ + ! total energy = -10.0 Ry + total stress (Ry/bohr**3) (kbar) P= 0.0 + 0 0 0 1.0 0.0 0.0 + 0 0 0 0.0 1.0 0.0 + 0 0 0 0.0 0.0 1.0 + End of BFGS Geometry Optimization + JOB DONE. +""" + with patch.dict(os.environ, endpoint_env, clear=False): + submitted = rcac_service_gateway(request) + assert submitted["job_id"] == "9002" + runner = (workdir / "run-elastic-task.sh").read_text(encoding="utf-8") + assert "mpirun pw.x -in relax.in" in runner + for relative in expected: + output = workdir / relative + output.parent.mkdir(parents=True) + output.write_text(qe_output, encoding="utf-8") + for index in range(2): + (workdir / f"slurm-9002_{index}.out").write_text("", encoding="utf-8") + (workdir / f"slurm-9002_{index}.err").write_text("", encoding="utf-8") + retained = [ + "elastic-inputs.tar.gz", + *expected, + "slurm-9002_0.out", + "slurm-9002_0.err", + "slurm-9002_1.out", + "slurm-9002_1.err", + ] + collected = rcac_service_gateway( + { + "action": "collect", + "payload": { + "cluster": "gautschi", + "run_id": run_id, + "job_id": "9002", + "tool_name": "elasticdft", + "expected_artifacts": expected, + "retained_artifacts": retained, + }, + } + ) + assert set(collected["parsed_outputs"]) == set(expected) + assert len(collected["artifact_manifest"]) == len(retained) + 2 + assert all( + item["storage_uri"].startswith("depot://") + for item in collected["artifact_manifest"] + ) diff --git a/api/tests/test_rcac_hardening.py b/api/tests/test_rcac_hardening.py new file mode 100644 index 0000000..2e93182 --- /dev/null +++ b/api/tests/test_rcac_hardening.py @@ -0,0 +1,298 @@ +"""Fault/authorization tests: no real cluster, endpoint, or production DB.""" +import base64 +import hashlib +import json +import os +import pwd +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +from unittest.mock import MagicMock, Mock +from uuid import uuid4 + +import pytest +from computations.artifacts import fetch_depot_artifact +from computations.globus_gateway import rcac_service_gateway, GlobusGatewayError +from computations.worker import ComputationWorker +from computations.rcac import configured_clusters +from test_computation_worker import _clusters, FakeGateway +from test_computations_router import ComputationRouterSafetyTests + + +@pytest.fixture +def archive(tmp_path, monkeypatch): + run_id = str(uuid4()) + work = tmp_path / "scratch" / run_id + work.mkdir(parents=True) + (work / ".dt-slurm-job-id").write_text("42") + for key, value in {"DT_CLUSTER_NAME": "negishi", "DT_WORK_ROOT": str(work.parent), + "DT_ARTIFACT_ROOT": str(tmp_path / "archive")}.items(): + monkeypatch.setenv(key, value) + return run_id, work + + +def invoke(action, run_id, **payload): + return rcac_service_gateway({"action": action, "payload": { + "cluster": "negishi", "run_id": run_id, "job_id": "42", **payload}}) + + +def test_terminal_accounting_includes_times_and_array_identity(archive): + run_id, _ = archive + calls = [] + def command(argv, **kwargs): + calls.append(argv) + assert kwargs["env"]["TZ"] == "UTC" + output = f"42_0|COMPLETED|2026-09-23T01:00:00|2026-09-23T01:01:00|0:0|2026-09-23T00:59:00|dt-run:{run_id}\n" + return SimpleNamespace(returncode=0, stdout=output if argv[0] == "sacct" else "", stderr="") + with patch("subprocess.run", side_effect=command): + result = invoke("status_many", run_id, jobs=[{"run_id": run_id, "job_id": "42"}]) + assert result["accounting"]["42"][0]["started_at"] == "2026-09-23T01:00:00+00:00" + assert result["states"]["42"] == ["COMPLETED"] + assert len(calls) == 2 + + +def test_scheduler_comment_mismatch_is_rejected(archive): + run_id, _ = archive + result = SimpleNamespace(returncode=0, stdout="42|COMPLETED|Unknown|Unknown|0:0|Unknown|another-run", stderr="") + with patch("subprocess.run", return_value=result), pytest.raises(RuntimeError, match="attribution"): + invoke("status_many", run_id, jobs=[{"run_id": run_id, "job_id": "42"}]) + + +@pytest.mark.parametrize("array", [False, True]) +def test_live_queue_authenticates_jobs_while_accounting_comment_is_empty(archive, array): + from computations.slurm import aggregate_states, SlurmState + run_id, _ = archive + raw_id = "42_1" if array else "42" + accounting = f"{raw_id}|PENDING|Unknown|Unknown|0:0|Unknown|\n" + if array: + accounting += f"42_0|COMPLETED|Unknown|Unknown|0:0|Unknown|dt-run:{run_id}\n" + queue = f"42|PENDING|dt-run:{run_id}|{pwd.getpwuid(os.getuid()).pw_name}\n" + def command(argv, **kwargs): + return SimpleNamespace(returncode=0, stdout=accounting if argv[0] == "sacct" else queue, stderr="") + with patch("subprocess.run", side_effect=command): + result = invoke("status_many", run_id, jobs=[{"run_id": run_id, "job_id": "42"}]) + assert aggregate_states(result["states"]["42"]) == SlurmState.PENDING + assert all(record["state"] == "COMPLETED" for record in result["accounting"]["42"]) + + +@pytest.mark.parametrize("comment,owner", [("another-run", None), (None, "another-user")]) +def test_live_queue_rejects_conflicting_attribution(archive, comment, owner): + run_id, _ = archive + queue = f"42|RUNNING|{comment or 'dt-run:' + run_id}|{owner or pwd.getpwuid(os.getuid()).pw_name}\n" + def command(argv, **kwargs): + return SimpleNamespace(returncode=0, stdout="" if argv[0] == "sacct" else queue, stderr="") + with patch("subprocess.run", side_effect=command), pytest.raises(RuntimeError, match="attribution"): + invoke("status_many", run_id, jobs=[{"run_id": run_id, "job_id": "42"}]) + + +def test_terminal_record_without_comment_waits_for_attributed_accounting(archive): + run_id, _ = archive + def command(argv, **kwargs): + return SimpleNamespace(returncode=0, stdout="42|COMPLETED|Unknown|Unknown|0:0|Unknown|\n" if argv[0] == "sacct" else "", stderr="") + with patch("subprocess.run", side_effect=command): + result = invoke("status_many", run_id, jobs=[{"run_id": run_id, "job_id": "42"}]) + assert result == {"states": {"42": ["UNKNOWN"]}, "accounting": {"42": []}} + + +def test_partial_failure_archives_logs_before_scientific_parsing(archive): + run_id, work = archive + (work / "slurm.err").write_text("out of memory") + (work / "vcrelax.stdout").write_text("") + result = invoke("collect", run_id, tool_name="cellrelaxdft", expected_artifacts=["vcrelax.stdout"], + retained_artifacts=["slurm.err", "slurm.out", "vcrelax.stdout"]) + assert {a["name"] for a in result["artifact_manifest"]} == {"slurm.err", "vcrelax.stdout"} + assert "slurm.out" in result["missing_artifacts"] + assert result["collection_errors"] + assert result["parsed_outputs"] == {} + + +@pytest.mark.parametrize("name", ["../secret", "/etc/passwd", "credentials.json", "task_000/elastic_inputs/strain_001/relax.stdout"]) +def test_collection_cannot_expand_allowlist_via_expected_outputs(archive, name): + run_id, _ = archive + with pytest.raises(RuntimeError): + invoke("collect", run_id, tool_name="elasticdft", expected_artifacts=[name], retained_artifacts=[name]) + + +def test_small_artifact_roundtrip_and_integrity_after_scratch_is_gone(archive, tmp_path, monkeypatch): + run_id, work = archive + data = b"diagnostic\n" * 30000 + (work / "slurm.err").write_bytes(data) + collected = invoke("collect", run_id, tool_name="cellrelaxdft", expected_artifacts=["vcrelax.stdout"], + retained_artifacts=["slurm.err"], diagnostics_only=True) + artifact = collected["artifact_manifest"][0] + cluster = SimpleNamespace(artifact_root=str(tmp_path / "archive"), name="negishi", endpoint_id="endpoint", gateway_function_id="gateway") + class LocalGateway: + def invoke(self, **kwargs): + return "task", rcac_service_gateway({"action": kwargs["action"], "payload": kwargs["payload"]}) + run = {"id": run_id, "scheduler_job_id": "42"} + (work / ".dt-slurm-job-id").unlink() + with fetch_depot_artifact(run, artifact, cluster, LocalGateway()) as handle: + assert handle.read() == data + (tmp_path / "archive" / run_id / "slurm.err").write_bytes(b"tampered") + with pytest.raises(RuntimeError, match="checksum"): + fetch_depot_artifact(run, artifact, cluster, LocalGateway()) + + +def test_artifact_symlink_escape_is_rejected(archive, tmp_path): + run_id, _ = archive + root = tmp_path / "archive" / run_id + root.mkdir(parents=True) + secret = tmp_path / "secret" + secret.write_text("private") + (root / "slurm.err").symlink_to(secret) + with pytest.raises(RuntimeError, match="unavailable"): + invoke("read_artifact", run_id, name="slurm.err", size_bytes=7, sha256="a" * 64, offset=0, length=7) + + +def test_cross_project_download_denied_before_endpoint_call(): + client = ComputationRouterSafetyTests.client() + with patch("routers.computations.feature_enabled", return_value=True), \ + patch("routers.computations.get_run", side_effect=PermissionError("denied")), \ + patch("routers.computations.GlobusComputeGateway") as gateway: + assert client.get("/api/computations/runs/123/artifacts/0").status_code == 403 + gateway.assert_not_called() + + +@pytest.mark.parametrize("invalid", ["uri", "oversize", "checksum"]) +def test_invalid_manifest_rejected_without_contacting_endpoint(invalid): + run_id = str(uuid4()) + artifact = {"name": "slurm.err", "size_bytes": 1, "sha256": "a" * 64, + "storage_uri": f"depot:///archive/{run_id}/slurm.err"} + if invalid == "uri": artifact["storage_uri"] = "depot:///another-project/slurm.err" + if invalid == "oversize": artifact["size_bytes"] = 8 * 1024 * 1024 + 1 + if invalid == "checksum": artifact["sha256"] = "invalid" + with pytest.raises(ValueError): + fetch_depot_artifact({"id": run_id}, artifact, SimpleNamespace(artifact_root="/archive"), None) + + +def test_short_job_gets_scheduler_start_even_without_running_poll(monkeypatch): + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters()) + run = {"id": str(uuid4()), "cluster": "negishi", "status": "submitted", "scheduler_job_id": "42"} + worker = ComputationWorker(gateway=FakeGateway()) + result = {"states": {"42": ["COMPLETED"]}, "accounting": {"42": [{ + "started_at": "2026-09-23T01:00:00+00:00", "ended_at": "2026-09-23T01:01:00+00:00"}]}} + with patch.object(worker, "_invoke", return_value=("task", result)), patch("computations.worker.update_run") as update: + worker._advance_cluster("negishi", [run]) + values = update.call_args.kwargs["values"] + assert values["started_at"].isoformat() == "2026-09-23T01:00:00+00:00" + assert values["external_workflow_ref_json"]["scheduler_ended_at"] == "2026-09-23T01:01:00+00:00" + assert update.call_args.kwargs["status"] == "collecting" + + +def test_cancel_waits_for_terminal_accounting_and_keeps_failure_logs(monkeypatch, tmp_path): + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters()) + run = {"id": str(uuid4()), "cluster": "negishi", "status": "cancelling", "cancel_requested": True, + "scheduler_job_id": "42", "external_workflow_ref_json": {}, "input_json": {}, + "prepared_manifest_json": {"tool_name": "cellrelaxdft", "expected_artifacts": ["vcrelax.stdout"], "ntasks": 64, "time_limit": "24:00:00"}} + def update(_, status=None, values=None, **kwargs): + if status: run["status"] = status + run.update(values or {}) + gateway = FakeGateway() + worker = ComputationWorker(gateway=gateway) + worker.work_root = tmp_path + with patch("computations.worker.update_run", side_effect=update): + worker._cancel(run) + assert run["status"] == "cancelling" + with patch.object(worker, "_invoke", return_value=("task", {"states": {"42": ["CANCELLED"]}})): + worker._cancel(run) + assert run["status"] == "collecting" + assert run["cancel_requested"] is False + # A fresh process can restore the manifest without staged local files. + worker = ComputationWorker(gateway=gateway) + worker.work_root = tmp_path + worker._collect(run) + assert run["status"] == "cancelled" + assert run["artifact_manifest_json"] + assert "output_json" not in run + assert [c["action"] for c in gateway.calls] == ["cancel", "collect"] + + +@pytest.mark.parametrize("status,cancel_requested", [ + ("collecting", False), ("cancelling", True), ("submitted", True), ("cancelled", False), +]) +def test_repeated_cancellation_does_not_restart_collection(status, cancel_requested): + from computations.store import request_cancel + row = {"id": str(uuid4()), "project_id": "pilot", "status": status, + "cancel_requested": cancel_requested} + conn = MagicMock() + cur = conn.cursor.return_value.__enter__.return_value + cur.fetchone.return_value = row + user = object() + with patch("computations.store.get_pg_worker_connection", return_value=conn), \ + patch("computations.store._ensure_project_write_access") as authorize, \ + patch("computations.store._insert_event") as event: + result = request_cancel(row["id"], user) + authorize.assert_called_once_with(cur, project_id="pilot", user=user) + assert result["status"] == status + assert cur.execute.call_count == 1 # row lock/read only; no backward transition + event.assert_not_called() + conn.close.assert_called_once() + + +def test_endpoint_auth_failure_is_not_retried_or_exposed(): + from test_rcac_gateway import FakeClient + from computations.globus_gateway import GlobusComputeGateway + client = FakeClient({}) + with patch.object(client, "run", side_effect=RuntimeError("secret-token-value")) as submit: + with pytest.raises(GlobusGatewayError) as caught: + GlobusComputeGateway(client=client).invoke(endpoint_id=str(uuid4()), function_id=str(uuid4()), action="submit", payload={}) + assert submit.call_count == 1 + assert "secret-token-value" not in str(caught.value) + + +def test_stolen_batch_claim_is_not_executed(): + with patch("computations.worker.feature_enabled", return_value=True), \ + patch("computations.worker.claim_active_runs", return_value=[{"id": str(uuid4())}]), \ + patch("computations.worker.renew_run_claim", return_value=None), \ + patch.object(ComputationWorker, "_submit") as submit: + assert ComputationWorker().run_cycle() + submit.assert_not_called() + + +def test_unknown_status_uses_bounded_transport_recovery(monkeypatch): + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters()) + worker = ComputationWorker() + run = {"id": str(uuid4()), "scheduler_job_id": "42", "cluster": "negishi"} + with patch.object(worker, "_invoke", return_value=("task", {"states": {"42": ["UNKNOWN"]}})), \ + patch.object(worker, "_handle_error") as handle: + worker._advance_cluster("negishi", [run]) + from computations.globus_gateway import GlobusTransientError + assert isinstance(handle.call_args.args[1], GlobusTransientError) + + +def test_unknown_submission_is_not_falsely_marked_cancelled(monkeypatch): + monkeypatch.setenv("RCAC_CLUSTERS_JSON", _clusters()) + worker = ComputationWorker() + run = {"id": str(uuid4()), "cluster": "negishi", "external_workflow_ref_json": {"submission_requested": True}} + from computations.globus_gateway import GlobusTransientError + with patch.object(worker, "_invoke", return_value=("task", {"submission_outcome": "unknown"})), \ + patch("computations.worker.update_run") as update: + with pytest.raises(GlobusTransientError): + worker._cancel(run) + update.assert_not_called() + + +def test_recover_submission_only_reads_marker(archive): + run_id, work = archive + assert invoke("recover_submission", run_id)["job_id"] == "42" + (work / ".dt-slurm-job-id").unlink() + assert invoke("recover_submission", run_id) == {"submission_outcome": "unknown"} + + +def test_submission_budget_is_atomic_and_project_restricted(monkeypatch): + from computations.store import _enforce_submission_budget, SubmissionCapacityError + monkeypatch.setenv("RCAC_ALLOWED_PROJECT_IDS", "pilot") + monkeypatch.setenv("RCAC_MAX_ACTIVE_RUNS_PER_USER", "2") + monkeypatch.setenv("RCAC_MAX_ACTIVE_RUNS_PER_CLUSTER", "1") + cursor = Mock() + with pytest.raises(PermissionError): + _enforce_submission_budget(cursor, user_id="student", project_id="production", cluster_name="negishi") + cursor.execute.assert_not_called() + cursor.fetchone.return_value = {"user_count": 0, "cluster_count": 1} + with pytest.raises(SubmissionCapacityError): + _enforce_submission_budget(cursor, user_id="student", project_id="pilot", cluster_name="negishi") + assert "pg_advisory_xact_lock" in cursor.execute.call_args_list[0].args[0] + cursor.fetchone.return_value = {"user_count": 0, "cluster_count": 0} + _enforce_submission_budget(cursor, user_id="student", project_id="pilot", cluster_name="negishi") diff --git a/api/tests/test_row_level_security_pg.py b/api/tests/test_row_level_security_pg.py index e014286..4e26017 100644 --- a/api/tests/test_row_level_security_pg.py +++ b/api/tests/test_row_level_security_pg.py @@ -65,6 +65,21 @@ def _user(user_id: str, role: str, org: str) -> PlatformUser: @unittest.skipUnless(TEST_PORT, "set DT_RLS_TEST_PG_PORT to run RLS integration tests") class RowLevelSecurityIntegrationTests(unittest.TestCase): + def setUp(self): + # These tests cover access rules and proposal-job mechanics with a + # faked optimizer. Which equipment the etch-rate model supports is + # covered in test_proposal_objectives, so resolve any request here. + resolver = patch( + "metadata_pg.resolve_optimization_request", + side_effect=lambda **kwargs: { + "constraints": {}, + "domain_config": {}, + "objectives": None, + }, + ) + resolver.start() + self.addCleanup(resolver.stop) + @classmethod def setUpClass(cls): import data_loader_pg @@ -117,6 +132,64 @@ def setUpClass(cls): finally: conn.close() + # ── physical vs computational (scripts/add_asset_kinds.sql) ────────── + + def test_templates_take_the_kind_of_their_equipment(self): + import data_loader_pg + from metadata_pg import create_experiment_type_pg + + qe_id = f"qe_rls_{uuid.uuid4().hex[:6]}" + conn = data_loader_pg.get_pg_migration_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO equipment_metadata ( + domain_id, equipment_name, owner_id, owner_org, status, asset_kind + ) + VALUES (%s, 'QE on RLS', %s, '', 'approved', 'computational') + """, + (qe_id, self.olivia.id), + ) + conn.commit() + finally: + conn.close() + + # A user-made template (for example a fork) on a QE installation is + # computational even though nothing in the payload says so. + qe_type = create_experiment_type_pg( + payload={"type_name": "forked relax", "equipment_id": qe_id}, + user=self.alice, + )["type_id"] + physical_type = self._create_template(self.alice) + + conn = data_loader_pg.get_pg_migration_connection() + try: + with conn.cursor() as cur: + cur.execute( + "SELECT id, execution_kind FROM experiment_types WHERE id IN (%s, %s)", + (qe_type, physical_type), + ) + kinds = dict(cur.fetchall()) + self.assertEqual(kinds, {qe_type: "computational", physical_type: "physical"}) + + # The kind is derived, so it cannot be overwritten directly. + cur.execute( + "UPDATE experiment_types SET execution_kind = 'physical' WHERE id = %s", + (qe_type,), + ) + cur.execute("SELECT execution_kind FROM experiment_types WHERE id = %s", (qe_type,)) + self.assertEqual(cur.fetchone()[0], "computational") + + with self.assertRaises(psycopg2.errors.CheckViolation): + cur.execute( + "UPDATE equipment_metadata SET asset_kind = 'digital' WHERE domain_id = %s", + (qe_id,), + ) + conn.rollback() + finally: + conn.close() + # ── helpers ────────────────────────────────────────────────────────── def _create_project(self, owner: PlatformUser, access: str = "private") -> str: @@ -167,7 +240,9 @@ def _create_experiment( "selected_equipment": self.equipment_id, "project_id": project_id, "type_id": type_id, - "planned_parameters": [{"name": "pressure", "planned_value": "10"}], + # An optimization campaign searches a Variable input's range, so + # it takes no point value for it. + "planned_parameters": [] if generate else [{"name": "pressure", "planned_value": "10"}], "optimization_context": {"mode": "optimization" if generate else "none"}, "generate_proposals": generate, "client_request_id": client_request_id, diff --git a/geddes/k8s/09-rcac-worker.example.yaml b/geddes/k8s/09-rcac-worker.example.yaml new file mode 100644 index 0000000..2d5d6f5 --- /dev/null +++ b/geddes/k8s/09-rcac-worker.example.yaml @@ -0,0 +1,144 @@ +# REVIEW-ONLY EXAMPLE — do not apply until the RCAC service account, Globus +# confidential client, and both single-user endpoints have been provisioned. +# +# This Deployment is intentionally isolated from the existing DT workloads. +# It creates no per-user Secrets and needs no Kubernetes API permissions. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dt-computation-worker + namespace: ncn-digitaltwins-zchen-sim2l +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: dt-computation-config + namespace: ncn-digitaltwins-zchen-sim2l +data: + # Leave false until every prerequisite and the deployment itself are approved. + RCAC_COMPUTE_ENABLED: "false" + RCAC_SERVICE_ACCOUNT_USERNAME: REPLACE_WITH_APPROVED_SERVICE_ACCOUNT + GLOBUS_COMPUTE_CLIENT_ID: 2fea1cb8-521c-4f61-9b0b-5d7226389f66 + GLOBUS_COMPUTE_GATEWAY_FUNCTION_ID: REPLACE_WITH_REGISTERED_FUNCTION_UUID + RCAC_LOCAL_WORK_ROOT: /var/lib/dt-computation-worker/runs + RCAC_CLUSTERS_JSON: >- + {"negishi":{"display_name":"Negishi","host":"negishi.rcac.purdue.edu","endpoint_id":"REPLACE_WITH_NEGISHI_ENDPOINT_UUID","qe_version":"7.1","module_dependencies":["gcc/12.2.0","openmpi/4.1.4"],"qe_module":"quantum-espresso/7.1","mpi_launcher":"srun","allocation":"ncn","partition":"cpu","qos":"normal","remote_root":"/scratch/negishi/REPLACE_WITH_SERVICE_ACCOUNT/digital-twins/runs","pseudo_root":"/depot/ncn/data/digital-twins/pseudopotentials","artifact_root":"/depot/ncn/data/digital-twins/runs"},"gautschi":{"display_name":"Gautschi","host":"gautschi.rcac.purdue.edu","endpoint_id":"REPLACE_WITH_GAUTSCHI_ENDPOINT_UUID","qe_version":"7.3.1","module_dependencies":["gcc/11.4.1","openmpi/4.1.6"],"qe_module":"quantum-espresso/7.3.1","mpi_launcher":"mpirun","allocation":"ncn","partition":"cpu","qos":"normal","remote_root":"/scratch/gautschi/REPLACE_WITH_SERVICE_ACCOUNT/digital-twins/runs","pseudo_root":"/depot/ncn/data/digital-twins/pseudopotentials","artifact_root":"/depot/ncn/data/digital-twins/runs"}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dt-computation-worker + namespace: ncn-digitaltwins-zchen-sim2l + labels: + app: dt-computation-worker +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: dt-computation-worker + template: + metadata: + labels: + app: dt-computation-worker + spec: + serviceAccountName: dt-computation-worker + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: worker + image: geddes-registry.rcac.purdue.edu/sdx/dt-computation-worker@sha256:13f977ced56cf95a52d3f78ff3a6c5d34cef83baff0abdce5bf6f14b16863f27 + imagePullPolicy: IfNotPresent + args: ["python", "-m", "computations.worker", "--poll-seconds", "30"] + envFrom: + - secretRef: + name: dt-computation-database + - configMapRef: + name: dt-computation-config + env: + # Background role (not superuser, not BYPASSRLS); the row-level + # security script grants it the computation-table policies. The + # worker receives no migration (table-owner) credentials. + - name: PG_WORKER_USER + value: api_worker + - name: PG_WORKER_PASS + valueFrom: + secretKeyRef: + name: dt-computation-database + key: PG_WORKER_PASS + - name: GLOBUS_COMPUTE_CLIENT_SECRET_FILE + value: /var/run/secrets/globus/client-secret + - name: MATERIALS_PROJECT_API_KEY + valueFrom: + secretKeyRef: + name: dt-computation-materials-project + key: api-key + volumeMounts: + - name: globus-client-secret + mountPath: /var/run/secrets/globus + readOnly: true + - name: work + mountPath: /var/lib/dt-computation-worker + - name: temporary + mountPath: /tmp + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + volumes: + - name: globus-client-secret + secret: + secretName: dt-globus-confidential-client + items: + - key: client-secret + path: client-secret + mode: 0400 + - name: work + emptyDir: + sizeLimit: 8Gi + # Globus stores its token cache below HOME=/tmp, and scientific Python + # libraries need temporary files with the root filesystem read-only. + - name: temporary + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: sdx-registry-secret +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dt-computation-worker-deny-ingress + namespace: ncn-digitaltwins-zchen-sim2l +spec: + podSelector: + matchLabels: + app: dt-computation-worker + policyTypes: ["Ingress"] + ingress: [] + +# Required Secret keys (values are never committed): +# dt-globus-confidential-client/client-secret +# dt-computation-materials-project/api-key +# dt-computation-database: the existing worker DB connection variables only +# +# The API Deployment also needs the non-secret RCAC service-account/cluster +# values when the feature is eventually enabled, but it does not need either +# external API secret. Keep RCAC_COMPUTE_ENABLED=false everywhere until the +# readiness checklist has been approved. diff --git a/geddes/k8s/09a-rcac-worker-bootstrap.yaml b/geddes/k8s/09a-rcac-worker-bootstrap.yaml new file mode 100644 index 0000000..6633a89 --- /dev/null +++ b/geddes/k8s/09a-rcac-worker-bootstrap.yaml @@ -0,0 +1,23 @@ +# Safe bootstrap resources for the isolated RCAC computation namespace. +# These resources contain no credentials and do not start a workload. +# Apply the full 09-rcac-worker manifest only after the RCAC service account, +# Globus client/function, and both endpoint UUIDs have been provisioned. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dt-computation-worker + namespace: ncn-digitaltwins-zchen-sim2l +automountServiceAccountToken: false +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dt-computation-worker-deny-ingress + namespace: ncn-digitaltwins-zchen-sim2l +spec: + podSelector: + matchLabels: + app: dt-computation-worker + policyTypes: ["Ingress"] + ingress: [] diff --git a/geddes/k8s/pilots/computations/Dockerfile.api b/geddes/k8s/pilots/computations/Dockerfile.api new file mode 100644 index 0000000..9d5a1c5 --- /dev/null +++ b/geddes/k8s/pilots/computations/Dockerfile.api @@ -0,0 +1,13 @@ +# Login-validation pilot: no production datasets, domain fixtures or credentials. +# Build from repository root, not this directory. +FROM python:3.12-slim +WORKDIR /app +COPY api/requirements.txt ./requirements.txt +RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir globus-compute-sdk==3.16.1 +COPY api/ ./ +COPY azure/common/ /app/common/ +COPY azure/process_etcher_data/ /app/legacy_processors/process_etcher_data/ +RUN mkdir -p /app/data /app/domains/configs +ENV PYTHONDONTWRITEBYTECODE=1 +CMD ["uvicorn", "main:app", "--host", "127.0.0.1", "--port", "8000", "--workers", "1"] diff --git a/geddes/k8s/pilots/computations/ELASTICDFT_CLAUDE_REVIEW_2026-10-06.md b/geddes/k8s/pilots/computations/ELASTICDFT_CLAUDE_REVIEW_2026-10-06.md new file mode 100644 index 0000000..caf4104 --- /dev/null +++ b/geddes/k8s/pilots/computations/ELASTICDFT_CLAUDE_REVIEW_2026-10-06.md @@ -0,0 +1,58 @@ +# ElasticDFT diagnosis and accepted Claude implementation + +Verified October 7, 2026, after rollout at 14:35 UTC (10:35 America/Indiana/Indianapolis). + +**Claude's reviewed implementation is deployed and verified on the private pilot.** Both endpoints use the new gateway; all four containers are healthy. Resource controls, inherited parent settings, per-member diagnostics and FAQ updates are live. No scientific jobs were submitted or rerun. Refresh the project page to load the update. + +## Confirmed cause + +Run `47548c30-99b1-416c-b757-6931e047aece`, parent `931dcf71-0319-4544-850c-f5566aac09f3`, Negishi Slurm job `43864570`. + +All 24 archived QE outputs matched their recorded SHA-256. The 21 failed members explicitly report BFGS failure and exit `3:0`; members 8, 11 and 13 converged. Failed members stopped after 4–10 BFGS steps, below the 100-step limit. Inputs used energy `1e-10` Ry and force `1e-5` Ry/bohr thresholds. These observations support a convergence failure; there is no evidence of an out-of-memory failure. No thresholds were silently changed and no elastic result is claimed. + +[Detailed findings](elastic-review-2026-10-06/ACTUAL_FAILURE_FINDINGS.md) and the archived-output replay preserve the evidence. The historical run now includes the evidence-based diagnosis and per-member records in the live database and API exports. + +## Review agreement + +Real Claude Opus 5.5 with `xhigh` effort authored the production changes in an isolated copy. Codex reviewed and tested each revision and returned concrete regressions. Rounds 2 and 3 corrected public serialization, archive evidence validation and confinement, preservation of previous diagnostics, and accounting labels. Claude's final round-4 audit found no remaining blocker; Codex agrees. The assessment is in [round-004.result.json](elastic-review-2026-10-06/review-queue/round-004.result.json). + +The accepted change provides bounded ranks, walltime, memory and elastic concurrency controls; separate PostgreSQL JSONB resource persistence; API/worker/gateway validation; read-only inherited parent settings; per-member diagnostics and exports; checksum-verified historical enrichment; and student FAQ/handoff updates. One node remains fixed by pilot policy. + +All 19 accepted source/test files match their recorded hashes after transfer; the handoff was transferred separately. Existing unrelated changes were preserved. No branch commit or push was made. See the [accepted manifest](elastic-review-2026-10-06/accepted-source-manifest.json), [patch](elastic-review-2026-10-06/claude-reviewed.patch), and [implementation notes](elastic-review-2026-10-06/CLAUDE_FINAL_IMPLEMENTATION_NOTES.md). + +## Validation + +- Final revision: **242 Python tests and 5 subtests passed**. +- Final revision: **5 PostgreSQL integration tests passed**, using the pilot's Linux runtime and a disposable database. Coverage includes resource persistence/worker submission, cache behavior, enrichment and repeated retrieval/export, idempotence, and unchanged scientific data/status/timestamps. +- TypeScript, targeted ESLint and Next.js production builds passed. +- The actual React form passed Chromium checks for relaxation and ElasticDFT payloads, invalid ranks, legacy disabled fields, inherited settings, diagnostic links and a 390-pixel viewport without horizontal overflow. API responses were fixtures; this does not establish a new live nanoHUB login or successful scientific run. +- The exact archived failed QE output is rejected as unconverged and diagnosed as a BFGS failure. + +Logs and browser evidence are under [elastic-review-2026-10-06](elastic-review-2026-10-06). Earlier revision-1 database/browser blockers were resolved with isolated Docker test environments. + +## Verified live rollout + +New gateway function: `8448efe7-5697-4dc5-86bb-c9ab9ce8b7bc`. +Gateway SHA-256: `4bb305d8aaf82eb50589d14512353efd35b8d0925bef0c173a00ca6527bbd131`. + +Both endpoints passed [capability checks](elastic-review-2026-10-06/live-gateway-capabilities.json): protocol 2, one node, 64-rank ceiling, four-member concurrency ceiling, adjustable walltime, maximum 2010 MB/CPU, diagnostics and archive evidence enabled. The CPU partitions' default remains 1917 MB/CPU. Live API policies match those limits. + +The web, API and worker images in [rollout-status.json](elastic-review-2026-10-06/rollout-status.json) are deployed by immutable digest. API and worker preserve the previous runtime dependencies. The PostgreSQL image and persistent volumes are unchanged. Pod `dt-computation-pilot-5cdccfc498-z7d6t` is 4/4 ready with zero restarts. All 19 accepted source hashes still match the reviewed manifest. + +[Live API/database verification](elastic-review-2026-10-06/live-release-verification.json) passed: + +- All 72 historical evidence files verified; no files unavailable. Twenty-one members have `qe_bfgs_failed`, three completed. +- Original status, timestamps, inputs, outputs, fingerprint, manifests and external scheduler data are unchanged. The original error remains in enrichment provenance. +- Exactly one audit event was added (7 to 8). Repeating enrichment returned `unchanged`, with the event count still 8. +- Repeated API retrieval, diagnostics download and full result export retain the diagnosis. A downloaded raw QE output matched its recorded SHA-256 and contained the BFGS failure. +- The new `resource_request_json` column is non-null JSONB. All 19 original runs remain (15 ingested, 3 cancelled, 1 failed). +- `ngholiza` remains Administrator/project PI; `desai224` and `jam19` remain Researchers/project members. Anonymous run access is denied (403). +- The HTTPS ingress is reachable from the deployed API; an unauthenticated FAQ request redirects to login. No student identity or nanoHUB session was fabricated for verification. + +The final [Chromium check with deployed API data](elastic-review-2026-10-06/live-data-browser-result.json) passed: actual form controls submit the intended payload to intercepted fixture routes, inherited settings are visible, BFGS findings/exit codes and diagnostics links render, and the 390-pixel viewport has no horizontal overflow or JavaScript errors. This is a component check using a live-data snapshot, not a new authenticated end-to-end student session. Desktop and mobile screenshots are adjacent to that result. + +## Endpoint restart correction + +The first terminal script updated Negishi's files but stopped at restart because the cluster DNS alias selected a different login host. Both old endpoints remained online. The corrected operational script verifies the actual process identity before writing and connects to `login03.negishi.rcac.purdue.edu` and `login01.gautschi.rcac.purdue.edu`, using the existing trusted cluster host keys. The user's second terminal run completed both restarts. Its backups and log are preserved in `endpoint-activation-20261007T143232Z.log`; the earlier failed attempt is retained for audit. No approval blocker remains for this rollout. + +The task-owned disposable PostgreSQL test container is stopped. Existing development containers were not changed. The source branch remains uncommitted, with unrelated user edits preserved. Numerical convergence and scientific validity still require the students' review; this rollout does not claim a successful new simulation or validate elastic constants. diff --git a/geddes/k8s/pilots/computations/LOGIN_VALIDATION_2026-09-28.md b/geddes/k8s/pilots/computations/LOGIN_VALIDATION_2026-09-28.md new file mode 100644 index 0000000..f045445 --- /dev/null +++ b/geddes/k8s/pilots/computations/LOGIN_VALIDATION_2026-09-28.md @@ -0,0 +1,82 @@ +# Isolated pilot login validation — September 28, 2026 + +Historical login-stage record. The later compute deployment, completed gates, +and current student access are tracked in +[PILOT_VALIDATION_2026-09-28.md](PILOT_VALIDATION_2026-09-28.md). + +Application source: `feature/sim2l-rcac-integration`, `db5d72a`. +Deployment helpers added locally; no merge to dev or production rollout. +Namespace: `ncn-digitaltwins-zchen-sim2l`. +URL: https://dt-sim2l-pilot.geddes.rcac.purdue.edu/login + +## Deployed + +- One `dt-computation-pilot` Deployment (Recreate), three non-root containers: + web, API, PostgreSQL. No worker, no RCAC submission credentials. +- New `dt-computation-pilot-data` 5 GiB PVC; existing historical DB not reused. +- Independent `dt-computation-pilot-runtime` credentials, reused on rerun. + Existing `dt-computation-pilot-saml` signing credentials preserved unchanged. +- New DB `dt_computation_pilot`, private project `RCAC-PILOT`, zero memberships. +- API and PostgreSQL bind to loopback; Service exposes only port 3000. +- `api_client` and `dt_pilot_owner`: no superuser, create-role, create-database + or bypass-RLS privileges. Owner role performs platform schema migrations. +- No production resource updates or scientific Slurm jobs in this rollout. + +Pinned images: + +```text +geddes-registry.rcac.purdue.edu/sdx/dt-web@sha256:90970eabf7462d58d8d3ba1958a4e930b97baedd588446a9793af766a78e15d5 +geddes-registry.rcac.purdue.edu/sdx/dt-api@sha256:2636c963ec7b0cc78f9801d9538e2797c35719118827b5f77b4427af6abf89b2 +postgres@sha256:efedf3595f1d6f415c08568ba171029bf54052e754cc9f030e3f2412b21f3d67 +``` + +## Verified + +- Fresh Next.js production image build, including TypeScript, passed. +- Pilot-specific API image build passed. Standard API Dockerfile's local CSV + dependency was not copied or fabricated for this pilot. +- Kubernetes server-side dry run passed; all three containers became Ready. +- API restarted once while its new database initialized, then stayed healthy. +- API `/api/health`: HTTP 200, `api_client` on `dt_computation_pilot`, no missing + required tables. No physical CSV or production SQLite dataset present. +- HTTPS certificate verification passed. `/login`: 200; `/`: login redirect; + anonymous `/api/dt/projects`: 401. +- Live SAML metadata SP certificate matches the public registration attachment. +- Login emits an RSA-SHA256 signed request, independently verified against that + certificate; issuer and ACS refer only to the pilot hostname. +- nanoHUB returns HTTP 302 to `https://nanohub.org/login` for that request. +- ACS GET: 405; invalid assertion POST: login-error redirect, no session cookie. +- Resource usage reservation: one Pod, 2304 MiB memory limits, 3 CPU limits, + 5 GiB storage, within namespace quota. + +Public checks can be repeated without credentials: + +```sh +node geddes/k8s/pilots/computations/check_login.mjs \ + /Users/ngholiza/Mxene/DT-computational-pilot-SAML-registration/sp-certificate.pem +``` + +## Still pending + +- Logout and a second real login. Initial real sign-in is confirmed below. +- Match Justin/Rushik's nanoHUB-returned identities to the supplied emails before + granting private-project membership. No guessed identities or admin grants. +- Worker deployment, new hardened gateway registration, authenticated workflow + submission/cancellation/download tests, persistent API upload/config storage, + dependency security review and scientific acceptance remain separate gates. + +Do not describe this as ready for students to run simulations yet. No SAML +signature, issuer, audience or request-correlation validation was relaxed. + +## Real sign-in confirmed + +Navid reported returning from nanoHUB to the pilot. Read-only verification found +the canonical user `ngholiza` with the expected Purdue email, active researcher +role, and last activity at 2026-09-28 14:25:46 UTC. Web logs show authenticated +API forwarding under that identity. This confirms the real sign-in path, beyond +the earlier unauthenticated redirect checks. The earlier invalid-XML log entry +was produced by the deliberate negative assertion test, not this login. + +No additional nanoHUB-to-pilot network opening was made. Project membership +remains empty and cluster submission remains disabled. This result does not +establish student membership, logout, or computational workflow readiness. diff --git a/geddes/k8s/pilots/computations/PILOT_EVIDENCE_2026-09-28.json b/geddes/k8s/pilots/computations/PILOT_EVIDENCE_2026-09-28.json new file mode 100644 index 0000000..9d1f939 --- /dev/null +++ b/geddes/k8s/pilots/computations/PILOT_EVIDENCE_2026-09-28.json @@ -0,0 +1,839 @@ +{ + "generated_at_utc": "2026-09-29T02:10:08.440339+00:00", + "checks": { + "download:7318d304-4505-42af-8f13-27f3c9ab590c": "sha256 matched", + "isolation:7318d304-4505-42af-8f13-27f3c9ab590c": "outsider denied", + "download:c75d97bf-2f1a-4f9f-b5f7-271ced2dd002": "sha256 matched", + "isolation:c75d97bf-2f1a-4f9f-b5f7-271ced2dd002": "outsider denied", + "relax": "passed", + "download:ff79c453-47f3-4422-9a96-b80112ee551b": "sha256 matched", + "isolation:ff79c453-47f3-4422-9a96-b80112ee551b": "outsider denied", + "download:ba53a97f-b743-4362-8477-dfb0553b6ebb": "sha256 matched", + "isolation:ba53a97f-b743-4362-8477-dfb0553b6ebb": "outsider denied", + "elastic": "passed", + "capacity": "third simultaneous submission rejected with 429", + "download:4aa6848d-0c00-40d4-9070-f07c3e921a03": "sha256 matched", + "isolation:4aa6848d-0c00-40d4-9070-f07c3e921a03": "outsider denied", + "download:c59b7b4e-b101-48c8-9740-a779d722665d": "sha256 matched", + "isolation:c59b7b4e-b101-48c8-9740-a779d722665d": "outsider denied", + "relax-fresh": "passed", + "download:ddbe609d-545c-4544-a204-21e94fe01b62": "sha256 matched", + "isolation:ddbe609d-545c-4544-a204-21e94fe01b62": "outsider denied", + "download:ad61e7cb-3c36-4df0-9b15-2b2c6a507058": "sha256 matched", + "isolation:ad61e7cb-3c36-4df0-9b15-2b2c6a507058": "outsider denied", + "2d-parent-rejected": "passed", + "2d": "passed", + "download:ad727315-73b2-4612-8614-609c742c0a8d": "sha256 matched", + "isolation:ad727315-73b2-4612-8614-609c742c0a8d": "outsider denied", + "download:e8cf97bc-0326-4eeb-84a8-3963b8725420": "sha256 matched", + "isolation:e8cf97bc-0326-4eeb-84a8-3963b8725420": "outsider denied", + "cache": "passed", + "cancel": "passed", + "cancel-idempotent:7a44e1b4-d493-4495-9d56-3776e8831199": "collection continued", + "cancel-idempotent:e2542dd7-94d9-4abf-932b-305ca9fea527": "collection continued", + "download:7a44e1b4-d493-4495-9d56-3776e8831199": "sha256 matched", + "isolation:7a44e1b4-d493-4495-9d56-3776e8831199": "outsider denied", + "download:e2542dd7-94d9-4abf-932b-305ca9fea527": "sha256 matched", + "isolation:e2542dd7-94d9-4abf-932b-305ca9fea527": "outsider denied", + "elastic-fresh": "passed", + "member:desai224": "project and results accessible; researcher/member roles", + "member:jam19": "project and results accessible; researcher/member roles", + "outsider-project-hidden": "passed", + "cancelled-artifact-downloads": "Negishi and Gautschi SHA-256 matched", + "final-api-health": "passed", + "final-image-data-exclusion": "API and worker /app/data and /app/local_data empty", + "real-login-logout": "three canonical SAML accounts; Navid confirmed logout and re-login" + }, + "runs": { + "relax:negishi": { + "id": "7318d304-4505-42af-8f13-27f3c9ab590c", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "negishi", + "status": "data_ingested", + "scheduler_job_id": "43537557", + "started_at": "2026-09-28T20:36:54+00:00", + "completed_at": "2026-09-28T20:40:18.078427+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [ + "7.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "43537557", + "ended_at": "2026-09-28T20:37:50+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:36:54+00:00", + "submitted_at": "2026-09-28T20:36:54+00:00" + } + ] + }, + "relax:gautschi": { + "id": "c75d97bf-2f1a-4f9f-b5f7-271ced2dd002", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "gautschi", + "status": "data_ingested", + "scheduler_job_id": "16748345", + "started_at": "2026-09-28T20:37:44+00:00", + "completed_at": "2026-09-28T20:40:19.509825+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [ + "7.3.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "16748345", + "ended_at": "2026-09-28T20:37:52+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:37:44+00:00", + "submitted_at": "2026-09-28T20:37:01+00:00" + } + ] + }, + "elastic:negishi": { + "id": "ff79c453-47f3-4422-9a96-b80112ee551b", + "tool_name": "elasticdft", + "tool_revision": "r27-dt2", + "cluster": "negishi", + "status": "data_ingested", + "scheduler_job_id": "43537708", + "started_at": "2026-09-28T20:41:22+00:00", + "completed_at": "2026-09-28T20:51:23.721954+00:00", + "upstream_run_id": "7318d304-4505-42af-8f13-27f3c9ab590c", + "artifact_count": 51, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 12, + "qe_versions": [ + "7.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "43537708_0", + "ended_at": "2026-09-28T20:41:35+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:22+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_1", + "ended_at": "2026-09-28T20:41:35+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:22+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_2", + "ended_at": "2026-09-28T20:41:32+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:22+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_3", + "ended_at": "2026-09-28T20:41:35+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:22+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_4", + "ended_at": "2026-09-28T20:41:50+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:35+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_5", + "ended_at": "2026-09-28T20:41:50+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:35+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_6", + "ended_at": "2026-09-28T20:41:47+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:35+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_7", + "ended_at": "2026-09-28T20:41:52+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:35+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_8", + "ended_at": "2026-09-28T20:42:08+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:48+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_9", + "ended_at": "2026-09-28T20:42:18+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:52+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_10", + "ended_at": "2026-09-28T20:42:16+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:52+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43537708_11", + "ended_at": "2026-09-28T20:42:10+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:52+00:00", + "submitted_at": "2026-09-28T20:41:20+00:00" + } + ] + }, + "elastic:gautschi": { + "id": "ba53a97f-b743-4362-8477-dfb0553b6ebb", + "tool_name": "elasticdft", + "tool_revision": "r27-dt2", + "cluster": "gautschi", + "status": "data_ingested", + "scheduler_job_id": "16748361", + "started_at": "2026-09-28T20:41:45+00:00", + "completed_at": "2026-09-28T20:51:26.245631+00:00", + "upstream_run_id": "c75d97bf-2f1a-4f9f-b5f7-271ced2dd002", + "artifact_count": 51, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 12, + "qe_versions": [ + "7.3.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "16748361_0", + "ended_at": "2026-09-28T20:41:53+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:45+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_1", + "ended_at": "2026-09-28T20:41:57+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:45+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_2", + "ended_at": "2026-09-28T20:41:57+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:45+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_3", + "ended_at": "2026-09-28T20:41:57+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:41:45+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_4", + "ended_at": "2026-09-28T20:42:51+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:42:44+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_5", + "ended_at": "2026-09-28T20:42:55+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:42:44+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_6", + "ended_at": "2026-09-28T20:42:55+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:42:44+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_7", + "ended_at": "2026-09-28T20:42:55+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:42:44+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_8", + "ended_at": "2026-09-28T20:43:56+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:43:44+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_9", + "ended_at": "2026-09-28T20:43:56+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:43:44+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_10", + "ended_at": "2026-09-28T20:43:56+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:43:44+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16748361_11", + "ended_at": "2026-09-28T20:44:51+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:44:44+00:00", + "submitted_at": "2026-09-28T20:41:21+00:00" + } + ] + }, + "relax-fresh:negishi": { + "id": "4aa6848d-0c00-40d4-9070-f07c3e921a03", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "negishi", + "status": "data_ingested", + "scheduler_job_id": "43537887", + "started_at": "2026-09-28T20:52:01+00:00", + "completed_at": "2026-09-28T20:53:14.808404+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [ + "7.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "43537887", + "ended_at": "2026-09-28T20:52:22+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:52:01+00:00", + "submitted_at": "2026-09-28T20:52:00+00:00" + } + ] + }, + "relax-fresh:gautschi": { + "id": "c59b7b4e-b101-48c8-9740-a779d722665d", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "gautschi", + "status": "data_ingested", + "scheduler_job_id": "16748409", + "started_at": "2026-09-28T20:52:11+00:00", + "completed_at": "2026-09-28T20:53:20.510875+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [ + "7.3.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "16748409", + "ended_at": "2026-09-28T20:52:20+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:52:11+00:00", + "submitted_at": "2026-09-28T20:52:02+00:00" + } + ] + }, + "2d:negishi": { + "id": "ddbe609d-545c-4544-a204-21e94fe01b62", + "tool_name": "cellrelaxdft2d", + "tool_revision": "justin-dt1", + "cluster": "negishi", + "status": "data_ingested", + "scheduler_job_id": "43537943", + "started_at": "2026-09-28T20:53:53+00:00", + "completed_at": "2026-09-28T20:54:58.551287+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [ + "7.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "43537943", + "ended_at": "2026-09-28T20:54:20+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:53:53+00:00", + "submitted_at": "2026-09-28T20:53:52+00:00" + } + ] + }, + "2d:gautschi": { + "id": "ad61e7cb-3c36-4df0-9b15-2b2c6a507058", + "tool_name": "cellrelaxdft2d", + "tool_revision": "justin-dt1", + "cluster": "gautschi", + "status": "data_ingested", + "scheduler_job_id": "16748419", + "started_at": "2026-09-28T20:54:15+00:00", + "completed_at": "2026-09-28T20:55:36.151478+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [ + "7.3.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "16748419", + "ended_at": "2026-09-28T20:54:30+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:54:15+00:00", + "submitted_at": "2026-09-28T20:53:53+00:00" + } + ] + }, + "cache:negishi": { + "id": "ad727315-73b2-4612-8614-609c742c0a8d", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "negishi", + "status": "data_ingested", + "scheduler_job_id": "", + "started_at": "", + "completed_at": "2026-09-28T20:56:06.909536+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": true, + "collection_status": null, + "execution_environment_count": 0, + "qe_versions": [], + "scheduler_accounting": [] + }, + "cache:gautschi": { + "id": "e8cf97bc-0326-4eeb-84a8-3963b8725420", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "gautschi", + "status": "data_ingested", + "scheduler_job_id": "", + "started_at": "", + "completed_at": "2026-09-28T20:56:07.642246+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": true, + "collection_status": null, + "execution_environment_count": 0, + "qe_versions": [], + "scheduler_accounting": [] + }, + "cancel:negishi": { + "id": "14d90ed8-9952-415c-8f83-c99e1134be82", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "negishi", + "status": "data_ingested", + "scheduler_job_id": "43537980", + "started_at": "2026-09-28T20:56:41+00:00", + "completed_at": "2026-09-28T21:27:41.327989+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [ + "7.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "43537980", + "ended_at": "2026-09-28T20:57:01+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:56:41+00:00", + "submitted_at": "2026-09-28T20:56:40+00:00" + } + ] + }, + "cancel:gautschi": { + "id": "c087cbf5-019f-4ba3-a5cd-f753c91b5629", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "gautschi", + "status": "data_ingested", + "scheduler_job_id": "16748432", + "started_at": "2026-09-28T20:56:44+00:00", + "completed_at": "2026-09-28T21:27:39.342193+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [ + "7.3.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "16748432", + "ended_at": "2026-09-28T20:56:57+00:00", + "exit_code": "0:0", + "started_at": "2026-09-28T20:56:44+00:00", + "submitted_at": "2026-09-28T20:56:42+00:00" + } + ] + }, + "cancel-confirmed:negishi": { + "id": "ee3359c9-9928-4b08-bc0e-6b2fd607ebfa", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "negishi", + "status": "cancelled", + "scheduler_job_id": "43545163", + "started_at": "2026-09-29T01:58:50+00:00", + "completed_at": "2026-09-29T02:00:36.583795+00:00", + "upstream_run_id": "", + "artifact_count": 7, + "convergence": null, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 1, + "qe_versions": [], + "scheduler_accounting": [ + { + "state": "CANCELLED by 2305096", + "job_id": "43545163", + "ended_at": "2026-09-29T01:59:28+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T01:58:50+00:00", + "submitted_at": "2026-09-29T01:58:48+00:00" + } + ] + }, + "cancel-confirmed:gautschi": { + "id": "23ca2b89-33ed-4f1a-9b69-001b5cc63b8f", + "tool_name": "cellrelaxdft", + "tool_revision": "r99-dt2", + "cluster": "gautschi", + "status": "cancelled", + "scheduler_job_id": "16750610", + "started_at": "", + "completed_at": "2026-09-29T02:00:35.248693+00:00", + "upstream_run_id": "", + "artifact_count": 3, + "convergence": null, + "cache_hit": false, + "collection_status": "partial", + "execution_environment_count": 0, + "qe_versions": [], + "scheduler_accounting": [ + { + "state": "CANCELLED by 2305096", + "job_id": "16750610", + "ended_at": "2026-09-29T01:59:27+00:00", + "exit_code": "0:0", + "started_at": null, + "submitted_at": "2026-09-29T01:58:55+00:00" + } + ] + }, + "elastic-fresh:negishi": { + "id": "7a44e1b4-d493-4495-9d56-3776e8831199", + "tool_name": "elasticdft", + "tool_revision": "r27-dt2", + "cluster": "negishi", + "status": "data_ingested", + "scheduler_job_id": "43545244", + "started_at": "2026-09-29T02:01:07+00:00", + "completed_at": "2026-09-29T02:04:36.505439+00:00", + "upstream_run_id": "4aa6848d-0c00-40d4-9070-f07c3e921a03", + "artifact_count": 51, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 12, + "qe_versions": [ + "7.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "43545244_0", + "ended_at": "2026-09-29T02:01:20+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:01:07+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_1", + "ended_at": "2026-09-29T02:01:20+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:01:07+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_2", + "ended_at": "2026-09-29T02:01:20+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:01:07+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_3", + "ended_at": "2026-09-29T02:01:17+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:01:07+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_4", + "ended_at": "2026-09-29T02:03:06+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:51+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_5", + "ended_at": "2026-09-29T02:03:09+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:51+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_6", + "ended_at": "2026-09-29T02:03:03+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:51+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_7", + "ended_at": "2026-09-29T02:03:06+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:51+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_8", + "ended_at": "2026-09-29T02:03:24+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:03:04+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_9", + "ended_at": "2026-09-29T02:03:27+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:03:09+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_10", + "ended_at": "2026-09-29T02:03:31+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:03:09+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + }, + { + "state": "COMPLETED", + "job_id": "43545244_11", + "ended_at": "2026-09-29T02:03:30+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:03:09+00:00", + "submitted_at": "2026-09-29T02:01:07+00:00" + } + ] + }, + "elastic-fresh:gautschi": { + "id": "e2542dd7-94d9-4abf-932b-305ca9fea527", + "tool_name": "elasticdft", + "tool_revision": "r27-dt2", + "cluster": "gautschi", + "status": "data_ingested", + "scheduler_job_id": "16750617", + "started_at": "2026-09-29T02:01:46+00:00", + "completed_at": "2026-09-29T02:05:09.648322+00:00", + "upstream_run_id": "c59b7b4e-b101-48c8-9740-a779d722665d", + "artifact_count": 51, + "convergence": true, + "cache_hit": false, + "collection_status": "complete", + "execution_environment_count": 12, + "qe_versions": [ + "7.3.1" + ], + "scheduler_accounting": [ + { + "state": "COMPLETED", + "job_id": "16750617_0", + "ended_at": "2026-09-29T02:01:55+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:01:46+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_1", + "ended_at": "2026-09-29T02:01:54+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:01:46+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_2", + "ended_at": "2026-09-29T02:01:55+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:01:46+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_3", + "ended_at": "2026-09-29T02:01:54+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:01:46+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_4", + "ended_at": "2026-09-29T02:02:53+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:45+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_5", + "ended_at": "2026-09-29T02:02:53+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:45+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_6", + "ended_at": "2026-09-29T02:02:52+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:45+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_7", + "ended_at": "2026-09-29T02:02:53+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:45+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_8", + "ended_at": "2026-09-29T02:03:04+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:53+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_9", + "ended_at": "2026-09-29T02:03:04+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:53+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_10", + "ended_at": "2026-09-29T02:03:02+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:02:53+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + }, + { + "state": "COMPLETED", + "job_id": "16750617_11", + "ended_at": "2026-09-29T02:03:53+00:00", + "exit_code": "0:0", + "started_at": "2026-09-29T02:03:45+00:00", + "submitted_at": "2026-09-29T02:01:11+00:00" + } + ] + } + } +} diff --git a/geddes/k8s/pilots/computations/PILOT_VALIDATION_2026-09-28.md b/geddes/k8s/pilots/computations/PILOT_VALIDATION_2026-09-28.md new file mode 100644 index 0000000..45ee639 --- /dev/null +++ b/geddes/k8s/pilots/computations/PILOT_VALIDATION_2026-09-28.md @@ -0,0 +1,152 @@ +# Computational pilot validation — September 28, 2026 + +Dates use Indiana local time; late checks are September 29 in UTC. + +Status: **ready for Rushik and Justin to begin the private scientific pilot review**. +Both `desai224` and `jam19` have verified ordinary `member` access to `RCAC-PILOT`; +their global role remains `researcher`. Project listing, run listing, and JSON +exports passed for both accounts. Nonmembers cannot discover the private project. +Scientific acceptance remains the purpose of Rushik/Justin's review. + +Pilot: https://dt-sim2l-pilot.geddes.rcac.purdue.edu/projects/RCAC-PILOT +Namespace: `ncn-digitaltwins-zchen-sim2l`. Project: `RCAC-PILOT`. +Branch: `feature/sim2l-rcac-integration`, based on `db5d72a` with local pilot fixes. +No production deployment, merge into `dev`, or messages to reviewers were made. + +## Deployment + +One `Recreate` Deployment, four non-root containers: web, API, PostgreSQL, worker. +Only the web port is exposed. API and database bind to loopback. Independent +runtime/SAML credentials are retained; no demo mode or student test sessions. +Three persistent 5 GiB volumes retain the database, application data, and worker +staging. Worker state also resumes from the database. Limits total 4 CPUs/4 GiB, +within the existing one-Pod namespace quota. Observed post-rollout usage was +46 MiB web, 169 MiB API, 21 MiB PostgreSQL, and 272 MiB worker during +validation. This is a smoke-test observation, not a large-material load test. + +September 28 pinned images (the web image was subsequently updated in the +[September 29 convergence UI release](UI_VALIDATION_2026-09-29.md)): + +```text +web: geddes-registry.rcac.purdue.edu/sdx/dt-web@sha256:4121a40add3580ec03a661a91899ebe84733f59d5e76fb6b84aa8f1dc82328a9 +api: geddes-registry.rcac.purdue.edu/sdx/dt-api@sha256:8bca6183eff29720846b6d1cdcbb1a44860293420f38a61344bfdd289dc6b985 +worker: geddes-registry.rcac.purdue.edu/sdx/dt-computation-worker@sha256:28f1e3010ea7f012576582b7d3de7f90351fa04bedd8f0d571e2bfb4d01734a2 +postgres: postgres@sha256:efedf3595f1d6f415c08568ba171029bf54052e754cc9f030e3f2412b21f3d67 +``` + +Gateway: `31dc6ef0-162e-4d9b-8785-08153e6bfdfe`. +Gateway source SHA-256: `67a3b906e1e22324efd0a6fc0bc9291ff48bc8376f001a74db1ca70c1880634e`. +Both dedicated `dtwin0` endpoint allowlists include it. Prior gateway +`674b04c4-267d-45af-aaec-79c184a9c7f2`, images, and configuration are retained; +the old ConfigMap is `dt-computation-pilot-compute-674b04c4`. + +## Defect found and corrected during live validation + +Slurm `sacct` returned blank comments for pending/running jobs and pending array +members, while `squeue` returned the correct `dt-run:` and `dtwin0` owner. +The previous gateway treated the blank accounting field as conflicting attribution, +so the initial relaxations and elasticity runs were marked as transport failures +even though the calculations completed successfully. + +The corrected gateway checks live queue comment and owner, rejects any conflicting +nonempty accounting comment, and treats unattributed terminal accounting as unknown +until attribution arrives. It does not silently accept a blank terminal identity. +The four original jobs were recovered through the API's retry-collection route +using their existing Slurm IDs. Fresh runs verify normal operation of the fix. + +## Cancellation correction + +The first cancellation smoke jobs completed in 13–20 seconds, before the +worker could send `scancel`. Slurm correctly reported them as completed. The +harness repeatedly requested cancellation after they entered collection, exposing +a backwards state transition in the API. Repeated cancellation is now idempotent +for already requested/cancelling runs and during collection; finished results or +diagnostics cannot be pushed back into cancelling. The harness sends cancellation +once and uses a longer calculation with a ten-minute cancellation-test deadline. +The early-completed runs and their event history are retained, not relabeled as +successful cancellation tests. + +## Verification + +- 79 targeted API/gateway/worker/schema/monolayer tests passed, including nine new + cases for empty accounting attribution, array siblings, conflicting identities, + and idempotent cancellation during collection. +- Production Next.js build/TypeScript and lint on the affected UI files passed. +- `npm audit --omit=dev`: zero known vulnerabilities. Plotly is now 4.1.1 and + Node 22 is used by the web image. Deployed API/worker `pip check` passed. +- Kubernetes server dry run and final rollout passed. Recreate required normal + RWO-volume detach/reattach; no data volumes were deleted or reinitialized. + The API restarted once while PostgreSQL started, then became healthy. +- HTTPS certificate, matching SAML certificate, signed Redirect request, anonymous + route protection, forged-header rejection and invalid assertion rejection passed. +- Navid, `desai224`, and `jam19` have real SAML-created researcher accounts. + Navid confirmed successful logout and re-login in this conversation. +- Final gateway read-only checks passed on both endpoints. + +The live harness checks ordinary project authorization, scheduler timestamps, +execution identity, convergence, observed QE versions and executable hashes, +raw-output download sizes/hashes, JSON exports, cache reuse, capacity rejection, +2D-parent rejection, and cancelled-run diagnostics. Its resumable operator report +is `/var/lib/dt-api/pilot-validation-20260928.json` on the application PVC. +Operator API identity checks are separate from the real browser sign-ins; no +browser click-through under the students' private sessions is claimed. + +## Image data isolation + +The final packaging audit found generated local upload files and a mock database +in the broad `COPY api/` build context. `.dockerignore` now excludes `api/data` +and `api/local_data` for both API and worker images. Pilot uploads/configuration +remain on the dedicated application PVC; the image is not a data-distribution +mechanism. Local image verification confirms both directories contain no files and all +computation-module hashes match the tested source. Final deployed API and worker checks also found zero files in these directories. + +## Final run evidence + +All 16 validation requests are terminal: 14 ingested (including two cache hits) and +two confirmed cancelled. The two earlier short cancellation attempts finished +before Slurm received cancellation and remain recorded as completed. + +| Phase | Cluster | Status | Slurm ID | Artifacts | +| --- | --- | --- | --- | --- | +| relax | negishi | data_ingested | 43537557 | 7 | +| relax | gautschi | data_ingested | 16748345 | 7 | +| elastic | negishi | data_ingested | 43537708 | 51 | +| elastic | gautschi | data_ingested | 16748361 | 51 | +| relax-fresh | negishi | data_ingested | 43537887 | 7 | +| relax-fresh | gautschi | data_ingested | 16748409 | 7 | +| 2d | negishi | data_ingested | 43537943 | 7 | +| 2d | gautschi | data_ingested | 16748419 | 7 | +| cache | negishi | data_ingested | cache; no job | 7 | +| cache | gautschi | data_ingested | cache; no job | 7 | +| cancel | negishi | data_ingested | 43537980 | 7 | +| cancel | gautschi | data_ingested | 16748432 | 7 | +| cancel-confirmed | negishi | cancelled | 43545163 | 7 | +| cancel-confirmed | gautschi | cancelled | 16750610 | 3 | +| elastic-fresh | negishi | data_ingested | 43545244 | 51 | +| elastic-fresh | gautschi | data_ingested | 16750617 | 51 | + +The [machine-readable evidence](PILOT_EVIDENCE_2026-09-28.json) records run UUIDs, +upstream links, accounting, observed QE versions, and every phase/access check. +Fresh elastic runs also passed a late cancellation request during collection: +collection continued without reverting to cancelling. + +Review package: `Mxene/DT-computational-scientific-review-20260928.zip`. +It contains six fresh converged runs (three tools × two clusters), included parent +runs, checksummed artifacts, sanitized observed runtimes, and Justin’s supplied +scientific reference. Requester identities, secrets, and private paths are omitted. + +ZIP SHA-256: `6646dc85f8d089275cecbe44d7de6b75177fa8fc0c178baf38679274b7deb052`. +Scientific acceptance is pending; no numerical equivalence is inferred. + +## Rollback and review + +Disable new submissions on the **API container only**, leaving the worker enabled +to finish or cancel/collect active jobs. Scaling a worker down does not cancel +Slurm jobs. Retain all three PVCs and Depot archives. Previous pinned API/worker +images and the versioned compute ConfigMap permit recovery; the previous gateway +has the accounting-comment defect and should not be reopened to students. + +The [handoff guide](../../../../COMPUTATIONAL_PILOT_HANDOFF.md) explains the review +workflow. Scientific approval and convergence studies remain pending. These +infrastructure checks do not validate arbitrary materials, 2D elasticity, or +third-order tensors. diff --git a/geddes/k8s/pilots/computations/README.md b/geddes/k8s/pilots/computations/README.md new file mode 100644 index 0000000..08c2783 --- /dev/null +++ b/geddes/k8s/pilots/computations/README.md @@ -0,0 +1,161 @@ +# Isolated computational student pilot + +This is **not** the production DT deployment. Never apply the parent directory's +production manifests as part of this pilot. No merge into `dev` is required. + +Target namespace: `ncn-digitaltwins-zchen-sim2l`. +Pilot host: `dt-sim2l-pilot.geddes.rcac.purdue.edu`. +Ingress class `nginx` is Purdue-network/VPN-only, per the +[Geddes web-server guide](https://docs.rcac.purdue.edu/userguides/geddes/examples/webserver/). +The `routing.yaml` file only reserves routing; it does not deploy a web server. +Do not tell students the pilot is ready until all gates below pass. + +## Current deployment (September 28) + +The isolated stack now includes web, API, PostgreSQL and one worker, with three +persistent volumes and a versioned Globus gateway on both RCAC clusters. See +[current validation/access status](PILOT_VALIDATION_2026-09-28.md) and the +[student handoff](../../../../COMPUTATIONAL_PILOT_HANDOFF.md). That report +supersedes the historical login-only stage below. + +Provision with `provision_compute.py` using pinned image digests, after rendering +and server-dry-running its manifests. `prepare_gateway.py` retains the previous +compute ConfigMap when an explicitly identified function is replaced. Permit +the new function only with `allow_gateway.py` on the two dedicated endpoints, +reload them, and complete `check_gateway.py` before rolling the pilot. + +Run `validate_pilot.py` inside the API container, one phase at a time: +`relax`, `elastic`, `relax-fresh`, `elastic-fresh`, `2d`, `cache`, `cancel`. +The report on the application PVC prevents duplicate submissions when resuming. +Phases make real, bounded Slurm requests; do not run concurrent copies against +the same report. `open_pilot.py` requires all phases to pass before adding +ordinary members whose canonical IDs/emails match already registered users. +Its default is a read-only preflight; `--apply` grants and verifies access. + +## Historical login-only stage (superseded) + +The pilot now serves `/login` (HTTP 200) and live SAML metadata. Daniel reports +the SP metadata is registered. `check_login.mjs` verifies the live public +certificate against the attachment, the signed Redirect request, anonymous +route protection and invalid-assertion rejection. nanoHUB redirects the signed +request to its login page; this is **not** proof of a successful real login. +Navid subsequently completed a real login: the pilot recorded canonical user +`ngholiza` and forwarded authenticated API requests. No extra network opening +was needed. Logout and student identities/project membership still need +validation. Do not share passwords, session cookies or SAML assertions. + +This deployment is deliberately **login validation only**, with +`RCAC_COMPUTE_ENABLED=false`, no worker and no Globus/Materials Project secrets. +The production namespace and previous E2E database are untouched. A fresh +`dt_computation_pilot` database resides in the pilot Pod with API/web containers, +backed by a dedicated 5 GiB PVC. PostgreSQL and API listen only on loopback; +only web port 3000 is exposed. Both API database roles are non-superusers, with +the migration role owning only this new database's schema. Total container +memory limits are 2304 MiB, CPU limits 3, active Pods 1. + +`provision_login.py` creates namespace-local independent runtime credentials, +reuses existing credentials on rerun, reuses the full initialization schema +without its shared-role password-reset block, and requires image digests. Its +default mode prints non-secret manifests; `--apply` provisions only this pilot. +Postgres initialization runs once on a new volume; do not delete/recreate the +PVC to troubleshoot application errors. API startup performs normal migrations. +The API writable configuration/upload directory is ephemeral at this stage; +do not ingest research data until persistent application storage is provisioned. + +Use `Dockerfile.api` here to build the pilot API without physical datasets or +domain fixtures. Image digests and verification are recorded in +`LOGIN_VALIDATION_2026-09-28.md`. The private project `RCAC-PILOT` exists with no +members yet. Grant membership using actual SAML-resolved identities, not guessed +usernames. Completing this login gate does **not** complete the worker rollout, +dependency review, cluster smoke tests or scientific review below. + +## Release gates + +1. Complete local tests/build and review the branch diff. The pending merge is + `dev` **into** the feature branch, not the feature branch into `dev`. +2. Build fresh API/worker/web images, pin their digests, and register the reviewed + gateway source. Preserve the old function and images for rollback; permit the + new function on only the dedicated `dtwin0` endpoints. Test submit, cancellation, + partial collection and downloads against RCAC before opening student access. +3. Provision independent web/session/system/ingestion credentials. Do not copy + production session keys, use signed test cookies for students, or enable + `DEMO_MODE`. Secrets and mounts must all belong to the pilot namespace. +4. Use an isolated, fully migrated DT database. Existing `dt_rcac_e2e` contains + historical evidence; its minimal bootstrap alone is **not** a complete SAML/UI + database. Verify user roles, project queries and all startup health checks before + using it. Never run the initialization script against the production DB or alter + the shared `api_client` role/password as part of pilot setup. +5. Create a new private `RCAC-PILOT` project in the isolated DB. Grant Justin/Rushik + membership using the canonical identity returned by a **real** nanoHUB login. + Do not guess their usernames from their names or give them administrator roles. +6. Register the separate SAML SP with nanoHUB and verify a real login/logout. The + public metadata and SP certificate may be shared; the SP private key must not. +7. Verify DNS and HTTPS from a student's Purdue/VPN network. An Ingress object alone + does not prove DNS or TLS readiness. If DNS does not appear, ask RCAC to route + this hostname to the private Geddes ingress. Do not change the production host. +8. Walk through the [review checklist](../../../../COMPUTATIONAL_SCIENTIFIC_REVIEW.md) + with Navid, then hand over the URL, project name and supported workflow list. + +## Runtime layout and guards + +Observed quota September 23: **one active Pod, 4 GiB memory limits, 7 CPUs**. +Use a single-replica `Recreate` Deployment with separate web, API and worker +containers in one Pod; do not raise project quota or scale a production deployment. +Suggested memory limits: web 768 MiB, API 1024 MiB, worker 2048 MiB (3840 MiB total). +These fit the quota but still need a real memory-use test. Bind the API to +`127.0.0.1:8000` and set web `DT_API_URL_INTERNAL=http://127.0.0.1:8000/api`. +Expose **only the web port** through the Service. No API/DB/worker ingress. + +Run as non-root with no service-account token, no added capabilities, read-only +root filesystems and bounded temporary volumes. Only worker/API containers that +need Globus downloads receive the namespace-local client secret; only the worker +receives the Materials Project key. Web receives neither external credential nor +DB credentials. The API and worker use an ordinary, pilot-DB-scoped role. + +Enable these only in that pilot environment after validation: + +```text +RCAC_COMPUTE_ENABLED=true +RCAC_ARTIFACT_DOWNLOADS_ENABLED=true +RCAC_ALLOWED_PROJECT_IDS=RCAC-PILOT +RCAC_MAX_ACTIVE_RUNS_PER_USER=2 +RCAC_MAX_ACTIVE_RUNS_PER_CLUSTER=1 +RCAC_SERVICE_ACCOUNT_USERNAME=dtwin0 +``` + +Leave the existing DT feature flag disabled. The new capacity limits are atomic +across concurrent submissions and return HTTP 429 when occupied. One elastic +workflow may contain up to four simultaneous 64-rank members; array membership +is not the same as one MPI allocation. Keep a single worker process and 30-second +centralized polling. Lease renewal reduces stale batch handling but is not a claim +of validated multi-worker/high-availability operation. + +Raw downloads are limited to 8 MiB per artifact and two concurrent transfers per +API process. Larger artifacts remain in Depot for authorized retrieval. A scratch +path alone is not persistent storage. Generated wavefunction/restart directories +are not included in the selected-artifact archive. + +## SAML registration message (not sent) + +> Please register a separate nanoHUB SAML service provider for the NCN Digital +> Twin computational student pilot, without changing the existing DT registration. +> Base URL: https://dt-sim2l-pilot.geddes.rcac.purdue.edu +> Entity ID/metadata: https://dt-sim2l-pilot.geddes.rcac.purdue.edu/api/auth/saml/metadata +> ACS (HTTP-POST): https://dt-sim2l-pilot.geddes.rcac.purdue.edu/api/auth/saml/acs +> Logout (HTTP-Redirect): https://dt-sim2l-pilot.geddes.rcac.purdue.edu/api/auth/saml/sls +> We will supply a separate public SP certificate/metadata, never the private key. +> Please confirm signing requirements and the canonical username/email attributes. + +The current public IdP metadata is available at +https://nanohub.org/saml/idp/metadata. Pin the public signing certificate from the +approved metadata and confirm it with the nanoHUB administrator; do not relax +issuer, audience, request-ID or assertion-signature validation to get login working. + +## Rollback + +First disable new submissions at the pilot API. Do not treat scaling the worker +down as cancellation of running Slurm jobs. Allow the existing worker to finish +monitoring/archiving, or explicitly cancel the selected pilot run IDs and confirm +their terminal Slurm states. Then scale **only** `dt-computation-pilot` to zero. +Keep the pilot DB, Depot archives and endpoint configurations for recovery. Never +delete the namespace or change the production DT services as a rollback step. diff --git a/geddes/k8s/pilots/computations/RUN_VALUES_FIXES_2026-09-29.md b/geddes/k8s/pilots/computations/RUN_VALUES_FIXES_2026-09-29.md new file mode 100644 index 0000000..f01be62 --- /dev/null +++ b/geddes/k8s/pilots/computations/RUN_VALUES_FIXES_2026-09-29.md @@ -0,0 +1,107 @@ +# Unit experiment run values — September 29, 2026 + +**Deployed and verified on the private pilot.** The final pod is 4/4 ready with +zero restarts. All 17 saved runs, the three computational steps, and Rushik's and +Justin's project memberships are intact. HTTPS serves the exact updated UI +chunks. No new live calculation was submitted. Claude's final implementation +sign-off remains blocked by its credit balance, as described below. + +The seven findings in the [Claude/Codex review](../../../../CLAUDE_UNIT_EXPERIMENT_REVIEW_2026-09-29.md) +have been addressed in the working tree of `feature/sim2l-rcac-integration` +(HEAD `db5d72ae120d63ceca959e590951a69fbcee2b49`, with uncommitted changes). +This report covers the run-entry, computation and proposal paths reviewed here; +it is not a full audit of every change on this branch. + +## Actual value entry + +- Computational process steps offer **New RCAC Run**, which selects the matching + tool in **Registered computational experiments**. Bookmarked generic URLs for + computational templates show the route to that form. The generic API rejects + those templates with HTTP 409, without creating a run plan. +- Ordinary unit experiments offer **Run** in the library with project selection, + or **New Run** on the project. **Run Values → Planned Values** contains a field + for each selected equipment input and additional input. Saving creates a run + plan; it does not operate physical equipment. +- Actual values use the declared type: number, integer, text, boolean, date/time, + JSON object or JSON array. Units come from the definition. Finite numbers, + bounds, integers, allowed values and required inputs are validated. Additional + inputs are optional unless marked Required. Their type, requirement and numeric + bounds can be authored in the definition UI. +- Backend validation details are displayed and the save button recovers after + errors. View / Copy describes ordinary definition behavior accurately. Broken + definition and fork links now use `/experiment` routes. + +## Execution and optimization + +- Fractional `KE_cutoff` values reach `ecutwfc` unchanged. Adapter version 2 is + part of the input fingerprint, preventing old cached results from being reused. + Old fractional-cutoff relaxations and unverified fractional upstreams cannot + become ElasticDFT parents. Integral historical parents remain eligible. A + future adapter bump preserves eligibility of verified corrected parents. +- Initial computation defaults and tool switching both use the API schema. + Scientific notation, visible convergence controls, advanced JSON and mobile + layouts remain supported. Manual tool selection also updates the URL, so + clicking a process step again reselects the requested tool. +- Optimization pins Fixed values and searches Variable ranges. Every model + feature must be explicitly selected. Missing, inverted, nonfinite and + out-of-equipment ranges are rejected. Additional/sample context cannot pin + optimizer features. +- The selected equipment's registered configuration supplies limits and targets; + its own legacy file is used only if no registered record exists. The available + training model is the etcher model. Other equipment is rejected unless an + operator explicitly maps it via `RECIPE_PROPOSAL_ETCHER_EQUIPMENT_IDS` and gives + it a compatible configuration. Matching field names alone never enable it. + Maximize/minimize directions change GPR and RF ranking. The established etcher + default path remains available, with output bounds checked. + +## Claude implementation and independent review + +Two real Claude Code sessions implemented disjoint areas (`web/**` and `api/**`). +Codex reviewed their changes, ran tests and sent concrete revision requests. +Claude completed the UI revision. The backend session stopped partway through +its second revision with **“Credit balance is too low.”** Codex finished the +equipment-config validation, range checks, exact integer handling, historical +fingerprint tests, UI JSON checks and tool-link behavior, and reviewed the final +changes. **Final Claude agreement on the completed implementation is pending.** +The earlier agreement on review findings must not be read as implementation +sign-off. + +The [evidence directory](run-values-evidence-2026-09-29/) contains Claude's +implementation notes, revision prompts, browser captures, screenshots and source +hashes. No credentials or full agent transcripts are included. + +## Verification + +- 216 targeted Python tests and 17 subtests passed across run validation, + objective ranking, closed-loop regression, security, QE, worker, router and + gateway suites. After the final exact-integer/error-text adjustment, the 59 + affected tests and 13 subtests passed again. +- Four PostgreSQL integration tests passed in a disposable database: computation + API/worker lifecycle and typed run-value persistence. Invalid values leave no + partial run or process-step attachment. +- Chrome checks render actual React pages, the actual API client and stylesheet + using isolated HTTP fixtures. They verify payloads, required/typed input + validation, error recovery, authoring/copy metadata, project selection, links, + all-variable optimization, computational routing, fractional cutoff/scientific + notation, consistent defaults, JSON synchronization and mobile field widths. +- TypeScript and ESLint passed for revised forms, components and helpers. The + project page retains one pre-existing `set-state-in-effect` lint error at its + process-step draft synchronization effect; that unrelated code was not changed. + Linux production builds passed for web, API and worker. `git diff --check` + passed. +- No live calculation was submitted for these checks. Local browser fixtures do + not establish a student-session walkthrough or scientific equivalence. + +## Deployment + +The target is only `dt-computation-pilot` in namespace +`ncn-digitaltwins-zchen-sim2l`, context `geddes`. The existing PostgreSQL image, +PVCs and project membership are preserved. The pre-rollout API check recorded +17 saved runs (14 ingested, 3 cancelled), no active calculations and the three +existing computational process steps. + +Final image pins, readiness, HTTPS assets and saved-run comparison are recorded +in [deployment.json](run-values-evidence-2026-09-29/deployment.json). +The final pod is `dt-computation-pilot-57949548f9-vmc6h`. Anonymous project access +still redirects to sign-in (307), and anonymous computation API access returns +401. The temporary test database was removed after verification. diff --git a/geddes/k8s/pilots/computations/STUDENT_FAQ_OWNER_ACCESS_2026-09-29.md b/geddes/k8s/pilots/computations/STUDENT_FAQ_OWNER_ACCESS_2026-09-29.md new file mode 100644 index 0000000..45d509e --- /dev/null +++ b/geddes/k8s/pilots/computations/STUDENT_FAQ_OWNER_ACCESS_2026-09-29.md @@ -0,0 +1,60 @@ +# Student FAQ and owner access — September 29, 2026 + +The private pilot now serves a searchable +[Computational Workflow FAQ](https://dt-sim2l-pilot.geddes.rcac.purdue.edu/help/computational-workflows?project_id=RCAC-PILOT). +Students can open **Help & FAQ** in the sidebar or **Computational workflow FAQ** +beside the project computation form. The latter preserves the project context +and provides links to each tool's run form. + +The FAQ contains 19 answers covering sign-in and project access, workflow +selection, definitions versus actual runs, visible input fields and advanced +JSON, pilot example settings, submission and statuses, cancellation, cached +results, downloads, scientific review, elastic parent selection, retrieval +retries, and reporting problems. Example settings match the handoff and current +tool defaults; they are explicitly distinguished from scientifically validated +convergence settings. Search, keyboard-operated expandable answers, and a +horizontally scrollable example table work on narrow screens. + +## Owner access + +At the owner's explicit request, the existing active canonical account +`ngholiza` was promoted from `researcher` to `admin` in the isolated pilot +database. **Administrator** is the highest implemented platform role and enables +all current platform permission flags. No new role or client-side identity +exception was introduced. The separate `RCAC-PILOT` project role remains PI; +Rushik (`desai224`) and Justin (`jam19`) remain project members. + +The database change and its reason were recorded in `user_role_audit`, entry +`9938cbc6-f866-462d-961e-42590c0bae16`. The internal identity lookup used by sign-in +returns `admin`; Users, Reviews, and Execution Queue API reads return HTTP 200 +using the account's database-confirmed role. These checks passed again after +the deployment restarted. Refresh the page to reload the role; if an existing +browser still shows Researcher, sign out and sign back in. + +## Verification and deployment + +- TypeScript, ESLint for the changed UI files, `git diff --check`, and the Linux + production web build passed. +- Chrome exercised the real FAQ, project form, application shell, sidebar, and + role permissions with isolated session/API fixtures. Checks covered all four + roles, Administrator navigation, 19 answers, keyboard expansion, search and + empty-result recovery, context-preserving links, and mobile overflow with + expanded and collapsed sidebars. +- The running server contains the FAQ route. HTTPS serves the exact checksummed + FAQ, sidebar, and project-form chunks. Anonymous page requests redirect to + sign-in; anonymous computation API requests return HTTP 401. +- The final pod `dt-computation-pilot-84f6658677-fmxzm` has four ready containers + and zero restarts. The web image is pinned to + `sha256:6974e931319b41b6be21c7c080fc3790d5dfe0a758f4ed2223add38835e81fa5`. + The API, worker, and PostgreSQL image pins are unchanged. +- All 17 saved run IDs and project memberships are preserved. No calculations + were active during the rollout and no new live simulation was submitted. + +The target is `dt-computation-pilot`, namespace `ncn-digitaltwins-zchen-sim2l`, +context `geddes`. This update and role grant apply to the private pilot. +Browser fixtures do not constitute a student-session walkthrough or a new +scientific validation. No student browser session was impersonated. + +[Sanitized verification evidence](faq-owner-evidence-2026-09-29/) includes source +hashes, screenshots, browser results, the audited role change, internal API +checks, deployment status, and HTTPS asset verification. diff --git a/geddes/k8s/pilots/computations/UI_VALIDATION_2026-09-29.md b/geddes/k8s/pilots/computations/UI_VALIDATION_2026-09-29.md new file mode 100644 index 0000000..c0cab14 --- /dev/null +++ b/geddes/k8s/pilots/computations/UI_VALIDATION_2026-09-29.md @@ -0,0 +1,61 @@ +# Convergence form correction — September 29, 2026 + +Status: **deployed and verified on the private pilot**. Implementation, browser +checks, production image build, and rollout passed. Purdue network connectivity +was restored before publishing. + +The prior experiment form exposed cutoff and k-points while placing convergence +thresholds and ionic steps inside a collapsed JSON editor. The form now displays +**Convergence and iteration limits** with SCF, energy, force, pressure, and maximum +ionic steps. The handoff uses the same labels. Existing scientific defaults are +preserved; the coarse silicon example remains an explicit reviewer choice. + +Numeric drafts preserve scientific notation while typing. Empty, incomplete, +nonfinite, fractional-integer, and out-of-range values are rejected before +submission. Bounds come from the selected tool's API schema. The advanced JSON +editor and visible controls remain synchronized. The initial cutoff is 50 Ry; +5 Ry is below the schema's 10 Ry minimum. + +## Verification + +- Isolated Chrome test used the actual React component, API client, and Tailwind + stylesheet with local HTTP fixtures generated from repository API schemas. + It did not use a student session or submit any live calculation. +- Typed the handoff's 30 Ry, 2×2×2, `1e-8`, `1e-8`, `1e-4`, `1.0`, and 50 values + character by character; confirmed all fields were visible with JSON collapsed + and captured the corresponding numeric POST payload. +- Verified partial exponents, empty cutoff, cutoff 5, fractional ionic steps, + excessive energy tolerance, and non-object JSON cannot submit. +- Checked JSON-to-form and form-to-JSON synchronization, 2D defaults and fixed + `k_z=1`, and ElasticDFT's existing parent-selection submission. +- Inspected desktop (1280 px) and mobile (390 px) screenshots; all numeric fields + fit their viewport. No browser page errors occurred. +- All three captured relaxation payloads passed the actual Python API validators. +- ESLint, TypeScript, and `git diff --check` passed. The normal production Next.js + build passed in the Linux Docker image. The host Turbopack build was blocked by + local process/port restrictions; the container build had no such failure. + +[Desktop screenshot](ui-evidence-2026-09-29/convergence-desktop.png) · +[Mobile screenshot](ui-evidence-2026-09-29/convergence-mobile.png) · +[Browser check evidence](ui-evidence-2026-09-29/results.json) + +## Deployed release + +Published image: `geddes-registry.rcac.purdue.edu/sdx/dt-web:pilot-convergence-20260929`. +Deployed manifest-list digest: +`sha256:255dc94052a787c45d0262a73cea8ed47497bea888f545868926cb916bd66f9f`. + +Only the private pilot's web image was changed. There were no active calculations +before rollout. Its single-Pod `Recreate` deployment restarted the companion +containers while retaining the API, worker, database images and configuration +recorded in the September 28 report. The new pod is +`dt-computation-pilot-58fd84ccd6-z85wm`: 4/4 ready, zero restarts. Initial image-pull +timeouts cleared on retry; no scheduling or volume changes were needed. + +Post-rollout checks confirmed all 17 saved runs are available, none active, and +the live API publishes the expected convergence schemas. The public HTTPS URL +serves `/_next/static/chunks/1g3smfyt5qf6r.js` containing all five controls; its +SHA-256 matches the deployed container: +`a7d5029bd2098b2c13da89bbd43665b6af5598eb9db1727664b066b307e3ff39`. +Anonymous project access redirects to sign-in (307), and the computation API +rejects anonymous access (401). Student browser sessions were not impersonated. diff --git a/geddes/k8s/pilots/computations/allow_gateway.py b/geddes/k8s/pilots/computations/allow_gateway.py new file mode 100644 index 0000000..731ecf9 --- /dev/null +++ b/geddes/k8s/pilots/computations/allow_gateway.py @@ -0,0 +1,34 @@ +"""Run via stdin on dtwin0's endpoint host; append one reviewed function UUID. + +Preserves an exclusive timestamped backup and all existing allowed functions. +Does not restart services or submit jobs. Only the two named pilot configs. +""" +import datetime +import os +import pathlib +import pwd +import sys +import uuid + +assert pwd.getpwuid(os.getuid()).pw_name == "dtwin0" +cluster, function_id = sys.argv[1:] +assert cluster in {"negishi", "gautschi"} +assert str(uuid.UUID(function_id)) == function_id +path = pathlib.Path.home() / ".globus_compute" / ("dt-" + cluster) / "config.yaml" +old = path.read_text() +if function_id in old: + print("Function already permitted") +else: + assert old.count("allowed_functions:\n") == 1 + assert "type: LocalProvider" in old + backup = path.with_name("config.yaml.backup-" + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")) + with backup.open("x") as stream: + stream.write(old) + os.chmod(backup, 0o600) + updated = old.replace("allowed_functions:\n", "allowed_functions:\n - " + function_id + "\n", 1) + temporary = path.with_name("config.yaml.pending-" + function_id) + with temporary.open("x") as stream: + stream.write(updated) + os.chmod(temporary, 0o600) + temporary.replace(path) + print("Added reviewed function; backup:", backup.name) diff --git a/geddes/k8s/pilots/computations/check_gateway.py b/geddes/k8s/pilots/computations/check_gateway.py new file mode 100644 index 0000000..c055c1c --- /dev/null +++ b/geddes/k8s/pilots/computations/check_gateway.py @@ -0,0 +1,25 @@ +"""Read-only live checks of the newly registered pilot gateway on both clusters.""" +import base64 +import json +import os +import sys +from provision_login import ROOT, NAME, kubectl + +sys.path.insert(0, str(ROOT / "api")) +from computations.globus_gateway import GlobusComputeGateway + +config = json.loads(kubectl("get", "configmap", NAME + "-compute", "-o", "json"))["data"] +secret = json.loads(kubectl("get", "secret", "dt-globus-confidential-client", "-o", "json")) +os.environ["GLOBUS_COMPUTE_CLIENT_ID"] = config["GLOBUS_COMPUTE_CLIENT_ID"] +os.environ["GLOBUS_COMPUTE_CLIENT_SECRET"] = base64.b64decode(secret["data"]["client-secret"]).decode().strip() +gateway = GlobusComputeGateway() +historical = { + "negishi": {"run_id": "44e09c05-6c10-4724-9fa2-638fd71859ea", "job_id": "43375490"}, + "gautschi": {"run_id": "8956cf41-00c1-45c7-9b9c-41c6244a7d96", "job_id": "16619101"}, +} +for name, cluster in json.loads(config["RCAC_CLUSTERS_JSON"]).items(): + task, result = gateway.invoke(endpoint_id=cluster["endpoint_id"], function_id=cluster["gateway_function_id"], + action="status_many", payload={"cluster": name, "jobs": [historical[name]]}) + assert "COMPLETED" in result["states"][historical[name]["job_id"]] + assert result["accounting"][historical[name]["job_id"]][0]["started_at"] + print(json.dumps({"cluster": name, "task_id": task, "read_only_gateway": "passed"}), flush=True) diff --git a/geddes/k8s/pilots/computations/check_login.mjs b/geddes/k8s/pilots/computations/check_login.mjs new file mode 100644 index 0000000..4725fdb --- /dev/null +++ b/geddes/k8s/pilots/computations/check_login.mjs @@ -0,0 +1,63 @@ +// Public-endpoint checks only. Never prints cookies, assertions or request tokens. +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { X509Certificate, verify } from 'node:crypto'; +import { inflateRawSync } from 'node:zlib'; + +const base = 'https://dt-sim2l-pilot.geddes.rcac.purdue.edu'; +const request = (path, options = {}) => fetch(new URL(path, base), { + redirect: 'manual', signal: AbortSignal.timeout(20000), ...options, +}); +let res = await request('/login'); +assert.equal(res.status, 200); +res = await request('/'); +assert.equal(res.status, 307); +assert.match(res.headers.get('location'), /login/); +res = await request('/api/dt/dataset/v2/projects'); +assert.equal(res.status, 401); +res = await request('/api/dt/computations/projects/RCAC-PILOT/runs', { + headers: {'X-User-Id': 'ngholiza', 'X-User-Role': 'admin', 'X-System-Token': 'invalid'}, +}); +assert.equal(res.status, 401, 'Public proxy must reject forged identity headers without a real session'); +res = await request('/api/auth/saml/metadata'); +assert.equal(res.status, 200); +const metadata = await res.text(); +assert.ok(metadata.includes(`entityID="${base}/api/auth/saml/metadata"`)); +assert.ok(metadata.includes(`${base}/api/auth/saml/acs`)); +const embedded = metadata.match(/<(?:\w+:)?X509Certificate>([\s\S]*?)<\/(?:\w+:)?X509Certificate>/)[1]; +const certificate = new X509Certificate(Buffer.from(embedded.replace(/\s/g, ''), 'base64')); +if (process.argv[2]) { + assert.equal(certificate.fingerprint256, new X509Certificate(readFileSync(process.argv[2])).fingerprint256); +} +res = await request('/api/auth/saml/login?callbackUrl=%2Fprojects'); +assert.equal(res.status, 302); +const target = new URL(res.headers.get('location')); +assert.equal(target.origin, 'https://nanohub.org'); +assert.equal(target.pathname, '/saml/idp/login'); +const query = target.search.slice(1).split('&'); +const signed = ['SAMLRequest', 'RelayState', 'SigAlg'].map(k => query.find(x => x.startsWith(k + '='))).join('&'); +assert.equal(target.searchParams.get('SigAlg'), 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha256'); +assert.ok(verify('RSA-SHA256', Buffer.from(signed), certificate.publicKey, + Buffer.from(target.searchParams.get('Signature'), 'base64'))); +const xml = inflateRawSync(Buffer.from(target.searchParams.get('SAMLRequest'), 'base64')).toString(); +assert.ok(xml.includes(`${base}/api/auth/saml/acs`)); +assert.ok(xml.includes(`${base}/api/auth/saml/metadata`)); +console.log('PASS: HTTPS login, protected routes, matching SP certificate, signed request and pilot ACS.'); + +// No credentials: only check whether the IdP recognizes the registration. +res = await fetch(target, {redirect: 'manual', signal: AbortSignal.timeout(20000)}); +const location = res.headers.get('location'); +const body = await res.text(); +console.log('nanoHUB unauthenticated response:', res.status, + location ? new URL(location, target).origin + new URL(location, target).pathname : '(no redirect)', + 'title:', body.match(/]*>([^<]*)<\/title>/i)?.[1] ?? '(none)'); +if (/unknown service provider|invalid signature|service provider not found/i.test(body)) { + throw new Error('nanoHUB rejected the service provider or request signature'); +} +res = await request('/api/auth/saml/acs'); +assert.equal(res.status, 405); +res = await request('/api/auth/saml/acs', {method: 'POST', body: new URLSearchParams({SAMLResponse: 'invalid'})}); +assert.equal(res.status, 303); +assert.ok(!res.headers.get('set-cookie')); +assert.match(res.headers.get('location'), /SAMLAuthenticationFailed/); +console.log('PASS: ACS rejects invalid responses without issuing a session. Real user login remains necessary.'); diff --git a/geddes/k8s/pilots/computations/open_pilot.py b/geddes/k8s/pilots/computations/open_pilot.py new file mode 100644 index 0000000..4096061 --- /dev/null +++ b/geddes/k8s/pilots/computations/open_pilot.py @@ -0,0 +1,76 @@ +"""Grant ordinary project access after validation, inside the isolated API Pod. + +Use --member canonical_id=email for accounts already established by real SAML +sign-in. Default is a read-only preflight; --apply adds project memberships. +Never creates users, changes global roles, or handles browser sessions. +""" +import argparse +import json +import os +from pathlib import Path + +import requests +from data_loader_pg import get_pg_superuser_connection +from psycopg2.extras import RealDictCursor + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--member", action="append", required=True) + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + assert os.environ["PG_DB"] == "dt_computation_pilot" + assert os.environ["PG_HOST"] == "127.0.0.1" + assert os.environ["RCAC_ALLOWED_PROJECT_IDS"] == "RCAC-PILOT" + report_path = Path("/var/lib/dt-api/pilot-validation-20260928.json") + report = json.loads(report_path.read_text()) + for phase in ("relax", "relax-fresh", "elastic", "elastic-fresh", "2d", "cache", "cancel"): + assert report["checks"].get(phase) == "passed", f"Validation not complete: {phase}" + expected = dict(item.split("=", 1) for item in args.member) + conn = get_pg_superuser_connection() + try: + with conn, conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute("SELECT access_mode FROM projects WHERE id='RCAC-PILOT' FOR UPDATE") + assert cur.fetchone()["access_mode"] == "private" + cur.execute("SELECT count(*) AS n FROM computation_runs WHERE status NOT IN ('data_ingested','failed','cancelled')") + assert cur.fetchone()["n"] == 0, "Finish validation jobs before opening access" + cur.execute("SELECT id,email,role,status FROM users WHERE id = ANY(%s)", (list(expected),)) + users = cur.fetchall() + assert len(users) == len(expected), "Every member must already have signed in" + for user in users: + assert user["email"].casefold() == expected[user["id"]].casefold(), "Identity/email mismatch" + assert user["role"] == "researcher" and user["status"] == "active" + if args.apply: + cur.execute("""INSERT INTO project_members (project_id,nanohub_user_id,role) + VALUES ('RCAC-PILOT',%s,'member') ON CONFLICT DO NOTHING""", (user["id"],)) + cur.execute("SELECT role FROM project_members WHERE project_id='RCAC-PILOT' AND nanohub_user_id=%s", (user["id"],)) + assert cur.fetchone()["role"] == "member", "Unexpected existing project privilege" + if args.apply: + cur.execute("""UPDATE projects SET description=%s WHERE id='RCAC-PILOT'""", + ("Private RCAC computational pilot for scientific review: 3D relaxation, 2D monolayer relaxation, and 3D second-order elasticity. Scientific acceptance pending.",)) + finally: + conn.close() + if not args.apply: + print(json.dumps({"preflight": "passed", "members": sorted(expected)})) + return + + # Operator-level API checks of authorization; these do not substitute for + # the students' real SAML logins and do not create test session cookies. + base = "http://127.0.0.1:8000/api" + run_id = report["runs"]["relax-fresh:negishi"]["id"] + for user in users: + headers = {"X-System-Token": os.environ["DT_SYSTEM_TOKEN"], + "X-User-Id": user["id"], "X-User-Email": user["email"], + "X-User-Role": user["role"]} + response = requests.get(base + "/dataset/v2/projects", headers=headers, timeout=30) + assert response.ok and any(p["id"] == "RCAC-PILOT" for p in response.json()) + for route in ("/computations/projects/RCAC-PILOT/runs", + f"/computations/runs/{run_id}/export"): + assert requests.get(base + route, headers=headers, timeout=30).ok, route + report["checks"]["member:" + user["id"]] = "project and results accessible; researcher/member roles" + print(json.dumps({"member": user["id"], "project": "RCAC-PILOT", "access": "verified"})) + report_path.write_text(json.dumps(report, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/geddes/k8s/pilots/computations/prepare_gateway.py b/geddes/k8s/pilots/computations/prepare_gateway.py new file mode 100644 index 0000000..0bc8467 --- /dev/null +++ b/geddes/k8s/pilots/computations/prepare_gateway.py @@ -0,0 +1,70 @@ +"""Register the reviewed gateway once and create only pilot compute configuration. + +Credentials are consumed in memory, never printed or copied into a ConfigMap. +Endpoint allowlist changes are a separate, explicit operation. +""" +import base64 +import argparse +import hashlib +import json +import os +import sys + +from provision_login import ROOT, NAME, kubectl, resource + +sys.path.insert(0, str(ROOT / "api")) +from computations.globus_gateway import register_gateway_function + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--replace-function-id", help="Expected current function UUID; preserve its config for rollback") + args = parser.parse_args() + digest = hashlib.sha256((ROOT / "api/computations/globus_gateway.py").read_bytes()).hexdigest() + name = NAME + "-compute" + prior = kubectl("get", "configmap", name, "--ignore-not-found", "-o", "json") + if prior: + old = json.loads(prior) + if old["metadata"].get("annotations", {}).get("dt/gateway-sha256") == digest: + print(old["data"]["GLOBUS_COMPUTE_GATEWAY_FUNCTION_ID"]) + return + if old["data"]["GLOBUS_COMPUTE_GATEWAY_FUNCTION_ID"] != args.replace_function_id: + raise SystemExit("Source differs; use --replace-function-id with the reviewed current UUID") + backup_name = name + "-" + args.replace_function_id[:8] + backup = kubectl("get", "configmap", backup_name, "--ignore-not-found", "-o", "json") + if backup: + if json.loads(backup)["data"] != old["data"]: + raise SystemExit("Existing rollback configuration differs; refusing to overwrite") + else: + document = resource("ConfigMap", backup_name, data=old["data"]) + document["metadata"]["annotations"] = old["metadata"].get("annotations", {}) + kubectl("create", "-f", "-", document=document) + original = old["data"] + else: + if args.replace_function_id: + raise SystemExit("No current pilot configuration to replace") + original = json.loads(kubectl("get", "configmap", "dt-computation-config-live-e2e", "-o", "json"))["data"] + secret = json.loads(kubectl("get", "secret", "dt-globus-confidential-client", "-o", "json")) + os.environ["GLOBUS_COMPUTE_CLIENT_ID"] = original["GLOBUS_COMPUTE_CLIENT_ID"] + os.environ["GLOBUS_COMPUTE_CLIENT_SECRET"] = base64.b64decode(secret["data"]["client-secret"]).decode().strip() + function_id = register_gateway_function() + clusters = json.loads(original["RCAC_CLUSTERS_JSON"]) + for cluster in clusters.values(): + cluster["gateway_function_id"] = function_id + data = {**original, "RCAC_CLUSTERS_JSON": json.dumps(clusters), + "GLOBUS_COMPUTE_GATEWAY_FUNCTION_ID": function_id, + "GLOBUS_COMPUTE_CLIENT_SECRET_FILE": "/run/globus/client-secret", + "RCAC_ARTIFACT_DOWNLOADS_ENABLED": "true", + "RCAC_ALLOWED_PROJECT_IDS": "RCAC-PILOT", + "RCAC_MAX_ACTIVE_RUNS_PER_USER": "2", + "RCAC_MAX_ACTIVE_RUNS_PER_CLUSTER": "1"} + config = resource("ConfigMap", name, data=data) + config["metadata"]["annotations"] = {"dt/gateway-sha256": digest} + if prior: + config["metadata"]["resourceVersion"] = old["metadata"]["resourceVersion"] + kubectl("replace" if prior else "create", "-f", "-", document=config) + print(function_id) + + +if __name__ == "__main__": + main() diff --git a/geddes/k8s/pilots/computations/provision_compute.py b/geddes/k8s/pilots/computations/provision_compute.py new file mode 100644 index 0000000..0b83418 --- /dev/null +++ b/geddes/k8s/pilots/computations/provision_compute.py @@ -0,0 +1,63 @@ +"""Upgrade ONLY the isolated pilot; use --apply after a server dry run. + +Retains its existing DB, SAML keys, sessions and users. No production references. +Student memberships must be granted separately after live validation. +""" +import argparse +import copy +import json +import re + +from provision_login import NAME, documents, env, kubectl, mount, resource, resources + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for n in ("web", "api", "worker", "postgres"): + parser.add_argument("--" + n + "-image", required=True) + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + for image in (args.web_image, args.api_image, args.worker_image, args.postgres_image): + if not re.fullmatch(r"[a-zA-Z0-9./_-]+@sha256:[a-f0-9]{64}", image): + parser.error("Use immutable image digests") + # Existing database bootstrap/PVC must not be reinitialized or replaced. + deployment = documents(args.web_image, args.api_image, args.postgres_image)[-1] + pod = deployment["spec"]["template"]["spec"] + api = pod["containers"][1] + api["env"] = [v for v in api["env"] if v["name"] != "RCAC_COMPUTE_ENABLED"] + api["env"].append(env("GLOBUS_COMPUTE_USER_DIR", "/tmp/dt-globus-client")) + api["envFrom"] = [{"configMapRef": {"name": NAME + "-compute"}}] + api["volumeMounts"].append(mount("globus", "/run/globus", True)) + worker = copy.deepcopy(api) + worker.update(name="worker", image=args.worker_image, + command=["python", "-m", "computations.worker", "--poll-seconds", "30"], + resources=resources("1792Mi")) + # The worker is not a web server. Runs resume from database state. + worker.pop("readinessProbe") + worker.pop("startupProbe") + worker["env"] = [v for v in worker["env"] if v["name"] not in {"DT_SYSTEM_TOKEN", "INGESTION_TOKEN"}] + worker["env"].append({"name": "MATERIALS_PROJECT_API_KEY", "valueFrom": {"secretKeyRef": { + "name": "dt-computation-materials-project", "key": "api-key"}}}) + worker["volumeMounts"] = [mount("globus", "/run/globus", True), mount("worker-tmp", "/tmp"), + mount("worker-data", "/var/lib/dt-computation-worker")] + pod["containers"].append(worker) + pod["volumes"] = [v for v in pod["volumes"] if v["name"] != "api-data"] + [ + {"name": "api-data", "persistentVolumeClaim": {"claimName": NAME + "-application-data"}}, + {"name": "worker-data", "persistentVolumeClaim": {"claimName": NAME + "-worker-data"}}, + {"name": "worker-tmp", "emptyDir": {"sizeLimit": "512Mi"}}, + {"name": "globus", "secret": {"secretName": "dt-globus-confidential-client", "defaultMode": 0o440}}] + pvcs = [resource("PersistentVolumeClaim", NAME + "-" + n, spec={ + "accessModes": ["ReadWriteOnce"], "storageClassName": "geddes-standard-singlenode", + "resources": {"requests": {"storage": "5Gi"}}}) for n in ("application-data", "worker-data")] + manifests = [*pvcs, deployment] + if not args.apply: + print(json.dumps({"apiVersion": "v1", "kind": "List", "items": manifests}, indent=2)) + return + kubectl("get", "configmap", NAME + "-compute", "-o", "name") + kubectl("get", "secret", NAME + "-runtime", "-o", "name") + for manifest in manifests: + print(kubectl("apply", "-f", "-", document=manifest).strip()) + + +if __name__ == "__main__": + main() diff --git a/geddes/k8s/pilots/computations/provision_login.py b/geddes/k8s/pilots/computations/provision_login.py new file mode 100644 index 0000000..e0c3794 --- /dev/null +++ b/geddes/k8s/pilots/computations/provision_login.py @@ -0,0 +1,164 @@ +"""Provision ONLY the isolated login pilot; never enable cluster submission. + +Uses a new PostgreSQL instance/PVC, not the production or historical E2E DB. +Secret values are generated in memory and passed to kubectl on stdin. +Existing secrets are reused without reading/printing values or rotating them. +Images must be immutable digest references. Run with --apply after inspection. +This bootstrap is superseded by provision_compute.py once compute is installed. +""" +import argparse +import json +import pathlib +import re +import secrets +import subprocess + +NAMESPACE = "ncn-digitaltwins-zchen-sim2l" +NAME = "dt-computation-pilot" +RUNTIME_SECRET = NAME + "-runtime" +ROOT = pathlib.Path(__file__).resolve().parents[4] + + +def kubectl(*args, document=None): + result = subprocess.run( + ["kubectl", "--context", "geddes", "--request-timeout=30s", "-n", NAMESPACE, *args], + input=json.dumps(document) if document is not None else None, + text=True, capture_output=True, + ) + if result.returncode: + # API errors may echo a submitted Secret. Never print those payloads. + raise RuntimeError(f"kubectl {args[0]} failed (exit {result.returncode}); inspect resource events") + return result.stdout + + +def resource(kind, name, **fields): + versions = {"Deployment": "apps/v1", "NetworkPolicy": "networking.k8s.io/v1"} + return {"apiVersion": versions.get(kind, "v1"), "kind": kind, + "metadata": {"name": name, "namespace": NAMESPACE}, **fields} + + +def env(name, value): + return {"name": name, "value": value} + + +def secret_env(name, key=None): + return {"name": name, "valueFrom": {"secretKeyRef": { + "name": RUNTIME_SECRET, "key": key or name}}} + + +def mount(name, path, readonly=False): + return {"name": name, "mountPath": path, "readOnly": readonly} + + +def resources(memory): + return {"requests": {"cpu": "100m", "memory": "128Mi"}, + "limits": {"cpu": "1", "memory": memory}} + + +def documents(web, api, postgres): + # Reuse full platform schema, but not its shared-cluster role/password block. + schema = (ROOT / "api/scripts/init_db.sql").read_text() + schema = schema[schema.index("-- Table: projects"):] + initialization = r"""\set ON_ERROR_STOP on +\getenv api_password PG_API_PASSWORD +\getenv owner_password PG_OWNER_PASSWORD +CREATE ROLE api_client LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS PASSWORD :'api_password'; +CREATE ROLE dt_pilot_owner LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS PASSWORD :'owner_password'; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +ALTER SCHEMA public OWNER TO dt_pilot_owner; +SET ROLE dt_pilot_owner; +""" + schema + """ +INSERT INTO projects (id, name, description, access_mode) +VALUES ('RCAC-PILOT', 'Computational Student Pilot', + 'Isolated pilot. Login validation only; computation submission disabled.', 'private'); +""" + config = resource("ConfigMap", NAME + "-bootstrap", data={"01-schema.sql": initialization}) + pvc = resource("PersistentVolumeClaim", NAME + "-data", spec={ + "accessModes": ["ReadWriteOnce"], "storageClassName": "geddes-standard-singlenode", + "resources": {"requests": {"storage": "5Gi"}}}) + security = {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, + "capabilities": {"drop": ["ALL"]}, "runAsNonRoot": True} + web_env = [env("AUTH_URL", "https://dt-sim2l-pilot.geddes.rcac.purdue.edu"), + env("AUTH_TRUST_HOST", "true"), env("NEXT_TELEMETRY_DISABLED", "1"), + env("DT_API_URL_INTERNAL", "http://127.0.0.1:8000/api"), + env("SAML_SP_PRIVATE_KEY_PATH", "/run/saml/sp-private-key.pem"), + env("SAML_SP_CERTIFICATE_PATH", "/run/saml/sp-certificate.pem"), + env("SAML_IDP_CERTIFICATE_PATH", "/run/saml/idp-certificate.pem"), + secret_env("AUTH_SECRET"), secret_env("DT_SYSTEM_TOKEN")] + api_env = [env("PG_HOST", "127.0.0.1"), env("PG_PORT", "5432"), + env("PG_DB", "dt_computation_pilot"), env("PG_USER", "api_client"), + env("PG_SUPER_USER", "dt_pilot_owner"), env("RCAC_COMPUTE_ENABLED", "false"), + env("USE_MOCK_DATABASE", "false"), env("DT_CONFIGS_DIR", "/var/lib/dt-api/configs"), + env("DT_UPLOADS_DIR", "/var/lib/dt-api/uploads"), + *[secret_env(k) for k in ("PG_PASS", "PG_SUPER_PASS", "DT_SYSTEM_TOKEN", "INGESTION_TOKEN")]] + api_probe = {"exec": {"command": ["python", "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/health', timeout=4)"]}, + "timeoutSeconds": 5, "periodSeconds": 10} + containers = [ + {"name": "web", "image": web, "env": web_env, + "securityContext": {**security, "runAsUser": 1000, "runAsGroup": 1000}, + "ports": [{"containerPort": 3000}], "resources": resources("768Mi"), + "volumeMounts": [mount("saml", "/run/saml", True), mount("web-tmp", "/tmp")], + "readinessProbe": {"httpGet": {"path": "/api/auth/saml/metadata", "port": 3000}, "periodSeconds": 10}, + "startupProbe": {"httpGet": {"path": "/login", "port": 3000}, "periodSeconds": 5, "failureThreshold": 60}}, + {"name": "api", "image": api, "env": api_env, + "securityContext": {**security, "runAsUser": 1000, "runAsGroup": 1000}, + "resources": resources("1024Mi"), + "volumeMounts": [mount("api-tmp", "/tmp"), mount("api-data", "/var/lib/dt-api")], + "readinessProbe": api_probe, + "startupProbe": {**api_probe, "failureThreshold": 60}}, + {"name": "postgres", "image": postgres, + "args": ["postgres", "-c", "listen_addresses=127.0.0.1", "-c", "max_connections=40", "-c", "shared_buffers=128MB"], + "securityContext": {**security, "runAsUser": 999, "runAsGroup": 1000}, + "resources": resources("512Mi"), + "env": [env("POSTGRES_DB", "dt_computation_pilot"), env("PGDATA", "/var/lib/postgresql/data/pgdata"), + secret_env("POSTGRES_PASSWORD", "DB_ADMIN_PASSWORD"), + secret_env("PG_API_PASSWORD", "PG_PASS"), secret_env("PG_OWNER_PASSWORD", "PG_SUPER_PASS")], + "volumeMounts": [mount("database", "/var/lib/postgresql/data"), mount("pg-run", "/var/run/postgresql"), + mount("pg-tmp", "/tmp"), mount("bootstrap", "/docker-entrypoint-initdb.d", True)], + "readinessProbe": {"exec": {"command": ["pg_isready", "-h", "127.0.0.1", "-d", "dt_computation_pilot"]}, "periodSeconds": 10}}, + ] + deployment = resource("Deployment", NAME, spec={ + "replicas": 1, "strategy": {"type": "Recreate"}, "selector": {"matchLabels": {"app": NAME}}, + "template": {"metadata": {"labels": {"app": NAME}}, "spec": { + "automountServiceAccountToken": False, "terminationGracePeriodSeconds": 30, + "securityContext": {"fsGroup": 1000, "seccompProfile": {"type": "RuntimeDefault"}}, + "imagePullSecrets": [{"name": "sdx-registry-secret"}], "containers": containers, + "volumes": [{"name": "database", "persistentVolumeClaim": {"claimName": NAME + "-data"}}, + {"name": "saml", "secret": {"secretName": NAME + "-saml", "defaultMode": 0o440}}, + {"name": "bootstrap", "configMap": {"name": NAME + "-bootstrap"}}, + *[{"name": n, "emptyDir": {"sizeLimit": "256Mi"}} for n in + ("web-tmp", "api-tmp", "api-data", "pg-run", "pg-tmp")]]}}}) + return [pvc, config, deployment] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("web", "api", "postgres"): + parser.add_argument("--" + name + "-image", required=True) + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + images = [args.web_image, args.api_image, args.postgres_image] + if any(not re.fullmatch(r"[a-zA-Z0-9./_-]+@sha256:[a-f0-9]{64}", image) for image in images): + parser.error("All images must be pinned repository@sha256 references") + manifests = documents(*images) + if not args.apply: + print(json.dumps({"apiVersion": "v1", "kind": "List", "items": manifests}, indent=2)) + return + if kubectl("get", "configmap", NAME + "-compute", "--ignore-not-found", "-o", "name").strip(): + parser.error("Compute is already installed; use provision_compute.py. Refusing a login-only downgrade.") + # Verify the independently generated signing credentials exist before changes. + kubectl("get", "secret", NAME + "-saml", "-o", "name") + existing = kubectl("get", "secret", RUNTIME_SECRET, "--ignore-not-found", "-o", "name") + if not existing.strip(): + values = {key: secrets.token_urlsafe(48) for key in + ("AUTH_SECRET", "DT_SYSTEM_TOKEN", "INGESTION_TOKEN", "PG_PASS", "PG_SUPER_PASS", "DB_ADMIN_PASSWORD")} + kubectl("create", "-f", "-", document=resource("Secret", RUNTIME_SECRET, type="Opaque", stringData=values)) + print("Created independent pilot runtime credentials (values not displayed).") + for manifest in manifests: + print(kubectl("apply", "-f", "-", document=manifest).strip()) + + +if __name__ == "__main__": + main() diff --git a/geddes/k8s/pilots/computations/routing.yaml b/geddes/k8s/pilots/computations/routing.yaml new file mode 100644 index 0000000..a69bb13 --- /dev/null +++ b/geddes/k8s/pilots/computations/routing.yaml @@ -0,0 +1,42 @@ +# Isolated pilot only. No route points at the existing DT API or web service. +# This reserves routing; without the separately reviewed Deployment it serves +# no application and cannot submit jobs. nginx is Purdue-network/VPN-only. +apiVersion: v1 +kind: Service +metadata: + name: dt-computation-pilot + namespace: ncn-digitaltwins-zchen-sim2l +spec: + selector: + app: dt-computation-pilot + ports: + - name: web + port: 3000 + targetPort: 3000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: dt-computation-pilot + namespace: ncn-digitaltwins-zchen-sim2l + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-body-size: "10m" +spec: + ingressClassName: nginx + tls: + - hosts: + - dt-sim2l-pilot.geddes.rcac.purdue.edu + rules: + - host: dt-sim2l-pilot.geddes.rcac.purdue.edu + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: dt-computation-pilot + port: + name: web diff --git a/geddes/k8s/pilots/computations/validate_pilot.py b/geddes/k8s/pilots/computations/validate_pilot.py new file mode 100644 index 0000000..6f41b2a --- /dev/null +++ b/geddes/k8s/pilots/computations/validate_pilot.py @@ -0,0 +1,187 @@ +"""Run inside the isolated pilot API container via stdin for an authorized rollout. + +Exercises the deployed API and independently running worker, not an in-process +replacement. Attribution is Navid's verified pilot identity, never a student. +Writes a resumable report on the pilot application PVC. Polls the DB-backed API, +not Slurm; the worker alone polls Slurm every 30 seconds. +""" +import argparse +import hashlib +import json +import os +import pathlib +import time + +import requests + +assert os.environ["PG_DB"] == "dt_computation_pilot" +assert os.environ["PG_HOST"] == "127.0.0.1" +assert os.environ["RCAC_SERVICE_ACCOUNT_USERNAME"] == "dtwin0" +base = "http://127.0.0.1:8000/api/computations" +headers = {"X-System-Token": os.environ["DT_SYSTEM_TOKEN"], "X-User-Id": "ngholiza", + "X-User-Email": "ngholiza@purdue.edu", "X-User-Name": "Navid Gholizadeh", + "X-User-Role": "researcher", "X-User-Org": "Purdue University"} +path = pathlib.Path("/var/lib/dt-api/pilot-validation-20260928.json") +report = json.loads(path.read_text()) if path.exists() else {"runs": {}, "checks": {}} +terminal = {"data_ingested", "failed", "cancelled"} +clusters = ("negishi", "gautschi") +cell_inputs = {"mp_id": "mp-149", "KE_cutoff": 30, "k_x": 2, "k_y": 2, "k_z": 2, + "scf_conv": 1e-8, "energy_conv": 1e-8, "force_conv": 1e-4, + "pressure_conv": 1.0, "i_steps": 50} + + +def save(): + pending = path.with_suffix(".pending") + pending.write_text(json.dumps(report, indent=2) + "\n") + pending.replace(path) + + +def emit(**data): + print(json.dumps(data), flush=True) + + +def request(method, route, expected=None, **kwargs): + response = requests.request(method, base + route, headers=headers, timeout=300, **kwargs) + if expected is not None: + assert response.status_code == expected, (route, response.status_code, response.text[:500]) + else: + assert response.ok, (route, response.status_code, response.text[:500]) + return response + + +def submit(label, cluster, tool, inputs): + key = label + ":" + cluster + if key not in report["runs"]: + row = request("POST", "/projects/RCAC-PILOT/runs/" + tool, + json={"cluster": cluster, "inputs": inputs}, expected=202).json() + report["runs"][key] = {"id": row["id"], "cluster": cluster, "tool": tool} + save() + emit(event="submitted", label=key, run_id=row["id"]) + return report["runs"][key]["id"] + + +def wait_all(ids, expected="data_ingested", cancel=False): + deadline = time.monotonic() + (600 if cancel else 1800) + previous = {} + cancellation_requested = set() + collection_checked = set() + while True: + rows = [request("GET", "/runs/" + run_id).json() for run_id in ids] + for row in rows: + if previous.get(row["id"]) != row["status"]: + emit(event="state", run_id=row["id"], status=row["status"], job_id=row["scheduler_job_id"]) + previous[row["id"]] = row["status"] + if row["status"] == "collecting" and row["id"] not in collection_checked: + # Slurm is already terminal. A late/repeated click must not + # interrupt result collection or change the scheduler outcome. + repeated = request("POST", "/runs/" + row["id"] + "/cancel", expected=202).json() + assert repeated["status"] in terminal | {"collecting"}, repeated["status"] + collection_checked.add(row["id"]) + report["checks"]["cancel-idempotent:" + row["id"]] = "collection continued" + save() + if (cancel and row["scheduler_job_id"] and row["status"] not in terminal | {"collecting", "cancelling"} + and row["id"] not in cancellation_requested and not row.get("cancel_requested") + and not row["external_workflow_ref"].get("cancellation_sent")): + request("POST", "/runs/" + row["id"] + "/cancel", expected=202) + cancellation_requested.add(row["id"]) + emit(event="cancel_requested", run_id=row["id"]) + if row["status"] in terminal: + entry = next(v for v in report["runs"].values() if v["id"] == row["id"]) + entry.update(status=row["status"], scheduler_job_id=row["scheduler_job_id"], + artifact_count=len(row["artifact_manifest"]), error=row.get("error")) + save() + if all(row["status"] in terminal for row in rows): + for row in rows: + assert row["status"] == expected, (row["id"], row["status"], row.get("error")) + return rows + if time.monotonic() > deadline: + for row in rows: + if row["status"] not in terminal: + request("POST", "/runs/" + row["id"] + "/cancel", expected=202) + raise TimeoutError("Validation deadline; cancellation requested. Confirm terminal states before leaving.") + time.sleep(5 if cancel else 20) + + +def check_result(row, cache=False): + assert row["requested_by"] == "ngholiza" + assert row["execution_identity"] == "dtwin0" + assert row["output"]["convergence"] is True + if cache: + assert row["external_workflow_ref"]["cache_hit"] is True + assert not row["scheduler_job_id"] + else: + assert row["scheduler_job_id"] and row["started_at"] and row["completed_at"] + assert row["slurm_comment"] == "dt-run:" + row["id"] + assert row["external_workflow_ref"]["scheduler_accounting"] + environments = row["external_workflow_ref"]["execution_environments"] + assert environments and all(item["qe_executable_sha256"] for item in environments) + assert row["external_workflow_ref"]["qe_versions_from_outputs"] + assert row["artifact_manifest"] + # Download one real archived QE output through the deployed API. + for index, artifact in enumerate(row["artifact_manifest"]): + if artifact.get("uri") and artifact["name"].endswith("stdout"): + response = request("GET", f"/runs/{row['id']}/artifacts/{index}") + assert hashlib.sha256(response.content).hexdigest() == artifact["sha256"] + assert len(response.content) == artifact["size_bytes"] + report["checks"]["download:" + row["id"]] = "sha256 matched" + break + else: + raise AssertionError("No downloadable QE output") + assert request("GET", "/runs/" + row["id"] + "/export").json() + outsider = {**headers, "X-User-Id": "pilot-validation-outsider", "X-User-Email": "outsider@example.invalid"} + for route in (f"/runs/{row['id']}", f"/runs/{row['id']}/export", f"/runs/{row['id']}/artifacts/0"): + denied = requests.get(base + route, headers=outsider, timeout=15) + assert denied.status_code in (403, 404), (route, denied.status_code) + report["checks"]["isolation:" + row["id"]] = "outsider denied" + save() + + +parser = argparse.ArgumentParser() +parser.add_argument("--phase", choices=["relax", "relax-fresh", "elastic", "elastic-fresh", "2d", "cache", "cancel"], required=True) +args = parser.parse_args() +request("GET", "/tools") +if args.phase in {"relax", "relax-fresh"}: + inputs = cell_inputs if args.phase == "relax" else {**cell_inputs, "KE_cutoff": 31} + ids = [submit(args.phase, c, "cellrelaxdft", inputs) for c in clusters] + if all(request("GET", "/runs/" + run_id).json()["status"] not in terminal for run_id in ids): + request("POST", "/projects/RCAC-PILOT/runs/cellrelaxdft", expected=429, + json={"cluster": "negishi", "inputs": inputs}) + report["checks"]["capacity"] = "third simultaneous submission rejected with 429" + save() + for row in wait_all(ids): + check_result(row) +elif args.phase in {"elastic", "elastic-fresh"}: + parent_label = "relax" if args.phase == "elastic" else "relax-fresh" + ids = [submit(args.phase, c, "elasticdft", { + "mp_id": "mp-149", "upstream_execution_id": report["runs"][parent_label + ":" + c]["id"], + "diagonal_strain_magnitudes": [-0.005, 0.005], + "offdiagonal_strain_magnitudes": [-0.005, 0.005]}) for c in clusters] + for row in wait_all(ids): + check_result(row) + assert len(row["output"]["elastic_tensor"]) == 6 + assert len(row["output"]["member_statuses"]) == 12 + assert row["upstream_run_id"] == report["runs"][parent_label + ":" + row["cluster"]]["id"] +elif args.phase == "2d": + ids = [submit("2d", c, "cellrelaxdft2d", {"mp_id": "mp-48"}) for c in clusters] + for row in wait_all(ids): + check_result(row) + assert row["output"]["dimensionality"] == 2 + invalid_parent = request("POST", "/projects/RCAC-PILOT/runs/elasticdft", expected=400, + json={"cluster": "negishi", "inputs": {"mp_id": "mp-48", "upstream_execution_id": ids[0]}}) + assert "3D" in invalid_parent.text or "cellrelaxdft" in invalid_parent.text.lower() + report["checks"]["2d-parent-rejected"] = "passed" +elif args.phase == "cache": + ids = [submit("cache", c, "cellrelaxdft", cell_inputs) for c in clusters] + for row in wait_all(ids): + check_result(row, cache=True) +elif args.phase == "cancel": + ids = [submit("cancel-confirmed", c, "cellrelaxdft", {**cell_inputs, "KE_cutoff": 180, "k_x": 16, "k_y": 16, "k_z": 16}) for c in clusters] + for row in wait_all(ids, expected="cancelled", cancel=True): + assert row["scheduler_job_id"] + assert row["external_workflow_ref"]["scheduler_accounting"] + assert row["external_workflow_ref"]["collection_status"] in {"complete", "partial"} + assert row["artifact_manifest"], "Cancelled runs must retain available diagnostics" + assert not row["output"], "Cancelled runs must not publish successful scientific outputs" +report["checks"][args.phase] = "passed" +save() +emit(event="phase_passed", phase=args.phase) diff --git a/web/Dockerfile b/web/Dockerfile index 4a5b76e..9b1edb3 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -5,7 +5,7 @@ # Push: docker push geddes-registry.rcac.purdue.edu/sdx/dt-web:v18 # ── Stage 1: Build ────────────────────────────────────────────────────────── -FROM node:20-alpine AS builder +FROM node:22-alpine AS builder WORKDIR /app # Install deps first (cache layer) @@ -17,7 +17,7 @@ COPY . . RUN npm run build # ── Stage 2: Runtime ──────────────────────────────────────────────────────── -FROM node:20-alpine AS runner +FROM node:22-alpine AS runner WORKDIR /app ENV NODE_ENV=production diff --git a/web/app/equipment/list/page.tsx b/web/app/equipment/list/page.tsx index 3a4c9d1..a192935 100644 --- a/web/app/equipment/list/page.tsx +++ b/web/app/equipment/list/page.tsx @@ -15,6 +15,7 @@ import { Plus, Search, Loader2, + Server, } from "lucide-react"; import { useState, useEffect } from "react"; @@ -97,8 +98,34 @@ export default function EquipmentListPage() { {searchQuery ? "No equipment matches your search." : "No equipment registered for your organization yet."} ) : ( +
+ {[ + { + kind: "physical", + title: "Physical equipment", + note: "Lab instruments. Their runs are measured data.", + }, + { + kind: "computational", + title: "Computational tools", + note: "Simulation software installed on RCAC clusters. Run them from a project's Registered computational experiments.", + }, + ].map((section) => { + const items = filtered.filter( + (eq) => (eq.asset_kind || "physical") === section.kind, + ); + if (items.length === 0) return null; + return ( +
+
+

+ {section.title} ({items.length}) +

+

{section.note}

+
- {filtered.map((eq) => { + {items.map((eq) => { + const computational = eq.asset_kind === "computational"; const domainId = eq.domain_id || eq.id || "unknown-equipment"; const equipmentName = eq.display_name || eq.name || domainId; const status = STATUS_STYLES[eq.status || "approved"] || STATUS_STYLES.approved; @@ -129,6 +156,8 @@ export default function EquipmentListPage() { > {eq.status === "maintenance" ? ( + ) : computational ? ( + ) : ( )} @@ -161,7 +190,7 @@ export default function EquipmentListPage() {
- {totalRunsLabel} runs + {computational ? (eq.model ? `Version ${eq.model}` : "—") : `${totalRunsLabel} runs`}
@@ -188,6 +217,10 @@ export default function EquipmentListPage() { ); })}
+
+ ); + })} +
)} ); diff --git a/web/app/experiment/new/page.tsx b/web/app/experiment/new/page.tsx index 3eebc44..88d12ea 100644 --- a/web/app/experiment/new/page.tsx +++ b/web/app/experiment/new/page.tsx @@ -6,9 +6,9 @@ import { useAuth } from "@/lib/auth-context"; import Link from "next/link"; import { useSearchParams } from "next/navigation"; import { - createExperiment, - createExperimentType, newClientRequestId, + saveExperimentType, + saveProjectRun, getEquipmentDetail, getExperimentTypes, getEquipmentListSimple, @@ -18,10 +18,16 @@ import { type EquipmentOutput, type EquipmentInfo, type EquipmentSimple, + type ExperimentTypeParameterInput, type ExperimentTypeRecord, type SampleRecord, type V2Project, } from "@/lib/api-client"; +import { + computationalTemplateInfo, + computationRunHref, + type ComputationToolName, +} from "@/lib/computation-templates"; import { ChevronRight, ChevronLeft, @@ -70,12 +76,192 @@ interface TypeParam { description: string; equipment_parameter_id?: string; recipe_settable?: boolean; + required?: boolean; + // Allowed values kept as defined and saved back under the same key. + allowed_values?: unknown[]; + allowed_key?: AllowedValuesKey; } -interface PlannedParam { +type AllowedValuesKey = "allowed_values" | "enum" | "options"; + +type ValueKind = "number" | "integer" | "boolean" | "datetime" | "object" | "array" | "string" | "unsupported"; + +// One value field per selected equipment input and per additional input. +interface RunInputRow { + key: string; name: string; - planned_value: string; + label: string; + kind: ValueKind; + declaredType: string; unit: string; + mode: TypeParam["mode"]; + min_value: string; + max_value: string; + required: boolean; + allowed?: unknown[]; + source: "equipment" | "additional"; + description: string; +} + +// Types an author can pick for an additional input (aliases are also read). +const ADDITIONAL_INPUT_TYPES = [ + { value: "string", label: "Text" }, + { value: "float", label: "Number" }, + { value: "integer", label: "Whole number" }, + { value: "boolean", label: "True / false" }, + { value: "datetime", label: "Date / time" }, + { value: "object", label: "JSON object" }, + { value: "array", label: "JSON array" }, +]; + +interface ComputationalTemplateNotice { + id: string; + name: string; + tool: ComputationToolName | null; + inputs: ExperimentTypeParameterInput[]; +} + +const NUMBER_TEXT = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/; + +// Mirrors _VALUE_TYPE_ALIASES in api/metadata_pg.py. A missing type means +// float for equipment inputs and string for additional inputs. +function valueKind(type: string | undefined, fallback: "float" | "string"): ValueKind { + switch ((type || "").trim().toLowerCase() || fallback) { + case "float": + case "double": + case "number": + case "numeric": + case "real": + case "decimal": + return "number"; + case "int": + case "integer": + return "integer"; + case "bool": + case "boolean": + return "boolean"; + case "string": + case "str": + case "text": + case "enum": + case "categorical": + return "string"; + case "datetime": + case "date": + case "timestamp": + return "datetime"; + case "object": + case "map": + case "dict": + case "json": + return "object"; + case "array": + case "list": + return "array"; + default: + return "unsupported"; + } +} + +const KIND_LABEL: Record = { + number: "number", + integer: "whole number", + boolean: "true / false", + datetime: "ISO 8601 date or date-time", + object: "JSON object", + array: "JSON array", + string: "text", + unsupported: "unsupported type", +}; + +function allowedValuesOf(param: ExperimentTypeParameterInput): { key: AllowedValuesKey; values: unknown[] } | null { + for (const key of ["allowed_values", "enum", "options"] as const) { + const values = param[key]; + if (Array.isArray(values) && values.length > 0) return { key, values }; + } + return null; +} + +/** Definition payload: restore allowed values under the key they were read from. */ +function definitionPayloadParam(param: TypeParam): ExperimentTypeParameterInput { + const { allowed_values: allowed, allowed_key: key, ...rest } = param; + const payload: ExperimentTypeParameterInput = { ...rest }; + if (allowed && allowed.length > 0) payload[key || "allowed_values"] = allowed; + return payload; +} + +/** Wire encoding: strings verbatim, booleans true/false, objects/arrays as JSON. */ +function encodeValue(value: unknown): string { + if (value === null || value === undefined) return ""; + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return JSON.stringify(value); +} + +function optionalText(value: unknown): string { + return value === null || value === undefined ? "" : String(value); +} + +function containsNonfiniteNumber(value: unknown): boolean { + if (typeof value === "number") return !Number.isFinite(value); + if (value && typeof value === "object") return Object.values(value).some(containsNonfiniteNumber); + return false; +} + +function runValueError(row: RunInputRow, raw: string): string | null { + const value = row.kind === "string" ? raw : raw.trim(); + if (row.kind === "unsupported") { + // No value can be entered or sent; a required one blocks the run. + return row.required + ? `This input declares an unsupported value type "${row.declaredType}", so no value can be entered. The unit experiment definition must be corrected.` + : null; + } + if (!value.trim()) return row.required ? "Enter a value." : null; + const allowed = row.allowed && row.allowed.length > 0 ? row.allowed : null; + switch (row.kind) { + case "number": + case "integer": { + const numeric = Number(value); + if (!NUMBER_TEXT.test(value) || !Number.isFinite(numeric)) return "Enter a finite number (for example 50 or 1e-8)."; + if (row.kind === "integer" && !Number.isInteger(numeric)) return "Enter a whole number."; + const min = row.min_value.trim() === "" ? NaN : Number(row.min_value); + const max = row.max_value.trim() === "" ? NaN : Number(row.max_value); + if (Number.isFinite(min) && numeric < min) return `Must be at least ${row.min_value}.`; + if (Number.isFinite(max) && numeric > max) return `Must be at most ${row.max_value}.`; + if (allowed && !allowed.some((option) => typeof option !== "boolean" && Number(encodeValue(option)) === numeric)) { + return "Choose one of the listed values."; + } + return null; + } + case "boolean": + if (value !== "true" && value !== "false") return "Choose true or false."; + return allowed && !allowed.some((option) => encodeValue(option).trim().toLowerCase() === value) + ? "Choose one of the listed values." + : null; + case "object": + case "array": { + try { + const parsed = JSON.parse(value); + if (containsNonfiniteNumber(parsed)) return "JSON numbers must be finite."; + const isArray = Array.isArray(parsed); + if (row.kind === "array" ? !isArray : isArray || parsed === null || typeof parsed !== "object") { + return row.kind === "array" ? "Enter a JSON array, for example [1, 2]." : 'Enter a JSON object, for example {"key": 1}.'; + } + return null; + } catch { + return "Enter valid JSON."; + } + } + default: + // Text and date/time. Date/time format is checked by the API. + return allowed && !allowed.some((option) => encodeValue(option).trim() === value.trim()) + ? "Choose one of the listed values." + : null; + } +} + +function runValueKey(source: RunInputRow["source"], name: string) { + return `${source}:${name}`; } interface FormData { @@ -94,7 +280,8 @@ interface FormData { selected_project: string; experiment_name: string; planned_date: string; - planned_parameters: PlannedParam[]; + // Keyed by runValueKey(); encoded strings as sent in planned_parameters. + run_values: Record; optimization_mode: RunPurposeMode; primary_target: string; primary_objective: "maximize" | "minimize"; @@ -118,7 +305,7 @@ const INITIAL_DATA: FormData = { selected_project: "", experiment_name: "", planned_date: "", - planned_parameters: [], + run_values: {}, optimization_mode: "none", primary_target: "", primary_objective: "maximize", @@ -131,7 +318,13 @@ const INITIAL_DATA: FormData = { function NewExperimentContent() { const { user } = useAuth(); const searchParams = useSearchParams(); - const projectRunMode = Boolean(searchParams.get("project_id") && searchParams.get("type_id")); + const projectFromUrl = searchParams.get("project_id") || ""; + const typeFromUrl = searchParams.get("type_id") || ""; + // Project steps pass project_id; the library Run action passes mode=run + // and lets the user choose the project. + const projectRunMode = Boolean(typeFromUrl && (projectFromUrl || searchParams.get("mode") === "run")); + // Opened from an existing definition without run intent: view it, or save changes as a copy. + const viewingDefinition = Boolean(typeFromUrl) && !projectRunMode; // Editing an existing template saves it in place as a new version instead // of creating a copy. The backend allows only the owner or an admin. const editTypeId = projectRunMode ? "" : searchParams.get("edit_type") || ""; @@ -139,13 +332,17 @@ function NewExperimentContent() { const activeSteps = projectRunMode ? [ { id: 2, label: "Run Values", icon: ClipboardList }, - { id: 3, label: "Review & Run", icon: FileCheck }, + { id: 3, label: "Review & Save", icon: FileCheck }, ] : [ { id: 1, label: "Unit Experiment Definition", icon: FlaskConical }, { id: 3, label: editMode ? "Review & Save" : "Review & Publish", icon: FileCheck }, ]; const prefillApplied = useRef(false); + const [computationalTemplate, setComputationalTemplate] = useState(null); + const [prefillError, setPrefillError] = useState(null); + const [loadedTemplateName, setLoadedTemplateName] = useState(""); + const [showRunValueErrors, setShowRunValueErrors] = useState(false); // One idempotency key per run submission, kept until it succeeds so a // retry after a lost response cannot create a second run. const experimentRequestKey = useRef(null); @@ -214,6 +411,7 @@ function NewExperimentContent() { const paramTypeId = editTypeId || searchParams.get("type_id"); if (!paramEquipmentId && !paramTypeId) return; prefillApplied.current = true; + const notFound = `Unit experiment "${paramTypeId}" was not found or is not visible to your account.`; void (async () => { let effectiveEquipmentId = paramEquipmentId || ""; let templateToApply: ExperimentTypeRecord | null = null; @@ -222,12 +420,19 @@ function NewExperimentContent() { templateToApply = (allTypes ?? []).find((t) => t.id === paramTypeId) || null; effectiveEquipmentId = templateToApply?.equipment_id || ""; } + if (templateToApply && computationalTemplateInfo(templateToApply).computational) { + applyExperimentType(templateToApply); + if (paramProjectId) setFormData((prev) => ({ ...prev, selected_project: paramProjectId })); + return; + } if (!effectiveEquipmentId) { if (editMode) { setSubmitResult({ status: "error", message: "This unit experiment was not found or you do not have access to edit it.", }); + } else if (paramTypeId) { + setPrefillError(notFound); } return; } @@ -239,7 +444,11 @@ function NewExperimentContent() { // Wait for unit experiments to load, then select the requested one. const types = await getExperimentTypes(effectiveEquipmentId); setExperimentTypes(types ?? []); - const match = (types ?? []).find((t) => t.id === paramTypeId) || templateToApply; + let match = (types ?? []).find((t) => t.id === paramTypeId) || templateToApply; + if (!match) { + const allTypes = await getExperimentTypes(); + match = (allTypes ?? []).find((t) => t.id === paramTypeId) || null; + } if (match) { applyExperimentType(match); if (editMode) setEditTemplate(match); @@ -248,11 +457,10 @@ function NewExperimentContent() { status: "error", message: "This unit experiment was not found or you do not have access to edit it.", }); + } else { + setPrefillError(notFound); } } - if (paramProjectId && paramTypeId && !editMode) { - setCurrentStep(2); - } })(); // eslint-disable-next-line react-hooks/exhaustive-deps }, [equipmentLoading, projectsLoading, searchParams]); @@ -295,13 +503,65 @@ function NewExperimentContent() { : selectedEquipmentDetail ? [] : []; - const selectedExperimentInputOptions = formData.type_parameters - .filter((parameter) => parameter.name && parameter.mode !== "not_used") - .map((parameter) => ({ + const optimizationRun = formData.optimization_mode === "optimization"; + // Requiredness matches the API: every selected equipment input needs a + // value (Variable ones are exempt in optimization, see rowUsesRange); + // additional inputs are optional unless the definition says required. + const selectedEquipmentInputs = formData.type_parameters.filter( + (parameter) => parameter.name && parameter.mode !== "not_used", + ); + const runInputRows: RunInputRow[] = [ + ...selectedEquipmentInputs.map((parameter) => ({ + key: runValueKey("equipment", parameter.name), name: parameter.name, label: equipmentFeatureOptions.find((feature) => feature.name === parameter.name)?.label || parameter.name, + kind: valueKind(parameter.type, "float"), + declaredType: parameter.type || "float", unit: parameter.unit || "", - })); + mode: parameter.mode, + min_value: parameter.min_value, + max_value: parameter.max_value, + required: true, + allowed: parameter.allowed_values, + source: "equipment" as const, + description: parameter.description, + })), + ...formData.additional_inputs + // The API ignores an additional input that repeats an equipment input name. + .filter((parameter) => parameter.name.trim() && !selectedEquipmentInputs.some((item) => item.name === parameter.name)) + .map((parameter) => ({ + key: runValueKey("additional", parameter.name), + name: parameter.name, + label: parameter.name, + kind: valueKind(parameter.type, "string"), + declaredType: parameter.type || "string", + unit: parameter.unit || "", + mode: "fixed" as const, + min_value: parameter.min_value, + max_value: parameter.max_value, + required: parameter.required === true, + allowed: parameter.allowed_values, + source: "additional" as const, + description: parameter.description, + })), + ]; + // In optimization campaigns Variable equipment inputs are search ranges, + // not point values; the backend reads their ranges from the definition. + const rowUsesRange = (row: RunInputRow) => + optimizationRun && row.source === "equipment" && row.mode === "variable"; + const runValueErrors: Record = {}; + for (const row of runInputRows) { + if (rowUsesRange(row)) continue; + const message = runValueError(row, formData.run_values[row.key] ?? ""); + if (message) runValueErrors[row.key] = message; + } + const plannedParameters = runInputRows + .filter((row) => !rowUsesRange(row) && row.kind !== "unsupported") + .map((row) => { + const raw = formData.run_values[row.key] ?? ""; + return { name: row.name, planned_value: row.kind === "string" ? raw : raw.trim(), unit: row.unit }; + }) + .filter((param) => param.planned_value.trim() !== ""); const selectedExperimentOutputOptions = formData.outputs .filter((output) => output.name && !output.artifact) .map((output) => ({ @@ -371,12 +631,41 @@ function NewExperimentContent() { })), outputs: [], additional_inputs: [], - planned_parameters: [], + run_values: {}, selected_sample_ids: [], })); }; const applyExperimentType = (template: ExperimentTypeRecord) => { + const computation = computationalTemplateInfo(template); + if (computation.computational) { + // The generic wizard cannot run or faithfully copy these templates. + setComputationalTemplate({ + id: template.id, + name: template.name, + tool: computation.tool, + inputs: template.type_parameters || [], + }); + return; + } + setComputationalTemplate(null); + setLoadedTemplateName(template.name); + // Legacy definitions use default_value; catalog metadata uses default. + const definedDefault = (param: ExperimentTypeParameterInput) => + param.default_value !== undefined && param.default_value !== null && param.default_value !== "" + ? encodeValue(param.default_value) + : encodeValue(param.default); + // Keep bounds and allowed values under whichever alias the definition used. + const constraints = (param: ExperimentTypeParameterInput) => { + const allowed = allowedValuesOf(param); + return { + min_value: optionalText(param.min_value) || optionalText(param.minimum), + max_value: optionalText(param.max_value) || optionalText(param.maximum), + required: typeof param.required === "boolean" ? param.required : undefined, + allowed_values: allowed?.values, + allowed_key: allowed?.key, + }; + }; const nextTypeParameters: TypeParam[] = (template.type_parameters || []).map((param) => ({ name: param.name, type: param.type || "float", @@ -390,14 +679,35 @@ function NewExperimentContent() { : param.is_variable ? "variable" : "fixed", - default_value: param.default_value || "", - min_value: param.min_value || "", - max_value: param.max_value || "", + default_value: definedDefault(param), source: param.source === "sample" ? "sample" : "equipment", description: param.description || "", equipment_parameter_id: param.equipment_parameter_id || "", recipe_settable: param.recipe_settable !== false, + ...constraints(param), + })); + const nextAdditionalInputs: TypeParam[] = (template.additional_inputs || []).map((param) => ({ + name: param.name, + type: param.type || "string", + unit: param.unit || "", + is_variable: false, + mode: "fixed", + default_value: definedDefault(param), + source: "sample", + description: param.description || "", + recipe_settable: true, + ...constraints(param), })); + // Every defined value starts at its default; the user edits per run. + const initialRunValues: Record = {}; + for (const param of nextTypeParameters) { + if (param.name && param.mode !== "not_used") { + initialRunValues[runValueKey("equipment", param.name)] = param.default_value; + } + } + for (const param of nextAdditionalInputs) { + if (param.name.trim()) initialRunValues[runValueKey("additional", param.name)] = param.default_value; + } setFormData((prev) => ({ ...prev, template_mode: "existing", @@ -420,28 +730,8 @@ function NewExperimentContent() { ] : nextTypeParameters, outputs: template.outputs || [], - additional_inputs: (template.additional_inputs || []).map((param) => ({ - name: param.name, - type: param.type || "string", - unit: param.unit || "", - is_variable: false, - mode: "fixed", - default_value: param.default_value || "", - min_value: "", - max_value: "", - source: "sample", - description: param.description || "", - recipe_settable: true, - })), - planned_parameters: projectRunMode - ? nextTypeParameters - .filter((param) => param.name && param.mode !== "not_used") - .map((param) => ({ - name: param.name, - planned_value: param.default_value, - unit: param.unit, - })) - : [], + additional_inputs: nextAdditionalInputs, + run_values: projectRunMode ? initialRunValues : {}, })); }; @@ -486,7 +776,7 @@ function NewExperimentContent() { selected_type_id: "", additional_inputs: [ ...prev.additional_inputs, - { name: "", type: "string", unit: "", is_variable: false, mode: "fixed", default_value: "", min_value: "", max_value: "", source: "sample", description: "", recipe_settable: true }, + { name: "", type: "string", unit: "", is_variable: false, mode: "fixed", default_value: "", min_value: "", max_value: "", source: "sample", description: "", recipe_settable: true, required: false }, ], })); }; @@ -494,7 +784,16 @@ function NewExperimentContent() { const updateAdditionalInput = (index: number, field: keyof TypeParam, value: string | boolean) => { setFormData((prev) => { const params = [...prev.additional_inputs]; - params[index] = { ...params[index], [field]: value, source: "sample", is_variable: false, mode: "fixed" }; + const nextParam: TypeParam = { ...params[index], [field]: value, source: "sample", is_variable: false, mode: "fixed" }; + if (field === "type" && typeof value === "string") { + const kind = valueKind(value, "string"); + // Bounds apply only to numbers; don't keep hidden stale ones. + if (kind !== "number" && kind !== "integer") { + nextParam.min_value = ""; + nextParam.max_value = ""; + } + } + params[index] = nextParam; return { ...prev, template_mode: "new", selected_type_id: "", additional_inputs: params }; }); }; @@ -508,238 +807,194 @@ function NewExperimentContent() { })); }; - /* ─── Planned Parameter management ─── */ - const addPlannedParam = () => { - setFormData((prev) => ({ - ...prev, - planned_parameters: [...prev.planned_parameters, { name: "", planned_value: "", unit: "" }], - })); + /* ─── Per-run values ─── */ + const updateRunValue = (key: string, value: string) => { + setFormData((prev) => ({ ...prev, run_values: { ...prev.run_values, [key]: value } })); }; - const updatePlannedParam = (index: number, field: keyof PlannedParam, value: string) => { - setFormData((prev) => { - const params = [...prev.planned_parameters]; - const nextParam = { ...params[index], [field]: value }; - if (field === "name") { - const option = selectedExperimentInputOptions.find((feature) => feature.name === value); - if (option) nextParam.unit = option.unit; - } - params[index] = nextParam; - return { ...prev, planned_parameters: params }; - }); + const firstRunValueError = () => { + const key = Object.keys(runValueErrors)[0]; + const row = runInputRows.find((item) => item.key === key); + return row ? `${row.label}: ${runValueErrors[key]}` : null; }; - const removePlannedParam = (index: number) => { - setFormData((prev) => ({ - ...prev, - planned_parameters: prev.planned_parameters.filter((_, i) => i !== index), - })); - }; - - /* ─── Auto-populate planned params from type params ─── */ - const populatePlannedFromType = () => { - const planned = formData.type_parameters - .filter((p) => p.mode === "fixed" && p.name) - .map((p) => ({ - name: p.name, - planned_value: p.default_value, - unit: p.unit, - })); - updateField("planned_parameters", planned); - }; - - const handleSubmit = async (createInvocation: boolean) => { - setSubmitting(true); - setSubmitResult(null); - - if ( - !formData.type_name.trim() || - !formData.selected_equipment - ) { - setSubmitResult({ - status: "error", - message: "Unit Experiment name and equipment are required.", - }); - setSubmitting(false); + const saveDefinition = async () => { + if (!formData.type_name.trim() || !formData.selected_equipment) { + setSubmitResult({ status: "error", message: "Unit Experiment name and equipment are required." }); return; } - - if ( - createInvocation && - (!formData.experiment_name.trim() || !formData.selected_project) - ) { + if (editMode) { + await saveEditedDefinition(); + return; + } + if (formData.template_mode === "existing" && formData.selected_type_id) { + // Nothing changed, so there is nothing to save; say so plainly. setSubmitResult({ - status: "error", - message: "Project and run name are required when running an unit experiment.", + status: "success", + message: `Unit Experiment "${formData.type_name}" is unchanged and already in the library.`, + href: "/experiment", + linkLabel: "View unit experiments.", }); - setSubmitting(false); return; } - - if ( - createInvocation && - !formData.planned_parameters.some((param) => param.name.trim() && param.planned_value.trim()) - ) { + const result = await saveExperimentType(definitionPayload()); + if (!result.ok || !result.data) { setSubmitResult({ status: "error", - message: "Enter at least one planned parameter value before running the unit experiment.", + message: `Unit Experiment was not saved: ${result.error || "no response from the server"}.`, }); - setSubmitting(false); return; } + setSubmitResult({ + status: "success", + message: viewingDefinition + ? `Saved as a new unit experiment "${formData.type_name}". The original${loadedTemplateName ? ` "${loadedTemplateName}"` : ""} is unchanged.` + : `Unit Experiment "${formData.type_name}" saved. It is reusable across projects.`, + href: "/experiment", + linkLabel: "View unit experiments.", + }); + }; - if (projectRunMode && !formData.selected_type_id) { + const saveEditedDefinition = async () => { + if (!editTemplate) { setSubmitResult({ status: "error", - message: "Select an existing unit experiment before creating a project run.", + message: "The unit experiment is still loading. Wait for it to load before saving.", }); - setSubmitting(false); return; } + const updated = await updateExperimentType(editTypeId, { + ...definitionPayload(), + // The backend stores these as sent, so carry them over. + author_name: editTemplate.author_name || "", + author_email: editTemplate.author_email || "", + change_summary: changeSummary.trim(), + }); + setSubmitResult( + updated.ok + ? { + status: "success", + message: `Unit Experiment "${formData.type_name}" saved as version ${updated.data?.version ?? ""}.`, + href: "/experiment", + linkLabel: "View unit experiments.", + } + : { + status: "error", + message: updated.error || "Failed to save the unit experiment.", + }, + ); + }; - if ( - createInvocation && - formData.optimization_mode === "optimization" && - !formData.primary_target - ) { + const definitionPayload = () => ({ + type_name: formData.type_name, + type_description: formData.type_description, + scientific_objective: formData.scientific_objective, + equipment_id: formData.selected_equipment, + type_parameters: formData.type_parameters + .filter((param) => param.name.trim() && param.mode !== "not_used") + .map((param) => definitionPayloadParam({ + ...param, + is_variable: param.recipe_settable !== false && param.mode === "variable", + mode: + param.recipe_settable === false && param.mode === "variable" + ? "fixed" + : param.mode, + })), + outputs: formData.outputs.filter((output) => output.name.trim()), + additional_inputs: formData.additional_inputs + .filter((param) => param.name.trim()) + .map((param) => definitionPayloadParam({ ...param, required: param.required === true })), + visibility: formData.visibility, + primary_target: "", + primary_objective: "", + secondary_target: "", + secondary_objective: "", + }); + + const saveRun = async () => { + if (!formData.selected_type_id) { setSubmitResult({ status: "error", - message: "Choose a primary optimization target before starting an optimization campaign.", + message: prefillError || "The unit experiment has not loaded, so no run can be saved.", }); - setSubmitting(false); return; } - - const typePayload = { - type_name: formData.type_name, - type_description: formData.type_description, - scientific_objective: formData.scientific_objective, - equipment_id: formData.selected_equipment, - type_parameters: formData.type_parameters - .filter((param) => param.name.trim() && param.mode !== "not_used") - .map((param) => ({ - ...param, - is_variable: param.recipe_settable !== false && param.mode === "variable", - mode: - param.recipe_settable === false && param.mode === "variable" - ? "fixed" - : param.mode, - })), - outputs: formData.outputs.filter((output) => output.name.trim()), - additional_inputs: formData.additional_inputs.filter((param) => param.name.trim()), - visibility: formData.visibility, - primary_target: "", - primary_objective: "", - secondary_target: "", - secondary_objective: "", - }; - - if (editMode) { - if (!editTemplate) { - setSubmitResult({ - status: "error", - message: "The unit experiment is still loading. Wait for it to load before saving.", - }); - setSubmitting(false); - return; - } - const updated = await updateExperimentType(editTypeId, { - ...typePayload, - // The backend stores these as sent, so carry them over. - author_name: editTemplate.author_name || "", - author_email: editTemplate.author_email || "", - change_summary: changeSummary.trim(), - }); - setSubmitResult( - updated.ok - ? { - status: "success", - message: `Unit Experiment "${formData.type_name}" saved as version ${updated.data?.version ?? ""}.`, - href: "/experiment", - linkLabel: "View unit experiments.", - } - : { - status: "error", - message: updated.error || "Failed to save the unit experiment.", - }, - ); - setSubmitting(false); + if (!formData.experiment_name.trim() || !formData.selected_project) { + setSubmitResult({ status: "error", message: "Choose a project and enter a run name." }); return; } - - const typeResult = (projectRunMode || formData.template_mode === "existing") && formData.selected_type_id - ? { status: "reused", type_id: formData.selected_type_id } - : await createExperimentType(typePayload); - - if (!typeResult) { - setSubmitResult({ - status: "error", - message: "Failed to save the unit experiment. Check the backend connection.", - }); - setSubmitting(false); + const valueError = firstRunValueError(); + if (valueError) { + setShowRunValueErrors(true); + setSubmitResult({ status: "error", message: `Correct the run values before saving. ${valueError}` }); return; } - - if (!createInvocation) { + if (optimizationRun && !formData.primary_target) { setSubmitResult({ - status: "success", - message: - formData.template_mode === "existing" - ? `Unit Experiment "${formData.type_name}" is ready to reuse.` - : `Unit Experiment "${formData.type_name}" saved. It is reusable across projects.`, - href: "/unit experiment", - linkLabel: "View unit experiments.", + status: "error", + message: "Choose a primary optimization target before starting an optimization campaign.", }); - setSubmitting(false); return; } - if (!experimentRequestKey.current) { experimentRequestKey.current = newClientRequestId(); } - const experimentResult = await createExperiment({ + const result = await saveProjectRun({ client_request_id: experimentRequestKey.current, experiment_name: formData.experiment_name, selected_equipment: formData.selected_equipment, project_id: formData.selected_project, - type_id: typeResult.type_id, + type_id: formData.selected_type_id, planned_date: formData.planned_date, - planned_parameters: formData.planned_parameters.filter((param) => param.name.trim()), + planned_parameters: plannedParameters, optimization_context: { mode: formData.optimization_mode, - primary_target: formData.optimization_mode === "optimization" ? formData.primary_target : "", - primary_objective: formData.optimization_mode === "optimization" ? formData.primary_objective : "", - secondary_target: formData.optimization_mode === "optimization" ? formData.secondary_target : "", - secondary_objective: formData.optimization_mode === "optimization" ? formData.secondary_objective : "", + primary_target: optimizationRun ? formData.primary_target : "", + primary_objective: optimizationRun ? formData.primary_objective : "", + secondary_target: optimizationRun ? formData.secondary_target : "", + secondary_objective: optimizationRun ? formData.secondary_objective : "", notes: formData.optimization_notes, }, sample_ids: formData.selected_sample_ids, }); - - if (!experimentResult) { + const experimentResult = result.data; + if (!result.ok || !experimentResult) { setSubmitResult({ status: "error", - message: "Unit Experiment saved, but the project run failed to save.", + message: `The run was not saved: ${result.error || "no response from the server"}.`, }); - setSubmitting(false); return; } - experimentRequestKey.current = null; + const saved = `Run plan "${formData.experiment_name}" saved as ${experimentResult.execution_request_id || "a new request"}.`; setSubmitResult({ status: "success", message: experimentResult.optimization_status === "queued" - ? `Run "${formData.experiment_name}" submitted as ${experimentResult.execution_request_id || "a new request"}. Recipe generation is running in the background.` + ? `${saved} Recipe generation is running in the background.` : experimentResult.optimization_status === "completed" - ? `Run "${formData.experiment_name}" submitted as ${experimentResult.execution_request_id || "a new request"}. Generated ${experimentResult.proposal_count || 0} proposed recipes.` + ? `${saved} Generated ${experimentResult.proposal_count || 0} proposed recipes.` : experimentResult.optimization_status === "skipped" - ? `Run "${formData.experiment_name}" submitted as ${experimentResult.execution_request_id || "a new request"}. Recipe generation was not requested for this run.` - : `Run "${formData.experiment_name}" submitted, but automatic recipe generation did not complete.`, + ? `${saved} Recipe generation was not requested for this run.` + : `${saved} Automatic recipe generation did not complete.`, href: `/projects/${encodeURIComponent(formData.selected_project)}`, linkLabel: "View project.", }); - setSubmitting(false); + }; + + const handleSubmit = async (createInvocation: boolean) => { + setSubmitting(true); + setSubmitResult(null); + try { + await (createInvocation ? saveRun() : saveDefinition()); + } catch (err) { + setSubmitResult({ + status: "error", + message: err instanceof Error ? err.message : "Unexpected error while saving.", + }); + } finally { + setSubmitting(false); + } }; const inputClass = @@ -761,6 +1016,21 @@ function NewExperimentContent() {

+ {viewingDefinition && loadedTemplateName && ( +
+ Viewing “{loadedTemplateName}”. Changing any field and saving creates a new + unit experiment; the original is not modified. To enter values for a run, + use Run in the unit experiment library or + New Run on a project process step. +
+ )} + {prefillError && ( +
+ + {prefillError} +
+ )} +
@@ -974,46 +1244,120 @@ function NewExperimentContent() {

Additional Inputs / Samples

- Declare non-equipment inputs required to run this unit experiment, - such as wafer, sample, substrate, mask, or material batch. + Declare non-equipment inputs for this unit experiment, such as wafer, + sample, substrate, mask, or material batch. Each run can record a + value of the declared type; mark an input Required if every run must + provide one.

-
+ - {formData.additional_inputs.map((param, i) => ( -
- updateAdditionalInput(i, "name", e.target.value)} - placeholder="e.g., Wafer" - className={inputClass} - /> - updateAdditionalInput(i, "unit", e.target.value)} - placeholder="e.g., sample ID" - className={inputClass} - /> - updateAdditionalInput(i, "description", e.target.value)} - placeholder="Required sample/material context" - className={inputClass} - /> - -
- ))} + {formData.additional_inputs.map((param, i) => { + const kind = valueKind(param.type, "string"); + const numeric = kind === "number" || kind === "integer"; + const typeValue = param.type || "string"; + const knownType = ADDITIONAL_INPUT_TYPES.some((option) => option.value === typeValue); + const rowLabel = param.name.trim() || `additional input ${i + 1}`; + return ( +
+
+ updateAdditionalInput(i, "name", e.target.value)} + placeholder="e.g., Wafer" + className={inputClass} + /> + + updateAdditionalInput(i, "unit", e.target.value)} + placeholder="e.g., sample ID" + className={inputClass} + /> + updateAdditionalInput(i, "description", e.target.value)} + placeholder="Sample/material context" + className={inputClass} + /> + + +
+ {(numeric || (param.allowed_values && param.allowed_values.length > 0)) && ( +
+ {numeric && ( + <> + updateAdditionalInput(i, "min_value", e.target.value)} + placeholder="Min (optional)" + className={inputClass} + /> + updateAdditionalInput(i, "max_value", e.target.value)} + placeholder="Max (optional)" + className={inputClass} + /> + + )} + {param.allowed_values && param.allowed_values.length > 0 && ( +

+ Allowed values (kept from the definition): {param.allowed_values.map((value) => encodeValue(value)).join(", ")} +

+ )} +
+ )} +
+ ); + })}
+ {prefillError && ( +
+ + {prefillError} +
+ )} +
- + updateField("experiment_name", e.target.value)} @@ -1111,8 +1463,8 @@ function NewExperimentContent() {
- - {projectRunMode && selectedProject ? ( + + {projectFromUrl && selectedProject ? (
{selectedProject.name} ({selectedProject.access})
@@ -1137,6 +1489,7 @@ function NewExperimentContent() {
) : ( updatePlannedParam(i, "name", e.target.value)} - className={inputClass} - > - - {selectedExperimentInputOptions.map((feature) => ( - - ))} - - updatePlannedParam(i, "planned_value", e.target.value)} - placeholder="55" - className={inputClass} - /> - updatePlannedParam(i, "unit", e.target.value)} - placeholder="sccm" - className={cn(inputClass, "text-[hsl(var(--muted-foreground))]")} - /> - -
- ))} - - - {selectedExperimentInputOptions.length === 0 && ( -

- Select at least one unit experiment input as Fixed or Variable before adding planned values. -

- )} -
-

Run Purpose

@@ -1406,6 +1681,132 @@ function NewExperimentContent() {
)}
+ + {/* One value per defined input, including additional inputs */} +
+
+

Planned Values

+

+ {optimizationRun + ? "Fixed inputs and additional inputs use the values entered here. Variable inputs take no point value: the optimizer searches the range defined in the unit experiment." + : "Enter the value used in this run for every input. Values start at the unit experiment defaults; units come from the definition."} +

+
+ {runInputRows.length === 0 ? ( +

+ This unit experiment defines no inputs. +

+ ) : ( +
+ + {runInputRows.map((row, index) => { + const inputId = `run-value-${index}`; + const hintId = `${inputId}-hint`; + const errorId = `${inputId}-error`; + const value = formData.run_values[row.key] ?? ""; + const usesRange = rowUsesRange(row); + const error = usesRange ? undefined : runValueErrors[row.key]; + const showError = Boolean(error) && (showRunValueErrors || value.trim() !== "" || row.kind === "unsupported"); + const allowedOptions = (row.allowed || []).map((option) => encodeValue(option)); + const describedBy = showError ? `${hintId} ${errorId}` : hintId; + const hasRange = row.min_value.trim() !== "" || row.max_value.trim() !== ""; + const rangeText = `${row.min_value || "—"} to ${row.max_value || "—"}`; + const controlProps = { + id: inputId, + "aria-describedby": describedBy, + "aria-invalid": showError || undefined, + "aria-required": row.required || undefined, + className: cn(inputClass, showError && "border-red-500"), + }; + return ( +
+
+ {usesRange ? ( + {row.label} + ) : ( + + )} +

+ {row.source === "additional" ? "Additional input" : row.mode === "variable" ? "Variable" : "Fixed"} + {row.kind === "unsupported" ? ` · unsupported type "${row.declaredType}"` : ` · ${KIND_LABEL[row.kind]}`} + {hasRange ? ` · range ${rangeText}` : ""} + {row.required ? "" : " · optional"} + {row.description ? ` · ${row.description}` : ""} +

+
+
+ {usesRange ? ( +

+ {row.min_value.trim() && row.max_value.trim() + ? `Optimizer searches ${rangeText}` + : "No complete range is defined, so the optimizer cannot search this input."} +

+ ) : row.kind === "unsupported" ? ( + + ) : allowedOptions.length > 0 ? ( + + ) : row.kind === "boolean" ? ( + + ) : row.kind === "object" || row.kind === "array" ? ( +