diff --git a/api/data_loader_pg.py b/api/data_loader_pg.py index 191a87d..3988293 100644 --- a/api/data_loader_pg.py +++ b/api/data_loader_pg.py @@ -854,6 +854,227 @@ def ensure_equipment_run_trace_events_pg() -> None: conn.close() +def ensure_upload_experiment_and_run_files_pg() -> None: + """Link uploaded runs to an experiment and store per-run file outputs. + + - ``equipment_runs.experiment_type_id`` / ``data_uploads.experiment_type_id`` + record the experiment (``experiment_types.id``) an upload was processed + into. ``''`` means "no experiment", matching ``execution_request_id``. It + is a plain label rather than a foreign key so deleting a template never + deletes or orphans measured runs. + - ``data_upload_files`` stages the attachments (e.g. XRD pattern CSVs) sent + with an upload until it is processed. Like ``data_uploads`` it has no RLS; + the API authorizes access through the owning upload record. + - ``equipment_run_files`` holds each run's file outputs. Content is kept in + Postgres (as the model registry does) because the API has no durable file + volume; visibility follows the run's project like the trace tables. + + Idempotent — safe to call on every startup. + """ + conn = get_pg_migration_connection() + try: + with conn.cursor() as cur: + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "experiment_type_id VARCHAR(255) NOT NULL DEFAULT ''" + ) + # Version of the experiment template the run was captured under, so + # old results keep their old schema when the template evolves. + cur.execute( + "ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS " + "experiment_type_version INT" + ) + # Both columns are metadata-only changes on Postgres 11+ (constant + # default / nullable). Commit them now so their ACCESS EXCLUSIVE lock is + # released before the index scans the table, and build the index + # concurrently so reads and writes continue meanwhile. + conn.commit() + conn.autocommit = True + with conn.cursor() as cur: + # A cancelled concurrent build leaves an INVALID index behind that + # IF NOT EXISTS would skip forever; drop it so it's rebuilt. + cur.execute( + """ + SELECT i.indisvalid + FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relname = 'idx_equipment_runs_experiment_type_id' + """ + ) + existing = cur.fetchone() + if existing is not None and not existing[0]: + cur.execute( + "DROP INDEX CONCURRENTLY IF EXISTS " + "idx_equipment_runs_experiment_type_id" + ) + cur.execute( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS " + "idx_equipment_runs_experiment_type_id " + "ON equipment_runs(experiment_type_id) " + "WHERE experiment_type_id <> ''" + ) + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + "ALTER TABLE data_uploads ADD COLUMN IF NOT EXISTS " + "experiment_type_id VARCHAR(255) NOT NULL DEFAULT ''" + ) + cur.execute( + """ + CREATE TABLE IF NOT EXISTS data_upload_files ( + id BIGSERIAL PRIMARY KEY, + upload_id UUID NOT NULL + REFERENCES data_uploads(id) ON DELETE CASCADE, + relative_path TEXT NOT NULL, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(255) NOT NULL DEFAULT '', + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + content BYTEA NOT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + UNIQUE (upload_id, relative_path) + ) + """ + ) + cur.execute( + "GRANT SELECT, INSERT, UPDATE, DELETE ON data_upload_files TO api_client" + ) + cur.execute( + "GRANT USAGE, SELECT ON SEQUENCE data_upload_files_id_seq TO api_client" + ) + cur.execute( + """ + CREATE TABLE IF NOT EXISTS equipment_run_files ( + id BIGSERIAL PRIMARY KEY, + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + output_name VARCHAR(255) NOT NULL, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(255) NOT NULL DEFAULT '', + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + content BYTEA NOT NULL, + upload_id UUID, + source_path TEXT NOT NULL DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + UNIQUE (equipment_run_id, output_name) + ) + """ + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_equipment_run_files_upload_id " + "ON equipment_run_files(upload_id)" + ) + # Which registered samples a run produced (role 'output') or used + # (role 'input'). The output of one unit process is the input of the + # next, which is how runs are connected into a sample's history. + cur.execute( + """ + CREATE TABLE IF NOT EXISTS equipment_run_samples ( + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_id VARCHAR(80) NOT NULL + REFERENCES samples(id) ON DELETE CASCADE, + role VARCHAR(20) NOT NULL, + field_name VARCHAR(255) NOT NULL, + upload_id UUID, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (equipment_run_id, role, field_name), + CHECK (role IN ('input', 'output')) + ) + """ + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_equipment_run_samples_sample_id " + "ON equipment_run_samples(sample_id)" + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_equipment_run_samples_upload_id " + "ON equipment_run_samples(upload_id)" + ) + cur.execute("ALTER TABLE equipment_run_samples ENABLE ROW LEVEL SECURITY") + cur.execute("GRANT SELECT ON equipment_run_samples TO api_client") + cur.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_samples' + AND policyname = 'equipment_run_sample_visibility_policy' + ) THEN + CREATE POLICY equipment_run_sample_visibility_policy + ON equipment_run_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; + END + $$; + """ + ) + cur.execute("ALTER TABLE equipment_run_files ENABLE ROW LEVEL SECURITY") + cur.execute("GRANT SELECT ON equipment_run_files TO api_client") + cur.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_files' + AND policyname = 'equipment_run_file_visibility_policy' + ) THEN + CREATE POLICY equipment_run_file_visibility_policy + ON equipment_run_files + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_files.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 + FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; + END + $$; + """ + ) + conn.commit() + finally: + conn.close() + + def ensure_glance_ingestion_audit_pg() -> None: """Create connector audit storage; it is intentionally not API-readable.""" conn = get_pg_migration_connection() @@ -990,7 +1211,10 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: r.raw_payload_json #>> '{{glance,run_start_time}}', '' ) AS source_run_start_time, COALESCE(trace_counts.sample_count, 0)::int AS trace_sample_count, + COALESCE(file_counts.file_count, 0)::int AS file_count, r.execution_request_id, + r.experiment_type_id, + r.experiment_type_version, p.name AS project_name, r.equipment_id AS equipment_id, COALESCE(em.equipment_name, p.equipment_name, r.equipment_id) AS equipment_name, @@ -1005,6 +1229,11 @@ def _equipment_runs_base_select(*, include_raw_payload: bool = False) -> str: FROM equipment_run_trace_samples trace_sample WHERE trace_sample.equipment_run_id = r.id ) trace_counts ON true + LEFT JOIN LATERAL ( + SELECT COUNT(*)::int AS file_count + FROM equipment_run_files run_file + WHERE run_file.equipment_run_id = r.id + ) file_counts ON true """ @@ -1052,6 +1281,10 @@ def _shape_equipment_run_record(row) -> Dict[str, Any]: else: rec["source_run_id"] = None rec["trace_sample_count"] = int(rec.get("trace_sample_count") or 0) + rec["file_count"] = int(rec.get("file_count") or 0) + rec["experiment_type_id"] = str(rec.get("experiment_type_id") or "") + version = rec.get("experiment_type_version") + rec["experiment_type_version"] = int(version) if version is not None else None rec["equipment_name"] = rec.get("equipment_name") or rec.get("equipment_id") or "" raw_payload = rec.pop("raw_payload_json", None) if raw_payload is not None: @@ -1116,13 +1349,38 @@ def sync_equipment_runs_pg( catalog URL). Any rows left over from a previous, longer version of the upload are pruned afterwards. """ - from psycopg2.extras import execute_values - if not rows: return 0 conn = get_pg_worker_connection() try: + with conn.cursor() as cur: + count = _sync_equipment_rows( + cur, + equipment_id=equipment_id, + rows=rows, + upload_id=upload_id, + upload_filename=upload_filename, + ) + conn.commit() + return count + finally: + conn.close() + + +def _sync_equipment_rows( + cur, + *, + equipment_id: str, + rows: List[Dict[str, Any]], + upload_id: Optional[str], + upload_filename: str, +) -> int: + """Body of ``sync_equipment_runs_pg`` on a caller-owned cursor, so upload + processing can sync runs, files and sample links in one transaction.""" + from psycopg2.extras import execute_values + + if True: insert_rows = [] for index, row in enumerate(rows): insert_rows.append(( @@ -1140,9 +1398,11 @@ def sync_equipment_runs_pg( index, upload_filename, str(row.get("source", "data_upload") or "data_upload"), + str(row.get("experiment_type_id", "") or ""), + row.get("experiment_type_version"), )) - with conn.cursor() as cur: + if True: execute_values( cur, """ @@ -1150,7 +1410,8 @@ def sync_equipment_runs_pg( equipment_id, project_id, lotname, run_date, is_outlier, is_calibration_recipe, outlier_type, inputs_json, outputs_json, raw_payload_json, - upload_id, row_index, upload_filename, source + upload_id, row_index, upload_filename, source, + experiment_type_id, experiment_type_version ) VALUES %s ON CONFLICT (upload_id, row_index) WHERE upload_id IS NOT NULL DO UPDATE SET @@ -1165,7 +1426,17 @@ def sync_equipment_runs_pg( outputs_json = EXCLUDED.outputs_json, raw_payload_json = EXCLUDED.raw_payload_json, upload_filename = EXCLUDED.upload_filename, - source = EXCLUDED.source + source = EXCLUDED.source, + experiment_type_id = EXCLUDED.experiment_type_id, + -- Keep the version a run was first captured under when it's + -- re-processed into the same experiment; a template edit in + -- between must not relabel historical data. + experiment_type_version = CASE + WHEN equipment_runs.experiment_type_id = EXCLUDED.experiment_type_id + AND equipment_runs.experiment_type_version IS NOT NULL + THEN equipment_runs.experiment_type_version + ELSE EXCLUDED.experiment_type_version + END """, insert_rows, ) @@ -1176,10 +1447,7 @@ def sync_equipment_runs_pg( "DELETE FROM equipment_runs WHERE upload_id = %s AND row_index >= %s", (upload_id, len(insert_rows)), ) - conn.commit() return len(insert_rows) - finally: - conn.close() def _normalize_trace_parameter_key(value: Any) -> str: @@ -2288,13 +2556,40 @@ def merge_equipment_runs_pg( if not rows: return 0 - side_key = "inputs" if kind == "input" else "outputs" - raw_key = "input_raw" if kind == "input" else "output_raw" - conn = get_pg_worker_connection() try: - affected = 0 with conn.cursor() as cur: + affected = _merge_equipment_rows( + cur, + equipment_id=equipment_id, + kind=kind, + rows=rows, + upload_id=upload_id, + upload_filename=upload_filename, + ) + conn.commit() + return affected + finally: + conn.close() + + +def _merge_equipment_rows( + cur, + *, + equipment_id: str, + kind: str, + rows: List[Dict[str, Any]], + upload_id: Optional[str], + upload_filename: str, +) -> int: + """Body of ``merge_equipment_runs_pg`` on a caller-owned cursor. The + advisory lock it takes is transaction-scoped, so it is held until the + caller commits (including any file/sample reconciliation).""" + side_key = "inputs" if kind == "input" else "outputs" + raw_key = "input_raw" if kind == "input" else "output_raw" + if True: + affected = 0 + if True: # Serialize all merges for this equipment+project for the duration of # the transaction. SELECT ... FOR UPDATE alone can't prevent two # concurrent merges (e.g. a matching input file and output file @@ -2327,6 +2622,8 @@ def merge_equipment_runs_pg( is_outlier = bool(row.get("is_outlier", False)) is_calib = bool(row.get("is_calibration_recipe", False)) outlier_type = str(row.get("outlier_type", "") or "") + experiment_type_id = str(row.get("experiment_type_id", "") or "") + experiment_type_version = row.get("experiment_type_version") # Resolve and lock exactly one merge target so a pre-existing # duplicate natural key isn't mass-updated and a concurrent merge @@ -2359,20 +2656,49 @@ def merge_equipment_runs_pg( raw_payload_json = raw_payload_json || %s, is_outlier = %s, is_calibration_recipe = %s, - outlier_type = %s + outlier_type = %s, + experiment_type_id = COALESCE( + NULLIF(%s, ''), experiment_type_id + ), + experiment_type_version = CASE + WHEN %s <> '' + AND (experiment_type_id <> %s + OR experiment_type_version IS NULL) + THEN %s + ELSE experiment_type_version + END WHERE id = %s """, - (payload, raw_payload, is_outlier, is_calib, outlier_type, target_id), + ( + payload, raw_payload, is_outlier, is_calib, + outlier_type, experiment_type_id, + experiment_type_id, experiment_type_id, + experiment_type_version, target_id, + ), ) else: cur.execute( """ UPDATE equipment_runs SET outputs_json = %s, - raw_payload_json = raw_payload_json || %s + raw_payload_json = raw_payload_json || %s, + experiment_type_id = COALESCE( + NULLIF(%s, ''), experiment_type_id + ), + experiment_type_version = CASE + WHEN %s <> '' + AND (experiment_type_id <> %s + OR experiment_type_version IS NULL) + THEN %s + ELSE experiment_type_version + END WHERE id = %s """, - (payload, raw_payload, target_id), + ( + payload, raw_payload, experiment_type_id, + experiment_type_id, experiment_type_id, + experiment_type_version, target_id, + ), ) affected += 1 continue @@ -2385,19 +2711,473 @@ def merge_equipment_runs_pg( equipment_id, project_id, lotname, run_date, is_outlier, is_calibration_recipe, outlier_type, inputs_json, outputs_json, raw_payload_json, - upload_id, row_index, upload_filename, source - ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'data_upload') + upload_id, row_index, upload_filename, source, + experiment_type_id, experiment_type_version + ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'data_upload',%s,%s) """, ( equipment_id, project_id, lotname, run_date, is_outlier, is_calib, outlier_type, inputs_json, outputs_json, raw_payload, - upload_id, index, upload_filename, + upload_id, index, upload_filename, experiment_type_id, + experiment_type_version, ), ) affected += 1 - conn.commit() return affected + + +def replace_upload_files_pg(upload_id: str, files: List[Dict[str, Any]]) -> int: + """Replace the staged attachments of an upload in one transaction. + + Each file dict carries relative_path, filename, content_type, size_bytes, + sha256 and content (bytes). Callers authorize access to the upload first; + ``data_upload_files`` has no RLS, like ``data_uploads``. + """ + from psycopg2 import Binary + + conn = get_pg_connection() + try: + with conn.cursor() as cur: + # Lock the upload row: processing claims it with an UPDATE on the + # same row, so replacement and processing can't interleave. + cur.execute( + "SELECT status FROM data_uploads WHERE id = %s FOR UPDATE", + (upload_id,), + ) + row = cur.fetchone() + if row is None: + raise RuntimeError("Upload not found") + if row[0] == "processing": + raise RuntimeError( + "This upload is being processed; add attachments once it finishes." + ) + cur.execute("DELETE FROM data_upload_files WHERE upload_id = %s", (upload_id,)) + for item in files: + cur.execute( + """ + INSERT INTO data_upload_files ( + upload_id, relative_path, filename, content_type, + size_bytes, sha256, content + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + upload_id, + item["relative_path"], + item["filename"], + item.get("content_type") or "", + int(item["size_bytes"]), + item["sha256"], + Binary(item["content"]), + ), + ) + conn.commit() + return len(files) + finally: + conn.close() + + +def list_upload_files_pg(upload_id: str) -> List[Dict[str, Any]]: + """Staged attachment metadata for an upload (no file content).""" + conn = get_pg_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + SELECT id, relative_path, filename, content_type, size_bytes, sha256 + FROM data_upload_files + WHERE upload_id = %s + ORDER BY relative_path + """, + (upload_id,), + ) + rows = [dict(row) for row in cur.fetchall()] + conn.commit() + return rows + finally: + conn.close() + + +def attach_equipment_run_files_pg( + *, + equipment_id: str, + upload_id: str, + kind: str, + links: List[Dict[str, Any]], +) -> int: + """Copy staged upload attachments onto the runs an upload produced. + + Each link names the upload ``row_index`` (and the run's natural key: + project_id, lotname, run_date), the file-typed ``output_name`` and the + staged ``upload_file_id``. Combined ("all") uploads resolve their runs by the + stable ``(upload_id, row_index)`` identity; split uploads by the same natural + key ``merge_equipment_runs_pg`` merges on. Files this upload attached earlier + are replaced, so re-processing is idempotent. The bytes are copied inside + Postgres (INSERT ... SELECT) rather than round-tripped through the API. + """ + conn = get_pg_worker_connection() + try: + with conn.cursor() as cur: + cur.execute( + "DELETE FROM equipment_run_files WHERE upload_id = %s", (upload_id,) + ) + attached = _attach_run_files( + cur, + equipment_id=equipment_id, + upload_id=upload_id, + kind=kind, + links=links, + ) + conn.commit() + return attached + finally: + conn.close() + + +def _attach_run_files( + cur, + *, + equipment_id: str, + upload_id: str, + kind: str, + links: List[Dict[str, Any]], +) -> int: + """Insert one file per link on a caller-owned cursor (no cleanup).""" + attached = 0 + if True: + if True: + for link in links: + run_id = _resolve_upload_run_id( + cur, + equipment_id=equipment_id, + upload_id=upload_id, + kind=kind, + link=link, + ) + if run_id is None: + raise ValueError( + f"No run was found for upload row {link['row_index']}" + ) + cur.execute( + """ + INSERT INTO equipment_run_files ( + equipment_run_id, output_name, filename, content_type, + size_bytes, sha256, content, upload_id, source_path + ) + SELECT %s, %s, f.filename, f.content_type, f.size_bytes, + f.sha256, f.content, f.upload_id, f.relative_path + FROM data_upload_files f + WHERE f.id = %s AND f.upload_id = %s + ON CONFLICT (equipment_run_id, output_name) DO UPDATE SET + filename = EXCLUDED.filename, + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + sha256 = EXCLUDED.sha256, + content = EXCLUDED.content, + upload_id = EXCLUDED.upload_id, + source_path = EXCLUDED.source_path, + created_at = CURRENT_TIMESTAMP + """, + ( + run_id, + link["output_name"], + link["upload_file_id"], + upload_id, + ), + ) + if cur.rowcount != 1: + raise ValueError( + f"Attachment for upload row {link['row_index']} is missing" + ) + attached += 1 + return attached + + +def _resolve_upload_run_id( + cur, + *, + equipment_id: str, + upload_id: str, + kind: str, + link: Dict[str, Any], +) -> Optional[int]: + """Locate the run an upload row produced: combined uploads by their stable + ``(upload_id, row_index)``, split uploads by the natural key that + ``merge_equipment_runs_pg`` merges on.""" + if kind == "all": + cur.execute( + "SELECT id FROM equipment_runs WHERE upload_id = %s AND row_index = %s", + (upload_id, link["row_index"]), + ) + else: + cur.execute( + """ + SELECT id FROM equipment_runs + WHERE equipment_id = %s + AND project_id IS NOT DISTINCT FROM %s + AND lotname IS NOT DISTINCT FROM %s + AND run_date IS NOT DISTINCT FROM %s + ORDER BY id + LIMIT 1 + """, + ( + equipment_id, + link.get("project_id"), + link.get("lotname"), + link.get("run_date") or None, + ), + ) + match = cur.fetchone() + return int(match[0]) if match else None + + +def find_samples_pg(sample_ids: List[str]) -> Dict[str, str]: + """Map each existing sample id to its project (worker connection; callers + decide what the caller may see). Unknown ids are absent from the result.""" + ids = sorted({sample_id for sample_id in sample_ids if sample_id}) + if not ids: + return {} + conn = get_pg_worker_connection() + try: + with conn.cursor() as cur: + cur.execute( + "SELECT id, project_id FROM samples WHERE id = ANY(%s)", + (ids,), + ) + found = {str(row[0]): str(row[1]) for row in cur.fetchall()} + conn.commit() + return found + finally: + conn.close() + + +def link_equipment_run_samples_pg( + *, + equipment_id: str, + upload_id: str, + kind: str, + owner_id: str, + owner_org: str, + links: List[Dict[str, Any]], +) -> int: + """Register the samples an upload's runs produced and link runs to the + samples they produced or used, in one transaction. + + Output links create the sample (platform-issued id) in the run's project if + it doesn't exist yet; input links must name a sample already registered in + the same project. Links this upload made earlier are replaced, so + re-processing is idempotent. Samples themselves are never deleted here: a + sample is a physical object that later experiments may reference. + """ + conn = get_pg_worker_connection() + try: + with conn.cursor() as cur: + cur.execute( + "DELETE FROM equipment_run_samples WHERE upload_id = %s", (upload_id,) + ) + linked = _link_run_samples( + cur, + equipment_id=equipment_id, + upload_id=upload_id, + kind=kind, + owner_id=owner_id, + owner_org=owner_org, + links=links, + ) + conn.commit() + return linked + finally: + conn.close() + + +def _link_run_samples( + cur, + *, + equipment_id: str, + upload_id: str, + kind: str, + owner_id: str, + owner_org: str, + links: List[Dict[str, Any]], +) -> int: + """Register output samples and insert run-sample links on a caller-owned + cursor (no cleanup). Missing and foreign samples raise the same error so it + can't be used to probe other projects' sample IDs.""" + linked = 0 + if True: + if True: + for link in links: + run_id = _resolve_upload_run_id( + cur, + equipment_id=equipment_id, + upload_id=upload_id, + kind=kind, + link=link, + ) + if run_id is None: + raise ValueError(f"No run was found for upload row {link['row_index']}") + sample_id = link["sample_id"] + project_id = link.get("project_id") + if link["role"] == "output": + cur.execute( + """ + INSERT INTO samples ( + id, project_id, name, sample_type, status, + metadata_json, owner_id, owner_org + ) VALUES (%s, %s, %s, %s, 'active', %s, %s, %s) + ON CONFLICT (id) DO NOTHING + """, + ( + sample_id, + project_id, + link.get("sample_name") or sample_id, + link.get("sample_type") or "", + Json( + { + "id_scheme": "igsn", + "igsn_registered": False, + "produced_by": { + "equipment_id": equipment_id, + "lotname": link.get("lotname"), + "field": link["field_name"], + }, + } + ), + owner_id, + owner_org, + ), + ) + cur.execute("SELECT project_id FROM samples WHERE id = %s", (sample_id,)) + found = cur.fetchone() + if found is None or str(found[0]) != str(project_id): + raise ValueError( + f"Sample {sample_id} is not an available sample in this project" + ) + cur.execute( + """ + INSERT INTO equipment_run_samples ( + equipment_run_id, sample_id, role, field_name, upload_id + ) VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (equipment_run_id, role, field_name) DO UPDATE SET + sample_id = EXCLUDED.sample_id, + upload_id = EXCLUDED.upload_id, + created_at = CURRENT_TIMESTAMP + """, + (run_id, sample_id, link["role"], link["field_name"], upload_id), + ) + linked += 1 + return linked + + +def ingest_equipment_upload_pg( + *, + equipment_id: str, + kind: str, + rows: List[Dict[str, Any]], + upload_id: str, + upload_filename: str, + file_links: List[Dict[str, Any]], + sample_links: List[Dict[str, Any]], + owner_id: str, + owner_org: str, +) -> int: + """Ingest one processed generic-equipment upload atomically. + + Runs, their stored files and their sample links are written in a single + transaction, so a failure leaves no partial result, and a split merge keeps + its advisory lock until everything is reconciled. On every run this upload + touches, the whole side it replaces is reconciled (not just links it made + earlier, and not just fields in the current schema): combined and output + uploads replace the run's outputs, so all its stored files and output + sample links are rebuilt; combined and input uploads replace its inputs, + so its input sample links are rebuilt. The other side is left alone. + """ + if not rows: + return 0 + conn = get_pg_worker_connection() + try: + with conn.cursor() as cur: + if kind in ("input", "output"): + count = _merge_equipment_rows( + cur, + equipment_id=equipment_id, + kind=kind, + rows=rows, + upload_id=upload_id, + upload_filename=upload_filename, + ) + else: + count = _sync_equipment_rows( + cur, + equipment_id=equipment_id, + rows=rows, + upload_id=upload_id, + upload_filename=upload_filename, + ) + touched: List[int] = [] + for index, row in enumerate(rows): + run_id = _resolve_upload_run_id( + cur, + equipment_id=equipment_id, + upload_id=upload_id, + kind=kind, + link={ + "row_index": index, + "project_id": row.get("project_id"), + "lotname": row.get("lotname"), + "run_date": row.get("run_date"), + }, + ) + if run_id is not None: + touched.append(run_id) + replaces_outputs = kind in ("all", "output") + replaces_inputs = kind in ("all", "input") + # Files are outputs: anything this upload attached before (incl. + # pruned rows), plus every file on the touched runs when this upload + # replaces their outputs. + cur.execute( + """ + DELETE FROM equipment_run_files + WHERE upload_id = %s + OR (%s AND equipment_run_id = ANY(%s)) + """, + (upload_id, replaces_outputs, touched), + ) + _attach_run_files( + cur, + equipment_id=equipment_id, + upload_id=upload_id, + kind=kind, + links=file_links, + ) + cur.execute( + """ + DELETE FROM equipment_run_samples + WHERE upload_id = %s + OR ( + equipment_run_id = ANY(%s) + AND ( + (role = 'output' AND %s) + OR (role = 'input' AND %s) + ) + ) + """, + (upload_id, touched, replaces_outputs, replaces_inputs), + ) + _link_run_samples( + cur, + equipment_id=equipment_id, + upload_id=upload_id, + kind=kind, + owner_id=owner_id, + owner_org=owner_org, + links=sample_links, + ) + conn.commit() + return count + except Exception: + conn.rollback() + raise finally: conn.close() @@ -2523,6 +3303,7 @@ def get_runs_list_pg( is_admin: bool = False, project_id: Optional[str] = None, include_raw_payload: bool = False, + experiment_type_id: Optional[str] = None, ) -> List[Dict[str, Any]]: """ Fetch runs from Postgres. @@ -2553,25 +3334,41 @@ def get_runs_list_pg( base_params.append(project_id) where_clause = (" WHERE " + " AND ".join(conditions)) if conditions else "" + # Experiment grouping only exists on the generic equipment_runs table; + # etcher_runs have no experiment link, so an experiment filter returns + # generic runs only. + etcher_where_clause = where_clause + generic_where_clause = where_clause + etcher_params = list(base_params) + generic_params = list(base_params) + if experiment_type_id: + etcher_where_clause = ( + (where_clause + " AND ") if where_clause else " WHERE " + ) + "false" + generic_where_clause = ( + (where_clause + " AND ") if where_clause else " WHERE " + ) + "r.experiment_type_id = %s" + generic_params.append(experiment_type_id) + # Over-fetch (limit + offset) from each source so a merged, globally # date-sorted page is still correct after slicing in Python. page_cap = max(0, limit) + max(0, offset) etcher_query = ( _run_base_select(include_raw_payload=include_raw_payload) - + where_clause + + etcher_where_clause + " ORDER BY r.run_date DESC NULLS LAST LIMIT %s" ) generic_query = ( _equipment_runs_base_select(include_raw_payload=include_raw_payload) - + where_clause + + generic_where_clause + " ORDER BY r.run_date DESC NULLS LAST LIMIT %s" ) with conn.cursor(cursor_factory=RealDictCursor) as cur: - cur.execute(etcher_query, tuple(base_params + [page_cap])) + cur.execute(etcher_query, tuple(etcher_params + [page_cap])) etcher_rows = cur.fetchall() - cur.execute(generic_query, tuple(base_params + [page_cap])) + cur.execute(generic_query, tuple(generic_params + [page_cap])) generic_rows = cur.fetchall() canonical_records = [_shape_run_record(row) for row in etcher_rows] @@ -2705,6 +3502,12 @@ def get_run_detail_pg( conn.close() return None record = _shape_equipment_run_record(row) + record["files"] = _equipment_run_file_list( + conn, run_id - EQUIPMENT_RUN_ID_OFFSET + ) + record["samples"] = _equipment_run_sample_list( + conn, run_id - EQUIPMENT_RUN_ID_OFFSET + ) source_run_id = record.get("source_run_id") if ( str(record.get("source") or "").startswith("glance") @@ -2755,6 +3558,174 @@ def get_run_detail_pg( return None +def _equipment_run_file_list(conn, equipment_run_id: int) -> List[Dict[str, Any]]: + """File-output metadata (no content) for one generic run, on ``conn`` so + the same RLS scope as the run itself applies.""" + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + SELECT id, output_name, filename, content_type, size_bytes, sha256, + source_path, created_at + FROM equipment_run_files + WHERE equipment_run_id = %s + ORDER BY output_name + """, + (equipment_run_id,), + ) + rows = [dict(row) for row in cur.fetchall()] + for row in rows: + if row.get("created_at") is not None and hasattr(row["created_at"], "isoformat"): + row["created_at"] = row["created_at"].isoformat() + row["size_bytes"] = int(row.get("size_bytes") or 0) + return rows + + +def _equipment_run_sample_list(conn, equipment_run_id: int) -> List[Dict[str, Any]]: + """Samples a generic run produced (role 'output') or used ('input'), on + ``conn`` so the run's RLS scope applies.""" + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + SELECT sample_id, role, field_name + FROM equipment_run_samples + WHERE equipment_run_id = %s + ORDER BY role DESC, field_name + """, + (equipment_run_id,), + ) + return [dict(row) for row in cur.fetchall()] + + +def get_sample_lineage_pg( + sample_id: str, + nanohub_user_id: Optional[str] = None, + is_admin: bool = False, +) -> Optional[Dict[str, Any]]: + """A sample and every visible run that produced or used it, oldest first. + + This is how one unit process is followed into the next (synthesis -> XRD): + the producing run lists the sample as an output, downstream runs as an + input. Returns ``None`` when the sample doesn't exist or isn't visible.""" + # Explicit project visibility (open project, or a member), in addition to + # RLS, so the endpoint stays safe if RLS enforcement is switched off. + visible_sql = "" + visible_params: List[Any] = [] + if not is_admin: + visible_sql = """ + AND EXISTS ( + SELECT 1 FROM projects vis_p + WHERE vis_p.id = {column} + AND ( + vis_p.access_mode = 'open' + OR EXISTS ( + SELECT 1 FROM project_members vis_pm + WHERE vis_pm.project_id = vis_p.id + AND vis_pm.nanohub_user_id = %s + ) + ) + ) + """ + visible_params = [nanohub_user_id or ""] + conn = _scoped_connection(nanohub_user_id, is_admin) + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + SELECT id, project_id, name, sample_type, external_id, + status, metadata_json, created_at + FROM samples + WHERE id = %s + """ + + visible_sql.format(column="samples.project_id"), + [sample_id, *visible_params], + ) + sample = cur.fetchone() + if not sample: + conn.commit() + return None + cur.execute( + """ + SELECT equipment_run_id, role, field_name + FROM equipment_run_samples + WHERE sample_id = %s + """, + (sample_id,), + ) + links = [dict(row) for row in cur.fetchall()] + run_ids = sorted({int(link["equipment_run_id"]) for link in links}) + records: List[Dict[str, Any]] = [] + if run_ids: + cur.execute( + _equipment_runs_base_select() + + " WHERE r.id = ANY(%s)" + + visible_sql.format(column="r.project_id") + + " ORDER BY r.run_date ASC NULLS LAST, r.id", + [run_ids, *visible_params], + ) + records = [_shape_equipment_run_record(row) for row in cur.fetchall()] + conn.commit() + finally: + conn.close() + + by_run: Dict[int, List[Dict[str, Any]]] = {} + for link in links: + by_run.setdefault(int(link["equipment_run_id"]) + EQUIPMENT_RUN_ID_OFFSET, []).append( + {"role": link["role"], "field_name": link["field_name"]} + ) + for record in records: + record["sample_links"] = by_run.get(int(record["run_id"]), []) + created_at = sample.get("created_at") + return { + "sample": { + "id": sample["id"], + "project_id": sample["project_id"], + "name": sample["name"], + "sample_type": sample.get("sample_type") or "", + "external_id": sample.get("external_id") or "", + "status": sample.get("status") or "active", + "metadata": sample.get("metadata_json") or {}, + "created_at": created_at.isoformat() if hasattr(created_at, "isoformat") else created_at, + }, + "runs": records, + } + + +def get_equipment_run_file_pg( + run_id: int, + file_id: int, + nanohub_user_id: Optional[str] = None, + is_admin: bool = False, +) -> Optional[Dict[str, Any]]: + """Return one stored file output (metadata + ``content`` bytes) of a + generic run, or ``None`` when the run/file doesn't exist or the caller + can't see the run's project. RLS on ``equipment_run_files`` enforces the + same project visibility as the run, so a hidden run's files can't be + probed by id.""" + if run_id < EQUIPMENT_RUN_ID_OFFSET: + return None + conn = _scoped_connection(nanohub_user_id, is_admin) + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + """ + SELECT id, output_name, filename, content_type, size_bytes, + sha256, content + FROM equipment_run_files + WHERE id = %s AND equipment_run_id = %s + """, + (file_id, run_id - EQUIPMENT_RUN_ID_OFFSET), + ) + row = cur.fetchone() + conn.commit() + finally: + conn.close() + if not row: + return None + record = dict(row) + record["content"] = bytes(record["content"]) + return record + + def get_equipment_run_trace_pg( run_id: int, *, diff --git a/api/db_migrations.py b/api/db_migrations.py index 4c9fac5..b1b9a8b 100644 --- a/api/db_migrations.py +++ b/api/db_migrations.py @@ -169,6 +169,7 @@ def run_schema_migrations() -> None: ensure_etcher_recipe_name_column_pg, ensure_etcher_run_file_refs_pg, ensure_glance_ingestion_audit_pg, + ensure_upload_experiment_and_run_files_pg, get_pg_migration_connection, ) from metadata_pg import ( @@ -221,6 +222,8 @@ def run_schema_migrations() -> None: ensure_equipment_access_table_pg() ensure_data_uploads_kind_column_pg() ensure_data_uploads_processing_started_at_pg() + # After data_uploads and equipment_runs exist (both referenced by FKs). + ensure_upload_experiment_and_run_files_pg() ensure_ml_endpoint_cache_table_pg() ensure_platform_notifications_table_pg() ensure_role_change_requests_table_pg() diff --git a/api/main.py b/api/main.py index 75622c1..de5b08b 100644 --- a/api/main.py +++ b/api/main.py @@ -49,6 +49,8 @@ "equipment_runs", "equipment_run_trace_samples", "equipment_run_trace_events", + "equipment_run_files", + "equipment_run_samples", "glance_ingestion_audit", "samples", "experiment_samples", @@ -60,6 +62,7 @@ "experiment_recipe_proposals", "purr_publication_requests", "data_uploads", + "data_upload_files", "ingestion_logs", "equipment_access", ) diff --git a/api/metadata_pg.py b/api/metadata_pg.py index 0d3be9a..acbeb4c 100644 --- a/api/metadata_pg.py +++ b/api/metadata_pg.py @@ -2525,12 +2525,12 @@ def list_experiment_types_pg( if equipment_id: conditions.append("(equipment_id = %s OR equipment_id = '')") params.append(equipment_id) - if user.role != "admin": - conditions.append( - "(owner_id = %s OR visibility IN ('shared', 'published') OR owner_org = %s)" - ) - params.extend([user.id, user.organization]) - where_sql = f"WHERE {' AND '.join(conditions)}" if conditions else "" + where_sql = f"WHERE {' AND '.join(conditions)}" if conditions else "WHERE true" + # Same rule as everywhere else templates are used (and as RLS): own, + # shared/published, same organization, or attached to a readable project. + visibility_sql, visibility_params = _visible_experiment_type_filter(user) + where_sql += visibility_sql + params.extend(visibility_params) try: with conn.cursor(cursor_factory=RealDictCursor) as cur: @@ -2549,6 +2549,60 @@ def list_experiment_types_pg( conn.close() +def get_upload_target_experiment_pg( + *, + type_id: str, + user: PlatformUser, + project_id: str, +) -> dict[str, Any] | None: + """ + Return the experiment template an upload may be processed into, or None + when ``user`` can't see it. + + The result carries ``equipment_id``, ``owner_id`` and + ``attached_to_project`` (whether ``project_id`` uses the template through + ``project_experiments``) so the caller can decide whether the user may file + runs under it. Visibility uses the same rule as the template library. + """ + visibility_sql, visibility_params = _visible_experiment_type_filter(user) + conn = get_pg_connection(user) + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + f""" + SELECT experiment_types.id, + experiment_types.name, + experiment_types.equipment_id, + experiment_types.owner_id, + experiment_types.version, + EXISTS ( + SELECT 1 + FROM project_experiments target_pe + WHERE target_pe.type_id = experiment_types.id + AND target_pe.project_id = %s + ) AS attached_to_project + FROM experiment_types + WHERE experiment_types.id = %s + {visibility_sql} + """, + [project_id, type_id, *visibility_params], + ) + row = cur.fetchone() + conn.commit() + finally: + conn.close() + if not row: + return None + return { + "id": row["id"], + "name": row["name"] or row["id"], + "equipment_id": str(row["equipment_id"] or ""), + "owner_id": row["owner_id"], + "version": int(row["version"]) if row.get("version") is not None else None, + "attached_to_project": bool(row["attached_to_project"]), + } + + def list_public_experiment_types_pg( *, equipment_id: str = "", @@ -2906,6 +2960,22 @@ def _ensure_project_write_access(cur, *, project_id: str, user: PlatformUser) -> raise PermissionError("Only project members or admins can change this project") +def user_can_write_project_pg(*, project_id: str, user: PlatformUser) -> bool: + """Whether ``user`` may add data to ``project_id`` (admin, or the project's + PI or a member). Viewers of open projects can't.""" + conn = get_pg_connection(user) + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + _ensure_project_write_access(cur, project_id=project_id, user=user) + except PermissionError: + return False + conn.commit() + return True + finally: + conn.close() + + def _ensure_project_publish_access(cur, *, project_id: str, user: PlatformUser) -> None: role = _ensure_project_access(cur, project_id=project_id, user=user) if user.role == "admin" or role == "pi": @@ -6840,7 +6910,13 @@ def list_uploads_pg(user: PlatformUser) -> list[dict[str, Any]]: kind, owner_id, owner_org, - uploaded_at + uploaded_at, + experiment_type_id, + ( + SELECT COUNT(*) + FROM data_upload_files f + WHERE f.upload_id = data_uploads.id + ) AS attachment_count FROM data_uploads ORDER BY uploaded_at DESC """ @@ -6875,6 +6951,8 @@ def list_uploads_pg(user: PlatformUser) -> list[dict[str, Any]]: "owner_id": record["owner_id"], "owner_org": record["owner_org"], "uploaded_at": _iso(record["uploaded_at"]), + "experiment_type_id": record.get("experiment_type_id") or "", + "attachment_count": int(record.get("attachment_count") or 0), } ) return uploads diff --git a/api/routers/dataset_v2.py b/api/routers/dataset_v2.py index 1a7e42a..19c9a9d 100644 --- a/api/routers/dataset_v2.py +++ b/api/routers/dataset_v2.py @@ -19,6 +19,8 @@ add_project_member_pg, create_project_pg, delete_project_pg, + get_equipment_run_file_pg, + get_sample_lineage_pg, get_equipment_run_trace_pg, get_equipment_recipe_pg, get_glance_ingestion_project_pg, @@ -723,6 +725,7 @@ def runs( limit: int = 200, offset: int = 0, project_id: Optional[str] = None, + experiment_type_id: Optional[str] = None, platform_user: PlatformUser = Depends(get_platform_user), ): """ @@ -735,11 +738,15 @@ def runs( ``project_id`` (optional) narrows the list to a single project. Combined with RLS, a PI asking for a project they don't belong to receives an empty list — the same shape as for any inaccessible project. + + ``experiment_type_id`` (optional) narrows the list to runs uploaded into + that experiment. """ is_admin = platform_user.role == "admin" logger.info( - "[dataset/v2/runs] Fetching runs for user=%s role=%s admin_bypass=%s project_id=%s", - platform_user.id, platform_user.role, is_admin, project_id, + "[dataset/v2/runs] Fetching runs for user=%s role=%s admin_bypass=%s " + "project_id=%s experiment_type_id=%s", + platform_user.id, platform_user.role, is_admin, project_id, experiment_type_id, ) return get_runs_list_pg( nanohub_user_id=platform_user.id, @@ -749,6 +756,7 @@ def runs( offset=offset, is_admin=is_admin, project_id=project_id, + experiment_type_id=(experiment_type_id or "").strip() or None, ) @@ -1003,6 +1011,79 @@ def run_trace( return record +_SAFE_DOWNLOAD_NAME = re.compile(r"[^A-Za-z0-9._ -]+") +# Types served as-is; anything else (e.g. HTML/SVG an uploader could plant) is +# sent as an opaque download. +_SAFE_DOWNLOAD_TYPES = frozenset( + { + "text/csv", + "text/plain", + "application/json", + "application/pdf", + "image/png", + "image/jpeg", + "image/tiff", + } +) + + +@router.get("/runs/{run_id}/files/{file_id}") +def run_file_download( + run_id: int, + file_id: int, + platform_user: PlatformUser = Depends(get_platform_user), +): + """Download one file output (e.g. an XRD pattern) stored with a run. + + Same visibility as the run itself; returns 404 when the run or file is + missing or hidden, without saying which.""" + is_admin = platform_user.role == "admin" + record = get_equipment_run_file_pg( + run_id=run_id, + file_id=file_id, + nanohub_user_id=platform_user.id, + is_admin=is_admin, + ) + if record is None: + raise HTTPException(status_code=404, detail="File not found") + filename = _SAFE_DOWNLOAD_NAME.sub("_", str(record["filename"] or "file")).strip() or "file" + return Response( + content=record["content"], + media_type=( + record.get("content_type") + if record.get("content_type") in _SAFE_DOWNLOAD_TYPES + else "application/octet-stream" + ), + headers={ + "Content-Disposition": f'attachment; filename="{filename}"', + "X-Content-Type-Options": "nosniff", + "X-Content-SHA256": str(record.get("sha256") or ""), + }, + ) + + +@router.get("/samples/{sample_id}/lineage") +def sample_lineage( + sample_id: str, + platform_user: PlatformUser = Depends(get_platform_user), +): + """A sample and the visible runs that produced it or used it (e.g. the + furnace run that made a powder and the XRD scans of that powder).""" + from sample_ids import normalize_sample_id + + canonical = normalize_sample_id(sample_id) + if not canonical: + raise HTTPException(status_code=404, detail="Sample not found") + record = get_sample_lineage_pg( + canonical, + nanohub_user_id=platform_user.id, + is_admin=platform_user.role == "admin", + ) + if record is None: + raise HTTPException(status_code=404, detail="Sample not found") + return record + + @router.post("/glance/traces/sync") async def sync_glance_traces( request: Request, diff --git a/api/routers/upload.py b/api/routers/upload.py index c2e2e00..1521ebe 100644 --- a/api/routers/upload.py +++ b/api/routers/upload.py @@ -7,8 +7,12 @@ import json import logging import math +import mimetypes import os +import posixpath +import struct import threading +import zipfile from datetime import datetime from decimal import Decimal, InvalidOperation from pathlib import Path @@ -22,20 +26,25 @@ from domain_configs import derive_expected_columns, load_domain_config from data_loader_pg import ( + find_samples_pg, get_pg_connection, get_projects_list_pg, - merge_equipment_runs_pg, + ingest_equipment_upload_pg, + list_upload_files_pg, + replace_upload_files_pg, sync_equipment_run_traces_pg, - sync_equipment_runs_pg, sync_runs_pg, ) from metadata_pg import ( approved_equipment_available_to_user, create_upload_record_pg, get_equipment_pg, + get_upload_target_experiment_pg, list_uploads_pg, recover_stale_db3_uploads_pg, + user_can_write_project_pg, ) +from sample_ids import SAMPLE_ID_TYPES, normalize_sample_id, run_sample_id from security import PlatformUser, get_platform_user from glance_db3.trace_import import extract_db3_trace @@ -46,6 +55,8 @@ class ProcessUploadPayload(BaseModel): project_id: str + # Optional experiment (experiment_types.id) the uploaded runs belong to. + experiment_type_id: str = "" # Upload "kinds": a combined single file ("all", the legacy/etcher shape) or a @@ -120,6 +131,87 @@ def resolve_uploads_dir() -> Path: CSV_EXCEL_UPLOAD_LIMIT_BYTES = 50 * 1024 * 1024 DB3_UPLOAD_CHUNK_BYTES = 1024 * 1024 +# Output types whose CSV cell names a file sent alongside the upload (e.g. an +# XRD pattern) rather than holding the value itself. +FILE_OUTPUT_TYPES = frozenset({"file", "artifact"}) +MAX_ATTACHMENT_COUNT = 5000 + + +def _env_megabytes(name: str, default: int) -> int: + raw = os.getenv(name, "").strip() + try: + value = int(raw) if raw else default + except ValueError: + value = default + return max(1, value) * 1024 * 1024 + + +def _attachment_file_limit_bytes() -> int: + return _env_megabytes("DT_UPLOAD_ATTACHMENT_MAX_MB", 25) + + +def _linked_files_limit_bytes() -> int: + """Cap on the bytes one processing run may copy into run files (the same + attachment referenced by many runs is stored once per run).""" + return _env_megabytes("DT_UPLOAD_LINKED_FILES_MAX_MB", 256) + + +def _attachment_total_limit_bytes() -> int: + # Default stays under the 70 MB request-body cap of the cluster ingress + # (geddes/k8s/pilots/db3/ingress.yaml), like the DB3 upload limit. + return _env_megabytes("DT_UPLOAD_ATTACHMENTS_TOTAL_MAX_MB", 64) + + +def _normalize_attachment_path(value: Any) -> str: + """Normalize a referenced/archived path to a safe relative POSIX path. + + Returns "" for empty values and for anything that is absolute or escapes + its folder ("..").""" + text = str(value or "").strip().replace("\\", "/") + if not text: + return "" + if text.startswith("/") or (len(text) > 1 and text[1] == ":"): + return "" + normalized = posixpath.normpath(text) + if normalized in (".", "") or normalized.startswith("../") or normalized == "..": + return "" + return normalized + + +def _is_ignored_archive_member(path: str) -> bool: + """macOS archive metadata that isn't a real attachment.""" + parts = path.split("/") + return parts[0] == "__MACOSX" or parts[-1].startswith("._") or parts[-1] == ".DS_Store" + + +def _resolve_attachment( + reference: Any, + staged: list[dict[str, Any]], +) -> tuple[dict[str, Any] | None, str]: + """Find the staged attachment a file-output cell refers to. + + Matching order: the exact relative path; then a unique staged path that + ends with the reference (the archive may wrap files in a top-level folder); + then a unique file name. Returns (file, "") or (None, reason).""" + ref = _normalize_attachment_path(reference) + if not ref: + return None, f"'{reference}' is not a valid relative file path" + exact = [f for f in staged if f["relative_path"] == ref] + if len(exact) == 1: + return exact[0], "" + suffix = [f for f in staged if f["relative_path"].endswith("/" + ref)] + if len(suffix) == 1: + return suffix[0], "" + if len(suffix) > 1: + return None, f"'{reference}' matches more than one attached file" + base = posixpath.basename(ref) + by_name = [f for f in staged if posixpath.basename(f["relative_path"]) == base] + if len(by_name) == 1: + return by_name[0], "" + if len(by_name) > 1: + return None, f"'{reference}' matches more than one attached file named '{base}'" + return None, f"'{reference}' was not found among the attached files" + def _db3_upload_enabled() -> bool: return os.getenv("DT_ENABLE_DB3_UPLOAD", "").strip().casefold() in { @@ -538,7 +630,7 @@ def _update_upload_record( conn.close() -def _claim_upload_for_processing(upload_id: str) -> bool: +def _claim_upload_for_processing(upload_id: str, experiment_type_id: str = "") -> bool: """Atomically transition an upload into ``processing``. Returns True only if this caller won the claim. The conditional UPDATE makes @@ -552,11 +644,12 @@ def _claim_upload_for_processing(upload_id: str) -> bool: """ UPDATE data_uploads SET status = 'processing', errors_json = '[]'::jsonb, - processing_started_at = CURRENT_TIMESTAMP + processing_started_at = CURRENT_TIMESTAMP, + experiment_type_id = %s WHERE id = %s AND status <> 'processing' RETURNING id """, - (upload_id,), + (experiment_type_id, upload_id), ) claimed = cur.fetchone() is not None conn.commit() @@ -1520,6 +1613,10 @@ def _coerce_typed(value: Any, declared_type: str) -> Any: t = (declared_type or "").strip().lower() if t in {"string", "str", "text", "categorical", "category", "id", "identifier"}: return text + if t in SAMPLE_ID_TYPES: + # Identifiers stay text (never coerced to numbers); they are + # canonicalized when resolved into platform sample IDs. + return text if t in {"bool", "boolean"}: low = text.lower() if low in {"true", "1", "yes", "y", "t"}: @@ -1795,6 +1892,8 @@ def _execute_upload_processing( upload_record: dict[str, Any], project_id: str, prior_warnings: list[str], + experiment_type_id: str = "", + experiment_type_version: int | None = None, ) -> None: """Execute an upload with a live processing lease for DB3 imports.""" is_db3 = Path(upload_record["filename"]).suffix.lower() == ".db3" @@ -1804,6 +1903,8 @@ def _execute_upload_processing( upload_record=upload_record, project_id=project_id, prior_warnings=prior_warnings, + experiment_type_id=experiment_type_id, + experiment_type_version=experiment_type_version, ) return @@ -1841,6 +1942,8 @@ def _execute_upload_processing_impl( upload_record: dict[str, Any], project_id: str, prior_warnings: list[str], + experiment_type_id: str = "", + experiment_type_version: int | None = None, ) -> None: """Parse and ingest a stored upload, updating its status to ``processed`` or ``failed``. Runs in a background task (Starlette executes sync background @@ -2006,6 +2109,12 @@ def _fail(errors: list[str], row_count: int) -> None: ) return + file_links: list[dict[str, Any]] = [] + sample_links: list[dict[str, Any]] = [] + file_outputs: list[str] = [] + output_sample_fields: list[str] = [] + input_sample_fields: list[str] = [] + attachment_warnings: list[str] = [] try: if is_canonical_etcher: rows, errors = _csv_upload_to_sync_rows( @@ -2021,6 +2130,30 @@ def _fail(errors: list[str], row_count: int) -> None: project_id=project_id, kind=kind, ) + for row in rows: + row["experiment_type_id"] = experiment_type_id + row["experiment_type_version"] = ( + experiment_type_version if experiment_type_id else None + ) + if not errors and rows: + output_sample_fields, input_sample_fields = _sample_fields(config, kind) + sample_links, errors = _sample_links( + rows=rows, + equipment_id=equipment_id, + output_fields=output_sample_fields, + input_fields=input_sample_fields, + ) + if not errors and sample_links: + errors = _check_sample_links(sample_links) + if not errors and rows: + file_outputs = _file_output_names(config, kind) + staged = list_upload_files_pg(upload_id) if file_outputs else [] + file_links, link_errors, attachment_warnings = _file_output_links( + rows=rows, + file_outputs=file_outputs, + staged=staged, + ) + errors = link_errors except Exception as exc: # noqa: BLE001 - surface parse failures to the UI logger.exception("Upload %s parsing failed", upload_id) _fail([f"Processing failed: {exc}"], 0) @@ -2036,35 +2169,553 @@ def _fail(errors: list[str], row_count: int) -> None: try: if is_canonical_etcher: inserted = sync_runs_pg(rows) - elif kind in ("input", "output"): - inserted = merge_equipment_runs_pg( - equipment_id=equipment_id, - kind=kind, - rows=rows, - upload_id=upload_id, - upload_filename=upload_record["filename"], - ) else: - inserted = sync_equipment_runs_pg( + # Runs, stored files and sample links commit together (or not at + # all), and every touched run is reconciled even with no links. + inserted = ingest_equipment_upload_pg( equipment_id=equipment_id, + kind=kind, rows=rows, upload_id=upload_id, upload_filename=upload_record["filename"], + file_links=file_links, + sample_links=sample_links, + owner_id=str(upload_record.get("owner_id") or ""), + owner_org=str(upload_record.get("owner_org") or ""), ) except Exception as exc: # noqa: BLE001 - surface sync failures to the UI logger.exception("Upload %s sync failed", upload_id) _fail([f"Database sync failed: {exc}"], len(rows)) return - logger.info("Upload %s processed: %s row(s) ingested", upload_id, inserted) + logger.info( + "Upload %s processed: %s row(s) ingested, %s file(s) attached, %s sample link(s)", + upload_id, + inserted, + len(file_links), + len(sample_links), + ) _update_upload_record( upload_id=upload_id, status="processed", - errors=prior_warnings[:50], + errors=(prior_warnings + attachment_warnings)[:50], row_count=len(rows), ) +def _check_zip_directory(data: bytes, part_name: str, max_entries: int) -> None: + """Count a zip's central-directory entries without allocating them. + + ``zipfile.ZipFile`` builds an object per entry before any limit can be + checked, so a small archive with a huge directory could exhaust memory. + This walks the raw directory records (46-byte headers plus their name, + extra and comment lengths) and rejects archives with more than + ``max_entries`` entries. To be sure it checks the same directory + ``zipfile`` will read, it only accepts the plain layout: no ZIP64 records + (never needed under the upload size limit, and they override the ordinary + end record), no data prepended to the archive (``zipfile`` shifts offsets + for that), the directory ending exactly at the end record, and a walked + entry count equal to the declared one.""" + + def reject(detail: str) -> None: + raise HTTPException(status_code=400, detail=f"'{part_name}' {detail}") + + eocd = data.rfind(b"PK\x05\x06", max(0, len(data) - 65557)) + if eocd < 0 or eocd + 22 > len(data): + reject("is not a valid zip archive.") + ( + _sig, _disk, _cd_disk, _entries_disk, entries_total, + cd_size, cd_offset, _comment_len, + ) = struct.unpack("<4s4H2LH", data[eocd:eocd + 22]) + zip64_locator = eocd >= 20 and data[eocd - 20:eocd - 16] == b"PK\x06\x07" + if ( + zip64_locator + or entries_total == 0xFFFF + or cd_size == 0xFFFFFFFF + or cd_offset == 0xFFFFFFFF + ): + reject("is a ZIP64 archive, which isn't supported; use a standard zip.") + if entries_total > max_entries: + reject(f"has too many entries (at most {max_entries}).") + end = cd_offset + cd_size + if end != eocd: + # Either the directory isn't where the end record says, or data was + # prepended (zipfile would then read a shifted directory). + reject("has an unsupported or inconsistent zip layout.") + position, count = cd_offset, 0 + while position < end: + if data[position:position + 4] != b"PK\x01\x02" or position + 46 > end: + reject("has an inconsistent zip directory.") + name_len, extra_len, comment_len = struct.unpack( + "<3H", data[position + 28:position + 34] + ) + position += 46 + name_len + extra_len + comment_len + count += 1 + if count > max_entries: + reject(f"has too many entries (at most {max_entries}).") + if position != end or count != entries_total: + reject("has an inconsistent zip directory.") + + +def _read_upload_with_limit(source: Any, max_bytes: int) -> bytes: + """Read an uploaded part into memory, refusing anything over ``max_bytes``.""" + source.seek(0) + data = source.read(max_bytes + 1) + if len(data) > max_bytes: + raise ValueError("too_large") + return data + + +def _attachment_record(relative_path: str, content: bytes) -> dict[str, Any]: + filename = posixpath.basename(relative_path) + return { + "relative_path": relative_path, + "filename": filename, + "content_type": mimetypes.guess_type(filename)[0] or "application/octet-stream", + "size_bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "content": content, + } + + +def _collect_attachments(parts: list[tuple[str, bytes]]) -> list[dict[str, Any]]: + """Expand uploaded parts (plain files or .zip archives) into attachment + records, enforcing per-file, total and count limits. Raises HTTPException + with an actionable message on any violation.""" + per_file = _attachment_file_limit_bytes() + total_limit = _attachment_total_limit_bytes() + records: dict[str, dict[str, Any]] = {} + total = 0 + + def add(relative_path: str, content: bytes) -> None: + nonlocal total + if len(content) > per_file: + raise HTTPException( + status_code=413, + detail=( + f"'{relative_path}' exceeds the " + f"{per_file // (1024 * 1024)} MB per-file limit." + ), + ) + if relative_path in records: + raise HTTPException( + status_code=400, + detail=f"Attachment '{relative_path}' appears more than once.", + ) + if len(records) >= MAX_ATTACHMENT_COUNT: + raise HTTPException( + status_code=400, + detail=f"At most {MAX_ATTACHMENT_COUNT} attachments are allowed per upload.", + ) + total += len(content) + if total > total_limit: + raise HTTPException( + status_code=413, + detail=( + "Attachments exceed the " + f"{total_limit // (1024 * 1024)} MB total limit for one upload." + ), + ) + records[relative_path] = _attachment_record(relative_path, content) + + for part_name, data in parts: + if Path(part_name).suffix.lower() == ".zip": + _check_zip_directory(data, part_name, MAX_ATTACHMENT_COUNT * 2) + try: + archive = zipfile.ZipFile(io.BytesIO(data)) + except zipfile.BadZipFile as exc: + raise HTTPException( + status_code=400, detail=f"'{part_name}' is not a valid zip archive." + ) from exc + with archive: + for info in archive.infolist(): + if info.is_dir(): + continue + relative_path = _normalize_attachment_path(info.filename) + if not relative_path: + raise HTTPException( + status_code=400, + detail=f"Archive entry '{info.filename}' has an unsafe path.", + ) + if _is_ignored_archive_member(relative_path): + continue + if info.file_size > per_file: + raise HTTPException( + status_code=413, + detail=( + f"'{relative_path}' exceeds the " + f"{per_file // (1024 * 1024)} MB per-file limit." + ), + ) + # Read with a hard cap: the declared size in a crafted + # archive can't be trusted (zip bombs). + with archive.open(info) as member: + content = member.read(per_file + 1) + if len(content) > per_file: + raise HTTPException( + status_code=413, + detail=( + f"'{relative_path}' exceeds the " + f"{per_file // (1024 * 1024)} MB per-file limit." + ), + ) + add(relative_path, content) + continue + relative_path = _normalize_attachment_path(posixpath.basename(part_name or "")) + if not relative_path: + raise HTTPException(status_code=400, detail="Each attachment needs a file name.") + add(relative_path, data) + + if not records: + raise HTTPException(status_code=400, detail="No attachment files were provided.") + return list(records.values()) + + +def _assert_attachments_supported(upload_record: dict[str, Any]) -> None: + if Path(upload_record["filename"]).suffix.lower() != ".csv": + raise HTTPException( + status_code=400, + detail="Attachments can only be added to CSV uploads.", + ) + if upload_record["equipment_id"] == "etcher": + raise HTTPException( + status_code=400, + detail="The canonical etcher upload doesn't support file attachments.", + ) + + +@router.post("/uploads/{upload_id}/attachments") +async def upload_attachments( + upload_id: str, + files: list[UploadFile] = File(...), + user: PlatformUser = Depends(get_platform_user), +): + """ + Attach the files a CSV upload refers to (e.g. one XRD pattern per run). + + Send individual files and/or .zip archives. Each call replaces the upload's + previous attachments. When the upload is processed, every output registered + with type "file" is matched to an attachment by the path written in its CSV + cell, and the file is stored with that run. + """ + upload_record = await run_in_threadpool(_load_upload_record, upload_id, user) + _assert_attachments_supported(upload_record) + if upload_record["status"] == "processing": + raise HTTPException( + status_code=409, + detail="This upload is being processed; add attachments once it finishes.", + ) + + total_limit = _attachment_total_limit_bytes() + parts: list[tuple[str, bytes]] = [] + read_total = 0 + for item in files: + try: + data = await run_in_threadpool( + _read_upload_with_limit, item.file, total_limit - read_total + ) + except ValueError as exc: + raise HTTPException( + status_code=413, + detail=( + "Attachments exceed the " + f"{total_limit // (1024 * 1024)} MB total limit for one upload." + ), + ) from exc + read_total += len(data) + parts.append((item.filename or "", data)) + + # Archive parsing is CPU-bound; keep it off the event loop. + records = await run_in_threadpool(_collect_attachments, parts) + try: + await run_in_threadpool(replace_upload_files_pg, upload_id, records) + except RuntimeError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return { + "upload_id": upload_id, + "attachment_count": len(records), + "total_bytes": sum(r["size_bytes"] for r in records), + "files": [r["relative_path"] for r in records[:50]], + "message": ( + "Attachments saved. Process (or re-process) the upload to link them to runs." + ), + } + + +@router.get("/uploads/{upload_id}/attachments") +def list_upload_attachments( + upload_id: str, + user: PlatformUser = Depends(get_platform_user), +): + """List an upload's staged attachments (names, sizes, checksums).""" + upload_record = _load_upload_record(upload_id, user) + _assert_attachments_supported(upload_record) + files = list_upload_files_pg(upload_id) + return { + "upload_id": upload_id, + "attachment_count": len(files), + "files": [ + { + "relative_path": f["relative_path"], + "filename": f["filename"], + "size_bytes": int(f["size_bytes"] or 0), + "sha256": f["sha256"], + } + for f in files + ], + } + + +def _sample_fields(config: dict[str, Any], kind: str) -> tuple[list[str], list[str]]: + """(output, input) columns registered with the sample-ID type that this + upload kind carries.""" + input_names, output_names = _config_input_output_names(config) + type_by_name = _config_type_by_name(config) + + def is_sample(name: str) -> bool: + return type_by_name.get(name, "").strip().lower() in SAMPLE_ID_TYPES + + outputs = [n for n in output_names if is_sample(n)] if kind in ("all", "output") else [] + inputs = [n for n in input_names if is_sample(n)] if kind in ("all", "input") else [] + return outputs, inputs + + +def _sample_links( + *, + rows: list[dict[str, Any]], + equipment_id: str, + output_fields: list[str], + input_fields: list[str], +) -> tuple[list[dict[str, Any]], list[str]]: + """Resolve each row's sample-ID cells into platform sample IDs. + + An output sample-ID cell gets the platform-issued ID (IGSN syntax) minted + from the run's identity, so re-processing never issues a second ID for the + same sample. The cell may be left blank or carry exactly that ID (e.g. + prepared ahead of time by a script with ``sample_ids.run_sample_id``); + anything else is rejected, so students can't invent sample IDs. Input + sample-ID cells name the sample a run used: the output of an earlier unit + process. The resolved IDs replace the cell values, so runs always show + canonical IDs.""" + links: list[dict[str, Any]] = [] + errors: list[str] = [] + minted_by: dict[str, int] = {} + for index, row in enumerate(rows): + label = row.get("lotname") or f"row {index + 1}" + for role, fields, side in ( + ("output", output_fields, "outputs"), + ("input", input_fields, "inputs"), + ): + values = row.setdefault(side, {}) + for field in fields: + raw = values.get(field) + if role == "output": + sample_id = run_sample_id( + equipment_id=equipment_id, + project_id=str(row.get("project_id") or ""), + lotname=str(row.get("lotname") or ""), + run_date=row.get("run_date"), + field=field, + ) + if raw not in (None, "") and normalize_sample_id(raw) != sample_id: + errors.append( + f"Run '{label}': {field} '{raw}' is not the ID the platform " + "issues for this run. Leave the cell blank and the platform " + "will issue it." + ) + continue + prior = minted_by.get(sample_id) + if prior is not None: + errors.append( + f"Run '{label}': {field} can't be issued because row " + f"{prior + 1} has the same lot name and run date; give " + "each run a unique lot name or run date." + ) + continue + minted_by[sample_id] = index + else: + if raw in (None, ""): + continue + sample_id = normalize_sample_id(raw) + if not sample_id: + errors.append( + f"Run '{label}': {field} '{raw}' is not a valid sample ID" + ) + continue + values[field] = sample_id + links.append( + { + "row_index": index, + "project_id": row.get("project_id"), + "lotname": row.get("lotname"), + "run_date": row.get("run_date"), + "field_name": field, + "role": role, + "sample_id": sample_id, + "sample_name": str(row.get("lotname") or sample_id), + } + ) + if len(errors) > 50: + errors.append("... (more errors truncated)") + return links, errors + return links, errors + + +def _check_sample_links(links: list[dict[str, Any]]) -> list[str]: + """Reject links before any write: inputs must name a sample already + registered in the same project; outputs can't reuse another project's + sample. Missing and foreign samples get the same message, so uploads can't + be used to probe other projects' sample IDs.""" + found = find_samples_pg([link["sample_id"] for link in links]) + errors: list[str] = [] + for link in links: + label = link.get("lotname") or f"row {link['row_index'] + 1}" + sample_project = found.get(link["sample_id"]) + foreign = sample_project is not None and sample_project != str(link.get("project_id")) + if foreign or (link["role"] == "input" and sample_project is None): + errors.append( + f"Run '{label}': {link['field_name']} {link['sample_id']} is not an " + "available sample in this project (process the run that produced it " + "first)." + ) + if len(errors) > 50: + errors.append("... (more errors truncated)") + break + return errors + + +def _file_output_names(config: dict[str, Any], kind: str) -> list[str]: + """Outputs registered as files, for uploads that carry outputs.""" + if kind == "input": + return [] + _, output_names = _config_input_output_names(config) + type_by_name = _config_type_by_name(config) + return [ + name + for name in output_names + if type_by_name.get(name, "").strip().lower() in FILE_OUTPUT_TYPES + ] + + +def _file_output_links( + *, + rows: list[dict[str, Any]], + file_outputs: list[str], + staged: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[str], list[str]]: + """Match each row's file-output cells to staged attachments. + + Returns (links, errors, warnings). Without any attachments the cells keep + their old meaning (plain text), so existing uploads are unaffected; a + warning notes that no files were stored.""" + links: list[dict[str, Any]] = [] + errors: list[str] = [] + warnings: list[str] = [] + if not file_outputs: + return links, errors, warnings + referenced = 0 + for index, row in enumerate(rows): + outputs = row.get("outputs") or {} + for name in file_outputs: + value = outputs.get(name) + if value in (None, ""): + continue + referenced += 1 + if not staged: + continue + match, reason = _resolve_attachment(value, staged) + if match is None: + errors.append( + f"Run '{row.get('lotname') or index + 1}': {name} {reason}" + ) + if len(errors) > 50: + errors.append("... (more errors truncated)") + return links, errors, warnings + continue + links.append( + { + "row_index": index, + "project_id": row.get("project_id"), + "lotname": row.get("lotname"), + "run_date": row.get("run_date"), + "output_name": name, + "upload_file_id": match["id"], + "size_bytes": int(match.get("size_bytes") or 0), + } + ) + linked_bytes = sum(int(link.get("size_bytes") or 0) for link in links) + budget = _linked_files_limit_bytes() + if linked_bytes > budget: + errors.append( + "The file outputs reference " + f"{linked_bytes // (1024 * 1024)} MB in total (counting a file once per run " + f"that uses it); the limit is {budget // (1024 * 1024)} MB per upload." + ) + if referenced and not staged: + warnings.append( + f"{referenced} file output value(s) ({', '.join(file_outputs)}) were " + "stored as text only because no attachments were added to this upload." + ) + return links, errors, warnings + + +def _validate_upload_experiment( + *, + experiment_type_id: str, + upload_record: dict[str, Any], + project_id: str, + user: PlatformUser, +) -> tuple[str, int | None]: + """Check the experiment an upload is filed under; returns its id and + current version, or ("", None) when no experiment was chosen. + + The experiment must be visible to the caller, belong to the upload's + equipment (or be equipment-agnostic), and the caller must own it, be an + admin, or be uploading into a project that already uses it. That keeps + someone who can merely see a shared template from filing runs under it.""" + if not experiment_type_id: + return "", None + if Path(upload_record["filename"]).suffix.lower() == ".db3": + raise HTTPException( + status_code=400, + detail="DB3 imports can't be filed under an experiment yet.", + ) + if upload_record["equipment_id"] == "etcher": + raise HTTPException( + status_code=400, + detail="Canonical etcher uploads can't be filed under an experiment.", + ) + experiment = get_upload_target_experiment_pg( + type_id=experiment_type_id, + user=user, + project_id=project_id, + ) + if experiment is None: + raise HTTPException(status_code=404, detail="Selected experiment was not found") + experiment_equipment = experiment["equipment_id"] + if experiment_equipment and experiment_equipment != upload_record["equipment_id"]: + raise HTTPException( + status_code=400, + detail=( + f"Experiment '{experiment['name']}' uses equipment " + f"'{experiment_equipment}', not '{upload_record['equipment_id']}'." + ), + ) + if not ( + user.role == "admin" + or experiment["owner_id"] == user.id + or experiment["attached_to_project"] + ): + raise HTTPException( + status_code=403, + detail=( + "Only the experiment's owner or an admin can file runs under it, " + "unless the selected project already uses this experiment." + ), + ) + return experiment["id"], experiment.get("version") + + @router.post("/uploads/{upload_id}/process") def process_upload( upload_id: str, @@ -2085,6 +2736,14 @@ def process_upload( if not project_id: raise HTTPException(status_code=400, detail="project_id is required") project = _assert_project_visible(project_id, user) + # Processing writes runs (and may register samples) in the project, so it + # needs write access, not just visibility (open projects are visible to + # non-members). + if not user_can_write_project_pg(project_id=project_id, user=user): + raise HTTPException( + status_code=403, + detail="Only members of the selected project can add data to it.", + ) if _db3_user_is_allowed(user): recover_stale_db3_uploads_pg() @@ -2117,9 +2776,16 @@ def process_upload( ), ) + experiment_type_id, experiment_type_version = _validate_upload_experiment( + experiment_type_id=(payload.experiment_type_id or "").strip(), + upload_record=upload_record, + project_id=project_id, + user=user, + ) + # Atomically claim the upload; if another request already moved it into # "processing", don't enqueue a second background ingestion. - if not _claim_upload_for_processing(upload_id): + if not _claim_upload_for_processing(upload_id, experiment_type_id): raise HTTPException( status_code=409, detail="This upload is already being processed.", @@ -2130,6 +2796,8 @@ def process_upload( upload_record=upload_record, project_id=project_id, prior_warnings=prior_warnings, + experiment_type_id=experiment_type_id, + experiment_type_version=experiment_type_version, ) return { @@ -2137,6 +2805,8 @@ def process_upload( "upload_id": upload_id, "equipment_id": equipment_id, "project_id": project_id, + "experiment_type_id": experiment_type_id, + "experiment_type_version": experiment_type_version, "inserted": 0, "row_count": 0, "errors": [], diff --git a/api/sample_ids.py b/api/sample_ids.py new file mode 100644 index 0000000..9113a6c --- /dev/null +++ b/api/sample_ids.py @@ -0,0 +1,107 @@ +"""Platform-issued sample identifiers in IGSN syntax. + +Samples are identified by the platform, never by a student-chosen name, so two +samples can't collide and one experiment's output sample can be the next +experiment's input (synthesis -> XRD, etch -> profilometry). + +IDs follow IGSN syntax: an uppercase registrant namespace followed by a local +code (here 10 Crockford base32 characters). Until the namespace is registered +with an IGSN allocating agent (DataCite or SESAR), the IDs are unique within +the platform but don't resolve globally yet; registering them later doesn't +change the identifiers. The namespace is set by ``DT_IGSN_NAMESPACE``. +""" +from __future__ import annotations + +import hashlib +import os +import re + +# Crockford base32: no I, L, O or U, so codes are unambiguous when read aloud +# or copied by hand. +_CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" +LOCAL_CODE_LENGTH = 10 +DEFAULT_NAMESPACE = "PUDT" +_NAMESPACE_RE = re.compile(r"^[A-Z][A-Z0-9]{1,9}$") +_SAMPLE_ID_RE = re.compile(r"^[A-Z0-9][A-Z0-9.\-]{5,63}$") +SAMPLE_ID_TYPES = frozenset({"sample_id", "sample"}) + + +def igsn_namespace() -> str: + """The configured IGSN namespace (falls back to the default if invalid).""" + value = os.getenv("DT_IGSN_NAMESPACE", "").strip().upper() + return value if _NAMESPACE_RE.match(value) else DEFAULT_NAMESPACE + + +def mint_sample_id(seed: str, namespace: str | None = None) -> str: + """Deterministically mint an IGSN-syntax sample ID from ``seed``. + + The same seed always yields the same ID, so re-processing an upload never + issues a second identity for the same physical sample.""" + digest = hashlib.sha256(seed.encode("utf-8")).digest() + number = int.from_bytes(digest[:8], "big") + chars = [] + for _ in range(LOCAL_CODE_LENGTH): + chars.append(_CROCKFORD[number & 31]) + number >>= 5 + return (namespace or igsn_namespace()) + "".join(reversed(chars)) + + +def normalize_sample_id(value: object) -> str: + """Return the canonical (uppercase) sample ID, or "" if ``value`` isn't a + syntactically valid IGSN-style identifier.""" + text = str(value or "").strip().upper() + if text.startswith("IGSN:"): + text = text[5:].strip() + return text if _SAMPLE_ID_RE.match(text) else "" + + +def canonical_run_instant(value: object) -> str: + """Canonical text for a run timestamp, so every spelling of the same + instant seeds the same sample ID and different instants never collide. + + The result is UTC with microsecond precision (``...T00:00:00.000000Z``). + Timestamps without an offset are read as UTC, matching how the platform's + database session stores them. Unparseable text is used as-is.""" + from datetime import datetime, timezone + + text = str(value or "").strip() + if not text: + return "" + candidate = text[:-1] + "+00:00" if text.endswith("Z") else text + try: + parsed = datetime.fromisoformat(candidate) + except ValueError: + return text + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + utc = parsed.astimezone(timezone.utc).replace(tzinfo=None) + return utc.isoformat(timespec="microseconds") + "Z" + + +def run_sample_id( + *, + equipment_id: str, + project_id: str, + lotname: str, + run_date: object, + field: str, + namespace: str | None = None, +) -> str: + """The platform sample ID for the sample a run produced in ``field``. + + It depends only on the run's identity (equipment, project, lot name, run + instant) and the output field, so re-processing, a split input/output + upload, or a script preparing linked uploads ahead of time (e.g. the legacy + XRD converter) all arrive at the same ID. Uploads may only carry this ID + (or leave the cell blank); arbitrary student-chosen IDs are rejected.""" + seed = "|".join( + [ + "run-sample", + equipment_id, + project_id, + lotname, + canonical_run_instant(run_date), + field, + ] + ) + return mint_sample_id(seed, namespace) diff --git a/api/scripts/add_row_level_security.sql b/api/scripts/add_row_level_security.sql index ebcbc8c..b32416f 100644 --- a/api/scripts/add_row_level_security.sql +++ b/api/scripts/add_row_level_security.sql @@ -547,6 +547,18 @@ DROP POLICY IF EXISTS dt_trace_samples_worker ON equipment_run_trace_samples; CREATE POLICY dt_trace_samples_worker ON equipment_run_trace_samples FOR ALL TO api_worker USING (true) WITH CHECK (true); +DROP POLICY IF EXISTS dt_equipment_run_files_admin ON equipment_run_files; +CREATE POLICY dt_equipment_run_files_admin ON equipment_run_files FOR SELECT USING (app_is_admin()); +DROP POLICY IF EXISTS dt_equipment_run_files_worker ON equipment_run_files; +CREATE POLICY dt_equipment_run_files_worker ON equipment_run_files + FOR ALL TO api_worker USING (true) WITH CHECK (true); + +DROP POLICY IF EXISTS dt_equipment_run_samples_admin ON equipment_run_samples; +CREATE POLICY dt_equipment_run_samples_admin ON equipment_run_samples FOR SELECT USING (app_is_admin()); +DROP POLICY IF EXISTS dt_equipment_run_samples_worker ON equipment_run_samples; +CREATE POLICY dt_equipment_run_samples_worker ON equipment_run_samples + FOR ALL TO api_worker USING (true) WITH CHECK (true); + DROP POLICY IF EXISTS dt_trace_events_admin ON equipment_run_trace_events; CREATE POLICY dt_trace_events_admin ON equipment_run_trace_events FOR SELECT USING (app_is_admin()); DROP POLICY IF EXISTS dt_trace_events_worker ON equipment_run_trace_events; diff --git a/api/scripts/add_upload_experiment_and_run_files.sql b/api/scripts/add_upload_experiment_and_run_files.sql new file mode 100644 index 0000000..0aa2b37 --- /dev/null +++ b/api/scripts/add_upload_experiment_and_run_files.sql @@ -0,0 +1,138 @@ +-- Uploads filed under an experiment, and file outputs stored per run. +-- Documentation/fresh-setup twin of +-- data_loader_pg.ensure_upload_experiment_and_run_files_pg(), which the +-- migration Job (python -m db_migrations) applies in production. +-- Safe to run multiple times. + +-- Experiment (experiment_types.id) an upload was processed into; '' = none. +-- A plain label, not a foreign key, so deleting a template never deletes runs. +ALTER TABLE equipment_runs + ADD COLUMN IF NOT EXISTS experiment_type_id VARCHAR(255) NOT NULL DEFAULT ''; +-- The Python migration builds this index CONCURRENTLY (outside a +-- transaction) so production reads/writes aren't blocked during the scan. +CREATE INDEX IF NOT EXISTS idx_equipment_runs_experiment_type_id + ON equipment_runs(experiment_type_id) + WHERE experiment_type_id <> ''; +ALTER TABLE data_uploads + ADD COLUMN IF NOT EXISTS experiment_type_id VARCHAR(255) NOT NULL DEFAULT ''; + +-- Attachments sent with an upload, staged until it is processed. Like +-- data_uploads there is no RLS: the API authorizes through the upload record. +CREATE TABLE IF NOT EXISTS data_upload_files ( + id BIGSERIAL PRIMARY KEY, + upload_id UUID NOT NULL REFERENCES data_uploads(id) ON DELETE CASCADE, + relative_path TEXT NOT NULL, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(255) NOT NULL DEFAULT '', + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + content BYTEA NOT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + UNIQUE (upload_id, relative_path) +); +GRANT SELECT, INSERT, UPDATE, DELETE ON data_upload_files TO api_client; +GRANT USAGE, SELECT ON SEQUENCE data_upload_files_id_seq TO api_client; + +-- A run's file outputs (e.g. its XRD pattern). Content lives in Postgres, as +-- for trained models, because the API has no durable file volume. +CREATE TABLE IF NOT EXISTS equipment_run_files ( + id BIGSERIAL PRIMARY KEY, + equipment_run_id BIGINT NOT NULL + REFERENCES equipment_runs(id) ON DELETE CASCADE, + output_name VARCHAR(255) NOT NULL, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(255) NOT NULL DEFAULT '', + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + content BYTEA NOT NULL, + upload_id UUID, + source_path TEXT NOT NULL DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + UNIQUE (equipment_run_id, output_name) +); +CREATE INDEX IF NOT EXISTS idx_equipment_run_files_upload_id + ON equipment_run_files(upload_id); +ALTER TABLE equipment_run_files ENABLE ROW LEVEL SECURITY; +GRANT SELECT ON equipment_run_files TO api_client; + +-- Visible exactly when the run's project is (open, or the user is a member). +-- Admin and api_worker policies live in add_row_level_security.sql. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_files' + AND policyname = 'equipment_run_file_visibility_policy' + ) THEN + CREATE POLICY equipment_run_file_visibility_policy ON equipment_run_files + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_files.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; +END +$$; + +-- Version of the experiment template each run was captured under. +ALTER TABLE equipment_runs ADD COLUMN IF NOT EXISTS experiment_type_version INT; + +-- Samples a run produced ('output') or used ('input'), linking unit processes +-- (e.g. furnace synthesis -> XRD) through platform-issued sample IDs +-- (IGSN syntax, see api/sample_ids.py). +CREATE TABLE IF NOT EXISTS equipment_run_samples ( + equipment_run_id BIGINT NOT NULL REFERENCES equipment_runs(id) ON DELETE CASCADE, + sample_id VARCHAR(80) NOT NULL REFERENCES samples(id) ON DELETE CASCADE, + role VARCHAR(20) NOT NULL, + field_name VARCHAR(255) NOT NULL, + upload_id UUID, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (equipment_run_id, role, field_name), + CHECK (role IN ('input', 'output')) +); +CREATE INDEX IF NOT EXISTS idx_equipment_run_samples_sample_id ON equipment_run_samples(sample_id); +CREATE INDEX IF NOT EXISTS idx_equipment_run_samples_upload_id ON equipment_run_samples(upload_id); +ALTER TABLE equipment_run_samples ENABLE ROW LEVEL SECURITY; +GRANT SELECT ON equipment_run_samples TO api_client; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'equipment_run_samples' + AND policyname = 'equipment_run_sample_visibility_policy' + ) THEN + CREATE POLICY equipment_run_sample_visibility_policy ON equipment_run_samples + FOR SELECT + USING ( + EXISTS ( + SELECT 1 + FROM equipment_runs r + JOIN projects p ON p.id = r.project_id + WHERE r.id = equipment_run_samples.equipment_run_id + AND ( + p.access_mode = 'open' + OR EXISTS ( + SELECT 1 FROM project_members pm + WHERE pm.project_id = p.id + AND pm.nanohub_user_id = current_setting('app.current_user', true) + ) + ) + ) + ); + END IF; +END +$$; diff --git a/api/tests/test_etcher_realtime_smoke.py b/api/tests/test_etcher_realtime_smoke.py index 2a72ae9..d7dbb04 100644 --- a/api/tests/test_etcher_realtime_smoke.py +++ b/api/tests/test_etcher_realtime_smoke.py @@ -84,6 +84,7 @@ def fake_get_runs_list_pg( is_admin=False, project_id=None, include_raw_payload=False, + experiment_type_id=None, ): rows = synced_rows if include_outliers else [ row diff --git a/api/tests/test_glance_closed_loop.py b/api/tests/test_glance_closed_loop.py index d76b223..e822710 100644 --- a/api/tests/test_glance_closed_loop.py +++ b/api/tests/test_glance_closed_loop.py @@ -1212,6 +1212,10 @@ def execute(self, query, params=None): def fetchone(self): return self.row + def fetchall(self): + # Run detail also lists the run's stored file outputs. + return [] + def __enter__(self): return self diff --git a/api/tests/test_upload_experiment_artifacts.py b/api/tests/test_upload_experiment_artifacts.py new file mode 100644 index 0000000..93f35aa --- /dev/null +++ b/api/tests/test_upload_experiment_artifacts.py @@ -0,0 +1,657 @@ +"""Uploads filed under an experiment, with file outputs stored per run.""" +import io +import os +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +from fastapi import HTTPException +from fastapi.testclient import TestClient + +os.environ.setdefault("DT_SYSTEM_TOKEN", "test-system-token") +os.environ.setdefault("INGESTION_TOKEN", "test-ingestion-token") +os.environ.setdefault("PG_PASS", "test-api-password") +os.environ.setdefault("PG_SUPER_PASS", "test-super-password") + +API_DIR = Path(__file__).resolve().parents[1] +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + +import main # noqa: E402 +from routers import dataset_v2, upload # noqa: E402 +from security import PlatformUser # noqa: E402 + +SYSTEM_TOKEN = os.environ["DT_SYSTEM_TOKEN"] +OWNER_HEADERS = { + "X-System-Token": SYSTEM_TOKEN, + "X-User-Id": "owner-1", + "X-User-Email": "owner@example.com", + "X-User-Name": "Equipment Owner", + "X-User-Role": "equipment_owner", + "X-User-Org": "Birck", +} + +OWNER = PlatformUser( + id="owner-1", email="o@example.com", name="Owner", role="equipment_owner", + organization="Birck", +) +ADMIN = PlatformUser( + id="admin-1", email="a@example.com", name="Admin", role="admin", + organization="Birck", +) +OTHER = PlatformUser( + id="other-1", email="x@example.com", name="Other", role="researcher", + organization="Birck", +) + +FURNACE_CONFIG = { + "data": {"lotname_column": "LOTNAME", "timestamp_column": "run_date"}, + "columns": [ + {"name": "LOTNAME", "type": "string"}, + {"name": "run_date", "type": "datetime"}, + ], + "parameters": [{"name": "Temperature", "type": "float"}], + "outputs": [ + {"name": "Sample ID", "type": "string"}, + {"name": "XRD Pattern", "type": "file"}, + ], +} + + +def _staged(*paths): + return [ + {"id": index + 1, "relative_path": path, "filename": path.rsplit("/", 1)[-1]} + for index, path in enumerate(paths) + ] + + +def _zip(entries): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name, data in entries.items(): + archive.writestr(name, data) + return buffer.getvalue() + + +class AttachmentPathTests(unittest.TestCase): + def test_normalizes_relative_paths(self): + self.assertEqual(upload._normalize_attachment_path("patterns/a.csv"), "patterns/a.csv") + self.assertEqual(upload._normalize_attachment_path("./patterns//a.csv"), "patterns/a.csv") + self.assertEqual(upload._normalize_attachment_path("patterns\\a.csv"), "patterns/a.csv") + + def test_rejects_unsafe_paths(self): + for value in ("", " ", "/etc/passwd", "../a.csv", "a/../../b.csv", "C:/x.csv", ".."): + self.assertEqual(upload._normalize_attachment_path(value), "", value) + + def test_resolves_exact_suffix_and_name_matches(self): + staged = _staged("batch1/patterns/a.csv", "batch1/patterns/b.csv", "c.csv") + self.assertEqual(upload._resolve_attachment("c.csv", staged)[0]["id"], 3) + self.assertEqual(upload._resolve_attachment("patterns/a.csv", staged)[0]["id"], 1) + self.assertEqual(upload._resolve_attachment("old/dir/b.csv", staged)[0]["id"], 2) + + def test_reports_missing_ambiguous_and_unsafe_references(self): + staged = _staged("x/a.csv", "y/a.csv") + match, reason = upload._resolve_attachment("a.csv", staged) + self.assertIsNone(match) + self.assertIn("more than one", reason) + match, reason = upload._resolve_attachment("z.csv", staged) + self.assertIsNone(match) + self.assertIn("not found", reason) + match, reason = upload._resolve_attachment("../a.csv", staged) + self.assertIsNone(match) + self.assertIn("not a valid", reason) + + +class CollectAttachmentTests(unittest.TestCase): + def test_expands_zip_and_skips_macos_metadata(self): + data = _zip( + { + "batch1/patterns/a.csv": b"1,2\n", + "__MACOSX/batch1/._a.csv": b"junk", + "batch1/.DS_Store": b"junk", + } + ) + records = upload._collect_attachments([("files.zip", data), ("extra.csv", b"3,4\n")]) + paths = sorted(r["relative_path"] for r in records) + self.assertEqual(paths, ["batch1/patterns/a.csv", "extra.csv"]) + record = next(r for r in records if r["relative_path"] == "extra.csv") + self.assertEqual(record["size_bytes"], 4) + self.assertEqual(len(record["sha256"]), 64) + self.assertEqual(record["content_type"], "text/csv") + + def test_rejects_traversal_duplicates_and_bad_zip(self): + with self.assertRaises(HTTPException) as ctx: + upload._collect_attachments([("x.zip", _zip({"../evil.csv": b"x"}))]) + self.assertEqual(ctx.exception.status_code, 400) + with self.assertRaises(HTTPException) as ctx: + upload._collect_attachments([("a.csv", b"1"), ("dir/a.csv", b"2")]) + self.assertIn("more than once", ctx.exception.detail) + with self.assertRaises(HTTPException) as ctx: + upload._collect_attachments([("x.zip", b"not a zip")]) + self.assertIn("not a valid zip", ctx.exception.detail) + with self.assertRaises(HTTPException): + upload._collect_attachments([]) + + def test_plain_files_respect_the_per_file_limit(self): + with patch.dict(os.environ, {"DT_UPLOAD_ATTACHMENT_MAX_MB": "1"}): + with self.assertRaises(HTTPException) as ctx: + upload._collect_attachments([("big.csv", b"x" * (1024 * 1024 + 1))]) + self.assertEqual(ctx.exception.status_code, 413) + + def test_rejects_archives_with_too_many_entries(self): + with patch.object(upload, "MAX_ATTACHMENT_COUNT", 2): + with self.assertRaises(HTTPException) as ctx: + upload._collect_attachments( + [("x.zip", _zip({f"__MACOSX/._{i}": b"" for i in range(5)}))] + ) + self.assertIn("too many entries", ctx.exception.detail) + + def test_directory_is_counted_before_zipfile_allocates(self): + data = _zip({f"f{i}.csv": b"1" for i in range(5)}) + with patch.object(upload, "MAX_ATTACHMENT_COUNT", 2), \ + patch.object(upload.zipfile, "ZipFile") as zip_ctor: + with self.assertRaises(HTTPException): + upload._collect_attachments([("x.zip", data)]) + zip_ctor.assert_not_called() + + def test_zip64_and_prefixed_archives_are_rejected_before_construction(self): + import struct as _struct + + data = _zip({"a.csv": b"1"}) + eocd = data.rfind(b"PK\x05\x06") + # A ZIP64 locator right before the ordinary end record. + locator = b"PK\x06\x07" + b"\0" * 16 + zip64 = data[:eocd] + locator + data[eocd:] + # Data prepended to an otherwise valid archive. + prefixed = b"junk" * 10 + data + # Declared entry count that disagrees with the directory. + fields = list(_struct.unpack("<4s4H2LH", data[eocd:eocd + 22])) + fields[4] = fields[3] = 2 + miscounted = data[:eocd] + _struct.pack("<4s4H2LH", *fields) + data[eocd + 22:] + for bad in (zip64, prefixed, miscounted): + with patch.object(upload.zipfile, "ZipFile") as zip_ctor: + with self.assertRaises(HTTPException): + upload._collect_attachments([("x.zip", bad)]) + zip_ctor.assert_not_called() + + def test_inconsistent_or_truncated_archives_are_rejected(self): + data = _zip({"a.csv": b"1"}) + with self.assertRaises(HTTPException): + upload._collect_attachments([("x.zip", data[:-30] + b"PK\x05\x06" + b"\0" * 18)]) + with self.assertRaises(HTTPException): + upload._collect_attachments([("x.zip", b"PK\x05\x06" + b"\0" * 4 + b"\xff\xff" * 2 + b"\0" * 10)]) + + def test_enforces_size_limits(self): + big = b"x" * (1024 * 1024 + 1) + with patch.dict(os.environ, {"DT_UPLOAD_ATTACHMENT_MAX_MB": "1"}): + with self.assertRaises(HTTPException) as ctx: + upload._collect_attachments([("x.zip", _zip({"big.csv": big}))]) + self.assertEqual(ctx.exception.status_code, 413) + with patch.dict(os.environ, {"DT_UPLOAD_ATTACHMENTS_TOTAL_MAX_MB": "1"}): + with self.assertRaises(HTTPException) as ctx: + upload._collect_attachments([("a.csv", big[:700000]), ("b.csv", big[:700000])]) + self.assertEqual(ctx.exception.status_code, 413) + + +class FileOutputLinkTests(unittest.TestCase): + def test_file_output_names_accept_file_and_artifact_types(self): + config = dict(FURNACE_CONFIG) + config["outputs"] = FURNACE_CONFIG["outputs"] + [{"name": "Report", "type": "Artifact"}] + self.assertEqual(upload._file_output_names(config, "all"), ["XRD Pattern", "Report"]) + self.assertEqual(upload._file_output_names(config, "input"), []) + + def test_without_attachments_values_stay_text_with_a_warning(self): + rows = [{"lotname": "L1", "outputs": {"XRD Pattern": "patterns/a.csv"}}] + links, errors, warnings = upload._file_output_links( + rows=rows, file_outputs=["XRD Pattern"], staged=[] + ) + self.assertEqual((links, errors), ([], [])) + self.assertEqual(len(warnings), 1) + self.assertEqual(rows[0]["outputs"]["XRD Pattern"], "patterns/a.csv") + + def test_links_rows_and_reports_missing_files(self): + rows = [ + {"lotname": "L1", "project_id": "p", "run_date": "2026-05-08T00:00:00", + "outputs": {"XRD Pattern": "patterns/a.csv"}}, + {"lotname": "L2", "outputs": {"XRD Pattern": ""}}, + {"lotname": "L3", "outputs": {"XRD Pattern": "patterns/missing.csv"}}, + ] + links, errors, warnings = upload._file_output_links( + rows=rows, file_outputs=["XRD Pattern"], staged=_staged("batch/patterns/a.csv") + ) + self.assertEqual( + links, + [{"row_index": 0, "project_id": "p", "lotname": "L1", + "run_date": "2026-05-08T00:00:00", "output_name": "XRD Pattern", + "upload_file_id": 1, "size_bytes": 0}], + ) + self.assertEqual(len(errors), 1) + self.assertIn("L3", errors[0]) + self.assertEqual(warnings, []) + + +class ExperimentValidationTests(unittest.TestCase): + record = {"equipment_id": "furnace", "filename": "runs.csv"} + + def _validate(self, user, experiment, record=None): + with patch.object(upload, "get_upload_target_experiment_pg", return_value=experiment): + return upload._validate_upload_experiment( + experiment_type_id="exp_1", + upload_record=record or self.record, + project_id="proj", + user=user, + ) + + def _experiment(self, **overrides): + base = {"id": "exp_1", "name": "Legacy", "equipment_id": "furnace", + "owner_id": "owner-1", "version": 4, "attached_to_project": False} + base.update(overrides) + return base + + def test_blank_experiment_is_allowed_and_skips_lookup(self): + with patch.object(upload, "get_upload_target_experiment_pg") as lookup: + result = upload._validate_upload_experiment( + experiment_type_id="", upload_record=self.record, project_id="p", user=OTHER + ) + self.assertEqual(result, ("", None)) + lookup.assert_not_called() + + def test_owner_admin_and_project_users_may_file_runs(self): + expected = ("exp_1", 4) + self.assertEqual(self._validate(OWNER, self._experiment()), expected) + self.assertEqual(self._validate(ADMIN, self._experiment()), expected) + self.assertEqual( + self._validate(OTHER, self._experiment(attached_to_project=True)), expected + ) + self.assertEqual(self._validate(OWNER, self._experiment(equipment_id="")), expected) + + def test_rejects_hidden_foreign_and_mismatched_experiments(self): + cases = [ + (OWNER, None, None, 404), + (OTHER, self._experiment(), None, 403), + (OWNER, self._experiment(equipment_id="etcher_01"), None, 400), + (OWNER, self._experiment(), {"equipment_id": "furnace", "filename": "t.db3"}, 400), + (OWNER, self._experiment(), {"equipment_id": "etcher", "filename": "r.csv"}, 400), + ] + for user, experiment, record, status in cases: + with self.assertRaises(HTTPException) as ctx: + self._validate(user, experiment, record) + self.assertEqual(ctx.exception.status_code, status) + + +class ProcessingTests(unittest.TestCase): + def _run(self, *, staged, kind="all", experiment="exp_1", ingest_error=None): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp, "runs.csv") + path.write_text( + "LOTNAME,run_date,Temperature,Sample ID,XRD Pattern\n" + "L1,2026-05-08,1600,S1,patterns/a.csv\n" + "L2,2026-05-09,1500,S2,patterns/b.csv\n" + ) + record = {"equipment_id": "furnace", "filename": "runs.csv", + "storage_path": str(path), "kind": kind} + with patch.object(upload, "_load_equipment_config", return_value=FURNACE_CONFIG), \ + patch.object(upload, "list_upload_files_pg", return_value=staged), \ + patch.object(upload, "find_samples_pg", return_value={}), \ + patch.object(upload, "ingest_equipment_upload_pg", return_value=2, + side_effect=ingest_error) as ingest, \ + patch.object(upload, "_update_upload_record") as update: + upload._execute_upload_processing_impl( + upload_id="u1", + upload_record=record, + project_id="proj", + prior_warnings=[], + experiment_type_id=experiment, + experiment_type_version=4 if experiment else None, + ) + return ingest, update + + def test_rows_are_tagged_and_files_linked_in_one_ingest(self): + ingest, update = self._run(staged=_staged("patterns/a.csv", "patterns/b.csv")) + kwargs = ingest.call_args.kwargs + self.assertEqual({row["experiment_type_id"] for row in kwargs["rows"]}, {"exp_1"}) + self.assertEqual({row["experiment_type_version"] for row in kwargs["rows"]}, {4}) + self.assertEqual( + [(l["row_index"], l["upload_file_id"]) for l in kwargs["file_links"]], + [(0, 1), (1, 2)], + ) + self.assertEqual(kwargs["kind"], "all") + self.assertEqual(kwargs["sample_links"], []) + self.assertEqual(update.call_args.kwargs["status"], "processed") + self.assertEqual(update.call_args.kwargs["errors"], []) + + def test_missing_attachment_fails_before_any_write(self): + ingest, update = self._run(staged=_staged("patterns/a.csv")) + ingest.assert_not_called() + self.assertEqual(update.call_args.kwargs["status"], "failed") + self.assertIn("patterns/b.csv", update.call_args.kwargs["errors"][0]) + + def test_no_attachments_keeps_text_and_still_reconciles(self): + ingest, update = self._run(staged=[], experiment="") + kwargs = ingest.call_args.kwargs + self.assertEqual(kwargs["file_links"], []) + self.assertEqual(update.call_args.kwargs["status"], "processed") + self.assertIn("stored as text only", update.call_args.kwargs["errors"][0]) + self.assertEqual({row["experiment_type_id"] for row in kwargs["rows"]}, {""}) + + def test_output_only_upload_is_ingested_as_a_split_merge(self): + ingest, _ = self._run( + staged=_staged("patterns/a.csv", "patterns/b.csv"), kind="output" + ) + kwargs = ingest.call_args.kwargs + self.assertEqual(kwargs["kind"], "output") + self.assertEqual(kwargs["file_links"][0]["lotname"], "L1") + + def test_ingest_failure_marks_the_upload_failed(self): + _, update = self._run( + staged=_staged("patterns/a.csv", "patterns/b.csv"), + ingest_error=ValueError("boom"), + ) + self.assertEqual(update.call_args.kwargs["status"], "failed") + + def test_linked_file_budget_is_enforced(self): + staged = [ + {"id": 1, "relative_path": "big.csv", "size_bytes": 600 * 1024 * 1024}, + ] + rows = [ + {"lotname": f"L{i}", "outputs": {"XRD Pattern": "big.csv"}} for i in range(2) + ] + with patch.dict(os.environ, {"DT_UPLOAD_LINKED_FILES_MAX_MB": "1000"}): + _, errors, _ = upload._file_output_links( + rows=rows, file_outputs=["XRD Pattern"], staged=staged + ) + self.assertEqual(len(errors), 1) + self.assertIn("limit is 1000 MB", errors[0]) + + +SYNTHESIS_CONFIG = { + "data": {"lotname_column": "LOTNAME", "timestamp_column": "run_date"}, + "parameters": [{"name": "Temperature", "type": "float"}], + "outputs": [{"name": "Sample ID", "type": "sample_id"}], +} +XRD_CONFIG = { + "data": {"lotname_column": "LOTNAME", "timestamp_column": "run_date"}, + "parameters": [{"name": "Sample ID", "type": "sample_id"}, + {"name": "Loading", "type": "string"}], + "outputs": [{"name": "XRD Pattern", "type": "file"}], +} + + +class SampleIdTests(unittest.TestCase): + def test_minted_ids_are_deterministic_igsn_style(self): + from sample_ids import mint_sample_id, normalize_sample_id + + first = mint_sample_id("seed", namespace="PUDT") + self.assertEqual(first, mint_sample_id("seed", namespace="PUDT")) + self.assertNotEqual(first, mint_sample_id("other", namespace="PUDT")) + self.assertRegex(first, r"^PUDT[0-9A-HJKMNP-TV-Z]{10}$") + self.assertEqual(normalize_sample_id(f" igsn:{first.lower()} "), first) + for bad in ("", "abc", "has space 123", "x" * 80, "../etc"): + self.assertEqual(normalize_sample_id(bad), "", bad) + + def test_namespace_comes_from_environment(self): + from sample_ids import igsn_namespace, mint_sample_id + + with patch.dict(os.environ, {"DT_IGSN_NAMESPACE": "abc1"}): + self.assertEqual(igsn_namespace(), "ABC1") + self.assertTrue(mint_sample_id("s").startswith("ABC1")) + with patch.dict(os.environ, {"DT_IGSN_NAMESPACE": "bad-ns!"}): + self.assertEqual(igsn_namespace(), "PUDT") + + def test_equal_instants_give_the_same_run_sample_id(self): + from sample_ids import canonical_run_instant, run_sample_id + + self.assertEqual( + canonical_run_instant("2026-05-08T00:00:00+00:00"), + canonical_run_instant("2026-05-07T20:00:00-04:00"), + ) + self.assertEqual(canonical_run_instant("2026-05-08"), "2026-05-08T00:00:00.000000Z") + self.assertEqual( + canonical_run_instant("2026-05-08T00:00:00"), + canonical_run_instant("2026-05-08T00:00:00+00:00"), + ) + self.assertNotEqual( + canonical_run_instant("2026-05-08T00:00:00.100000Z"), + canonical_run_instant("2026-05-08T00:00:00.900000Z"), + ) + common = dict(equipment_id="f", project_id="p", lotname="L1", field="Sample ID") + self.assertEqual( + run_sample_id(run_date="2026-05-08T00:00:00Z", **common), + run_sample_id(run_date="2026-05-07T20:00:00-04:00", **common), + ) + self.assertNotEqual( + run_sample_id(run_date="2026-05-08", **common), + run_sample_id(run_date="2026-05-08", **dict(common, project_id="other")), + ) + + def test_outputs_get_the_platform_id_and_invented_ids_are_rejected(self): + from sample_ids import run_sample_id + + issued = run_sample_id( + equipment_id="furnace", project_id="p", lotname="L2", + run_date="2026-05-09T00:00:00", field="Sample ID", + ) + rows = [ + {"lotname": "L1", "project_id": "p", "run_date": "2026-05-08T00:00:00", + "inputs": {}, "outputs": {"Sample ID": ""}}, + {"lotname": "L2", "project_id": "p", "run_date": "2026-05-09T00:00:00", + "inputs": {}, "outputs": {"Sample ID": issued.lower()}}, + {"lotname": "L3", "project_id": "p", "run_date": "2026-05-10T00:00:00", + "inputs": {}, "outputs": {"Sample ID": "MY-SAMPLE"}}, + ] + links, errors = upload._sample_links( + rows=rows, equipment_id="furnace", output_fields=["Sample ID"], input_fields=[] + ) + self.assertTrue(rows[0]["outputs"]["Sample ID"].startswith("PUDT")) + self.assertEqual(rows[1]["outputs"]["Sample ID"], issued) + self.assertEqual([l["role"] for l in links], ["output", "output"]) + self.assertEqual(len(errors), 1) + self.assertIn("not the ID the platform issues", errors[0]) + again = [dict(rows[0], outputs={"Sample ID": ""})] + links_again, _ = upload._sample_links( + rows=again, equipment_id="furnace", output_fields=["Sample ID"], input_fields=[] + ) + self.assertEqual(links_again[0]["sample_id"], links[0]["sample_id"]) + + def test_invalid_and_duplicate_samples_are_errors(self): + rows = [ + {"lotname": "L1", "project_id": "p", "run_date": "d", + "inputs": {"Sample ID": "not valid!"}, "outputs": {"Sample ID": ""}}, + {"lotname": "L1", "project_id": "p", "run_date": "d", + "inputs": {}, "outputs": {"Sample ID": ""}}, + ] + _, errors = upload._sample_links( + rows=rows, equipment_id="x", output_fields=["Sample ID"], + input_fields=["Sample ID"], + ) + self.assertEqual(len(errors), 2) + self.assertIn("not a valid sample ID", errors[0]) + self.assertIn("same lot name and run date", errors[1]) + + def test_missing_and_foreign_samples_get_the_same_error(self): + links = [ + {"row_index": 0, "lotname": "X1", "project_id": "p", "field_name": "Sample ID", + "role": "input", "sample_id": "PUDT0000000001"}, + {"row_index": 1, "lotname": "X2", "project_id": "p", "field_name": "Sample ID", + "role": "input", "sample_id": "PUDT0000000002"}, + {"row_index": 2, "lotname": "X3", "project_id": "p", "field_name": "Sample ID", + "role": "output", "sample_id": "PUDT0000000003"}, + {"row_index": 3, "lotname": "X4", "project_id": "p", "field_name": "Sample ID", + "role": "input", "sample_id": "PUDT0000000004"}, + ] + found = {"PUDT0000000002": "other", "PUDT0000000003": "other", + "PUDT0000000004": "p"} + with patch.object(upload, "find_samples_pg", return_value=found): + errors = upload._check_sample_links(links) + self.assertEqual(len(errors), 3) + for error in errors: + self.assertIn("not an available sample in this project", error) + self.assertNotIn("different project", " ".join(errors)) + + def _process(self, config, csv_text, found=None): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp, "runs.csv") + path.write_text(csv_text) + record = {"equipment_id": "eq", "filename": "runs.csv", "storage_path": str(path), + "kind": "all", "owner_id": "owner-1", "owner_org": "Birck"} + staged = _staged("patterns/a.csv") + with patch.object(upload, "_load_equipment_config", return_value=config), \ + patch.object(upload, "list_upload_files_pg", return_value=staged), \ + patch.object(upload, "find_samples_pg", return_value=found or {}), \ + patch.object(upload, "ingest_equipment_upload_pg", return_value=1) as ingest, \ + patch.object(upload, "_update_upload_record") as update: + upload._execute_upload_processing_impl( + upload_id="u1", upload_record=record, project_id="proj", prior_warnings=[] + ) + return ingest, update + + def test_synthesis_then_xrd_links_through_the_sample(self): + ingest, update = self._process( + SYNTHESIS_CONFIG, "LOTNAME,run_date,Temperature,Sample ID\nL1,2026-05-08,1600,\n" + ) + self.assertEqual(update.call_args.kwargs["status"], "processed") + kwargs = ingest.call_args.kwargs + sample_id = kwargs["rows"][0]["outputs"]["Sample ID"] + self.assertTrue(sample_id.startswith("PUDT")) + self.assertEqual(kwargs["owner_id"], "owner-1") + self.assertEqual(kwargs["sample_links"][0]["role"], "output") + + ingest, update = self._process( + XRD_CONFIG, + f"LOTNAME,run_date,Sample ID,Loading,XRD Pattern\nX1,2026-05-10,{sample_id.lower()},side,patterns/a.csv\n", + found={sample_id: "proj"}, + ) + self.assertEqual(update.call_args.kwargs["status"], "processed") + kwargs = ingest.call_args.kwargs + self.assertEqual(kwargs["rows"][0]["inputs"]["Sample ID"], sample_id) + self.assertEqual(kwargs["sample_links"][0]["role"], "input") + + def test_xrd_with_unknown_sample_fails_before_writing(self): + ingest, update = self._process( + XRD_CONFIG, + "LOTNAME,run_date,Sample ID,Loading,XRD Pattern\nX1,2026-05-10,PUDT0000000009,side,patterns/a.csv\n", + ) + ingest.assert_not_called() + self.assertEqual(update.call_args.kwargs["status"], "failed") + self.assertIn("not an available sample", update.call_args.kwargs["errors"][0]) + + +class EndpointTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = TestClient(main.app) + + def _record(self, **overrides): + record = {"id": "u1", "equipment_id": "furnace", "filename": "runs.csv", + "storage_path": "/x/runs.csv", "status": "success", + "owner_id": "owner-1", "owner_org": "Birck", "errors": [], "kind": "all"} + record.update(overrides) + return record + + def test_attachments_are_staged(self): + with patch.object(upload, "_load_upload_record", return_value=self._record()), \ + patch.object(upload, "replace_upload_files_pg", return_value=2) as replace: + response = self.client.post( + "/api/data/uploads/u1/attachments", + headers=OWNER_HEADERS, + files=[("files", ("patterns.zip", _zip({"patterns/a.csv": b"1,2\n", + "patterns/b.csv": b"3,4\n"}), + "application/zip"))], + ) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(response.json()["attachment_count"], 2) + staged = replace.call_args.args[1] + self.assertEqual(sorted(r["relative_path"] for r in staged), + ["patterns/a.csv", "patterns/b.csv"]) + + def test_attachments_rejected_while_processing_or_for_db3(self): + for record, status in ( + (self._record(status="processing"), 409), + (self._record(filename="t.db3"), 400), + ): + with patch.object(upload, "_load_upload_record", return_value=record), \ + patch.object(upload, "replace_upload_files_pg") as replace: + response = self.client.post( + "/api/data/uploads/u1/attachments", + headers=OWNER_HEADERS, + files=[("files", ("a.csv", b"1", "text/csv"))], + ) + self.assertEqual(response.status_code, status, response.text) + replace.assert_not_called() + + def test_process_passes_validated_experiment_to_background_job(self): + project = {"id": "proj", "equipment_id": ""} + with patch.object(upload, "_assert_project_visible", return_value=project), \ + patch.object(upload, "user_can_write_project_pg", return_value=True), \ + patch.object(upload, "_db3_user_is_allowed", return_value=False), \ + patch.object(upload, "_load_upload_record", return_value=self._record()), \ + patch.object(upload, "_validate_upload_experiment", return_value=("exp_1", 4)), \ + patch.object(upload, "_claim_upload_for_processing", return_value=True) as claim, \ + patch.object(upload, "_execute_upload_processing") as execute: + response = self.client.post( + "/api/data/uploads/u1/process", + headers=OWNER_HEADERS, + json={"project_id": "proj", "experiment_type_id": "exp_1"}, + ) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(response.json()["experiment_type_id"], "exp_1") + claim.assert_called_once_with("u1", "exp_1") + self.assertEqual(execute.call_args.kwargs["experiment_type_id"], "exp_1") + self.assertEqual(execute.call_args.kwargs["experiment_type_version"], 4) + + def test_process_requires_project_write_access(self): + project = {"id": "proj", "equipment_id": ""} + with patch.object(upload, "_assert_project_visible", return_value=project), \ + patch.object(upload, "user_can_write_project_pg", return_value=False), \ + patch.object(upload, "_claim_upload_for_processing") as claim: + response = self.client.post( + "/api/data/uploads/u1/process", + headers=OWNER_HEADERS, + json={"project_id": "proj"}, + ) + self.assertEqual(response.status_code, 403) + claim.assert_not_called() + + def test_attachment_replacement_conflict_is_reported(self): + with patch.object(upload, "_load_upload_record", return_value=self._record()), \ + patch.object(upload, "replace_upload_files_pg", + side_effect=RuntimeError("This upload is being processed")): + response = self.client.post( + "/api/data/uploads/u1/attachments", + headers=OWNER_HEADERS, + files=[("files", ("a.csv", b"1", "text/csv"))], + ) + self.assertEqual(response.status_code, 409) + + def test_file_download_is_safe_and_hides_missing_files(self): + stored = {"id": 7, "filename": 'pat"tern<1>.html', "content_type": "text/html", + "sha256": "ab" * 32, "content": b"x"} + with patch.object(dataset_v2, "get_equipment_run_file_pg", return_value=stored): + response = self.client.get( + "/api/dataset/v2/runs/5000000001/files/7", headers=OWNER_HEADERS + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, b"x") + self.assertEqual(response.headers["content-type"], "application/octet-stream") + self.assertEqual(response.headers["content-disposition"], + 'attachment; filename="pat_tern_1_.html"') + with patch.object(dataset_v2, "get_equipment_run_file_pg", return_value=None): + response = self.client.get( + "/api/dataset/v2/runs/5000000001/files/8", headers=OWNER_HEADERS + ) + self.assertEqual(response.status_code, 404) + + def test_run_list_forwards_experiment_filter(self): + with patch.object(dataset_v2, "get_runs_list_pg", return_value=[]) as runs: + response = self.client.get( + "/api/dataset/v2/runs?experiment_type_id=exp_1", headers=OWNER_HEADERS + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(runs.call_args.kwargs["experiment_type_id"], "exp_1") + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_upload_experiment_artifacts_pg.py b/api/tests/test_upload_experiment_artifacts_pg.py new file mode 100644 index 0000000..fa5a320 --- /dev/null +++ b/api/tests/test_upload_experiment_artifacts_pg.py @@ -0,0 +1,576 @@ +""" +Integration tests for experiment-targeted uploads and stored run files against +a real, disposable Postgres (same setup and safety marker as +test_row_level_security_pg.py). Skipped unless DT_RLS_TEST_PG_PORT is set. +""" +from __future__ import annotations + +import hashlib +import os +import sys +import unittest +import uuid +from pathlib import Path +from unittest.mock import patch + +API_ROOT = Path(__file__).resolve().parents[1] +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +TEST_PORT = os.getenv("DT_RLS_TEST_PG_PORT", "").strip() + +if TEST_PORT: + os.environ["PG_HOST"] = os.getenv("DT_RLS_TEST_PG_HOST", "127.0.0.1") + os.environ["PG_PORT"] = TEST_PORT + os.environ["PG_DB"] = os.getenv("DT_RLS_TEST_PG_DB", "digital_twin") + os.environ["PG_USER"] = "api_client" + os.environ["PG_PASS"] = os.environ["DT_RLS_TEST_API_PASS"] + os.environ["PG_SUPER_USER"] = os.getenv("DT_RLS_TEST_SUPER_USER", "postgres") + os.environ["PG_SUPER_PASS"] = os.environ["DT_RLS_TEST_SUPER_PASS"] + os.environ["PG_WORKER_USER"] = "api_worker" + os.environ["PG_WORKER_PASS"] = os.environ["DT_RLS_TEST_WORKER_PASS"] + os.environ["USE_MOCK_DATABASE"] = "false" + os.environ.setdefault("DT_SYSTEM_TOKEN", "rls-test-system-token") + os.environ.setdefault("INGESTION_TOKEN", "rls-test-ingestion-token") + +from security import PlatformUser # noqa: E402 + +OFFSET = 5_000_000_000 + + +def _user(user_id: str, role: str, org: str) -> PlatformUser: + return PlatformUser( + id=user_id, email=f"{user_id}@example.test", name=user_id.split("-")[0].title(), + role=role, organization=org, + ) + + +def _file(path: str, content: bytes) -> dict: + return { + "relative_path": path, + "filename": path.rsplit("/", 1)[-1], + "content_type": "text/csv", + "size_bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "content": content, + } + + +@unittest.skipUnless(TEST_PORT, "set DT_RLS_TEST_PG_PORT to run Postgres integration tests") +class UploadExperimentArtifactsPgTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + import data_loader_pg + import db_migrations + + conn = data_loader_pg.get_pg_migration_connection() + try: + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('public.rls_test_marker')") + if cur.fetchone()[0] is None: + raise RuntimeError( + "Refusing to run: the target database has no rls_test_marker table" + ) + finally: + conn.close() + + with patch.dict(os.environ, {"DB_RLS_ENFORCEMENT": "enforce"}): + db_migrations.run_schema_migrations() + # A second run must be a no-op (idempotent migrations). + db_migrations.run_schema_migrations() + + suffix = uuid.uuid4().hex[:6] + cls.alice = _user(f"alice-{suffix}", "pi", "LabA") + cls.bob = _user(f"bob-{suffix}", "pi", "LabB") + cls.admin = _user(f"admin-{suffix}", "admin", "Ops") + cls.equipment_id = f"furnace_{suffix}" + + conn = data_loader_pg.get_pg_migration_connection() + try: + with conn.cursor() as cur: + for person in (cls.alice, cls.bob, cls.admin): + cur.execute( + "INSERT INTO users (id, name, email, role, organization) " + "VALUES (%s, %s, %s, %s, %s)", + (person.id, person.name, person.email, person.role, person.organization), + ) + cur.execute( + "INSERT INTO equipment_metadata " + "(domain_id, equipment_name, owner_id, owner_org, status) " + "VALUES (%s, 'Test Furnace', %s, '', 'approved')", + (cls.equipment_id, cls.alice.id), + ) + conn.commit() + finally: + conn.close() + + from data_loader_pg import create_project_pg + from metadata_pg import create_experiment_type_pg + + cls.project_id = create_project_pg( + name=f"Furnace {suffix}", description="artifact test", + equipment_id=cls.equipment_id, equipment_name="Test Furnace", + access_mode="private", nanohub_user_id=cls.alice.id, + pi_name=cls.alice.name, creator_role=cls.alice.role, + )["id"] + cls.type_id = create_experiment_type_pg( + payload={ + "type_name": f"legacy xrd {suffix}", + "equipment_id": cls.equipment_id, + "visibility": "private", + "type_parameters": [ + {"name": "Temperature", "mode": "variable", "min_value": 1000, "max_value": 1700}, + ], + }, + user=cls.alice, + )["type_id"] + + def _new_upload(self, kind: str = "all") -> str: + import data_loader_pg + + upload_id = str(uuid.uuid4()) + conn = data_loader_pg.get_pg_migration_connection() + try: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO data_uploads ( + id, equipment_id, filename, stored_as, storage_path, + size_bytes, row_count, columns_json, errors_json, status, + kind, owner_id, owner_org + ) VALUES (%s, %s, 'runs.csv', 'runs.csv', '/tmp/runs.csv', + 10, 2, '[]'::jsonb, '[]'::jsonb, 'success', %s, %s, 'LabA') + """, + (upload_id, self.equipment_id, kind, self.alice.id), + ) + conn.commit() + finally: + conn.close() + return upload_id + + def _rows(self, experiment: str = "") -> list[dict]: + return [ + {"project_id": self.project_id, "lotname": "L1", "run_date": "2026-05-08T00:00:00", + "inputs": {"Temperature": 1600}, "outputs": {"XRD Pattern": "patterns/a.csv"}, + "raw": {}, "experiment_type_id": experiment}, + {"project_id": self.project_id, "lotname": "L2", "run_date": "2026-05-09T00:00:00", + "inputs": {"Temperature": 1500}, "outputs": {"XRD Pattern": "patterns/b.csv"}, + "raw": {}, "experiment_type_id": experiment}, + ] + + def test_full_flow_tags_runs_stores_files_and_respects_visibility(self): + from data_loader_pg import ( + attach_equipment_run_files_pg, + get_equipment_run_file_pg, + get_run_detail_pg, + get_runs_list_pg, + list_upload_files_pg, + replace_upload_files_pg, + sync_equipment_runs_pg, + ) + from metadata_pg import list_uploads_pg + + upload_id = self._new_upload() + replace_upload_files_pg( + upload_id, + [_file("batch/patterns/a.csv", b"10,1\n10.02,2\n"), + _file("batch/patterns/b.csv", b"10,3\n")], + ) + staged = {f["relative_path"]: f for f in list_upload_files_pg(upload_id)} + self.assertEqual(len(staged), 2) + uploads = {u["id"]: u for u in list_uploads_pg(self.alice)} + self.assertEqual(uploads[upload_id]["attachment_count"], 2) + + sync_equipment_runs_pg( + equipment_id=self.equipment_id, rows=self._rows(self.type_id), + upload_id=upload_id, upload_filename="runs.csv", + ) + attached = attach_equipment_run_files_pg( + equipment_id=self.equipment_id, upload_id=upload_id, kind="all", + links=[ + {"row_index": 0, "output_name": "XRD Pattern", + "upload_file_id": staged["batch/patterns/a.csv"]["id"]}, + {"row_index": 1, "output_name": "XRD Pattern", + "upload_file_id": staged["batch/patterns/b.csv"]["id"]}, + ], + ) + self.assertEqual(attached, 2) + + # Other tests in this class share the experiment; look at this upload's runs. + runs = [ + r for r in get_runs_list_pg( + nanohub_user_id=self.alice.id, experiment_type_id=self.type_id, limit=200, + ) + if r["lot_name"] in ("L1", "L2") + ] + self.assertEqual(sorted(r["lot_name"] for r in runs), ["L1", "L2"]) + self.assertTrue(all(r["experiment_type_id"] == self.type_id for r in runs)) + self.assertTrue(all(r["file_count"] == 1 for r in runs)) + self.assertEqual( + get_runs_list_pg(nanohub_user_id=self.alice.id, experiment_type_id="nope"), [] + ) + # Bob is not a member of the private project: no runs, no files. + self.assertEqual( + get_runs_list_pg(nanohub_user_id=self.bob.id, experiment_type_id=self.type_id), [] + ) + + run = next(r for r in runs if r["lot_name"] == "L1") + detail = get_run_detail_pg(run_id=run["run_id"], nanohub_user_id=self.alice.id) + self.assertEqual(len(detail["files"]), 1) + file_meta = detail["files"][0] + self.assertEqual(file_meta["output_name"], "XRD Pattern") + self.assertEqual(file_meta["source_path"], "batch/patterns/a.csv") + self.assertNotIn("content", file_meta) + + stored = get_equipment_run_file_pg( + run_id=run["run_id"], file_id=file_meta["id"], nanohub_user_id=self.alice.id + ) + self.assertEqual(stored["content"], b"10,1\n10.02,2\n") + self.assertEqual(stored["sha256"], hashlib.sha256(b"10,1\n10.02,2\n").hexdigest()) + self.assertIsNone( + get_equipment_run_file_pg( + run_id=run["run_id"], file_id=file_meta["id"], nanohub_user_id=self.bob.id + ) + ) + self.assertIsNotNone( + get_equipment_run_file_pg( + run_id=run["run_id"], file_id=file_meta["id"], + nanohub_user_id=self.admin.id, is_admin=True, + ) + ) + # A file id paired with the wrong run is not served. + other = next(r for r in runs if r["lot_name"] == "L2") + self.assertIsNone( + get_equipment_run_file_pg( + run_id=other["run_id"], file_id=file_meta["id"], nanohub_user_id=self.alice.id + ) + ) + + # Re-processing without the experiment and without links clears both, + # keeping the same run ids. + sync_equipment_runs_pg( + equipment_id=self.equipment_id, rows=self._rows(""), + upload_id=upload_id, upload_filename="runs.csv", + ) + attach_equipment_run_files_pg( + equipment_id=self.equipment_id, upload_id=upload_id, kind="all", links=[], + ) + detail = get_run_detail_pg(run_id=run["run_id"], nanohub_user_id=self.alice.id) + self.assertEqual(detail["experiment_type_id"], "") + self.assertEqual(detail["files"], []) + + def test_split_output_upload_attaches_by_natural_key(self): + from data_loader_pg import ( + attach_equipment_run_files_pg, + get_run_detail_pg, + get_runs_list_pg, + list_upload_files_pg, + merge_equipment_runs_pg, + replace_upload_files_pg, + ) + + input_upload = self._new_upload("input") + rows = [ + {"project_id": self.project_id, "lotname": "S1", "run_date": "2026-06-01T00:00:00", + "inputs": {"Temperature": 1400}, "outputs": {}, "raw": {}, + "experiment_type_id": self.type_id}, + ] + merge_equipment_runs_pg( + equipment_id=self.equipment_id, kind="input", rows=rows, + upload_id=input_upload, upload_filename="in.csv", + ) + output_upload = self._new_upload("output") + replace_upload_files_pg(output_upload, [_file("s1.csv", b"20,5\n")]) + staged = list_upload_files_pg(output_upload) + out_rows = [dict(rows[0], inputs={}, outputs={"XRD Pattern": "s1.csv"}, + experiment_type_id="")] + merge_equipment_runs_pg( + equipment_id=self.equipment_id, kind="output", rows=out_rows, + upload_id=output_upload, upload_filename="out.csv", + ) + attach_equipment_run_files_pg( + equipment_id=self.equipment_id, upload_id=output_upload, kind="output", + links=[{"row_index": 0, "project_id": self.project_id, "lotname": "S1", + "run_date": "2026-06-01T00:00:00", "output_name": "XRD Pattern", + "upload_file_id": staged[0]["id"]}], + ) + runs = [ + r for r in get_runs_list_pg(nanohub_user_id=self.alice.id, project_id=self.project_id) + if r["lot_name"] == "S1" + ] + self.assertEqual(len(runs), 1) + # An output file without an experiment keeps the input file's experiment. + self.assertEqual(runs[0]["experiment_type_id"], self.type_id) + detail = get_run_detail_pg(run_id=runs[0]["run_id"], nanohub_user_id=self.alice.id) + self.assertEqual([f["filename"] for f in detail["files"]], ["s1.csv"]) + + def test_missing_run_or_foreign_attachment_raises(self): + from data_loader_pg import attach_equipment_run_files_pg, list_upload_files_pg + from data_loader_pg import replace_upload_files_pg, sync_equipment_runs_pg + + first = self._new_upload() + second = self._new_upload() + replace_upload_files_pg(second, [_file("x.csv", b"1\n")]) + foreign_file_id = list_upload_files_pg(second)[0]["id"] + sync_equipment_runs_pg( + equipment_id=self.equipment_id, rows=self._rows()[:1], + upload_id=first, upload_filename="runs.csv", + ) + with self.assertRaises(ValueError): + attach_equipment_run_files_pg( + equipment_id=self.equipment_id, upload_id=first, kind="all", + links=[{"row_index": 0, "output_name": "XRD Pattern", + "upload_file_id": foreign_file_id}], + ) + with self.assertRaises(ValueError): + attach_equipment_run_files_pg( + equipment_id=self.equipment_id, upload_id=first, kind="all", + links=[{"row_index": 9, "output_name": "XRD Pattern", + "upload_file_id": foreign_file_id}], + ) + + def test_synthesis_to_xrd_sample_chain_and_lineage(self): + from data_loader_pg import ( + find_samples_pg, + get_run_detail_pg, + get_runs_list_pg, + get_sample_lineage_pg, + link_equipment_run_samples_pg, + sync_equipment_runs_pg, + ) + from sample_ids import mint_sample_id + + sample_id = mint_sample_id(f"pg-test|{uuid.uuid4().hex}") + synth_upload = self._new_upload() + synth_rows = [ + {"project_id": self.project_id, "lotname": "SYN1", + "run_date": "2026-07-01T00:00:00", "inputs": {"Temperature": 1600}, + "outputs": {"Sample ID": sample_id}, "raw": {}, + "experiment_type_id": self.type_id, "experiment_type_version": 3}, + ] + sync_equipment_runs_pg( + equipment_id=self.equipment_id, rows=synth_rows, + upload_id=synth_upload, upload_filename="synth.csv", + ) + link_equipment_run_samples_pg( + equipment_id=self.equipment_id, upload_id=synth_upload, kind="all", + owner_id=self.alice.id, owner_org="LabA", + links=[{"row_index": 0, "project_id": self.project_id, "lotname": "SYN1", + "field_name": "Sample ID", "role": "output", + "sample_id": sample_id, "sample_name": "SYN1"}], + ) + self.assertEqual(find_samples_pg([sample_id]), {sample_id: self.project_id}) + + xrd_upload = self._new_upload() + sync_equipment_runs_pg( + equipment_id=self.equipment_id, + rows=[{"project_id": self.project_id, "lotname": "XRD1", + "run_date": "2026-07-02T00:00:00", + "inputs": {"Sample ID": sample_id}, "outputs": {}, "raw": {}}], + upload_id=xrd_upload, upload_filename="xrd.csv", + ) + link_equipment_run_samples_pg( + equipment_id=self.equipment_id, upload_id=xrd_upload, kind="all", + owner_id=self.alice.id, owner_org="LabA", + links=[{"row_index": 0, "project_id": self.project_id, "lotname": "XRD1", + "field_name": "Sample ID", "role": "input", "sample_id": sample_id}], + ) + # Re-linking is idempotent. + link_equipment_run_samples_pg( + equipment_id=self.equipment_id, upload_id=xrd_upload, kind="all", + owner_id=self.alice.id, owner_org="LabA", + links=[{"row_index": 0, "project_id": self.project_id, "lotname": "XRD1", + "field_name": "Sample ID", "role": "input", "sample_id": sample_id}], + ) + + lineage = get_sample_lineage_pg(sample_id, nanohub_user_id=self.alice.id) + self.assertEqual(lineage["sample"]["project_id"], self.project_id) + self.assertEqual([r["lot_name"] for r in lineage["runs"]], ["SYN1", "XRD1"]) + self.assertEqual( + [r["sample_links"][0]["role"] for r in lineage["runs"]], ["output", "input"] + ) + self.assertIsNone(get_sample_lineage_pg(sample_id, nanohub_user_id=self.bob.id)) + + synth_run = next( + r for r in get_runs_list_pg(nanohub_user_id=self.alice.id, project_id=self.project_id) + if r["lot_name"] == "SYN1" + ) + self.assertEqual(synth_run["experiment_type_version"], 3) + detail = get_run_detail_pg(run_id=synth_run["run_id"], nanohub_user_id=self.alice.id) + self.assertEqual( + detail["samples"], + [{"sample_id": sample_id, "role": "output", "field_name": "Sample ID"}], + ) + + # An input naming an unregistered sample is refused. + with self.assertRaises(ValueError): + link_equipment_run_samples_pg( + equipment_id=self.equipment_id, upload_id=xrd_upload, kind="all", + owner_id=self.alice.id, owner_org="LabA", + links=[{"row_index": 0, "project_id": self.project_id, "lotname": "XRD1", + "field_name": "Sample ID", "role": "input", + "sample_id": mint_sample_id("never-made")}], + ) + + def _set_upload_status(self, upload_id: str, status: str) -> None: + import data_loader_pg + + conn = data_loader_pg.get_pg_migration_connection() + try: + with conn.cursor() as cur: + cur.execute("UPDATE data_uploads SET status = %s WHERE id = %s", (status, upload_id)) + conn.commit() + finally: + conn.close() + + def _ingest(self, **overrides): + from data_loader_pg import ingest_equipment_upload_pg + + args = dict( + equipment_id=self.equipment_id, kind="all", upload_filename="runs.csv", + file_links=[], sample_links=[], owner_id=self.alice.id, + owner_org="LabA", + ) + args.update(overrides) + return ingest_equipment_upload_pg(**args) + + def test_ingest_is_atomic_and_reconciles_replacement_uploads(self): + from data_loader_pg import ( + get_run_detail_pg, + get_runs_list_pg, + list_upload_files_pg, + replace_upload_files_pg, + ) + + lot = f"ATOM-{uuid.uuid4().hex[:6]}" + run_date = "2026-08-01T00:00:00" + input_upload = self._new_upload("input") + rows = [{"project_id": self.project_id, "lotname": lot, "run_date": run_date, + "inputs": {"Temperature": 1500}, "outputs": {}, "raw": {}, + "experiment_type_id": self.type_id, "experiment_type_version": 1}] + self._ingest(kind="input", rows=rows, upload_id=input_upload) + + # Output upload A attaches a pattern. + upload_a = self._new_upload("output") + replace_upload_files_pg(upload_a, [_file("a.csv", b"1,1\n")]) + file_id = list_upload_files_pg(upload_a)[0]["id"] + out_rows = [dict(rows[0], inputs={}, outputs={"XRD Pattern": "a.csv"}, + experiment_type_id="")] + self._ingest(kind="output", rows=out_rows, upload_id=upload_a, + file_links=[{"row_index": 0, "project_id": self.project_id, + "lotname": lot, "run_date": run_date, + "output_name": "XRD Pattern", "upload_file_id": file_id}]) + + def run(): + return next( + r for r in get_runs_list_pg(nanohub_user_id=self.alice.id, + project_id=self.project_id) + if r["lot_name"] == lot + ) + + self.assertEqual(run()["file_count"], 1) + + # A failing ingest (bad link) leaves nothing half-written. + upload_bad = self._new_upload("output") + with self.assertRaises(ValueError): + self._ingest(kind="output", rows=[dict(out_rows[0], outputs={"XRD Pattern": "x"})], + upload_id=upload_bad, + file_links=[{"row_index": 0, "project_id": self.project_id, + "lotname": lot, "run_date": run_date, + "output_name": "XRD Pattern", + "upload_file_id": 999999999}]) + detail = get_run_detail_pg(run_id=run()["run_id"], nanohub_user_id=self.alice.id) + self.assertEqual(detail["outputs"], {"XRD Pattern": "a.csv"}) + self.assertEqual(len(detail["files"]), 1) + + # Replacement output upload B without the file removes A's file. + upload_b = self._new_upload("output") + self._ingest(kind="output", rows=[dict(out_rows[0], outputs={})], upload_id=upload_b) + self.assertEqual(run()["file_count"], 0) + + # Re-processing the input after a template edit keeps the captured version. + self._ingest(kind="input", rows=[dict(rows[0], experiment_type_version=2)], + upload_id=input_upload) + self.assertEqual(run()["experiment_type_version"], 1) + + def test_replacement_clears_links_from_fields_no_longer_registered(self): + from data_loader_pg import get_run_detail_pg, get_runs_list_pg + from sample_ids import run_sample_id + + lot = f"OBS-{uuid.uuid4().hex[:6]}" + run_date = "2026-08-03T00:00:00" + input_upload = self._new_upload("input") + base = {"project_id": self.project_id, "lotname": lot, "run_date": run_date, + "inputs": {}, "outputs": {}, "raw": {}} + self._ingest(kind="input", rows=[base], upload_id=input_upload) + old_field_sample = run_sample_id( + equipment_id=self.equipment_id, project_id=self.project_id, lotname=lot, + run_date=run_date, field="Old Sample", + ) + upload_a = self._new_upload("output") + self._ingest( + kind="output", rows=[dict(base, outputs={"Old Sample": old_field_sample})], + upload_id=upload_a, + sample_links=[{"row_index": 0, "project_id": self.project_id, "lotname": lot, + "run_date": run_date, "field_name": "Old Sample", + "role": "output", "sample_id": old_field_sample}], + ) + run = next(r for r in get_runs_list_pg(nanohub_user_id=self.alice.id, + project_id=self.project_id) + if r["lot_name"] == lot) + detail = get_run_detail_pg(run_id=run["run_id"], nanohub_user_id=self.alice.id) + self.assertEqual(len(detail["samples"]), 1) + # The field was dropped from the schema; a new output upload replaces + # the outputs and must not keep the obsolete link. + upload_b = self._new_upload("output") + self._ingest(kind="output", rows=[dict(base, outputs={"Points": 10})], + upload_id=upload_b) + detail = get_run_detail_pg(run_id=run["run_id"], nanohub_user_id=self.alice.id) + self.assertEqual(detail["samples"], []) + + def test_combined_upload_keeps_version_on_reprocess(self): + from data_loader_pg import get_runs_list_pg + + lot = f"VER-{uuid.uuid4().hex[:6]}" + upload_id = self._new_upload() + row = {"project_id": self.project_id, "lotname": lot, + "run_date": "2026-08-02T00:00:00", "inputs": {}, "outputs": {}, "raw": {}, + "experiment_type_id": self.type_id, "experiment_type_version": 1} + self._ingest(rows=[row], upload_id=upload_id) + self._ingest(rows=[dict(row, experiment_type_version=5)], upload_id=upload_id) + found = [r for r in get_runs_list_pg(nanohub_user_id=self.alice.id, + project_id=self.project_id) + if r["lot_name"] == lot] + self.assertEqual(found[0]["experiment_type_version"], 1) + + def test_attachments_cannot_be_replaced_while_processing(self): + from data_loader_pg import replace_upload_files_pg + + upload_id = self._new_upload() + self._set_upload_status(upload_id, "processing") + with self.assertRaises(RuntimeError): + replace_upload_files_pg(upload_id, [_file("a.csv", b"1\n")]) + + def test_upload_target_experiment_visibility(self): + from metadata_pg import get_upload_target_experiment_pg + + found = get_upload_target_experiment_pg( + type_id=self.type_id, user=self.alice, project_id=self.project_id + ) + self.assertEqual(found["owner_id"], self.alice.id) + self.assertEqual(found["equipment_id"], self.equipment_id) + self.assertFalse(found["attached_to_project"]) + self.assertIsNone( + get_upload_target_experiment_pg( + type_id=self.type_id, user=self.bob, project_id=self.project_id + ) + ) + self.assertIsNotNone( + get_upload_target_experiment_pg( + type_id=self.type_id, user=self.admin, project_id=self.project_id + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/legacy_xrd/convert_legacy_xrd.py b/scripts/legacy_xrd/convert_legacy_xrd.py new file mode 100644 index 0000000..1a3899b --- /dev/null +++ b/scripts/legacy_xrd/convert_legacy_xrd.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +"""Convert the Anasori lab's legacy MAX-phase XRD exports into platform uploads. + +One-off ingestion helper for the Anasori lab's legacy MAX-phase data. Each raw +export is named + + MMDDYY_Composition_Temperature_Time_Initials[_extra tags].csv + +e.g. ``050826_(TiVNbZrCrMoHfTaW)4AlC3_1600_4h_TH.csv``, and holds a headerless +2-theta / intensity scan (one quoted ``2thetacounts`` field per line). + +The data is split into the two unit processes it records, linked through a +sample ID (as agreed with Prof. Strachan, 2026-10-08): + +1. ``synthesis.csv`` -- one row per physical sample made in the High + Temperature Furnace (Indy): the recipe read from the filename, and the + sample that came out, with a platform sample ID (IGSN syntax). Files that + differ only by extra tags (e.g. a quick scan and a long rescan) are the same + sample. +2. ``xrd.csv`` -- one row per XRD scan: the sample ID it measured, the scan + settings, and the pattern file (``patterns/.csv``, also bundled as + ``patterns.zip`` to attach to the XRD upload). + +Each sample gets exactly the ID the platform issues for that synthesis run +(``sample_ids.run_sample_id``: furnace, project, lot name, run date), so the +XRD upload can reference samples before the synthesis upload is processed, and +re-running gives the same IDs. The project and furnace must be the ones the +synthesis upload will be processed into. Nothing here talks to the platform: +it only reads the given folder and writes to ``--out``. + +Usage: + python convert_legacy_xrd.py RAW_DIR --project-id PROJECT --out OUT_DIR + [--furnace-id high_temperature_furnace_1_indy] [--dry-run] +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import statistics +import sys +import zipfile +from dataclasses import dataclass, field +from datetime import date +from fractions import Fraction +from pathlib import Path + +# The platform's sample-ID minting (api/sample_ids.py), so IDs issued here use +# exactly the platform's syntax and namespace rules. +_API_DIR = Path(__file__).resolve().parents[2] / "api" +if str(_API_DIR) not in sys.path: + sys.path.insert(0, str(_API_DIR)) +from sample_ids import run_sample_id # noqa: E402 + +DEFAULT_FURNACE_ID = "high_temperature_furnace_1_indy" +SAMPLE_FIELD = "Sample ID" + +# Initials confirmed by Tufail Hassan (email, 2026-10-05). +RESEARCHERS = { + "TH": "Tufail Hassan", + "PN": "Poulomi Nandi", + "KK": "Krutarth Kamath", + "MS": "Muhammad Sharif", + "CW": "Caleb Wasserbeck", +} + +# Scan settings by step size (degrees 2-theta). Times are as reported by the +# lab (effective time per step); quick scans are routine, long ones are the +# ~2 h scans used for Rietveld refinement. +SCAN_TYPES = { + Fraction("0.02"): ("quick", 49.0), + Fraction("0.015"): ("long", 361.24), +} + +# Conditions the lab reports as constant for its MAX synthesis runs. +LEGACY_CONSTANTS = { + "Atmosphere": "Ar", + "Heating Rate": 3.5, # degC/min + "Cooling Rate": 5.0, # degC/min + "Crucible": "Alumina", + "Radiation": "Cu K-alpha", + "Furnace": "High Temperature Furnace_1_Indy", +} + +ELEMENTS = set( + """H He Li Be B C N O F Ne Na Mg Al Si P S Cl Ar K Ca Sc Ti V Cr Mn Fe Co + Ni Cu Zn Ga Ge As Se Br Kr Rb Sr Y Zr Nb Mo Tc Ru Rh Pd Ag Cd In Sn Sb Te I + Xe Cs Ba La Ce Pr Nd Pm Sm Eu Gd Tb Dy Ho Er Tm Yb Lu Hf Ta W Re Os Ir Pt + Au Hg Tl Pb Bi Po At Rn Fr Ra Ac Th Pa U Np Pu""".split() +) + +# Unit process 1: synthesis in the furnace (one row per sample made). +SYNTHESIS_COLUMNS = [ + "LOTNAME", + "run_date", + "Temperature", + "Time", + "Composition", + "Composition Elements", + "Researcher", + "Atmosphere", + "Heating Rate", + "Cooling Rate", + "Crucible", + "Sample ID", + "Temperature Profile", + "Source File", +] + +# Unit process 2: XRD characterization (one row per scan). +XRD_COLUMNS = [ + "LOTNAME", + "run_date", + "Sample ID", + "Loading", + "Radiation", + "Scan Type", + "Step Size", + "Time per Step", + "Two Theta Min", + "Two Theta Max", + "Points", + "Researcher", + "XRD Pattern", + "Source File", + "Tags", +] + +_SUBSCRIPTS = str.maketrans("₀₁₂₃₄₅₆₇₈₉", "0123456789") +_DATE_RE = re.compile(r"^\d{6}$") +_TEMP_RE = re.compile(r"^(\d{3,4}(?:\.\d+)?)\s*(?:°?C)?$", re.IGNORECASE) +_TIME_RE = re.compile( + r"^(\d+(?:\.\d+)?)\s*(h|hr|hrs|hour|hours|m|min|mins|minute|minutes)$", + re.IGNORECASE, +) +_INITIALS_RE = re.compile(r"^[A-Z]{2,3}$") +_NUMBER_RE = re.compile(r"\d+(?:\.\d+)?") + + +class FormulaError(ValueError): + pass + + +def parse_formula(text: str) -> dict[str, Fraction]: + """Return atoms per formula unit for each element. + + Handles plain formulas (``Ti3AlC2``, ``Mo2Ti2AlC3``), explicit ratios + (``Ti1V1Mo0.5W1.5AlC3``) and bracketed groups. A group whose members carry + no counts is equimolar: ``(TiVNbZrCrMoHfTaW)4AlC3`` gives each metal 4/9. + Members with explicit counts are multiplied by the group's count, so + ``(Ti0.5V0.5)2AlC`` gives Ti 1, V 1. + """ + text = text.translate(_SUBSCRIPTS).strip() + if not text: + raise FormulaError("empty composition") + pos = 0 + + def read_number() -> Fraction | None: + nonlocal pos + m = _NUMBER_RE.match(text, pos) + if not m: + return None + pos = m.end() + return Fraction(m.group(0)) + + def read_group(closing: str | None) -> list[tuple[str, Fraction | None]]: + """Parse until ``closing``; return (element, count-or-None) terms.""" + nonlocal pos + terms: list[tuple[str, Fraction | None]] = [] + while pos < len(text): + ch = text[pos] + if closing and ch == closing: + pos += 1 + return terms + if ch in "([": + pos += 1 + inner = read_group(")" if ch == "(" else "]") + mult = read_number() or Fraction(1) + for element, amount in _resolve_group(inner, mult).items(): + terms.append((element, amount)) + continue + m = re.match(r"[A-Z][a-z]?", text[pos:]) + if not m: + raise FormulaError(f"unexpected {ch!r} at position {pos} in {text!r}") + symbol = m.group(0) + if symbol not in ELEMENTS: + raise FormulaError(f"unknown element {symbol!r} in {text!r}") + pos += len(symbol) + terms.append((symbol, read_number())) + if closing: + raise FormulaError(f"unclosed bracket in {text!r}") + return terms + + terms = read_group(None) + totals: dict[str, Fraction] = {} + for element, count in terms: + totals[element] = totals.get(element, Fraction(0)) + (count or Fraction(1)) + return totals + + +def _resolve_group( + terms: list[tuple[str, Fraction | None]], mult: Fraction +) -> dict[str, Fraction]: + if not terms: + raise FormulaError("empty bracket group") + out: dict[str, Fraction] = {} + if all(count is None for _, count in terms): + share = mult / len(terms) + for element, _ in terms: + out[element] = out.get(element, Fraction(0)) + share + else: + for element, count in terms: + out[element] = out.get(element, Fraction(0)) + (count or Fraction(1)) * mult + return out + + +_PRECURSOR_RE = re.compile(r"^(\d+(?:\.\d+)?)([A-Z][a-z]?)$") + + +def composition_elements(label: str) -> dict[str, Fraction]: + """Best-effort chemistry from a composition label; {} if there is none. + + Handles a plain formula (``(TiVNbZrCrMoHfTaW)4AlC3``), a label whose + ``-``-separated parts include precursor ratios (``G1-#4-3Ti-1.9Al-2C`` + gives Ti 3, Al 1.9, C 2), and a label with a formula among other parts + (``(9)-6M-(TiVNbMoTaW)4AlC3``, ``Ti3AlC2-Molten Salt``); in the last + case the first part that is a multi-element formula is used. + """ + try: + return parse_formula(label) + except FormulaError: + pass + parts = [part.strip() for part in label.split("-") if part.strip()] + precursors: dict[str, Fraction] = {} + for part in parts: + m = _PRECURSOR_RE.match(part) + if m and m.group(2) in ELEMENTS: + precursors[m.group(2)] = precursors.get(m.group(2), Fraction(0)) + Fraction(m.group(1)) + if len(precursors) >= 2: + return precursors + # Words inside a part count too, so "(TiVCrNbMoW)4AlC3 MAX" still finds + # the formula. + for piece in [p for part in parts for p in (part, *part.split())]: + try: + amounts = parse_formula(piece) + except FormulaError: + continue + if len(amounts) >= 2: + return amounts + return {} + + +@dataclass +class ParsedName: + stem: str + run_date: date | None = None + composition: str = "" + elements: dict[str, Fraction] = field(default_factory=dict) + temperature_c: float | None = None + time_min: float | None = None + initials: str = "" + tags: list[str] = field(default_factory=list) + problems: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + +def parse_filename(stem: str) -> ParsedName: + """Classify each ``_``-separated token by its shape rather than position, + so names that omit temperature or time still parse.""" + result = ParsedName(stem=stem) + tokens = [t.strip() for t in stem.split("_") if t.strip()] + if not tokens: + result.problems.append("empty filename") + return result + + first = tokens.pop(0) + if _DATE_RE.match(first): + month, day, year = int(first[:2]), int(first[2:4]), 2000 + int(first[4:]) + try: + result.run_date = date(year, month, day) + except ValueError: + result.problems.append(f"invalid MMDDYY date {first!r}") + else: + result.problems.append(f"first part {first!r} is not an MMDDYY date") + tokens.insert(0, first) + + # The token right after the date is the composition label (the lab puts + # the whole identifier between the first two underscores), unless the + # name skipped it and went straight to temperature or time. + if tokens and not _TEMP_RE.match(tokens[0]) and not _TIME_RE.match(tokens[0]): + result.composition = tokens.pop(0).translate(_SUBSCRIPTS) + result.elements = composition_elements(result.composition) + if not result.elements: + result.warnings.append( + f"no chemical formula found in {result.composition!r}; kept as a label only" + ) + + for token in tokens: + if result.temperature_c is None and (m := _TEMP_RE.match(token)): + result.temperature_c = float(m.group(1)) + continue + if result.time_min is None and (m := _TIME_RE.match(token)): + value, unit = float(m.group(1)), m.group(2).lower() + result.time_min = value * 60 if unit.startswith("h") else value + continue + if not result.initials and _INITIALS_RE.match(token): + result.initials = token + continue + result.tags.append(token) + + if not result.composition: + result.problems.append("no composition found") + if result.temperature_c is None: + result.warnings.append("no temperature in filename") + if result.time_min is None: + result.warnings.append("no time in filename") + elif result.time_min != int(result.time_min): + result.warnings.append( + f"time {result.time_min} min is not a whole number (registered as integer)" + ) + if not result.initials: + result.warnings.append("no researcher initials in filename") + elif result.initials not in RESEARCHERS: + result.warnings.append(f"unknown initials {result.initials!r}") + if result.tags: + result.warnings.append(f"extra tags {result.tags} (repeat scan?)") + return result + + +@dataclass +class Pattern: + two_theta: list[float] + intensity: list[float] + + @property + def step(self) -> float: + diffs = [b - a for a, b in zip(self.two_theta, self.two_theta[1:])] + return statistics.median(diffs) if diffs else 0.0 + + +def read_pattern(path: Path) -> Pattern: + """Read a raw export: one ``2thetacounts`` pair per line, optionally + wrapped in quotes, separated by tab/comma/space. Non-numeric lines (a + header, if one was added) are skipped.""" + two_theta: list[float] = [] + intensity: list[float] = [] + with path.open("r", encoding="utf-8-sig", errors="replace") as handle: + for line in handle: + cleaned = line.strip().strip('"').strip() + if not cleaned: + continue + parts = [p for p in re.split(r"[\t,;\s]+", cleaned.replace('"', "")) if p] + if len(parts) < 2: + continue + try: + x, y = float(parts[0]), float(parts[1]) + except ValueError: + continue + two_theta.append(round(x, 4)) + intensity.append(y) + return Pattern(two_theta, intensity) + + +def classify_scan(step: float) -> tuple[str, float | None]: + for known, (label, seconds) in SCAN_TYPES.items(): + if abs(step - float(known)) < 1e-4: + return label, seconds + return "unknown", None + + +def _fmt_amount(value: Fraction) -> float: + return round(float(value), 6) + + +def _fmt_number(value: float | None) -> str: + if value is None: + return "" + return str(int(value)) if value == int(value) else str(value) + + +def sample_key(parsed: ParsedName) -> str: + """Identity of the physical sample behind a scan: the filename without its + extra tags, so a quick scan and a long rescan of the same powder share it.""" + parts = parsed.stem.split("_") + tags = set(parsed.tags) + return "_".join(part for part in parts if part not in tags) if tags else parsed.stem + + +def convert( + raw_dir: Path, + out_dir: Path, + dry_run: bool, + *, + project_id: str, + furnace_id: str = DEFAULT_FURNACE_ID, + namespace: str | None = None, +) -> int: + files = sorted( + p for p in raw_dir.iterdir() if p.is_file() and p.suffix.lower() in {".csv", ".txt", ".xy"} + ) + if not files: + print(f"No .csv/.txt/.xy files found in {raw_dir}", file=sys.stderr) + return 1 + + synthesis_rows: dict[str, dict[str, str]] = {} + xrd_rows: list[dict[str, str]] = [] + issues: list[dict[str, str]] = [] + seen_scans: dict[str, str] = {} + patterns: dict[str, list[tuple[float, float]]] = {} + + for path in files: + parsed = parse_filename(path.stem) + pattern = read_pattern(path) + if len(pattern.two_theta) < 10: + parsed.problems.append(f"only {len(pattern.two_theta)} numeric data rows") + scan_type, time_per_step = classify_scan(pattern.step) + if scan_type == "unknown" and pattern.two_theta: + parsed.warnings.append(f"unrecognized step size {pattern.step:.4f}") + + scan = parsed.stem + if scan in seen_scans: + parsed.problems.append(f"duplicate scan name (also {seen_scans[scan]})") + seen_scans.setdefault(scan, path.name) + + for level, messages in (("error", parsed.problems), ("warning", parsed.warnings)): + for message in messages: + issues.append({"file": path.name, "level": level, "issue": message}) + if parsed.problems: + continue + + key = sample_key(parsed) + run_date = parsed.run_date.isoformat() if parsed.run_date else "" + sample_id = run_sample_id( + equipment_id=furnace_id, + project_id=project_id, + lotname=key, + run_date=run_date, + field=SAMPLE_FIELD, + namespace=namespace, + ) + researcher = RESEARCHERS.get(parsed.initials, "") + synthesis = synthesis_rows.get(key) + if synthesis is None: + synthesis = { + "LOTNAME": key, + "run_date": run_date, + "Temperature": _fmt_number(parsed.temperature_c), + "Time": _fmt_number(parsed.time_min), + "Composition": parsed.composition, + "Composition Elements": json.dumps( + {el: _fmt_amount(n) for el, n in parsed.elements.items()} + ), + "Researcher": researcher, + "Atmosphere": LEGACY_CONSTANTS["Atmosphere"], + "Heating Rate": _fmt_number(LEGACY_CONSTANTS["Heating Rate"]), + "Cooling Rate": _fmt_number(LEGACY_CONSTANTS["Cooling Rate"]), + "Crucible": LEGACY_CONSTANTS["Crucible"], + "Sample ID": sample_id, + # Not recorded for legacy runs. + "Temperature Profile": "", + "Source File": path.name, + } + synthesis_rows[key] = synthesis + else: + synthesis["Source File"] += f" {path.name}" + issues.append( + { + "file": path.name, + "level": "warning", + "issue": f"treated as another scan of sample {key} ({sample_id})", + } + ) + + pattern_name = f"{scan}.csv" + patterns[pattern_name] = list(zip(pattern.two_theta, pattern.intensity)) + xrd_rows.append( + { + "LOTNAME": scan, + # The scan date isn't recorded; the synthesis date is used. + "run_date": run_date, + "Sample ID": sample_id, + "Loading": "", + "Radiation": LEGACY_CONSTANTS["Radiation"], + "Scan Type": scan_type, + "Step Size": f"{pattern.step:.4g}" if pattern.two_theta else "", + "Time per Step": _fmt_number(time_per_step), + "Two Theta Min": _fmt_number(min(pattern.two_theta)), + "Two Theta Max": _fmt_number(max(pattern.two_theta)), + "Points": str(len(pattern.two_theta)), + "Researcher": researcher, + "XRD Pattern": f"patterns/{pattern_name}", + "Source File": path.name, + "Tags": " ".join(parsed.tags), + } + ) + + if not dry_run: + patterns_dir = out_dir / "patterns" + patterns_dir.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(out_dir / "patterns.zip", "w", zipfile.ZIP_DEFLATED) as archive: + for name, points in patterns.items(): + lines = ["two_theta_deg,intensity_counts"] + [ + f"{x},{_fmt_number(y)}" for x, y in points + ] + text = "\n".join(lines) + "\n" + (patterns_dir / name).write_text(text, encoding="utf-8") + archive.writestr(f"patterns/{name}", text) + for filename, columns, rows in ( + ("synthesis.csv", SYNTHESIS_COLUMNS, list(synthesis_rows.values())), + ("xrd.csv", XRD_COLUMNS, xrd_rows), + ("issues.csv", ["file", "level", "issue"], issues), + ): + with (out_dir / filename).open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=columns) + writer.writeheader() + writer.writerows(rows) + + errors = sum(1 for i in issues if i["level"] == "error") + warnings = len(issues) - errors + print( + f"{len(files)} files: {len(xrd_rows)} XRD scans of {len(synthesis_rows)} samples, " + f"{len(files) - len(xrd_rows)} rejected, {warnings} warnings" + ) + for issue in issues: + print(f" [{issue['level']}] {issue['file']}: {issue['issue']}") + if not dry_run: + print( + f"Wrote {out_dir / 'synthesis.csv'}, {out_dir / 'xrd.csv'}, " + f"{out_dir / 'patterns.zip'} and {out_dir / 'issues.csv'}" + ) + return 0 if errors == 0 else 2 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("raw_dir", type=Path, help="folder of raw XRD exports") + parser.add_argument("--out", type=Path, default=Path("legacy_xrd_out"), help="output folder") + parser.add_argument("--dry-run", action="store_true", help="report only; write nothing") + parser.add_argument( + "--project-id", + required=True, + help="platform project the synthesis and XRD uploads will be processed into", + ) + parser.add_argument( + "--furnace-id", + default=DEFAULT_FURNACE_ID, + help=f"equipment id of the synthesis furnace (default {DEFAULT_FURNACE_ID})", + ) + parser.add_argument( + "--namespace", + default=None, + help="IGSN namespace for sample IDs (default: DT_IGSN_NAMESPACE or PUDT, as the platform)", + ) + args = parser.parse_args(argv) + if not args.raw_dir.is_dir(): + parser.error(f"{args.raw_dir} is not a folder") + return convert( + args.raw_dir, + args.out, + args.dry_run, + project_id=args.project_id, + furnace_id=args.furnace_id, + namespace=args.namespace, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/legacy_xrd/test_convert_legacy_xrd.py b/scripts/legacy_xrd/test_convert_legacy_xrd.py new file mode 100644 index 0000000..1da9f45 --- /dev/null +++ b/scripts/legacy_xrd/test_convert_legacy_xrd.py @@ -0,0 +1,159 @@ +"""Tests for the legacy XRD converter. Run: python -m pytest scripts/legacy_xrd""" + +import csv +import sys +import tempfile +import unittest +import zipfile +from datetime import date +from fractions import Fraction +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from convert_legacy_xrd import FormulaError, convert, parse_filename, parse_formula # noqa: E402 + +RAW_LINES = [f'"{3 + i * 0.02:.16g}\t{100 + i}"\r\n' for i in range(50)] + + +class ParseFormulaTests(unittest.TestCase): + def test_equimolar_bracket(self): + amounts = parse_formula("(TiVNbZrCrMoHfTaW)4AlC3") + for metal in ("Ti", "V", "Nb", "Zr", "Cr", "Mo", "Hf", "Ta", "W"): + self.assertEqual(amounts[metal], Fraction(4, 9)) + self.assertEqual(amounts["Al"], 1) + self.assertEqual(amounts["C"], 3) + + def test_plain_formulas(self): + self.assertEqual(parse_formula("Ti3AlC2"), {"Ti": 3, "Al": 1, "C": 2}) + self.assertEqual(parse_formula("Mo2Ti2AlC3"), {"Mo": 2, "Ti": 2, "Al": 1, "C": 3}) + + def test_explicit_ratios(self): + amounts = parse_formula("Ti1V1Mo0.5W1.5AlC3") + self.assertEqual(amounts["Mo"], Fraction(1, 2)) + self.assertEqual(amounts["W"], Fraction(3, 2)) + + def test_bracket_with_counts_is_multiplied(self): + self.assertEqual(parse_formula("(Ti0.5V0.5)2AlC"), {"Ti": 1, "V": 1, "Al": 1, "C": 1}) + + def test_unicode_subscripts(self): + self.assertEqual(parse_formula("Ti₃AlC₂"), {"Ti": 3, "Al": 1, "C": 2}) + + def test_rejects_unknown_element_and_unclosed_bracket(self): + with self.assertRaises(FormulaError): + parse_formula("Xq2AlC") + with self.assertRaises(FormulaError): + parse_formula("(TiV2AlC") + + +class ParseFilenameTests(unittest.TestCase): + def test_standard_name(self): + p = parse_filename("050826_(TiVNbZrCrMoHfTaW)4AlC3_1600_4h_TH") + self.assertEqual(p.run_date, date(2026, 5, 8)) + self.assertEqual(p.temperature_c, 1600) + self.assertEqual(p.time_min, 240) + self.assertEqual(p.initials, "TH") + self.assertEqual(p.problems, []) + self.assertEqual(p.warnings, []) + + def test_decimal_hours_and_minutes(self): + self.assertEqual(parse_filename("010224_Ti3AlC2_1500_2.5h_KK").time_min, 150) + self.assertEqual(parse_filename("010224_Ti3AlC2_1500_90min_KK").time_min, 90) + + def test_missing_temperature_and_time_are_warnings(self): + p = parse_filename("010224_Ti3AlC2_MS") + self.assertEqual(p.problems, []) + self.assertIsNone(p.temperature_c) + self.assertIsNone(p.time_min) + self.assertEqual(p.initials, "MS") + self.assertEqual(len(p.warnings), 2) + + def test_extra_tags_are_kept_and_flagged(self): + p = parse_filename("010224_Ti3AlC2_1500_4h_PN_rescan") + self.assertEqual(p.tags, ["rescan"]) + self.assertTrue(any("rescan" in w for w in p.warnings)) + + def test_lab_composition_labels(self): + # Real names from the lab's first batch (email 2026-10-06). + p = parse_filename("050826_(9)-6M-(TiVNbMoTaW)4AlC3_1600C_4h_TH") + self.assertEqual(p.composition, "(9)-6M-(TiVNbMoTaW)4AlC3") + self.assertEqual(p.elements["Mo"], Fraction(2, 3)) + self.assertEqual((p.temperature_c, p.time_min, p.initials), (1600, 240, "TH")) + self.assertEqual(p.problems, []) + self.assertEqual(parse_filename("100526_Nb2AlN-NbN-1.5Al_1550C_4h_TH").elements, {"Nb": 2, "Al": 1, "N": 1}) + self.assertEqual(parse_filename("100526_Ti3AlC2-Molten Salt_1000C_10h_TH").time_min, 600) + self.assertEqual( + parse_filename("100526_G1-#4-3Ti-1.9Al-2C_1400C_4h_KK").elements, + {"Ti": 3, "Al": Fraction("1.9"), "C": 2}, + ) + + def test_label_without_formula_is_kept_with_warning(self): + p = parse_filename("100526_Mystery-Batch_1400C_4h_TH") + self.assertEqual(p.composition, "Mystery-Batch") + self.assertEqual(p.elements, {}) + self.assertEqual(p.problems, []) + self.assertTrue(any("label only" in w for w in p.warnings)) + + def test_bad_date_is_an_error(self): + self.assertTrue(parse_filename("133024_Ti3AlC2_1500_4h_TH").problems) + + +class ConvertTests(unittest.TestCase): + def test_splits_into_synthesis_and_xrd_linked_by_sample_id(self): + with tempfile.TemporaryDirectory() as tmp: + raw, out = Path(tmp, "raw"), Path(tmp, "out") + raw.mkdir() + Path(raw, "050826_(TiVNbZrCrMoHfTaW)4AlC3_1600_4h_TH.csv").write_text("".join(RAW_LINES)) + # A second scan of the same powder (extra tag) is the same sample. + Path(raw, "050826_(TiVNbZrCrMoHfTaW)4AlC3_1600_4h_TH_rescan.csv").write_text( + "".join(RAW_LINES) + ) + Path(raw, "notes_without_date.csv").write_text("".join(RAW_LINES)) + self.assertEqual(convert(raw, out, dry_run=False, project_id="proj_1", namespace="PUDT"), 2) + + with (out / "synthesis.csv").open() as handle: + synthesis = list(csv.DictReader(handle)) + with (out / "xrd.csv").open() as handle: + xrd = list(csv.DictReader(handle)) + self.assertEqual(len(synthesis), 1) + self.assertEqual(len(xrd), 2) + sample = synthesis[0] + self.assertEqual(sample["run_date"], "2026-05-08") + self.assertEqual(sample["Time"], "240") + self.assertEqual(sample["Researcher"], "Tufail Hassan") + self.assertRegex(sample["Sample ID"], r"^PUDT[0-9A-Z]{10}$") + self.assertEqual({row["Sample ID"] for row in xrd}, {sample["Sample ID"]}) + self.assertEqual(xrd[0]["Scan Type"], "quick") + self.assertEqual(xrd[1]["Tags"], "rescan") + + pattern = (out / xrd[0]["XRD Pattern"]).read_text().splitlines() + self.assertEqual(pattern[0], "two_theta_deg,intensity_counts") + self.assertEqual(pattern[1], "3.0,100") + self.assertEqual(len(pattern), 51) + with zipfile.ZipFile(out / "patterns.zip") as archive: + self.assertEqual( + sorted(archive.namelist()), sorted(row["XRD Pattern"] for row in xrd) + ) + + # It's exactly the ID the platform issues for that synthesis run. + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "api")) + from sample_ids import run_sample_id + + self.assertEqual( + sample["Sample ID"], + run_sample_id( + equipment_id="high_temperature_furnace_1_indy", project_id="proj_1", + lotname=sample["LOTNAME"], run_date="2026-05-08T00:00:00", + field="Sample ID", namespace="PUDT", + ), + ) + + # Re-running issues the same sample ID. + out2 = Path(tmp, "out2") + convert(raw, out2, dry_run=False, project_id="proj_1", namespace="PUDT") + with (out2 / "synthesis.csv").open() as handle: + self.assertEqual(next(csv.DictReader(handle))["Sample ID"], sample["Sample ID"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/app/data/catalog/[id]/page.tsx b/web/app/data/catalog/[id]/page.tsx index af70098..d3e1b80 100644 --- a/web/app/data/catalog/[id]/page.tsx +++ b/web/app/data/catalog/[id]/page.tsx @@ -3,7 +3,7 @@ import { use, useEffect, useState } from "react"; import Link from "next/link"; import { cn } from "@/lib/utils"; -import { getV2Run, type V2Run } from "@/lib/api-client"; +import { getV2Run, runFileDownloadUrl, type V2Run } from "@/lib/api-client"; import { RunTraceExplorer } from "@/components/run-trace-explorer"; import { ArrowLeft, @@ -464,6 +464,64 @@ export default function RunDetailPage({ params }: { params: Promise<{ id: string )} + {run.experiment_type_id && ( +

+ Experiment: + {run.experiment_type_id} + {run.experiment_type_version != null && ( + (version {run.experiment_type_version}) + )} +

+ )} + {(run.samples || []).length > 0 && ( +
+

+ Samples +

+
    + {(run.samples || []).map((sample) => ( +
  • + + {sample.role === "output" ? "Produced" : "Used"} ({sample.field_name}):{" "} + + {sample.sample_id} +
    • + ))} +
+
+ )} + {(run.files || []).length > 0 && ( +
+

+ Stored Files +

+
+ {(run.files || []).map((storedFile) => ( +
+
+

{storedFile.filename}

+

+ {storedFile.output_name} · {(storedFile.size_bytes / 1024).toFixed(1)} KB +

+

+ SHA-256 {storedFile.sha256} +

+
+ + Download + +
+ ))} +
+
+ )} {(run.file_refs || []).length > 0 && (

diff --git a/web/app/data/upload/page.tsx b/web/app/data/upload/page.tsx index 139a2a2..392582d 100644 --- a/web/app/data/upload/page.tsx +++ b/web/app/data/upload/page.tsx @@ -4,12 +4,15 @@ import { useState, useRef, useEffect } from "react"; import { cn } from "@/lib/utils"; import { getEquipmentListSimple, + getExperimentTypes, getUploadCapabilities, getUploads, getV2Projects, processUpload, + uploadAttachments, uploadFile, type EquipmentSimple, + type ExperimentTypeRecord, type PersistedUpload, type UploadResult, type UploadKind, @@ -26,6 +29,7 @@ import { Cpu, Download, Loader2, + Paperclip, Play, } from "lucide-react"; @@ -42,8 +46,12 @@ interface UploadedFile { errors?: string[]; message?: string; timestamp: string; + attachmentCount?: number; + experimentTypeId?: string; } +const KEEP_EXPERIMENT = "__keep__"; + function isDb3File(filename: string): boolean { return filename.toLowerCase().endsWith(".db3"); } @@ -86,6 +94,8 @@ function mapPersistedUpload(upload: PersistedUpload): UploadedFile { ? "Processing failed. Review the errors below." : "Upload status is unavailable.", timestamp: upload.uploaded_at, + attachmentCount: upload.attachment_count ?? 0, + experimentTypeId: upload.experiment_type_id ?? "", }; } @@ -134,6 +144,16 @@ export default function DataUploadPage() { const [equipmentLoading, setEquipmentLoading] = useState(true); const [selectedProject, setSelectedProject] = useState(""); const [processingUploadId, setProcessingUploadId] = useState(null); + const [experimentState, setExperimentState] = useState<{ + equipmentId: string; + items: ExperimentTypeRecord[]; + }>({ equipmentId: "", items: [] }); + // KEEP_EXPERIMENT re-uses each upload's earlier experiment (none for a new + // upload); "" explicitly files runs under no experiment. + const [selectedExperiment, setSelectedExperiment] = useState(KEEP_EXPERIMENT); + const [attachingUploadId, setAttachingUploadId] = useState(null); + const [attachTargetId, setAttachTargetId] = useState(null); + const attachInputRef = useRef(null); const [uploadKind, setUploadKind] = useState("input"); const [uploadCapabilities, setUploadCapabilities] = useState({ db3_upload: { enabled: false, max_mb: 64, equipment_ids: [] }, @@ -175,6 +195,28 @@ export default function DataUploadPage() { void fetchEquipment(); }, []); + // Experiments that runs uploaded for the selected equipment can be filed + // under. Equipment-agnostic templates are included by the API. + useEffect(() => { + if (!selectedEquipment || selectedEquipment === "etcher") return; + let cancelled = false; + void getExperimentTypes(selectedEquipment).then((data) => { + if (!cancelled) { + setExperimentState({ equipmentId: selectedEquipment, items: data ?? [] }); + } + }); + return () => { + cancelled = true; + }; + }, [selectedEquipment]); + const experiments = + experimentState.equipmentId === selectedEquipment ? experimentState.items : []; + + const selectEquipment = (equipmentId: string) => { + setSelectedEquipment(equipmentId); + setSelectedExperiment(KEEP_EXPERIMENT); + }; + // Validation and processing run server-side and can take minutes for large // files. Poll the uploads list while anything is in-flight so the UI shows // real progress (and reconciles a slow/dropped request) without the user @@ -320,7 +362,17 @@ export default function DataUploadPage() { : file ) ); - const result = await processUpload(uploadId, selectedProject); + // The experiment list belongs to the selected equipment, so only file + // uploads of that equipment under it. + // Re-processing keeps the upload's earlier experiment unless another one + // is selected. + const upload = files.find((file) => file.id === uploadId); + const pickerApplies = upload?.equipmentId === selectedEquipment; + const experimentTypeId = + pickerApplies && selectedExperiment !== KEEP_EXPERIMENT + ? selectedExperiment + : upload?.experimentTypeId || ""; + const result = await processUpload(uploadId, selectedProject, experimentTypeId); setProcessingUploadId(null); // Processing now runs in the background: a successful kick-off returns @@ -354,6 +406,35 @@ export default function DataUploadPage() { } }; + const startAttach = (uploadId: string) => { + setAttachTargetId(uploadId); + attachInputRef.current?.click(); + }; + + const handleAttachSelect = async (e: React.ChangeEvent) => { + const uploadId = attachTargetId; + const chosen = e.target.files ? Array.from(e.target.files) : []; + e.target.value = ""; + setAttachTargetId(null); + if (!uploadId || chosen.length === 0) return; + setAttachingUploadId(uploadId); + const result = await uploadAttachments(uploadId, chosen); + setAttachingUploadId(null); + setFiles((prev) => + prev.map((file) => { + if (file.id !== uploadId) return file; + if (!result.data) { + return { ...file, errors: [result.error || "Attaching files failed."] }; + } + return { + ...file, + attachmentCount: result.data.attachment_count, + message: `${result.data.attachment_count} file(s) attached. Process the upload to link them to its runs.`, + }; + }) + ); + }; + const handleDrop = (e: React.DragEvent) => { e.preventDefault(); setDragOver(false); @@ -437,7 +518,7 @@ export default function DataUploadPage() { {equipment.map((eq) => (

{/* Upload kind: split tools accept an inputs-only or outputs-only @@ -553,6 +657,14 @@ export default function DataUploadPage() { + + {/* Upload Queue */} {files.length > 0 && (
@@ -606,6 +718,17 @@ export default function DataUploadPage() { {file.kind} )} + {(file.attachmentCount ?? 0) > 0 && ( + + + {file.attachmentCount} attached + + )} + {file.experimentTypeId && ( + + exp: {file.experimentTypeId} + + )}
{file.status === "uploading" && ( @@ -648,10 +771,40 @@ export default function DataUploadPage() { )} - {file.id && file.status === "success" && ( + {file.id && + file.equipmentId !== "etcher" && + file.kind !== "input" && + file.name.toLowerCase().endsWith(".csv") && + (file.status === "success" || file.status === "failed" || file.status === "processed") && ( + + )} + + {/* A failed upload can be retried, e.g. after attaching the + files its CSV refers to; processing is idempotent. */} + {file.id && + (file.status === "success" || + ((file.status === "failed" || file.status === "processed") && + !isDb3File(file.name) && + file.equipmentId !== "etcher")) && ( )} @@ -693,7 +846,7 @@ export default function DataUploadPage() {

3. Processing

- Processing runs in the background and the status updates here automatically — no need to reload. Input and output CSV files are merged by lot name and run date. Excel files are stored for provenance but are not ingested automatically. + Processing runs in the background and the status updates here automatically — no need to reload. Input and output CSV files are merged by lot name and run date. Excel files are stored for provenance but are not ingested automatically. For outputs registered with type "file" (for example an XRD pattern), put each file's path in the CSV cell, attach the files (or one .zip, up to 64 MB) before processing, and they are stored with their runs. {uploadCapabilities.db3_upload.enabled && " The temporary DB3 pilot creates one catalog row per physical GLANCE run and stores every timestamped sample as its linked trace."}

diff --git a/web/app/equipment/register/page.tsx b/web/app/equipment/register/page.tsx index 01477b9..84f93e6 100644 --- a/web/app/equipment/register/page.tsx +++ b/web/app/equipment/register/page.tsx @@ -186,6 +186,9 @@ const PARAMETER_TYPE_OPTIONS = [ "boolean", "array", "map", + // A platform-issued sample ID (IGSN syntax). As an output the platform issues + // it; as an input it must name a sample an earlier process produced. + "sample_id", ]; const OUTPUT_TYPE_OPTIONS = [...PARAMETER_TYPE_OPTIONS, "file"]; diff --git a/web/lib/api-client.ts b/web/lib/api-client.ts index b7e309b..4f7d9f1 100644 --- a/web/lib/api-client.ts +++ b/web/lib/api-client.ts @@ -133,6 +133,22 @@ export interface V2Project { optimization_iterations: number; } +/** A file output (e.g. an XRD pattern) stored with a generic equipment run. */ +export interface V2RunStoredFile { + id: number; + output_name: string; + filename: string; + content_type: string; + size_bytes: number; + sha256: string; + source_path: string; + created_at: string | null; +} + +export function runFileDownloadUrl(runId: number, fileId: number) { + return `${API_URL}/dataset/v2/runs/${encodeURIComponent(String(runId))}/files/${encodeURIComponent(String(fileId))}`; +} + export interface V2RunFileRef { id: string; run_id: number; @@ -172,6 +188,15 @@ export interface V2Run { pi_name: string | null; project_access: V2Project["access"] | null; file_refs: V2RunFileRef[]; + /** Experiment (experiment_types.id) the run was uploaded into; "" if none. */ + experiment_type_id?: string; + file_count?: number; + /** Stored file outputs; returned by the run detail endpoint only. */ + files?: V2RunStoredFile[]; + /** Experiment template version the run was captured under. */ + experiment_type_version?: number | null; + /** Samples the run produced (output) or used (input); run detail only. */ + samples?: { sample_id: string; role: "input" | "output"; field_name: string }[]; source?: string; upload_filename?: string; raw_payload?: { @@ -497,6 +522,7 @@ export interface GetV2RunsOptions { limit?: number; offset?: number; project_id?: string; + experiment_type_id?: string; } export function getV2Runs(options: GetV2RunsOptions = {}) { @@ -513,6 +539,9 @@ export function getV2Runs(options: GetV2RunsOptions = {}) { if (options.project_id) { params.set("project_id", options.project_id); } + if (options.experiment_type_id) { + params.set("experiment_type_id", options.experiment_type_id); + } const query = params.toString(); const path = query ? `/dataset/v2/runs?${query}` : "/dataset/v2/runs"; return apiFetch(path); @@ -1497,6 +1526,8 @@ export interface PersistedUpload { owner_id: string; owner_org: string; uploaded_at: string; + experiment_type_id?: string; + attachment_count?: number; } export interface UploadCapabilities { @@ -1512,12 +1543,21 @@ export interface ProcessUploadResult { upload_id: string; equipment_id: string; project_id: string; + experiment_type_id?: string; inserted: number; row_count: number; errors: string[]; message: string; } +export interface UploadAttachmentsResult { + upload_id: string; + attachment_count: number; + total_bytes: number; + files: string[]; + message: string; +} + export interface UploadFileResponse { data: UploadResult | null; error: string | null; @@ -1573,14 +1613,54 @@ export function getUploadCapabilities() { return apiFetch("/data/upload-capabilities"); } -export function processUpload(uploadId: string, projectId: string) { +export function processUpload( + uploadId: string, + projectId: string, + experimentTypeId = "", +) { return apiJson( `/data/uploads/${encodeURIComponent(uploadId)}/process`, "POST", - { project_id: projectId }, + { project_id: projectId, experiment_type_id: experimentTypeId }, ); } +/** + * Attach the files a CSV upload refers to (plain files and/or .zip archives). + * Replaces the upload's earlier attachments; process the upload afterwards to + * link them to its runs. + */ +export async function uploadAttachments( + uploadId: string, + files: File[], +): Promise<{ data: UploadAttachmentsResult | null; error: string | null }> { + try { + const formData = new FormData(); + for (const file of files) { + formData.append("files", file); + } + const response = await fetch( + `${API_URL}/data/uploads/${encodeURIComponent(uploadId)}/attachments`, + { method: "POST", body: formData, cache: "no-store" }, + ); + if (!response.ok) { + let error = `Attaching files failed (HTTP ${response.status}).`; + try { + const body = (await response.json()) as { detail?: unknown }; + if (typeof body.detail === "string" && body.detail.trim()) { + error = body.detail; + } + } catch { + // The response was not JSON; keep the status-based message. + } + return { data: null, error }; + } + return { data: (await response.json()) as UploadAttachmentsResult, error: null }; + } catch { + return { data: null, error: "API unavailable — is the FastAPI server running?" }; + } +} + export interface EtcherProcessorStatus { available: boolean; missing_required_env: string[];