Upload into experiments, store file outputs, issue sample IDs

api/db_migrations.py

@@ -169,6 +169,7 @@ def run_schema_migrations() -> None:
         ensure_etcher_recipe_name_column_pg,
         ensure_etcher_run_file_refs_pg,
         ensure_glance_ingestion_audit_pg,
+        ensure_upload_experiment_and_run_files_pg,
         get_pg_migration_connection,
     )
     from metadata_pg import (
@@ -221,6 +222,8 @@ def run_schema_migrations() -> None:
         ensure_equipment_access_table_pg()
         ensure_data_uploads_kind_column_pg()
         ensure_data_uploads_processing_started_at_pg()
+        # After data_uploads and equipment_runs exist (both referenced by FKs).
+        ensure_upload_experiment_and_run_files_pg()
         ensure_ml_endpoint_cache_table_pg()
         ensure_platform_notifications_table_pg()
         ensure_role_change_requests_table_pg()

api/main.py

@@ -49,6 +49,8 @@
     "equipment_runs",
     "equipment_run_trace_samples",
     "equipment_run_trace_events",
+    "equipment_run_files",
+    "equipment_run_samples",
     "glance_ingestion_audit",
     "samples",
     "experiment_samples",
@@ -60,6 +62,7 @@
     "experiment_recipe_proposals",
     "purr_publication_requests",
     "data_uploads",
+    "data_upload_files",
     "ingestion_logs",
     "equipment_access",
 )

api/metadata_pg.py

@@ -2525,12 +2525,12 @@ def list_experiment_types_pg(
     if equipment_id:
         conditions.append("(equipment_id = %s OR equipment_id = '')")
         params.append(equipment_id)
-    if user.role != "admin":
-        conditions.append(
-            "(owner_id = %s OR visibility IN ('shared', 'published') OR owner_org = %s)"
-        )
-        params.extend([user.id, user.organization])
-    where_sql = f"WHERE {' AND '.join(conditions)}" if conditions else ""
+    where_sql = f"WHERE {' AND '.join(conditions)}" if conditions else "WHERE true"
+    # Same rule as everywhere else templates are used (and as RLS): own,
+    # shared/published, same organization, or attached to a readable project.
+    visibility_sql, visibility_params = _visible_experiment_type_filter(user)
+    where_sql += visibility_sql
+    params.extend(visibility_params)
 
     try:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
@@ -2549,6 +2549,60 @@ def list_experiment_types_pg(
         conn.close()
 
 
+def get_upload_target_experiment_pg(
+    *,
+    type_id: str,
+    user: PlatformUser,
+    project_id: str,
+) -> dict[str, Any] | None:
+    """
+    Return the experiment template an upload may be processed into, or None
+    when ``user`` can't see it.
+
+    The result carries ``equipment_id``, ``owner_id`` and
+    ``attached_to_project`` (whether ``project_id`` uses the template through
+    ``project_experiments``) so the caller can decide whether the user may file
+    runs under it. Visibility uses the same rule as the template library.
+    """
+    visibility_sql, visibility_params = _visible_experiment_type_filter(user)
+    conn = get_pg_connection(user)
+    try:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute(
+                f"""
+                SELECT experiment_types.id,
+                       experiment_types.name,
+                       experiment_types.equipment_id,
+                       experiment_types.owner_id,
+                       experiment_types.version,
+                       EXISTS (
+                           SELECT 1
+                           FROM project_experiments target_pe
+                           WHERE target_pe.type_id = experiment_types.id
+                             AND target_pe.project_id = %s
+                       ) AS attached_to_project
+                FROM experiment_types
+                WHERE experiment_types.id = %s
+                {visibility_sql}
+                """,
+                [project_id, type_id, *visibility_params],
+            )
+            row = cur.fetchone()
+        conn.commit()
+    finally:
+        conn.close()
+    if not row:
+        return None
+    return {
+        "id": row["id"],
+        "name": row["name"] or row["id"],
+        "equipment_id": str(row["equipment_id"] or ""),
+        "owner_id": row["owner_id"],
+        "version": int(row["version"]) if row.get("version") is not None else None,
+        "attached_to_project": bool(row["attached_to_project"]),
+    }
+
+
 def list_public_experiment_types_pg(
     *,
     equipment_id: str = "",
@@ -2906,6 +2960,22 @@ def _ensure_project_write_access(cur, *, project_id: str, user: PlatformUser) ->
     raise PermissionError("Only project members or admins can change this project")
 
 
+def user_can_write_project_pg(*, project_id: str, user: PlatformUser) -> bool:
+    """Whether ``user`` may add data to ``project_id`` (admin, or the project's
+    PI or a member). Viewers of open projects can't."""
+    conn = get_pg_connection(user)
+    try:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            try:
+                _ensure_project_write_access(cur, project_id=project_id, user=user)
+            except PermissionError:
+                return False
+        conn.commit()
+        return True
+    finally:
+        conn.close()
+
+
 def _ensure_project_publish_access(cur, *, project_id: str, user: PlatformUser) -> None:
     role = _ensure_project_access(cur, project_id=project_id, user=user)
     if user.role == "admin" or role == "pi":
@@ -6840,7 +6910,13 @@ def list_uploads_pg(user: PlatformUser) -> list[dict[str, Any]]:
                     kind,
                     owner_id,
                     owner_org,
-                    uploaded_at
+                    uploaded_at,
+                    experiment_type_id,
+                    (
+                        SELECT COUNT(*)
+                        FROM data_upload_files f
+                        WHERE f.upload_id = data_uploads.id
+                    ) AS attachment_count
                 FROM data_uploads
                 ORDER BY uploaded_at DESC
                 """
@@ -6875,6 +6951,8 @@ def list_uploads_pg(user: PlatformUser) -> list[dict[str, Any]]:
                     "owner_id": record["owner_id"],
                     "owner_org": record["owner_org"],
                     "uploaded_at": _iso(record["uploaded_at"]),
+                    "experiment_type_id": record.get("experiment_type_id") or "",
+                    "attachment_count": int(record.get("attachment_count") or 0),
                 }
             )
         return uploads

api/routers/dataset_v2.py

@@ -19,6 +19,8 @@
     add_project_member_pg,
     create_project_pg,
     delete_project_pg,
+    get_equipment_run_file_pg,
+    get_sample_lineage_pg,
     get_equipment_run_trace_pg,
     get_equipment_recipe_pg,
     get_glance_ingestion_project_pg,
@@ -723,6 +725,7 @@ def runs(
     limit: int = 200,
     offset: int = 0,
     project_id: Optional[str] = None,
+    experiment_type_id: Optional[str] = None,
     platform_user: PlatformUser = Depends(get_platform_user),
 ):
     """
@@ -735,11 +738,15 @@ def runs(
     ``project_id`` (optional) narrows the list to a single project. Combined
     with RLS, a PI asking for a project they don't belong to receives an
     empty list — the same shape as for any inaccessible project.
+
+    ``experiment_type_id`` (optional) narrows the list to runs uploaded into
+    that experiment.
     """
     is_admin = platform_user.role == "admin"
     logger.info(
-        "[dataset/v2/runs] Fetching runs for user=%s role=%s admin_bypass=%s project_id=%s",
-        platform_user.id, platform_user.role, is_admin, project_id,
+        "[dataset/v2/runs] Fetching runs for user=%s role=%s admin_bypass=%s "
+        "project_id=%s experiment_type_id=%s",
+        platform_user.id, platform_user.role, is_admin, project_id, experiment_type_id,
     )
     return get_runs_list_pg(
         nanohub_user_id=platform_user.id,
@@ -749,6 +756,7 @@ def runs(
         offset=offset,
         is_admin=is_admin,
         project_id=project_id,
+        experiment_type_id=(experiment_type_id or "").strip() or None,
     )
 
 
@@ -1003,6 +1011,79 @@ def run_trace(
     return record
 
 
+_SAFE_DOWNLOAD_NAME = re.compile(r"[^A-Za-z0-9._ -]+")
+# Types served as-is; anything else (e.g. HTML/SVG an uploader could plant) is
+# sent as an opaque download.
+_SAFE_DOWNLOAD_TYPES = frozenset(
+    {
+        "text/csv",
+        "text/plain",
+        "application/json",
+        "application/pdf",
+        "image/png",
+        "image/jpeg",
+        "image/tiff",
+    }
+)
+
+
+@router.get("/runs/{run_id}/files/{file_id}")
+def run_file_download(
+    run_id: int,
+    file_id: int,
+    platform_user: PlatformUser = Depends(get_platform_user),
+):
+    """Download one file output (e.g. an XRD pattern) stored with a run.
+
+    Same visibility as the run itself; returns 404 when the run or file is
+    missing or hidden, without saying which."""
+    is_admin = platform_user.role == "admin"
+    record = get_equipment_run_file_pg(
+        run_id=run_id,
+        file_id=file_id,
+        nanohub_user_id=platform_user.id,
+        is_admin=is_admin,
+    )
+    if record is None:
+        raise HTTPException(status_code=404, detail="File not found")
+    filename = _SAFE_DOWNLOAD_NAME.sub("_", str(record["filename"] or "file")).strip() or "file"
+    return Response(
+        content=record["content"],
+        media_type=(
+            record.get("content_type")
+            if record.get("content_type") in _SAFE_DOWNLOAD_TYPES
+            else "application/octet-stream"
+        ),
+        headers={
+            "Content-Disposition": f'attachment; filename="{filename}"',
+            "X-Content-Type-Options": "nosniff",
+            "X-Content-SHA256": str(record.get("sha256") or ""),
+        },
+    )
+
+
+@router.get("/samples/{sample_id}/lineage")
+def sample_lineage(
+    sample_id: str,
+    platform_user: PlatformUser = Depends(get_platform_user),
+):
+    """A sample and the visible runs that produced it or used it (e.g. the
+    furnace run that made a powder and the XRD scans of that powder)."""
+    from sample_ids import normalize_sample_id
+
+    canonical = normalize_sample_id(sample_id)
+    if not canonical:
+        raise HTTPException(status_code=404, detail="Sample not found")
+    record = get_sample_lineage_pg(
+        canonical,
+        nanohub_user_id=platform_user.id,
+        is_admin=platform_user.role == "admin",
+    )
+    if record is None:
+        raise HTTPException(status_code=404, detail="Sample not found")
+    return record
+
+
 @router.post("/glance/traces/sync")
 async def sync_glance_traces(
     request: Request,