Data read modified to work with the new pipeline

Nolte-Group · Oct 1, 2025 · f4418e2 · f4418e2
1 parent a94fffa
commit f4418e2
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/analysis.py b/analysis.py
@@ -45,7 +45,12 @@ def pca(data, features):
 
 if __name__ == "__main__":
     spark = SparkSession.builder.appName("train").getOrCreate()
-    data = load(spark, split=[0.9, 0.5, 0.5])
+    SPLIT = [0.9, 0.05, 0.05]
+    load_from_scratch = False
+    if load_from_scratch:
+        data = etl(spark, split=SPLIT)
+    else:
+        data = read(spark)
 
     pca(data)
     category_distribution(data)