Added an argument that allows output files to be read in by Dask (usi…

…ng open_mfdataset), which is needed for very large data files.
MintonGroup · Feb 19, 2023 · 91d2ad8 · 91d2ad8
1 parent 175d83c
commit 91d2ad8
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/python/swiftest/swiftest/io.py b/python/swiftest/swiftest/io.py
@@ -842,7 +842,9 @@ def swiftest2xr(param, verbose=True):
 
     if ((param['OUT_TYPE'] == 'NETCDF_DOUBLE') or (param['OUT_TYPE'] == 'NETCDF_FLOAT')):
         if verbose: print('\nCreating Dataset from NetCDF file')
-        ds = xr.open_dataset(param['BIN_OUT'], mask_and_scale=False)
+        #ds = xr.open_dataset(param['BIN_OUT'], mask_and_scale=False)
+        ds = xr.open_mfdataset(param['BIN_OUT'], parallel=True, engine='h5netcdf', mask_and_scale=False)
+
         ds = process_netcdf_input(ds, param)
     else:
         print(f"Error encountered. OUT_TYPE {param['OUT_TYPE']} not recognized.")

diff --git a/python/swiftest/swiftest/tool.py b/python/swiftest/swiftest/tool.py
@@ -18,7 +18,7 @@ def magnitude(ds,x):
     dim = "space"
     ord = None
     return xr.apply_ufunc(
-        np.linalg.norm, ds[x], input_core_dims=[[dim]], kwargs={"ord": ord, "axis": -1}
+        np.linalg.norm, ds[x], input_core_dims=[[dim]], kwargs={"ord": ord, "axis": -1}, dask="parallelized"
     )
 
 def wrap_angle(angle):