From 91d2ad86f68be00584ef5512dd55da03edebdf89 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Sun, 19 Feb 2023 11:08:00 -0500 Subject: [PATCH] Added an argument that allows output files to be read in by Dask (using open_mfdataset), which is needed for very large data files. --- python/swiftest/swiftest/io.py | 4 +++- python/swiftest/swiftest/tool.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/swiftest/swiftest/io.py b/python/swiftest/swiftest/io.py index 98e6c154a..d070bbb91 100644 --- a/python/swiftest/swiftest/io.py +++ b/python/swiftest/swiftest/io.py @@ -842,7 +842,9 @@ def swiftest2xr(param, verbose=True): if ((param['OUT_TYPE'] == 'NETCDF_DOUBLE') or (param['OUT_TYPE'] == 'NETCDF_FLOAT')): if verbose: print('\nCreating Dataset from NetCDF file') - ds = xr.open_dataset(param['BIN_OUT'], mask_and_scale=False) + #ds = xr.open_dataset(param['BIN_OUT'], mask_and_scale=False) + ds = xr.open_mfdataset(param['BIN_OUT'], parallel=True, engine='h5netcdf', mask_and_scale=False) + ds = process_netcdf_input(ds, param) else: print(f"Error encountered. OUT_TYPE {param['OUT_TYPE']} not recognized.") diff --git a/python/swiftest/swiftest/tool.py b/python/swiftest/swiftest/tool.py index 7fe2c8884..225619204 100644 --- a/python/swiftest/swiftest/tool.py +++ b/python/swiftest/swiftest/tool.py @@ -18,7 +18,7 @@ def magnitude(ds,x): dim = "space" ord = None return xr.apply_ufunc( - np.linalg.norm, ds[x], input_core_dims=[[dim]], kwargs={"ord": ord, "axis": -1} + np.linalg.norm, ds[x], input_core_dims=[[dim]], kwargs={"ord": ord, "axis": -1}, dask="parallelized" ) def wrap_angle(angle):