add chunks auto to open_mfdataset to avoid memory problems

Author: SarahAlidoost

PyStemmusScope/global_data/cams_co2.py

@@ -60,7 +60,7 @@ def extract_cams_data(
     Returns:
         DataArray containing the CO2 concentration.
     """
-    ds = xr.open_mfdataset(files_cams)
+    ds = xr.open_mfdataset(files_cams, chunks='auto')
 
     check_cams_dataset(cams_data=ds, latlon=latlon, time_range=time_range)
 

PyStemmusScope/global_data/cci_landcover.py

@@ -62,7 +62,7 @@ def extract_landcover_data(
     Returns:
         Dictionary containing IGBP and LCCS land cover classes.
     """
-    cci_dataset = xr.open_mfdataset(files_cci)
+    cci_dataset = xr.open_mfdataset(files_cci, chunks='auto')
 
     check_cci_dataset(cci_dataset, latlon, time_range)  # Assert spatial/temporal bounds
 

PyStemmusScope/global_data/copernicus_lai.py

@@ -60,7 +60,7 @@ def extract_lai_data(
     Returns:
         DataArray containing the LAI of the specified site for the given time range.
     """
-    ds = xr.open_mfdataset(files_lai)
+    ds = xr.open_mfdataset(files_lai, chunks='auto')
 
     check_lai_dataset(ds, latlon, time_range)
 

PyStemmusScope/global_data/era5.py

@@ -124,7 +124,8 @@ def get_era5_dataset(
     """
     tol = RESOLUTION_ERA5 if name == "ERA5" else RESOLUTION_ERA5LAND
 
-    ds = xr.open_mfdataset(files)
+    ds = xr.open_mfdataset(files, chunks='auto')
+
     check_era5_dataset(ds, name, latlon, time_range)
 
     try: