Merge pull request #48 from EcoExtreML/forcing_mat

Avoid locale issues in generating forcing .mat files

Author: BSchilperoort

PyStemmusScope/forcing_io.py

@@ -2,6 +2,7 @@
 import hdf5storage
 import numpy as np
 import xarray as xr
+from . import utils
 from . import variable_conversion as vc
 
 
@@ -159,7 +160,8 @@ def prepare_global_variables(data, input_path, config):
 
     matfiledata['Dur_tot'] = float(total_duration) # Matlab expects a 'double'
 
-    hdf5storage.savemat(input_path / 'forcing_globals.mat', matfiledata, appendmat=False)
+    hdf5storage.savemat(input_path / "forcing_globals.mat", matfiledata, appendmat=False)
+    utils.remove_dates_from_header(input_path / "forcing_globals.mat")
 
 
 def prepare_forcing(config):

PyStemmusScope/soil_io.py

@@ -227,3 +227,4 @@ def prepare_soil_data(config):
     hdf5storage.savemat(
         Path(config["InputPath"]) / "soil_parameters.mat", mdict=matfiledata, appendmat=False,
     )
+    utils.remove_dates_from_header(Path(config["InputPath"]) / "soil_parameters.mat")

PyStemmusScope/utils.py

@@ -67,3 +67,31 @@ def to_absolute_path(
         must_exist = os_name() == 'nt'
 
     return pathlike.expanduser().resolve(strict=must_exist)
+
+
+def remove_dates_from_header(filename):
+    """Removes the datetime string from the .mat file header.
+
+    MATLAB raises an error when some characters are non-UTF-8 (?), e.g. Chinese month
+    names. This function removes this part of the file header to avoid this problem.
+
+    Args:
+        filename (Path): Valid path to the .mat file
+    """
+    with open(filename, "rb") as f:
+        data = f.read()
+
+    # Get locations of date string in header
+    start_datestring = data[:128].find(b"Created on:") + 12
+    end_datestring = data[:128].find(b"HDF5") - 1
+
+    # Rebuild the data, with the dates in the header removed
+    sanitized_data = (
+        data[:start_datestring] +
+        b' '*len(data[start_datestring:end_datestring]) +
+        data[end_datestring:]
+        )
+
+    # Overwrite the old file
+    with open(filename, 'wb') as file:
+        file.write(sanitized_data)