Skip to content

Commit 10675aa

Browse files
author
Yang Liu
committed
update notebook to run model
1 parent dc96c36 commit 10675aa

1 file changed

Lines changed: 55 additions & 75 deletions

File tree

‎notebooks/run_model_in_notebook.ipynb‎

Lines changed: 55 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,6 @@
2929
"STEMMUS_SCOPE executable file is located in the **private** repository on GitHub https://github.com/EcoExtreML/STEMMUS_SCOPE. You need to clone the repository locally and specify the path to it in the cell below. Make sure you have right access to the repository. "
3030
]
3131
},
32-
{
33-
"cell_type": "code",
34-
"execution_count": 2,
35-
"metadata": {},
36-
"outputs": [],
37-
"source": [
38-
"# path to executable\n",
39-
"path_to_exefile = \"STEMMUS_SCOPE/exe\""
40-
]
41-
},
4232
{
4333
"cell_type": "markdown",
4434
"metadata": {},
@@ -48,36 +38,42 @@
4838
},
4939
{
5040
"cell_type": "code",
51-
"execution_count": 5,
41+
"execution_count": 2,
5242
"metadata": {},
5343
"outputs": [],
5444
"source": [
5545
"# the user must provide the correct path\n",
5646
"# path to config file\n",
57-
"path_to_config_file = \"config_file_template.txt\"\n",
47+
"path_to_config_file = \"path_to_config_file\"\n",
5848
"# Instantiate working directories handler from PyStemmusScope\n",
5949
"config = PyStemmusScope.read_config(path_to_config_file)"
6050
]
6151
},
6252
{
6353
"cell_type": "code",
64-
"execution_count": 6,
54+
"execution_count": 3,
6555
"metadata": {},
6656
"outputs": [
6757
{
6858
"data": {
6959
"text/plain": [
70-
"{'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
71-
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/',\n",
72-
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/',\n",
60+
"{'WorkDir': '/scratch-shared/ecoextreml/stemmus_scope/',\n",
61+
" 'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
7362
" 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n",
7463
" 'ForcingFileName': 'FI-Hyy_1996-2014_FLUXNET2015_Met.nc',\n",
75-
" 'VegetationPropertyPath': '/projects/0/einf2480/model_parameters/vegetation_property/',\n",
64+
" 'directional': '/projects/0/einf2480/model_parameters/vegetation_property/directional/',\n",
65+
" 'fluspect_parameters': '/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/',\n",
66+
" 'leafangles': '/projects/0/einf2480/model_parameters/vegetation_property/leafangles/',\n",
67+
" 'radiationdata': '/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/',\n",
68+
" 'soil_spectrum': '/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/',\n",
69+
" 'input_data': '/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx',\n",
7670
" 'InitialConditionPath': '/projects/0/einf2480/soil_initialcondition/',\n",
77-
" 'DurationSize': '17520'}"
71+
" 'NumberOfTimeSteps': '17520',\n",
72+
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/FI-Hyy_2022-05-11-1705/',\n",
73+
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/FI-Hyy_2022-05-11-1705/'}"
7874
]
7975
},
80-
"execution_count": 6,
76+
"execution_count": 3,
8177
"metadata": {},
8278
"output_type": "execute_result"
8379
}
@@ -88,22 +84,29 @@
8884
},
8985
{
9086
"cell_type": "code",
91-
"execution_count": 18,
87+
"execution_count": 4,
9288
"metadata": {},
9389
"outputs": [
9490
{
9591
"data": {
9692
"text/plain": [
97-
"{'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
98-
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/',\n",
99-
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/',\n",
93+
"{'WorkDir': '/scratch-shared/ecoextreml/stemmus_scope/',\n",
94+
" 'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
10095
" 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n",
10196
" 'ForcingFileName': 'FI-Hyy_1996-2014_FLUXNET2015_Met.nc',\n",
102-
" 'VegetationPropertyPath': '/projects/0/einf2480/model_parameters/vegetation_property/',\n",
103-
" 'DurationSize': '5'}"
97+
" 'directional': '/projects/0/einf2480/model_parameters/vegetation_property/directional/',\n",
98+
" 'fluspect_parameters': '/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/',\n",
99+
" 'leafangles': '/projects/0/einf2480/model_parameters/vegetation_property/leafangles/',\n",
100+
" 'radiationdata': '/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/',\n",
101+
" 'soil_spectrum': '/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/',\n",
102+
" 'input_data': '/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx',\n",
103+
" 'InitialConditionPath': '/projects/0/einf2480/soil_initialcondition/',\n",
104+
" 'NumberOfTimeSteps': '5',\n",
105+
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/FI-Hyy_2022-05-11-1705/',\n",
106+
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/FI-Hyy_2022-05-11-1705/'}"
104107
]
105108
},
106-
"execution_count": 18,
109+
"execution_count": 4,
107110
"metadata": {},
108111
"output_type": "execute_result"
109112
}
@@ -123,24 +126,22 @@
123126
},
124127
{
125128
"cell_type": "code",
126-
"execution_count": 10,
129+
"execution_count": 5,
127130
"metadata": {},
128131
"outputs": [
129132
{
130133
"name": "stdout",
131134
"output_type": "stream",
132135
"text": [
133-
"Prepare work directory /home/sarah/temp/ecoextreml/input/NL-Hor_20220519_1455 for the station: NL-Hor\n",
134-
"Prepare work directory /home/sarah/temp/ecoextreml/input/ZA-Kru_20220519_1455 for the station: ZA-Kru\n"
136+
"/scratch-shared/ecoextreml/stemmus_scope/input/ZA-Kru_20220621_1334 /scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_20220621_1334 /scratch-shared/ecoextreml/stemmus_scope/input/ZA-Kru_20220621_1334/ZA-Kru_20220621_1334_config.txt\n"
135137
]
136138
}
137139
],
138140
"source": [
139141
"# specify the forcing filenames\n",
140142
"# forcing_filenames_list is by default none, if full_run is true, then all stations with forcing\n",
141143
"# listed in the ForcingPath will be used.\n",
142-
"forcing_filenames_list = [\"NL-Hor_2008-2011_FLUXNET2015_Met.nc\",\n",
143-
" \"ZA-Kru_2000-2002_FLUXNET2015_Met.nc\"]\n",
144+
"forcing_filenames_list = [\"ZA-Kru_2000-2002_FLUXNET2015_Met.nc\"]\n",
144145
"\n",
145146
"full_run = False\n",
146147
"if full_run:\n",
@@ -167,45 +168,34 @@
167168
},
168169
{
169170
"cell_type": "code",
170-
"execution_count": 13,
171+
"execution_count": null,
171172
"metadata": {},
172-
"outputs": [
173-
{
174-
"name": "stdout",
175-
"output_type": "stream",
176-
"text": [
177-
"/usr/local/MATLAB/MATLAB_Runtime/v910/runtime/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/bin/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/sys/os/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/extern/bin/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/sys/opengl/lib/glnxa64\n"
178-
]
179-
}
180-
],
173+
"outputs": [],
181174
"source": [
182175
"# # Set LD_LIBRARY_PATH\n",
183-
"matlab_path = !whereis MATLAB\n",
184-
"matlab_path = matlab_path.s.split(\": \")[1]\n",
185-
"os.environ['LD_LIBRARY_PATH'] = (\n",
186-
" f\"{matlab_path}/MATLAB_Runtime/v910/runtime/glnxa64:\"\n",
187-
" f\"{matlab_path}/MATLAB_Runtime/v910/bin/glnxa64:\"\n",
188-
" f\"{matlab_path}/MATLAB_Runtime/v910/sys/os/glnxa64:\"\n",
189-
" f\"{matlab_path}/MATLAB_Runtime/v910/extern/bin/glnxa64:\"\n",
190-
" f\"{matlab_path}/MATLAB_Runtime/v910/sys/opengl/lib/glnxa64\")\n",
191-
"print(os.environ['LD_LIBRARY_PATH'])"
176+
"# matlab_path = !whereis MATLAB\n",
177+
"# matlab_path = matlab_path.s.split(\": \")[1]\n",
178+
"# os.environ['LD_LIBRARY_PATH'] = (\n",
179+
"# f\"{matlab_path}/MATLAB_Runtime/v910/runtime/glnxa64:\"\n",
180+
"# f\"{matlab_path}/MATLAB_Runtime/v910/bin/glnxa64:\"\n",
181+
"# f\"{matlab_path}/MATLAB_Runtime/v910/sys/os/glnxa64:\"\n",
182+
"# f\"{matlab_path}/MATLAB_Runtime/v910/extern/bin/glnxa64:\"\n",
183+
"# f\"{matlab_path}/MATLAB_Runtime/v910/sys/opengl/lib/glnxa64\")\n",
184+
"# print(os.environ['LD_LIBRARY_PATH'])"
192185
]
193186
},
194187
{
195188
"cell_type": "code",
196-
"execution_count": 14,
189+
"execution_count": 6,
197190
"metadata": {},
198191
"outputs": [
199192
{
200193
"name": "stdout",
201194
"output_type": "stream",
202195
"text": [
203-
"Run STEMMUS-SCOPE with the forcing: NL-Hor_2008-2011_FLUXNET2015_Met.nc\n",
204-
"Reading config from /home/sarah/temp/ecoextreml/input/NL-Hor_20220519_1455/NL-Hor_20220519_1455_config.txt\n",
205-
"\n",
206-
" The calculations start now \n",
207196
"Run STEMMUS-SCOPE with the forcing: ZA-Kru_2000-2002_FLUXNET2015_Met.nc\n",
208-
"Reading config from /home/sarah/temp/ecoextreml/input/ZA-Kru_20220519_1455/ZA-Kru_20220519_1455_config.txt\n",
197+
"Opening log file: /scratch-shared/ecoextreml/stemmus_scope/input/ZA-Kru_20220621_1334/java.log.20020\n",
198+
"Reading config from /scratch-shared/ecoextreml/stemmus_scope/input/ZA-Kru_20220621_1334/ZA-Kru_20220621_1334_config.txt\n",
209199
"\n",
210200
" The calculations start now \n",
211201
" The calculations end now \r"
@@ -214,13 +204,14 @@
214204
],
215205
"source": [
216206
"# generate a text file\n",
207+
"path_to_model = \"path_to_STEMMUS_SCOPE_repository\"\n",
217208
"for ncfile in forcing_filenames_list:\n",
218209
" path_to_config = config_path_dict[ncfile]\n",
219210
" # set matlab log dir to slurm, otherwise java.log files are created in user home dir\n",
220211
" os.environ['MATLAB_LOG_DIR'] = str(input_dir_dict[ncfile])\n",
221212
" # run the model\n",
222213
" print(f\"Run STEMMUS-SCOPE with the forcing: {ncfile}\")\n",
223-
" result = subprocess.run([f\"STEMMUS_SCOPE {path_to_config}\"], cwd = path_to_exefile, shell=True)\n",
214+
" result = subprocess.run([f\"exe/STEMMUS_SCOPE {path_to_config}\"], cwd = Path(path_to_model), shell=True)\n",
224215
" result.check_returncode()"
225216
]
226217
},
@@ -233,42 +224,31 @@
233224
},
234225
{
235226
"cell_type": "code",
236-
"execution_count": 15,
227+
"execution_count": 7,
237228
"metadata": {},
238229
"outputs": [
239230
{
240231
"name": "stdout",
241232
"output_type": "stream",
242233
"text": [
243-
"/home/sarah/temp/ecoextreml/output/NL-Hor_2022-05-19-1457/ECdata.csv\n",
244-
"Reading variable metadata from /home/sarah/GitHub/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
245-
"Creating /home/sarah/temp/ecoextreml/output/NL-Hor_2022-05-19-1457/NL-Hor_2022-05-19-1457_STEMMUS_SCOPE.nc \n",
246-
"Reading data from file: radiation.csv\n",
247-
"Reading data from file: fluxes.csv\n",
248-
"Reading data from file: surftemp.csv\n",
249-
"Reading data from file: Sim_Temp.csv\n",
250-
"Reading data from file: Sim_Theta.csv\n",
251-
"Reading data from file: aerodyn.csv\n",
252-
"Reading data from file: ECdata.csv\n",
253-
"Done writing /home/sarah/temp/ecoextreml/output/NL-Hor_2022-05-19-1457/NL-Hor_2022-05-19-1457_STEMMUS_SCOPE.nc\n",
254-
"/home/sarah/temp/ecoextreml/output/ZA-Kru_2022-05-19-1458/ECdata.csv\n",
255-
"Reading variable metadata from /home/sarah/GitHub/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
256-
"Creating /home/sarah/temp/ecoextreml/output/ZA-Kru_2022-05-19-1458/ZA-Kru_2022-05-19-1458_STEMMUS_SCOPE.nc \n",
234+
"/scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_20220621_1334/ECdata.csv\n",
235+
"Reading variable metadata from /home/yangl/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
236+
"Creating /scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_20220621_1334/ZA-Kru_20220621_1334_STEMMUS_SCOPE.nc \n",
257237
"Reading data from file: radiation.csv\n",
258238
"Reading data from file: fluxes.csv\n",
259239
"Reading data from file: surftemp.csv\n",
260240
"Reading data from file: Sim_Temp.csv\n",
261241
"Reading data from file: Sim_Theta.csv\n",
262242
"Reading data from file: aerodyn.csv\n",
263243
"Reading data from file: ECdata.csv\n",
264-
"Done writing /home/sarah/temp/ecoextreml/output/ZA-Kru_2022-05-19-1458/ZA-Kru_2022-05-19-1458_STEMMUS_SCOPE.nc\n"
244+
"Done writing /scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_20220621_1334/ZA-Kru_20220621_1334_STEMMUS_SCOPE.nc\n"
265245
]
266246
}
267247
],
268248
"source": [
269249
"# convert csv files to nc files\n",
270-
"path_to_generate_netcdf_script = \"STEMMUS_SCOPE/utils/csv_to_nc/generate_netcdf_files.py\"\n",
271-
"path_to_Variables_will_be_in_NetCDF_file = \"STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\"\n",
250+
"path_to_generate_netcdf_script = \"path_to_generate_netcdf_script\"\n",
251+
"path_to_Variables_will_be_in_NetCDF_file = \"path_to_Variables_will_be_in_NetCDF_file\"\n",
272252
"for ncfile in forcing_filenames_list:\n",
273253
" path_to_config = config_path_dict[ncfile]\n",
274254
" result = subprocess.run([\"python\", path_to_generate_netcdf_script,\n",

0 commit comments

Comments
 (0)