Skip to content

Commit 7be45f7

Browse files
author
Yang Liu
committed
update notebook to run model for developers
1 parent 10675aa commit 7be45f7

1 file changed

Lines changed: 71 additions & 135 deletions

File tree

‎notebooks/run_model_in_notebook_dev.ipynb‎

Lines changed: 71 additions & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,9 @@
1515
"outputs": [],
1616
"source": [
1717
"import os\n",
18-
"import shutil\n",
19-
"import numpy as np\n",
2018
"from pathlib import Path\n",
2119
"import subprocess\n",
22-
"import time"
20+
"import PyStemmusScope"
2321
]
2422
},
2523
{
@@ -38,7 +36,7 @@
3836
"outputs": [],
3937
"source": [
4038
"# path to model repository\n",
41-
"path_to_model = Path(\"path_to_STEMMUS_SCOPE_repository\")"
39+
"path_to_model = \"path_to_STEMMUS_SCOPE_repository\""
4240
]
4341
},
4442
{
@@ -57,27 +55,21 @@
5755
"name": "stdout",
5856
"output_type": "stream",
5957
"text": [
60-
"Already on 'name_of_your_branch'\n",
61-
"On branch name_of_your_branch\n",
58+
"Already on 'python-compliant'\n",
59+
"Your branch is up to date with 'origin/python-compliant'.\n",
60+
"On branch python-compliant\n",
61+
"Your branch is up to date with 'origin/python-compliant'.\n",
62+
"\n",
6263
"Untracked files:\n",
6364
" (use \"git add <file>...\" to include in what will be committed)\n",
65+
"\t\u001b[31m.ipynb_checkpoints/\u001b[m\n",
6466
"\t\u001b[31mexe/mccExcludedFiles.log\u001b[m\n",
6567
"\t\u001b[31mexe/readme.txt\u001b[m\n",
6668
"\t\u001b[31mexe/requiredMCRProducts.txt\u001b[m\n",
6769
"\t\u001b[31mexe/run_STEMMUS_SCOPE.sh\u001b[m\n",
6870
"\t\u001b[31mexe/unresolvedSymbols.txt\u001b[m\n",
69-
"\t\u001b[31mjava.log.42488\u001b[m\n",
70-
"\t\u001b[31mslurm/\u001b[m\n",
71-
"\t\u001b[31mslurm_901092.out\u001b[m\n",
72-
"\t\u001b[31mslurm_901245.out\u001b[m\n",
73-
"\t\u001b[31mslurm_901391.out\u001b[m\n",
74-
"\t\u001b[31mslurm_901417.out\u001b[m\n",
75-
"\t\u001b[31mslurm_901577.out\u001b[m\n",
76-
"\t\u001b[31mslurm_902191.out\u001b[m\n",
77-
"\t\u001b[31mslurm_908852.out\u001b[m\n",
78-
"\t\u001b[31mtest_run_stemmus_scope_snellius.sh\u001b[m\n",
71+
"\t\u001b[31mtest_config.txt\u001b[m\n",
7972
"\t\u001b[31mutils/csv_to_nc/__pycache__/\u001b[m\n",
80-
"\t\u001b[31mutils/get_info_stemmus_scope_snellius.sh\u001b[m\n",
8173
"\n",
8274
"nothing added to commit but untracked files present (use \"git add\" to track)\n"
8375
]
@@ -104,13 +96,9 @@
10496
"source": [
10597
"# the user must provide the correct path\n",
10698
"# path to config file\n",
107-
"config_file_path = path_to_model / \"config_file_snellius.txt\"\n",
108-
"# create an empty dict\n",
109-
"config = {}\n",
110-
"with open(config_file_path, \"r\") as f:\n",
111-
" for line in f:\n",
112-
" (key, val) = line.split(\"=\")\n",
113-
" config[key] = val.rstrip('\\n')"
99+
"path_to_config_file = \"path_to_config_file\"\n",
100+
"# Instantiate working directories handler from PyStemmusScope\n",
101+
"config = PyStemmusScope.read_config(path_to_config_file)"
114102
]
115103
},
116104
{
@@ -121,13 +109,20 @@
121109
{
122110
"data": {
123111
"text/plain": [
124-
"{'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
125-
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/',\n",
126-
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/',\n",
112+
"{'WorkDir': '/scratch-shared/ecoextreml/stemmus_scope/',\n",
113+
" 'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
127114
" 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n",
128115
" 'ForcingFileName': 'FI-Hyy_1996-2014_FLUXNET2015_Met.nc',\n",
129-
" 'VegetationPropertyPath': '/projects/0/einf2480/model_parameters/vegetation_property/',\n",
130-
" 'DurationSize': '17520'}"
116+
" 'directional': '/projects/0/einf2480/model_parameters/vegetation_property/directional/',\n",
117+
" 'fluspect_parameters': '/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/',\n",
118+
" 'leafangles': '/projects/0/einf2480/model_parameters/vegetation_property/leafangles/',\n",
119+
" 'radiationdata': '/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/',\n",
120+
" 'soil_spectrum': '/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/',\n",
121+
" 'input_data': '/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx',\n",
122+
" 'InitialConditionPath': '/projects/0/einf2480/soil_initialcondition/',\n",
123+
" 'NumberOfTimeSteps': '17520',\n",
124+
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/FI-Hyy_2022-05-11-1705/',\n",
125+
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/FI-Hyy_2022-05-11-1705/'}"
131126
]
132127
},
133128
"execution_count": 5,
@@ -147,13 +142,20 @@
147142
{
148143
"data": {
149144
"text/plain": [
150-
"{'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
151-
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/',\n",
152-
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/',\n",
145+
"{'WorkDir': '/scratch-shared/ecoextreml/stemmus_scope/',\n",
146+
" 'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
153147
" 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n",
154148
" 'ForcingFileName': 'FI-Hyy_1996-2014_FLUXNET2015_Met.nc',\n",
155-
" 'VegetationPropertyPath': '/projects/0/einf2480/model_parameters/vegetation_property/',\n",
156-
" 'DurationSize': '5'}"
149+
" 'directional': '/projects/0/einf2480/model_parameters/vegetation_property/directional/',\n",
150+
" 'fluspect_parameters': '/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/',\n",
151+
" 'leafangles': '/projects/0/einf2480/model_parameters/vegetation_property/leafangles/',\n",
152+
" 'radiationdata': '/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/',\n",
153+
" 'soil_spectrum': '/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/',\n",
154+
" 'input_data': '/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx',\n",
155+
" 'InitialConditionPath': '/projects/0/einf2480/soil_initialcondition/',\n",
156+
" 'NumberOfTimeSteps': '5',\n",
157+
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/FI-Hyy_2022-05-11-1705/',\n",
158+
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/FI-Hyy_2022-05-11-1705/'}"
157159
]
158160
},
159161
"execution_count": 6,
@@ -163,7 +165,7 @@
163165
],
164166
"source": [
165167
"# edit config\n",
166-
"config[\"DurationSize\"] = \"5\"\n",
168+
"config[\"NumberOfTimeSteps\"] = \"5\"\n",
167169
"config"
168170
]
169171
},
@@ -178,53 +180,20 @@
178180
"cell_type": "code",
179181
"execution_count": 7,
180182
"metadata": {},
181-
"outputs": [],
182-
"source": [
183-
"def input_dir(ncfile, config):\n",
184-
" \"\"\"Create input directory and prepare input files\n",
185-
" \"\"\"\n",
186-
" # get start time with the format Y-M-D-HM\n",
187-
" timestamp = time.strftime('%Y%m%d_%H%M')\n",
188-
" station_name = ncfile.split('_')[0]\n",
189-
" # create input directory\n",
190-
" work_dir = Path(f\"{config['InputPath']}{station_name}_{timestamp}\")\n",
191-
" Path(work_dir).mkdir(parents=True, exist_ok=True)\n",
192-
" print(f\"Prepare work directory {work_dir} for the station: {station_name}\")\n",
193-
" # copy model parameters to work directory\n",
194-
" shutil.copytree(config[\"VegetationPropertyPath\"], work_dir, dirs_exist_ok=True)\n",
195-
" # update config file for ForcingFileName and InputPath\n",
196-
" config_file_path = Path(work_dir, f\"{station_name}_{timestamp}_config.txt\")\n",
197-
" with open(config_file_path, 'w') as f:\n",
198-
" for i in config.keys():\n",
199-
" if i == \"ForcingFileName\":\n",
200-
" f.write(i + \"=\" + ncfile + \"\\n\")\n",
201-
" elif i == \"InputPath\":\n",
202-
" f.write(i + \"=\" + str(work_dir) + \"/\" + \"\\n\")\n",
203-
" else:\n",
204-
" f.write(i + \"=\" + config[i] + \"\\n\")\n",
205-
"\n",
206-
" return work_dir, config_file_path\n",
207-
" "
208-
]
209-
},
210-
{
211-
"cell_type": "code",
212-
"execution_count": 8,
213-
"metadata": {},
214183
"outputs": [
215184
{
216185
"name": "stdout",
217186
"output_type": "stream",
218187
"text": [
219-
"Prepare work directory /scratch-shared/ecoextreml/stemmus_scope/input/NL-Hor_20220517_1757 for the station: NL-Hor\n",
220-
"Prepare work directory /scratch-shared/ecoextreml/stemmus_scope/input/ZA-Kru_20220517_1757 for the station: ZA-Kru\n"
188+
"/scratch-shared/ecoextreml/stemmus_scope/input/FI-Hyy_20220621_1415 /scratch-shared/ecoextreml/stemmus_scope/output/FI-Hyy_20220621_1415 /scratch-shared/ecoextreml/stemmus_scope/input/FI-Hyy_20220621_1415/FI-Hyy_20220621_1415_config.txt\n"
221189
]
222190
}
223191
],
224192
"source": [
225193
"# specify the forcing filename\n",
226194
"forcing_filename = config[\"ForcingFileName\"]\n",
227-
"work_dir, config_path = input_dir(forcing_filename, config)"
195+
"input_dir, output_dir, config_path = PyStemmusScope.create_io_dir(forcing_filename, config)\n",
196+
"print(input_dir, output_dir, config_path)"
228197
]
229198
},
230199
{
@@ -236,16 +205,15 @@
236205
},
237206
{
238207
"cell_type": "code",
239-
"execution_count": 9,
208+
"execution_count": 8,
240209
"metadata": {},
241210
"outputs": [
242211
{
243212
"name": "stdout",
244213
"output_type": "stream",
245214
"text": [
246-
"Run STEMMUS-SCOPE with the forcing: NL-Hor_2008-2011_FLUXNET2015_Met.nc\n",
247215
"MATLAB is selecting SOFTWARE OPENGL rendering.\n",
248-
"Opening log file: /scratch-shared/ecoextreml/stemmus_scope/input/NL-Hor_20220517_1757/java.log.43865\n",
216+
"Opening log file: /scratch-shared/ecoextreml/stemmus_scope/input/FI-Hyy_20220621_1415/java.log.45704\n",
249217
"\n",
250218
" < M A T L A B (R) >\n",
251219
" Copyright 1984-2021 The MathWorks, Inc.\n",
@@ -256,23 +224,7 @@
256224
"To get started, type doc.\n",
257225
"For product information, visit www.mathworks.com.\n",
258226
" \n",
259-
"Reading config from /scratch-shared/ecoextreml/stemmus_scope/input/NL-Hor_20220517_1757/NL-Hor_20220517_1757_config.txt\n",
260-
"\n",
261-
" The calculations start now \n",
262-
"Run STEMMUS-SCOPE with the forcing: ZA-Kru_2000-2002_FLUXNET2015_Met.nc\n",
263-
"MATLAB is selecting SOFTWARE OPENGL rendering.\n",
264-
"Opening log file: /scratch-shared/ecoextreml/stemmus_scope/input/ZA-Kru_20220517_1757/java.log.44704\n",
265-
"\n",
266-
" < M A T L A B (R) >\n",
267-
" Copyright 1984-2021 The MathWorks, Inc.\n",
268-
" R2021a Update 3 (9.10.0.1684407) 64-bit (glnxa64)\n",
269-
" May 27, 2021\n",
270-
"\n",
271-
" \n",
272-
"To get started, type doc.\n",
273-
"For product information, visit www.mathworks.com.\n",
274-
" \n",
275-
"Reading config from /scratch-shared/ecoextreml/stemmus_scope/input/ZA-Kru_20220517_1757/ZA-Kru_20220517_1757_config.txt\n",
227+
"Reading config from /scratch-shared/ecoextreml/stemmus_scope/input/FI-Hyy_20220621_1415/FI-Hyy_20220621_1415_config.txt\n",
276228
"\n",
277229
" The calculations start now \n",
278230
" The calculations end now \r"
@@ -281,10 +233,10 @@
281233
],
282234
"source": [
283235
"# generate a text file\n",
284-
"path_to_code = path_to_model / \"src\"\n",
236+
"path_to_code = \"path_to_model_src\"\n",
285237
"\n",
286238
"# set matlab log dir to slurm, otherwise java.log files are created in user home dir\n",
287-
"os.environ['MATLAB_LOG_DIR'] = str(work_dir)\n",
239+
"os.environ['MATLAB_LOG_DIR'] = str(input_dir)\n",
288240
"\n",
289241
"# set matlab arguments\n",
290242
"path_to_config = f\"'{config_path}'\"\n",
@@ -312,37 +264,27 @@
312264
"name": "stdout",
313265
"output_type": "stream",
314266
"text": [
315-
"/scratch-shared/ecoextreml/stemmus_scope/output/NL-Hor_2022-05-17-1758/ECdata.csv\n",
316-
"Reading variable metadata from /home/alidoost/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
317-
"Creating /scratch-shared/ecoextreml/stemmus_scope/output/NL-Hor_2022-05-17-1758/NL-Hor_2022-05-17-1758_STEMMUS_SCOPE.nc \n",
318-
"Reading data from file: radiation.csv\n",
319-
"Reading data from file: fluxes.csv\n",
320-
"Reading data from file: surftemp.csv\n",
321-
"Reading data from file: Sim_Temp.csv\n",
322-
"Reading data from file: Sim_Theta.csv\n",
323-
"Reading data from file: aerodyn.csv\n",
324-
"Reading data from file: ECdata.csv\n",
325-
"Done writing /scratch-shared/ecoextreml/stemmus_scope/output/NL-Hor_2022-05-17-1758/NL-Hor_2022-05-17-1758_STEMMUS_SCOPE.nc\n",
326-
"/scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-05-17-1758/ECdata.csv\n",
327-
"Reading variable metadata from /home/alidoost/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
328-
"Creating /scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-05-17-1758/ZA-Kru_2022-05-17-1758_STEMMUS_SCOPE.nc \n",
267+
"/scratch-shared/ecoextreml/stemmus_scope/output/FI-Hyy_20220621_1415/ECdata.csv\n",
268+
"Reading variable metadata from /home/yangl/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
269+
"Creating /scratch-shared/ecoextreml/stemmus_scope/output/FI-Hyy_20220621_1415/FI-Hyy_20220621_1415_STEMMUS_SCOPE.nc \n",
329270
"Reading data from file: radiation.csv\n",
330271
"Reading data from file: fluxes.csv\n",
331272
"Reading data from file: surftemp.csv\n",
332273
"Reading data from file: Sim_Temp.csv\n",
333274
"Reading data from file: Sim_Theta.csv\n",
334275
"Reading data from file: aerodyn.csv\n",
335276
"Reading data from file: ECdata.csv\n",
336-
"Done writing /scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-05-17-1758/ZA-Kru_2022-05-17-1758_STEMMUS_SCOPE.nc\n"
277+
"Done writing /scratch-shared/ecoextreml/stemmus_scope/output/FI-Hyy_20220621_1415/FI-Hyy_20220621_1415_STEMMUS_SCOPE.nc\n"
337278
]
338279
}
339280
],
340281
"source": [
341282
"# convert csv files to nc files\n",
342-
"path_to_utils = Path(path_to_model, \"utils/csv_to_nc\")\n",
343-
"result = subprocess.run([\"python\", Path(path_to_utils, \"generate_netcdf_files.py\"),\n",
283+
"path_to_generate_netcdf_script = \"path_to_generate_netcdf_script\"\n",
284+
"path_to_Variables_will_be_in_NetCDF_file = \"path_to_Variables_will_be_in_NetCDF_file\"\n",
285+
"result = subprocess.run([\"python\", path_to_generate_netcdf_script,\n",
344286
" \"--config_file\", config_path, \"--variable_file\",\n",
345-
" Path(path_to_utils, \"Variables_will_be_in_NetCDF_file.csv\")])\n",
287+
" path_to_Variables_will_be_in_NetCDF_file])\n",
346288
"result.check_returncode()"
347289
]
348290
},
@@ -362,7 +304,7 @@
362304
"name": "stdout",
363305
"output_type": "stream",
364306
"text": [
365-
"Opening log file: ./java.log.45777\n"
307+
"Opening log file: ./java.log.52377\n"
366308
]
367309
}
368310
],
@@ -389,32 +331,25 @@
389331
"name": "stdout",
390332
"output_type": "stream",
391333
"text": [
392-
"On branch name_of_your_branch\n",
334+
"On branch python-compliant\n",
335+
"Your branch is up to date with 'origin/python-compliant'.\n",
336+
"\n",
393337
"Changes not staged for commit:\n",
394338
" (use \"git add <file>...\" to update what will be committed)\n",
395339
" (use \"git restore <file>...\" to discard changes in working directory)\n",
396340
"\t\u001b[31mmodified: exe/STEMMUS_SCOPE\u001b[m\n",
397341
"\n",
398342
"Untracked files:\n",
399343
" (use \"git add <file>...\" to include in what will be committed)\n",
344+
"\t\u001b[31m.ipynb_checkpoints/\u001b[m\n",
400345
"\t\u001b[31mexe/mccExcludedFiles.log\u001b[m\n",
401346
"\t\u001b[31mexe/readme.txt\u001b[m\n",
402347
"\t\u001b[31mexe/requiredMCRProducts.txt\u001b[m\n",
403348
"\t\u001b[31mexe/run_STEMMUS_SCOPE.sh\u001b[m\n",
404349
"\t\u001b[31mexe/unresolvedSymbols.txt\u001b[m\n",
405-
"\t\u001b[31mjava.log.42488\u001b[m\n",
406-
"\t\u001b[31mjava.log.45777\u001b[m\n",
407-
"\t\u001b[31mslurm/\u001b[m\n",
408-
"\t\u001b[31mslurm_901092.out\u001b[m\n",
409-
"\t\u001b[31mslurm_901245.out\u001b[m\n",
410-
"\t\u001b[31mslurm_901391.out\u001b[m\n",
411-
"\t\u001b[31mslurm_901417.out\u001b[m\n",
412-
"\t\u001b[31mslurm_901577.out\u001b[m\n",
413-
"\t\u001b[31mslurm_902191.out\u001b[m\n",
414-
"\t\u001b[31mslurm_908852.out\u001b[m\n",
415-
"\t\u001b[31mtest_run_stemmus_scope_snellius.sh\u001b[m\n",
350+
"\t\u001b[31mjava.log.52377\u001b[m\n",
351+
"\t\u001b[31mtest_config.txt\u001b[m\n",
416352
"\t\u001b[31mutils/csv_to_nc/__pycache__/\u001b[m\n",
417-
"\t\u001b[31mutils/get_info_stemmus_scope_snellius.sh\u001b[m\n",
418353
"\n",
419354
"no changes added to commit (use \"git add\" and/or \"git commit -a\")\n"
420355
]
@@ -423,27 +358,28 @@
423358
"source": [
424359
"!git -C {path_to_model} status"
425360
]
426-
},
427-
{
428-
"cell_type": "code",
429-
"execution_count": null,
430-
"metadata": {},
431-
"outputs": [],
432-
"source": []
433361
}
434362
],
435363
"metadata": {
436364
"interpreter": {
437365
"hash": "28b136f154b3384fcb2854e5626613232692c304dbc5315064ef4c9363104a2c"
438366
},
439367
"kernelspec": {
440-
"display_name": "Python 3.7.6 64-bit ('esmvaltool': conda)",
368+
"display_name": "Python 3 (ipykernel)",
441369
"language": "python",
442-
"name": "python37664bitesmvaltoolconda8248b33b29dd45388bf12a38f96e3b70"
370+
"name": "python3"
443371
},
444372
"language_info": {
373+
"codemirror_mode": {
374+
"name": "ipython",
375+
"version": 3
376+
},
377+
"file_extension": ".py",
378+
"mimetype": "text/x-python",
445379
"name": "python",
446-
"version": ""
380+
"nbconvert_exporter": "python",
381+
"pygments_lexer": "ipython3",
382+
"version": "3.9.12"
447383
},
448384
"metadata": {
449385
"interpreter": {

0 commit comments

Comments
 (0)