Skip to content

Commit 2767265

Browse files
SarahAlidoostSarahAlidoost
authored
Merge pull request #5 from EcoExtreML/run_nb_locally
Run the notebook locally
2 parents d5cd35a + c11dfc9 commit 2767265

1 file changed

Lines changed: 81 additions & 31 deletions

File tree

‎notebooks/run_model_in_notebook.ipynb‎

Lines changed: 81 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"metadata": {},
66
"source": [
77
"## Run the STEMMUS_SCOPE model\n",
8-
"Steps to run the STEMMUS_SCOPE model, including preprocessing and postprocessing, on Surf super computer Snellius."
8+
"Steps to run the STEMMUS_SCOPE model, including preprocessing and postprocessing, on Surf super computer Snellius or on a unix system."
99
]
1010
},
1111
{
@@ -50,7 +50,7 @@
5050
},
5151
{
5252
"cell_type": "code",
53-
"execution_count": 5,
53+
"execution_count": 16,
5454
"metadata": {},
5555
"outputs": [],
5656
"source": [
@@ -67,7 +67,7 @@
6767
},
6868
{
6969
"cell_type": "code",
70-
"execution_count": 6,
70+
"execution_count": 17,
7171
"metadata": {},
7272
"outputs": [
7373
{
@@ -82,7 +82,7 @@
8282
" 'DurationSize': '17520'}"
8383
]
8484
},
85-
"execution_count": 6,
85+
"execution_count": 17,
8686
"metadata": {},
8787
"output_type": "execute_result"
8888
}
@@ -93,29 +93,29 @@
9393
},
9494
{
9595
"cell_type": "code",
96-
"execution_count": 10,
96+
"execution_count": 18,
9797
"metadata": {},
9898
"outputs": [
9999
{
100100
"data": {
101101
"text/plain": [
102-
"{'SoilPropertyPath': '/home/sarah/temp/ecoextreml/data/model_parameters/soil_property/',\n",
103-
" 'InputPath': '/home/sarah/temp/ecoextreml/input/',\n",
104-
" 'OutputPath': '/home/sarah/temp/ecoextreml/output/',\n",
105-
" 'ForcingPath': '/home/sarah/temp/ecoextreml/data/forcing/plumber2_data/',\n",
102+
"{'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n",
103+
" 'InputPath': '/scratch-shared/ecoextreml/stemmus_scope/input/',\n",
104+
" 'OutputPath': '/scratch-shared/ecoextreml/stemmus_scope/output/',\n",
105+
" 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n",
106106
" 'ForcingFileName': 'FI-Hyy_1996-2014_FLUXNET2015_Met.nc',\n",
107-
" 'VegetationPropertyPath': '/home/sarah/temp/ecoextreml/data/model_parameters/vegetation_property/',\n",
108-
" 'DurationSize': '20'}"
107+
" 'VegetationPropertyPath': '/projects/0/einf2480/model_parameters/vegetation_property/',\n",
108+
" 'DurationSize': '5'}"
109109
]
110110
},
111-
"execution_count": 10,
111+
"execution_count": 18,
112112
"metadata": {},
113113
"output_type": "execute_result"
114114
}
115115
],
116116
"source": [
117117
"# edit config\n",
118-
"config[\"DurationSize\"] = \"20\"\n",
118+
"config[\"DurationSize\"] = \"5\"\n",
119119
"config"
120120
]
121121
},
@@ -128,7 +128,7 @@
128128
},
129129
{
130130
"cell_type": "code",
131-
"execution_count": 11,
131+
"execution_count": 9,
132132
"metadata": {},
133133
"outputs": [],
134134
"source": [
@@ -161,15 +161,15 @@
161161
},
162162
{
163163
"cell_type": "code",
164-
"execution_count": 19,
164+
"execution_count": 10,
165165
"metadata": {},
166166
"outputs": [
167167
{
168168
"name": "stdout",
169169
"output_type": "stream",
170170
"text": [
171-
"Prepare work directory /home/sarah/temp/ecoextreml/input/NL-Hor_20220513_1256 for the station: NL-Hor\n",
172-
"Prepare work directory /home/sarah/temp/ecoextreml/input/ZA-Kru_20220513_1256 for the station: ZA-Kru\n"
171+
"Prepare work directory /home/sarah/temp/ecoextreml/input/NL-Hor_20220519_1455 for the station: NL-Hor\n",
172+
"Prepare work directory /home/sarah/temp/ecoextreml/input/ZA-Kru_20220519_1455 for the station: ZA-Kru\n"
173173
]
174174
}
175175
],
@@ -182,7 +182,7 @@
182182
"\n",
183183
"full_run = False\n",
184184
"if full_run:\n",
185-
" forcing_filenames_list = Path(config[\"ForcingPath\"]).iterdir()\n",
185+
" forcing_filenames_list = [file.name for file in Path(config[\"ForcingPath\"]).iterdir()]\n",
186186
"\n",
187187
"config_path_dict = {}\n",
188188
"work_dir_dict = {}\n",
@@ -194,17 +194,60 @@
194194
"cell_type": "markdown",
195195
"metadata": {},
196196
"source": [
197-
"#### Run the model"
197+
"#### Run the model\n",
198+
"\n",
199+
"If you run the model locally, you need to first intsall MATLAB Runtime, see instructions in the [readme](https://github.com/EcoExtreML/processing#readme). Then set `LD_LIBRARY_PATH`. To do this, uncomment the cell below and run it:"
198200
]
199201
},
200202
{
201203
"cell_type": "code",
202-
"execution_count": null,
204+
"execution_count": 13,
203205
"metadata": {},
204-
"outputs": [],
206+
"outputs": [
207+
{
208+
"name": "stdout",
209+
"output_type": "stream",
210+
"text": [
211+
"/usr/local/MATLAB/MATLAB_Runtime/v910/runtime/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/bin/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/sys/os/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/extern/bin/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/sys/opengl/lib/glnxa64\n"
212+
]
213+
}
214+
],
215+
"source": [
216+
"# # Set LD_LIBRARY_PATH\n",
217+
"matlab_path = !whereis MATLAB\n",
218+
"matlab_path = matlab_path.s.split(\": \")[1]\n",
219+
"os.environ['LD_LIBRARY_PATH'] = (\n",
220+
" f\"{matlab_path}/MATLAB_Runtime/v910/runtime/glnxa64:\"\n",
221+
" f\"{matlab_path}/MATLAB_Runtime/v910/bin/glnxa64:\"\n",
222+
" f\"{matlab_path}/MATLAB_Runtime/v910/sys/os/glnxa64:\"\n",
223+
" f\"{matlab_path}/MATLAB_Runtime/v910/extern/bin/glnxa64:\"\n",
224+
" f\"{matlab_path}/MATLAB_Runtime/v910/sys/opengl/lib/glnxa64\")\n",
225+
"print(os.environ['LD_LIBRARY_PATH'])"
226+
]
227+
},
228+
{
229+
"cell_type": "code",
230+
"execution_count": 14,
231+
"metadata": {},
232+
"outputs": [
233+
{
234+
"name": "stdout",
235+
"output_type": "stream",
236+
"text": [
237+
"Run STEMMUS-SCOPE with the forcing: NL-Hor_2008-2011_FLUXNET2015_Met.nc\n",
238+
"Reading config from /home/sarah/temp/ecoextreml/input/NL-Hor_20220519_1455/NL-Hor_20220519_1455_config.txt\n",
239+
"\n",
240+
" The calculations start now \n",
241+
"Run STEMMUS-SCOPE with the forcing: ZA-Kru_2000-2002_FLUXNET2015_Met.nc\n",
242+
"Reading config from /home/sarah/temp/ecoextreml/input/ZA-Kru_20220519_1455/ZA-Kru_20220519_1455_config.txt\n",
243+
"\n",
244+
" The calculations start now \n",
245+
" The calculations end now \r"
246+
]
247+
}
248+
],
205249
"source": [
206250
"# generate a text file\n",
207-
"str.split(\"/\")[-1]\n",
208251
"for ncfile in forcing_filenames_list:\n",
209252
" path_to_config = config_path_dict[ncfile]\n",
210253
" # set matlab log dir to slurm, otherwise java.log files are created in user home dir\n",
@@ -224,35 +267,35 @@
224267
},
225268
{
226269
"cell_type": "code",
227-
"execution_count": 26,
270+
"execution_count": 15,
228271
"metadata": {},
229272
"outputs": [
230273
{
231274
"name": "stdout",
232275
"output_type": "stream",
233276
"text": [
234-
"/scratch-shared/ecoextreml/stemmus_scope/output/NL-Hor_2022-05-11-1610/ECdata.csv\n",
235-
"Reading variable metadata from /home/yangl/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
236-
"Creating /scratch-shared/ecoextreml/stemmus_scope/output/NL-Hor_2022-05-11-1610/NL-Hor_2022-05-11-1610_STEMMUS_SCOPE.nc \n",
277+
"/home/sarah/temp/ecoextreml/output/NL-Hor_2022-05-19-1457/ECdata.csv\n",
278+
"Reading variable metadata from /home/sarah/GitHub/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
279+
"Creating /home/sarah/temp/ecoextreml/output/NL-Hor_2022-05-19-1457/NL-Hor_2022-05-19-1457_STEMMUS_SCOPE.nc \n",
237280
"Reading data from file: radiation.csv\n",
238281
"Reading data from file: fluxes.csv\n",
239282
"Reading data from file: surftemp.csv\n",
240283
"Reading data from file: Sim_Temp.csv\n",
241284
"Reading data from file: Sim_Theta.csv\n",
242285
"Reading data from file: aerodyn.csv\n",
243286
"Reading data from file: ECdata.csv\n",
244-
"Done writing /scratch-shared/ecoextreml/stemmus_scope/output/NL-Hor_2022-05-11-1610/NL-Hor_2022-05-11-1610_STEMMUS_SCOPE.nc\n",
245-
"/scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-05-11-1611/ECdata.csv\n",
246-
"Reading variable metadata from /home/yangl/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
247-
"Creating /scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-05-11-1611/ZA-Kru_2022-05-11-1611_STEMMUS_SCOPE.nc \n",
287+
"Done writing /home/sarah/temp/ecoextreml/output/NL-Hor_2022-05-19-1457/NL-Hor_2022-05-19-1457_STEMMUS_SCOPE.nc\n",
288+
"/home/sarah/temp/ecoextreml/output/ZA-Kru_2022-05-19-1458/ECdata.csv\n",
289+
"Reading variable metadata from /home/sarah/GitHub/STEMMUS_SCOPE/utils/csv_to_nc/Variables_will_be_in_NetCDF_file.csv\n",
290+
"Creating /home/sarah/temp/ecoextreml/output/ZA-Kru_2022-05-19-1458/ZA-Kru_2022-05-19-1458_STEMMUS_SCOPE.nc \n",
248291
"Reading data from file: radiation.csv\n",
249292
"Reading data from file: fluxes.csv\n",
250293
"Reading data from file: surftemp.csv\n",
251294
"Reading data from file: Sim_Temp.csv\n",
252295
"Reading data from file: Sim_Theta.csv\n",
253296
"Reading data from file: aerodyn.csv\n",
254297
"Reading data from file: ECdata.csv\n",
255-
"Done writing /scratch-shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-05-11-1611/ZA-Kru_2022-05-11-1611_STEMMUS_SCOPE.nc\n"
298+
"Done writing /home/sarah/temp/ecoextreml/output/ZA-Kru_2022-05-19-1458/ZA-Kru_2022-05-19-1458_STEMMUS_SCOPE.nc\n"
256299
]
257300
}
258301
],
@@ -266,6 +309,13 @@
266309
" Path(path_to_utils, \"Variables_will_be_in_NetCDF_file.csv\")])\n",
267310
" result.check_returncode()"
268311
]
312+
},
313+
{
314+
"cell_type": "code",
315+
"execution_count": null,
316+
"metadata": {},
317+
"outputs": [],
318+
"source": []
269319
}
270320
],
271321
"metadata": {

0 commit comments

Comments
 (0)