Skip to content

Commit 485165b

Browse files
williamhobbswilliamhobbs
committed
added goes4
1 parent 2ca40aa commit 485165b

1 file changed

Lines changed: 389 additions & 0 deletions

File tree

‎pvlib/iotools/goes4.py‎

Lines changed: 389 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,389 @@
1+
"""
2+
Get NSRDB GOES V4.0.0
3+
see https://developer.nrel.gov/docs/solar/nsrdb/nsrdb-GOES-aggregated-v4-0-0-download/
4+
"""
5+
6+
import io
7+
import requests
8+
import pandas as pd
9+
from json import JSONDecodeError
10+
import warnings
11+
from pvlib._deprecation import pvlibDeprecationWarning
12+
13+
NSRDB_API_BASE = "https://developer.nrel.gov"
14+
GOES_URL = NSRDB_API_BASE + "/api/nsrdb/v2/solar/nsrdb-GOES-aggregated-v4-0-0-download.csv"
15+
TMY_URL = NSRDB_API_BASE + "/api/nsrdb/v2/solar/nsrdb-GOES-tmy-v4-0-0-download.csv"
16+
GOESCONUS_URL = NSRDB_API_BASE + "/api/nsrdb/v2/solar/nsrdb-GOES-conus-v4-0-0-download.csv"
17+
18+
ATTRIBUTES = (
19+
'air_temperature', 'dew_point', 'dhi', 'dni', 'ghi', 'surface_albedo',
20+
'surface_pressure', 'wind_direction', 'wind_speed')
21+
PVLIB_PYTHON = 'pvlib python'
22+
23+
# Dictionary mapping GOES4 response names to pvlib names
24+
VARIABLE_MAP = {
25+
'GHI': 'ghi',
26+
'DHI': 'dhi',
27+
'DNI': 'dni',
28+
'Clearsky GHI': 'ghi_clear',
29+
'Clearsky DHI': 'dhi_clear',
30+
'Clearsky DNI': 'dni_clear',
31+
'Solar Zenith Angle': 'solar_zenith',
32+
'Temperature': 'temp_air',
33+
'Dew Point': 'temp_dew',
34+
'Relative Humidity': 'relative_humidity',
35+
'Pressure': 'pressure',
36+
'Wind Speed': 'wind_speed',
37+
'Wind Direction': 'wind_direction',
38+
'Surface Albedo': 'albedo',
39+
'Precipitable Water': 'precipitable_water',
40+
'AOD': 'aod',
41+
'Alpha': 'alpha',
42+
'Asymmetry': 'asymmetry',
43+
}
44+
45+
# Dictionary mapping pvlib names to GOES4 request names
46+
# Note, GOES4 uses different names for the same variables in the
47+
# response and the request
48+
REQUEST_VARIABLE_MAP = {
49+
'ghi': 'ghi',
50+
'dhi': 'dhi',
51+
'dni': 'dni',
52+
'ghi_clear': 'clearsky_ghi',
53+
'dhi_clear': 'clearsky_dhi',
54+
'dni_clear': 'clearsky_dni',
55+
'zenith': 'solar_zenith_angle',
56+
'temp_air': 'air_temperature',
57+
'temp_dew': 'dew_point',
58+
'relative_humidity': 'relative_humidity',
59+
'pressure': 'surface_pressure',
60+
'wind_speed': 'wind_speed',
61+
'wind_direction': 'wind_direction',
62+
'albedo': 'surface_albedo',
63+
'precipitable_water': 'total_precipitable_water',
64+
'aod': 'aod',
65+
'alpha': 'alpha',
66+
'asymmetry': 'asymmetry',
67+
}
68+
69+
70+
def get_goes4(latitude, longitude, api_key, email, names='tmy', interval=60,
71+
attributes=ATTRIBUTES, leap_day=True, full_name=PVLIB_PYTHON,
72+
affiliation=PVLIB_PYTHON, map_variables=True, url=None,
73+
timeout=30):
74+
"""
75+
Retrieve NSRDB GOES4 timeseries weather data from the GOES4 API. The NSRDB
76+
is described in [1]_ and the GOES4 API is described in [2]_, [3]_, and [4]_.
77+
78+
Parameters
79+
----------
80+
latitude : float or int
81+
in decimal degrees, between -90 and 90, north is positive
82+
longitude : float or int
83+
in decimal degrees, between -180 and 180, east is positive
84+
api_key : str
85+
NREL Developer Network API key
86+
email : str
87+
NREL API uses this to automatically communicate messages back
88+
to the user only if necessary
89+
names : str, default 'tmy'
90+
GOES4 API parameter specifing year (e.g. ``2020``) or TMY variant
91+
to download (e.g. ``'tmy'`` or ``'tgy-2022'``). The allowed values
92+
update periodically, so consult the NSRDB references below for the
93+
current set of options.
94+
interval : int, {60, 5, 15, 30}
95+
interval size in minutes, must be 5, 15, 30 or 60. Must be 60 for
96+
typical year requests (i.e., tmy/tgy/tdy).
97+
attributes : list of str, optional
98+
meteorological fields to fetch. If not specified, defaults to
99+
``pvlib.iotools.goes4.ATTRIBUTES``. See references [2]_, [3]_, and [4]_
100+
for lists of available fields. Alternatively, pvlib names may also be
101+
used (e.g. 'ghi' rather than 'GHI'); see :const:`REQUEST_VARIABLE_MAP`.
102+
To retrieve all available fields, set ``attributes=[]``.
103+
leap_day : bool, default : True
104+
include leap day in the results. Only used for single-year requests
105+
(i.e., it is ignored for tmy/tgy/tdy requests).
106+
full_name : str, default 'pvlib python'
107+
optional
108+
affiliation : str, default 'pvlib python'
109+
optional
110+
map_variables : bool, default True
111+
When true, renames columns of the Dataframe to pvlib variable names
112+
where applicable. See variable :const:`VARIABLE_MAP`.
113+
url : str, optional
114+
API endpoint URL. If not specified, the endpoint is determined from
115+
the ``names`` and ``interval`` parameters.
116+
timeout : int, default 30
117+
time in seconds to wait for server response before timeout
118+
119+
Returns
120+
-------
121+
data : pandas.DataFrame
122+
timeseries data from NREL GOES4
123+
metadata : dict
124+
metadata from NREL GOES4 about the record, see
125+
:func:`pvlib.iotools.parse_goes4` for fields
126+
127+
Raises
128+
------
129+
requests.HTTPError
130+
if the request response status is not ok, then the ``'errors'`` field
131+
from the JSON response or any error message in the content will be
132+
raised as an exception, for example if the `api_key` was rejected or if
133+
the coordinates were not found in the NSRDB
134+
135+
Notes
136+
-----
137+
The required NREL developer key, `api_key`, is available for free by
138+
registering at the `NREL Developer Network <https://developer.nrel.gov/>`_.
139+
140+
.. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may
141+
result in rejected requests.
142+
143+
.. warning:: GOES4 is limited to data found in the NSRDB, please consult the
144+
references below for locations with available data. Additionally,
145+
querying data with < 30-minute resolution uses a different API endpoint
146+
with fewer available fields (see [4]_).
147+
148+
See Also
149+
--------
150+
pvlib.iotools.read_goes4, pvlib.iotools.parse_goes4
151+
152+
References
153+
----------
154+
155+
.. [1] `NREL National Solar Radiation Database (NSRDB)
156+
<https://nsrdb.nrel.gov/>`_
157+
.. [2] `NSRDB GOES Aggregated V4.0.0
158+
<https://developer.nrel.gov/docs/solar/nsrdb/nsrdb-GOES-aggregated-v4-0-0-download/>`_
159+
.. [3] `NSRDB GOES Tmy V4.0.0
160+
<https://developer.nrel.gov/docs/solar/nsrdb/nsrdb-GOES-tmy-v4-0-0-download/>`_
161+
.. [4] `NSRDB GOES Conus V4.0.0
162+
<https://developer.nrel.gov/docs/solar/nsrdb/nsrdb-GOES-conus-v4-0-0-download/>`_
163+
"""
164+
# The well know text (WKT) representation of geometry notation is strict.
165+
# A POINT object is a string with longitude first, then the latitude, with
166+
# four decimals each, and exactly one space between them.
167+
longitude = ('%9.4f' % longitude).strip()
168+
latitude = ('%8.4f' % latitude).strip()
169+
# TODO: make format_WKT(object_type, *args) in tools.py
170+
171+
# convert to string to accomodate integer years being passed in
172+
names = str(names)
173+
174+
# convert pvlib names in attributes to psm3 convention
175+
attributes = [REQUEST_VARIABLE_MAP.get(a, a) for a in attributes]
176+
177+
# required query-string parameters for request to PSM3 API
178+
params = {
179+
'api_key': api_key,
180+
'full_name': full_name,
181+
'email': email,
182+
'affiliation': affiliation,
183+
'reason': PVLIB_PYTHON,
184+
'mailing_list': 'false',
185+
'wkt': 'POINT(%s %s)' % (longitude, latitude),
186+
'names': names,
187+
'attributes': ','.join(attributes),
188+
'leap_day': str(leap_day).lower(),
189+
'utc': 'false',
190+
'interval': interval
191+
}
192+
# request CSV download from NREL PSM3
193+
if url is None:
194+
# determine the endpoint that suits the user inputs
195+
if any(prefix in names for prefix in ('tmy', 'tgy', 'tdy')):
196+
url = TMY_URL
197+
elif interval in (5, 15):
198+
url = GOESCONUS_URL
199+
else:
200+
url = GOES_URL
201+
202+
response = requests.get(url, params=params, timeout=timeout)
203+
if not response.ok:
204+
# if the API key is rejected, then the response status will be 403
205+
# Forbidden, and then the error is in the content and there is no JSON
206+
try:
207+
errors = response.json()['errors']
208+
except JSONDecodeError:
209+
errors = response.content.decode('utf-8')
210+
raise requests.HTTPError(errors, response=response)
211+
# the CSV is in the response content as a UTF-8 bytestring
212+
# to use pandas we need to create a file buffer from the response
213+
fbuf = io.StringIO(response.content.decode('utf-8'))
214+
return parse_goes4(fbuf, map_variables)
215+
216+
217+
def parse_goes4(fbuf, map_variables=True):
218+
"""
219+
Parse an NSRDB GOES4 weather file (formatted as SAM CSV). The NSRDB
220+
is described in [1]_ and the SAM CSV format is described in [2]_.
221+
222+
Parameters
223+
----------
224+
fbuf: file-like object
225+
File-like object containing data to read.
226+
map_variables: bool, default True
227+
When true, renames columns of the Dataframe to pvlib variable names
228+
where applicable. See variable :const:`VARIABLE_MAP`.
229+
230+
Returns
231+
-------
232+
data : pandas.DataFrame
233+
timeseries data from NREL GOES4
234+
metadata : dict
235+
metadata from NREL GOES4 about the record, see notes for fields
236+
237+
Notes
238+
-----
239+
The return is a tuple with two items. The first item is a dataframe with
240+
the GOES4 timeseries data.
241+
242+
The second item is a dictionary with metadata from NREL GOES4 about the
243+
record containing the following fields:
244+
245+
* Source
246+
* Location ID
247+
* City
248+
* State
249+
* Country
250+
* Latitude
251+
* Longitude
252+
* Time Zone
253+
* Elevation
254+
* Local Time Zone
255+
* Clearsky DHI Units
256+
* Clearsky DNI Units
257+
* Clearsky GHI Units
258+
* Dew Point Units
259+
* DHI Units
260+
* DNI Units
261+
* GHI Units
262+
* Solar Zenith Angle Units
263+
* Temperature Units
264+
* Pressure Units
265+
* Relative Humidity Units
266+
* Precipitable Water Units
267+
* Wind Direction Units
268+
* Wind Speed Units
269+
* Cloud Type -15
270+
* Cloud Type 0
271+
* Cloud Type 1
272+
* Cloud Type 2
273+
* Cloud Type 3
274+
* Cloud Type 4
275+
* Cloud Type 5
276+
* Cloud Type 6
277+
* Cloud Type 7
278+
* Cloud Type 8
279+
* Cloud Type 9
280+
* Cloud Type 10
281+
* Cloud Type 11
282+
* Cloud Type 12
283+
* Fill Flag 0
284+
* Fill Flag 1
285+
* Fill Flag 2
286+
* Fill Flag 3
287+
* Fill Flag 4
288+
* Fill Flag 5
289+
* Surface Albedo Units
290+
* Version
291+
292+
Examples
293+
--------
294+
>>> # Read a local GOES4 file:
295+
>>> with open(filename, 'r') as f: # doctest: +SKIP
296+
... df, metadata = iotools.parse_goes4(f) # doctest: +SKIP
297+
298+
See Also
299+
--------
300+
pvlib.iotools.read_goes4, pvlib.iotools.get_goes4
301+
302+
References
303+
----------
304+
.. [1] `NREL National Solar Radiation Database (NSRDB)
305+
<https://nsrdb.nrel.gov/>`_
306+
.. [2] `Standard Time Series Data File Format
307+
<https://web.archive.org/web/20170207203107/https://sam.nrel.gov/sites/default/files/content/documents/pdf/wfcsv.pdf>`_
308+
"""
309+
# The first 2 lines of the response are headers with metadata
310+
metadata_fields = fbuf.readline().split(',')
311+
metadata_fields[-1] = metadata_fields[-1].strip() # strip trailing newline
312+
metadata_values = fbuf.readline().split(',')
313+
metadata_values[-1] = metadata_values[-1].strip() # strip trailing newline
314+
metadata = dict(zip(metadata_fields, metadata_values))
315+
# the response is all strings, so set some metadata types to numbers
316+
metadata['Local Time Zone'] = int(metadata['Local Time Zone'])
317+
metadata['Time Zone'] = int(metadata['Time Zone'])
318+
metadata['Latitude'] = float(metadata['Latitude'])
319+
metadata['Longitude'] = float(metadata['Longitude'])
320+
metadata['Elevation'] = int(metadata['Elevation'])
321+
# get the column names so we can set the dtypes
322+
columns = fbuf.readline().split(',')
323+
columns[-1] = columns[-1].strip() # strip trailing newline
324+
# Since the header has so many columns, excel saves blank cols in the
325+
# data below the header lines.
326+
columns = [col for col in columns if col != '']
327+
dtypes = dict.fromkeys(columns, float) # all floats except datevec
328+
dtypes.update(Year=int, Month=int, Day=int, Hour=int, Minute=int)
329+
dtypes['Cloud Type'] = int
330+
dtypes['Fill Flag'] = int
331+
data = pd.read_csv(
332+
fbuf, header=None, names=columns, usecols=columns, dtype=dtypes,
333+
delimiter=',', lineterminator='\n') # skip carriage returns \r
334+
# the response 1st 5 columns are a date vector, convert to datetime
335+
dtidx = pd.to_datetime(
336+
data[['Year', 'Month', 'Day', 'Hour', 'Minute']])
337+
# in USA all timezones are integers
338+
tz = 'Etc/GMT%+d' % -metadata['Time Zone']
339+
data.index = pd.DatetimeIndex(dtidx).tz_localize(tz)
340+
341+
if map_variables:
342+
data = data.rename(columns=VARIABLE_MAP)
343+
metadata['latitude'] = metadata.pop('Latitude')
344+
metadata['longitude'] = metadata.pop('Longitude')
345+
metadata['altitude'] = metadata.pop('Elevation')
346+
347+
return data, metadata
348+
349+
350+
def read_goes4(filename, map_variables=True):
351+
"""
352+
Read an NSRDB GOES4 weather file (formatted as SAM CSV). The NSRDB
353+
is described in [1]_ and the SAM CSV format is described in [2]_.
354+
355+
.. versionchanged:: 0.9.0
356+
The function now returns a tuple where the first element is a dataframe
357+
and the second element is a dictionary containing metadata. Previous
358+
versions of this function had the return values switched.
359+
360+
Parameters
361+
----------
362+
filename: str
363+
Filename of a file containing data to read.
364+
map_variables: bool, default True
365+
When true, renames columns of the Dataframe to pvlib variable names
366+
where applicable. See variable :const:`VARIABLE_MAP`.
367+
368+
Returns
369+
-------
370+
data : pandas.DataFrame
371+
timeseries data from NREL GOES4
372+
metadata : dict
373+
metadata from NREL GOES4 about the record, see
374+
:func:`pvlib.iotools.parse_goes4` for fields
375+
376+
See Also
377+
--------
378+
pvlib.iotools.parse_goes4, pvlib.iotools.get_goes4
379+
380+
References
381+
----------
382+
.. [1] `NREL National Solar Radiation Database (NSRDB)
383+
<https://nsrdb.nrel.gov/>`_
384+
.. [2] `Standard Time Series Data File Format
385+
<https://web.archive.org/web/20170207203107/https://sam.nrel.gov/sites/default/files/content/documents/pdf/wfcsv.pdf>`_
386+
"""
387+
with open(str(filename), 'r') as fbuf:
388+
content = parse_goes4(fbuf, map_variables)
389+
return content

0 commit comments

Comments
 (0)