acis.py

import requests
import pandas as pd
import numpy as np


VARIABLE_MAP = {
    # time series names
    'pcpn': 'precipitation',
    'maxt': 'temp_air_max',
    'avgt': 'temp_air_average',
    'obst': 'temp_air_observation',
    'mint': 'temp_air_min',
    'cdd': 'cooling_degree_days',
    'hdd': 'heating_degree_days',
    'gdd': 'growing_degree_days',
    'snow': 'snowfall',
    'snwd': 'snowdepth',

    # metadata names
    'lat': 'latitude',
    'lon': 'longitude',
    'elev': 'altitude',
}


def _get_acis(start, end, params, map_variables, url, **kwargs):
    """
    generic helper for the public get_acis_X functions
    """
    params = {
        # use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted
        'sdate': pd.to_datetime(start).strftime('%Y-%m-%d'),
        'edate': pd.to_datetime(end).strftime('%Y-%m-%d'),
        'output': 'json',
        **params,  # endpoint-specific parameters
    }
    response = requests.post(url,
                             json=params,
                             headers={"Content-Type": "application/json"},
                             **kwargs)
    response.raise_for_status()
    payload = response.json()

    # somewhat inconveniently, the ACIS API tends to return errors as "valid"
    # responses instead of using proper HTTP error codes:
    if "error" in payload:
        raise requests.HTTPError(payload['error'], response=response)

    columns = ['date'] + [e['name'] for e in params['elems']]
    df = pd.DataFrame(payload['data'], columns=columns)
    df = df.set_index('date')
    df.index = pd.to_datetime(df.index)
    df.index.name = None

    metadata = payload['meta']

    try:
        # for StnData endpoint, unpack combination "ll" into lat, lon 
        metadata['lon'], metadata['lat'] = metadata.pop('ll')
    except KeyError:
        pass

    try:
        metadata['elev'] = metadata['elev'] * 0.3048  # feet to meters
    except KeyError:
        # some queries don't return elevation
        pass

    if map_variables:
        df = df.rename(columns=VARIABLE_MAP)

        for key in list(metadata.keys()):
            if key in VARIABLE_MAP:
                metadata[VARIABLE_MAP[key]] = metadata.pop(key)

    return df, metadata


def get_acis_prism(latitude, longitude, start, end, map_variables=True,
                   url="https://data.rcc-acis.org/GridData", **kwargs):
    """
    Retrieve estimated daily precipitation and temperature data from PRISM
    via the Applied Climate Information System (ACIS).

    ACIS [2]_, [3]_ aggregates and provides access to climate data
    from many underlying sources.  This function retrieves daily data from
    the Parameter-elevation Regressions on Independent Slopes Model
    (PRISM) [1]_, a gridded precipitation and temperature model
    from Oregon State University.

    Geographical coverage: US, Central America, and part of South America.
    Approximately 0° to 50° in latitude and -130° to -65° in longitude.

    Parameters
    ----------
    latitude : float
        in decimal degrees, between -90 and 90, north is positive
    longitude : float
        in decimal degrees, between -180 and 180, east is positive
    start : datetime-like
        First day of the requested period
    end : datetime-like
        Last day of the requested period
    map_variables : bool, default True
        When True, rename data columns and metadata keys to pvlib variable
        names where applicable. See variable :const:`VARIABLE_MAP`.
    url : str, default: 'https://data.rcc-acis.org/GridData'
        API endpoint URL
    kwargs:
        Optional parameters passed to ``requests.post``.

    Returns
    -------
    data : pandas.DataFrame
        Daily precipitation [mm], temperature [Celsius], and degree day
        [Celsius-days] data
    metadata : dict
        Metadata of the selected grid cell

    Raises
    ------
    requests.HTTPError
        A message from the ACIS server if the request is rejected

    Notes
    -----
    PRISM data is aggregated from 12:00 to 12:00 UTC, meaning data labeled
    May 26 reflects to the 24 hours ending at 7:00am Eastern Standard Time
    on May 26.

    References
    ----------
    .. [1] `PRISM <https://prism.oregonstate.edu/>`_
    .. [2] `ACIS Gridded Data <http://www.rcc-acis.org/docs_gridded.html>`_
    .. [3] `ACIS Web Services <http://www.rcc-acis.org/docs_webservices.html>`_

    Examples
    --------
    >>> from pvlib.iotools import get_acis_prism
    >>> df, meta = get_acis_prism(40, 80, '2020-01-01', '2020-12-31')
    """
    elems = [
        {"name": "pcpn", "interval": "dly", "units": "mm"},
        {"name": "maxt", "interval": "dly", "units": "degreeC"},
        {"name": "mint", "interval": "dly", "units": "degreeC"},
        {"name": "avgt", "interval": "dly", "units": "degreeC"},
        {"name": "cdd", "interval": "dly", "units": "degreeC"},
        {"name": "hdd", "interval": "dly", "units": "degreeC"},
        {"name": "gdd", "interval": "dly", "units": "degreeC"},
    ]
    params = {
        'loc': f"{longitude},{latitude}",
        'grid': "21",
        'elems': elems,
        'meta': ["ll", "elev"],
    }
    df, meta = _get_acis(start, end, params, map_variables, url, **kwargs)
    df = df.replace(-999, np.nan)
    return df, meta


def get_acis_nrcc(latitude, longitude, start, end, grid, map_variables=True,
                  url="https://data.rcc-acis.org/GridData", **kwargs):
    """
    Retrieve estimated daily precipitation and temperature data from the
    Northeast Regional Climate Center via the Applied Climate
    Information System (ACIS).

    ACIS [2]_, [3]_ aggregates and provides access to climate data
    from many underlying sources.  This function retrieves daily data from
    Cornell's Northeast Regional Climate Center (NRCC) [1]_.

    Geographical coverage: US, Central America, and part of South America.
    Approximately 0° to 50° in latitude and -130° to -65° in longitude.

    Parameters
    ----------
    latitude : float
        in decimal degrees, between -90 and 90, north is positive
    longitude : float
        in decimal degrees, between -180 and 180, east is positive
    start : datetime-like
        First day of the requested period
    end : datetime-like
        Last day of the requested period
    grid : int
        Options are either 1 (for "NRCC Interpolated") or 3
        (for "NRCC Hi-Resolution").  See [2]_ for details.
    map_variables : bool, default True
        When True, rename data columns and metadata keys to pvlib variable
        names where applicable. See variable :const:`VARIABLE_MAP`.
    url : str, default: 'https://data.rcc-acis.org/GridData'
        API endpoint URL
    kwargs:
        Optional parameters passed to ``requests.post``.

    Returns
    -------
    data : pandas.DataFrame
        Daily precipitation [mm], temperature [Celsius], and degree day
        [Celsius-days] data
    metadata : dict
        Metadata of the selected grid cell

    Raises
    ------
    requests.HTTPError
        A message from the ACIS server if the request is rejected

    Notes
    -----
    The returned values are 24-hour aggregates, but
    the aggregation period may not be midnight to midnight in local time.
    Check the ACIS and NRCC documentation for details.

    References
    ----------
    .. [1] `NRCC <http://www.nrcc.cornell.edu/>`_
    .. [2] `ACIS Gridded Data <http://www.rcc-acis.org/docs_gridded.html>`_
    .. [3] `ACIS Web Services <http://www.rcc-acis.org/docs_webservices.html>`_

    Examples
    --------
    >>> from pvlib.iotools import get_acis_nrcc
    >>> df, meta = get_acis_nrcc(40, -80, '2020-01-01', '2020-12-31', grid=1)
    """
    elems = [
        {"name": "pcpn", "interval": "dly", "units": "mm"},
        {"name": "maxt", "interval": "dly", "units": "degreeC"},
        {"name": "mint", "interval": "dly", "units": "degreeC"},
        {"name": "avgt", "interval": "dly", "units": "degreeC"},
        {"name": "cdd", "interval": "dly", "units": "degreeC"},
        {"name": "hdd", "interval": "dly", "units": "degreeC"},
        {"name": "gdd", "interval": "dly", "units": "degreeC"},
    ]
    params = {
        'loc': f"{longitude},{latitude}",
        'grid': grid,
        'elems': elems,
        'meta': ["ll", "elev"],
    }
    df, meta = _get_acis(start, end, params, map_variables, url, **kwargs)
    df = df.replace(-999, np.nan)
    return df, meta



def get_acis_mpe(latitude, longitude, start, end, map_variables=True,
                 url="https://data.rcc-acis.org/GridData", **kwargs):
    """
    Retrieve estimated daily Multi-sensor Precipitation Estimates
    via the Applied Climate Information System (ACIS).

    ACIS [2]_, [3]_ aggregates and provides access to climate data
    from many underlying sources.  This function retrieves daily data from
    the National Weather Service's Multi-sensor Precipitation Estimates
    (MPE) [1]_, a gridded precipitation model.

    This dataset covers the contiguous United States, Mexico, and parts of
    Central America.

    Parameters
    ----------
    latitude : float
        in decimal degrees, between -90 and 90, north is positive
    longitude : float
        in decimal degrees, between -180 and 180, east is positive
    start : datetime-like
        First day of the requested period
    end : datetime-like
        Last day of the requested period
    map_variables : bool, default True
        When True, rename data columns and metadata keys to pvlib variable
        names where applicable. See variable :const:`VARIABLE_MAP`.
    url : str, default: 'https://data.rcc-acis.org/GridData'
        API endpoint URL
    kwargs:
        Optional parameters passed to ``requests.post``.

    Returns
    -------
    data : pandas.DataFrame
        Daily precipitation [mm] data
    metadata : dict
        Coordinates of the selected grid cell

    Raises
    ------
    requests.HTTPError
        A message from the ACIS server if the request is rejected

    Notes
    -----
    The returned values are 24-hour aggregates, but
    the aggregation period may not be midnight to midnight in local time.
    Check the ACIS and MPE documentation for details.

    References
    ----------
    .. [1] `Multisensor Precipitation Estimates
           <https://www.weather.gov/marfc/Multisensor_Precipitation>`_
    .. [2] `ACIS Gridded Data <http://www.rcc-acis.org/docs_gridded.html>`_
    .. [3] `ACIS Web Services <http://www.rcc-acis.org/docs_webservices.html>`_

    Examples
    --------
    >>> from pvlib.iotools import get_acis_mpe
    >>> df, meta = get_acis_mpe(40, -80, '2020-01-01', '2020-12-31')
    """
    elems = [
        # only precipitation is supported in this dataset
        {"name": "pcpn", "interval": "dly", "units": "mm"},
    ]
    params = {
        'loc': f"{longitude},{latitude}",
        'grid': "2",
        'elems': elems,
        'meta': ["ll"],  # "elev" is not supported for this dataset
    }
    df, meta = _get_acis(start, end, params, map_variables, url, **kwargs)
    df = df.replace(-999, np.nan)
    return df, meta


def get_acis_station_data(station, start, end, trace_val=0.001,
                          map_variables=True,
                          url="https://data.rcc-acis.org/StnData", **kwargs):
    """
    Retrieve weather station climate records via the Applied Climate
    Information System (ACIS).

    ACIS [1]_, [2]_ aggregates and provides access to climate data
    from many underlying sources.  This function retrieves measurements
    from ground stations belonging to various global networks.

    This function can query data from stations all over the world.
    The stations available in a given area can be listed using
    :py:func:`get_acis_available_stations`.

    Parameters
    ----------
    station : str
        Identifier code for the station to query. Identifiers from many
        station networks are accepted, including WBAN, COOP, FAA, WMO, GHCN,
        and others.  See [1]_ and [2]_ for details.
    start : datetime-like
        First day of the requested period
    end : datetime-like
        Last day of the requested period
    map_variables : bool, default True
        When True, rename data columns and metadata keys to pvlib variable
        names where applicable. See variable :const:`VARIABLE_MAP`.
    trace_val : float, default 0.001
        Value to replace "trace" values in the precipitation data
    url : str, default: 'https://data.rcc-acis.org/GridData'
        API endpoint URL
    kwargs:
        Optional parameters passed to ``requests.post``.

    Returns
    -------
    data : pandas.DataFrame
        Daily precipitation [mm], temperature [Celsius], snow [mm], and
        degree day [Celsius-days] data
    metadata : dict
        station metadata

    Raises
    ------
    requests.HTTPError
        A message from the ACIS server if the request is rejected

    See Also
    --------
    get_acis_available_stations

    References
    ----------
    .. [1] `ACIS Web Services <http://www.rcc-acis.org/docs_webservices.html>`_
    .. [2] `ACIS Metadata <http://www.rcc-acis.org/docs_metadata.html>`_

    Examples
    --------
    >>> # Using an FAA code (Chicago O'Hare airport)
    >>> from pvlib.iotools import get_acis_station_data
    >>> df, meta = get_acis_station_data('ORD', '2020-01-01', '2020-12-31')
    >>>
    >>> # Look up available stations in a lat/lon rectangle, with data
    >>> # available in the specified date range:
    >>> from pvlib.iotools import get_acis_available_stations
    >>> stations = get_acis_available_stations([39.5, 40.5], [-80.5, -79.5],
    ...                                        '2020-01-01', '2020-01-03')
    >>> stations['sids'][0]
    ['369367 2', 'USC00369367 6', 'WYNP1 7']
    >>> df, meta = get_acis_station_data('369367', '2020-01-01', '2020-01-03')
    """
    elems = [
        {"name": "maxt", "interval": "dly", "units": "degreeC"},
        {"name": "mint", "interval": "dly", "units": "degreeC"},
        {"name": "avgt", "interval": "dly", "units": "degreeC"},
        {"name": "obst", "interval": "dly", "units": "degreeC"},
        {"name": "pcpn", "interval": "dly", "units": "mm"},
        {"name": "snow", "interval": "dly", "units": "cm"},
        {"name": "snwd", "interval": "dly", "units": "cm"},
        {"name": "cdd", "interval": "dly", "units": "degreeC"},
        {"name": "hdd", "interval": "dly", "units": "degreeC"},
        {"name": "gdd", "interval": "dly", "units": "degreeC"},
    ]
    params = {
        'sid': str(station),
        'elems': elems,
        'meta': ('name,state,sids,sid_dates,ll,elev,uid,county,'
                 'climdiv,valid_daterange,tzo,network')
    }
    df, metadata = _get_acis(start, end, params, map_variables, url, **kwargs)
    df = df.mask(df == 'M', np.nan)
    df = df.mask(df == 'T', trace_val)
    df = df.astype(float)
    return df, metadata


def get_acis_available_stations(latitude_range, longitude_range,
                                start=None, end=None,
                                url="https://data.rcc-acis.org/StnMeta",
                                **kwargs):
    """
    List weather stations in a given area available from the
    Applied Climate Information System (ACIS).

    The ``sids`` returned by this function can be used with
    :py:func:`get_acis_station_data` to retrieve weather measurements
    from the station.

    Parameters
    ----------
    latitude_range : list
        A 2-element list of [southern bound, northern bound]
        in decimal degrees, between -90 and 90, north is positive
    longitude_range : list
        A 2-element list of [western bound, eastern bound]
        in decimal degrees, between -180 and 180, east is positive
    start : datetime-like, optional
        If specified, return only stations that have data between ``start`` and
        ``end``.  If not specified, all stations in the region are returned.
    end : datetime-like, optional
        See ``start``
    url : str, default: 'https://data.rcc-acis.org/StnMeta'
        API endpoint URL
    kwargs:
        Optional parameters passed to ``requests.post``.

    Returns
    -------
    stations : pandas.DataFrame
        A dataframe of station metadata, one row per station.
        The ``sids`` column contains IDs that can be used with 
        :py:func:`get_acis_station_data`.

    Raises
    ------
    requests.HTTPError
        A message from the ACIS server if the request is rejected

    See Also
    --------
    get_acis_station_data

    References
    ----------
    .. [1] `ACIS Web Services <http://www.rcc-acis.org/docs_webservices.html>`_
    .. [2] `ACIS Metadata <http://www.rcc-acis.org/docs_metadata.html>`_

    Examples
    --------
    >>> # Look up available stations in a lat/lon rectangle, with data
    >>> # available in the specified date range:
    >>> from pvlib.iotools import get_acis_available_stations
    >>> stations = get_acis_available_stations([39.5, 40.5], [-80.5, -79.5],
    ...                                        '2020-01-01', '2020-01-03')
    >>> stations['sids'][0]
    ['369367 2', 'USC00369367 6', 'WYNP1 7']
    """
    bbox = "{},{},{},{}".format(
        longitude_range[0],
        latitude_range[0],
        longitude_range[1],
        latitude_range[1],
    )
    params = {
        "bbox": bbox,
        "meta": ("name,state,sids,sid_dates,ll,elev,"
                 "uid,county,climdiv,tzo,network"),    
    }
    if start is not None and end is not None:
        params['elems'] = ['maxt', 'mint', 'avgt', 'obst',
                           'pcpn', 'snow', 'snwd']
        params['sdate'] = pd.to_datetime(start).strftime('%Y-%m-%d')
        params['edate'] = pd.to_datetime(end).strftime('%Y-%m-%d')

    response = requests.post(url,
                             json=params,
                             headers={"Content-Type": "application/json"},
                             **kwargs)
    response.raise_for_status()
    payload = response.json()
    if "error" in payload:
        raise requests.HTTPError(payload['error'], response=response)

    metadata = payload['meta']
    for station_record in metadata:
        station_record['altitude'] = station_record.pop('elev')
        station_record['longitude'], station_record['latitude'] = \
            station_record.pop('ll')

    df = pd.DataFrame(metadata)
    return df