__author__ = "Daniel Westwood"
__contact__ = "daniel.westwood@stfc.ac.uk"
__copyright__ = "Copyright 2024 United Kingdom Research and Innovation"
import logging
from collections.abc import Mapping
from functools import partial
import numpy as np
import requests
from xarray import conventions
from xarray.backends import BackendEntrypoint, StoreBackendEntrypoint
from xarray.backends.common import AbstractDataStore
from xarray.coders import CFDatetimeCoder
from xarray.coding.times import decode_cf_datetime
from xarray.coding.variables import lazy_elemwise_func
from xarray.core import indexing
from xarray.core.dataset import Dataset
from xarray.core.formatting import first_n_items, last_item
from xarray.core.variable import Variable
from xarray.namedarray.pycompat import to_numpy
from cfapyx.datastore import CFADataStore
from cfapyx.utils import conform_data_to_units, logstream
from cfapyx.wrappers import FragmentArrayWrapper
logger = logging.getLogger(__name__)
logger.addHandler(logstream)
logger.propagate = False
def fetch_remote_as_bytes(remote_url: str) -> bytes:
response = requests.get(remote_url)
response.raise_for_status()
# Fetch bytes content of remote CFA file
return response.content
[docs]
def open_cfa_dataset(
filename_or_obj,
drop_variables=None,
mask_and_scale=False,
decode_times=True,
concat_characters=None,
decode_coords=None,
use_cftime=None,
decode_timedelta=None,
cfa_options: dict = None,
group=None,
):
"""
Top-level function which opens a CFA dataset using Xarray.
Creates a CFA Datastore
from the ``filename_or_obj`` provided, then passes this to a CFA
StoreBackendEntrypoint to create an Xarray Dataset. Most parameters are
not handled by CFA, so only the CFA-relevant ones are described here.
:param filename_or_obj: (str) The path to a CFA-netCDF file to be
opened by Xarray
:param cfa_options: (dict) A set of kwargs provided to CFA which
provide additional configurations. Currently implemented are:
substitutions (dict), decode_cfa (bool)
:param group: (str) The name or path to a NetCDF group.
CFA can handle opening from specific groups and will inherit both
``group`` and ``global`` dimensions/attributes.
:returns: An xarray.Dataset object composed of xarray.DataArray objects
representing the different NetCDF variables and dimensions. CFA aggregated
variables are decoded unless the ``decode_cfa`` parameter in ``cfa_options``
is false.
"""
cfa_options = cfa_options or {}
# Add loading CFA file from remote location
if isinstance(filename_or_obj, str):
if "https://" in filename_or_obj or "s3://" in filename_or_obj:
filename_or_obj = fetch_remote_as_bytes(filename_or_obj)
# Load the CFA datastore from the provided file (object not supported).
store = CFADataStore.open(filename_or_obj, group=group)
# Expands cfa_options into individual kwargs for the store.
store.cfa_options = cfa_options
# Mask/scale decoding now done internally.
store.mask_and_scale = mask_and_scale
# Xarray makes use of StoreBackendEntrypoints to provide the Dataset 'ds'
store_entrypoint = CFAStoreBackendEntrypoint()
ds = store_entrypoint.open_dataset(
store,
decode_times=decode_times,
concat_characters=concat_characters,
decode_coords=decode_coords,
drop_variables=drop_variables,
use_cftime=use_cftime,
decode_timedelta=decode_timedelta,
)
return ds
[docs]
class CFANetCDFBackendEntrypoint(BackendEntrypoint):
"""Open CFA-netCDF files (.nca) using 'cfapyx' in Xarray"""
description = 'Open CFA-netCDF files (.nca) using "cfapyx" in Xarray'
url = "https://cedadev.github.io/CFAPyX/"
[docs]
def open_dataset(
self,
filename_or_obj,
*,
drop_variables=None,
mask_and_scale=False,
decode_times=True,
concat_characters=None,
decode_coords=None,
use_cftime=None,
decode_timedelta=None,
cfa_options=None,
group=None,
# backend specific keyword arguments
# do not use 'chunks' or 'cache' here
):
"""
Returns a complete xarray representation of a CFA-netCDF dataset which
includes expanding/decoding CFA aggregated variables into proper arrays.
"""
cfa_options = cfa_options or {}
return open_cfa_dataset(
filename_or_obj,
drop_variables=drop_variables,
mask_and_scale=mask_and_scale,
decode_times=decode_times,
concat_characters=concat_characters,
decode_coords=decode_coords,
use_cftime=use_cftime,
decode_timedelta=decode_timedelta,
cfa_options=cfa_options,
group=group,
)
def _is_cfa_lazy_data(data) -> bool:
"""
Return True if ``data`` is backed by a lazy CFA ``FragmentArrayWrapper``.
"""
curr = data
while hasattr(curr, "array"):
if isinstance(curr, FragmentArrayWrapper):
return True
curr = curr.array
return isinstance(curr, FragmentArrayWrapper)
[docs]
class CFADatetimeCoder(CFDatetimeCoder):
"""
CFA-aware extension of Xarray's ``CFDatetimeCoder``.
Xarray's default ``CFDatetimeCoder`` decodes an ``example_value`` assembled
from the first and last items of a variable's array in order to infer the
decoded datetime dtype (e.g. ``datetime64[ns]`` vs ``object`` for
``cftime.datetime``). For CFA-aggregated variables such as ``time_bnds``,
fetching those items forces remote fragment files to be opened at dataset
open time even when the variable is never accessed.
``CFADatetimeCoder`` bypasses fragment fetching for CFA-aggregated variables
by generating ``example_value`` from the corresponding non-aggregated
coordinate variable (such as ``time``) already stored in the CFA dataset.
"""
def __init__(
self,
variables: Mapping[str, Variable] | None = None,
use_cftime: bool | None = None,
time_unit: str = "ns",
) -> None:
super().__init__(use_cftime=use_cftime, time_unit=time_unit)
self._coord_candidates = {}
if variables:
for k, v in variables.items():
if not _is_cfa_lazy_data(v._data):
self._coord_candidates[k] = (
v._data,
dict(v.attrs),
v.dims,
v.size,
)
def _find_time_coord(self, variable: Variable, name):
"""
Find a non-aggregated CF time coordinate variable corresponding to
``variable`` via CF ``bounds``/``climatology`` attributes or shared
dimensions.
"""
if name is not None:
for cand_name, cand_info in self._coord_candidates.items():
_, cand_attrs, _, _ = cand_info
cand_units = cand_attrs.get("units")
has_date_units = isinstance(cand_units, str) and "since" in cand_units
if has_date_units and (
cand_attrs.get("bounds") == name
or cand_attrs.get("climatology") == name
):
return cand_name, cand_info
for dim in variable.dims:
if dim in self._coord_candidates:
cand_info = self._coord_candidates[dim]
_, cand_attrs, _, _ = cand_info
cand_units = cand_attrs.get("units")
if isinstance(cand_units, str) and "since" in cand_units:
return dim, cand_info
return None, None
def _generate_example_value(self, variable: Variable, coord_info) -> np.ndarray:
"""
Generate a 1D ``example_value`` array from the first and last elements
of the non-aggregated coordinate variable.
"""
coord_data, coord_attrs, _, coord_size = coord_info
if variable.size == 0 or coord_size == 0:
return np.array([], dtype=variable.dtype)
coord_values = indexing.ImplicitToExplicitIndexingAdapter(
indexing.as_indexable(coord_data)
)
example_value = np.concatenate(
[
to_numpy(first_n_items(coord_values, 1)),
to_numpy(last_item(coord_values)),
]
)
if "scale_factor" in coord_attrs or "add_offset" in coord_attrs:
scale = coord_attrs.get("scale_factor", 1.0)
offset = coord_attrs.get("add_offset", 0.0)
example_value = example_value * scale + offset
var_units = variable.attrs.get("units")
var_calendar = variable.attrs.get("calendar")
coord_units = coord_attrs.get("units")
coord_calendar = coord_attrs.get("calendar")
if (
example_value.size > 0
and isinstance(coord_units, str)
and isinstance(var_units, str)
and (
coord_units != var_units
or (coord_calendar or "standard") != (var_calendar or "standard")
)
):
example_value = conform_data_to_units(
example_value,
coord_units,
var_units,
old_calendar=coord_calendar or "standard",
new_calendar=var_calendar or "standard",
)
if np.issubdtype(variable.dtype, np.number):
if not (
np.issubdtype(variable.dtype, np.integer)
and np.issubdtype(example_value.dtype, np.floating)
and np.isnan(example_value).any()
):
example_value = example_value.astype(variable.dtype)
return example_value
[docs]
def decode(self, variable: Variable, name=None) -> Variable:
"""
Custom Datetime decoder, defaults to CFDatetimeCoder.decode where
not using a CFA FragmentArrayWrapper for a variable."""
units = variable.attrs.get("units", None)
if not (isinstance(units, str) and "since" in units):
return variable
if not _is_cfa_lazy_data(variable._data):
return super().decode(variable, name=name)
coord_name, coord_info = self._find_time_coord(variable, name)
if coord_info is None:
return super().decode(variable, name=name)
example_value = self._generate_example_value(variable, coord_info)
logger.debug(
f"Bypassing fragment fetch for '{name}' using coordinate "
f"'{coord_name}' example_value={example_value}"
)
sample_var = Variable(
("time_sample",), example_value, variable.attrs, variable.encoding
)
decoded_sample = super().decode(sample_var, name=name)
if hasattr(decoded_sample._data, "func"):
transform = decoded_sample._data.func
else:
transform = partial(
decode_cf_datetime,
units=decoded_sample.encoding.get("units"),
calendar=decoded_sample.encoding.get("calendar"),
use_cftime=self.use_cftime,
time_unit=self.time_unit,
)
data = lazy_elemwise_func(variable._data, transform, decoded_sample.dtype)
return Variable(
variable.dims,
data,
decoded_sample.attrs,
decoded_sample.encoding,
fastpath=True,
)
def _resolve_cfa_decode_times(decode_times, use_cftime, variables):
"""
Wrap truthy ``decode_times`` specifications with ``CFADatetimeCoder`` so
CFA-aggregated lazy time variables generate ``example_value`` from their
coordinate dimension rather than fetching remote fragments.
"""
if not decode_times:
return decode_times, use_cftime
if isinstance(decode_times, CFDatetimeCoder):
if use_cftime is not None:
return decode_times, use_cftime
return (
CFADatetimeCoder(
variables=variables,
use_cftime=decode_times.use_cftime,
time_unit=decode_times.time_unit,
),
None,
)
if isinstance(decode_times, Mapping):
resolved = {}
for k in variables:
dt_k = decode_times.get(k, True)
uc_k = (
use_cftime.get(k, None)
if isinstance(use_cftime, Mapping)
else use_cftime
)
if not dt_k:
resolved[k] = False
elif isinstance(dt_k, CFDatetimeCoder):
if uc_k is not None:
return decode_times, use_cftime
resolved[k] = CFADatetimeCoder(
variables=variables,
use_cftime=dt_k.use_cftime,
time_unit=dt_k.time_unit,
)
else:
resolved[k] = CFADatetimeCoder(
variables=variables,
use_cftime=uc_k,
)
return resolved, None
if isinstance(use_cftime, Mapping):
resolved = {
k: CFADatetimeCoder(
variables=variables,
use_cftime=use_cftime.get(k, None),
)
for k in variables
}
return resolved, None
return CFADatetimeCoder(variables=variables, use_cftime=use_cftime), None
[docs]
class CFAStoreBackendEntrypoint(StoreBackendEntrypoint):
"""Open CFA-based Abstract Data Store"""
description = "Open CFA-based Abstract Data Store"
url = "https://cedadev.github.io/CFAPyX/"
[docs]
def open_dataset(
self,
cfa_xarray_store,
decode_times=True,
concat_characters=True,
decode_coords=True,
drop_variables=None,
use_cftime=None,
decode_timedelta=None,
) -> Dataset:
"""
Takes cfa_xarray_store of type AbstractDataStore and creates an
xarray.Dataset object. Most parameters are not handled by CFA, so only the
CFA-relevant ones are described here.
:param cfa_xarray_store: (obj) The CFA Datastore object which
loads and decodes CFA aggregated variables and dimensions.
:returns: An xarray.Dataset object composed of xarray.DataArray
objects representing the different NetCDF variables and dimensions.
CFA aggregated variables are decoded unless the ``decode_cfa``
parameter in ``cfa_options`` is false.
"""
assert isinstance(cfa_xarray_store, AbstractDataStore)
# Same as NetCDF4 operations, just with the CFA Datastore
vars, attrs = cfa_xarray_store.load()
encoding = cfa_xarray_store.get_encoding()
cfa_decode_times, cfa_use_cftime = _resolve_cfa_decode_times(
decode_times, use_cftime, vars
)
# Ensures variables/attributes comply with CF conventions.
vars, attrs, coord_names = conventions.decode_cf_variables(
vars,
attrs,
decode_times=cfa_decode_times,
concat_characters=concat_characters,
decode_coords=decode_coords,
drop_variables=drop_variables,
use_cftime=cfa_use_cftime,
decode_timedelta=decode_timedelta,
)
# Create the xarray.Dataset object here.
ds = Dataset(vars, attrs=attrs)
ds = ds.set_coords(coord_names.intersection(vars))
ds.set_close(cfa_xarray_store.close)
ds.encoding = encoding
return ds