Skip to content

Thredds

ThreddsConf

Bases: BaseModel

Thredds Config.

Parameters:

Name Type Description Default
url str

URL of Thredds server.

 required
depth int

Depth of catalog walk.

 1000
uri_term str

Attritube to use as uri.

 required
extra_terms list[KeyOutputKey]

List of extra attributes.

 []
Source code in stac_generator/plugins/inputs/thredds.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
class ThreddsConf(BaseModel):
    """Thredds Config."""

    url: str = Field(
        description="URL of Thredds server.",
    )
    depth: int = Field(
        default=1000,
        description="Depth of catalog walk.",
    )
    uri_term: str = Field(
        description="Attritube to use as uri.",
    )
    extra_terms: list[KeyOutputKey] = Field(
        default=[],
        description="List of extra attributes.",
    )

ThreddsInput

Bases: Input

Uses a Thredds Data Server <https://www.unidata.ucar.edu/software/tds/current/>_ as an event source.

Plugin name: thredds

Example Configuration

.. code-block:: yaml

- name: thredds
  conf:
    uri: test-url
    object_path_attr: access_urls.OPENDAP
Source code in stac_generator/plugins/inputs/thredds.py 
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
class ThreddsInput(Input):
    """
    Uses a `Thredds Data Server <https://www.unidata.ucar.edu/software/tds/current/>`_
    as an event source.

    **Plugin name:** ``thredds``

    Example Configuration:
        .. code-block:: yaml

            - name: thredds
              conf:
                uri: test-url
                object_path_attr: access_urls.OPENDAP

    """

    config_class = ThreddsConf

    def walk_tds(self, catalog: TDSCatalog, depth: int) -> Iterator:
        """
        Return a generator walking a THREDDS data catalog for datasets.

        :param cat: THREDDS catalog.
        :param depth: Maximum recursive depth.
        """
        yield from catalog.datasets.items()

        if depth > 0:
            for _, ref in catalog.catalog_refs.items():
                child = ref.follow()
                yield from self.walk_tds(catalog=child, depth=depth - 1)

    def get_sub_attr(self, obj: object, path: str):
        """
        Returns a child or sub-child attribute of a dict object.

        :param obj: Object
        :param path: 'attr1.attr2.etc'
        :return: obj.attr1.attr2.etc
        """
        attrs = path.split(".")

        for attr in attrs:
            if isinstance(obj, CaseInsensitiveDict):
                obj = obj[attr]
                continue

            obj = getattr(obj, attr)

        return obj

    def run(self):
        """
        Plugin's entrypoint.

        """
        total_generated = 0
        start = datetime.now()

        catalog = TDSCatalog(self.conf.url, **self.conf.thredds_kwargs)

        for _, dataset in self.walk_tds(catalog=catalog, depth=self.conf.depth):
            output = {"uri": self.get_sub_attr(dataset, self.conf.uri_term)}

            for extra_term in self.conf.extra_terms:
                output[extra_term.output_key] = self.get_sub_attr(dataset, extra_term.key)

            yield output
            total_generated += 1

        end = datetime.now()
        print(f"Processed {total_generated} records from {self.conf.url} in {end - start}")

get_sub_attr(obj, path)

Returns a child or sub-child attribute of a dict object.

:param obj: Object :param path: 'attr1.attr2.etc' :return: obj.attr1.attr2.etc

Source code in stac_generator/plugins/inputs/thredds.py 
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
def get_sub_attr(self, obj: object, path: str):
    """
    Returns a child or sub-child attribute of a dict object.

    :param obj: Object
    :param path: 'attr1.attr2.etc'
    :return: obj.attr1.attr2.etc
    """
    attrs = path.split(".")

    for attr in attrs:
        if isinstance(obj, CaseInsensitiveDict):
            obj = obj[attr]
            continue

        obj = getattr(obj, attr)

    return obj

run()

Plugin's entrypoint.

Source code in stac_generator/plugins/inputs/thredds.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def run(self):
    """
    Plugin's entrypoint.

    """
    total_generated = 0
    start = datetime.now()

    catalog = TDSCatalog(self.conf.url, **self.conf.thredds_kwargs)

    for _, dataset in self.walk_tds(catalog=catalog, depth=self.conf.depth):
        output = {"uri": self.get_sub_attr(dataset, self.conf.uri_term)}

        for extra_term in self.conf.extra_terms:
            output[extra_term.output_key] = self.get_sub_attr(dataset, extra_term.key)

        yield output
        total_generated += 1

    end = datetime.now()
    print(f"Processed {total_generated} records from {self.conf.url} in {end - start}")

walk_tds(catalog, depth)

Return a generator walking a THREDDS data catalog for datasets.

:param cat: THREDDS catalog. :param depth: Maximum recursive depth.

Source code in stac_generator/plugins/inputs/thredds.py 
64
65
66
67
68
69
70
71
72
73
74
75
76
def walk_tds(self, catalog: TDSCatalog, depth: int) -> Iterator:
    """
    Return a generator walking a THREDDS data catalog for datasets.

    :param cat: THREDDS catalog.
    :param depth: Maximum recursive depth.
    """
    yield from catalog.datasets.items()

    if depth > 0:
        for _, ref in catalog.catalog_refs.items():
            child = ref.follow()
            yield from self.walk_tds(catalog=child, depth=depth - 1)