LSST Applications 27.0.0,g0265f82a02+469cd937ee,g02d81e74bb+21ad69e7e1,g1470d8bcf6+cbe83ee85a,g2079a07aa2+e67c6346a6,g212a7c68fe+04a9158687,g2305ad1205+94392ce272,g295015adf3+81dd352a9d,g2bbee38e9b+469cd937ee,g337abbeb29+469cd937ee,g3939d97d7f+72a9f7b576,g487adcacf7+71499e7cba,g50ff169b8f+5929b3527e,g52b1c1532d+a6fc98d2e7,g591dd9f2cf+df404f777f,g5a732f18d5+be83d3ecdb,g64a986408d+21ad69e7e1,g858d7b2824+21ad69e7e1,g8a8a8dda67+a6fc98d2e7,g99cad8db69+f62e5b0af5,g9ddcbc5298+d4bad12328,ga1e77700b3+9c366c4306,ga8c6da7877+71e4819109,gb0e22166c9+25ba2f69a1,gb6a65358fc+469cd937ee,gbb8dafda3b+69d3c0e320,gc07e1c2157+a98bf949bb,gc120e1dc64+615ec43309,gc28159a63d+469cd937ee,gcf0d15dbbd+72a9f7b576,gdaeeff99f8+a38ce5ea23,ge6526c86ff+3a7c1ac5f1,ge79ae78c31+469cd937ee,gee10cc3b42+a6fc98d2e7,gf1cff7945b+21ad69e7e1,gfbcc870c63+9a11dc8c8f
LSST Data Management Base Package
Loading...
Searching...
No Matches
apdbIndex.py
Go to the documentation of this file.
1# This file is part of dax_apdb.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21
22from __future__ import annotations
23
24__all__ = ["ApdbIndex"]
25
26import io
27import logging
28import os
29from collections.abc import Mapping
30from typing import ClassVar
31
32import yaml
33from lsst.resources import ResourcePath
34from pydantic import TypeAdapter, ValidationError
35
36_LOG = logging.getLogger(__name__)
37
38
40 """Index of well-known Apdb instances.
41
42 Parameters
43 ----------
44 index_path : `str`, optional
45 Path to the index configuration file, if not provided then the value
46 of ``DAX_APDB_INDEX_URI`` environment variable is used to locate
47 configuration file.
48
49 The index is configured from a simple YAML file whose location is
50 determined from ``DAX_APDB_INDEX_URI`` environment variable. The content
51 of the index file is a mapping of the labels to URIs in YAML format, e.g.:
52
53 .. code-block:: yaml
54
55 dev: "/path/to/config-file.yaml"
56 "prod/pex_config": "s3://bucket/apdb-prod.py"
57 "prod/yaml": "s3://bucket/apdb-prod.yaml"
58
59 The labels in the index file consists of the label name and an optional
60 format name separated from label by slash. `get_apdb_uri` method can
61 use its ``format`` argument to return either a format-specific
62 configuration or a label-only configuration if format-specific is not
63 defined.
64 """
65
66 index_env_var: ClassVar[str] = "DAX_APDB_INDEX_URI"
67 """The name of the environment variable containing the URI of the index
68 configuration file.
69 """
70
71 _cache: Mapping[str, str] | None = None
72 """Contents of the index file cached in memory."""
73
74 def __init__(self, index_path: str | None = None):
75 self._index_path = index_path
76
77 def _read_index(self, index_path: str | None = None) -> Mapping[str, str]:
78 """Return contents of the index file.
79
80 Parameters
81 ----------
82 index_path : `str`, optional
83 Location of the index file, if not provided then default location
84 is used.
85
86 Returns
87 -------
88 entries : `~collections.abc.Mapping` [`str`, `str`]
89 All known entries. Can be empty if no index can be found.
90
91 Raises
92 ------
93 RuntimeError
94 Raised if ``index_path`` is not provided and environment variable
95 is not set.
96 TypeError
97 Raised if content of the configuration file is incorrect.
98 """
99 if self._cache_cache is not None:
100 return self._cache_cache
101 if index_path is None:
102 index_path = os.getenv(self.index_env_var)
103 if not index_path:
104 raise RuntimeError(
105 f"No repository index defined, environment variable {self.index_env_var} is not set."
106 )
107 index_uri = ResourcePath(index_path)
108 _LOG.debug("Opening YAML index file: %s", index_uri.geturl())
109 try:
110 content = index_uri.read()
111 except IsADirectoryError as exc:
112 raise FileNotFoundError(f"Index file {index_uri.geturl()} is a directory") from exc
113 stream = io.BytesIO(content)
114 if index_data := yaml.load(stream, Loader=yaml.SafeLoader):
115 try:
116 self._cache_cache = TypeAdapter(dict[str, str]).validate_python(index_data)
117 except ValidationError as e:
118 raise TypeError(f"Repository index {index_uri.geturl()} not in expected format") from e
119 else:
120 self._cache_cache = {}
121 return self._cache_cache
122
123 def get_apdb_uri(self, label: str, format: str | None = None) -> ResourcePath:
124 """Return URI for APDB configuration file given its label.
125
126 Parameters
127 ----------
128 label : `str`
129 Label of APDB instance.
130 format : `str`
131 Format of the APDB configuration file, arbitrary string. This can
132 be used to support an expected migration from pex_config to YAML
133 configuration for APDB, code that uses pex_config could provide
134 "pex_config" for ``format``. The actual key in the index is
135 either a slash-separated label and format, or, if that is missing,
136 just a label.
137
138 Returns
139 -------
140 uri : `~lsst.resources.ResourcePath`
141 URI for the configuration file for APDB instance.
142
143 Raises
144 ------
145 FileNotFoundError
146 Raised if an index is defined in the environment but it
147 can not be found.
148 ValueError
149 Raised if the label is not found in the index.
150 RuntimeError
151 Raised if ``index_path`` is not provided and environment variable
152 is not set.
153 TypeError
154 Raised if the format of the index file is incorrect.
155 """
156 index = self._read_index(self._index_path)
157 labels: list[str] = [label]
158 if format:
159 labels.insert(0, f"{label}/{format}")
160 for label in labels:
161 if (uri_str := index.get(label)) is not None:
162 return ResourcePath(uri_str)
163 if len(labels) == 1:
164 message = f"Label {labels[0]} is not defined in index file"
165 else:
166 labels_str = ", ".join(labels)
167 message = f"None of labels {labels_str} is defined in index file"
168 all_labels = set(index)
169 raise ValueError(f"{message}, labels known to index: {all_labels}")
170
171 def get_entries(self) -> Mapping[str, str]:
172 """Retrieve all entries defined in index.
173
174 Returns
175 -------
176 entries : `~collections.abc.Mapping` [`str`, `str`]
177 All known index entries.
178
179 Raises
180 ------
181 RuntimeError
182 Raised if ``index_path`` is not provided and environment variable
183 is not set.
184 TypeError
185 Raised if content of the configuration file is incorrect.
186 """
187 return self._read_index(self._index_path)
Mapping[str, str] _read_index(self, str|None index_path=None)
Definition apdbIndex.py:77
__init__(self, str|None index_path=None)
Definition apdbIndex.py:74
Mapping[str, str] get_entries(self)
Definition apdbIndex.py:171
ResourcePath get_apdb_uri(self, str label, str|None format=None)
Definition apdbIndex.py:123
daf::base::PropertySet * set
Definition fits.cc:931