LSSTApplications  17.0+11,17.0+34,17.0+56,17.0+57,17.0+59,17.0+7,17.0-1-g377950a+33,17.0.1-1-g114240f+2,17.0.1-1-g4d4fbc4+28,17.0.1-1-g55520dc+49,17.0.1-1-g5f4ed7e+52,17.0.1-1-g6dd7d69+17,17.0.1-1-g8de6c91+11,17.0.1-1-gb9095d2+7,17.0.1-1-ge9fec5e+5,17.0.1-1-gf4e0155+55,17.0.1-1-gfc65f5f+50,17.0.1-1-gfc6fb1f+20,17.0.1-10-g87f9f3f+1,17.0.1-11-ge9de802+16,17.0.1-16-ga14f7d5c+4,17.0.1-17-gc79d625+1,17.0.1-17-gdae4c4a+8,17.0.1-2-g26618f5+29,17.0.1-2-g54f2ebc+9,17.0.1-2-gf403422+1,17.0.1-20-g2ca2f74+6,17.0.1-23-gf3eadeb7+1,17.0.1-3-g7e86b59+39,17.0.1-3-gb5ca14a,17.0.1-3-gd08d533+40,17.0.1-30-g596af8797,17.0.1-4-g59d126d+4,17.0.1-4-gc69c472+5,17.0.1-6-g5afd9b9+4,17.0.1-7-g35889ee+1,17.0.1-7-gc7c8782+18,17.0.1-9-gc4bbfb2+3,w.2019.22
LSSTDataManagementBasePackage
shims.py
Go to the documentation of this file.
1 # This file is part of pipe_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Shim classes that provide (limited) Gen2 Butler interfaces to Gen3
23 data repositories.
24 
25 All of the classes here still operate on Gen3 data IDs - the shim layer
26 essentially assumes that Gen2 Tasks treat the data ID as an opaque blob, which
27 is usually (but not always true). When it isn't, the best approach now is
28 probably to use the GENERATION class attribute on the Butler classes to
29 special-case code for each generation.
30 """
31 
32 __all__ = ("ShimButler", "ShimButlerSubset", "ShimDataRef")
33 
34 from lsst.daf.persistence import NoResults
35 from lsst.daf.butler import StorageClassFactory
36 
37 
38 class ShimButler:
39  """A shim for a Gen2 `~lsst.daf.persistence.Butler` with a Gen3
40  `~lsst.daf.butler.Butler` backend.
41 
42  Parameters
43  ----------
44  butler3 : `lsst.daf.butler.Butler`
45  Generation 3 Butler instance.
46  """
47 
48  GENERATION = 2.5
49  """This is a Generation 2 shim for a Generation3 Butler.
50  """
51 
52  def __init__(self, butler3):
53  self._butler3 = butler3
54 
55  def _makeDataId(self, dataId=None, **rest):
56  """Construct a full data ID by merging the given arguments with the
57  internal data ID.
58  """
59  fullDataId = dict()
60  if dataId is not None:
61  fullDataId.update(dataId)
62  fullDataId.update(rest)
63  return fullDataId
64 
65  def _translateDatasetType(self, datasetType):
66  if "_" in datasetType:
67  if datasetType.endswith("_md"):
68  return f"{datasetType[:-3]}.metadata"
69  for component in StorageClassFactory().getStorageClass("Exposure").components:
70  suffix = f"_{component}"
71  if datasetType.endswith(suffix):
72  return "{}.{}".format(datasetType[:-len(suffix)], component)
73  return datasetType
74 
75  def datasetExists(self, datasetType, dataId=None, write=False, **rest):
76  """Check whether a datataset exists in the repository.
77 
78  Parameters
79  ----------
80  datasetType : `str`
81  Name of the Gen2 dataset type.
82  dataId : `dict` or `~lsst.daf.butler.DataId`, optional
83  A Generation 3 data ID that identifies the dataset.
84  write : `bool`
85  This option is provided for compatibility with
86  `lsst.daf.persistence.Butler`, but must be `False`.
87  rest
88  Additional key-value pairs to augment the given data ID.
89 
90  Returns
91  -------
92  exists : `bool`
93  `True` if the dataset is present in the repository, `False`
94  otherwise.
95  """
96  if write:
97  raise NotImplementedError("ShimButler cannot implement datasetExists with 'write=True'")
98  datasetType = self._translateDatasetType(datasetType)
99  try:
100  return self._butler3.datasetExists(datasetType, self._makeDataId(dataId, **rest))
101  except LookupError:
102  # Gen3 datasetExists raises if Dataset is not present in Registry;
103  # Gen2 does not distinguish between present in Datastore and
104  # present in Registry.
105  return False
106 
107  def get(self, datasetType, dataId=None, immediate=True, **rest):
108  """Retrieve a dataset.
109 
110  Parameters
111  ----------
112  datasetType : `str`
113  Name of the Gen2 dataset type.
114  dataId : `dict` or `~lsst.daf.butler.DataId`, optional
115  A Generation 3 data ID that identifies the dataset.
116  immediate : `bool`
117  This option is provided for compatibility with
118  `lsst.daf.persistence.Butler`, but is ignored.
119  rest
120  Additional key-value pairs to augment the given data ID.
121 
122  Returns
123  -------
124  dataset
125  Retrieved object.
126 
127  Raises
128  ------
129  `~lsst.daf.persistence.NoResults`
130  Raised if the dataset does not exist.
131  """
132  datasetType = self._translateDatasetType(datasetType)
133  fullDataId = self._makeDataId(dataId, **rest)
134  if datasetType.endswith("_sub"):
135  import lsst.afw.image
136  datasetType = datasetType[:-len("_sub")]
137  parameters = dict(bbox=fullDataId.pop("bbox"))
138  origin = fullDataId.pop("imageOrigin", lsst.afw.image.PARENT)
139  parameters["origin"] = origin
140  else:
141  parameters = {}
142  try:
143  return self._butler3.get(datasetType, fullDataId, parameters=parameters)
144  except (FileNotFoundError, LookupError) as err:
145  raise NoResults(str(err), datasetType, fullDataId)
146 
147  def put(self, obj, datasetType, dataId=None, doBackup=False, **rest):
148  """Write a dataset.
149 
150  Parameters
151  ----------
152  obj
153  Object to write.
154  datasetType : `str`
155  Name of the Gen2 dataset type.
156  dataId : `dict` or `~lsst.daf.butler.DataId`, optional
157  A Generation 3 data ID that identifies the dataset.
158  doBackup : `bool`
159  This option is provided for compatibility with
160  `lsst.daf.persistence.Butler`, but must be `False`.
161  rest
162  Additional key-value pairs to augment the given data ID.
163  """
164  if doBackup:
165  raise NotImplementedError("ShimButler cannot implement put with 'doBackup=True'")
166  datasetType = self._translateDatasetType(datasetType)
167  self._butler3.put(obj, datasetType, self._makeDataId(dataId, **rest))
168 
169  def dataRef(self, datasetType, level=None, dataId=None, **rest):
170  """Return a DataRef associated with the given dataset type and data ID.
171 
172  Parameters
173  ----------
174  datasetType : `str`
175  Name of the dataset type.
176  dataId : `dict` or `~lsst.daf.butler.DataId`, optional
177  A Generation 3 data ID that identifies the dataset.
178  level
179  This option is provided for compatibility with
180  `lsst.daf.persistence.Butler`, but must be `None`.
181  rest
182  Additional key-value pairs to augment the given data ID.
183  """
184  if level is not None:
185  raise NotImplementedError("ShimButler cannot implement dataRef with 'level != None'")
186  fullDataId = {}
187  if dataId is not None:
188  fullDataId.update(dataId)
189  fullDataId.update(rest)
190  return next(iter(ShimButlerSubset(self, datasetType, [fullDataId])))
191 
192 
194  """A shim for a Gen2 `~lsst.daf.persistence.ButlerSubset` with a Gen3
195  `~lsst.daf.butler.Butler` backend.
196 
197  Parameters
198  ----------
199  butler : `ShimButler`
200  Butler shim instance.
201  datasetType : `str`
202  Name of the dataset type.
203  dataIds : iterable of `dict` or `~lsst.daf.butler.DataId`
204  Generation 3 data IDs that define the data in this subset.
205  """
206 
207  GENERATION = 2.5
208  """This is a Generation 2 shim for a Generation3 Butler.
209  """
210 
211  def __init__(self, butler, datasetType, dataIds):
212  self.butler = butler
213  self.datasetType = datasetType
214  self._dataIds = tuple(dataIds)
215 
216  def __len__(self):
217  return len(self._dataIds)
218 
219  def __iter__(self):
220  for dataId in self._dataIds:
221  yield ShimDataRef(self, dataId)
222 
223 
225  """A shim for a Gen2 `~lsst.daf.persistence.ButlerDataRef` with a Gen3
226  `~lsst.daf.butler.Butler` backend.
227 
228  Parameters
229  ----------
230  butlerSubset : `ShimButlerSubset`
231  ButlerSubset shim instance. Sets the butler and default dataset type
232  used by the Dataref.
233  dataId : `dict` or `~lsst.daf.butler.DataId`
234  Generation 3 data ID associated with this reference.
235  """
236 
237  GENERATION = 2.5
238  """This is a Generation 2 shim for a Generation3 Butler.
239  """
240 
241  def __init__(self, butlerSubset, dataId):
242  self.butlerSubset = butlerSubset
243  self.dataId = dataId
244 
245  def get(self, datasetType=None, **rest):
246  """Retrieve a dataset.
247 
248  Parameters
249  ----------
250  datasetType : `str`, optional.
251  Name of the dataset type. Defaults to the dataset type used to
252  construct the `ShimButlerSubset`.
253  rest
254  Additional arguments forwarded to `ShimButler.get`.
255 
256  Returns
257  -------
258  dataset
259  Retrieved object.
260 
261  Raises
262  ------
263  `~lsst.daf.persistence.NoResults`
264  Raised if the dataset does not exist.
265  """
266  if datasetType is None:
267  datasetType = self.butlerSubset.datasetType
268  return self.butlerSubset.butler.get(datasetType, self.dataId, **rest)
269 
270  def put(self, obj, datasetType=None, doBackup=False, **rest):
271  """Write a dataset.
272 
273  Parameters
274  ----------
275  obj
276  Object to write.
277  datasetType : `str`, optional
278  Name of the dataset type. Defaults to the dataset type used to
279  construct the `ShimButlerSubset`.
280  doBackup : `bool`
281  This option is provided for compatibility with
282  `lsst.daf.persistence.ButlerDataRef`, but must be `False`.
283  rest
284  Additional arguments forwarded to `ShimButler.put`.
285  """
286  if datasetType is None:
287  datasetType = self.butlerSubset.datasetType
288  self.butlerSubset.butler.put(obj, datasetType, self.dataId, doBackup=doBackup, **rest)
289 
290  def datasetExists(self, datasetType=None, write=False, **rest):
291  """Check whether a datataset exists in the repository.
292 
293  Parameters
294  ----------
295  datasetType : `str`, optional
296  Name of the dataset type. Defaults to the dataset type used to
297  construct the `ShimButlerSubset`.
298  write : `bool`
299  This option is provided for compatibility with
300  `lsst.daf.persistence.ButlerDataRef`, but must be `False`.
301  rest
302  Additional arguments forwarded to `ShimButler.datasetExists`.
303 
304  Returns
305  -------
306  exists : `bool`
307  `True` if the dataset is present in the repository, `False`
308  otherwise.
309  """
310  if datasetType is None:
311  datasetType = self.butlerSubset.datasetType
312  return self.butlerSubset.butler.datasetExists(
313  datasetType, self.dataId, write=write, **rest)
314 
315  def getButler(self):
316  """Return the (shim) Butler used by this DataRef.
317  """
318  return self.butlerSubset.butler
def __init__(self, butler, datasetType, dataIds)
Definition: shims.py:211
def __init__(self, butlerSubset, dataId)
Definition: shims.py:241
def datasetExists(self, datasetType, dataId=None, write=False, rest)
Definition: shims.py:75
def put(self, obj, datasetType=None, doBackup=False, rest)
Definition: shims.py:270
def _makeDataId(self, dataId=None, rest)
Definition: shims.py:55
def __init__(self, butler3)
Definition: shims.py:52
def get(self, datasetType=None, rest)
Definition: shims.py:245
def get(self, datasetType, dataId=None, immediate=True, rest)
Definition: shims.py:107
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168
def dataRef(self, datasetType, level=None, dataId=None, rest)
Definition: shims.py:169
def _translateDatasetType(self, datasetType)
Definition: shims.py:65
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...
def datasetExists(self, datasetType=None, write=False, rest)
Definition: shims.py:290
def put(self, obj, datasetType, dataId=None, doBackup=False, rest)
Definition: shims.py:147