LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
butlerLocation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2008, 2009, 2010 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 
25 
26 """This module defines the ButlerLocation class."""
27 
28 import lsst.daf.base as dafBase
29 
30 import os
31 import yaml
32 
33 from . import iterify, doImport
34 
35 
37  """Initializer
38 
39  Parameters
40  ----------
41  assembler : function object
42  Function object or importable string to a function object that can be called with the assembler
43  signature: (dataId, componentDict, cls).
44  disassembler : function object
45  Function object or importable string to a function object that can be called with the disassembler
46  signature: (object, dataId, componentDict).
47  python : class object
48  A python class object or importable string to a class object that can be used by the assembler to
49  instantiate an object to be returned.
50  dataId : dict or DataId
51  The dataId that is used to look up components.
52  mapper : Mapper instance
53  A reference to the mapper that created this ButlerComposite object.
54  """
55 
56  class ComponentInfo():
57  """Information about a butler composite object. Some details come from the policy and some are filled
58  in by the butler. Component info is used while assembling and disassembling a composite object in
59  butler. It is used as an input to assemblers and disassemblers (which are part of the butler public
60  API).
61 
62  Parameters
63  ----------
64  datasetType : string
65  The datasetType of the component.
66  obj : object instance
67  The python object instance that is this component.
68  setter : string
69  The name of the function in the parent object to set this component.
70  Optional - may be None
71  getter : string
72  The name of the function in the parent object to get this component.
73  Optional - may be None
74  subset : bool
75  If true, indicates that the obj should be a list of objects found via butlerSubset.
76  inputOnly : bool
77  If true, indicates that the obj should not be serialized when performing a butler.put.
78  """
79  def __init__(self, datasetType, obj, setter, getter, subset, inputOnly):
80  self.datasetTypedatasetType = datasetType
81  self.objobj = obj
82  self.settersetter = setter
83  self.gettergetter = getter
84  self.subsetsubset = subset
85  self.inputOnlyinputOnly = inputOnly
86 
87  def __repr__(self):
88  return 'ComponentInfo(datasetType:%s, obj:%s, setter:%s, getter:%s, subset:%s)' % \
89  (self.datasetTypedatasetType, self.objobj, self.settersetter, self.gettergetter, self.subsetsubset)
90 
91  def __repr__(self):
92  return 'ButlerComposite(assembler:%s, disassembler:%s, python:%s, dataId:%s, mapper:%s, ' \
93  'componentInfo:%s, repository:%s)' % \
94  (self.assemblerassembler,
95  self.disassemblerdisassembler,
96  self.pythonpython,
97  self.dataIddataId,
98  self.mappermapper,
99  self.componentInfocomponentInfo,
100  self.repositoryrepository)
101 
102  def __init__(self, assembler, disassembler, python, dataId, mapper):
103  self.assemblerassembler = doImport(assembler) if isinstance(assembler, str) else assembler
104  self.disassemblerdisassembler = doImport(disassembler) if isinstance(disassembler, str) else disassembler
105  self.pythonpython = doImport(python) if isinstance(python, str) else python
106  self.dataIddataId = dataId
107  self.mappermapper = mapper
108  self.componentInfocomponentInfo = {}
109  self.repositoryrepository = None
110 
111  def add(self, id, datasetType, setter, getter, subset, inputOnly):
112  """Add a description of a component needed to fetch the composite dataset.
113 
114  Parameters
115  ----------
116  id : string
117  The name of the component in the policy definition.
118  datasetType : string
119  The name of the datasetType of the component.
120  setter : string or None
121  The name of the function used to set this component into the python type that contains it.
122  Specifying a setter is optional, use None if the setter won't be specified or used.
123  getter : string or None
124  The name of the function used to get this component from the python type that contains it.
125  Specifying a setter is optional, use None if the setter won't be specified or used.
126  subset : bool
127  If true, indicates that the obj should be a list of objects found via butlerSubset.
128  inputOnly : bool
129  If true, indicates that the obj should not be serialized when performing a butler.put.
130  """
131  self.componentInfocomponentInfo[id] = ButlerComposite.ComponentInfo(datasetType=datasetType,
132  obj=None,
133  setter=setter,
134  getter=getter,
135  subset=subset,
136  inputOnly=inputOnly)
137 
138  def setRepository(self, repository):
139  self.repositoryrepository = repository
140 
141  def getRepository(self):
142  return self.repositoryrepository
143 
144  def getPythonType(self):
145  return self.pythonpython
146 
147 
148 class ButlerLocation(yaml.YAMLObject):
149  """ButlerLocation is a struct-like class that holds information needed to
150  persist and retrieve an object using the LSST Persistence Framework.
151 
152  Mappers should create and return ButlerLocations from their
153  map_{datasetType} methods.
154 
155  Parameters
156  ----------
157  pythonType - string or class instance
158  This is the type of python object that should be created when reading the location.
159 
160  cppType - string or None
161  The type of cpp object represented by the location (optional, may be None)
162 
163  storageName - string
164  The type of storage the object is in or should be place into.
165 
166  locationList - list of string
167  A list of URI to place the object or where the object might be found. (Typically when reading the
168  length is expected to be exactly 1).
169 
170  dataId - dict
171  The dataId that was passed in when mapping the location. This may include keys that were not used for
172  mapping this location.
173 
174  mapper - mapper class instance
175  The mapper object that mapped this location.
176 
177  storage - storage class instance
178  The storage interface that can be used to read or write this location.
179 
180  usedDataId - dict
181  The dataId components that were used to map this location. If the mapper had to look up keys those
182  will be in this dict (even though they may not appear in the dataId parameter). If the dataId
183  parameter contained keys that were not required to map this item then those keys will NOT be in this
184  parameter.
185 
186  datasetType - string
187  The datasetType that this location represents.
188 
189  additionalData : `lsst.daf.base.PropertySet`, optional
190  Additional metadata to be passed to the persistence framework,
191  or `None`.
192  """
193 
194  yaml_tag = u"!ButlerLocation"
195  try:
196  # PyYAML >=5.1 prefers a different loader
197  yaml_loader = yaml.UnsafeLoader
198  except AttributeError:
199  yaml_loader = yaml.Loader
200  yaml_dumper = yaml.Dumper
201 
202  def __repr__(self):
203  return \
204  'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, storage=%r, locationList=%r,' \
205  ' additionalData=%r, mapper=%r, dataId=%r)' % \
206  (self.pythonTypepythonType, self.cppTypecppType, self.storageNamestorageName, self.storagestorage, self.locationListlocationList,
207  self.additionalDataadditionalData, self.mappermapper, self.dataIddataId)
208 
209  def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage,
210  usedDataId=None, datasetType=None, additionalData=None):
211  self.pythonTypepythonType = pythonType
212  self.cppTypecppType = cppType
213  self.storageNamestorageName = storageName
214  self.mappermapper = mapper
215  self.storagestorage = storage
216  self.locationListlocationList = iterify(locationList)
217  self.additionalDataadditionalData = additionalData if additionalData else dafBase.PropertySet()
218  for k, v in dataId.items():
219  self.additionalDataadditionalData.set(k, v)
220  self.dataIddataId = dataId
221  self.usedDataIdusedDataId = usedDataId
222  self.datasetTypedatasetType = datasetType
223 
224  def __str__(self):
225  s = "%s at %s(%s)" % (self.pythonTypepythonType, self.storageNamestorageName,
226  ", ".join(self.locationListlocationList))
227  return s
228 
229  @staticmethod
230  def to_yaml(dumper, obj):
231  """Representer for dumping to YAML
232  :param dumper:
233  :param obj:
234  :return:
235  """
236  return dumper.represent_mapping(ButlerLocation.yaml_tag,
237  {'pythonType': obj.pythonType, 'cppType': obj.cppType,
238  'storageName': obj.storageName,
239  'locationList': obj.locationList, 'mapper': obj.mapper,
240  'storage': obj.storage, 'dataId': obj.dataId})
241 
242  @staticmethod
243  def from_yaml(loader, node):
244  obj = loader.construct_mapping(node)
245  return ButlerLocation(**obj)
246 
247  def setRepository(self, repository):
248  self.repositoryrepository = repository
249 
250  def getRepository(self):
251  return self.repositoryrepository
252 
253  def getPythonType(self):
254  return self.pythonTypepythonType
255 
256  def getCppType(self):
257  return self.cppTypecppType
258 
259  def getStorageName(self):
260  return self.storageNamestorageName
261 
262  def getLocations(self):
263  return self.locationListlocationList
264 
266  return [os.path.join(self.storagestorage.root, loc) for loc in self.getLocationsgetLocations()]
267 
268  def getAdditionalData(self):
269  return self.additionalDataadditionalData
270 
271  def getStorage(self):
272  return self.storagestorage
Class for storing generic metadata.
Definition: PropertySet.h:66
def __init__(self, datasetType, obj, setter, getter, subset, inputOnly)
def __init__(self, assembler, disassembler, python, dataId, mapper)
def add(self, id, datasetType, setter, getter, subset, inputOnly)
def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage, usedDataId=None, datasetType=None, additionalData=None)
daf::base::PropertySet * set
Definition: fits.cc:912