LSSTApplications  19.0.0-14-gb0260a2+72efe9b372,20.0.0+7927753e06,20.0.0+8829bf0056,20.0.0+995114c5d2,20.0.0+b6f4b2abd1,20.0.0+bddc4f4cbe,20.0.0-1-g253301a+8829bf0056,20.0.0-1-g2b7511a+0d71a2d77f,20.0.0-1-g5b95a8c+7461dd0434,20.0.0-12-g321c96ea+23efe4bbff,20.0.0-16-gfab17e72e+fdf35455f6,20.0.0-2-g0070d88+ba3ffc8f0b,20.0.0-2-g4dae9ad+ee58a624b3,20.0.0-2-g61b8584+5d3db074ba,20.0.0-2-gb780d76+d529cf1a41,20.0.0-2-ged6426c+226a441f5f,20.0.0-2-gf072044+8829bf0056,20.0.0-2-gf1f7952+ee58a624b3,20.0.0-20-geae50cf+e37fec0aee,20.0.0-25-g3dcad98+544a109665,20.0.0-25-g5eafb0f+ee58a624b3,20.0.0-27-g64178ef+f1f297b00a,20.0.0-3-g4cc78c6+e0676b0dc8,20.0.0-3-g8f21e14+4fd2c12c9a,20.0.0-3-gbd60e8c+187b78b4b8,20.0.0-3-gbecbe05+48431fa087,20.0.0-38-ge4adf513+a12e1f8e37,20.0.0-4-g97dc21a+544a109665,20.0.0-4-gb4befbc+087873070b,20.0.0-4-gf910f65+5d3db074ba,20.0.0-5-gdfe0fee+199202a608,20.0.0-5-gfbfe500+d529cf1a41,20.0.0-6-g64f541c+d529cf1a41,20.0.0-6-g9a5b7a1+a1cd37312e,20.0.0-68-ga3f3dda+5fca18c6a4,20.0.0-9-g4aef684+e18322736b,w.2020.45
LSSTDataManagementBasePackage
butlerLocation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2008, 2009, 2010 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 
25 
26 """This module defines the ButlerLocation class."""
27 
28 import lsst.daf.base as dafBase
29 
30 import os
31 import yaml
32 
33 from . import iterify, doImport
34 
35 
37  """Initializer
38 
39  Parameters
40  ----------
41  assembler : function object
42  Function object or importable string to a function object that can be called with the assembler
43  signature: (dataId, componentDict, cls).
44  disassembler : function object
45  Function object or importable string to a function object that can be called with the disassembler
46  signature: (object, dataId, componentDict).
47  python : class object
48  A python class object or importable string to a class object that can be used by the assembler to
49  instantiate an object to be returned.
50  dataId : dict or DataId
51  The dataId that is used to look up components.
52  mapper : Mapper instance
53  A reference to the mapper that created this ButlerComposite object.
54  """
55 
56  class ComponentInfo():
57  """Information about a butler composite object. Some details come from the policy and some are filled
58  in by the butler. Component info is used while assembling and disassembling a composite object in
59  butler. It is used as an input to assemblers and disassemblers (which are part of the butler public
60  API).
61 
62  Parameters
63  ----------
64  datasetType : string
65  The datasetType of the component.
66  obj : object instance
67  The python object instance that is this component.
68  setter : string
69  The name of the function in the parent object to set this component.
70  Optional - may be None
71  getter : string
72  The name of the function in the parent object to get this component.
73  Optional - may be None
74  subset : bool
75  If true, indicates that the obj should be a list of objects found via butlerSubset.
76  inputOnly : bool
77  If true, indicates that the obj should not be serialized when performing a butler.put.
78  """
79  def __init__(self, datasetType, obj, setter, getter, subset, inputOnly):
80  self.datasetType = datasetType
81  self.obj = obj
82  self.setter = setter
83  self.getter = getter
84  self.subset = subset
85  self.inputOnly = inputOnly
86 
87  def __repr__(self):
88  return 'ComponentInfo(datasetType:%s, obj:%s, setter:%s, getter:%s, subset:%s)' % \
89  (self.datasetType, self.obj, self.setter, self.getter, self.subset)
90 
91  def __repr__(self):
92  return 'ButlerComposite(assembler:%s, disassembler:%s, python:%s, dataId:%s, mapper:%s, ' \
93  'componentInfo:%s, repository:%s)' % \
94  (self.assembler,
95  self.disassembler,
96  self.python,
97  self.dataId,
98  self.mapper,
99  self.componentInfo,
100  self.repository)
101 
102  def __init__(self, assembler, disassembler, python, dataId, mapper):
103  self.assembler = doImport(assembler) if isinstance(assembler, str) else assembler
104  self.disassembler = doImport(disassembler) if isinstance(disassembler, str) else disassembler
105  self.python = doImport(python) if isinstance(python, str) else python
106  self.dataId = dataId
107  self.mapper = mapper
108  self.componentInfo = {}
109  self.repository = None
110 
111  def add(self, id, datasetType, setter, getter, subset, inputOnly):
112  """Add a description of a component needed to fetch the composite dataset.
113 
114  Parameters
115  ----------
116  id : string
117  The name of the component in the policy definition.
118  datasetType : string
119  The name of the datasetType of the component.
120  setter : string or None
121  The name of the function used to set this component into the python type that contains it.
122  Specifying a setter is optional, use None if the setter won't be specified or used.
123  getter : string or None
124  The name of the function used to get this component from the python type that contains it.
125  Specifying a setter is optional, use None if the setter won't be specified or used.
126  subset : bool
127  If true, indicates that the obj should be a list of objects found via butlerSubset.
128  inputOnly : bool
129  If true, indicates that the obj should not be serialized when performing a butler.put.
130  """
131  self.componentInfo[id] = ButlerComposite.ComponentInfo(datasetType=datasetType,
132  obj=None,
133  setter=setter,
134  getter=getter,
135  subset=subset,
136  inputOnly=inputOnly)
137 
138  def setRepository(self, repository):
139  self.repository = repository
140 
141  def getRepository(self):
142  return self.repository
143 
144  def getPythonType(self):
145  return self.python
146 
147 
148 class ButlerLocation(yaml.YAMLObject):
149  """ButlerLocation is a struct-like class that holds information needed to
150  persist and retrieve an object using the LSST Persistence Framework.
151 
152  Mappers should create and return ButlerLocations from their
153  map_{datasetType} methods.
154 
155  Parameters
156  ----------
157  pythonType - string or class instance
158  This is the type of python object that should be created when reading the location.
159 
160  cppType - string or None
161  The type of cpp object represented by the location (optional, may be None)
162 
163  storageName - string
164  The type of storage the object is in or should be place into.
165 
166  locationList - list of string
167  A list of URI to place the object or where the object might be found. (Typically when reading the
168  length is expected to be exactly 1).
169 
170  dataId - dict
171  The dataId that was passed in when mapping the location. This may include keys that were not used for
172  mapping this location.
173 
174  mapper - mapper class instance
175  The mapper object that mapped this location.
176 
177  storage - storage class instance
178  The storage interface that can be used to read or write this location.
179 
180  usedDataId - dict
181  The dataId components that were used to map this location. If the mapper had to look up keys those
182  will be in this dict (even though they may not appear in the dataId parameter). If the dataId
183  parameter contained keys that were not required to map this item then those keys will NOT be in this
184  parameter.
185 
186  datasetType - string
187  The datasetType that this location represents.
188 
189  additionalData : `lsst.daf.base.PropertySet`, optional
190  Additional metadata to be passed to the persistence framework,
191  or `None`.
192  """
193 
194  yaml_tag = u"!ButlerLocation"
195  try:
196  # PyYAML >=5.1 prefers a different loader
197  yaml_loader = yaml.FullLoader
198  except AttributeError:
199  yaml_loader = yaml.Loader
200  yaml_dumper = yaml.Dumper
201 
202  def __repr__(self):
203  return \
204  'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, storage=%r, locationList=%r,' \
205  ' additionalData=%r, mapper=%r, dataId=%r)' % \
206  (self.pythonType, self.cppType, self.storageName, self.storage, self.locationList,
207  self.additionalData, self.mapper, self.dataId)
208 
209  def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage,
210  usedDataId=None, datasetType=None, additionalData=None):
211  self.pythonType = pythonType
212  self.cppType = cppType
213  self.storageName = storageName
214  self.mapper = mapper
215  self.storage = storage
216  self.locationList = iterify(locationList)
217  self.additionalData = additionalData if additionalData else dafBase.PropertySet()
218  for k, v in dataId.items():
219  self.additionalData.set(k, v)
220  self.dataId = dataId
221  self.usedDataId = usedDataId
222  self.datasetType = datasetType
223 
224  def __str__(self):
225  s = "%s at %s(%s)" % (self.pythonType, self.storageName,
226  ", ".join(self.locationList))
227  return s
228 
229  @staticmethod
230  def to_yaml(dumper, obj):
231  """Representer for dumping to YAML
232  :param dumper:
233  :param obj:
234  :return:
235  """
236  return dumper.represent_mapping(ButlerLocation.yaml_tag,
237  {'pythonType': obj.pythonType, 'cppType': obj.cppType,
238  'storageName': obj.storageName,
239  'locationList': obj.locationList, 'mapper': obj.mapper,
240  'storage': obj.storage, 'dataId': obj.dataId})
241 
242  @staticmethod
243  def from_yaml(loader, node):
244  obj = loader.construct_mapping(node)
245  return ButlerLocation(**obj)
246 
247  def setRepository(self, repository):
248  self.repository = repository
249 
250  def getRepository(self):
251  return self.repository
252 
253  def getPythonType(self):
254  return self.pythonType
255 
256  def getCppType(self):
257  return self.cppType
258 
259  def getStorageName(self):
260  return self.storageName
261 
262  def getLocations(self):
263  return self.locationList
264 
266  return [os.path.join(self.storage.root, l) for l in self.getLocations()]
267 
268  def getAdditionalData(self):
269  return self.additionalData
270 
271  def getStorage(self):
272  return self.storage
lsst::daf::persistence.butlerLocation.ButlerLocation.__repr__
def __repr__(self)
Definition: butlerLocation.py:202
lsst::daf::persistence.butlerLocation.ButlerComposite.assembler
assembler
Definition: butlerLocation.py:103
lsst::daf::persistence.butlerLocation.ButlerLocation.getLocations
def getLocations(self)
Definition: butlerLocation.py:262
lsst::daf::persistence.butlerLocation.ButlerComposite.getPythonType
def getPythonType(self)
Definition: butlerLocation.py:144
lsst::daf::persistence.butlerLocation.ButlerComposite.setRepository
def setRepository(self, repository)
Definition: butlerLocation.py:138
lsst::daf::persistence.butlerLocation.ButlerComposite
Definition: butlerLocation.py:36
lsst::daf::persistence.butlerLocation.ButlerLocation.storageName
storageName
Definition: butlerLocation.py:212
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo.subset
subset
Definition: butlerLocation.py:84
lsst::daf::persistence.butlerLocation.ButlerLocation.cppType
cppType
Definition: butlerLocation.py:211
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo.getter
getter
Definition: butlerLocation.py:83
lsst::daf::persistence.butlerLocation.ButlerComposite.add
def add(self, id, datasetType, setter, getter, subset, inputOnly)
Definition: butlerLocation.py:111
lsst::daf::persistence.butlerLocation.ButlerLocation.getStorage
def getStorage(self)
Definition: butlerLocation.py:271
lsst::daf::persistence.butlerLocation.ButlerLocation.pythonType
pythonType
Definition: butlerLocation.py:210
lsst::daf::persistence.butlerLocation.ButlerLocation
Definition: butlerLocation.py:148
lsst::daf::persistence.butlerLocation.ButlerLocation.getStorageName
def getStorageName(self)
Definition: butlerLocation.py:259
lsst::daf::persistence.butlerLocation.ButlerLocation.getRepository
def getRepository(self)
Definition: butlerLocation.py:250
lsst::daf::persistence.butlerLocation.ButlerComposite.__repr__
def __repr__(self)
Definition: butlerLocation.py:91
lsst::daf::persistence.butlerLocation.ButlerLocation.mapper
mapper
Definition: butlerLocation.py:213
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo.datasetType
datasetType
Definition: butlerLocation.py:80
lsst::daf::persistence.butlerLocation.ButlerLocation.__init__
def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage, usedDataId=None, datasetType=None, additionalData=None)
Definition: butlerLocation.py:209
lsst::daf::persistence.butlerLocation.ButlerLocation.additionalData
additionalData
Definition: butlerLocation.py:216
lsst::daf::persistence.butlerLocation.ButlerComposite.repository
repository
Definition: butlerLocation.py:109
lsst::daf::persistence.butlerLocation.ButlerComposite.python
python
Definition: butlerLocation.py:105
lsst::daf::persistence.butlerLocation.ButlerComposite.disassembler
disassembler
Definition: butlerLocation.py:104
lsst::daf::persistence.butlerLocation.ButlerLocation.repository
repository
Definition: butlerLocation.py:248
lsst::daf::persistence.butlerLocation.ButlerComposite.dataId
dataId
Definition: butlerLocation.py:106
lsst.pex.config.config.doImport
doImport
Definition: config.py:48
lsst::daf::persistence.butlerLocation.ButlerLocation.getLocationsWithRoot
def getLocationsWithRoot(self)
Definition: butlerLocation.py:265
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo
Definition: butlerLocation.py:56
lsst::daf::persistence.butlerLocation.ButlerLocation.locationList
locationList
Definition: butlerLocation.py:215
lsst::daf::persistence.butlerLocation.ButlerLocation.setRepository
def setRepository(self, repository)
Definition: butlerLocation.py:247
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo.setter
setter
Definition: butlerLocation.py:82
lsst::daf::base
Definition: Utils.h:47
lsst::daf::persistence.butlerLocation.ButlerLocation.to_yaml
def to_yaml(dumper, obj)
Definition: butlerLocation.py:230
lsst::daf::persistence.butlerLocation.ButlerComposite.mapper
mapper
Definition: butlerLocation.py:107
lsst::daf::persistence.butlerLocation.ButlerLocation.storage
storage
Definition: butlerLocation.py:214
lsst::daf::persistence.butlerLocation.ButlerLocation.usedDataId
usedDataId
Definition: butlerLocation.py:220
lsst::daf::persistence.butlerLocation.ButlerComposite.componentInfo
componentInfo
Definition: butlerLocation.py:108
lsst::daf::persistence.butlerLocation.ButlerLocation.__str__
def __str__(self)
Definition: butlerLocation.py:224
lsst::daf::base::PropertySet
Class for storing generic metadata.
Definition: PropertySet.h:67
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo.inputOnly
inputOnly
Definition: butlerLocation.py:85
lsst::daf::persistence.butlerLocation.ButlerLocation.from_yaml
def from_yaml(loader, node)
Definition: butlerLocation.py:243
lsst::daf::persistence.butlerLocation.ButlerLocation.dataId
dataId
Definition: butlerLocation.py:219
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo.obj
obj
Definition: butlerLocation.py:81
lsst::daf::persistence.butlerLocation.ButlerLocation.getAdditionalData
def getAdditionalData(self)
Definition: butlerLocation.py:268
lsst::daf::persistence.butlerLocation.ButlerComposite.__init__
def __init__(self, assembler, disassembler, python, dataId, mapper)
Definition: butlerLocation.py:102
lsst::daf::persistence.butlerLocation.ButlerComposite.getRepository
def getRepository(self)
Definition: butlerLocation.py:141
lsst::daf::persistence.utils.iterify
def iterify(x)
Definition: utils.py:49
lsst::daf::persistence.butlerLocation.ButlerLocation.getPythonType
def getPythonType(self)
Definition: butlerLocation.py:253
lsst::daf::persistence.butlerLocation.ButlerLocation.getCppType
def getCppType(self)
Definition: butlerLocation.py:256
lsst::daf::persistence.butlerLocation.ButlerLocation.datasetType
datasetType
Definition: butlerLocation.py:221
set
daf::base::PropertySet * set
Definition: fits.cc:912
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo.__repr__
def __repr__(self)
Definition: butlerLocation.py:87
lsst::daf::persistence.butlerLocation.ButlerComposite.ComponentInfo.__init__
def __init__(self, datasetType, obj, setter, getter, subset, inputOnly)
Definition: butlerLocation.py:79