LSSTApplications  19.0.0-14-gb0260a2+72efe9b372,20.0.0+7927753e06,20.0.0+8829bf0056,20.0.0+995114c5d2,20.0.0+b6f4b2abd1,20.0.0+bddc4f4cbe,20.0.0-1-g253301a+8829bf0056,20.0.0-1-g2b7511a+0d71a2d77f,20.0.0-1-g5b95a8c+7461dd0434,20.0.0-12-g321c96ea+23efe4bbff,20.0.0-16-gfab17e72e+fdf35455f6,20.0.0-2-g0070d88+ba3ffc8f0b,20.0.0-2-g4dae9ad+ee58a624b3,20.0.0-2-g61b8584+5d3db074ba,20.0.0-2-gb780d76+d529cf1a41,20.0.0-2-ged6426c+226a441f5f,20.0.0-2-gf072044+8829bf0056,20.0.0-2-gf1f7952+ee58a624b3,20.0.0-20-geae50cf+e37fec0aee,20.0.0-25-g3dcad98+544a109665,20.0.0-25-g5eafb0f+ee58a624b3,20.0.0-27-g64178ef+f1f297b00a,20.0.0-3-g4cc78c6+e0676b0dc8,20.0.0-3-g8f21e14+4fd2c12c9a,20.0.0-3-gbd60e8c+187b78b4b8,20.0.0-3-gbecbe05+48431fa087,20.0.0-38-ge4adf513+a12e1f8e37,20.0.0-4-g97dc21a+544a109665,20.0.0-4-gb4befbc+087873070b,20.0.0-4-gf910f65+5d3db074ba,20.0.0-5-gdfe0fee+199202a608,20.0.0-5-gfbfe500+d529cf1a41,20.0.0-6-g64f541c+d529cf1a41,20.0.0-6-g9a5b7a1+a1cd37312e,20.0.0-68-ga3f3dda+5fca18c6a4,20.0.0-9-g4aef684+e18322736b,w.2020.45
LSSTDataManagementBasePackage
storageInterface.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2017 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from abc import ABCMeta, abstractmethod
25 
26 
27 class NoRepositroyAtRoot(RuntimeError):
28  pass
29 
30 
32  """Defines the interface for a connection to a Storage location.
33 
34  Parameters
35  ----------
36  uri : string
37  URI or path that is used as the storage location.
38  create : bool
39  If True The StorageInterface subclass should create a new
40  repository at the root location. If False then a new repository
41  will not be created.
42 
43  Raises
44  ------
45  NoRepositroyAtRoot
46  If create is False and a repository does not exist at the root
47  specified by uri then NoRepositroyAtRoot is raised.
48  """
49  __metaclass__ = ABCMeta
50 
51  def __init__(self, uri, create):
52  """initialzer"""
53  pass
54 
55  @classmethod
56  def _readFormatters(cls):
57  """Getter for the container of read formatters of a StorageInterface subclass.
58 
59  Returns
60  -------
61  dict
62  The read formatters container belonging to the class type.
63  """
64  try:
65  return cls._readFormattersDict
66  except AttributeError:
67  cls._readFormattersDict = {}
68  return cls._readFormattersDict
69 
70  @classmethod
71  def _writeFormatters(cls):
72  """Getter for the container of write formatters of a StorageInterface subclass.
73 
74  Returns
75  -------
76  dict
77  The write formatters container belonging to the class type.
78  """
79  try:
80  return cls._writeFormattersDict
81  except AttributeError:
82  cls._writeFormattersDict = {}
83  return cls._writeFormattersDict
84 
85  @classmethod
86  def getReadFormatter(cls, objType):
87  """Search in the registered formatters for the objType read formatter.
88 
89  Parameters
90  ----------
91  objType : class type
92  The type of class to find a formatter for.
93 
94  Returns
95  -------
96  formatter callable
97  The formatter callable used to read the object from the storageInterface.
98  """
99  return cls._readFormatters().get(objType, None)
100 
101  @classmethod
102  def getWriteFormatter(cls, objType):
103  """Search in the registered formatters for the objType write formatter.
104 
105  Parameters
106  ----------
107  objType : class type
108  The type of class to find a formatter for.
109 
110  Returns
111  -------
112  formatter callable
113  The formatter callable used to write the object to the storageInterface.
114  """
115  return cls._writeFormatters().get(objType, None)
116 
117  @classmethod
118  def registerFormatters(cls, formatable, readFormatter=None, writeFormatter=None):
119  """Register read and/or write formatters for a storageInterface subclass
120 
121  Parameters
122  ----------
123  cls : StorageInterface subclass
124  The type of StorageInterface the formatter is being registered for.
125  formatable : class object
126  The class object whose instances can be formatted by the formatter.
127  readFormatter : a read formatter callable
128  The formatter function that can be used by the StorageInterface instance to read the object from
129  the storage.
130  writeFormatter : a write formatter callable
131  The formatter function that can be used by the StorageInterface instance to write the object to
132  the storage.
133 
134  Raises
135  ------
136  RuntimeError
137  For each object type and StorageInterface subclass the read and write formatters should only be
138  registered once. If a second registration occurs for either a RuntimeError is raised.
139  """
140  def register(formatable, formatter, formatters, storageInterface):
141  if formatable in formatters:
142  raise RuntimeError(("Registration of second formatter {} for formattable {} in "
143  " storageInterface {}").format(formatter, formatable, storageInterface))
144  formatters[formatable] = formatter
145 
146  if readFormatter:
147  formatters = cls._readFormatters()
148  register(formatable, readFormatter, formatters, cls)
149  if writeFormatter:
150  formatters = cls._writeFormatters()
151  register(formatable, writeFormatter, formatters, cls)
152 
153  @abstractmethod
154  def write(self, butlerLocation, obj):
155  """Writes an object to a location and persistence format specified by ButlerLocation
156 
157  Parameters
158  ----------
159  butlerLocation : ButlerLocation
160  The location & formatting for the object to be written.
161  obj : object instance
162  The object to be written.
163  """
164 
165  @abstractmethod
166  def read(self, butlerLocation):
167  """Read from a butlerLocation.
168 
169  Parameters
170  ----------
171  butlerLocation : ButlerLocation
172  The location & formatting for the object(s) to be read.
173 
174  Returns
175  -------
176  A list of objects as described by the butler location. One item for
177  each location in butlerLocation.getLocations()
178  """
179 
180  @abstractmethod
181  def getLocalFile(self, path):
182  """Get a handle to a local copy of the file, downloading it to a
183  temporary if needed.
184 
185  Parameters
186  ----------
187  path : string
188  A path to the the file in storage, relative to root.
189 
190  Returns
191  -------
192  A handle to a local copy of the file. If storage is remote it will be
193  a temporary file. If storage is local it may be the original file or
194  a temporary file. The file name can be gotten via the 'name' property
195  of the returned object.
196  """
197 
198  @abstractmethod
199  def exists(self, location):
200  """Check if location exists.
201 
202  Parameters
203  ----------
204  location : ButlerLocation or string
205  A a string or a ButlerLocation that describes the location of an
206  object in this storage.
207 
208  Returns
209  -------
210  bool
211  True if exists, else False.
212  """
213 
214  @abstractmethod
215  def instanceSearch(self, path):
216  """Search for the given path in this storage instance.
217 
218  If the path contains an HDU indicator (a number in brackets before the
219  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
220  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
221  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
222 
223  Parameters
224  ----------
225  path : string
226  A filename (and optionally prefix path) to search for within root.
227 
228  Returns
229  -------
230  string or None
231  The location that was found, or None if no location was found.
232  """
233 
234  @classmethod
235  @abstractmethod
236  def search(cls, root, path):
237  """Look for the given path in the current root.
238 
239  Also supports searching for the path in Butler v1 repositories by
240  following the Butler v1 _parent symlink
241 
242  If the path contains an HDU indicator (a number in brackets, e.g.
243  'foo.fits[1]', this will be stripped when searching and so
244  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
245  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
246 
247  Parameters
248  ----------
249  root : string
250  The path to the root directory.
251  path : string
252  The path to the file within the root directory.
253 
254  Returns
255  -------
256  string or None
257  The location that was found, or None if no location was found.
258  """
259 
260  @abstractmethod
261  def copyFile(self, fromLocation, toLocation):
262  """Copy a file from one location to another on the local filesystem.
263 
264  Parameters
265  ----------
266  fromLocation : string
267  Path and name of existing file.
268  toLocation : string
269  Path and name of new file.
270 
271  Returns
272  -------
273  None
274  """
275 
276  @abstractmethod
277  def locationWithRoot(self, location):
278  """Get the full path to the location.
279 
280  Parameters
281  ----------
282  location : string
283  Path to a location within the repository relative to repository
284  root.
285 
286  Returns
287  -------
288  string
289  Absolute path to to the locaiton within the repository.
290  """
291 
292  @classmethod
293  @abstractmethod
294  def getRepositoryCfg(cls, uri):
295  """Get a persisted RepositoryCfg
296 
297  Parameters
298  ----------
299  uri : URI or path to a RepositoryCfg
300  Description
301 
302  Returns
303  -------
304  A RepositoryCfg instance or None
305  """
306 
307  @classmethod
308  @abstractmethod
309  def putRepositoryCfg(cls, cfg, loc=None):
310  """Serialize a RepositoryCfg to a location.
311 
312  When loc == cfg.root, the RepositoryCfg is to be written at the root
313  location of the repository. In that case, root is not written, it is
314  implicit in the location of the cfg. This allows the cfg to move from
315  machine to machine without modification.
316 
317  Parameters
318  ----------
319  cfg : RepositoryCfg instance
320  The RepositoryCfg to be serailized.
321  loc : string, optional
322  The URI location (can be relative path) to write the RepositoryCfg.
323  If loc is None, the location will be read from the root parameter
324  of loc.
325 
326  Returns
327  -------
328  None
329  """
330 
331  @classmethod
332  @abstractmethod
333  def getMapperClass(cls, root):
334  """Get the mapper class associated with a repository root.
335 
336  Parameters
337  ----------
338  root : string
339  The location of a persisted RepositoryCfg is (new style repos).
340 
341  Returns
342  -------
343  A class object or a class instance, depending on the state of the
344  mapper when the repository was created.
345  """
346 
347  # Optional: Only needs to work if relative paths are sensical on this
348  # storage type and for the case where fromPath and toPath are of the same
349  # storage type.
350  @classmethod
351  def relativePath(cls, fromPath, toPath):
352  """Get a relative path from a location to a location.
353 
354  Parameters
355  ----------
356  fromPath : string
357  A path at which to start. It can be a relative path or an
358  absolute path.
359  toPath : string
360  A target location. It can be a relative path or an absolute path.
361 
362  Returns
363  -------
364  string
365  A relative path that describes the path from fromPath to toPath.
366  """
367  return toPath
368 
369  # Optional: Only needs to work if relative paths and absolute paths are
370  # sensical on this storage type and for the case where fromPath and toPath
371  # are of the same storage type.
372  @classmethod
373  def absolutePath(cls, fromPath, relativePath):
374  """Get an absolute path for the path from fromUri to toUri
375 
376  Parameters
377  ----------
378  fromPath : the starting location
379  A location at which to start. It can be a relative path or an
380  absolute path.
381  relativePath : the location relative to fromPath
382  A relative path.
383 
384  Returns
385  -------
386  string
387  Path that is an absolute path representation of fromPath +
388  relativePath, if one exists. If relativePath is absolute or if
389  fromPath is not related to relativePath then relativePath will be
390  returned.
391  """
392  return relativePath
lsst::daf::persistence.storageInterface.StorageInterface.getMapperClass
def getMapperClass(cls, root)
Definition: storageInterface.py:333
lsst::daf::persistence.storageInterface.NoRepositroyAtRoot
Definition: storageInterface.py:27
lsst::daf::persistence.storageInterface.StorageInterface.write
def write(self, butlerLocation, obj)
Definition: storageInterface.py:154
lsst::daf::persistence.storageInterface.StorageInterface._writeFormattersDict
_writeFormattersDict
Definition: storageInterface.py:82
lsst::daf::persistence.storageInterface.StorageInterface.search
def search(cls, root, path)
Definition: storageInterface.py:236
lsst::daf::persistence.storageInterface.StorageInterface.copyFile
def copyFile(self, fromLocation, toLocation)
Definition: storageInterface.py:261
lsst::daf::persistence.storageInterface.StorageInterface.__init__
def __init__(self, uri, create)
Definition: storageInterface.py:51
lsst::daf::persistence.storageInterface.StorageInterface.putRepositoryCfg
def putRepositoryCfg(cls, cfg, loc=None)
Definition: storageInterface.py:309
lsst::daf::persistence.storageInterface.StorageInterface.getWriteFormatter
def getWriteFormatter(cls, objType)
Definition: storageInterface.py:102
lsst.pex.config.history.format
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
lsst::daf::persistence.storageInterface.StorageInterface.getLocalFile
def getLocalFile(self, path)
Definition: storageInterface.py:181
lsst::daf::persistence.storageInterface.StorageInterface.instanceSearch
def instanceSearch(self, path)
Definition: storageInterface.py:215
lsst::daf::persistence.storageInterface.StorageInterface.getReadFormatter
def getReadFormatter(cls, objType)
Definition: storageInterface.py:86
lsst::daf::persistence.storageInterface.StorageInterface._readFormatters
def _readFormatters(cls)
Definition: storageInterface.py:56
lsst::daf::persistence.storageInterface.StorageInterface.absolutePath
def absolutePath(cls, fromPath, relativePath)
Definition: storageInterface.py:373
lsst::daf::persistence.storageInterface.StorageInterface.getRepositoryCfg
def getRepositoryCfg(cls, uri)
Definition: storageInterface.py:294
lsst::daf::persistence.storageInterface.StorageInterface.read
def read(self, butlerLocation)
Definition: storageInterface.py:166
lsst::daf::persistence.storageInterface.StorageInterface.exists
def exists(self, location)
Definition: storageInterface.py:199
lsst::daf::persistence.storageInterface.StorageInterface.registerFormatters
def registerFormatters(cls, formatable, readFormatter=None, writeFormatter=None)
Definition: storageInterface.py:118
lsst::daf::persistence.storageInterface.StorageInterface.relativePath
def relativePath(cls, fromPath, toPath)
Definition: storageInterface.py:351
lsst.gdb.ip.diffim.printers.register
def register(obj)
Definition: printers.py:20
lsst::daf::persistence.storageInterface.StorageInterface.locationWithRoot
def locationWithRoot(self, location)
Definition: storageInterface.py:277
lsst::daf::persistence.storageInterface.StorageInterface._readFormattersDict
_readFormattersDict
Definition: storageInterface.py:67
lsst::daf::persistence.storageInterface.StorageInterface
Definition: storageInterface.py:31
lsst::daf::persistence.storageInterface.StorageInterface._writeFormatters
def _writeFormatters(cls)
Definition: storageInterface.py:71