LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
Public Member Functions | Static Public Member Functions | Public Attributes | List of all members
lsst.daf.persistence.posixStorage.PosixStorage Class Reference
Inheritance diagram for lsst.daf.persistence.posixStorage.PosixStorage:
lsst.daf.persistence.storageInterface.StorageInterface

Public Member Functions

def __init__ (self, uri, create)
 
def __repr__ (self)
 
def write (self, butlerLocation, obj)
 
def read (self, butlerLocation)
 
def butlerLocationExists (self, location)
 
def exists (self, location)
 
def locationWithRoot (self, location)
 
def copyFile (self, fromLocation, toLocation)
 
def getLocalFile (self, path)
 
def instanceSearch (self, path)
 
def getReadFormatter (cls, objType)
 
def getWriteFormatter (cls, objType)
 
def registerFormatters (cls, formatable, readFormatter=None, writeFormatter=None)
 
def getRepositoryCfg (cls, uri)
 
def putRepositoryCfg (cls, cfg, loc=None)
 
def getMapperClass (cls, root)
 
def relativePath (cls, fromPath, toPath)
 
def absolutePath (cls, fromPath, relativePath)
 

Static Public Member Functions

def relativePath (fromPath, toPath)
 
def absolutePath (fromPath, relativePath)
 
def getRepositoryCfg (uri)
 
def putRepositoryCfg (cfg, loc=None)
 
def getMapperClass (root)
 
def getParentSymlinkPath (root)
 
def v1RepoExists (root)
 
def search (root, path, searchParents=False)
 
def storageExists (uri)
 

Public Attributes

 log
 
 root
 

Detailed Description

Defines the interface for a storage location on the local filesystem.

Parameters
----------
uri : string
    URI or path that is used as the storage location.
create : bool
    If True a new repository will be created at the root location if it
    does not exist. If False then a new repository will not be created.

Raises
------
NoRepositroyAtRoot
    If create is False and a repository does not exist at the root
    specified by uri then NoRepositroyAtRoot is raised.

Definition at line 45 of file posixStorage.py.

Constructor & Destructor Documentation

◆ __init__()

def lsst.daf.persistence.posixStorage.PosixStorage.__init__ (   self,
  uri,
  create 
) 
initialzer

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 63 of file posixStorage.py.

63  def __init__(self, uri, create):
64  self.log = Log.getLogger("daf.persistence.butler")
65  self.root = self._pathFromURI(uri)
66  if self.root and not os.path.exists(self.root):
67  if not create:
68  raise NoRepositroyAtRoot("No repository at {}".format(uri))
69  safeMakeDir(self.root)
70 
lsst.pex.config.history.format
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174

Member Function Documentation

◆ __repr__()

def lsst.daf.persistence.posixStorage.PosixStorage.__repr__ (   self)

Definition at line 71 of file posixStorage.py.

71  def __repr__(self):
72  return 'PosixStorage(root=%s)' % self.root
73 

◆ absolutePath() [1/2]

def lsst.daf.persistence.storageInterface.StorageInterface.absolutePath (   cls,
  fromPath,
  relativePath 
)
inherited 
Get an absolute path for the path from fromUri to toUri

Parameters
----------
fromPath : the starting location
    A location at which to start. It can be a relative path or an
    absolute path.
relativePath : the location relative to fromPath
    A relative path.

Returns
-------
string
    Path that is an absolute path representation of fromPath +
    relativePath, if one exists. If relativePath is absolute or if
    fromPath is not related to relativePath then relativePath will be
    returned.

Definition at line 373 of file storageInterface.py.

373  def absolutePath(cls, fromPath, relativePath):
374  """Get an absolute path for the path from fromUri to toUri
375 
376  Parameters
377  ----------
378  fromPath : the starting location
379  A location at which to start. It can be a relative path or an
380  absolute path.
381  relativePath : the location relative to fromPath
382  A relative path.
383 
384  Returns
385  -------
386  string
387  Path that is an absolute path representation of fromPath +
388  relativePath, if one exists. If relativePath is absolute or if
389  fromPath is not related to relativePath then relativePath will be
390  returned.
391  """
392  return relativePath

◆ absolutePath() [2/2]

def lsst.daf.persistence.posixStorage.PosixStorage.absolutePath (   fromPath,
  relativePath 
)
static 
Get an absolute path for the path from fromUri to toUri

Parameters
----------
fromPath : the starting location
    A location at which to start. It can be a relative path or an
    absolute path.
relativePath : the location relative to fromPath
    A relative path.

Returns
-------
string
    Path that is an absolute path representation of fromPath +
    relativePath, if one exists. If relativePath is absolute or if
    fromPath is not related to relativePath then relativePath will be
    returned.

Definition at line 100 of file posixStorage.py.

100  def absolutePath(fromPath, relativePath):
101  """Get an absolute path for the path from fromUri to toUri
102 
103  Parameters
104  ----------
105  fromPath : the starting location
106  A location at which to start. It can be a relative path or an
107  absolute path.
108  relativePath : the location relative to fromPath
109  A relative path.
110 
111  Returns
112  -------
113  string
114  Path that is an absolute path representation of fromPath +
115  relativePath, if one exists. If relativePath is absolute or if
116  fromPath is not related to relativePath then relativePath will be
117  returned.
118  """
119  if os.path.isabs(relativePath):
120  return relativePath
121  fromPath = os.path.realpath(fromPath)
122  return os.path.normpath(os.path.join(fromPath, relativePath))
123 

◆ butlerLocationExists()

def lsst.daf.persistence.posixStorage.PosixStorage.butlerLocationExists (   self,
  location 
) 
Implementation of PosixStorage.exists for ButlerLocation objects.

Definition at line 283 of file posixStorage.py.

283  def butlerLocationExists(self, location):
284  """Implementation of PosixStorage.exists for ButlerLocation objects.
285  """
286  storageName = location.getStorageName()
287  if storageName not in ('FitsStorage',
288  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
289  'YamlStorage', 'ParquetStorage', 'MatplotlibStorage'):
290  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
291  return False
292  for locationString in location.getLocations():
293  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
294  obj = self.instanceSearch(path=logLoc)
295  if obj:
296  return True
297  return False
298 

◆ copyFile()

def lsst.daf.persistence.posixStorage.PosixStorage.copyFile (   self,
  fromLocation,
  toLocation 
) 
Copy a file from one location to another on the local filesystem.

Parameters
----------
fromLocation : path
    Path and name of existing file.
 toLocation : path
    Path and name of new file.

Returns
-------
None

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 351 of file posixStorage.py.

351  def copyFile(self, fromLocation, toLocation):
352  """Copy a file from one location to another on the local filesystem.
353 
354  Parameters
355  ----------
356  fromLocation : path
357  Path and name of existing file.
358  toLocation : path
359  Path and name of new file.
360 
361  Returns
362  -------
363  None
364  """
365  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
366 

◆ exists()

def lsst.daf.persistence.posixStorage.PosixStorage.exists (   self,
  location 
) 
Check if location exists.

Parameters
----------
location : ButlerLocation or string
    A a string or a ButlerLocation that describes the location of an
    object in this storage.

Returns
-------
bool
    True if exists, else False.

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 299 of file posixStorage.py.

299  def exists(self, location):
300  """Check if location exists.
301 
302  Parameters
303  ----------
304  location : ButlerLocation or string
305  A a string or a ButlerLocation that describes the location of an
306  object in this storage.
307 
308  Returns
309  -------
310  bool
311  True if exists, else False.
312  """
313  if isinstance(location, ButlerLocation):
314  return self.butlerLocationExists(location)
315 
316  obj = self.instanceSearch(path=location)
317  return bool(obj)
318 

◆ getLocalFile()

def lsst.daf.persistence.posixStorage.PosixStorage.getLocalFile (   self,
  path 
) 
Get a handle to a local copy of the file, downloading it to a
temporary if needed.

Parameters
----------
A path the the file in storage, relative to root.

Returns
-------
A handle to a local copy of the file. If storage is remote it will be
a temporary file. If storage is local it may be the original file or
a temporary file. The file name can be gotten via the 'name' property
of the returned object.

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 367 of file posixStorage.py.

367  def getLocalFile(self, path):
368  """Get a handle to a local copy of the file, downloading it to a
369  temporary if needed.
370 
371  Parameters
372  ----------
373  A path the the file in storage, relative to root.
374 
375  Returns
376  -------
377  A handle to a local copy of the file. If storage is remote it will be
378  a temporary file. If storage is local it may be the original file or
379  a temporary file. The file name can be gotten via the 'name' property
380  of the returned object.
381  """
382  p = os.path.join(self.root, path)
383  try:
384  return open(p)
385  except IOError as e:
386  if e.errno == 2: # 'No such file or directory'
387  return None
388  else:
389  raise e
390 

◆ getMapperClass() [1/2]

def lsst.daf.persistence.storageInterface.StorageInterface.getMapperClass (   cls,
  root 
)
inherited 
Get the mapper class associated with a repository root.

Parameters
----------
root : string
    The location of a persisted RepositoryCfg is (new style repos).

Returns
-------
A class object or a class instance, depending on the state of the
mapper when the repository was created.

Definition at line 333 of file storageInterface.py.

333  def getMapperClass(cls, root):
334  """Get the mapper class associated with a repository root.
335 
336  Parameters
337  ----------
338  root : string
339  The location of a persisted RepositoryCfg is (new style repos).
340 
341  Returns
342  -------
343  A class object or a class instance, depending on the state of the
344  mapper when the repository was created.
345  """
346 

◆ getMapperClass() [2/2]

def lsst.daf.persistence.posixStorage.PosixStorage.getMapperClass (   root)
static 
Get the mapper class associated with a repository root.

Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
new code and repositories; they should use the Repository parentCfg mechanism.

Parameters
----------
root : string
    The location of a persisted ReositoryCfg is (new style repos), or
    the location where a _mapper file is (old style repos).

Returns
-------
A class object or a class instance, depending on the state of the
mapper when the repository was created.

Definition at line 164 of file posixStorage.py.

164  def getMapperClass(root):
165  """Get the mapper class associated with a repository root.
166 
167  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
168  new code and repositories; they should use the Repository parentCfg mechanism.
169 
170  Parameters
171  ----------
172  root : string
173  The location of a persisted ReositoryCfg is (new style repos), or
174  the location where a _mapper file is (old style repos).
175 
176  Returns
177  -------
178  A class object or a class instance, depending on the state of the
179  mapper when the repository was created.
180  """
181  if not (root):
182  return None
183 
184  cfg = PosixStorage.getRepositoryCfg(root)
185  if cfg is not None:
186  return cfg.mapper
187 
188  # Find a "_mapper" file containing the mapper class name
189  basePath = root
190  mapperFile = "_mapper"
191  while not os.path.exists(os.path.join(basePath, mapperFile)):
192  # Break abstraction by following _parent links from CameraMapper
193  if os.path.exists(os.path.join(basePath, "_parent")):
194  basePath = os.path.join(basePath, "_parent")
195  else:
196  mapperFile = None
197  break
198 
199  if mapperFile is not None:
200  mapperFile = os.path.join(basePath, mapperFile)
201 
202  # Read the name of the mapper class and instantiate it
203  with open(mapperFile, "r") as f:
204  mapperName = f.readline().strip()
205  components = mapperName.split(".")
206  if len(components) <= 1:
207  raise RuntimeError("Unqualified mapper name %s in %s" %
208  (mapperName, mapperFile))
209  pkg = importlib.import_module(".".join(components[:-1]))
210  return getattr(pkg, components[-1])
211 
212  return None
213 
strip
bool strip
Definition: fits.cc:911

◆ getParentSymlinkPath()

def lsst.daf.persistence.posixStorage.PosixStorage.getParentSymlinkPath (   root)
static 
For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
symlink.

Parameters
----------
root : string
    A path to the folder on the local filesystem.

Returns
-------
string or None
    A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
    symlink at root.

Definition at line 215 of file posixStorage.py.

215  def getParentSymlinkPath(root):
216  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
217  symlink.
218 
219  Parameters
220  ----------
221  root : string
222  A path to the folder on the local filesystem.
223 
224  Returns
225  -------
226  string or None
227  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
228  symlink at root.
229  """
230  linkpath = os.path.join(root, '_parent')
231  if os.path.exists(linkpath):
232  try:
233  return os.readlink(os.path.join(root, '_parent'))
234  except OSError:
235  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
236  # location. Allow that; return the path of that folder.
237  return os.path.join(root, '_parent')
238  return None
239 

◆ getReadFormatter()

def lsst.daf.persistence.storageInterface.StorageInterface.getReadFormatter (   cls,
  objType 
)
inherited 
Search in the registered formatters for the objType read formatter.

Parameters
----------
objType : class type
    The type of class to find a formatter for.

Returns
-------
formatter callable
    The formatter callable used to read the object from the storageInterface.

Definition at line 86 of file storageInterface.py.

86  def getReadFormatter(cls, objType):
87  """Search in the registered formatters for the objType read formatter.
88 
89  Parameters
90  ----------
91  objType : class type
92  The type of class to find a formatter for.
93 
94  Returns
95  -------
96  formatter callable
97  The formatter callable used to read the object from the storageInterface.
98  """
99  return cls._readFormatters().get(objType, None)
100 

◆ getRepositoryCfg() [1/2]

def lsst.daf.persistence.storageInterface.StorageInterface.getRepositoryCfg (   cls,
  uri 
)
inherited 
Get a persisted RepositoryCfg

Parameters
----------
uri : URI or path to a RepositoryCfg
    Description

Returns
-------
A RepositoryCfg instance or None

Definition at line 294 of file storageInterface.py.

294  def getRepositoryCfg(cls, uri):
295  """Get a persisted RepositoryCfg
296 
297  Parameters
298  ----------
299  uri : URI or path to a RepositoryCfg
300  Description
301 
302  Returns
303  -------
304  A RepositoryCfg instance or None
305  """
306 

◆ getRepositoryCfg() [2/2]

def lsst.daf.persistence.posixStorage.PosixStorage.getRepositoryCfg (   uri)
static 
Get a persisted RepositoryCfg

Parameters
----------
uri : URI or path to a RepositoryCfg
    Description

Returns
-------
A RepositoryCfg instance or None

Definition at line 125 of file posixStorage.py.

125  def getRepositoryCfg(uri):
126  """Get a persisted RepositoryCfg
127 
128  Parameters
129  ----------
130  uri : URI or path to a RepositoryCfg
131  Description
132 
133  Returns
134  -------
135  A RepositoryCfg instance or None
136  """
137  storage = Storage.makeFromURI(uri)
138  location = ButlerLocation(pythonType=RepositoryCfg,
139  cppType=None,
140  storageName=None,
141  locationList='repositoryCfg.yaml',
142  dataId={},
143  mapper=None,
144  storage=storage,
145  usedDataId=None,
146  datasetType=None)
147  return storage.read(location)
148 

◆ getWriteFormatter()

def lsst.daf.persistence.storageInterface.StorageInterface.getWriteFormatter (   cls,
  objType 
)
inherited 
Search in the registered formatters for the objType write formatter.

Parameters
----------
objType : class type
    The type of class to find a formatter for.

Returns
-------
formatter callable
    The formatter callable used to write the object to the storageInterface.

Definition at line 102 of file storageInterface.py.

102  def getWriteFormatter(cls, objType):
103  """Search in the registered formatters for the objType write formatter.
104 
105  Parameters
106  ----------
107  objType : class type
108  The type of class to find a formatter for.
109 
110  Returns
111  -------
112  formatter callable
113  The formatter callable used to write the object to the storageInterface.
114  """
115  return cls._writeFormatters().get(objType, None)
116 

◆ instanceSearch()

def lsst.daf.persistence.posixStorage.PosixStorage.instanceSearch (   self,
  path 
) 
Search for the given path in this storage instance.

If the path contains an HDU indicator (a number in brackets before the
dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
will match filenames without the HDU indicator, e.g. 'foo.fits'. The
path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].

Parameters
----------
path : string
    A filename (and optionally prefix path) to search for within root.

Returns
-------
string or None
    The location that was found, or None if no location was found.

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 391 of file posixStorage.py.

391  def instanceSearch(self, path):
392  """Search for the given path in this storage instance.
393 
394  If the path contains an HDU indicator (a number in brackets before the
395  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
396  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
397  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
398 
399  Parameters
400  ----------
401  path : string
402  A filename (and optionally prefix path) to search for within root.
403 
404  Returns
405  -------
406  string or None
407  The location that was found, or None if no location was found.
408  """
409  return self.search(self.root, path)
410 

◆ locationWithRoot()

def lsst.daf.persistence.posixStorage.PosixStorage.locationWithRoot (   self,
  location 
) 
Get the full path to the location.

:param location:
:return:

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 319 of file posixStorage.py.

319  def locationWithRoot(self, location):
320  """Get the full path to the location.
321 
322  :param location:
323  :return:
324  """
325  return os.path.join(self.root, location)
326 

◆ putRepositoryCfg() [1/2]

def lsst.daf.persistence.posixStorage.PosixStorage.putRepositoryCfg (   cfg,
  loc = None 
)
static

Definition at line 150 of file posixStorage.py.

150  def putRepositoryCfg(cfg, loc=None):
151  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
152  location = ButlerLocation(pythonType=RepositoryCfg,
153  cppType=None,
154  storageName=None,
155  locationList='repositoryCfg.yaml',
156  dataId={},
157  mapper=None,
158  storage=storage,
159  usedDataId=None,
160  datasetType=None)
161  storage.write(location, cfg)
162 

◆ putRepositoryCfg() [2/2]

def lsst.daf.persistence.storageInterface.StorageInterface.putRepositoryCfg (   cls,
  cfg,
  loc = None 
)
inherited 
Serialize a RepositoryCfg to a location.

When loc == cfg.root, the RepositoryCfg is to be written at the root
location of the repository. In that case, root is not written, it is
implicit in the location of the cfg. This allows the cfg to move from
machine to machine without modification.

Parameters
----------
cfg : RepositoryCfg instance
    The RepositoryCfg to be serailized.
loc : string, optional
    The URI location (can be relative path) to write the RepositoryCfg.
    If loc is None, the location will be read from the root parameter
    of loc.

Returns
-------
None

Definition at line 309 of file storageInterface.py.

309  def putRepositoryCfg(cls, cfg, loc=None):
310  """Serialize a RepositoryCfg to a location.
311 
312  When loc == cfg.root, the RepositoryCfg is to be written at the root
313  location of the repository. In that case, root is not written, it is
314  implicit in the location of the cfg. This allows the cfg to move from
315  machine to machine without modification.
316 
317  Parameters
318  ----------
319  cfg : RepositoryCfg instance
320  The RepositoryCfg to be serailized.
321  loc : string, optional
322  The URI location (can be relative path) to write the RepositoryCfg.
323  If loc is None, the location will be read from the root parameter
324  of loc.
325 
326  Returns
327  -------
328  None
329  """
330 

◆ read()

def lsst.daf.persistence.posixStorage.PosixStorage.read (   self,
  butlerLocation 
) 
Read from a butlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location & formatting for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 262 of file posixStorage.py.

262  def read(self, butlerLocation):
263  """Read from a butlerLocation.
264 
265  Parameters
266  ----------
267  butlerLocation : ButlerLocation
268  The location & formatting for the object(s) to be read.
269 
270  Returns
271  -------
272  A list of objects as described by the butler location. One item for
273  each location in butlerLocation.getLocations()
274  """
275  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
276  if not readFormatter:
277  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
278  if readFormatter:
279  return readFormatter(butlerLocation)
280 
281  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
282 
read
std::shared_ptr< table::io::Persistable > read(table::io::InputArchive const &archive, table::io::CatalogVector const &catalogs) const override
Definition: warpExposure.cc:0

◆ registerFormatters()

def lsst.daf.persistence.storageInterface.StorageInterface.registerFormatters (   cls,
  formatable,
  readFormatter = None,
  writeFormatter = None 
)
inherited 
Register read and/or write formatters for a storageInterface subclass

Parameters
----------
cls : StorageInterface subclass
    The type of StorageInterface the formatter is being registered for.
formatable : class object
    The class object whose instances can be formatted by the formatter.
readFormatter : a read formatter callable
    The formatter function that can be used by the StorageInterface instance to read the object from
    the storage.
writeFormatter : a write formatter callable
    The formatter function that can be used by the StorageInterface instance to write the object to
    the storage.

Raises
------
RuntimeError
    For each object type and StorageInterface subclass the read and write formatters should only be
    registered once. If a second registration occurs for either a RuntimeError is raised.

Definition at line 118 of file storageInterface.py.

118  def registerFormatters(cls, formatable, readFormatter=None, writeFormatter=None):
119  """Register read and/or write formatters for a storageInterface subclass
120 
121  Parameters
122  ----------
123  cls : StorageInterface subclass
124  The type of StorageInterface the formatter is being registered for.
125  formatable : class object
126  The class object whose instances can be formatted by the formatter.
127  readFormatter : a read formatter callable
128  The formatter function that can be used by the StorageInterface instance to read the object from
129  the storage.
130  writeFormatter : a write formatter callable
131  The formatter function that can be used by the StorageInterface instance to write the object to
132  the storage.
133 
134  Raises
135  ------
136  RuntimeError
137  For each object type and StorageInterface subclass the read and write formatters should only be
138  registered once. If a second registration occurs for either a RuntimeError is raised.
139  """
140  def register(formatable, formatter, formatters, storageInterface):
141  if formatable in formatters:
142  raise RuntimeError(("Registration of second formatter {} for formattable {} in "
143  " storageInterface {}").format(formatter, formatable, storageInterface))
144  formatters[formatable] = formatter
145 
146  if readFormatter:
147  formatters = cls._readFormatters()
148  register(formatable, readFormatter, formatters, cls)
149  if writeFormatter:
150  formatters = cls._writeFormatters()
151  register(formatable, writeFormatter, formatters, cls)
152 

◆ relativePath() [1/2]

def lsst.daf.persistence.storageInterface.StorageInterface.relativePath (   cls,
  fromPath,
  toPath 
)
inherited 
Get a relative path from a location to a location.

Parameters
----------
fromPath : string
    A path at which to start. It can be a relative path or an
    absolute path.
toPath : string
    A target location. It can be a relative path or an absolute path.

Returns
-------
string
    A relative path that describes the path from fromPath to toPath.

Definition at line 351 of file storageInterface.py.

351  def relativePath(cls, fromPath, toPath):
352  """Get a relative path from a location to a location.
353 
354  Parameters
355  ----------
356  fromPath : string
357  A path at which to start. It can be a relative path or an
358  absolute path.
359  toPath : string
360  A target location. It can be a relative path or an absolute path.
361 
362  Returns
363  -------
364  string
365  A relative path that describes the path from fromPath to toPath.
366  """
367  return toPath
368 

◆ relativePath() [2/2]

def lsst.daf.persistence.posixStorage.PosixStorage.relativePath (   fromPath,
  toPath 
)
static 
Get a relative path from a location to a location.

Parameters
----------
fromPath : string
    A path at which to start. It can be a relative path or an
    absolute path.
toPath : string
    A target location. It can be a relative path or an absolute path.

Returns
-------
string
    A relative path that describes the path from fromPath to toPath.

Definition at line 80 of file posixStorage.py.

80  def relativePath(fromPath, toPath):
81  """Get a relative path from a location to a location.
82 
83  Parameters
84  ----------
85  fromPath : string
86  A path at which to start. It can be a relative path or an
87  absolute path.
88  toPath : string
89  A target location. It can be a relative path or an absolute path.
90 
91  Returns
92  -------
93  string
94  A relative path that describes the path from fromPath to toPath.
95  """
96  fromPath = os.path.realpath(fromPath)
97  return os.path.relpath(toPath, fromPath)
98 

◆ search()

def lsst.daf.persistence.posixStorage.PosixStorage.search (   root,
  path,
  searchParents = False 
)
static 
Look for the given path in the current root.

Also supports searching for the path in Butler v1 repositories by
following the Butler v1 _parent symlink

If the path contains an HDU indicator (a number in brackets, e.g.
'foo.fits[1]', this will be stripped when searching and so
will match filenames without the HDU indicator, e.g. 'foo.fits'. The
path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].

Parameters
----------
root : string
    The path to the root directory.
path : string
    The path to the file within the root directory.
searchParents : bool, optional
    For Butler v1 repositories only, if true and a _parent symlink
    exists, then the directory at _parent will be searched if the file
    is not found in the root repository. Will continue searching the
    parent of the parent until the file is found or no additional
    parent exists.

Returns
-------
string or None
    The location that was found, or None if no location was found.

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 412 of file posixStorage.py.

412  def search(root, path, searchParents=False):
413  """Look for the given path in the current root.
414 
415  Also supports searching for the path in Butler v1 repositories by
416  following the Butler v1 _parent symlink
417 
418  If the path contains an HDU indicator (a number in brackets, e.g.
419  'foo.fits[1]', this will be stripped when searching and so
420  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
421  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
422 
423  Parameters
424  ----------
425  root : string
426  The path to the root directory.
427  path : string
428  The path to the file within the root directory.
429  searchParents : bool, optional
430  For Butler v1 repositories only, if true and a _parent symlink
431  exists, then the directory at _parent will be searched if the file
432  is not found in the root repository. Will continue searching the
433  parent of the parent until the file is found or no additional
434  parent exists.
435 
436  Returns
437  -------
438  string or None
439  The location that was found, or None if no location was found.
440  """
441  # Separate path into a root-equivalent prefix (in dir) and the rest
442  # (left in path)
443  rootDir = root
444  # First remove trailing slashes (#2527)
445  while len(rootDir) > 1 and rootDir[-1] == '/':
446  rootDir = rootDir[:-1]
447 
448  if not path.startswith('/'):
449  # Most common case is a relative path from a template
450  pathPrefix = None
451  elif path.startswith(rootDir + "/"):
452  # Common case; we have the same root prefix string
453  path = path[len(rootDir + '/'):]
454  pathPrefix = rootDir
455  elif rootDir == "/" and path.startswith("/"):
456  path = path[1:]
457  pathPrefix = None
458  else:
459  # Search for prefix that is the same as root
460  pathPrefix = os.path.dirname(path)
461  while pathPrefix != "" and pathPrefix != "/":
462  if os.path.realpath(pathPrefix) == os.path.realpath(root):
463  break
464  pathPrefix = os.path.dirname(pathPrefix)
465  if pathPrefix == "/":
466  path = path[1:]
467  elif pathPrefix != "":
468  path = path[len(pathPrefix)+1:]
469 
470  # Now search for the path in the root or its parents
471  # Strip off any cfitsio bracketed extension if present
472  strippedPath = path
473  pathStripped = None
474  firstBracket = path.find("[")
475  if firstBracket != -1:
476  strippedPath = path[:firstBracket]
477  pathStripped = path[firstBracket:]
478 
479  dir = rootDir
480  while True:
481  paths = glob.glob(os.path.join(dir, strippedPath))
482  if len(paths) > 0:
483  if pathPrefix != rootDir:
484  paths = [p[len(rootDir+'/'):] for p in paths]
485  if pathStripped is not None:
486  paths = [p + pathStripped for p in paths]
487  return paths
488  if searchParents:
489  dir = os.path.join(dir, "_parent")
490  if not os.path.exists(dir):
491  return None
492  else:
493  return None
494 

◆ storageExists()

def lsst.daf.persistence.posixStorage.PosixStorage.storageExists (   uri)
static 
Ask if a storage at the location described by uri exists

Parameters
----------
root : string
    URI to the the root location of the storage

Returns
-------
bool
    True if the storage exists, false if not

Definition at line 496 of file posixStorage.py.

496  def storageExists(uri):
497  """Ask if a storage at the location described by uri exists
498 
499  Parameters
500  ----------
501  root : string
502  URI to the the root location of the storage
503 
504  Returns
505  -------
506  bool
507  True if the storage exists, false if not
508  """
509  return os.path.exists(PosixStorage._pathFromURI(uri))
510 
511 

◆ v1RepoExists()

def lsst.daf.persistence.posixStorage.PosixStorage.v1RepoExists (   root)
static 
Test if a Version 1 Repository exists.

Version 1 Repositories only exist in posix storages, do not have a
RepositoryCfg file, and contain either a registry.sqlite3 file, a
_mapper file, or a _parent link.

Parameters
----------
root : string
    A path to a folder on the local filesystem.

Returns
-------
bool
    True if the repository at root exists, else False.

Definition at line 328 of file posixStorage.py.

328  def v1RepoExists(root):
329  """Test if a Version 1 Repository exists.
330 
331  Version 1 Repositories only exist in posix storages, do not have a
332  RepositoryCfg file, and contain either a registry.sqlite3 file, a
333  _mapper file, or a _parent link.
334 
335  Parameters
336  ----------
337  root : string
338  A path to a folder on the local filesystem.
339 
340  Returns
341  -------
342  bool
343  True if the repository at root exists, else False.
344  """
345  return os.path.exists(root) and (
346  os.path.exists(os.path.join(root, "registry.sqlite3"))
347  or os.path.exists(os.path.join(root, "_mapper"))
348  or os.path.exists(os.path.join(root, "_parent"))
349  )
350 

◆ write()

def lsst.daf.persistence.posixStorage.PosixStorage.write (   self,
  butlerLocation,
  obj 
) 
Writes an object to a location and persistence format specified by
ButlerLocation

Parameters
----------
butlerLocation : ButlerLocation
    The location & formatting for the object to be written.
obj : object instance
    The object to be written.

Reimplemented from lsst.daf.persistence.storageInterface.StorageInterface.

Definition at line 240 of file posixStorage.py.

240  def write(self, butlerLocation, obj):
241  """Writes an object to a location and persistence format specified by
242  ButlerLocation
243 
244  Parameters
245  ----------
246  butlerLocation : ButlerLocation
247  The location & formatting for the object to be written.
248  obj : object instance
249  The object to be written.
250  """
251  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
252 
253  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
254  if not writeFormatter:
255  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
256  if writeFormatter:
257  writeFormatter(butlerLocation, obj)
258  return
259 
260  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
261 
lsst::afw::image::write
void write(OutputArchiveHandle &handle) const override

Member Data Documentation

◆ log

lsst.daf.persistence.posixStorage.PosixStorage.log

Definition at line 64 of file posixStorage.py.

◆ root

lsst.daf.persistence.posixStorage.PosixStorage.root

Definition at line 65 of file posixStorage.py.

The documentation for this class was generated from the following file:
Generated on Fri Nov 26 2021 10:16:18 for LSST Applications by doxygen 1.9.1