LSSTApplications  17.0+11,17.0+34,17.0+56,17.0+57,17.0+59,17.0+7,17.0-1-g377950a+33,17.0.1-1-g114240f+2,17.0.1-1-g4d4fbc4+28,17.0.1-1-g55520dc+49,17.0.1-1-g5f4ed7e+52,17.0.1-1-g6dd7d69+17,17.0.1-1-g8de6c91+11,17.0.1-1-gb9095d2+7,17.0.1-1-ge9fec5e+5,17.0.1-1-gf4e0155+55,17.0.1-1-gfc65f5f+50,17.0.1-1-gfc6fb1f+20,17.0.1-10-g87f9f3f+1,17.0.1-11-ge9de802+16,17.0.1-16-ga14f7d5c+4,17.0.1-17-gc79d625+1,17.0.1-17-gdae4c4a+8,17.0.1-2-g26618f5+29,17.0.1-2-g54f2ebc+9,17.0.1-2-gf403422+1,17.0.1-20-g2ca2f74+6,17.0.1-23-gf3eadeb7+1,17.0.1-3-g7e86b59+39,17.0.1-3-gb5ca14a,17.0.1-3-gd08d533+40,17.0.1-30-g596af8797,17.0.1-4-g59d126d+4,17.0.1-4-gc69c472+5,17.0.1-6-g5afd9b9+4,17.0.1-7-g35889ee+1,17.0.1-7-gc7c8782+18,17.0.1-9-gc4bbfb2+3,w.2019.22
LSSTDataManagementBasePackage
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import re
30 import urllib.parse
31 import glob
32 import shutil
33 import yaml
34 
35 from . import (LogicalLocation, Policy,
36  StorageInterface, Storage, ButlerLocation,
37  NoRepositroyAtRoot, RepositoryCfg, doImport)
38 from lsst.log import Log
39 import lsst.pex.policy as pexPolicy
40 from .safeFileIo import SafeFilename, safeMakeDir
41 
42 
43 __all__ = ["PosixStorage"]
44 
45 
47  """Defines the interface for a storage location on the local filesystem.
48 
49  Parameters
50  ----------
51  uri : string
52  URI or path that is used as the storage location.
53  create : bool
54  If True a new repository will be created at the root location if it
55  does not exist. If False then a new repository will not be created.
56 
57  Raises
58  ------
59  NoRepositroyAtRoot
60  If create is False and a repository does not exist at the root
61  specified by uri then NoRepositroyAtRoot is raised.
62  """
63 
64  def __init__(self, uri, create):
65  self.log = Log.getLogger("daf.persistence.butler")
66  self.root = self._pathFromURI(uri)
67  if self.root and not os.path.exists(self.root):
68  if not create:
69  raise NoRepositroyAtRoot("No repository at {}".format(uri))
70  safeMakeDir(self.root)
71 
72  def __repr__(self):
73  return 'PosixStorage(root=%s)' % self.root
74 
75  @staticmethod
76  def _pathFromURI(uri):
77  """Get the path part of the URI"""
78  return urllib.parse.urlparse(uri).path
79 
80  @staticmethod
81  def relativePath(fromPath, toPath):
82  """Get a relative path from a location to a location.
83 
84  Parameters
85  ----------
86  fromPath : string
87  A path at which to start. It can be a relative path or an
88  absolute path.
89  toPath : string
90  A target location. It can be a relative path or an absolute path.
91 
92  Returns
93  -------
94  string
95  A relative path that describes the path from fromPath to toPath.
96  """
97  fromPath = os.path.realpath(fromPath)
98  return os.path.relpath(toPath, fromPath)
99 
100  @staticmethod
101  def absolutePath(fromPath, relativePath):
102  """Get an absolute path for the path from fromUri to toUri
103 
104  Parameters
105  ----------
106  fromPath : the starting location
107  A location at which to start. It can be a relative path or an
108  absolute path.
109  relativePath : the location relative to fromPath
110  A relative path.
111 
112  Returns
113  -------
114  string
115  Path that is an absolute path representation of fromPath +
116  relativePath, if one exists. If relativePath is absolute or if
117  fromPath is not related to relativePath then relativePath will be
118  returned.
119  """
120  if os.path.isabs(relativePath):
121  return relativePath
122  fromPath = os.path.realpath(fromPath)
123  return os.path.normpath(os.path.join(fromPath, relativePath))
124 
125  @staticmethod
127  """Get a persisted RepositoryCfg
128 
129  Parameters
130  ----------
131  uri : URI or path to a RepositoryCfg
132  Description
133 
134  Returns
135  -------
136  A RepositoryCfg instance or None
137  """
138  storage = Storage.makeFromURI(uri)
139  location = ButlerLocation(pythonType=RepositoryCfg,
140  cppType=None,
141  storageName=None,
142  locationList='repositoryCfg.yaml',
143  dataId={},
144  mapper=None,
145  storage=storage,
146  usedDataId=None,
147  datasetType=None)
148  return storage.read(location)
149 
150  @staticmethod
151  def putRepositoryCfg(cfg, loc=None):
152  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
153  location = ButlerLocation(pythonType=RepositoryCfg,
154  cppType=None,
155  storageName=None,
156  locationList='repositoryCfg.yaml',
157  dataId={},
158  mapper=None,
159  storage=storage,
160  usedDataId=None,
161  datasetType=None)
162  storage.write(location, cfg)
163 
164  @staticmethod
165  def getMapperClass(root):
166  """Get the mapper class associated with a repository root.
167 
168  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
169  new code and repositories; they should use the Repository parentCfg mechanism.
170 
171  Parameters
172  ----------
173  root : string
174  The location of a persisted ReositoryCfg is (new style repos), or
175  the location where a _mapper file is (old style repos).
176 
177  Returns
178  -------
179  A class object or a class instance, depending on the state of the
180  mapper when the repository was created.
181  """
182  if not (root):
183  return None
184 
185  cfg = PosixStorage.getRepositoryCfg(root)
186  if cfg is not None:
187  return cfg.mapper
188 
189  # Find a "_mapper" file containing the mapper class name
190  basePath = root
191  mapperFile = "_mapper"
192  while not os.path.exists(os.path.join(basePath, mapperFile)):
193  # Break abstraction by following _parent links from CameraMapper
194  if os.path.exists(os.path.join(basePath, "_parent")):
195  basePath = os.path.join(basePath, "_parent")
196  else:
197  mapperFile = None
198  break
199 
200  if mapperFile is not None:
201  mapperFile = os.path.join(basePath, mapperFile)
202 
203  # Read the name of the mapper class and instantiate it
204  with open(mapperFile, "r") as f:
205  mapperName = f.readline().strip()
206  components = mapperName.split(".")
207  if len(components) <= 1:
208  raise RuntimeError("Unqualified mapper name %s in %s" %
209  (mapperName, mapperFile))
210  pkg = importlib.import_module(".".join(components[:-1]))
211  return getattr(pkg, components[-1])
212 
213  return None
214 
215  @staticmethod
217  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
218  symlink.
219 
220  Parameters
221  ----------
222  root : string
223  A path to the folder on the local filesystem.
224 
225  Returns
226  -------
227  string or None
228  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
229  symlink at root.
230  """
231  linkpath = os.path.join(root, '_parent')
232  if os.path.exists(linkpath):
233  try:
234  return os.readlink(os.path.join(root, '_parent'))
235  except OSError:
236  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
237  # location. Allow that; return the path of that folder.
238  return os.path.join(root, '_parent')
239  return None
240 
241  def write(self, butlerLocation, obj):
242  """Writes an object to a location and persistence format specified by
243  ButlerLocation
244 
245  Parameters
246  ----------
247  butlerLocation : ButlerLocation
248  The location & formatting for the object to be written.
249  obj : object instance
250  The object to be written.
251  """
252  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
253 
254  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
255  if not writeFormatter:
256  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
257  if writeFormatter:
258  writeFormatter(butlerLocation, obj)
259  return
260 
261  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
262 
263  def read(self, butlerLocation):
264  """Read from a butlerLocation.
265 
266  Parameters
267  ----------
268  butlerLocation : ButlerLocation
269  The location & formatting for the object(s) to be read.
270 
271  Returns
272  -------
273  A list of objects as described by the butler location. One item for
274  each location in butlerLocation.getLocations()
275  """
276  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
277  if not readFormatter:
278  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
279  if readFormatter:
280  return readFormatter(butlerLocation)
281 
282  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
283 
284  def butlerLocationExists(self, location):
285  """Implementation of PosixStorage.exists for ButlerLocation objects.
286  """
287  storageName = location.getStorageName()
288  if storageName not in ('FitsStorage', 'PafStorage',
289  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
290  'YamlStorage', 'ParquetStorage', 'MatplotlibStorage'):
291  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
292  return False
293  for locationString in location.getLocations():
294  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
295  obj = self.instanceSearch(path=logLoc)
296  if obj:
297  return True
298  return False
299 
300  def exists(self, location):
301  """Check if location exists.
302 
303  Parameters
304  ----------
305  location : ButlerLocation or string
306  A a string or a ButlerLocation that describes the location of an
307  object in this storage.
308 
309  Returns
310  -------
311  bool
312  True if exists, else False.
313  """
314  if isinstance(location, ButlerLocation):
315  return self.butlerLocationExists(location)
316 
317  obj = self.instanceSearch(path=location)
318  return bool(obj)
319 
320  def locationWithRoot(self, location):
321  """Get the full path to the location.
322 
323  :param location:
324  :return:
325  """
326  return os.path.join(self.root, location)
327 
328  @staticmethod
329  def v1RepoExists(root):
330  """Test if a Version 1 Repository exists.
331 
332  Version 1 Repositories only exist in posix storages, do not have a
333  RepositoryCfg file, and contain either a registry.sqlite3 file, a
334  _mapper file, or a _parent link.
335 
336  Parameters
337  ----------
338  root : string
339  A path to a folder on the local filesystem.
340 
341  Returns
342  -------
343  bool
344  True if the repository at root exists, else False.
345  """
346  return os.path.exists(root) and (
347  os.path.exists(os.path.join(root, "registry.sqlite3")) or
348  os.path.exists(os.path.join(root, "_mapper")) or
349  os.path.exists(os.path.join(root, "_parent"))
350  )
351 
352  def copyFile(self, fromLocation, toLocation):
353  """Copy a file from one location to another on the local filesystem.
354 
355  Parameters
356  ----------
357  fromLocation : path
358  Path and name of existing file.
359  toLocation : path
360  Path and name of new file.
361 
362  Returns
363  -------
364  None
365  """
366  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
367 
368  def getLocalFile(self, path):
369  """Get a handle to a local copy of the file, downloading it to a
370  temporary if needed.
371 
372  Parameters
373  ----------
374  A path the the file in storage, relative to root.
375 
376  Returns
377  -------
378  A handle to a local copy of the file. If storage is remote it will be
379  a temporary file. If storage is local it may be the original file or
380  a temporary file. The file name can be gotten via the 'name' property
381  of the returned object.
382  """
383  p = os.path.join(self.root, path)
384  try:
385  return open(p)
386  except IOError as e:
387  if e.errno == 2: # 'No such file or directory'
388  return None
389  else:
390  raise e
391 
392  def instanceSearch(self, path):
393  """Search for the given path in this storage instance.
394 
395  If the path contains an HDU indicator (a number in brackets before the
396  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
397  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
398  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
399 
400  Parameters
401  ----------
402  path : string
403  A filename (and optionally prefix path) to search for within root.
404 
405  Returns
406  -------
407  string or None
408  The location that was found, or None if no location was found.
409  """
410  return self.search(self.root, path)
411 
412  @staticmethod
413  def search(root, path, searchParents=False):
414  """Look for the given path in the current root.
415 
416  Also supports searching for the path in Butler v1 repositories by
417  following the Butler v1 _parent symlink
418 
419  If the path contains an HDU indicator (a number in brackets, e.g.
420  'foo.fits[1]', this will be stripped when searching and so
421  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
422  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
423 
424  Parameters
425  ----------
426  root : string
427  The path to the root directory.
428  path : string
429  The path to the file within the root directory.
430  searchParents : bool, optional
431  For Butler v1 repositories only, if true and a _parent symlink
432  exists, then the directory at _parent will be searched if the file
433  is not found in the root repository. Will continue searching the
434  parent of the parent until the file is found or no additional
435  parent exists.
436 
437  Returns
438  -------
439  string or None
440  The location that was found, or None if no location was found.
441  """
442  # Separate path into a root-equivalent prefix (in dir) and the rest
443  # (left in path)
444  rootDir = root
445  # First remove trailing slashes (#2527)
446  while len(rootDir) > 1 and rootDir[-1] == '/':
447  rootDir = rootDir[:-1]
448 
449  if path.startswith(rootDir + "/"):
450  # Common case; we have the same root prefix string
451  path = path[len(rootDir + '/'):]
452  pathPrefix = rootDir
453  elif rootDir == "/" and path.startswith("/"):
454  path = path[1:]
455  pathPrefix = None
456  else:
457  # Search for prefix that is the same as root
458  pathPrefix = os.path.dirname(path)
459  while pathPrefix != "" and pathPrefix != "/":
460  if os.path.realpath(pathPrefix) == os.path.realpath(root):
461  break
462  pathPrefix = os.path.dirname(pathPrefix)
463  if pathPrefix == "/":
464  path = path[1:]
465  elif pathPrefix != "":
466  path = path[len(pathPrefix)+1:]
467 
468  # Now search for the path in the root or its parents
469  # Strip off any cfitsio bracketed extension if present
470  strippedPath = path
471  pathStripped = None
472  firstBracket = path.find("[")
473  if firstBracket != -1:
474  strippedPath = path[:firstBracket]
475  pathStripped = path[firstBracket:]
476 
477  dir = rootDir
478  while True:
479  paths = glob.glob(os.path.join(dir, strippedPath))
480  if len(paths) > 0:
481  if pathPrefix != rootDir:
482  paths = [p[len(rootDir+'/'):] for p in paths]
483  if pathStripped is not None:
484  paths = [p + pathStripped for p in paths]
485  return paths
486  if searchParents:
487  dir = os.path.join(dir, "_parent")
488  if not os.path.exists(dir):
489  return None
490  else:
491  return None
492 
493  @staticmethod
494  def storageExists(uri):
495  """Ask if a storage at the location described by uri exists
496 
497  Parameters
498  ----------
499  root : string
500  URI to the the root location of the storage
501 
502  Returns
503  -------
504  bool
505  True if the storage exists, false if not
506  """
507  return os.path.exists(PosixStorage._pathFromURI(uri))
508 
509 
510 def readConfigStorage(butlerLocation):
511  """Read an lsst.pex.config.Config from a butlerLocation.
512 
513  Parameters
514  ----------
515  butlerLocation : ButlerLocation
516  The location for the object(s) to be read.
517 
518  Returns
519  -------
520  A list of objects as described by the butler location. One item for
521  each location in butlerLocation.getLocations()
522  """
523  results = []
524  for locationString in butlerLocation.getLocations():
525  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
526  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
527  if not os.path.exists(logLoc.locString()):
528  raise RuntimeError("No such config file: " + logLoc.locString())
529  pythonType = butlerLocation.getPythonType()
530  if pythonType is not None:
531  if isinstance(pythonType, basestring):
532  pythonType = doImport(pythonType)
533  finalItem = pythonType()
534  finalItem.load(logLoc.locString())
535  results.append(finalItem)
536  return results
537 
538 
539 def writeConfigStorage(butlerLocation, obj):
540  """Writes an lsst.pex.config.Config object to a location specified by
541  ButlerLocation.
542 
543  Parameters
544  ----------
545  butlerLocation : ButlerLocation
546  The location for the object to be written.
547  obj : object instance
548  The object to be written.
549  """
550  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
551  with SafeFilename(filename) as locationString:
552  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
553  obj.save(logLoc.locString())
554 
555 
556 def readFitsStorage(butlerLocation):
557  """Read objects from a FITS file specified by ButlerLocation.
558 
559  The object is read using class or static method
560  ``readFitsWithOptions(path, options)``, if it exists, else
561  ``readFits(path)``. The ``options`` argument is the data returned by
562  ``butlerLocation.getAdditionalData()``.
563 
564  Parameters
565  ----------
566  butlerLocation : ButlerLocation
567  The location for the object(s) to be read.
568 
569  Returns
570  -------
571  A list of objects as described by the butler location. One item for
572  each location in butlerLocation.getLocations()
573  """
574  pythonType = butlerLocation.getPythonType()
575  if pythonType is not None:
576  if isinstance(pythonType, basestring):
577  pythonType = doImport(pythonType)
578  supportsOptions = hasattr(pythonType, "readFitsWithOptions")
579  if not supportsOptions:
580  from lsst.daf.base import PropertySet, PropertyList
581  if issubclass(pythonType, (PropertySet, PropertyList)):
582  from lsst.afw.image import readMetadata
583  reader = readMetadata
584  else:
585  reader = pythonType.readFits
586  results = []
587  additionalData = butlerLocation.getAdditionalData()
588  for locationString in butlerLocation.getLocations():
589  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
590  logLoc = LogicalLocation(locStringWithRoot, additionalData)
591  # test for existence of file, ignoring trailing [...]
592  # because that can specify the HDU or other information
593  filePath = re.sub(r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$", r"\1", logLoc.locString())
594  if not os.path.exists(filePath):
595  raise RuntimeError("No such FITS file: " + logLoc.locString())
596  if supportsOptions:
597  finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
598  else:
599  fileName = logLoc.locString()
600  mat = re.search(r"^(.*)\[(\d+)\]$", fileName)
601 
602  if mat and reader == readMetadata: # readMetadata() only understands the hdu argument, not [hdu]
603  fileName = mat.group(1)
604  hdu = int(mat.group(2))
605 
606  finalItem = reader(fileName, hdu=hdu)
607  else:
608  finalItem = reader(fileName)
609  results.append(finalItem)
610  return results
611 
612 
613 def writeFitsStorage(butlerLocation, obj):
614  """Writes an object to a FITS file specified by ButlerLocation.
615 
616  The object is written using method
617  ``writeFitsWithOptions(path, options)``, if it exists, else
618  ``writeFits(path)``. The ``options`` argument is the data returned by
619  ``butlerLocation.getAdditionalData()``.
620 
621  Parameters
622  ----------
623  butlerLocation : ButlerLocation
624  The location for the object to be written.
625  obj : object instance
626  The object to be written.
627  """
628  supportsOptions = hasattr(obj, "writeFitsWithOptions")
629  additionalData = butlerLocation.getAdditionalData()
630  locations = butlerLocation.getLocations()
631  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
632  logLoc = LogicalLocation(locationString, additionalData)
633  if supportsOptions:
634  obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
635  else:
636  obj.writeFits(logLoc.locString())
637 
638 
639 def readParquetStorage(butlerLocation):
640  """Read a catalog from a Parquet file specified by ButlerLocation.
641 
642  The object returned by this is expected to be a subtype
643  of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
644  that allows for lazy loading of the data.
645 
646  Parameters
647  ----------
648  butlerLocation : ButlerLocation
649  The location for the object(s) to be read.
650 
651  Returns
652  -------
653  A list of objects as described by the butler location. One item for
654  each location in butlerLocation.getLocations()
655  """
656  results = []
657  additionalData = butlerLocation.getAdditionalData()
658 
659  for locationString in butlerLocation.getLocations():
660  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
661  logLoc = LogicalLocation(locStringWithRoot, additionalData)
662  if not os.path.exists(logLoc.locString()):
663  raise RuntimeError("No such parquet file: " + logLoc.locString())
664 
665  pythonType = butlerLocation.getPythonType()
666  if pythonType is not None:
667  if isinstance(pythonType, basestring):
668  pythonType = doImport(pythonType)
669 
670  filename = logLoc.locString()
671 
672  # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
673  # filename should be the first kwarg, but being explicit here.
674  results.append(pythonType(filename=filename))
675 
676  return results
677 
678 
679 def writeParquetStorage(butlerLocation, obj):
680  """Writes pandas dataframe to parquet file.
681 
682  Parameters
683  ----------
684  butlerLocation : ButlerLocation
685  The location for the object(s) to be read.
686  obj : `lsst.qa.explorer.parquetTable.ParquetTable`
687  Wrapped DataFrame to write.
688 
689  """
690  additionalData = butlerLocation.getAdditionalData()
691  locations = butlerLocation.getLocations()
692  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
693  logLoc = LogicalLocation(locationString, additionalData)
694  filename = logLoc.locString()
695  obj.write(filename)
696 
697 
698 def writeYamlStorage(butlerLocation, obj):
699  """Writes an object to a YAML file specified by ButlerLocation.
700 
701  Parameters
702  ----------
703  butlerLocation : ButlerLocation
704  The location for the object to be written.
705  obj : object instance
706  The object to be written.
707  """
708  additionalData = butlerLocation.getAdditionalData()
709  locations = butlerLocation.getLocations()
710  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
711  logLoc = LogicalLocation(locationString, additionalData)
712  with open(logLoc.locString(), "w") as outfile:
713  yaml.dump(obj, outfile)
714 
715 
716 def readPickleStorage(butlerLocation):
717  """Read an object from a pickle file specified by ButlerLocation.
718 
719  Parameters
720  ----------
721  butlerLocation : ButlerLocation
722  The location for the object(s) to be read.
723 
724  Returns
725  -------
726  A list of objects as described by the butler location. One item for
727  each location in butlerLocation.getLocations()
728  """
729  # Create a list of Storages for the item.
730  results = []
731  additionalData = butlerLocation.getAdditionalData()
732  for locationString in butlerLocation.getLocations():
733  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
734  logLoc = LogicalLocation(locStringWithRoot, additionalData)
735  if not os.path.exists(logLoc.locString()):
736  raise RuntimeError("No such pickle file: " + logLoc.locString())
737  with open(logLoc.locString(), "rb") as infile:
738  # py3: We have to specify encoding since some files were written
739  # by python2, and 'latin1' manages that conversion safely. See:
740  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
741  if sys.version_info.major >= 3:
742  finalItem = pickle.load(infile, encoding="latin1")
743  else:
744  finalItem = pickle.load(infile)
745  results.append(finalItem)
746  return results
747 
748 
749 def writePickleStorage(butlerLocation, obj):
750  """Writes an object to a pickle file specified by ButlerLocation.
751 
752  Parameters
753  ----------
754  butlerLocation : ButlerLocation
755  The location for the object to be written.
756  obj : object instance
757  The object to be written.
758  """
759  additionalData = butlerLocation.getAdditionalData()
760  locations = butlerLocation.getLocations()
761  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
762  logLoc = LogicalLocation(locationString, additionalData)
763  with open(logLoc.locString(), "wb") as outfile:
764  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
765 
766 
767 def readFitsCatalogStorage(butlerLocation):
768  """Read a catalog from a FITS table specified by ButlerLocation.
769 
770  Parameters
771  ----------
772  butlerLocation : ButlerLocation
773  The location for the object(s) to be read.
774 
775  Returns
776  -------
777  A list of objects as described by the butler location. One item for
778  each location in butlerLocation.getLocations()
779  """
780  pythonType = butlerLocation.getPythonType()
781  if pythonType is not None:
782  if isinstance(pythonType, basestring):
783  pythonType = doImport(pythonType)
784  results = []
785  additionalData = butlerLocation.getAdditionalData()
786  for locationString in butlerLocation.getLocations():
787  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
788  logLoc = LogicalLocation(locStringWithRoot, additionalData)
789  if not os.path.exists(logLoc.locString()):
790  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
791  kwds = {}
792  if additionalData.exists("hdu"):
793  kwds["hdu"] = additionalData.getInt("hdu")
794  if additionalData.exists("flags"):
795  kwds["flags"] = additionalData.getInt("flags")
796  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
797  results.append(finalItem)
798  return results
799 
800 
801 def writeFitsCatalogStorage(butlerLocation, obj):
802  """Writes a catalog to a FITS table specified by ButlerLocation.
803 
804  Parameters
805  ----------
806  butlerLocation : ButlerLocation
807  The location for the object to be written.
808  obj : object instance
809  The object to be written.
810  """
811  additionalData = butlerLocation.getAdditionalData()
812  locations = butlerLocation.getLocations()
813  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
814  logLoc = LogicalLocation(locationString, additionalData)
815  if additionalData.exists("flags"):
816  kwds = dict(flags=additionalData.getInt("flags"))
817  else:
818  kwds = {}
819  obj.writeFits(logLoc.locString(), **kwds)
820 
821 
822 def readMatplotlibStorage(butlerLocation):
823  """Read from a butlerLocation (always fails for this storage type).
824 
825  Parameters
826  ----------
827  butlerLocation : ButlerLocation
828  The location for the object(s) to be read.
829 
830  Returns
831  -------
832  A list of objects as described by the butler location. One item for
833  each location in butlerLocation.getLocations()
834  """
835  raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
836 
837 
838 def writeMatplotlibStorage(butlerLocation, obj):
839  """Writes a matplotlib.figure.Figure to a location, using the template's
840  filename suffix to infer the file format.
841 
842  Parameters
843  ----------
844  butlerLocation : ButlerLocation
845  The location for the object to be written.
846  obj : matplotlib.figure.Figure
847  The object to be written.
848  """
849  additionalData = butlerLocation.getAdditionalData()
850  locations = butlerLocation.getLocations()
851  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
852  logLoc = LogicalLocation(locationString, additionalData)
853  # SafeFilename appends a random suffix, which corrupts the extension
854  # matplotlib uses to guess the file format.
855  # Instead, we extract the extension from the original location
856  # and pass that as the format directly.
857  _, ext = os.path.splitext(locations[0])
858  if ext:
859  ext = ext[1:] # strip off leading '.'
860  else:
861  # If there is no extension, we let matplotlib fall back to its
862  # default.
863  ext = None
864  obj.savefig(logLoc.locString(), format=ext)
865 
866 
867 def readPafStorage(butlerLocation):
868  """Read a policy from a PAF file specified by a ButlerLocation.
869 
870  Parameters
871  ----------
872  butlerLocation : ButlerLocation
873  The location for the object(s) to be read.
874 
875  Returns
876  -------
877  A list of objects as described by the butler location. One item for
878  each location in butlerLocation.getLocations()
879  """
880  results = []
881  for locationString in butlerLocation.getLocations():
882  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
883  butlerLocation.getAdditionalData())
884  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
885  results.append(finalItem)
886  return results
887 
888 
889 def readYamlStorage(butlerLocation):
890  """Read an object from a YAML file specified by a butlerLocation.
891 
892  Parameters
893  ----------
894  butlerLocation : ButlerLocation
895  The location for the object(s) to be read.
896 
897  Returns
898  -------
899  A list of objects as described by the butler location. One item for
900  each location in butlerLocation.getLocations()
901  """
902  results = []
903  for locationString in butlerLocation.getLocations():
904  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
905  butlerLocation.getAdditionalData())
906  if not os.path.exists(logLoc.locString()):
907  raise RuntimeError("No such YAML file: " + logLoc.locString())
908  # Butler Gen2 repository configurations are handled specially
909  if butlerLocation.pythonType == 'lsst.daf.persistence.RepositoryCfg':
910  finalItem = Policy(filePath=logLoc.locString())
911  else:
912  try:
913  # PyYAML >=5.1 prefers a different loader
914  loader = yaml.FullLoader
915  except AttributeError:
916  loader = yaml.Loader
917  with open(logLoc.locString(), "rb") as infile:
918  finalItem = yaml.load(infile, Loader=loader)
919  results.append(finalItem)
920  return results
921 
922 
923 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
924 PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
925 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
926 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
927 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
928 PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
929 PosixStorage.registerFormatters("PafStorage", readFormatter=readPafStorage)
930 PosixStorage.registerFormatters("YamlStorage", readYamlStorage, writeYamlStorage)
931 
932 Storage.registerStorageClass(scheme='', cls=PosixStorage)
933 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def writeMatplotlibStorage(butlerLocation, obj)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readParquetStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
Definition: Log.h:691
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168
def readFitsCatalogStorage(butlerLocation)
def doImport(pythonType)
Definition: utils.py:106
def readYamlStorage(butlerLocation)
def writeYamlStorage(butlerLocation, obj)
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...
def writeFitsStorage(butlerLocation, obj)
bool strip
Definition: fits.cc:883