lsst.daf.persistence.posixStorage Namespace Reference

Classes
class	PosixStorage

Functions
def	readConfigStorage (butlerLocation)
def	writeConfigStorage (butlerLocation, obj)
def	readFitsStorage (butlerLocation)
def	writeFitsStorage (butlerLocation, obj)
def	readParquetStorage (butlerLocation)
def	writeParquetStorage (butlerLocation, obj)
def	writeYamlStorage (butlerLocation, obj)
def	readPickleStorage (butlerLocation)
def	writePickleStorage (butlerLocation, obj)
def	readFitsCatalogStorage (butlerLocation)
def	writeFitsCatalogStorage (butlerLocation, obj)
def	readMatplotlibStorage (butlerLocation)
def	writeMatplotlibStorage (butlerLocation, obj)
def	readYamlStorage (butlerLocation)

Function Documentation

readConfigStorage()

def lsst.daf.persistence.posixStorage.readConfigStorage

(

butlerLocation

)

Read an lsst.pex.config.Config from a butlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 512 of file posixStorage.py.

def readConfigStorage(butlerLocation):
    """Read an lsst.pex.config.Config from a butlerLocation.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
 
    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    results = []
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such config file: " + logLoc.locString())
 
        # Automatically determine the Config class from the serialized form
        with open(logLoc.locString(), "r") as fd:
            config_py = fd.read()
        config = pexConfig.Config._fromPython(config_py)
 
        pythonType = butlerLocation.getPythonType()
        if pythonType is not None:
            if isinstance(pythonType, str):
                pythonType = doImport(pythonType)
            if not isinstance(config, pythonType):
                raise TypeError(f"Unexpected type of config: {type(config)}, expected {pythonType}")
 
        results.append(config)
    return results
 
 

readFitsCatalogStorage()

def lsst.daf.persistence.posixStorage.readFitsCatalogStorage

(

butlerLocation

)

Read a catalog from a FITS table specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 776 of file posixStorage.py.

def readFitsCatalogStorage(butlerLocation):
    """Read a catalog from a FITS table specified by ButlerLocation.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
 
    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    pythonType = butlerLocation.getPythonType()
    if pythonType is not None:
        if isinstance(pythonType, str):
            pythonType = doImport(pythonType)
    results = []
    additionalData = butlerLocation.getAdditionalData()
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, additionalData)
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
        kwds = {}
        if additionalData.exists("hdu"):
            kwds["hdu"] = additionalData.getInt("hdu")
        if additionalData.exists("flags"):
            kwds["flags"] = additionalData.getInt("flags")
        finalItem = pythonType.readFits(logLoc.locString(), **kwds)
        results.append(finalItem)
    return results
 
 

readFitsStorage()

def lsst.daf.persistence.posixStorage.readFitsStorage

(

butlerLocation

)

Read objects from a FITS file specified by ButlerLocation.

The object is read using class or static method
``readFitsWithOptions(path, options)``, if it exists, else
``readFits(path)``. The ``options`` argument is the data returned by
``butlerLocation.getAdditionalData()``.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 565 of file posixStorage.py.

def readFitsStorage(butlerLocation):
    """Read objects from a FITS file specified by ButlerLocation.
 
    The object is read using class or static method
    ``readFitsWithOptions(path, options)``, if it exists, else
    ``readFits(path)``. The ``options`` argument is the data returned by
    ``butlerLocation.getAdditionalData()``.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
 
    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    pythonType = butlerLocation.getPythonType()
    if pythonType is not None:
        if isinstance(pythonType, str):
            pythonType = doImport(pythonType)
    supportsOptions = hasattr(pythonType, "readFitsWithOptions")
    if not supportsOptions:
        from lsst.daf.base import PropertySet, PropertyList
        if issubclass(pythonType, (PropertySet, PropertyList)):
            from lsst.afw.fits import readMetadata
            reader = readMetadata
        else:
            reader = pythonType.readFits
    results = []
    additionalData = butlerLocation.getAdditionalData()
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, additionalData)
        # test for existence of file, ignoring trailing [...]
        # because that can specify the HDU or other information
        filePath = re.sub(r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$", r"\1", logLoc.locString())
        if not os.path.exists(filePath):
            raise RuntimeError("No such FITS file: " + logLoc.locString())
        if supportsOptions:
            finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
        else:
            fileName = logLoc.locString()
            mat = re.search(r"^(.*)\[(\d+)\]$", fileName)
 
            if mat and reader == readMetadata:  # readMetadata() only understands the hdu argument, not [hdu]
                fileName = mat.group(1)
                hdu = int(mat.group(2))
 
                finalItem = reader(fileName, hdu=hdu)
            else:
                finalItem = reader(fileName)
        results.append(finalItem)
    return results
 
 

readMatplotlibStorage()

def lsst.daf.persistence.posixStorage.readMatplotlibStorage

(

butlerLocation

)

Read from a butlerLocation (always fails for this storage type).

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 831 of file posixStorage.py.

def readMatplotlibStorage(butlerLocation):
    """Read from a butlerLocation (always fails for this storage type).
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
 
    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
 
 

readParquetStorage()

def lsst.daf.persistence.posixStorage.readParquetStorage

(

butlerLocation

)

Read a catalog from a Parquet file specified by ButlerLocation.

The object returned by this is expected to be a subtype
of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
that allows for lazy loading of the data.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 648 of file posixStorage.py.

def readParquetStorage(butlerLocation):
    """Read a catalog from a Parquet file specified by ButlerLocation.
 
    The object returned by this is expected to be a subtype
    of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
    that allows for lazy loading of the data.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
 
    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    results = []
    additionalData = butlerLocation.getAdditionalData()
 
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, additionalData)
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such parquet file: " + logLoc.locString())
 
        pythonType = butlerLocation.getPythonType()
        if pythonType is not None:
            if isinstance(pythonType, str):
                pythonType = doImport(pythonType)
 
        filename = logLoc.locString()
 
        # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
        #  filename should be the first kwarg, but being explicit here.
        results.append(pythonType(filename=filename))
 
    return results
 
 

readPickleStorage()

def lsst.daf.persistence.posixStorage.readPickleStorage

(

butlerLocation

)

Read an object from a pickle file specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 725 of file posixStorage.py.

def readPickleStorage(butlerLocation):
    """Read an object from a pickle file specified by ButlerLocation.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
 
    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    # Create a list of Storages for the item.
    results = []
    additionalData = butlerLocation.getAdditionalData()
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, additionalData)
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such pickle file: " + logLoc.locString())
        with open(logLoc.locString(), "rb") as infile:
            # py3: We have to specify encoding since some files were written
            # by python2, and 'latin1' manages that conversion safely. See:
            # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
            if sys.version_info.major >= 3:
                finalItem = pickle.load(infile, encoding="latin1")
            else:
                finalItem = pickle.load(infile)
        results.append(finalItem)
    return results
 
 

readYamlStorage()

def lsst.daf.persistence.posixStorage.readYamlStorage

(

butlerLocation

)

Read an object from a YAML file specified by a butlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 876 of file posixStorage.py.

def readYamlStorage(butlerLocation):
    """Read an object from a YAML file specified by a butlerLocation.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
 
    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    results = []
    for locationString in butlerLocation.getLocations():
        logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
                                 butlerLocation.getAdditionalData())
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such YAML file: " + logLoc.locString())
        # Butler Gen2 repository configurations are handled specially
        if butlerLocation.pythonType == 'lsst.daf.persistence.RepositoryCfg':
            finalItem = Policy(filePath=logLoc.locString())
        else:
            try:
                # PyYAML >=5.1 prefers a different loader
                loader = yaml.UnsafeLoader
            except AttributeError:
                loader = yaml.Loader
            with open(logLoc.locString(), "rb") as infile:
                finalItem = yaml.load(infile, Loader=loader)
        results.append(finalItem)
    return results
 
 
PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
PosixStorage.registerFormatters("YamlStorage", readYamlStorage, writeYamlStorage)
 
Storage.registerStorageClass(scheme='', cls=PosixStorage)
Storage.registerStorageClass(scheme='file', cls=PosixStorage)

writeConfigStorage()

def lsst.daf.persistence.posixStorage.writeConfigStorage	(		butlerLocation,
			obj
	)

Writes an lsst.pex.config.Config  object to a location specified by
ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 548 of file posixStorage.py.

def writeConfigStorage(butlerLocation, obj):
    """Writes an lsst.pex.config.Config  object to a location specified by
    ButlerLocation.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
    with SafeFilename(filename) as locationString:
        logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
        obj.save(logLoc.locString())
 
 

writeFitsCatalogStorage()

def lsst.daf.persistence.posixStorage.writeFitsCatalogStorage	(		butlerLocation,
			obj
	)

Writes a catalog to a FITS table specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 810 of file posixStorage.py.

def writeFitsCatalogStorage(butlerLocation, obj):
    """Writes a catalog to a FITS table specified by ButlerLocation.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        if additionalData.exists("flags"):
            kwds = dict(flags=additionalData.getInt("flags"))
        else:
            kwds = {}
        obj.writeFits(logLoc.locString(), **kwds)
 
 

writeFitsStorage()

def lsst.daf.persistence.posixStorage.writeFitsStorage	(		butlerLocation,
			obj
	)

Writes an object to a FITS file specified by ButlerLocation.

The object is written using method
``writeFitsWithOptions(path, options)``, if it exists, else
``writeFits(path)``. The ``options`` argument is the data returned by
``butlerLocation.getAdditionalData()``.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 622 of file posixStorage.py.

def writeFitsStorage(butlerLocation, obj):
    """Writes an object to a FITS file specified by ButlerLocation.
 
    The object is written using method
    ``writeFitsWithOptions(path, options)``, if it exists, else
    ``writeFits(path)``. The ``options`` argument is the data returned by
    ``butlerLocation.getAdditionalData()``.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    supportsOptions = hasattr(obj, "writeFitsWithOptions")
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        if supportsOptions:
            obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
        else:
            obj.writeFits(logLoc.locString())
 
 

writeMatplotlibStorage()

def lsst.daf.persistence.posixStorage.writeMatplotlibStorage	(		butlerLocation,
			obj
	)

Writes a matplotlib.figure.Figure to a location, using the template's
filename suffix to infer the file format.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : matplotlib.figure.Figure
    The object to be written.

Definition at line 847 of file posixStorage.py.

def writeMatplotlibStorage(butlerLocation, obj):
    """Writes a matplotlib.figure.Figure to a location, using the template's
    filename suffix to infer the file format.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : matplotlib.figure.Figure
        The object to be written.
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        # SafeFilename appends a random suffix, which corrupts the extension
        # matplotlib uses to guess the file format.
        # Instead, we extract the extension from the original location
        # and pass that as the format directly.
        _, ext = os.path.splitext(locations[0])
        if ext:
            ext = ext[1:]  # strip off leading '.'
        else:
            # If there is no extension, we let matplotlib fall back to its
            # default.
            ext = None
        obj.savefig(logLoc.locString(), format=ext)
 
 

writeParquetStorage()

def lsst.daf.persistence.posixStorage.writeParquetStorage	(		butlerLocation,
			obj
	)

Writes pandas dataframe to parquet file.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.
obj : `lsst.qa.explorer.parquetTable.ParquetTable`
    Wrapped DataFrame to write.

Definition at line 688 of file posixStorage.py.

def writeParquetStorage(butlerLocation, obj):
    """Writes pandas dataframe to parquet file.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
    obj : `lsst.qa.explorer.parquetTable.ParquetTable`
        Wrapped DataFrame to write.
 
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        filename = logLoc.locString()
        obj.write(filename)
 
 

writePickleStorage()

def lsst.daf.persistence.posixStorage.writePickleStorage	(		butlerLocation,
			obj
	)

Writes an object to a pickle file specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 758 of file posixStorage.py.

def writePickleStorage(butlerLocation, obj):
    """Writes an object to a pickle file specified by ButlerLocation.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        with open(logLoc.locString(), "wb") as outfile:
            pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
 
 

writeYamlStorage()

def lsst.daf.persistence.posixStorage.writeYamlStorage	(		butlerLocation,
			obj
	)

Writes an object to a YAML file specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 707 of file posixStorage.py.

def writeYamlStorage(butlerLocation, obj):
    """Writes an object to a YAML file specified by ButlerLocation.
 
    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        with open(logLoc.locString(), "w") as outfile:
            yaml.dump(obj, outfile)