lsst.daf.persistence.posixStorage Namespace Reference

Classes
class	PosixStorage

Functions
def	readConfigStorage (butlerLocation)
def	writeConfigStorage (butlerLocation, obj)
def	readFitsStorage (butlerLocation)
def	writeFitsStorage (butlerLocation, obj)
def	readParquetStorage (butlerLocation)
def	writeParquetStorage (butlerLocation, obj)
def	writeYamlStorage (butlerLocation, obj)
def	readPickleStorage (butlerLocation)
def	writePickleStorage (butlerLocation, obj)
def	readFitsCatalogStorage (butlerLocation)
def	writeFitsCatalogStorage (butlerLocation, obj)
def	readMatplotlibStorage (butlerLocation)
def	writeMatplotlibStorage (butlerLocation, obj)
def	readPafStorage (butlerLocation)
def	readYamlStorage (butlerLocation)

Function Documentation

readConfigStorage()

def lsst.daf.persistence.posixStorage.readConfigStorage

(

butlerLocation

)

Read an lsst.pex.config.Config from a butlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 510 of file posixStorage.py.

def readConfigStorage(butlerLocation):
    """Read an lsst.pex.config.Config from a butlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.

    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    results = []
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such config file: " + logLoc.locString())
        pythonType = butlerLocation.getPythonType()
        if pythonType is not None:
            if isinstance(pythonType, basestring):
                pythonType = doImport(pythonType)
        finalItem = pythonType()
        finalItem.load(logLoc.locString())
        results.append(finalItem)
    return results

readFitsCatalogStorage()

def lsst.daf.persistence.posixStorage.readFitsCatalogStorage

(

butlerLocation

)

Read a catalog from a FITS table specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 767 of file posixStorage.py.

def readFitsCatalogStorage(butlerLocation):
    """Read a catalog from a FITS table specified by ButlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.

    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    pythonType = butlerLocation.getPythonType()
    if pythonType is not None:
        if isinstance(pythonType, basestring):
            pythonType = doImport(pythonType)
    results = []
    additionalData = butlerLocation.getAdditionalData()
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, additionalData)
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
        kwds = {}
        if additionalData.exists("hdu"):
            kwds["hdu"] = additionalData.getInt("hdu")
        if additionalData.exists("flags"):
            kwds["flags"] = additionalData.getInt("flags")
        finalItem = pythonType.readFits(logLoc.locString(), **kwds)
        results.append(finalItem)
    return results

readFitsStorage()

def lsst.daf.persistence.posixStorage.readFitsStorage

(

butlerLocation

)

Read objects from a FITS file specified by ButlerLocation.

The object is read using class or static method
``readFitsWithOptions(path, options)``, if it exists, else
``readFits(path)``. The ``options`` argument is the data returned by
``butlerLocation.getAdditionalData()``.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 556 of file posixStorage.py.

def readFitsStorage(butlerLocation):
    """Read objects from a FITS file specified by ButlerLocation.

    The object is read using class or static method
    ``readFitsWithOptions(path, options)``, if it exists, else
    ``readFits(path)``. The ``options`` argument is the data returned by
    ``butlerLocation.getAdditionalData()``.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.

    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    pythonType = butlerLocation.getPythonType()
    if pythonType is not None:
        if isinstance(pythonType, basestring):
            pythonType = doImport(pythonType)
    supportsOptions = hasattr(pythonType, "readFitsWithOptions")
    if not supportsOptions:
        from lsst.daf.base import PropertySet, PropertyList
        if issubclass(pythonType, (PropertySet, PropertyList)):
            from lsst.afw.image import readMetadata
            reader = readMetadata
        else:
            reader = pythonType.readFits
    results = []
    additionalData = butlerLocation.getAdditionalData()
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, additionalData)
        # test for existence of file, ignoring trailing [...]
        # because that can specify the HDU or other information
        filePath = re.sub(r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$", r"\1", logLoc.locString())
        if not os.path.exists(filePath):
            raise RuntimeError("No such FITS file: " + logLoc.locString())
        if supportsOptions:
            finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
        else:
            fileName = logLoc.locString()
            mat = re.search(r"^(.*)\[(\d+)\]$", fileName)

            if mat and reader == readMetadata:  # readMetadata() only understands the hdu argument, not [hdu]
                fileName = mat.group(1)
                hdu = int(mat.group(2))

                finalItem = reader(fileName, hdu=hdu)
            else:
                finalItem = reader(fileName)
        results.append(finalItem)
    return results

readMatplotlibStorage()

def lsst.daf.persistence.posixStorage.readMatplotlibStorage

(

butlerLocation

)

Read from a butlerLocation (always fails for this storage type).

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 822 of file posixStorage.py.

def readMatplotlibStorage(butlerLocation):
    """Read from a butlerLocation (always fails for this storage type).

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.

    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")

readPafStorage()

def lsst.daf.persistence.posixStorage.readPafStorage

(

butlerLocation

)

Read a policy from a PAF file specified by a ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 867 of file posixStorage.py.

def readPafStorage(butlerLocation):
    """Read a policy from a PAF file specified by a ButlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.

    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    results = []
    for locationString in butlerLocation.getLocations():
        logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
                                 butlerLocation.getAdditionalData())
        finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
        results.append(finalItem)
    return results

readParquetStorage()

def lsst.daf.persistence.posixStorage.readParquetStorage

(

butlerLocation

)

Read a catalog from a Parquet file specified by ButlerLocation.

The object returned by this is expected to be a subtype
of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
that allows for lazy loading of the data.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 639 of file posixStorage.py.

def readParquetStorage(butlerLocation):
    """Read a catalog from a Parquet file specified by ButlerLocation.

    The object returned by this is expected to be a subtype
    of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
    that allows for lazy loading of the data.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.

    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    results = []
    additionalData = butlerLocation.getAdditionalData()

    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, additionalData)
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such parquet file: " + logLoc.locString())

        pythonType = butlerLocation.getPythonType()
        if pythonType is not None:
            if isinstance(pythonType, basestring):
                pythonType = doImport(pythonType)

        filename = logLoc.locString()

        # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
        #  filename should be the first kwarg, but being explicit here.
        results.append(pythonType(filename=filename))

    return results

readPickleStorage()

def lsst.daf.persistence.posixStorage.readPickleStorage

(

butlerLocation

)

Read an object from a pickle file specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 716 of file posixStorage.py.

def readPickleStorage(butlerLocation):
    """Read an object from a pickle file specified by ButlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.

    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    # Create a list of Storages for the item.
    results = []
    additionalData = butlerLocation.getAdditionalData()
    for locationString in butlerLocation.getLocations():
        locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
        logLoc = LogicalLocation(locStringWithRoot, additionalData)
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such pickle file: " + logLoc.locString())
        with open(logLoc.locString(), "rb") as infile:
            # py3: We have to specify encoding since some files were written
            # by python2, and 'latin1' manages that conversion safely. See:
            # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
            if sys.version_info.major >= 3:
                finalItem = pickle.load(infile, encoding="latin1")
            else:
                finalItem = pickle.load(infile)
        results.append(finalItem)
    return results

readYamlStorage()

def lsst.daf.persistence.posixStorage.readYamlStorage

(

butlerLocation

)

Read an object from a YAML file specified by a butlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.

Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()

Definition at line 889 of file posixStorage.py.

def readYamlStorage(butlerLocation):
    """Read an object from a YAML file specified by a butlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.

    Returns
    -------
    A list of objects as described by the butler location. One item for
    each location in butlerLocation.getLocations()
    """
    results = []
    for locationString in butlerLocation.getLocations():
        logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
                                 butlerLocation.getAdditionalData())
        if not os.path.exists(logLoc.locString()):
            raise RuntimeError("No such YAML file: " + logLoc.locString())
        # Butler Gen2 repository configurations are handled specially
        if butlerLocation.pythonType == 'lsst.daf.persistence.RepositoryCfg':
            finalItem = Policy(filePath=logLoc.locString())
        else:
            try:
                # PyYAML >=5.1 prefers a different loader
                loader = yaml.FullLoader
            except AttributeError:
                loader = yaml.Loader
            with open(logLoc.locString(), "rb") as infile:
                finalItem = yaml.load(infile, Loader=loader)
        results.append(finalItem)
    return results


PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
PosixStorage.registerFormatters("PafStorage", readFormatter=readPafStorage)
PosixStorage.registerFormatters("YamlStorage", readYamlStorage, writeYamlStorage)

Storage.registerStorageClass(scheme='', cls=PosixStorage)
Storage.registerStorageClass(scheme='file', cls=PosixStorage)

writeConfigStorage()

def lsst.daf.persistence.posixStorage.writeConfigStorage	(		butlerLocation,
			obj
	)

Writes an lsst.pex.config.Config  object to a location specified by
ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 539 of file posixStorage.py.

def writeConfigStorage(butlerLocation, obj):
    """Writes an lsst.pex.config.Config  object to a location specified by
    ButlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
    with SafeFilename(filename) as locationString:
        logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
        obj.save(logLoc.locString())

writeFitsCatalogStorage()

def lsst.daf.persistence.posixStorage.writeFitsCatalogStorage	(		butlerLocation,
			obj
	)

Writes a catalog to a FITS table specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 801 of file posixStorage.py.

def writeFitsCatalogStorage(butlerLocation, obj):
    """Writes a catalog to a FITS table specified by ButlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        if additionalData.exists("flags"):
            kwds = dict(flags=additionalData.getInt("flags"))
        else:
            kwds = {}
        obj.writeFits(logLoc.locString(), **kwds)

writeFitsStorage()

def lsst.daf.persistence.posixStorage.writeFitsStorage	(		butlerLocation,
			obj
	)

Writes an object to a FITS file specified by ButlerLocation.

The object is written using method
``writeFitsWithOptions(path, options)``, if it exists, else
``writeFits(path)``. The ``options`` argument is the data returned by
``butlerLocation.getAdditionalData()``.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 613 of file posixStorage.py.

def writeFitsStorage(butlerLocation, obj):
    """Writes an object to a FITS file specified by ButlerLocation.

    The object is written using method
    ``writeFitsWithOptions(path, options)``, if it exists, else
    ``writeFits(path)``. The ``options`` argument is the data returned by
    ``butlerLocation.getAdditionalData()``.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    supportsOptions = hasattr(obj, "writeFitsWithOptions")
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        if supportsOptions:
            obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
        else:
            obj.writeFits(logLoc.locString())

writeMatplotlibStorage()

def lsst.daf.persistence.posixStorage.writeMatplotlibStorage	(		butlerLocation,
			obj
	)

Writes a matplotlib.figure.Figure to a location, using the template's
filename suffix to infer the file format.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : matplotlib.figure.Figure
    The object to be written.

Definition at line 838 of file posixStorage.py.

def writeMatplotlibStorage(butlerLocation, obj):
    """Writes a matplotlib.figure.Figure to a location, using the template's
    filename suffix to infer the file format.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : matplotlib.figure.Figure
        The object to be written.
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        # SafeFilename appends a random suffix, which corrupts the extension
        # matplotlib uses to guess the file format.
        # Instead, we extract the extension from the original location
        # and pass that as the format directly.
        _, ext = os.path.splitext(locations[0])
        if ext:
            ext = ext[1:]  # strip off leading '.'
        else:
            # If there is no extension, we let matplotlib fall back to its
            # default.
            ext = None
        obj.savefig(logLoc.locString(), format=ext)

writeParquetStorage()

def lsst.daf.persistence.posixStorage.writeParquetStorage	(		butlerLocation,
			obj
	)

Writes pandas dataframe to parquet file.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object(s) to be read.
obj : `lsst.qa.explorer.parquetTable.ParquetTable`
    Wrapped DataFrame to write.

Definition at line 679 of file posixStorage.py.

def writeParquetStorage(butlerLocation, obj):
    """Writes pandas dataframe to parquet file.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object(s) to be read.
    obj : `lsst.qa.explorer.parquetTable.ParquetTable`
        Wrapped DataFrame to write.

    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        filename = logLoc.locString()
        obj.write(filename)

writePickleStorage()

def lsst.daf.persistence.posixStorage.writePickleStorage	(		butlerLocation,
			obj
	)

Writes an object to a pickle file specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 749 of file posixStorage.py.

def writePickleStorage(butlerLocation, obj):
    """Writes an object to a pickle file specified by ButlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        with open(logLoc.locString(), "wb") as outfile:
            pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)

writeYamlStorage()

def lsst.daf.persistence.posixStorage.writeYamlStorage	(		butlerLocation,
			obj
	)

Writes an object to a YAML file specified by ButlerLocation.

Parameters
----------
butlerLocation : ButlerLocation
    The location for the object to be written.
obj : object instance
    The object to be written.

Definition at line 698 of file posixStorage.py.

def writeYamlStorage(butlerLocation, obj):
    """Writes an object to a YAML file specified by ButlerLocation.

    Parameters
    ----------
    butlerLocation : ButlerLocation
        The location for the object to be written.
    obj : object instance
        The object to be written.
    """
    additionalData = butlerLocation.getAdditionalData()
    locations = butlerLocation.getLocations()
    with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
        logLoc = LogicalLocation(locationString, additionalData)
        with open(logLoc.locString(), "w") as outfile:
            yaml.dump(obj, outfile)

---

Generated on Thu Aug 8 2019 20:21:07 for LSSTApplications by   1.8.13