lsst.pipe.tasks.mergeDetections Namespace Reference

Classes
class	MergeDetectionsConnections

Functions
def	matchCatalogsExact (catalog1, catalog2, patch1=None, patch2=None)
def	write (self, patchRef, catalog)
	Write the output. More...
def	writeMetadata (self, dataRefList)
	No metadata to write, and not sure how to write it for a list of dataRefs. More...

Variables
	patch
	tract
	filter
	schema = self.merged.getPeakSchema()
	mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key
list	converted = []
	peak = oldFoot.getPeaks()[0]
	newFoot = afwDetect.Footprint(oldFoot.spans, schema)

Function Documentation

matchCatalogsExact()

def lsst.pipe.tasks.mergeDetections.matchCatalogsExact	(		catalog1,
			catalog2,
			patch1 = None,
			patch2 = None
	)

Match two catalogs derived from the same mergeDet catalog

When testing downstream features, like deblending methods/parameters
and measurement algorithms/parameters, it is useful to to compare
the same sources in two catalogs. In most cases this must be done
by matching on either RA/DEC or XY positions, which occassionally
will mismatch one source with another.

For a more robust solution, as long as the downstream catalog is
derived from the same mergeDet catalog, exact source matching
can be done via the unique ``(parent, deblend_peakID)``
combination. So this function performs this exact matching for
all sources both catalogs.

Parameters
----------
catalog1, catalog2 : `lsst.afw.table.SourceCatalog`
    The two catalogs to merge

patch1, patch2 : array of int
    Patch for each row, converted into an integer.
    In the gen3 butler this is done already, in gen2
    it is recommended to use `patch2Int`, assuming that
    the patches are the same structure as HSC, that range
    from '0,0' to '9,9'.

Returns
-------
result: list of `lsst.afw.table.SourceMatch`
    List of matches for each source (using an inner join).

Definition at line 45 of file mergeDetections.py.

def matchCatalogsExact(catalog1, catalog2, patch1=None, patch2=None):
    """Match two catalogs derived from the same mergeDet catalog
 
    When testing downstream features, like deblending methods/parameters
    and measurement algorithms/parameters, it is useful to to compare
    the same sources in two catalogs. In most cases this must be done
    by matching on either RA/DEC or XY positions, which occassionally
    will mismatch one source with another.
 
    For a more robust solution, as long as the downstream catalog is
    derived from the same mergeDet catalog, exact source matching
    can be done via the unique ``(parent, deblend_peakID)``
    combination. So this function performs this exact matching for
    all sources both catalogs.
 
    Parameters
    ----------
    catalog1, catalog2 : `lsst.afw.table.SourceCatalog`
        The two catalogs to merge
 
    patch1, patch2 : array of int
        Patch for each row, converted into an integer.
        In the gen3 butler this is done already, in gen2
        it is recommended to use `patch2Int`, assuming that
        the patches are the same structure as HSC, that range
        from '0,0' to '9,9'.
 
    Returns
    -------
    result: list of `lsst.afw.table.SourceMatch`
        List of matches for each source (using an inner join).
    """
    # Only match the individual sources, the parents will
    # already be matched by the mergeDet catalog
    sidx1 = catalog1["parent"] != 0
    sidx2 = catalog2["parent"] != 0
 
    # Create the keys used to merge the catalogs
    parents1 = np.array(catalog1["parent"][sidx1])
    peaks1 = np.array(catalog1["deblend_peakId"][sidx1])
    index1 = np.arange(len(catalog1))[sidx1]
    parents2 = np.array(catalog2["parent"][sidx2])
    peaks2 = np.array(catalog2["deblend_peakId"][sidx2])
    index2 = np.arange(len(catalog2))[sidx2]
 
    if patch1 is not None:
        if patch2 is None:
            msg = ("If the catalogs are from different patches then patch1 and patch2 must be specified"
                   ", got {} and {}").format(patch1, patch2)
            raise ValueError(msg)
        patch1 = patch1[sidx1]
        patch2 = patch2[sidx2]
 
        key1 = np.rec.array((parents1, peaks1, patch1, index1),
                            dtype=[('parent', np.int64), ('peakId', np.int32),
                                   ("patch", patch1.dtype), ("index", np.int32)])
        key2 = np.rec.array((parents2, peaks2, patch2, index2),
                            dtype=[('parent', np.int64), ('peakId', np.int32),
                                   ("patch", patch2.dtype), ("index", np.int32)])
        matchColumns = ("parent", "peakId", "patch")
    else:
        key1 = np.rec.array((parents1, peaks1, index1),
                            dtype=[('parent', np.int64), ('peakId', np.int32), ("index", np.int32)])
        key2 = np.rec.array((parents2, peaks2, index2),
                            dtype=[('parent', np.int64), ('peakId', np.int32), ("index", np.int32)])
        matchColumns = ("parent", "peakId")
    # Match the two keys.
    # This line performs an inner join on the structured
    # arrays `key1` and `key2`, which stores their indices
    # as columns in a structured array.
    matched = rec_join(matchColumns, key1, key2, jointype="inner")
 
    # Create the full index for both catalogs
    indices1 = matched["index1"]
    indices2 = matched["index2"]
 
    # Re-index the resulting catalogs
    matches = [
        afwTable.SourceMatch(catalog1[int(i1)], catalog2[int(i2)], 0.0)
        for i1, i2 in zip(indices1, indices2)
    ]
 
    return matches
 
 

write()

def lsst.pipe.tasks.mergeDetections.write	(		self,
			patchRef,
			catalog
	)

Write the output.

Parameters

[in]	patchRef	data reference for patch
[in]	catalog	catalog

We write as the dataset provided by the 'outputDataset' class variable.

Definition at line 477 of file mergeDetections.py.

    def write(self, patchRef, catalog):
        """!
        @brief Write the output.
 
        @param[in]  patchRef   data reference for patch
        @param[in]  catalog    catalog
 
        We write as the dataset provided by the 'outputDataset'
        class variable.
        """
        patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
        # since the filter isn't actually part of the data ID for the dataset we're saving,
        # it's confusing to see it in the log message, even if the butler simply ignores it.
        mergeDataId = patchRef.dataId.copy()
        del mergeDataId["filter"]
        self.log.info("Wrote merged catalog: %s", mergeDataId)
 

writeMetadata()

def lsst.pipe.tasks.mergeDetections.writeMetadata	(		self,
			dataRefList
	)

No metadata to write, and not sure how to write it for a list of dataRefs.

Definition at line 494 of file mergeDetections.py.

    def writeMetadata(self, dataRefList):
        """!
        @brief No metadata to write, and not sure how to write it for a list of dataRefs.
        """
        pass

Variable Documentation

converted

list lsst.pipe.tasks.mergeDetections.converted = []

Definition at line 466 of file mergeDetections.py.

filter

lsst.pipe.tasks.mergeDetections.filter

Definition at line 277 of file mergeDetections.py.

mergeKey

lsst.pipe.tasks.mergeDetections.mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key

Definition at line 465 of file mergeDetections.py.

newFoot

lsst.pipe.tasks.mergeDetections.newFoot = afwDetect.Footprint(oldFoot.spans, schema)

Definition at line 470 of file mergeDetections.py.

patch

lsst.pipe.tasks.mergeDetections.patch

Definition at line 277 of file mergeDetections.py.

peak

lsst.pipe.tasks.mergeDetections.peak = oldFoot.getPeaks()[0]

Examples

forEachPixel.cc.

Definition at line 469 of file mergeDetections.py.

schema

lsst.pipe.tasks.mergeDetections.schema = self.merged.getPeakSchema()

ConfigClass = MergeDetectionsConfig
RunnerClass = MergeSourcesRunner
_DefaultName = "mergeCoaddDetections"
inputDataset = "det"
outputDataset = "mergeDet"
makeIdFactory = _makeMakeIdFactory("MergedCoaddId")

@classmethod
def _makeArgumentParser(cls):
    return makeMergeArgumentParser(cls._DefaultName, cls.inputDataset)

def getInputSchema(self, butler=None, schema=None):
    return getInputSchema(self, butler, schema)

def __init__(self, butler=None, schema=None, initInputs=None, **kwargs):
    # Make PipelineTask-only wording less transitional after cmdlineTask is removed

super().__init__(**kwargs)
if initInputs is not None:
    schema = initInputs['schema'].schema

self.makeSubtask("skyObjects")
self.schema = self.getInputSchema(butler=butler, schema=schema)

filterNames = list(self.config.priorityList)
filterNames.append(self.config.skyFilterName)
self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
self.outputSchema = afwTable.SourceCatalog(self.schema)
self.outputPeakSchema = afwDetect.PeakCatalog(self.merged.getPeakSchema())

def runDataRef(self, patchRefList):
catalogs = dict(readCatalog(self, patchRef) for patchRef in patchRefList)
skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRefList[0])
idFactory = self.makeIdFactory(patchRefList[0])
skySeed = patchRefList[0].get(self.config.coaddName + "MergedCoaddId")
mergeCatalogStruct = self.run(catalogs, skyInfo, idFactory, skySeed)
self.write(patchRefList[0], mergeCatalogStruct.outputCatalog)

def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "tract_patch")
inputs["skySeed"] = exposureIdInfo.expId
inputs["idFactory"] = exposureIdInfo.makeSourceIdFactory()
catalogDict = {ref.dataId['band']: cat for ref, cat in zip(inputRefs.catalogs,
               inputs['catalogs'])}
inputs['catalogs'] = catalogDict
skyMap = inputs.pop('skyMap')
# Can use the first dataId to find the tract and patch being worked on
tractNumber = inputRefs.catalogs[0].dataId['tract']
tractInfo = skyMap[tractNumber]
patchInfo = tractInfo.getPatchInfo(inputRefs.catalogs[0].dataId['patch'])
skyInfo = Struct(
    skyMap=skyMap,
    tractInfo=tractInfo,
    patchInfo=patchInfo,
    wcs=tractInfo.getWcs(),
    bbox=patchInfo.getOuterBBox()
)
inputs['skyInfo'] = skyInfo

outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)

def run(self, catalogs, skyInfo, idFactory, skySeed):
r

# Convert distance to tract coordinate
tractWcs = skyInfo.wcs
peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()

# Put catalogs, filters in priority order
orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
orderedBands = [band for band in self.config.priorityList if band in catalogs.keys()]

mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
                                                self.schema, idFactory,
                                                samePeakDistance)

#
# Add extra sources that correspond to blank sky
#
skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
if skySourceFootprints:
    key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
    for foot in skySourceFootprints:
        s = mergedList.addNew()
        s.setFootprint(foot)
        s.set(key, True)

# Sort Peaks from brightest to faintest
for record in mergedList:
    record.getFootprint().sortPeaks()
self.log.info("Merged to %d sources", len(mergedList))
# Attempt to remove garbage peaks
self.cullPeaks(mergedList)
return Struct(outputCatalog=mergedList)

def cullPeaks(self, catalog):

keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
totalPeaks = 0
culledPeaks = 0
for parentSource in catalog:
    # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
    # to it (which is easier than deleting as we iterate).
    keptPeaks = parentSource.getFootprint().getPeaks()
    oldPeaks = list(keptPeaks)
    keptPeaks.clear()
    familySize = len(oldPeaks)
    totalPeaks += familySize
    for rank, peak in enumerate(oldPeaks):
        if ((rank < self.config.cullPeaks.rankSufficient)
            or (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient)
            or (rank < self.config.cullPeaks.rankConsidered
                and rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
            keptPeaks.append(peak)
        else:
            culledPeaks += 1
self.log.info("Culled %d of %d peaks", culledPeaks, totalPeaks)

def getSchemaCatalogs(self):

mergeDet = afwTable.SourceCatalog(self.schema)
peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
        self.config.coaddName + "Coadd_peak": peak}

def getSkySourceFootprints(self, mergedList, skyInfo, seed):

Definition at line 464 of file mergeDetections.py.

tract

lsst.pipe.tasks.mergeDetections.tract

Definition at line 277 of file mergeDetections.py.