lsst.ip.diffim.detectAndMeasure Namespace Reference

Classes
class	DetectAndMeasureConnections

Variables
	science : `lsst.afw.image.ExposureF`
	matchedTemplate : `lsst.afw.image.ExposureF`
	difference : `lsst.afw.image.ExposureF`
	idFactory : `lsst.afw.table.IdFactory`, optional
	measurementResults : `lsst.pipe.base.Struct`
	sources : `lsst.afw.table.SourceCatalog`
	positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
	negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
	diaSources : `lsst.afw.table.SourceCatalog`
	mask : `lsst.afw.image.Mask`
	seed : `int`
	wcs : `lsst.afw.geom.SkyWcs`
	maskedImage : `lsst.afw.image.maskedImage`
	streakInfo : `lsst.pipe.base.Struct`
	scoreExposure : `lsst.afw.image.ExposureF`

Variable Documentation

diaSources

lsst.ip.diffim.detectAndMeasure.diaSources : `lsst.afw.table.SourceCatalog`

def makeFootprints(sources):
    footprints = afwDetection.FootprintSet(difference.getBBox())
    footprints.setFootprints([src.getFootprint() for src in sources])
    return footprints

def deblend(footprints):

sources = afwTable.SourceCatalog(self.schema)
footprints.makeSources(sources)
self.deblend.run(exposure=difference, sources=sources)
self.setPrimaryFlags.run(sources)
children = sources["detect_isDeblendedSource"] == 1
sources = sources[children].copy(deep=True)
# Clear parents, so that measurement plugins behave correctly.
sources['parent'] = 0
return sources.copy(deep=True)

positives = deblend(positiveFootprints)
negatives = deblend(negativeFootprints)

sources = afwTable.SourceCatalog(self.schema)
sources.reserve(len(positives) + len(negatives))
sources.extend(positives, deep=True)
sources.extend(negatives, deep=True)
return sources, makeFootprints(positives), makeFootprints(negatives)

def _removeBadSources(self, diaSources):

selector = np.ones(len(diaSources), dtype=bool)
for flag in self.config.badSourceFlags:
    flags = diaSources[flag]
    nBad = np.count_nonzero(flags)
    if nBad > 0:
        self.log.debug("Found %d unphysical sources with flag %s.", nBad, flag)
        selector &= ~flags
nBadTotal = np.count_nonzero(~selector)
self.metadata["nRemovedBadFlaggedSources"] = nBadTotal
self.log.info("Removed %d unphysical sources.", nBadTotal)
return diaSources[selector].copy(deep=True)

def addSkySources(self, diaSources, mask, seed,
              subtask=None):

if subtask is None:
    subtask = self.skySources
skySourceFootprints = subtask.run(mask=mask, seed=seed, catalog=diaSources)
self.metadata[f"n_{subtask.getName()}"] = len(skySourceFootprints)

def measureDiaSources(self, diaSources, science, difference, matchedTemplate):

# Ensure that the required mask planes are present
for mp in self.config.measurement.plugins["base_PixelFlags"].masksFpAnywhere:
    difference.mask.addMaskPlane(mp)
# Note that this may not be correct if we convolved the science image.
# In the future we may wish to persist the matchedScience image.
self.measurement.run(diaSources, difference, science, matchedTemplate)
if self.config.doApCorr:
    apCorrMap = difference.getInfo().getApCorrMap()
    if apCorrMap is None:
        self.log.warning("Difference image does not have valid aperture correction; skipping.")
    else:
        self.applyApCorr.run(
            catalog=diaSources,
            apCorrMap=apCorrMap,
        )

def measureForcedSources(self, diaSources, science, wcs):

Definition at line 525 of file detectAndMeasure.py.

difference

lsst.ip.diffim.detectAndMeasure.difference : `lsst.afw.image.ExposureF`

if idFactory is None:
    idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()

self._prepareInputs(difference)

# Don't use the idFactory until after deblend+merge, so that we aren't
# generating ids that just get thrown away (footprint merge doesn't
# know about past ids).
table = afwTable.SourceTable.make(self.schema)
results = self.detection.run(
    table=table,
    exposure=difference,
    doSmooth=True,
)

sources, positives, negatives = self._deblend(difference,
                                              results.positive,
                                              results.negative)

return self.processResults(science, matchedTemplate, difference, sources, idFactory,
                           positiveFootprints=positives,
                           negativeFootprints=negatives)

def _prepareInputs(self, difference):

self.metadata["nUnmergedDiaSources"] = len(sources)
if self.config.doMerge:
    fpSet = positiveFootprints
    fpSet.merge(negativeFootprints, self.config.growFootprint,
                self.config.growFootprint, False)
    initialDiaSources = afwTable.SourceCatalog(self.schema)
    fpSet.makeSources(initialDiaSources)
    self.log.info("Merging detections into %d sources", len(initialDiaSources))
else:
    initialDiaSources = sources

# Assign source ids at the end: deblend/merge mean that we don't keep
# track of parents and children, we only care about the final ids.
for source in initialDiaSources:
    source.setId(idFactory())
# Ensure sources added after this get correct ids.
initialDiaSources.getTable().setIdFactory(idFactory)
initialDiaSources.setMetadata(self.algMetadata)

self.metadata["nMergedDiaSources"] = len(initialDiaSources)

if self.config.doMaskStreaks:
    streakInfo = self._runStreakMasking(difference.maskedImage)

if self.config.doSkySources:
    self.addSkySources(initialDiaSources, difference.mask, difference.info.id)

if not initialDiaSources.isContiguous():
    initialDiaSources = initialDiaSources.copy(deep=True)

self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
diaSources = self._removeBadSources(initialDiaSources)

if self.config.doForcedMeasurement:
    self.measureForcedSources(diaSources, science, difference.getWcs())

self.calculateMetrics(difference)

measurementResults = pipeBase.Struct(
    subtractedMeasuredExposure=difference,
    diaSources=diaSources,
)
if self.config.doMaskStreaks and self.config.writeStreakInfo:
    measurementResults.mergeItems(streakInfo, 'maskedStreaks')

return measurementResults

def _deblend(self, difference, positiveFootprints, negativeFootprints):

# Run forced psf photometry on the PVI at the diaSource locations.
# Copy the measured flux and error into the diaSource.
forcedSources = self.forcedMeasurement.generateMeasCat(science, diaSources, wcs)
self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
                  "ip_diffim_forced_PsfFlux_instFlux", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
                  "ip_diffim_forced_PsfFlux_instFluxErr", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
                  "ip_diffim_forced_PsfFlux_area", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
                  "ip_diffim_forced_PsfFlux_flag", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
                  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
                  "ip_diffim_forced_PsfFlux_flag_edge", True)
for diaSource, forcedSource in zip(diaSources, forcedSources):
    diaSource.assign(forcedSource, mapper)

def calculateMetrics(self, difference):

Definition at line 329 of file detectAndMeasure.py.

idFactory

lsst.ip.diffim.detectAndMeasure.idFactory : `lsst.afw.table.IdFactory`, optional

Definition at line 331 of file detectAndMeasure.py.

mask

lsst.ip.diffim.detectAndMeasure.mask : `lsst.afw.image.Mask`

Definition at line 555 of file detectAndMeasure.py.

maskedImage

lsst.ip.diffim.detectAndMeasure.maskedImage : `lsst.afw.image.maskedImage`

mask = difference.mask
badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
self.metadata["nGoodPixels"] = np.sum(~badPix)
self.metadata["nBadPixels"] = np.sum(badPix)
detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0
detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0
self.metadata["nPixelsDetectedPositive"] = np.sum(detPosPix)
self.metadata["nPixelsDetectedNegative"] = np.sum(detNegPix)
detPosPix &= badPix
detNegPix &= badPix
self.metadata["nBadPixelsDetectedPositive"] = np.sum(detPosPix)
self.metadata["nBadPixelsDetectedNegative"] = np.sum(detNegPix)

metricsMaskPlanes = list(mask.getMaskPlaneDict().keys())
for maskPlane in metricsMaskPlanes:
    try:
        self.metadata["%s_mask_fraction"%maskPlane.lower()] = evaluateMaskFraction(mask, maskPlane)
    except InvalidParameterError:
        self.metadata["%s_mask_fraction"%maskPlane.lower()] = -1
        self.log.info("Unable to calculate metrics for mask plane %s: not in image"%maskPlane)

def _runStreakMasking(self, maskedImage):

Definition at line 666 of file detectAndMeasure.py.

matchedTemplate

lsst.ip.diffim.detectAndMeasure.matchedTemplate : `lsst.afw.image.ExposureF`

Definition at line 326 of file detectAndMeasure.py.

measurementResults

lsst.ip.diffim.detectAndMeasure.measurementResults : `lsst.pipe.base.Struct`

Definition at line 338 of file detectAndMeasure.py.

negativeFootprints

lsst.ip.diffim.detectAndMeasure.negativeFootprints : `lsst.afw.detection.FootprintSet`, optional

Definition at line 413 of file detectAndMeasure.py.

positiveFootprints

lsst.ip.diffim.detectAndMeasure.positiveFootprints : `lsst.afw.detection.FootprintSet`, optional

Definition at line 411 of file detectAndMeasure.py.

science

lsst.ip.diffim.detectAndMeasure.science : `lsst.afw.image.ExposureF`

doMerge = pexConfig.Field(
    dtype=bool,
    default=True,
    doc="Merge positive and negative diaSources with grow radius "
        "set by growFootprint"
)
doForcedMeasurement = pexConfig.Field(
    dtype=bool,
    default=True,
    doc="Force photometer diaSource locations on PVI?")
doAddMetrics = pexConfig.Field(
    dtype=bool,
    default=False,
    doc="Add columns to the source table to hold analysis metrics?"
)
detection = pexConfig.ConfigurableField(
    target=SourceDetectionTask,
    doc="Final source detection for diaSource measurement",
)
deblend = pexConfig.ConfigurableField(
    target=lsst.meas.deblender.SourceDeblendTask,
    doc="Task to split blended sources into their components."
)
measurement = pexConfig.ConfigurableField(
    target=DipoleFitTask,
    doc="Task to measure sources on the difference image.",
)
doApCorr = lsst.pex.config.Field(
    dtype=bool,
    default=True,
    doc="Run subtask to apply aperture corrections"
)
applyApCorr = lsst.pex.config.ConfigurableField(
    target=ApplyApCorrTask,
    doc="Task to apply aperture corrections"
)
forcedMeasurement = pexConfig.ConfigurableField(
    target=ForcedMeasurementTask,
    doc="Task to force photometer science image at diaSource locations.",
)
growFootprint = pexConfig.Field(
    dtype=int,
    default=2,
    doc="Grow positive and negative footprints by this many pixels before merging"
)
diaSourceMatchRadius = pexConfig.Field(
    dtype=float,
    default=0.5,
    doc="Match radius (in arcseconds) for DiaSource to Source association"
)
doSkySources = pexConfig.Field(
    dtype=bool,
    default=False,
    doc="Generate sky sources?",
)
skySources = pexConfig.ConfigurableField(
    target=SkyObjectsTask,
    doc="Generate sky sources",
)
doMaskStreaks = pexConfig.Field(
    dtype=bool,
    default=True,
    doc="Turn on streak masking",
)
maskStreaks = pexConfig.ConfigurableField(
    target=MaskStreaksTask,
    doc="Subtask for masking streaks. Only used if doMaskStreaks is True. "
        "Adds a mask plane to an exposure, with the mask plane name set by streakMaskName.",
)
writeStreakInfo = pexConfig.Field(
    dtype=bool,
    default=False,
    doc="Record the parameters of any detected streaks. For LSST, this should be turned off except for "
        "development work."
)
setPrimaryFlags = pexConfig.ConfigurableField(
    target=SetPrimaryFlagsTask,
    doc="Task to add isPrimary and deblending-related flags to the catalog."
)
badSourceFlags = lsst.pex.config.ListField(
    dtype=str,
    doc="Sources with any of these flags set are removed before writing the output catalog.",
    default=("base_PixelFlags_flag_offimage",
             "base_PixelFlags_flag_interpolatedCenterAll",
             "base_PixelFlags_flag_badCenterAll",
             "base_PixelFlags_flag_edgeCenterAll",
             "base_PixelFlags_flag_saturatedCenterAll",
             ),
)
clearMaskPlanes = lsst.pex.config.ListField(
    dtype=str,
    doc="Mask planes to clear before running detection.",
    default=("DETECTED", "DETECTED_NEGATIVE", "NOT_DEBLENDED", "STREAK"),
)
idGenerator = DetectorVisitIdGeneratorConfig.make_field()

def setDefaults(self):
    # DiaSource Detection
    self.detection.thresholdPolarity = "both"
    self.detection.thresholdValue = 5.0
    self.detection.reEstimateBackground = False
    self.detection.thresholdType = "pixel_stdev"
    self.detection.excludeMaskPlanes = ["EDGE",
                                        "SAT",
                                        "BAD",
                                        "INTRP",
                                        "NO_DATA",
                                        ]

    self.measurement.plugins.names |= ["ext_trailedSources_Naive",
                                       "base_LocalPhotoCalib",
                                       "base_LocalWcs",
                                       "ext_shapeHSM_HsmSourceMoments",
                                       "ext_shapeHSM_HsmPsfMoments",
                                       "base_ClassificationSizeExtendedness",
                                       ]
    self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
    self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
    self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
    self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
    self.forcedMeasurement.copyColumns = {
        "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
    self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
    self.forcedMeasurement.slots.shape = None

    # Keep track of which footprints contain streaks
    self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [
        "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
    self.measurement.plugins["base_PixelFlags"].masksFpCenter = [
        "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
    self.skySources.avoidMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "NO_DATA", "EDGE"]

    # Set the streak mask along the entire fit line, not only where the
    # detected mask is set.
    self.maskStreaks.onlyMaskDetected = False


class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):

ConfigClass = DetectAndMeasureConfig
_DefaultName = "detectAndMeasure"

def __init__(self, **kwargs):
    super().__init__(**kwargs)
    self.schema = afwTable.SourceTable.makeMinimalSchema()
    # Add coordinate error fields:
    afwTable.CoordKey.addErrorFields(self.schema)

    self.algMetadata = dafBase.PropertyList()
    self.makeSubtask("detection", schema=self.schema)
    self.makeSubtask("deblend", schema=self.schema)
    self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True)
    self.makeSubtask("measurement", schema=self.schema,
                     algMetadata=self.algMetadata)
    if self.config.doApCorr:
        self.makeSubtask("applyApCorr", schema=self.measurement.schema)
    if self.config.doForcedMeasurement:
        self.schema.addField(
            "ip_diffim_forced_PsfFlux_instFlux", "D",
            "Forced PSF flux measured on the direct image.",
            units="count")
        self.schema.addField(
            "ip_diffim_forced_PsfFlux_instFluxErr", "D",
            "Forced PSF flux error measured on the direct image.",
            units="count")
        self.schema.addField(
            "ip_diffim_forced_PsfFlux_area", "F",
            "Forced PSF flux effective area of PSF.",
            units="pixel")
        self.schema.addField(
            "ip_diffim_forced_PsfFlux_flag", "Flag",
            "Forced PSF flux general failure flag.")
        self.schema.addField(
            "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
            "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
        self.schema.addField(
            "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
            "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
        self.makeSubtask("forcedMeasurement", refSchema=self.schema)

    self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
    self.schema.addField("srcMatchId", "L", "unique id of source match")
    if self.config.doSkySources:
        self.makeSubtask("skySources", schema=self.schema)
    if self.config.doMaskStreaks:
        self.makeSubtask("maskStreaks")

    # Check that the schema and config are consistent
    for flag in self.config.badSourceFlags:
        if flag not in self.schema:
            raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag)

    # initialize InitOutputs
    self.outputSchema = afwTable.SourceCatalog(self.schema)
    self.outputSchema.getTable().setMetadata(self.algMetadata)

def runQuantum(self, butlerQC: pipeBase.QuantumContext,
               inputRefs: pipeBase.InputQuantizedConnection,
               outputRefs: pipeBase.OutputQuantizedConnection):
    inputs = butlerQC.get(inputRefs)
    idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
    idFactory = idGenerator.make_table_id_factory()
    outputs = self.run(**inputs, idFactory=idFactory)
    butlerQC.put(outputs, outputRefs)

@timeMethod
def run(self, science, matchedTemplate, difference,
        idFactory=None):

# Check that we have a valid PSF now before we do more work
sigma = difference.psf.computeShape(difference.psf.getAveragePosition()).getDeterminantRadius()
if np.isnan(sigma):
    raise pipeBase.UpstreamFailureNoWorkFound("Invalid PSF detected! PSF width evaluates to NaN.")
# Ensure that we start with an empty detection and deblended mask.
mask = difference.mask
for mp in self.config.clearMaskPlanes:
    if mp not in mask.getMaskPlaneDict():
        mask.addMaskPlane(mp)
mask &= ~mask.getPlaneBitMask(self.config.clearMaskPlanes)

def processResults(self, science, matchedTemplate, difference, sources, idFactory,
               positiveFootprints=None, negativeFootprints=None,):

streaks = self.maskStreaks.run(maskedImage)
if self.config.writeStreakInfo:
    rhos = np.array([line.rho for line in streaks.lines])
    thetas = np.array([line.theta for line in streaks.lines])
    sigmas = np.array([line.sigma for line in streaks.lines])
    chi2s = np.array([line.reducedChi2 for line in streaks.lines])
    modelMaximums = np.array([line.modelMaximum for line in streaks.lines])
    streakInfo = {'rho': rhos, 'theta': thetas, 'sigma': sigmas, 'reducedChi2': chi2s,
                  'modelMaximum': modelMaximums}
else:
    streakInfo = {'rho': np.array([]), 'theta': np.array([]), 'sigma': np.array([]),
                  'reducedChi2': np.array([]), 'modelMaximum': np.array([])}
return pipeBase.Struct(maskedStreaks=streakInfo)


class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
scoreExposure = pipeBase.connectionTypes.Input(
doc="Maximum likelihood image for detection.",
dimensions=("instrument", "visit", "detector"),
storageClass="ExposureF",
name="{fakesType}{coaddName}Diff_scoreExp",
)


class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
                          pipelineConnections=DetectAndMeasureScoreConnections):
pass


class DetectAndMeasureScoreTask(DetectAndMeasureTask):

ConfigClass = DetectAndMeasureScoreConfig
_DefaultName = "detectAndMeasureScore"

@timeMethod
def run(self, science, matchedTemplate, difference, scoreExposure,
        idFactory=None):

Definition at line 324 of file detectAndMeasure.py.

scoreExposure

lsst.ip.diffim.detectAndMeasure.scoreExposure : `lsst.afw.image.ExposureF`

Definition at line 742 of file detectAndMeasure.py.

seed

lsst.ip.diffim.detectAndMeasure.seed : `int`

Definition at line 557 of file detectAndMeasure.py.

sources

lsst.ip.diffim.detectAndMeasure.sources : `lsst.afw.table.SourceCatalog`

Definition at line 406 of file detectAndMeasure.py.

streakInfo

lsst.ip.diffim.detectAndMeasure.streakInfo : `lsst.pipe.base.Struct`

Definition at line 672 of file detectAndMeasure.py.

wcs

lsst.ip.diffim.detectAndMeasure.wcs : `lsst.afw.geom.SkyWcs`

Definition at line 605 of file detectAndMeasure.py.