doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
doc="Add background to calexp before processing it. "
"Useful as ipDiffim does background matching.")
doUseRegister = pexConfig.Field(dtype=bool, default=False,
doc="Re-compute astrometry on the template. "
"Use image-to-image registration to align template with "
"science image (AL only).")
doDebugRegister = pexConfig.Field(dtype=bool, default=False,
doc="Writing debugging data for doUseRegister")
doSelectSources = pexConfig.Field(dtype=bool, default=False,
doc="Select stars to use for kernel fitting (AL only)")
doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
doc="Select stars of extreme color as part "
"of the control sample (AL only)")
doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
doc="Select stars that are variable to be part "
"of the control sample (AL only)")
doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
doPreConvolve = pexConfig.Field(dtype=bool, default=False,
doc="Not in use. Superseded by useScoreImageDetection.",
deprecated="This option superseded by useScoreImageDetection."
" Will be removed after v22.")
useScoreImageDetection = pexConfig.Field(
dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or "
"the Zogy score image. Use it for source detection (if doDetection=True).")
doWriteScoreExp = pexConfig.Field(
dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?")
doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
doc="Scale variance of the template before PSF matching")
doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
doc="Scale variance of the diffim before PSF matching. "
"You may do either this or template variance scaling, "
"or neither. (Doing both is a waste of CPU.)")
useGaussianForPreConvolution = pexConfig.Field(
dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution "
"(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
doDecorrelation = pexConfig.Field(dtype=bool, default=True,
doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
"kernel convolution (AL only)? If True, also update the diffim PSF.")
doMerge = pexConfig.Field(dtype=bool, default=True,
doc="Merge positive and negative diaSources with grow radius "
"set by growFootprint")
doMatchSources = pexConfig.Field(dtype=bool, default=False,
doc="Match diaSources with input calexp sources and ref catalog sources")
doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
doForcedMeasurement = pexConfig.Field(
dtype=bool,
default=True,
doc="Force photometer diaSource locations on PVI?")
doWriteSubtractedExp = pexConfig.Field(
dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
doWriteWarpedExp = pexConfig.Field(
dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
doc="Write warped and PSF-matched template coadd exposure?")
doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
doAddMetrics = pexConfig.Field(dtype=bool, default=False,
doc="Add columns to the source table to hold analysis metrics?")
doApplyFinalizedPsf = pexConfig.Field(
doc="Whether to apply finalized psf models and aperture correction map.",
dtype=bool,
default=False,
)
coaddName = pexConfig.Field(
doc="coadd name: typically one of deep, goodSeeing, or dcr",
dtype=str,
default="deep",
)
convolveTemplate = pexConfig.Field(
doc="Which image gets convolved (default = template)",
dtype=bool,
default=True
)
refObjLoader = pexConfig.ConfigurableField(
target=LoadIndexedReferenceObjectsTask,
doc="reference object loader",
)
astrometer = pexConfig.ConfigurableField(
target=AstrometryTask,
doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
)
sourceSelector = pexConfig.ConfigurableField(
target=ObjectSizeStarSelectorTask,
doc="Source selection algorithm",
)
subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
decorrelate = pexConfig.ConfigurableField(
target=DecorrelateALKernelSpatialTask,
doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
"If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
"default of 5.5).",
)
# Old style ImageMapper grid. ZogyTask has its own grid option
doSpatiallyVarying = pexConfig.Field(
dtype=bool,
default=False,
doc="Perform A&L decorrelation on a grid across the "
"image in order to allow for spatial variations. Zogy does not use this option."
)
detection = pexConfig.ConfigurableField(
target=SourceDetectionTask,
doc="Low-threshold detection for final measurement",
)
measurement = pexConfig.ConfigurableField(
target=DipoleFitTask,
doc="Enable updated dipole fitting method",
)
doApCorr = lsst.pex.config.Field(
dtype=bool,
default=True,
doc="Run subtask to apply aperture corrections"
)
applyApCorr = lsst.pex.config.ConfigurableField(
target=ApplyApCorrTask,
doc="Subtask to apply aperture corrections"
)
forcedMeasurement = pexConfig.ConfigurableField(
target=ForcedMeasurementTask,
doc="Subtask to force photometer PVI at diaSource location.",
)
getTemplate = pexConfig.ConfigurableField(
target=GetCoaddAsTemplateTask,
doc="Subtask to retrieve template exposure and sources",
)
scaleVariance = pexConfig.ConfigurableField(
target=ScaleVarianceTask,
doc="Subtask to rescale the variance of the template "
"to the statistically expected level"
)
controlStepSize = pexConfig.Field(
doc="What step size (every Nth one) to select a control sample from the kernelSources",
dtype=int,
default=5
)
controlRandomSeed = pexConfig.Field(
doc="Random seed for shuffing the control sample",
dtype=int,
default=10
)
register = pexConfig.ConfigurableField(
target=RegisterTask,
doc="Task to enable image-to-image image registration (warping)",
)
kernelSourcesFromRef = pexConfig.Field(
doc="Select sources to measure kernel from reference catalog if True, template if false",
dtype=bool,
default=False
)
templateSipOrder = pexConfig.Field(
dtype=int, default=2,
doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
)
growFootprint = pexConfig.Field(
dtype=int, default=2,
doc="Grow positive and negative footprints by this amount before merging"
)
diaSourceMatchRadius = pexConfig.Field(
dtype=float, default=0.5,
doc="Match radius (in arcseconds) for DiaSource to Source association"
)
requiredTemplateFraction = pexConfig.Field(
dtype=float, default=0.1,
doc="Do not attempt to run task if template covers less than this fraction of pixels."
"Setting to 0 will always attempt image subtraction"
)
doSkySources = pexConfig.Field(
dtype=bool,
default=False,
doc="Generate sky sources?",
)
skySources = pexConfig.ConfigurableField(
target=SkyObjectsTask,
doc="Generate sky sources",
)
def setDefaults(self):
# defaults are OK for catalog and diacatalog
self.subtract['al'].kernel.name = "AL"
self.subtract['al'].kernel.active.fitForBackground = True
self.subtract['al'].kernel.active.spatialKernelOrder = 1
self.subtract['al'].kernel.active.spatialBgOrder = 2
# DiaSource Detection
self.detection.thresholdPolarity = "both"
self.detection.thresholdValue = 5.0
self.detection.reEstimateBackground = False
self.detection.thresholdType = "pixel_stdev"
# Add filtered flux measurement, the correct measurement for pre-convolved images.
# Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
# To change that you must modify algorithms.names in the task's applyOverrides method,
# after the user has set doPreConvolve.
self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
self.measurement.plugins.names |= ['ext_trailedSources_Naive',
'base_LocalPhotoCalib',
'base_LocalWcs']
self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
self.forcedMeasurement.copyColumns = {
"id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
self.forcedMeasurement.slots.shape = None
# For shuffling the control sample
random.seed(self.controlRandomSeed)
def validate(self):
pexConfig.Config.validate(self)
if not self.doSubtract and not self.doDetection:
raise ValueError("Either doSubtract or doDetection must be enabled.")
if self.doMeasurement and not self.doDetection:
raise ValueError("Cannot run source measurement without source detection.")
if self.doMerge and not self.doDetection:
raise ValueError("Cannot run source merging without source detection.")
if self.doSkySources and not self.doDetection:
raise ValueError("Cannot run sky source creation without source detection.")
if self.doUseRegister and not self.doSelectSources:
raise ValueError("doUseRegister=True and doSelectSources=False. "
"Cannot run RegisterTask without selecting sources.")
if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
raise ValueError("Scaling the diffim variance and scaling the template variance "
"are both set. Please choose one or the other.")
# We cannot allow inconsistencies that would lead to None or not available output products
if self.subtract.name == 'zogy':
if self.doWriteMatchedExp:
raise ValueError("doWriteMatchedExp=True Matched exposure is not "
"calculated in zogy subtraction.")
if self.doAddMetrics:
raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
if self.doDecorrelation:
raise ValueError(
"doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
if self.doSelectSources:
raise ValueError(
"doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
if self.useGaussianForPreConvolution:
raise ValueError(
"useGaussianForPreConvolution=True This is an AL subtraction only option.")
else:
# AL only consistency checks
if self.useScoreImageDetection and not self.convolveTemplate:
raise ValueError(
"convolveTemplate=False and useScoreImageDetection=True "
"Pre-convolution and matching of the science image is not a supported operation.")
if self.doWriteSubtractedExp and self.useScoreImageDetection:
raise ValueError(
"doWriteSubtractedExp=True and useScoreImageDetection=True "
"Regular difference image is not calculated. "
"AL subtraction calculates either the regular difference image or the score image.")
if self.doWriteScoreExp and not self.useScoreImageDetection:
raise ValueError(
"doWriteScoreExp=True and useScoreImageDetection=False "
"Score image is not calculated. "
"AL subtraction calculates either the regular difference image or the score image.")
if self.doAddMetrics and not self.doSubtract:
raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
if self.useGaussianForPreConvolution and not self.useScoreImageDetection:
raise ValueError(
"useGaussianForPreConvolution=True and useScoreImageDetection=False "
"Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
@staticmethod
def getTargetList(parsedCmd, **kwargs):
return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
**kwargs)
class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
ConfigClass = ImageDifferenceConfig
RunnerClass = ImageDifferenceTaskRunner
_DefaultName = "imageDifference"
def __init__(self, butler=None, **kwargs):
super().__init__(**kwargs)
self.makeSubtask("getTemplate")
self.makeSubtask("subtract")
if self.config.subtract.name == 'al' and self.config.doDecorrelation:
self.makeSubtask("decorrelate")
if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
self.makeSubtask("scaleVariance")
if self.config.doUseRegister:
self.makeSubtask("register")
self.schema = afwTable.SourceTable.makeMinimalSchema()
if self.config.doSelectSources:
self.makeSubtask("sourceSelector")
if self.config.kernelSourcesFromRef:
self.makeSubtask('refObjLoader', butler=butler)
self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
self.algMetadata = dafBase.PropertyList()
if self.config.doDetection:
self.makeSubtask("detection", schema=self.schema)
if self.config.doMeasurement:
self.makeSubtask("measurement", schema=self.schema,
algMetadata=self.algMetadata)
if self.config.doApCorr:
self.makeSubtask("applyApCorr", schema=self.measurement.schema)
if self.config.doForcedMeasurement:
self.schema.addField(
"ip_diffim_forced_PsfFlux_instFlux", "D",
"Forced PSF flux measured on the direct image.",
units="count")
self.schema.addField(
"ip_diffim_forced_PsfFlux_instFluxErr", "D",
"Forced PSF flux error measured on the direct image.",
units="count")
self.schema.addField(
"ip_diffim_forced_PsfFlux_area", "F",
"Forced PSF flux effective area of PSF.",
units="pixel")
self.schema.addField(
"ip_diffim_forced_PsfFlux_flag", "Flag",
"Forced PSF flux general failure flag.")
self.schema.addField(
"ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
"Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
self.schema.addField(
"ip_diffim_forced_PsfFlux_flag_edge", "Flag",
"Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
self.makeSubtask("forcedMeasurement", refSchema=self.schema)
if self.config.doMatchSources:
self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
self.schema.addField("srcMatchId", "L", "unique id of source match")
if self.config.doSkySources:
self.makeSubtask("skySources")
self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
# initialize InitOutputs
self.outputSchema = afwTable.SourceCatalog(self.schema)
self.outputSchema.getTable().setMetadata(self.algMetadata)
@staticmethod
def makeIdFactory(expId, expBits):
return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
@lsst.utils.inheritDoc(pipeBase.PipelineTask)
def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
inputRefs: pipeBase.InputQuantizedConnection,
outputRefs: pipeBase.OutputQuantizedConnection):
inputs = butlerQC.get(inputRefs)
self.log.info("Processing %s", butlerQC.quantum.dataId)
finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None)
exposure = self.prepareCalibratedExposure(
inputs["exposure"],
finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog
)
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
returnMaxBits=True)
idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
if self.config.coaddName == 'dcr':
templateExposures = inputRefs.dcrCoadds
else:
templateExposures = inputRefs.coaddExposures
templateStruct = self.getTemplate.runQuantum(
exposure, butlerQC, inputRefs.skyMap, templateExposures
)
self.checkTemplateIsSufficient(templateStruct.exposure)
outputs = self.run(exposure=exposure,
templateExposure=templateStruct.exposure,
idFactory=idFactory)
# Consistency with runDataref gen2 handling
if outputs.diaSources is None:
del outputs.diaSources
butlerQC.put(outputs, outputRefs)
@timeMethod
def runDataRef(self, sensorRef, templateIdList=None):
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
subtractedExposure = None
selectSources = None
calexpBackgroundExposure = None
self.log.info("Processing %s", sensorRef.dataId)
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
expBits=sensorRef.get("ccdExposureId_bits"))
if self.config.doAddCalexpBackground:
calexpBackgroundExposure = sensorRef.get("calexpBackground")
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
# Retrieve the template image
template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
if sensorRef.datasetExists("src"):
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
if not self.config.doSubtract and self.config.doDetection:
# If we don't do subtraction, we need the subtracted exposure from the repo
subtractedExposure = sensorRef.get(subtractedExposureName)
# Both doSubtract and doDetection cannot be False
results = self.run(exposure=exposure,
selectSources=selectSources,
templateExposure=template.exposure,
templateSources=template.sources,
idFactory=idFactory,
calexpBackgroundExposure=calexpBackgroundExposure,
subtractedExposure=subtractedExposure)
if self.config.doWriteSources and results.diaSources is not None:
sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
if self.config.doWriteWarpedExp:
sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
if self.config.doWriteMatchedExp:
sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
if self.config.doAddMetrics and self.config.doSelectSources:
sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(results.subtractedExposure, subtractedExposureName)
if self.config.doWriteScoreExp:
sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
return results
def prepareCalibratedExposure(self, exposure, finalizedPsfApCorrCatalog=None):
Definition at line 668 of file imageDifference.py.
