24from lsstDebug
import getDebugFrame
29import lsst.pipe.base.connectionTypes
as cT
37from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
39from .measurePsf
import MeasurePsfTask
40from .repair
import RepairTask
41from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
43from lsst.utils.timer
import timeMethod
45__all__ = [
"CharacterizeImageConfig",
"CharacterizeImageTask"]
49 dimensions=(
"instrument",
"visit",
"detector")):
51 doc=
"Input exposure data",
53 storageClass=
"Exposure",
54 dimensions=[
"instrument",
"exposure",
"detector"],
56 characterized = cT.Output(
57 doc=
"Output characterized data.",
59 storageClass=
"ExposureF",
60 dimensions=[
"instrument",
"visit",
"detector"],
62 sourceCat = cT.Output(
63 doc=
"Output source catalog.",
65 storageClass=
"SourceCatalog",
66 dimensions=[
"instrument",
"visit",
"detector"],
68 backgroundModel = cT.Output(
69 doc=
"Output background model.",
70 name=
"icExpBackground",
71 storageClass=
"Background",
72 dimensions=[
"instrument",
"visit",
"detector"],
74 outputSchema = cT.InitOutput(
75 doc=
"Schema of the catalog produced by CharacterizeImage",
77 storageClass=
"SourceCatalog",
84 except pipeBase.ScalarError
as err:
85 raise pipeBase.ScalarError(
86 "CharacterizeImageTask can at present only be run on visits that are associated with "
87 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
88 "snap-combination step you probably want hasn't been configured to run between ISR and "
89 "this task (as of this writing, that would be because it hasn't been implemented yet)."
94 pipelineConnections=CharacterizeImageConnections):
96 """!Config for CharacterizeImageTask"""
97 doMeasurePsf = pexConfig.Field(
100 doc=
"Measure PSF? If False then for all subsequent operations use either existing PSF "
101 "model when present, or install simple PSF model when not (see installSimplePsf "
104 doWrite = pexConfig.Field(
107 doc=
"Persist results?",
109 doWriteExposure = pexConfig.Field(
112 doc=
"Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
114 psfIterations = pexConfig.RangeField(
118 doc=
"Number of iterations of detect sources, measure sources, "
119 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
120 "otherwise more may be wanted.",
122 background = pexConfig.ConfigurableField(
123 target=SubtractBackgroundTask,
124 doc=
"Configuration for initial background estimation",
126 detection = pexConfig.ConfigurableField(
127 target=SourceDetectionTask,
130 doDeblend = pexConfig.Field(
133 doc=
"Run deblender input exposure"
135 deblend = pexConfig.ConfigurableField(
136 target=SourceDeblendTask,
137 doc=
"Split blended source into their components"
139 measurement = pexConfig.ConfigurableField(
140 target=SingleFrameMeasurementTask,
141 doc=
"Measure sources"
143 doApCorr = pexConfig.Field(
146 doc=
"Run subtasks to measure and apply aperture corrections"
148 measureApCorr = pexConfig.ConfigurableField(
149 target=MeasureApCorrTask,
150 doc=
"Subtask to measure aperture corrections"
152 applyApCorr = pexConfig.ConfigurableField(
153 target=ApplyApCorrTask,
154 doc=
"Subtask to apply aperture corrections"
158 catalogCalculation = pexConfig.ConfigurableField(
159 target=CatalogCalculationTask,
160 doc=
"Subtask to run catalogCalculation plugins on catalog"
162 doComputeSummaryStats = pexConfig.Field(
165 doc=
"Run subtask to measure exposure summary statistics",
166 deprecated=(
"This subtask has been moved to CalibrateTask "
169 computeSummaryStats = pexConfig.ConfigurableField(
170 target=ComputeExposureSummaryStatsTask,
171 doc=
"Subtask to run computeSummaryStats on exposure",
172 deprecated=(
"This subtask has been moved to CalibrateTask "
175 useSimplePsf = pexConfig.Field(
178 doc=
"Replace the existing PSF model with a simplified version that has the same sigma "
179 "at the start of each PSF determination iteration? Doing so makes PSF determination "
180 "converge more robustly and quickly.",
182 installSimplePsf = pexConfig.ConfigurableField(
183 target=InstallGaussianPsfTask,
184 doc=
"Install a simple PSF model",
186 refObjLoader = pexConfig.ConfigurableField(
187 target=LoadIndexedReferenceObjectsTask,
188 deprecated=
"This field does nothing. Will be removed after v24 (see DM-34768).",
189 doc=
"reference object loader",
191 ref_match = pexConfig.ConfigurableField(
193 deprecated=
"This field was never usable. Will be removed after v24 (see DM-34768).",
194 doc=
"Task to load and match reference objects. Only used if measurePsf can use matches. "
195 "Warning: matching will only work well if the initial WCS is accurate enough "
196 "to give good matches (roughly: good to 3 arcsec across the CCD).",
198 measurePsf = pexConfig.ConfigurableField(
199 target=MeasurePsfTask,
202 repair = pexConfig.ConfigurableField(
204 doc=
"Remove cosmic rays",
206 requireCrForPsf = pexConfig.Field(
209 doc=
"Require cosmic ray detection and masking to run successfully before measuring the PSF."
211 checkUnitsParseStrict = pexConfig.Field(
212 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
221 self.
detectiondetection.thresholdValue = 5.0
222 self.
detectiondetection.includeThresholdMultiplier = 10.0
223 self.
detectiondetection.doTempLocalBackground =
False
236 "base_CircularApertureFlux",
241 raise RuntimeError(
"Must measure PSF to measure aperture correction, "
242 "because flags determined by PSF measurement are used to identify "
243 "sources used to measure aperture correction")
255 Measure bright sources and use this to estimate background
and PSF of an exposure
257 @anchor CharacterizeImageTask_
259 @section pipe_tasks_characterizeImage_Contents Contents
261 -
@ref pipe_tasks_characterizeImage_Purpose
262 -
@ref pipe_tasks_characterizeImage_Initialize
263 -
@ref pipe_tasks_characterizeImage_IO
264 -
@ref pipe_tasks_characterizeImage_Config
265 -
@ref pipe_tasks_characterizeImage_Debug
267 @section pipe_tasks_characterizeImage_Purpose Description
269 Given an exposure
with defects repaired (masked
and interpolated over, e.g.
as output by IsrTask):
270 - detect
and measure bright sources
272 - measure
and subtract background
275 @section pipe_tasks_characterizeImage_Initialize Task initialisation
277 @copydoc \_\_init\_\_
279 @section pipe_tasks_characterizeImage_IO Invoking the Task
281 If you want this task to unpersist inputs
or persist outputs, then call
282 the `runDataRef` method (a thin wrapper around the `run` method).
284 If you already have the inputs unpersisted
and do
not want to persist the output
285 then it
is more direct to call the `run` method:
287 @section pipe_tasks_characterizeImage_Config Configuration parameters
289 See
@ref CharacterizeImageConfig
291 @section pipe_tasks_characterizeImage_Debug Debug variables
293 The command line task interface supports a flag
294 `--debug` to
import `debug.py`
from your `$PYTHONPATH`; see
295 <a href=
"https://pipelines.lsst.io/modules/lsstDebug/">the lsstDebug documentation</a>
296 for more about `debug.py`.
298 CharacterizeImageTask has a debug dictionary
with the following keys:
301 <dd>int:
if specified, the frame of first debug image displayed (defaults to 1)
303 <dd>bool;
if True display image after each repair
in the measure PSF loop
305 <dd>bool;
if True display image after each background subtraction
in the measure PSF loop
307 <dd>bool;
if True display image
and sources at the end of each iteration of the measure PSF loop
308 See
@ref lsst.meas.astrom.displayAstrometry
for the meaning of the various symbols.
310 <dd>bool;
if True display image
and sources after PSF
is measured;
311 this will be identical to the final image displayed by measure_iter
if measure_iter
is true
313 <dd>bool;
if True display image
and sources after final repair
315 <dd>bool;
if True display image
and sources after final measurement
318 For example, put something like:
323 if name ==
"lsst.pipe.tasks.characterizeImage":
332 into your `debug.py` file
and run `calibrateTask.py`
with the `--debug` flag.
334 Some subtasks may have their own debug variables; see individual Task documentation.
339 ConfigClass = CharacterizeImageConfig
340 _DefaultName =
"characterizeImage"
341 RunnerClass = pipeBase.ButlerInitializedTaskRunner
344 inputs = butlerQC.get(inputRefs)
345 if 'exposureIdInfo' not in inputs.keys():
346 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
347 outputs = self.
runrun(**inputs)
348 butlerQC.put(outputs, outputRefs)
350 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
351 """!Construct a CharacterizeImageTask
353 @param[
in] butler A butler object
is passed to the refObjLoader constructor
in case
354 it
is needed to load catalogs. May be
None if a catalog-based star selector
is
355 not used,
if the reference object loader constructor does
not require a butler,
356 or if a reference object loader
is passed directly via the refObjLoader argument.
358 @param[
in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
359 external reference catalog to a catalog-based star selector. May be
None if a
360 catalog star selector
is not used
or the loader can be constructed
from the
363 @param[
in,out] kwargs other keyword arguments
for lsst.pipe.base.CmdLineTask
368 schema = SourceTable.makeMinimalSchema()
370 self.makeSubtask(
"background")
371 self.makeSubtask(
"installSimplePsf")
372 self.makeSubtask(
"repair")
373 self.makeSubtask(
"measurePsf", schema=self.
schemaschema)
375 if self.config.doMeasurePsf
and self.measurePsf.usesMatches:
377 self.makeSubtask(
'refObjLoader', butler=butler)
378 refObjLoader = self.refObjLoader
379 self.makeSubtask(
"ref_match", refObjLoader=refObjLoader)
381 self.makeSubtask(
'detection', schema=self.
schemaschema)
382 if self.config.doDeblend:
383 self.makeSubtask(
"deblend", schema=self.
schemaschema)
384 self.makeSubtask(
'measurement', schema=self.
schemaschema, algMetadata=self.
algMetadataalgMetadata)
385 if self.config.doApCorr:
386 self.makeSubtask(
'measureApCorr', schema=self.
schemaschema)
387 self.makeSubtask(
'applyApCorr', schema=self.
schemaschema)
388 self.makeSubtask(
'catalogCalculation', schema=self.
schemaschema)
391 self.
schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
396 outputCatSchema.getTable().setMetadata(self.
algMetadataalgMetadata)
397 return {
'outputSchema': outputCatSchema}
400 def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True):
401 """!Characterize a science image and, if wanted, persist the results
403 This simply unpacks the exposure and passes it to the characterize method to do the work.
405 @param[
in] dataRef: butler data reference
for science exposure
406 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar).
407 If
None then unpersist
from "postISRCCD".
408 The following changes are made, depending on the config:
409 - set psf to the measured PSF
410 - set apCorrMap to the measured aperture correction
411 - subtract background
412 - interpolate over cosmic rays
413 - update detection
and cosmic ray mask planes
414 @param[
in,out] background initial model of background already subtracted
from exposure
416 which
is typical
for image characterization.
417 A refined background model
is output.
418 @param[
in] doUnpersist
if True the exposure
is read
from the repository
419 and the exposure
and background arguments must be
None;
420 if False the exposure must be provided.
421 True is intended
for running
as a command-line task,
False for running
as a subtask
423 @return same data
as the characterize method
426 self.log.
info(
"Processing %s", dataRef.dataId)
429 if exposure
is not None or background
is not None:
430 raise RuntimeError(
"doUnpersist true; exposure and background must be None")
431 exposure = dataRef.get(
"postISRCCD", immediate=
True)
432 elif exposure
is None:
433 raise RuntimeError(
"doUnpersist false; exposure must be provided")
435 exposureIdInfo = dataRef.get(
"expIdInfo")
437 charRes = self.
runrun(
439 exposureIdInfo=exposureIdInfo,
440 background=background,
443 if self.config.doWrite:
444 dataRef.put(charRes.sourceCat,
"icSrc")
445 if self.config.doWriteExposure:
446 dataRef.put(charRes.exposure,
"icExp")
447 dataRef.put(charRes.background,
"icExpBackground")
452 def run(self, exposure, exposureIdInfo=None, background=None):
453 """!Characterize a science image
455 Peforms the following operations:
456 - Iterate the following config.psfIterations times, or once
if config.doMeasurePsf false:
457 - detect
and measure sources
and estimate PSF (see detectMeasureAndEstimatePsf
for details)
458 - interpolate over cosmic rays
459 - perform final measurement
461 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar).
462 The following changes are made:
465 - update detection
and cosmic ray mask planes
466 - subtract background
and interpolate over cosmic rays
467 @param[
in] exposureIdInfo ID info
for exposure (an lsst.obs.base.ExposureIdInfo).
468 If
not provided, returned SourceCatalog IDs will
not be globally unique.
469 @param[
in,out] background initial model of background already subtracted
from exposure
471 which
is typical
for image characterization.
473 @return pipe_base Struct containing these fields, all
from the final iteration
474 of detectMeasureAndEstimatePsf:
475 - exposure: characterized exposure; image
is repaired by interpolating over cosmic rays,
476 mask
is updated accordingly,
and the PSF model
is set
483 if not self.config.doMeasurePsf
and not exposure.hasPsf():
484 self.log.
info(
"CharacterizeImageTask initialized with 'simple' PSF.")
485 self.installSimplePsf.
run(exposure=exposure)
487 if exposureIdInfo
is None:
488 exposureIdInfo = ExposureIdInfo()
491 background = self.background.
run(exposure).background
493 psfIterations = self.config.psfIterations
if self.config.doMeasurePsf
else 1
494 for i
in range(psfIterations):
497 exposureIdInfo=exposureIdInfo,
498 background=background,
501 psf = dmeRes.exposure.getPsf()
503 psfAvgPos = psf.getAveragePosition()
504 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
505 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
506 medBackground = np.median(dmeRes.background.getImage().getArray())
507 self.log.
info(
"iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
508 i + 1, psfSigma, psfDimensions, medBackground)
509 if np.isnan(psfSigma):
510 raise RuntimeError(
"PSF sigma is NaN, cannot continue PSF determination.")
512 self.
displaydisplay(
"psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
515 self.repair.
run(exposure=dmeRes.exposure)
516 self.
displaydisplay(
"repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
520 self.measurement.
run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
521 exposureId=exposureIdInfo.expId)
522 if self.config.doApCorr:
523 apCorrMap = self.measureApCorr.
run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
524 dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
525 self.applyApCorr.
run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
526 self.catalogCalculation.
run(dmeRes.sourceCat)
528 self.
displaydisplay(
"measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
530 return pipeBase.Struct(
531 exposure=dmeRes.exposure,
532 sourceCat=dmeRes.sourceCat,
533 background=dmeRes.background,
534 psfCellSet=dmeRes.psfCellSet,
536 characterized=dmeRes.exposure,
537 backgroundModel=dmeRes.background
542 """!Perform one iteration of detect, measure and estimate PSF
544 Performs the following operations:
545 - if config.doMeasurePsf
or not exposure.hasPsf():
546 - install a simple PSF model (replacing the existing one,
if need be)
547 - interpolate over cosmic rays
with keepCRs=
True
548 - estimate background
and subtract it
from the exposure
549 - detect, deblend
and measure sources,
and subtract a refined background model;
550 -
if config.doMeasurePsf:
553 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar)
554 The following changes are made:
556 - update detection
and cosmic ray mask planes
557 - subtract background
558 @param[
in] exposureIdInfo ID info
for exposure (an lsst.obs_base.ExposureIdInfo)
559 @param[
in,out] background initial model of background already subtracted
from exposure
562 @return pipe_base Struct containing these fields, all
from the final iteration
563 of detect sources, measure sources
and estimate PSF:
564 - exposure characterized exposure; image
is repaired by interpolating over cosmic rays,
565 mask
is updated accordingly,
and the PSF model
is set
571 if not exposure.hasPsf()
or (self.config.doMeasurePsf
and self.config.useSimplePsf):
572 self.log.
info(
"PSF estimation initialized with 'simple' PSF")
573 self.installSimplePsf.
run(exposure=exposure)
576 if self.config.requireCrForPsf:
577 self.repair.
run(exposure=exposure, keepCRs=
True)
580 self.repair.
run(exposure=exposure, keepCRs=
True)
582 self.log.
warning(
"Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
583 self.config.repair.cosmicray.nCrPixelMax)
585 self.
displaydisplay(
"repair_iter", exposure=exposure)
587 if background
is None:
590 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
591 table = SourceTable.make(self.
schemaschema, sourceIdFactory)
594 detRes = self.detection.
run(table=table, exposure=exposure, doSmooth=
True)
595 sourceCat = detRes.sources
596 if detRes.fpSets.background:
597 for bg
in detRes.fpSets.background:
598 background.append(bg)
600 if self.config.doDeblend:
601 self.deblend.
run(exposure=exposure, sources=sourceCat)
603 self.measurement.
run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
605 measPsfRes = pipeBase.Struct(cellSet=
None)
606 if self.config.doMeasurePsf:
608 if self.measurePsf.usesMatches:
609 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
612 measPsfRes = self.measurePsf.
run(exposure=exposure, sources=sourceCat, matches=matches,
613 expId=exposureIdInfo.expId)
614 self.
displaydisplay(
"measure_iter", exposure=exposure, sourceCat=sourceCat)
616 return pipeBase.Struct(
619 background=background,
620 psfCellSet=measPsfRes.cellSet,
624 """Return a dict of empty catalogs for each catalog dataset produced by this task.
627 sourceCat.getTable().setMetadata(self.algMetadataalgMetadata)
628 return {
"icSrc": sourceCat}
630 def display(self, itemName, exposure, sourceCat=None):
631 """Display exposure and sources on next frame, if display of itemName has been requested
633 @param[
in] itemName name of item
in debugInfo
634 @param[
in] exposure exposure to display
635 @param[
in] sourceCat source catalog to display
A collection of SpatialCells covering an entire image.
Table class that contains measurements made on a single exposure.
Class for storing ordered metadata with comments.
Config for CharacterizeImageTask.
def adjustQuantum(self, inputs, outputs, label, dataId)
Measure bright sources and use this to estimate background and PSF of an exposure.
def getSchemaCatalogs(self)
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs)
Construct a CharacterizeImageTask.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getInitOutputDatasets(self)
def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True)
Characterize a science image and, if wanted, persist the results.
def run(self, exposure, exposureIdInfo=None, background=None)
Characterize a science image.
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background)
Perform one iteration of detect, measure and estimate PSF.
def display(self, itemName, exposure, sourceCat=None)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations.
def displayAstrometry(refCat=None, sourceCat=None, distortedCentroidKey=None, bbox=None, exposure=None, matches=None, frame=1, title="", pause=True)
def getDebugFrame(debugDisplay, name)