24 from lsstDebug
import getDebugFrame
29 import lsst.pipe.base.connectionTypes
as cT
37 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
39 from .measurePsf
import MeasurePsfTask
40 from .repair
import RepairTask
41 from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
43 from lsst.utils.timer
import timeMethod
45 __all__ = [
"CharacterizeImageConfig",
"CharacterizeImageTask"]
49 dimensions=(
"instrument",
"visit",
"detector")):
51 doc=
"Input exposure data",
53 storageClass=
"Exposure",
54 dimensions=[
"instrument",
"exposure",
"detector"],
56 characterized = cT.Output(
57 doc=
"Output characterized data.",
59 storageClass=
"ExposureF",
60 dimensions=[
"instrument",
"visit",
"detector"],
62 sourceCat = cT.Output(
63 doc=
"Output source catalog.",
65 storageClass=
"SourceCatalog",
66 dimensions=[
"instrument",
"visit",
"detector"],
68 backgroundModel = cT.Output(
69 doc=
"Output background model.",
70 name=
"icExpBackground",
71 storageClass=
"Background",
72 dimensions=[
"instrument",
"visit",
"detector"],
74 outputSchema = cT.InitOutput(
75 doc=
"Schema of the catalog produced by CharacterizeImage",
77 storageClass=
"SourceCatalog",
84 except pipeBase.ScalarError
as err:
85 raise pipeBase.ScalarError(
86 "CharacterizeImageTask can at present only be run on visits that are associated with "
87 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
88 "snap-combination step you probably want hasn't been configured to run between ISR and "
89 "this task (as of this writing, that would be because it hasn't been implemented yet)."
94 pipelineConnections=CharacterizeImageConnections):
96 """!Config for CharacterizeImageTask"""
97 doMeasurePsf = pexConfig.Field(
100 doc=
"Measure PSF? If False then for all subsequent operations use either existing PSF "
101 "model when present, or install simple PSF model when not (see installSimplePsf "
104 doWrite = pexConfig.Field(
107 doc=
"Persist results?",
109 doWriteExposure = pexConfig.Field(
112 doc=
"Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
114 psfIterations = pexConfig.RangeField(
118 doc=
"Number of iterations of detect sources, measure sources, "
119 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
120 "otherwise more may be wanted.",
122 background = pexConfig.ConfigurableField(
123 target=SubtractBackgroundTask,
124 doc=
"Configuration for initial background estimation",
126 detection = pexConfig.ConfigurableField(
127 target=SourceDetectionTask,
130 doDeblend = pexConfig.Field(
133 doc=
"Run deblender input exposure"
135 deblend = pexConfig.ConfigurableField(
136 target=SourceDeblendTask,
137 doc=
"Split blended source into their components"
139 measurement = pexConfig.ConfigurableField(
140 target=SingleFrameMeasurementTask,
141 doc=
"Measure sources"
143 doApCorr = pexConfig.Field(
146 doc=
"Run subtasks to measure and apply aperture corrections"
148 measureApCorr = pexConfig.ConfigurableField(
149 target=MeasureApCorrTask,
150 doc=
"Subtask to measure aperture corrections"
152 applyApCorr = pexConfig.ConfigurableField(
153 target=ApplyApCorrTask,
154 doc=
"Subtask to apply aperture corrections"
158 catalogCalculation = pexConfig.ConfigurableField(
159 target=CatalogCalculationTask,
160 doc=
"Subtask to run catalogCalculation plugins on catalog"
162 doComputeSummaryStats = pexConfig.Field(
165 doc=
"Run subtask to measure exposure summary statistics",
166 deprecated=(
"This subtask has been moved to CalibrateTask "
169 computeSummaryStats = pexConfig.ConfigurableField(
170 target=ComputeExposureSummaryStatsTask,
171 doc=
"Subtask to run computeSummaryStats on exposure",
172 deprecated=(
"This subtask has been moved to CalibrateTask "
175 useSimplePsf = pexConfig.Field(
178 doc=
"Replace the existing PSF model with a simplified version that has the same sigma "
179 "at the start of each PSF determination iteration? Doing so makes PSF determination "
180 "converge more robustly and quickly.",
182 installSimplePsf = pexConfig.ConfigurableField(
183 target=InstallGaussianPsfTask,
184 doc=
"Install a simple PSF model",
186 refObjLoader = pexConfig.ConfigurableField(
187 target=LoadIndexedReferenceObjectsTask,
188 doc=
"reference object loader",
190 ref_match = pexConfig.ConfigurableField(
192 doc=
"Task to load and match reference objects. Only used if measurePsf can use matches. "
193 "Warning: matching will only work well if the initial WCS is accurate enough "
194 "to give good matches (roughly: good to 3 arcsec across the CCD).",
196 measurePsf = pexConfig.ConfigurableField(
197 target=MeasurePsfTask,
200 repair = pexConfig.ConfigurableField(
202 doc=
"Remove cosmic rays",
204 requireCrForPsf = pexConfig.Field(
207 doc=
"Require cosmic ray detection and masking to run successfully before measuring the PSF."
209 checkUnitsParseStrict = pexConfig.Field(
210 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
219 self.
detectiondetection.thresholdValue = 5.0
220 self.
detectiondetection.includeThresholdMultiplier = 10.0
221 self.
detectiondetection.doTempLocalBackground =
False
234 "base_CircularApertureFlux",
239 raise RuntimeError(
"Must measure PSF to measure aperture correction, "
240 "because flags determined by PSF measurement are used to identify "
241 "sources used to measure aperture correction")
252 r"""!Measure bright sources and use this to estimate background and PSF of an exposure
254 @anchor CharacterizeImageTask_
256 @section pipe_tasks_characterizeImage_Contents Contents
258 - @ref pipe_tasks_characterizeImage_Purpose
259 - @ref pipe_tasks_characterizeImage_Initialize
260 - @ref pipe_tasks_characterizeImage_IO
261 - @ref pipe_tasks_characterizeImage_Config
262 - @ref pipe_tasks_characterizeImage_Debug
265 @section pipe_tasks_characterizeImage_Purpose Description
267 Given an exposure with defects repaired (masked and interpolated over, e.g. as output by IsrTask):
268 - detect and measure bright sources
270 - measure and subtract background
273 @section pipe_tasks_characterizeImage_Initialize Task initialisation
275 @copydoc \_\_init\_\_
277 @section pipe_tasks_characterizeImage_IO Invoking the Task
279 If you want this task to unpersist inputs or persist outputs, then call
280 the `runDataRef` method (a thin wrapper around the `run` method).
282 If you already have the inputs unpersisted and do not want to persist the output
283 then it is more direct to call the `run` method:
285 @section pipe_tasks_characterizeImage_Config Configuration parameters
287 See @ref CharacterizeImageConfig
289 @section pipe_tasks_characterizeImage_Debug Debug variables
291 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
292 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
294 CharacterizeImageTask has a debug dictionary with the following keys:
297 <dd>int: if specified, the frame of first debug image displayed (defaults to 1)
299 <dd>bool; if True display image after each repair in the measure PSF loop
301 <dd>bool; if True display image after each background subtraction in the measure PSF loop
303 <dd>bool; if True display image and sources at the end of each iteration of the measure PSF loop
304 See @ref lsst.meas.astrom.displayAstrometry for the meaning of the various symbols.
306 <dd>bool; if True display image and sources after PSF is measured;
307 this will be identical to the final image displayed by measure_iter if measure_iter is true
309 <dd>bool; if True display image and sources after final repair
311 <dd>bool; if True display image and sources after final measurement
314 For example, put something like:
318 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
319 if name == "lsst.pipe.tasks.characterizeImage":
326 lsstDebug.Info = DebugInfo
328 into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag.
330 Some subtasks may have their own debug variables; see individual Task documentation.
335 ConfigClass = CharacterizeImageConfig
336 _DefaultName =
"characterizeImage"
337 RunnerClass = pipeBase.ButlerInitializedTaskRunner
340 inputs = butlerQC.get(inputRefs)
341 if 'exposureIdInfo' not in inputs.keys():
342 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
343 outputs = self.
runrun(**inputs)
344 butlerQC.put(outputs, outputRefs)
346 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
347 """!Construct a CharacterizeImageTask
349 @param[in] butler A butler object is passed to the refObjLoader constructor in case
350 it is needed to load catalogs. May be None if a catalog-based star selector is
351 not used, if the reference object loader constructor does not require a butler,
352 or if a reference object loader is passed directly via the refObjLoader argument.
353 @param[in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
354 external reference catalog to a catalog-based star selector. May be None if a
355 catalog star selector is not used or the loader can be constructed from the
357 @param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
358 @param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
363 schema = SourceTable.makeMinimalSchema()
365 self.makeSubtask(
"background")
366 self.makeSubtask(
"installSimplePsf")
367 self.makeSubtask(
"repair")
368 self.makeSubtask(
"measurePsf", schema=self.
schemaschema)
369 if self.config.doMeasurePsf
and self.measurePsf.usesMatches:
371 self.makeSubtask(
'refObjLoader', butler=butler)
372 refObjLoader = self.refObjLoader
373 self.makeSubtask(
"ref_match", refObjLoader=refObjLoader)
375 self.makeSubtask(
'detection', schema=self.
schemaschema)
376 if self.config.doDeblend:
377 self.makeSubtask(
"deblend", schema=self.
schemaschema)
378 self.makeSubtask(
'measurement', schema=self.
schemaschema, algMetadata=self.
algMetadataalgMetadata)
379 if self.config.doApCorr:
380 self.makeSubtask(
'measureApCorr', schema=self.
schemaschema)
381 self.makeSubtask(
'applyApCorr', schema=self.
schemaschema)
382 self.makeSubtask(
'catalogCalculation', schema=self.
schemaschema)
385 self.
schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
390 outputCatSchema.getTable().setMetadata(self.
algMetadataalgMetadata)
391 return {
'outputSchema': outputCatSchema}
394 def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True):
395 """!Characterize a science image and, if wanted, persist the results
397 This simply unpacks the exposure and passes it to the characterize method to do the work.
399 @param[in] dataRef: butler data reference for science exposure
400 @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
401 If None then unpersist from "postISRCCD".
402 The following changes are made, depending on the config:
403 - set psf to the measured PSF
404 - set apCorrMap to the measured aperture correction
405 - subtract background
406 - interpolate over cosmic rays
407 - update detection and cosmic ray mask planes
408 @param[in,out] background initial model of background already subtracted from exposure
409 (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
410 which is typical for image characterization.
411 A refined background model is output.
412 @param[in] doUnpersist if True the exposure is read from the repository
413 and the exposure and background arguments must be None;
414 if False the exposure must be provided.
415 True is intended for running as a command-line task, False for running as a subtask
417 @return same data as the characterize method
420 self.log.
info(
"Processing %s", dataRef.dataId)
423 if exposure
is not None or background
is not None:
424 raise RuntimeError(
"doUnpersist true; exposure and background must be None")
425 exposure = dataRef.get(
"postISRCCD", immediate=
True)
426 elif exposure
is None:
427 raise RuntimeError(
"doUnpersist false; exposure must be provided")
429 exposureIdInfo = dataRef.get(
"expIdInfo")
431 charRes = self.
runrun(
433 exposureIdInfo=exposureIdInfo,
434 background=background,
437 if self.config.doWrite:
438 dataRef.put(charRes.sourceCat,
"icSrc")
439 if self.config.doWriteExposure:
440 dataRef.put(charRes.exposure,
"icExp")
441 dataRef.put(charRes.background,
"icExpBackground")
446 def run(self, exposure, exposureIdInfo=None, background=None):
447 """!Characterize a science image
449 Peforms the following operations:
450 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
451 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
452 - interpolate over cosmic rays
453 - perform final measurement
455 @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
456 The following changes are made:
459 - update detection and cosmic ray mask planes
460 - subtract background and interpolate over cosmic rays
461 @param[in] exposureIdInfo ID info for exposure (an lsst.obs.base.ExposureIdInfo).
462 If not provided, returned SourceCatalog IDs will not be globally unique.
463 @param[in,out] background initial model of background already subtracted from exposure
464 (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
465 which is typical for image characterization.
467 @return pipe_base Struct containing these fields, all from the final iteration
468 of detectMeasureAndEstimatePsf:
469 - exposure: characterized exposure; image is repaired by interpolating over cosmic rays,
470 mask is updated accordingly, and the PSF model is set
471 - sourceCat: detected sources (an lsst.afw.table.SourceCatalog)
472 - background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
473 - psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
477 if not self.config.doMeasurePsf
and not exposure.hasPsf():
478 self.log.
info(
"CharacterizeImageTask initialized with 'simple' PSF.")
479 self.installSimplePsf.
run(exposure=exposure)
481 if exposureIdInfo
is None:
482 exposureIdInfo = ExposureIdInfo()
485 background = self.background.
run(exposure).background
487 psfIterations = self.config.psfIterations
if self.config.doMeasurePsf
else 1
488 for i
in range(psfIterations):
491 exposureIdInfo=exposureIdInfo,
492 background=background,
495 psf = dmeRes.exposure.getPsf()
496 psfSigma = psf.computeShape().getDeterminantRadius()
497 psfDimensions = psf.computeImage().getDimensions()
498 medBackground = np.median(dmeRes.background.getImage().getArray())
499 self.log.
info(
"iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
500 i + 1, psfSigma, psfDimensions, medBackground)
502 self.
displaydisplay(
"psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
505 self.repair.
run(exposure=dmeRes.exposure)
506 self.
displaydisplay(
"repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
510 self.measurement.
run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
511 exposureId=exposureIdInfo.expId)
512 if self.config.doApCorr:
513 apCorrMap = self.measureApCorr.
run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
514 dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
515 self.applyApCorr.
run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
516 self.catalogCalculation.
run(dmeRes.sourceCat)
518 self.
displaydisplay(
"measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
520 return pipeBase.Struct(
521 exposure=dmeRes.exposure,
522 sourceCat=dmeRes.sourceCat,
523 background=dmeRes.background,
524 psfCellSet=dmeRes.psfCellSet,
526 characterized=dmeRes.exposure,
527 backgroundModel=dmeRes.background
532 """!Perform one iteration of detect, measure and estimate PSF
534 Performs the following operations:
535 - if config.doMeasurePsf or not exposure.hasPsf():
536 - install a simple PSF model (replacing the existing one, if need be)
537 - interpolate over cosmic rays with keepCRs=True
538 - estimate background and subtract it from the exposure
539 - detect, deblend and measure sources, and subtract a refined background model;
540 - if config.doMeasurePsf:
543 @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar)
544 The following changes are made:
546 - update detection and cosmic ray mask planes
547 - subtract background
548 @param[in] exposureIdInfo ID info for exposure (an lsst.obs_base.ExposureIdInfo)
549 @param[in,out] background initial model of background already subtracted from exposure
550 (an lsst.afw.math.BackgroundList).
552 @return pipe_base Struct containing these fields, all from the final iteration
553 of detect sources, measure sources and estimate PSF:
554 - exposure characterized exposure; image is repaired by interpolating over cosmic rays,
555 mask is updated accordingly, and the PSF model is set
556 - sourceCat detected sources (an lsst.afw.table.SourceCatalog)
557 - background model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
558 - psfCellSet spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
561 if not exposure.hasPsf()
or (self.config.doMeasurePsf
and self.config.useSimplePsf):
562 self.log.
info(
"PSF estimation initialized with 'simple' PSF")
563 self.installSimplePsf.
run(exposure=exposure)
566 if self.config.requireCrForPsf:
567 self.repair.
run(exposure=exposure, keepCRs=
True)
570 self.repair.
run(exposure=exposure, keepCRs=
True)
572 self.log.
warning(
"Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
573 self.config.repair.cosmicray.nCrPixelMax)
575 self.
displaydisplay(
"repair_iter", exposure=exposure)
577 if background
is None:
580 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
581 table = SourceTable.make(self.
schemaschema, sourceIdFactory)
584 detRes = self.detection.
run(table=table, exposure=exposure, doSmooth=
True)
585 sourceCat = detRes.sources
586 if detRes.fpSets.background:
587 for bg
in detRes.fpSets.background:
588 background.append(bg)
590 if self.config.doDeblend:
591 self.deblend.
run(exposure=exposure, sources=sourceCat)
593 self.measurement.
run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
595 measPsfRes = pipeBase.Struct(cellSet=
None)
596 if self.config.doMeasurePsf:
597 if self.measurePsf.usesMatches:
598 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
601 measPsfRes = self.measurePsf.
run(exposure=exposure, sources=sourceCat, matches=matches,
602 expId=exposureIdInfo.expId)
603 self.
displaydisplay(
"measure_iter", exposure=exposure, sourceCat=sourceCat)
605 return pipeBase.Struct(
608 background=background,
609 psfCellSet=measPsfRes.cellSet,
613 """Return a dict of empty catalogs for each catalog dataset produced by this task.
616 sourceCat.getTable().setMetadata(self.
algMetadataalgMetadata)
617 return {
"icSrc": sourceCat}
619 def display(self, itemName, exposure, sourceCat=None):
620 """Display exposure and sources on next frame, if display of itemName has been requested
622 @param[in] itemName name of item in debugInfo
623 @param[in] exposure exposure to display
624 @param[in] sourceCat source catalog to display
Class for storing ordered metadata with comments.
Config for CharacterizeImageTask.
def adjustQuantum(self, inputs, outputs, label, dataId)
Measure bright sources and use this to estimate background and PSF of an exposure.
def getSchemaCatalogs(self)
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs)
Construct a CharacterizeImageTask.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getInitOutputDatasets(self)
def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True)
Characterize a science image and, if wanted, persist the results.
def run(self, exposure, exposureIdInfo=None, background=None)
Characterize a science image.
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background)
Perform one iteration of detect, measure and estimate PSF.
def display(self, itemName, exposure, sourceCat=None)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations.
def displayAstrometry(refCat=None, sourceCat=None, distortedCentroidKey=None, bbox=None, exposure=None, matches=None, frame=1, title="", pause=True)
def getDebugFrame(debugDisplay, name)