24 from lsstDebug
import getDebugFrame
37 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
39 from .measurePsf
import MeasurePsfTask
40 from .repair
import RepairTask
41 from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
44 __all__ = [
"CharacterizeImageConfig",
"CharacterizeImageTask"]
48 dimensions=(
"instrument",
"visit",
"detector")):
50 doc=
"Input exposure data",
52 storageClass=
"Exposure",
53 dimensions=[
"instrument",
"exposure",
"detector"],
55 characterized = cT.Output(
56 doc=
"Output characterized data.",
58 storageClass=
"ExposureF",
59 dimensions=[
"instrument",
"visit",
"detector"],
61 sourceCat = cT.Output(
62 doc=
"Output source catalog.",
64 storageClass=
"SourceCatalog",
65 dimensions=[
"instrument",
"visit",
"detector"],
67 backgroundModel = cT.Output(
68 doc=
"Output background model.",
69 name=
"icExpBackground",
70 storageClass=
"Background",
71 dimensions=[
"instrument",
"visit",
"detector"],
73 outputSchema = cT.InitOutput(
74 doc=
"Schema of the catalog produced by CharacterizeImage",
76 storageClass=
"SourceCatalog",
79 def adjustQuantum(self, datasetRefMap: pipeBase.InputQuantizedConnection):
83 except pipeBase.ScalarError
as err:
84 raise pipeBase.ScalarError(
85 "CharacterizeImageTask can at present only be run on visits that are associated with "
86 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
87 "snap-combination step you probably want hasn't been configured to run between ISR and "
88 "this task (as of this writing, that would be because it hasn't been implemented yet)."
93 pipelineConnections=CharacterizeImageConnections):
95 """!Config for CharacterizeImageTask"""
96 doMeasurePsf = pexConfig.Field(
99 doc=
"Measure PSF? If False then for all subsequent operations use either existing PSF "
100 "model when present, or install simple PSF model when not (see installSimplePsf "
103 doWrite = pexConfig.Field(
106 doc=
"Persist results?",
108 doWriteExposure = pexConfig.Field(
111 doc=
"Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
113 psfIterations = pexConfig.RangeField(
117 doc=
"Number of iterations of detect sources, measure sources, "
118 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
119 "otherwise more may be wanted.",
121 background = pexConfig.ConfigurableField(
122 target=SubtractBackgroundTask,
123 doc=
"Configuration for initial background estimation",
125 detection = pexConfig.ConfigurableField(
126 target=SourceDetectionTask,
129 doDeblend = pexConfig.Field(
132 doc=
"Run deblender input exposure"
134 deblend = pexConfig.ConfigurableField(
135 target=SourceDeblendTask,
136 doc=
"Split blended source into their components"
138 measurement = pexConfig.ConfigurableField(
139 target=SingleFrameMeasurementTask,
140 doc=
"Measure sources"
142 doApCorr = pexConfig.Field(
145 doc=
"Run subtasks to measure and apply aperture corrections"
147 measureApCorr = pexConfig.ConfigurableField(
148 target=MeasureApCorrTask,
149 doc=
"Subtask to measure aperture corrections"
151 applyApCorr = pexConfig.ConfigurableField(
152 target=ApplyApCorrTask,
153 doc=
"Subtask to apply aperture corrections"
157 catalogCalculation = pexConfig.ConfigurableField(
158 target=CatalogCalculationTask,
159 doc=
"Subtask to run catalogCalculation plugins on catalog"
161 doComputeSummaryStats = pexConfig.Field(
164 doc=
"Run subtask to measure exposure summary statistics"
166 computeSummaryStats = pexConfig.ConfigurableField(
167 target=ComputeExposureSummaryStatsTask,
168 doc=
"Subtask to run computeSummaryStats on exposure"
170 useSimplePsf = pexConfig.Field(
173 doc=
"Replace the existing PSF model with a simplified version that has the same sigma "
174 "at the start of each PSF determination iteration? Doing so makes PSF determination "
175 "converge more robustly and quickly.",
177 installSimplePsf = pexConfig.ConfigurableField(
178 target=InstallGaussianPsfTask,
179 doc=
"Install a simple PSF model",
181 refObjLoader = pexConfig.ConfigurableField(
182 target=LoadIndexedReferenceObjectsTask,
183 doc=
"reference object loader",
185 ref_match = pexConfig.ConfigurableField(
187 doc=
"Task to load and match reference objects. Only used if measurePsf can use matches. "
188 "Warning: matching will only work well if the initial WCS is accurate enough "
189 "to give good matches (roughly: good to 3 arcsec across the CCD).",
191 measurePsf = pexConfig.ConfigurableField(
192 target=MeasurePsfTask,
195 repair = pexConfig.ConfigurableField(
197 doc=
"Remove cosmic rays",
199 requireCrForPsf = pexConfig.Field(
202 doc=
"Require cosmic ray detection and masking to run successfully before measuring the PSF."
204 checkUnitsParseStrict = pexConfig.Field(
205 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
214 self.
detectiondetection.thresholdValue = 5.0
215 self.
detectiondetection.includeThresholdMultiplier = 10.0
216 self.
detectiondetection.doTempLocalBackground =
False
229 "base_CircularApertureFlux",
234 raise RuntimeError(
"Must measure PSF to measure aperture correction, "
235 "because flags determined by PSF measurement are used to identify "
236 "sources used to measure aperture correction")
247 r"""!Measure bright sources and use this to estimate background and PSF of an exposure
249 @anchor CharacterizeImageTask_
251 @section pipe_tasks_characterizeImage_Contents Contents
253 - @ref pipe_tasks_characterizeImage_Purpose
254 - @ref pipe_tasks_characterizeImage_Initialize
255 - @ref pipe_tasks_characterizeImage_IO
256 - @ref pipe_tasks_characterizeImage_Config
257 - @ref pipe_tasks_characterizeImage_Debug
260 @section pipe_tasks_characterizeImage_Purpose Description
262 Given an exposure with defects repaired (masked and interpolated over, e.g. as output by IsrTask):
263 - detect and measure bright sources
265 - measure and subtract background
268 @section pipe_tasks_characterizeImage_Initialize Task initialisation
270 @copydoc \_\_init\_\_
272 @section pipe_tasks_characterizeImage_IO Invoking the Task
274 If you want this task to unpersist inputs or persist outputs, then call
275 the `runDataRef` method (a thin wrapper around the `run` method).
277 If you already have the inputs unpersisted and do not want to persist the output
278 then it is more direct to call the `run` method:
280 @section pipe_tasks_characterizeImage_Config Configuration parameters
282 See @ref CharacterizeImageConfig
284 @section pipe_tasks_characterizeImage_Debug Debug variables
286 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
287 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
289 CharacterizeImageTask has a debug dictionary with the following keys:
292 <dd>int: if specified, the frame of first debug image displayed (defaults to 1)
294 <dd>bool; if True display image after each repair in the measure PSF loop
296 <dd>bool; if True display image after each background subtraction in the measure PSF loop
298 <dd>bool; if True display image and sources at the end of each iteration of the measure PSF loop
299 See @ref lsst.meas.astrom.displayAstrometry for the meaning of the various symbols.
301 <dd>bool; if True display image and sources after PSF is measured;
302 this will be identical to the final image displayed by measure_iter if measure_iter is true
304 <dd>bool; if True display image and sources after final repair
306 <dd>bool; if True display image and sources after final measurement
309 For example, put something like:
313 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
314 if name == "lsst.pipe.tasks.characterizeImage":
321 lsstDebug.Info = DebugInfo
323 into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag.
325 Some subtasks may have their own debug variables; see individual Task documentation.
330 ConfigClass = CharacterizeImageConfig
331 _DefaultName =
"characterizeImage"
332 RunnerClass = pipeBase.ButlerInitializedTaskRunner
335 inputs = butlerQC.get(inputRefs)
336 if 'exposureIdInfo' not in inputs.keys():
337 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
338 outputs = self.
runrun(**inputs)
339 butlerQC.put(outputs, outputRefs)
341 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
342 """!Construct a CharacterizeImageTask
344 @param[in] butler A butler object is passed to the refObjLoader constructor in case
345 it is needed to load catalogs. May be None if a catalog-based star selector is
346 not used, if the reference object loader constructor does not require a butler,
347 or if a reference object loader is passed directly via the refObjLoader argument.
348 @param[in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
349 external reference catalog to a catalog-based star selector. May be None if a
350 catalog star selector is not used or the loader can be constructed from the
352 @param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
353 @param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
358 schema = SourceTable.makeMinimalSchema()
360 self.makeSubtask(
"background")
361 self.makeSubtask(
"installSimplePsf")
362 self.makeSubtask(
"repair")
363 self.makeSubtask(
"measurePsf", schema=self.
schemaschema)
364 if self.config.doMeasurePsf
and self.measurePsf.usesMatches:
366 self.makeSubtask(
'refObjLoader', butler=butler)
367 refObjLoader = self.refObjLoader
368 self.makeSubtask(
"ref_match", refObjLoader=refObjLoader)
370 self.makeSubtask(
'detection', schema=self.
schemaschema)
371 if self.config.doDeblend:
372 self.makeSubtask(
"deblend", schema=self.
schemaschema)
373 self.makeSubtask(
'measurement', schema=self.
schemaschema, algMetadata=self.
algMetadataalgMetadata)
374 if self.config.doApCorr:
375 self.makeSubtask(
'measureApCorr', schema=self.
schemaschema)
376 self.makeSubtask(
'applyApCorr', schema=self.
schemaschema)
377 self.makeSubtask(
'catalogCalculation', schema=self.
schemaschema)
378 if self.config.doComputeSummaryStats:
379 self.makeSubtask(
'computeSummaryStats')
382 self.
schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
387 outputCatSchema.getTable().setMetadata(self.
algMetadataalgMetadata)
388 return {
'outputSchema': outputCatSchema}
391 def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True):
392 """!Characterize a science image and, if wanted, persist the results
394 This simply unpacks the exposure and passes it to the characterize method to do the work.
396 @param[in] dataRef: butler data reference for science exposure
397 @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
398 If None then unpersist from "postISRCCD".
399 The following changes are made, depending on the config:
400 - set psf to the measured PSF
401 - set apCorrMap to the measured aperture correction
402 - subtract background
403 - interpolate over cosmic rays
404 - update detection and cosmic ray mask planes
405 @param[in,out] background initial model of background already subtracted from exposure
406 (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
407 which is typical for image characterization.
408 A refined background model is output.
409 @param[in] doUnpersist if True the exposure is read from the repository
410 and the exposure and background arguments must be None;
411 if False the exposure must be provided.
412 True is intended for running as a command-line task, False for running as a subtask
414 @return same data as the characterize method
417 self.log.
info(
"Processing %s" % (dataRef.dataId))
420 if exposure
is not None or background
is not None:
421 raise RuntimeError(
"doUnpersist true; exposure and background must be None")
422 exposure = dataRef.get(
"postISRCCD", immediate=
True)
423 elif exposure
is None:
424 raise RuntimeError(
"doUnpersist false; exposure must be provided")
426 exposureIdInfo = dataRef.get(
"expIdInfo")
428 charRes = self.
runrun(
430 exposureIdInfo=exposureIdInfo,
431 background=background,
434 if self.config.doWrite:
435 dataRef.put(charRes.sourceCat,
"icSrc")
436 if self.config.doWriteExposure:
437 dataRef.put(charRes.exposure,
"icExp")
438 dataRef.put(charRes.background,
"icExpBackground")
443 def run(self, exposure, exposureIdInfo=None, background=None):
444 """!Characterize a science image
446 Peforms the following operations:
447 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
448 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
449 - interpolate over cosmic rays
450 - perform final measurement
452 @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
453 The following changes are made:
456 - update detection and cosmic ray mask planes
457 - subtract background and interpolate over cosmic rays
458 @param[in] exposureIdInfo ID info for exposure (an lsst.obs.base.ExposureIdInfo).
459 If not provided, returned SourceCatalog IDs will not be globally unique.
460 @param[in,out] background initial model of background already subtracted from exposure
461 (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
462 which is typical for image characterization.
464 @return pipe_base Struct containing these fields, all from the final iteration
465 of detectMeasureAndEstimatePsf:
466 - exposure: characterized exposure; image is repaired by interpolating over cosmic rays,
467 mask is updated accordingly, and the PSF model is set
468 - sourceCat: detected sources (an lsst.afw.table.SourceCatalog)
469 - background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
470 - psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
474 if not self.config.doMeasurePsf
and not exposure.hasPsf():
475 self.log.
warn(
"Source catalog detected and measured with placeholder or default PSF")
476 self.installSimplePsf.
run(exposure=exposure)
478 if exposureIdInfo
is None:
479 exposureIdInfo = ExposureIdInfo()
482 background = self.background.
run(exposure).background
484 psfIterations = self.config.psfIterations
if self.config.doMeasurePsf
else 1
485 for i
in range(psfIterations):
488 exposureIdInfo=exposureIdInfo,
489 background=background,
492 psf = dmeRes.exposure.getPsf()
493 psfSigma = psf.computeShape().getDeterminantRadius()
494 psfDimensions = psf.computeImage().getDimensions()
495 medBackground = np.median(dmeRes.background.getImage().getArray())
496 self.log.
info(
"iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f" %
497 (i + 1, psfSigma, psfDimensions, medBackground))
499 self.
displaydisplay(
"psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
502 self.repair.
run(exposure=dmeRes.exposure)
503 self.
displaydisplay(
"repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
507 self.measurement.
run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
508 exposureId=exposureIdInfo.expId)
509 if self.config.doApCorr:
510 apCorrMap = self.measureApCorr.
run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
511 dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
512 self.applyApCorr.
run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
513 self.catalogCalculation.
run(dmeRes.sourceCat)
514 if self.config.doComputeSummaryStats:
515 summary = self.computeSummaryStats.
run(exposure=dmeRes.exposure,
516 sources=dmeRes.sourceCat,
517 background=dmeRes.background)
518 dmeRes.exposure.getInfo().setSummaryStats(summary)
520 self.
displaydisplay(
"measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
522 return pipeBase.Struct(
523 exposure=dmeRes.exposure,
524 sourceCat=dmeRes.sourceCat,
525 background=dmeRes.background,
526 psfCellSet=dmeRes.psfCellSet,
528 characterized=dmeRes.exposure,
529 backgroundModel=dmeRes.background
534 """!Perform one iteration of detect, measure and estimate PSF
536 Performs the following operations:
537 - if config.doMeasurePsf or not exposure.hasPsf():
538 - install a simple PSF model (replacing the existing one, if need be)
539 - interpolate over cosmic rays with keepCRs=True
540 - estimate background and subtract it from the exposure
541 - detect, deblend and measure sources, and subtract a refined background model;
542 - if config.doMeasurePsf:
545 @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar)
546 The following changes are made:
548 - update detection and cosmic ray mask planes
549 - subtract background
550 @param[in] exposureIdInfo ID info for exposure (an lsst.obs_base.ExposureIdInfo)
551 @param[in,out] background initial model of background already subtracted from exposure
552 (an lsst.afw.math.BackgroundList).
554 @return pipe_base Struct containing these fields, all from the final iteration
555 of detect sources, measure sources and estimate PSF:
556 - exposure characterized exposure; image is repaired by interpolating over cosmic rays,
557 mask is updated accordingly, and the PSF model is set
558 - sourceCat detected sources (an lsst.afw.table.SourceCatalog)
559 - background model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
560 - psfCellSet spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
563 if not exposure.hasPsf()
or (self.config.doMeasurePsf
and self.config.useSimplePsf):
564 self.log.
warn(
"Source catalog detected and measured with placeholder or default PSF")
565 self.installSimplePsf.
run(exposure=exposure)
568 if self.config.requireCrForPsf:
569 self.repair.
run(exposure=exposure, keepCRs=
True)
572 self.repair.
run(exposure=exposure, keepCRs=
True)
574 self.log.
warn(
"Skipping cosmic ray detection: Too many CR pixels (max %0.f)" %
575 self.config.repair.cosmicray.nCrPixelMax)
577 self.
displaydisplay(
"repair_iter", exposure=exposure)
579 if background
is None:
582 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
583 table = SourceTable.make(self.
schemaschema, sourceIdFactory)
586 detRes = self.detection.
run(table=table, exposure=exposure, doSmooth=
True)
587 sourceCat = detRes.sources
588 if detRes.fpSets.background:
589 for bg
in detRes.fpSets.background:
590 background.append(bg)
592 if self.config.doDeblend:
593 self.deblend.
run(exposure=exposure, sources=sourceCat)
595 self.measurement.
run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
597 measPsfRes = pipeBase.Struct(cellSet=
None)
598 if self.config.doMeasurePsf:
599 if self.measurePsf.usesMatches:
600 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
603 measPsfRes = self.measurePsf.
run(exposure=exposure, sources=sourceCat, matches=matches,
604 expId=exposureIdInfo.expId)
605 self.
displaydisplay(
"measure_iter", exposure=exposure, sourceCat=sourceCat)
607 return pipeBase.Struct(
610 background=background,
611 psfCellSet=measPsfRes.cellSet,
615 """Return a dict of empty catalogs for each catalog dataset produced by this task.
618 sourceCat.getTable().setMetadata(self.
algMetadataalgMetadata)
619 return {
"icSrc": sourceCat}
621 def display(self, itemName, exposure, sourceCat=None):
622 """Display exposure and sources on next frame, if display of itemName has been requested
624 @param[in] itemName name of item in debugInfo
625 @param[in] exposure exposure to display
626 @param[in] sourceCat source catalog to display
Class for storing ordered metadata with comments.
Config for CharacterizeImageTask.
def adjustQuantum(self, pipeBase.InputQuantizedConnection datasetRefMap)
Measure bright sources and use this to estimate background and PSF of an exposure.
def getSchemaCatalogs(self)
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs)
Construct a CharacterizeImageTask.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getInitOutputDatasets(self)
def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True)
Characterize a science image and, if wanted, persist the results.
def run(self, exposure, exposureIdInfo=None, background=None)
Characterize a science image.
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background)
Perform one iteration of detect, measure and estimate PSF.
def display(self, itemName, exposure, sourceCat=None)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations.
def displayAstrometry(refCat=None, sourceCat=None, distortedCentroidKey=None, bbox=None, exposure=None, matches=None, frame=1, title="", pause=True)
def getDebugFrame(debugDisplay, name)