24 from lsstDebug 
import getDebugFrame
 
   29 import lsst.pipe.base.connectionTypes 
as cT
 
   37 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
 
   39 from .measurePsf 
import MeasurePsfTask
 
   40 from .repair 
import RepairTask
 
   41 from .computeExposureSummaryStats 
import ComputeExposureSummaryStatsTask
 
   43 from lsst.utils.timer 
import timeMethod
 
   45 __all__ = [
"CharacterizeImageConfig", 
"CharacterizeImageTask"]
 
   49                                    dimensions=(
"instrument", 
"visit", 
"detector")):
 
   51         doc=
"Input exposure data",
 
   53         storageClass=
"Exposure",
 
   54         dimensions=[
"instrument", 
"exposure", 
"detector"],
 
   56     characterized = cT.Output(
 
   57         doc=
"Output characterized data.",
 
   59         storageClass=
"ExposureF",
 
   60         dimensions=[
"instrument", 
"visit", 
"detector"],
 
   62     sourceCat = cT.Output(
 
   63         doc=
"Output source catalog.",
 
   65         storageClass=
"SourceCatalog",
 
   66         dimensions=[
"instrument", 
"visit", 
"detector"],
 
   68     backgroundModel = cT.Output(
 
   69         doc=
"Output background model.",
 
   70         name=
"icExpBackground",
 
   71         storageClass=
"Background",
 
   72         dimensions=[
"instrument", 
"visit", 
"detector"],
 
   74     outputSchema = cT.InitOutput(
 
   75         doc=
"Schema of the catalog produced by CharacterizeImage",
 
   77         storageClass=
"SourceCatalog",
 
   84         except pipeBase.ScalarError 
as err:
 
   85             raise pipeBase.ScalarError(
 
   86                 "CharacterizeImageTask can at present only be run on visits that are associated with " 
   87                 "exactly one exposure.  Either this is not a valid exposure for this pipeline, or the " 
   88                 "snap-combination step you probably want hasn't been configured to run between ISR and " 
   89                 "this task (as of this writing, that would be because it hasn't been implemented yet)." 
   94                               pipelineConnections=CharacterizeImageConnections):
 
   96     """!Config for CharacterizeImageTask""" 
   97     doMeasurePsf = pexConfig.Field(
 
  100         doc=
"Measure PSF? If False then for all subsequent operations use either existing PSF " 
  101             "model when present, or install simple PSF model when not (see installSimplePsf " 
  104     doWrite = pexConfig.Field(
 
  107         doc=
"Persist results?",
 
  109     doWriteExposure = pexConfig.Field(
 
  112         doc=
"Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
 
  114     psfIterations = pexConfig.RangeField(
 
  118         doc=
"Number of iterations of detect sources, measure sources, " 
  119             "estimate PSF. If useSimplePsf is True then 2 should be plenty; " 
  120             "otherwise more may be wanted.",
 
  122     background = pexConfig.ConfigurableField(
 
  123         target=SubtractBackgroundTask,
 
  124         doc=
"Configuration for initial background estimation",
 
  126     detection = pexConfig.ConfigurableField(
 
  127         target=SourceDetectionTask,
 
  130     doDeblend = pexConfig.Field(
 
  133         doc=
"Run deblender input exposure" 
  135     deblend = pexConfig.ConfigurableField(
 
  136         target=SourceDeblendTask,
 
  137         doc=
"Split blended source into their components" 
  139     measurement = pexConfig.ConfigurableField(
 
  140         target=SingleFrameMeasurementTask,
 
  141         doc=
"Measure sources" 
  143     doApCorr = pexConfig.Field(
 
  146         doc=
"Run subtasks to measure and apply aperture corrections" 
  148     measureApCorr = pexConfig.ConfigurableField(
 
  149         target=MeasureApCorrTask,
 
  150         doc=
"Subtask to measure aperture corrections" 
  152     applyApCorr = pexConfig.ConfigurableField(
 
  153         target=ApplyApCorrTask,
 
  154         doc=
"Subtask to apply aperture corrections" 
  158     catalogCalculation = pexConfig.ConfigurableField(
 
  159         target=CatalogCalculationTask,
 
  160         doc=
"Subtask to run catalogCalculation plugins on catalog" 
  162     doComputeSummaryStats = pexConfig.Field(
 
  165         doc=
"Run subtask to measure exposure summary statistics",
 
  166         deprecated=(
"This subtask has been moved to CalibrateTask " 
  169     computeSummaryStats = pexConfig.ConfigurableField(
 
  170         target=ComputeExposureSummaryStatsTask,
 
  171         doc=
"Subtask to run computeSummaryStats on exposure",
 
  172         deprecated=(
"This subtask has been moved to CalibrateTask " 
  175     useSimplePsf = pexConfig.Field(
 
  178         doc=
"Replace the existing PSF model with a simplified version that has the same sigma " 
  179         "at the start of each PSF determination iteration? Doing so makes PSF determination " 
  180         "converge more robustly and quickly.",
 
  182     installSimplePsf = pexConfig.ConfigurableField(
 
  183         target=InstallGaussianPsfTask,
 
  184         doc=
"Install a simple PSF model",
 
  186     refObjLoader = pexConfig.ConfigurableField(
 
  187         target=LoadIndexedReferenceObjectsTask,
 
  188         doc=
"reference object loader",
 
  190     ref_match = pexConfig.ConfigurableField(
 
  192         doc=
"Task to load and match reference objects. Only used if measurePsf can use matches. " 
  193         "Warning: matching will only work well if the initial WCS is accurate enough " 
  194         "to give good matches (roughly: good to 3 arcsec across the CCD).",
 
  196     measurePsf = pexConfig.ConfigurableField(
 
  197         target=MeasurePsfTask,
 
  200     repair = pexConfig.ConfigurableField(
 
  202         doc=
"Remove cosmic rays",
 
  204     requireCrForPsf = pexConfig.Field(
 
  207         doc=
"Require cosmic ray detection and masking to run successfully before measuring the PSF." 
  209     checkUnitsParseStrict = pexConfig.Field(
 
  210         doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
 
  219         self.
detectiondetection.thresholdValue = 5.0
 
  220         self.
detectiondetection.includeThresholdMultiplier = 10.0
 
  221         self.
detectiondetection.doTempLocalBackground = 
False 
  234             "base_CircularApertureFlux",
 
  239             raise RuntimeError(
"Must measure PSF to measure aperture correction, " 
  240                                "because flags determined by PSF measurement are used to identify " 
  241                                "sources used to measure aperture correction")
 
  252     r"""!Measure bright sources and use this to estimate background and PSF of an exposure 
  254     @anchor CharacterizeImageTask_ 
  256     @section pipe_tasks_characterizeImage_Contents  Contents 
  258      - @ref pipe_tasks_characterizeImage_Purpose 
  259      - @ref pipe_tasks_characterizeImage_Initialize 
  260      - @ref pipe_tasks_characterizeImage_IO 
  261      - @ref pipe_tasks_characterizeImage_Config 
  262      - @ref pipe_tasks_characterizeImage_Debug 
  265     @section pipe_tasks_characterizeImage_Purpose  Description 
  267     Given an exposure with defects repaired (masked and interpolated over, e.g. as output by IsrTask): 
  268     - detect and measure bright sources 
  270     - measure and subtract background 
  273     @section pipe_tasks_characterizeImage_Initialize  Task initialisation 
  275     @copydoc \_\_init\_\_ 
  277     @section pipe_tasks_characterizeImage_IO  Invoking the Task 
  279     If you want this task to unpersist inputs or persist outputs, then call 
  280     the `runDataRef` method (a thin wrapper around the `run` method). 
  282     If you already have the inputs unpersisted and do not want to persist the output 
  283     then it is more direct to call the `run` method: 
  285     @section pipe_tasks_characterizeImage_Config  Configuration parameters 
  287     See @ref CharacterizeImageConfig 
  289     @section pipe_tasks_characterizeImage_Debug  Debug variables 
  291     The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag 
  292     `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`. 
  294     CharacterizeImageTask has a debug dictionary with the following keys: 
  297     <dd>int: if specified, the frame of first debug image displayed (defaults to 1) 
  299     <dd>bool; if True display image after each repair in the measure PSF loop 
  301     <dd>bool; if True display image after each background subtraction in the measure PSF loop 
  303     <dd>bool; if True display image and sources at the end of each iteration of the measure PSF loop 
  304         See @ref lsst.meas.astrom.displayAstrometry for the meaning of the various symbols. 
  306     <dd>bool; if True display image and sources after PSF is measured; 
  307         this will be identical to the final image displayed by measure_iter if measure_iter is true 
  309     <dd>bool; if True display image and sources after final repair 
  311     <dd>bool; if True display image and sources after final measurement 
  314     For example, put something like: 
  318             di = lsstDebug.getInfo(name)  # N.b. lsstDebug.Info(name) would call us recursively 
  319             if name == "lsst.pipe.tasks.characterizeImage": 
  326         lsstDebug.Info = DebugInfo 
  328     into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag. 
  330     Some subtasks may have their own debug variables; see individual Task documentation. 
  335     ConfigClass = CharacterizeImageConfig
 
  336     _DefaultName = 
"characterizeImage" 
  337     RunnerClass = pipeBase.ButlerInitializedTaskRunner
 
  340         inputs = butlerQC.get(inputRefs)
 
  341         if 'exposureIdInfo' not in inputs.keys():
 
  342             inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, 
"visit_detector")
 
  343         outputs = self.
runrun(**inputs)
 
  344         butlerQC.put(outputs, outputRefs)
 
  346     def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
 
  347         """!Construct a CharacterizeImageTask 
  349         @param[in] butler  A butler object is passed to the refObjLoader constructor in case 
  350             it is needed to load catalogs.  May be None if a catalog-based star selector is 
  351             not used, if the reference object loader constructor does not require a butler, 
  352             or if a reference object loader is passed directly via the refObjLoader argument. 
  353         @param[in] refObjLoader  An instance of LoadReferenceObjectsTasks that supplies an 
  354             external reference catalog to a catalog-based star selector.  May be None if a 
  355             catalog star selector is not used or the loader can be constructed from the 
  357         @param[in,out] schema  initial schema (an lsst.afw.table.SourceTable), or None 
  358         @param[in,out] kwargs  other keyword arguments for lsst.pipe.base.CmdLineTask 
  363             schema = SourceTable.makeMinimalSchema()
 
  365         self.makeSubtask(
"background")
 
  366         self.makeSubtask(
"installSimplePsf")
 
  367         self.makeSubtask(
"repair")
 
  368         self.makeSubtask(
"measurePsf", schema=self.
schemaschema)
 
  369         if self.config.doMeasurePsf 
and self.measurePsf.usesMatches:
 
  371                 self.makeSubtask(
'refObjLoader', butler=butler)
 
  372                 refObjLoader = self.refObjLoader
 
  373             self.makeSubtask(
"ref_match", refObjLoader=refObjLoader)
 
  375         self.makeSubtask(
'detection', schema=self.
schemaschema)
 
  376         if self.config.doDeblend:
 
  377             self.makeSubtask(
"deblend", schema=self.
schemaschema)
 
  378         self.makeSubtask(
'measurement', schema=self.
schemaschema, algMetadata=self.
algMetadataalgMetadata)
 
  379         if self.config.doApCorr:
 
  380             self.makeSubtask(
'measureApCorr', schema=self.
schemaschema)
 
  381             self.makeSubtask(
'applyApCorr', schema=self.
schemaschema)
 
  382         self.makeSubtask(
'catalogCalculation', schema=self.
schemaschema)
 
  385         self.
schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
 
  390         outputCatSchema.getTable().setMetadata(self.
algMetadataalgMetadata)
 
  391         return {
'outputSchema': outputCatSchema}
 
  394     def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True):
 
  395         """!Characterize a science image and, if wanted, persist the results 
  397         This simply unpacks the exposure and passes it to the characterize method to do the work. 
  399         @param[in] dataRef: butler data reference for science exposure 
  400         @param[in,out] exposure  exposure to characterize (an lsst.afw.image.ExposureF or similar). 
  401             If None then unpersist from "postISRCCD". 
  402             The following changes are made, depending on the config: 
  403             - set psf to the measured PSF 
  404             - set apCorrMap to the measured aperture correction 
  405             - subtract background 
  406             - interpolate over cosmic rays 
  407             - update detection and cosmic ray mask planes 
  408         @param[in,out] background  initial model of background already subtracted from exposure 
  409             (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted, 
  410             which is typical for image characterization. 
  411             A refined background model is output. 
  412         @param[in] doUnpersist  if True the exposure is read from the repository 
  413             and the exposure and background arguments must be None; 
  414             if False the exposure must be provided. 
  415             True is intended for running as a command-line task, False for running as a subtask 
  417         @return same data as the characterize method 
  420         self.log.
info(
"Processing %s", dataRef.dataId)
 
  423             if exposure 
is not None or background 
is not None:
 
  424                 raise RuntimeError(
"doUnpersist true; exposure and background must be None")
 
  425             exposure = dataRef.get(
"postISRCCD", immediate=
True)
 
  426         elif exposure 
is None:
 
  427             raise RuntimeError(
"doUnpersist false; exposure must be provided")
 
  429         exposureIdInfo = dataRef.get(
"expIdInfo")
 
  431         charRes = self.
runrun(
 
  433             exposureIdInfo=exposureIdInfo,
 
  434             background=background,
 
  437         if self.config.doWrite:
 
  438             dataRef.put(charRes.sourceCat, 
"icSrc")
 
  439             if self.config.doWriteExposure:
 
  440                 dataRef.put(charRes.exposure, 
"icExp")
 
  441                 dataRef.put(charRes.background, 
"icExpBackground")
 
  446     def run(self, exposure, exposureIdInfo=None, background=None):
 
  447         """!Characterize a science image 
  449         Peforms the following operations: 
  450         - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false: 
  451             - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) 
  452         - interpolate over cosmic rays 
  453         - perform final measurement 
  455         @param[in,out] exposure  exposure to characterize (an lsst.afw.image.ExposureF or similar). 
  456             The following changes are made: 
  459             - update detection and cosmic ray mask planes 
  460             - subtract background and interpolate over cosmic rays 
  461         @param[in] exposureIdInfo  ID info for exposure (an lsst.obs.base.ExposureIdInfo). 
  462             If not provided, returned SourceCatalog IDs will not be globally unique. 
  463         @param[in,out] background  initial model of background already subtracted from exposure 
  464             (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted, 
  465             which is typical for image characterization. 
  467         @return pipe_base Struct containing these fields, all from the final iteration 
  468         of detectMeasureAndEstimatePsf: 
  469         - exposure: characterized exposure; image is repaired by interpolating over cosmic rays, 
  470             mask is updated accordingly, and the PSF model is set 
  471         - sourceCat: detected sources (an lsst.afw.table.SourceCatalog) 
  472         - background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList) 
  473         - psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet) 
  477         if not self.config.doMeasurePsf 
and not exposure.hasPsf():
 
  478             self.log.
info(
"CharacterizeImageTask initialized with 'simple' PSF.")
 
  479             self.installSimplePsf.
run(exposure=exposure)
 
  481         if exposureIdInfo 
is None:
 
  482             exposureIdInfo = ExposureIdInfo()
 
  485         background = self.background.
run(exposure).background
 
  487         psfIterations = self.config.psfIterations 
if self.config.doMeasurePsf 
else 1
 
  488         for i 
in range(psfIterations):
 
  491                 exposureIdInfo=exposureIdInfo,
 
  492                 background=background,
 
  495             psf = dmeRes.exposure.getPsf()
 
  496             psfSigma = psf.computeShape().getDeterminantRadius()
 
  497             psfDimensions = psf.computeImage().getDimensions()
 
  498             medBackground = np.median(dmeRes.background.getImage().getArray())
 
  499             self.log.
info(
"iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
 
  500                           i + 1, psfSigma, psfDimensions, medBackground)
 
  502         self.
displaydisplay(
"psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
 
  505         self.repair.
run(exposure=dmeRes.exposure)
 
  506         self.
displaydisplay(
"repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
 
  510         self.measurement.
run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
 
  511                              exposureId=exposureIdInfo.expId)
 
  512         if self.config.doApCorr:
 
  513             apCorrMap = self.measureApCorr.
run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
 
  514             dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
 
  515             self.applyApCorr.
run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
 
  516         self.catalogCalculation.
run(dmeRes.sourceCat)
 
  518         self.
displaydisplay(
"measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
 
  520         return pipeBase.Struct(
 
  521             exposure=dmeRes.exposure,
 
  522             sourceCat=dmeRes.sourceCat,
 
  523             background=dmeRes.background,
 
  524             psfCellSet=dmeRes.psfCellSet,
 
  526             characterized=dmeRes.exposure,
 
  527             backgroundModel=dmeRes.background
 
  532         """!Perform one iteration of detect, measure and estimate PSF 
  534         Performs the following operations: 
  535         - if config.doMeasurePsf or not exposure.hasPsf(): 
  536             - install a simple PSF model (replacing the existing one, if need be) 
  537         - interpolate over cosmic rays with keepCRs=True 
  538         - estimate background and subtract it from the exposure 
  539         - detect, deblend and measure sources, and subtract a refined background model; 
  540         - if config.doMeasurePsf: 
  543         @param[in,out] exposure  exposure to characterize (an lsst.afw.image.ExposureF or similar) 
  544             The following changes are made: 
  546             - update detection and cosmic ray mask planes 
  547             - subtract background 
  548         @param[in] exposureIdInfo  ID info for exposure (an lsst.obs_base.ExposureIdInfo) 
  549         @param[in,out] background  initial model of background already subtracted from exposure 
  550             (an lsst.afw.math.BackgroundList). 
  552         @return pipe_base Struct containing these fields, all from the final iteration 
  553         of detect sources, measure sources and estimate PSF: 
  554         - exposure  characterized exposure; image is repaired by interpolating over cosmic rays, 
  555             mask is updated accordingly, and the PSF model is set 
  556         - sourceCat  detected sources (an lsst.afw.table.SourceCatalog) 
  557         - background  model of background subtracted from exposure (an lsst.afw.math.BackgroundList) 
  558         - psfCellSet  spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet) 
  561         if not exposure.hasPsf() 
or (self.config.doMeasurePsf 
and self.config.useSimplePsf):
 
  562             self.log.
info(
"PSF estimation initialized with 'simple' PSF")
 
  563             self.installSimplePsf.
run(exposure=exposure)
 
  566         if self.config.requireCrForPsf:
 
  567             self.repair.
run(exposure=exposure, keepCRs=
True)
 
  570                 self.repair.
run(exposure=exposure, keepCRs=
True)
 
  572                 self.log.
warning(
"Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
 
  573                                  self.config.repair.cosmicray.nCrPixelMax)
 
  575         self.
displaydisplay(
"repair_iter", exposure=exposure)
 
  577         if background 
is None:
 
  580         sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
 
  581         table = SourceTable.make(self.
schemaschema, sourceIdFactory)
 
  584         detRes = self.detection.
run(table=table, exposure=exposure, doSmooth=
True)
 
  585         sourceCat = detRes.sources
 
  586         if detRes.fpSets.background:
 
  587             for bg 
in detRes.fpSets.background:
 
  588                 background.append(bg)
 
  590         if self.config.doDeblend:
 
  591             self.deblend.
run(exposure=exposure, sources=sourceCat)
 
  593         self.measurement.
run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
 
  595         measPsfRes = pipeBase.Struct(cellSet=
None)
 
  596         if self.config.doMeasurePsf:
 
  597             if self.measurePsf.usesMatches:
 
  598                 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
 
  601             measPsfRes = self.measurePsf.
run(exposure=exposure, sources=sourceCat, matches=matches,
 
  602                                              expId=exposureIdInfo.expId)
 
  603         self.
displaydisplay(
"measure_iter", exposure=exposure, sourceCat=sourceCat)
 
  605         return pipeBase.Struct(
 
  608             background=background,
 
  609             psfCellSet=measPsfRes.cellSet,
 
  613         """Return a dict of empty catalogs for each catalog dataset produced by this task. 
  616         sourceCat.getTable().setMetadata(self.
algMetadataalgMetadata)
 
  617         return {
"icSrc": sourceCat}
 
  619     def display(self, itemName, exposure, sourceCat=None):
 
  620         """Display exposure and sources on next frame, if display of itemName has been requested 
  622         @param[in] itemName  name of item in debugInfo 
  623         @param[in] exposure  exposure to display 
  624         @param[in] sourceCat  source catalog to display 
Class for storing ordered metadata with comments.
Config for CharacterizeImageTask.
def adjustQuantum(self, inputs, outputs, label, dataId)
Measure bright sources and use this to estimate background and PSF of an exposure.
def getSchemaCatalogs(self)
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs)
Construct a CharacterizeImageTask.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getInitOutputDatasets(self)
def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True)
Characterize a science image and, if wanted, persist the results.
def run(self, exposure, exposureIdInfo=None, background=None)
Characterize a science image.
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background)
Perform one iteration of detect, measure and estimate PSF.
def display(self, itemName, exposure, sourceCat=None)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations.
def displayAstrometry(refCat=None, sourceCat=None, distortedCentroidKey=None, bbox=None, exposure=None, matches=None, frame=1, title="", pause=True)
def getDebugFrame(debugDisplay, name)