LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
27 import lsst.pipe.base.connectionTypes as cT
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask,
37  ApplyApCorrTask,
38  CatalogCalculationTask)
39 from lsst.meas.deblender import SourceDeblendTask
40 from lsst.utils.timer import timeMethod
41 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
42 from .fakes import BaseFakeSourcesTask
43 from .photoCal import PhotoCalTask
44 from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
45 
46 
47 __all__ = ["CalibrateConfig", "CalibrateTask"]
48 
49 
50 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
51  defaultTemplates={}):
52 
53  icSourceSchema = cT.InitInput(
54  doc="Schema produced by characterize image task, used to initialize this task",
55  name="icSrc_schema",
56  storageClass="SourceCatalog",
57  )
58 
59  outputSchema = cT.InitOutput(
60  doc="Schema after CalibrateTask has been initialized",
61  name="src_schema",
62  storageClass="SourceCatalog",
63  )
64 
65  exposure = cT.Input(
66  doc="Input image to calibrate",
67  name="icExp",
68  storageClass="ExposureF",
69  dimensions=("instrument", "visit", "detector"),
70  )
71 
72  background = cT.Input(
73  doc="Backgrounds determined by characterize task",
74  name="icExpBackground",
75  storageClass="Background",
76  dimensions=("instrument", "visit", "detector"),
77  )
78 
79  icSourceCat = cT.Input(
80  doc="Source catalog created by characterize task",
81  name="icSrc",
82  storageClass="SourceCatalog",
83  dimensions=("instrument", "visit", "detector"),
84  )
85 
86  astromRefCat = cT.PrerequisiteInput(
87  doc="Reference catalog to use for astrometry",
88  name="cal_ref_cat",
89  storageClass="SimpleCatalog",
90  dimensions=("skypix",),
91  deferLoad=True,
92  multiple=True,
93  )
94 
95  photoRefCat = cT.PrerequisiteInput(
96  doc="Reference catalog to use for photometric calibration",
97  name="cal_ref_cat",
98  storageClass="SimpleCatalog",
99  dimensions=("skypix",),
100  deferLoad=True,
101  multiple=True
102  )
103 
104  outputExposure = cT.Output(
105  doc="Exposure after running calibration task",
106  name="calexp",
107  storageClass="ExposureF",
108  dimensions=("instrument", "visit", "detector"),
109  )
110 
111  outputCat = cT.Output(
112  doc="Source catalog produced in calibrate task",
113  name="src",
114  storageClass="SourceCatalog",
115  dimensions=("instrument", "visit", "detector"),
116  )
117 
118  outputBackground = cT.Output(
119  doc="Background models estimated in calibration task",
120  name="calexpBackground",
121  storageClass="Background",
122  dimensions=("instrument", "visit", "detector"),
123  )
124 
125  matches = cT.Output(
126  doc="Source/refObj matches from the astrometry solver",
127  name="srcMatch",
128  storageClass="Catalog",
129  dimensions=("instrument", "visit", "detector"),
130  )
131 
132  matchesDenormalized = cT.Output(
133  doc="Denormalized matches from astrometry solver",
134  name="srcMatchFull",
135  storageClass="Catalog",
136  dimensions=("instrument", "visit", "detector"),
137  )
138 
139  def __init__(self, *, config=None):
140  super().__init__(config=config)
141 
142  if config.doAstrometry is False:
143  self.prerequisiteInputs.remove("astromRefCat")
144  if config.doPhotoCal is False:
145  self.prerequisiteInputs.remove("photoRefCat")
146 
147  if config.doWriteMatches is False or config.doAstrometry is False:
148  self.outputs.remove("matches")
149  if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
150  self.outputs.remove("matchesDenormalized")
151 
152 
153 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
154  """Config for CalibrateTask"""
155  doWrite = pexConfig.Field(
156  dtype=bool,
157  default=True,
158  doc="Save calibration results?",
159  )
160  doWriteHeavyFootprintsInSources = pexConfig.Field(
161  dtype=bool,
162  default=True,
163  doc="Include HeavyFootprint data in source table? If false then heavy "
164  "footprints are saved as normal footprints, which saves some space"
165  )
166  doWriteMatches = pexConfig.Field(
167  dtype=bool,
168  default=True,
169  doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
170  )
171  doWriteMatchesDenormalized = pexConfig.Field(
172  dtype=bool,
173  default=False,
174  doc=("Write reference matches in denormalized format? "
175  "This format uses more disk space, but is more convenient to "
176  "read. Ignored if doWriteMatches=False or doWrite=False."),
177  )
178  doAstrometry = pexConfig.Field(
179  dtype=bool,
180  default=True,
181  doc="Perform astrometric calibration?",
182  )
183  astromRefObjLoader = pexConfig.ConfigurableField(
184  target=LoadIndexedReferenceObjectsTask,
185  doc="reference object loader for astrometric calibration",
186  )
187  photoRefObjLoader = pexConfig.ConfigurableField(
188  target=LoadIndexedReferenceObjectsTask,
189  doc="reference object loader for photometric calibration",
190  )
191  astrometry = pexConfig.ConfigurableField(
192  target=AstrometryTask,
193  doc="Perform astrometric calibration to refine the WCS",
194  )
195  requireAstrometry = pexConfig.Field(
196  dtype=bool,
197  default=True,
198  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
199  "false."),
200  )
201  doPhotoCal = pexConfig.Field(
202  dtype=bool,
203  default=True,
204  doc="Perform phometric calibration?",
205  )
206  requirePhotoCal = pexConfig.Field(
207  dtype=bool,
208  default=True,
209  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
210  "false."),
211  )
212  photoCal = pexConfig.ConfigurableField(
213  target=PhotoCalTask,
214  doc="Perform photometric calibration",
215  )
216  icSourceFieldsToCopy = pexConfig.ListField(
217  dtype=str,
218  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
219  doc=("Fields to copy from the icSource catalog to the output catalog "
220  "for matching sources Any missing fields will trigger a "
221  "RuntimeError exception. Ignored if icSourceCat is not provided.")
222  )
223  matchRadiusPix = pexConfig.Field(
224  dtype=float,
225  default=3,
226  doc=("Match radius for matching icSourceCat objects to sourceCat "
227  "objects (pixels)"),
228  )
229  checkUnitsParseStrict = pexConfig.Field(
230  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
231  "'warn' or 'silent'"),
232  dtype=str,
233  default="raise",
234  )
235  detection = pexConfig.ConfigurableField(
236  target=SourceDetectionTask,
237  doc="Detect sources"
238  )
239  doDeblend = pexConfig.Field(
240  dtype=bool,
241  default=True,
242  doc="Run deblender input exposure"
243  )
244  deblend = pexConfig.ConfigurableField(
245  target=SourceDeblendTask,
246  doc="Split blended sources into their components"
247  )
248  doSkySources = pexConfig.Field(
249  dtype=bool,
250  default=True,
251  doc="Generate sky sources?",
252  )
253  skySources = pexConfig.ConfigurableField(
254  target=SkyObjectsTask,
255  doc="Generate sky sources",
256  )
257  measurement = pexConfig.ConfigurableField(
258  target=SingleFrameMeasurementTask,
259  doc="Measure sources"
260  )
261  setPrimaryFlags = pexConfig.ConfigurableField(
262  target=SetPrimaryFlagsTask,
263  doc=("Set flags for primary source classification in single frame "
264  "processing. True if sources are not sky sources and not a parent.")
265  )
266  doApCorr = pexConfig.Field(
267  dtype=bool,
268  default=True,
269  doc="Run subtask to apply aperture correction"
270  )
271  applyApCorr = pexConfig.ConfigurableField(
272  target=ApplyApCorrTask,
273  doc="Subtask to apply aperture corrections"
274  )
275  # If doApCorr is False, and the exposure does not have apcorrections
276  # already applied, the active plugins in catalogCalculation almost
277  # certainly should not contain the characterization plugin
278  catalogCalculation = pexConfig.ConfigurableField(
279  target=CatalogCalculationTask,
280  doc="Subtask to run catalogCalculation plugins on catalog"
281  )
282  doInsertFakes = pexConfig.Field(
283  dtype=bool,
284  default=False,
285  doc="Run fake sources injection task"
286  )
287  insertFakes = pexConfig.ConfigurableField(
288  target=BaseFakeSourcesTask,
289  doc="Injection of fake sources for testing purposes (must be "
290  "retargeted)"
291  )
292  doComputeSummaryStats = pexConfig.Field(
293  dtype=bool,
294  default=True,
295  doc="Run subtask to measure exposure summary statistics?"
296  )
297  computeSummaryStats = pexConfig.ConfigurableField(
298  target=ComputeExposureSummaryStatsTask,
299  doc="Subtask to run computeSummaryStats on exposure"
300  )
301  doWriteExposure = pexConfig.Field(
302  dtype=bool,
303  default=True,
304  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
305  "normal calexp but as a fakes_calexp."
306  )
307 
308  def setDefaults(self):
309  super().setDefaults()
310  self.detectiondetection.doTempLocalBackground = False
311  self.deblenddeblend.maxFootprintSize = 2000
312  self.measurementmeasurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
313 
314  def validate(self):
315  super().validate()
316  astromRefCatGen2 = getattr(self.astromRefObjLoaderastromRefObjLoader, "ref_dataset_name", None)
317  if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
318  raise ValueError(
319  f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
320  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
321  )
322  photoRefCatGen2 = getattr(self.photoRefObjLoaderphotoRefObjLoader, "ref_dataset_name", None)
323  if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
324  raise ValueError(
325  f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
326  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
327  )
328 
329 
330 
336 
337 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
338  r"""!Calibrate an exposure: measure sources and perform astrometric and
339  photometric calibration
340 
341  @anchor CalibrateTask_
342 
343  @section pipe_tasks_calibrate_Contents Contents
344 
345  - @ref pipe_tasks_calibrate_Purpose
346  - @ref pipe_tasks_calibrate_Initialize
347  - @ref pipe_tasks_calibrate_IO
348  - @ref pipe_tasks_calibrate_Config
349  - @ref pipe_tasks_calibrate_Metadata
350  - @ref pipe_tasks_calibrate_Debug
351 
352 
353  @section pipe_tasks_calibrate_Purpose Description
354 
355  Given an exposure with a good PSF model and aperture correction map
356  (e.g. as provided by @ref CharacterizeImageTask), perform the following
357  operations:
358  - Run detection and measurement
359  - Run astrometry subtask to fit an improved WCS
360  - Run photoCal subtask to fit the exposure's photometric zero-point
361 
362  @section pipe_tasks_calibrate_Initialize Task initialisation
363 
364  @copydoc \_\_init\_\_
365 
366  @section pipe_tasks_calibrate_IO Invoking the Task
367 
368  If you want this task to unpersist inputs or persist outputs, then call
369  the `runDataRef` method (a wrapper around the `run` method).
370 
371  If you already have the inputs unpersisted and do not want to persist the
372  output then it is more direct to call the `run` method:
373 
374  @section pipe_tasks_calibrate_Config Configuration parameters
375 
376  See @ref CalibrateConfig
377 
378  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
379 
380  Exposure metadata
381  <dl>
382  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
383  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
384  task
385  <dt>COLORTERM1 <dd>?? (always 0.0)
386  <dt>COLORTERM2 <dd>?? (always 0.0)
387  <dt>COLORTERM3 <dd>?? (always 0.0)
388  </dl>
389 
390  @section pipe_tasks_calibrate_Debug Debug variables
391 
392  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
393  interface supports a flag
394  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
395  for more about `debug.py`.
396 
397  CalibrateTask has a debug dictionary containing one key:
398  <dl>
399  <dt>calibrate
400  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
401  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
402  the meaning of the various symbols.
403  </dl>
404 
405  For example, put something like:
406  @code{.py}
407  import lsstDebug
408  def DebugInfo(name):
409  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
410  # call us recursively
411  if name == "lsst.pipe.tasks.calibrate":
412  di.display = dict(
413  calibrate = 1,
414  )
415 
416  return di
417 
418  lsstDebug.Info = DebugInfo
419  @endcode
420  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
421  flag.
422 
423  Some subtasks may have their own debug variables; see individual Task
424  documentation.
425  """
426 
427  # Example description used to live here, removed 2-20-2017 as per
428  # https://jira.lsstcorp.org/browse/DM-9520
429 
430  ConfigClass = CalibrateConfig
431  _DefaultName = "calibrate"
432  RunnerClass = pipeBase.ButlerInitializedTaskRunner
433 
434  def __init__(self, butler=None, astromRefObjLoader=None,
435  photoRefObjLoader=None, icSourceSchema=None,
436  initInputs=None, **kwargs):
437  """!Construct a CalibrateTask
438 
439  @param[in] butler The butler is passed to the refObjLoader constructor
440  in case it is needed. Ignored if the refObjLoader argument
441  provides a loader directly.
442  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
443  that supplies an external reference catalog for astrometric
444  calibration. May be None if the desired loader can be constructed
445  from the butler argument or all steps requiring a reference catalog
446  are disabled.
447  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
448  that supplies an external reference catalog for photometric
449  calibration. May be None if the desired loader can be constructed
450  from the butler argument or all steps requiring a reference catalog
451  are disabled.
452  @param[in] icSourceSchema schema for icSource catalog, or None.
453  Schema values specified in config.icSourceFieldsToCopy will be
454  taken from this schema. If set to None, no values will be
455  propagated from the icSourceCatalog
456  @param[in,out] kwargs other keyword arguments for
457  lsst.pipe.base.CmdLineTask
458  """
459  super().__init__(**kwargs)
460 
461  if icSourceSchema is None and butler is not None:
462  # Use butler to read icSourceSchema from disk.
463  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
464 
465  if icSourceSchema is None and butler is None and initInputs is not None:
466  icSourceSchema = initInputs['icSourceSchema'].schema
467 
468  if icSourceSchema is not None:
469  # use a schema mapper to avoid copying each field separately
470  self.schemaMapperschemaMapper = afwTable.SchemaMapper(icSourceSchema)
471  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
472  self.schemaMapperschemaMapper.addMinimalSchema(minimumSchema, False)
473 
474  # Add fields to copy from an icSource catalog
475  # and a field to indicate that the source matched a source in that
476  # catalog. If any fields are missing then raise an exception, but
477  # first find all missing fields in order to make the error message
478  # more useful.
479  self.calibSourceKeycalibSourceKey = self.schemaMapperschemaMapper.addOutputField(
480  afwTable.Field["Flag"]("calib_detected",
481  "Source was detected as an icSource"))
482  missingFieldNames = []
483  for fieldName in self.config.icSourceFieldsToCopy:
484  try:
485  schemaItem = icSourceSchema.find(fieldName)
486  except Exception:
487  missingFieldNames.append(fieldName)
488  else:
489  # field found; if addMapping fails then raise an exception
490  self.schemaMapperschemaMapper.addMapping(schemaItem.getKey())
491 
492  if missingFieldNames:
493  raise RuntimeError("isSourceCat is missing fields {} "
494  "specified in icSourceFieldsToCopy"
495  .format(missingFieldNames))
496 
497  # produce a temporary schema to pass to the subtasks; finalize it
498  # later
499  self.schemaschema = self.schemaMapperschemaMapper.editOutputSchema()
500  else:
501  self.schemaMapperschemaMapper = None
502  self.schemaschema = afwTable.SourceTable.makeMinimalSchema()
503  self.makeSubtask('detection', schema=self.schemaschema)
504 
505  self.algMetadataalgMetadata = dafBase.PropertyList()
506 
507  # Only create a subtask for fakes if configuration option is set
508  # N.B. the config for fake object task must be retargeted to a child
509  # of BaseFakeSourcesTask
510  if self.config.doInsertFakes:
511  self.makeSubtask("insertFakes")
512 
513  if self.config.doDeblend:
514  self.makeSubtask("deblend", schema=self.schemaschema)
515  if self.config.doSkySources:
516  self.makeSubtask("skySources")
517  self.skySourceKeyskySourceKey = self.schemaschema.addField("sky_source", type="Flag", doc="Sky objects.")
518  self.makeSubtask('measurement', schema=self.schemaschema,
519  algMetadata=self.algMetadataalgMetadata)
520  self.makeSubtask("setPrimaryFlags", schema=self.schemaschema, isSingleFrame=True)
521  if self.config.doApCorr:
522  self.makeSubtask('applyApCorr', schema=self.schemaschema)
523  self.makeSubtask('catalogCalculation', schema=self.schemaschema)
524 
525  if self.config.doAstrometry:
526  if astromRefObjLoader is None and butler is not None:
527  self.makeSubtask('astromRefObjLoader', butler=butler)
528  astromRefObjLoader = self.astromRefObjLoader
529  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
530  schema=self.schemaschema)
531  if self.config.doPhotoCal:
532  if photoRefObjLoader is None and butler is not None:
533  self.makeSubtask('photoRefObjLoader', butler=butler)
534  photoRefObjLoader = self.photoRefObjLoader
535  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
536  schema=self.schemaschema)
537  if self.config.doComputeSummaryStats:
538  self.makeSubtask('computeSummaryStats')
539 
540  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
541  raise RuntimeError("PipelineTask form of this task should not be initialized with "
542  "reference object loaders.")
543 
544  if self.schemaMapperschemaMapper is not None:
545  # finalize the schema
546  self.schemaschema = self.schemaMapperschemaMapper.getOutputSchema()
547  self.schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
548 
549  sourceCatSchema = afwTable.SourceCatalog(self.schemaschema)
550  sourceCatSchema.getTable().setMetadata(self.algMetadataalgMetadata)
551  self.outputSchemaoutputSchema = sourceCatSchema
552 
553  @timeMethod
554  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
555  doUnpersist=True):
556  """!Calibrate an exposure, optionally unpersisting inputs and
557  persisting outputs.
558 
559  This is a wrapper around the `run` method that unpersists inputs
560  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
561 
562  @param[in] dataRef butler data reference corresponding to a science
563  image
564  @param[in,out] exposure characterized exposure (an
565  lsst.afw.image.ExposureF or similar), or None to unpersist existing
566  icExp and icBackground. See `run` method for details of what is
567  read and written.
568  @param[in,out] background initial model of background already
569  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
570  None if no background has been subtracted, though that is unusual
571  for calibration. A refined background model is output. Ignored if
572  exposure is None.
573  @param[in] icSourceCat catalog from which to copy the fields specified
574  by icSourceKeys, or None;
575  @param[in] doUnpersist unpersist data:
576  - if True, exposure, background and icSourceCat are read from
577  dataRef and those three arguments must all be None;
578  - if False the exposure must be provided; background and
579  icSourceCat are optional. True is intended for running as a
580  command-line task, False for running as a subtask
581  @return same data as the calibrate method
582  """
583  self.log.info("Processing %s", dataRef.dataId)
584 
585  if doUnpersist:
586  if any(item is not None for item in (exposure, background,
587  icSourceCat)):
588  raise RuntimeError("doUnpersist true; exposure, background "
589  "and icSourceCat must all be None")
590  exposure = dataRef.get("icExp", immediate=True)
591  background = dataRef.get("icExpBackground", immediate=True)
592  icSourceCat = dataRef.get("icSrc", immediate=True)
593  elif exposure is None:
594  raise RuntimeError("doUnpersist false; exposure must be provided")
595 
596  exposureIdInfo = dataRef.get("expIdInfo")
597 
598  calRes = self.runrun(
599  exposure=exposure,
600  exposureIdInfo=exposureIdInfo,
601  background=background,
602  icSourceCat=icSourceCat,
603  )
604 
605  if self.config.doWrite:
606  self.writeOutputswriteOutputs(
607  dataRef=dataRef,
608  exposure=calRes.exposure,
609  background=calRes.background,
610  sourceCat=calRes.sourceCat,
611  astromMatches=calRes.astromMatches,
612  matchMeta=calRes.matchMeta,
613  )
614 
615  return calRes
616 
617  def runQuantum(self, butlerQC, inputRefs, outputRefs):
618  inputs = butlerQC.get(inputRefs)
619  inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector")
620 
621  if self.config.doAstrometry:
622  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
623  for ref in inputRefs.astromRefCat],
624  refCats=inputs.pop('astromRefCat'),
625  config=self.config.astromRefObjLoader, log=self.log)
626  self.astrometry.setRefObjLoader(refObjLoader)
627 
628  if self.config.doPhotoCal:
629  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
630  for ref in inputRefs.photoRefCat],
631  refCats=inputs.pop('photoRefCat'),
632  config=self.config.photoRefObjLoader,
633  log=self.log)
634  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
635 
636  outputs = self.runrun(**inputs)
637 
638  if self.config.doWriteMatches and self.config.doAstrometry:
639  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
640  normalizedMatches.table.setMetadata(outputs.matchMeta)
641  if self.config.doWriteMatchesDenormalized:
642  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
643  outputs.matchesDenormalized = denormMatches
644  outputs.matches = normalizedMatches
645  butlerQC.put(outputs, outputRefs)
646 
647  @timeMethod
648  def run(self, exposure, exposureIdInfo=None, background=None,
649  icSourceCat=None):
650  """!Calibrate an exposure (science image or coadd)
651 
652  @param[in,out] exposure exposure to calibrate (an
653  lsst.afw.image.ExposureF or similar);
654  in:
655  - MaskedImage
656  - Psf
657  out:
658  - MaskedImage has background subtracted
659  - Wcs is replaced
660  - PhotoCalib is replaced
661  @param[in] exposureIdInfo ID info for exposure (an
662  lsst.obs.base.ExposureIdInfo) If not provided, returned
663  SourceCatalog IDs will not be globally unique.
664  @param[in,out] background background model already subtracted from
665  exposure (an lsst.afw.math.BackgroundList). May be None if no
666  background has been subtracted, though that is unusual for
667  calibration. A refined background model is output.
668  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
669  from which we can copy some fields.
670 
671  @return pipe_base Struct containing these fields:
672  - exposure calibrate science exposure with refined WCS and PhotoCalib
673  - background model of background subtracted from exposure (an
674  lsst.afw.math.BackgroundList)
675  - sourceCat catalog of measured sources
676  - astromMatches list of source/refObj matches from the astrometry
677  solver
678  """
679  # detect, deblend and measure sources
680  if exposureIdInfo is None:
681  exposureIdInfo = ExposureIdInfo()
682 
683  if background is None:
684  background = BackgroundList()
685  sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
686  table = SourceTable.make(self.schemaschema, sourceIdFactory)
687  table.setMetadata(self.algMetadataalgMetadata)
688 
689  detRes = self.detection.run(table=table, exposure=exposure,
690  doSmooth=True)
691  sourceCat = detRes.sources
692  if detRes.fpSets.background:
693  for bg in detRes.fpSets.background:
694  background.append(bg)
695  if self.config.doSkySources:
696  skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
697  if skySourceFootprints:
698  for foot in skySourceFootprints:
699  s = sourceCat.addNew()
700  s.setFootprint(foot)
701  s.set(self.skySourceKeyskySourceKey, True)
702  if self.config.doDeblend:
703  self.deblend.run(exposure=exposure, sources=sourceCat)
704  self.measurement.run(
705  measCat=sourceCat,
706  exposure=exposure,
707  exposureId=exposureIdInfo.expId
708  )
709  if self.config.doApCorr:
710  self.applyApCorr.run(
711  catalog=sourceCat,
712  apCorrMap=exposure.getInfo().getApCorrMap()
713  )
714  self.catalogCalculation.run(sourceCat)
715 
716  self.setPrimaryFlags.run(sourceCat)
717 
718  if icSourceCat is not None and \
719  len(self.config.icSourceFieldsToCopy) > 0:
720  self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
721  sourceCat=sourceCat)
722 
723  # TODO DM-11568: this contiguous check-and-copy could go away if we
724  # reserve enough space during SourceDetection and/or SourceDeblend.
725  # NOTE: sourceSelectors require contiguous catalogs, so ensure
726  # contiguity now, so views are preserved from here on.
727  if not sourceCat.isContiguous():
728  sourceCat = sourceCat.copy(deep=True)
729 
730  # perform astrometry calibration:
731  # fit an improved WCS and update the exposure's WCS in place
732  astromMatches = None
733  matchMeta = None
734  if self.config.doAstrometry:
735  try:
736  astromRes = self.astrometry.run(
737  exposure=exposure,
738  sourceCat=sourceCat,
739  )
740  astromMatches = astromRes.matches
741  matchMeta = astromRes.matchMeta
742  except Exception as e:
743  if self.config.requireAstrometry:
744  raise
745  self.log.warning("Unable to perform astrometric calibration "
746  "(%s): attempting to proceed", e)
747 
748  # compute photometric calibration
749  if self.config.doPhotoCal:
750  try:
751  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
752  exposure.setPhotoCalib(photoRes.photoCalib)
753  # TODO: reword this to phrase it in terms of the calibration factor?
754  self.log.info("Photometric zero-point: %f",
755  photoRes.photoCalib.instFluxToMagnitude(1.0))
756  self.setMetadatasetMetadata(exposure=exposure, photoRes=photoRes)
757  except Exception as e:
758  if self.config.requirePhotoCal:
759  raise
760  self.log.warning("Unable to perform photometric calibration "
761  "(%s): attempting to proceed", e)
762  self.setMetadatasetMetadata(exposure=exposure, photoRes=None)
763 
764  if self.config.doInsertFakes:
765  self.insertFakes.run(exposure, background=background)
766 
767  table = SourceTable.make(self.schemaschema, sourceIdFactory)
768  table.setMetadata(self.algMetadataalgMetadata)
769 
770  detRes = self.detection.run(table=table, exposure=exposure,
771  doSmooth=True)
772  sourceCat = detRes.sources
773  if detRes.fpSets.background:
774  for bg in detRes.fpSets.background:
775  background.append(bg)
776  if self.config.doDeblend:
777  self.deblend.run(exposure=exposure, sources=sourceCat)
778  self.measurement.run(
779  measCat=sourceCat,
780  exposure=exposure,
781  exposureId=exposureIdInfo.expId
782  )
783  if self.config.doApCorr:
784  self.applyApCorr.run(
785  catalog=sourceCat,
786  apCorrMap=exposure.getInfo().getApCorrMap()
787  )
788  self.catalogCalculation.run(sourceCat)
789 
790  if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
791  self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
792  sourceCat=sourceCat)
793 
794  if self.config.doComputeSummaryStats:
795  summary = self.computeSummaryStats.run(exposure=exposure,
796  sources=sourceCat,
797  background=background)
798  exposure.getInfo().setSummaryStats(summary)
799 
800  frame = getDebugFrame(self._display, "calibrate")
801  if frame:
803  sourceCat=sourceCat,
804  exposure=exposure,
805  matches=astromMatches,
806  frame=frame,
807  pause=False,
808  )
809 
810  return pipeBase.Struct(
811  exposure=exposure,
812  background=background,
813  sourceCat=sourceCat,
814  astromMatches=astromMatches,
815  matchMeta=matchMeta,
816  # These are duplicate entries with different names for use with
817  # gen3 middleware
818  outputExposure=exposure,
819  outputCat=sourceCat,
820  outputBackground=background,
821  )
822 
823  def writeOutputs(self, dataRef, exposure, background, sourceCat,
824  astromMatches, matchMeta):
825  """Write output data to the output repository
826 
827  @param[in] dataRef butler data reference corresponding to a science
828  image
829  @param[in] exposure exposure to write
830  @param[in] background background model for exposure
831  @param[in] sourceCat catalog of measured sources
832  @param[in] astromMatches list of source/refObj matches from the
833  astrometry solver
834  """
835  dataRef.put(sourceCat, "src")
836  if self.config.doWriteMatches and astromMatches is not None:
837  normalizedMatches = afwTable.packMatches(astromMatches)
838  normalizedMatches.table.setMetadata(matchMeta)
839  dataRef.put(normalizedMatches, "srcMatch")
840  if self.config.doWriteMatchesDenormalized:
841  denormMatches = denormalizeMatches(astromMatches, matchMeta)
842  dataRef.put(denormMatches, "srcMatchFull")
843  if self.config.doWriteExposure:
844  dataRef.put(exposure, "calexp")
845  dataRef.put(background, "calexpBackground")
846 
847  def getSchemaCatalogs(self):
848  """Return a dict of empty catalogs for each catalog dataset produced
849  by this task.
850  """
851  sourceCat = afwTable.SourceCatalog(self.schemaschema)
852  sourceCat.getTable().setMetadata(self.algMetadataalgMetadata)
853  return {"src": sourceCat}
854 
855  def setMetadata(self, exposure, photoRes=None):
856  """!Set task and exposure metadata
857 
858  Logs a warning and continues if needed data is missing.
859 
860  @param[in,out] exposure exposure whose metadata is to be set
861  @param[in] photoRes results of running photoCal; if None then it was
862  not run
863  """
864  if photoRes is None:
865  return
866 
867  metadata = exposure.getMetadata()
868 
869  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
870  try:
871  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
872  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
873  except Exception:
874  self.log.warning("Could not set normalized MAGZERO in header: no "
875  "exposure time")
876  magZero = math.nan
877 
878  try:
879  metadata.set('MAGZERO', magZero)
880  metadata.set('MAGZERO_RMS', photoRes.sigma)
881  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
882  metadata.set('COLORTERM1', 0.0)
883  metadata.set('COLORTERM2', 0.0)
884  metadata.set('COLORTERM3', 0.0)
885  except Exception as e:
886  self.log.warning("Could not set exposure metadata: %s", e)
887 
888  def copyIcSourceFields(self, icSourceCat, sourceCat):
889  """!Match sources in icSourceCat and sourceCat and copy the specified fields
890 
891  @param[in] icSourceCat catalog from which to copy fields
892  @param[in,out] sourceCat catalog to which to copy fields
893 
894  The fields copied are those specified by `config.icSourceFieldsToCopy`
895  that actually exist in the schema. This was set up by the constructor
896  using self.schemaMapper.
897  """
898  if self.schemaMapperschemaMapper is None:
899  raise RuntimeError("To copy icSource fields you must specify "
900  "icSourceSchema nd icSourceKeys when "
901  "constructing this task")
902  if icSourceCat is None or sourceCat is None:
903  raise RuntimeError("icSourceCat and sourceCat must both be "
904  "specified")
905  if len(self.config.icSourceFieldsToCopy) == 0:
906  self.log.warning("copyIcSourceFields doing nothing because "
907  "icSourceFieldsToCopy is empty")
908  return
909 
910  mc = afwTable.MatchControl()
911  mc.findOnlyClosest = False # return all matched objects
912  matches = afwTable.matchXy(icSourceCat, sourceCat,
913  self.config.matchRadiusPix, mc)
914  if self.config.doDeblend:
915  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
916  # if deblended, keep children
917  matches = [m for m in matches if m[1].get(deblendKey) == 0]
918 
919  # Because we had to allow multiple matches to handle parents, we now
920  # need to prune to the best matches
921  # closest matches as a dict of icSourceCat source ID:
922  # (icSourceCat source, sourceCat source, distance in pixels)
923  bestMatches = {}
924  for m0, m1, d in matches:
925  id0 = m0.getId()
926  match = bestMatches.get(id0)
927  if match is None or d <= match[2]:
928  bestMatches[id0] = (m0, m1, d)
929  matches = list(bestMatches.values())
930 
931  # Check that no sourceCat sources are listed twice (we already know
932  # that each match has a unique icSourceCat source ID, due to using
933  # that ID as the key in bestMatches)
934  numMatches = len(matches)
935  numUniqueSources = len(set(m[1].getId() for m in matches))
936  if numUniqueSources != numMatches:
937  self.log.warning("%d icSourceCat sources matched only %d sourceCat "
938  "sources", numMatches, numUniqueSources)
939 
940  self.log.info("Copying flags from icSourceCat to sourceCat for "
941  "%d sources", numMatches)
942 
943  # For each match: set the calibSourceKey flag and copy the desired
944  # fields
945  for icSrc, src, d in matches:
946  src.setFlag(self.calibSourceKeycalibSourceKey, True)
947  # src.assign copies the footprint from icSrc, which we don't want
948  # (DM-407)
949  # so set icSrc's footprint to src's footprint before src.assign,
950  # then restore it
951  icSrcFootprint = icSrc.getFootprint()
952  try:
953  icSrc.setFootprint(src.getFootprint())
954  src.assign(icSrc, self.schemaMapperschemaMapper)
955  finally:
956  icSrc.setFootprint(icSrcFootprint)
afw::table::PointKey< int > dimensions
Definition: GaussianPsf.cc:48
Pass parameters to algorithms that match list of sources.
Definition: Match.h:45
A mapping between the keys of two Schemas, used to copy data between them.
Definition: SchemaMapper.h:21
Class for storing ordered metadata with comments.
Definition: PropertyList.h:68
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:337
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:824
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:555
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:436
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:855
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:617
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:888
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:649
daf::base::PropertyList * list
Definition: fits.cc:913
daf::base::PropertySet * set
Definition: fits.cc:912
BaseCatalog packMatches(std::vector< Match< Record1, Record2 > > const &matches)
Return a table representation of a MatchVector that can be used to persist it.
Definition: Match.cc:432
SourceMatchVector matchXy(SourceCatalog const &cat1, SourceCatalog const &cat2, double radius, MatchControl const &mc=MatchControl())
Compute all tuples (s1,s2,d) where s1 belings to cat1, s2 belongs to cat2 and d, the distance between...
Definition: Match.cc:305
bool any(CoordinateExpr< N > const &expr) noexcept
Return true if any elements are true.
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations.
def denormalizeMatches(matches, matchMeta=None)
def displayAstrometry(refCat=None, sourceCat=None, distortedCentroidKey=None, bbox=None, exposure=None, matches=None, frame=1, title="", pause=True)
Definition: display.py:34
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
def getDebugFrame(debugDisplay, name)
Definition: lsstDebug.py:95
A description of a field in a table.
Definition: Field.h:24