LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
forcedPhotCoadd.py
Go to the documentation of this file.
1 # This file is part of meas_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import lsst.pex.config
23 import lsst.coadd.utils
24 import lsst.afw.table
25 
26 import lsst.pipe.base as pipeBase
27 from lsst.obs.base import ExposureIdInfo
28 
29 from .references import MultiBandReferencesTask
30 from .forcedMeasurement import ForcedMeasurementTask
31 from .applyApCorr import ApplyApCorrTask
32 from .catalogCalculation import CatalogCalculationTask
33 
34 __all__ = ("ForcedPhotCoaddConfig", "ForcedPhotCoaddTask")
35 
36 
37 class ForcedPhotCoaddRunner(pipeBase.ButlerInitializedTaskRunner):
38  """Get the psfCache setting into ForcedPhotCoaddTask"""
39  @staticmethod
40  def getTargetList(parsedCmd, **kwargs):
41  return pipeBase.ButlerInitializedTaskRunner.getTargetList(parsedCmd,
42  psfCache=parsedCmd.psfCache)
43 
44 
45 class ForcedPhotCoaddConnections(pipeBase.PipelineTaskConnections,
46  dimensions=("band", "skymap", "tract", "patch"),
47  defaultTemplates={"inputCoaddName": "deep",
48  "outputCoaddName": "deep"}):
49  inputSchema = pipeBase.connectionTypes.InitInput(
50  doc="Schema for the input measurement catalogs.",
51  name="{inputCoaddName}Coadd_ref_schema",
52  storageClass="SourceCatalog",
53  )
54  outputSchema = pipeBase.connectionTypes.InitOutput(
55  doc="Schema for the output forced measurement catalogs.",
56  name="{outputCoaddName}Coadd_forced_src_schema",
57  storageClass="SourceCatalog",
58  )
59  exposure = pipeBase.connectionTypes.Input(
60  doc="Input exposure to perform photometry on.",
61  name="{inputCoaddName}Coadd",
62  storageClass="ExposureF",
63  dimensions=["band", "skymap", "tract", "patch"],
64  )
65  refCat = pipeBase.connectionTypes.Input(
66  doc="Catalog of shapes and positions at which to force photometry.",
67  name="{inputCoaddName}Coadd_ref",
68  storageClass="SourceCatalog",
69  dimensions=["skymap", "tract", "patch"],
70  )
71  refCatInBand = pipeBase.connectionTypes.Input(
72  doc="Catalog of shapes and positions in the band having forced photometry done",
73  name="{inputCoaddName}Coadd_meas",
74  storageClass="SourceCatalog",
75  dimensions=("band", "skymap", "tract", "patch")
76  )
77  refWcs = pipeBase.connectionTypes.Input(
78  doc="Reference world coordinate system.",
79  name="{inputCoaddName}Coadd.wcs",
80  storageClass="Wcs",
81  dimensions=["band", "skymap", "tract", "patch"],
82  ) # used in place of a skymap wcs because of DM-28880
83  measCat = pipeBase.connectionTypes.Output(
84  doc="Output forced photometry catalog.",
85  name="{outputCoaddName}Coadd_forced_src",
86  storageClass="SourceCatalog",
87  dimensions=["band", "skymap", "tract", "patch"],
88  )
89 
90 
91 class ForcedPhotCoaddConfig(pipeBase.PipelineTaskConfig,
92  pipelineConnections=ForcedPhotCoaddConnections):
94  target=MultiBandReferencesTask,
95  doc="subtask to retrieve reference source catalog"
96  )
98  target=ForcedMeasurementTask,
99  doc="subtask to do forced measurement"
100  )
101  coaddName = lsst.pex.config.Field(
102  doc="coadd name: typically one of deep or goodSeeing",
103  dtype=str,
104  default="deep",
105  )
106  doApCorr = lsst.pex.config.Field(
107  dtype=bool,
108  default=True,
109  doc="Run subtask to apply aperture corrections"
110  )
111  applyApCorr = lsst.pex.config.ConfigurableField(
112  target=ApplyApCorrTask,
113  doc="Subtask to apply aperture corrections"
114  )
115  catalogCalculation = lsst.pex.config.ConfigurableField(
116  target=CatalogCalculationTask,
117  doc="Subtask to run catalogCalculation plugins on catalog"
118  )
119  footprintDatasetName = lsst.pex.config.Field(
120  doc="Dataset (without coadd prefix) that should be used to obtain (Heavy)Footprints for sources. "
121  "Must have IDs that match those of the reference catalog."
122  "If None, Footprints will be generated by transforming the reference Footprints.",
123  dtype=str,
124  default="meas",
125  optional=True
126  )
127  hasFakes = lsst.pex.config.Field(
128  dtype=bool,
129  default=False,
130  doc="Should be set to True if fake sources have been inserted into the input data."
131  )
132 
133  def setDefaults(self):
134  # Docstring inherited.
135  # Make catalogCalculation a no-op by default as no modelFlux is setup by default in
136  # ForcedMeasurementTask
137  super().setDefaults()
138 
139  self.catalogCalculation.plugins.names = []
140  self.measurement.copyColumns["id"] = "id"
141  self.measurement.copyColumns["parent"] = "parent"
142  self.references.removePatchOverlaps = False # see validate() for why
143  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
144  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
145  'REJECTED', 'INEXACT_PSF']
146  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
147  'REJECTED', 'INEXACT_PSF']
148 
149  def validate(self):
150  super().validate()
151  if (self.measurement.doReplaceWithNoise and self.footprintDatasetName is not None
152  and self.references.removePatchOverlaps):
153  raise ValueError("Cannot use removePatchOverlaps=True with deblended footprints, as parent "
154  "sources may be rejected while their children are not.")
155 
156 
157 class ForcedPhotCoaddTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
158  """A command-line driver for performing forced measurement on coadd images.
159 
160  Parameters
161  ----------
162  butler : `lsst.daf.persistence.butler.Butler`, optional
163  A Butler which will be passed to the references subtask to allow it to
164  load its schema from disk. Optional, but must be specified if
165  ``refSchema`` is not; if both are specified, ``refSchema`` takes
166  precedence.
167  refSchema : `lsst.afw.table.Schema`, optional
168  The schema of the reference catalog, passed to the constructor of the
169  references subtask. Optional, but must be specified if ``butler`` is
170  not; if both are specified, ``refSchema`` takes precedence.
171  **kwds
172  Keyword arguments are passed to the supertask constructor.
173  """
174 
175  ConfigClass = ForcedPhotCoaddConfig
176  RunnerClass = ForcedPhotCoaddRunner
177  _DefaultName = "forcedPhotCoadd"
178  dataPrefix = "deepCoadd_"
179 
180  def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
181  super().__init__(**kwds)
182 
183  if initInputs is not None:
184  refSchema = initInputs['inputSchema'].schema
185 
186  self.makeSubtask("references", butler=butler, schema=refSchema)
187  if refSchema is None:
188  refSchema = self.references.schema
189  self.makeSubtask("measurement", refSchema=refSchema)
190  # It is necessary to get the schema internal to the forced measurement task until such a time
191  # that the schema is not owned by the measurement task, but is passed in by an external caller
192  if self.config.doApCorr:
193  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
194  self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
195  self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
196 
197  def runQuantum(self, butlerQC, inputRefs, outputRefs):
198  inputs = butlerQC.get(inputRefs)
199 
200  refCatInBand = inputs.pop('refCatInBand')
201  inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
202  inputs['exposure'],
203  inputs['refCat'],
204  refCatInBand,
205  inputs['refWcs'],
206  "tract_patch")
207  outputs = self.run(**inputs)
208  butlerQC.put(outputs, outputRefs)
209 
210  def generateMeasCat(self, exposureDataId, exposure, refCat, refCatInBand, refWcs, idPackerName):
211  """Generate a measurement catalog for Gen3.
212 
213  Parameters
214  ----------
215  exposureDataId : `DataId`
216  Butler dataId for this exposure.
217  exposure : `lsst.afw.image.exposure.Exposure`
218  Exposure to generate the catalog for.
219  refCat : `lsst.afw.table.SourceCatalog`
220  Catalog of shapes and positions at which to force photometry.
221  refCatInBand : `lsst.afw.table.SourceCatalog`
222  Catalog of shapes and position in the band forced photometry is
223  currently being performed
224  refWcs : `lsst.afw.image.SkyWcs`
225  Reference world coordinate system.
226  idPackerName : `str`
227  Type of ID packer to construct from the registry.
228 
229  Returns
230  -------
231  measCat : `lsst.afw.table.SourceCatalog`
232  Catalog of forced sources to measure.
233  expId : `int`
234  Unique binary id associated with the input exposure
235 
236  Raises
237  ------
238  LookupError
239  Raised if a footprint with a given source id was in the reference
240  catalog but not in the reference catalog in band (meaning there
241  was some sort of mismatch in the two input catalogs)
242  """
243  exposureIdInfo = ExposureIdInfo.fromDataId(exposureDataId, idPackerName)
244  idFactory = exposureIdInfo.makeSourceIdFactory()
245 
246  measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
247  idFactory=idFactory)
248  # attach footprints here, as the attachFootprints method is geared for gen2
249  # and is not worth modifying, as this can naturally live inside this method
250  for srcRecord in measCat:
251  fpRecord = refCatInBand.find(srcRecord.getId())
252  if fpRecord is None:
253  raise LookupError("Cannot find Footprint for source {}; please check that {} "
254  "IDs are compatible with reference source IDs"
255  .format(srcRecord.getId(), self.config.connections.refCatInBand))
256  srcRecord.setFootprint(fpRecord.getFootprint())
257  return measCat, exposureIdInfo.expId
258 
259  def runDataRef(self, dataRef, psfCache=None):
260  """Perform forced measurement on a single exposure.
261 
262  Parameters
263  ----------
264  dataRef : `lsst.daf.persistence.ButlerDataRef`
265  Passed to the ``references`` subtask to obtain the reference WCS,
266  the ``getExposure`` method (implemented by derived classes) to
267  read the measurment image, and the ``fetchReferences`` method to
268  get the exposure and load the reference catalog (see
269  :lsst-task`lsst.meas.base.references.CoaddSrcReferencesTask`).
270  Refer to derived class documentation for details of the datasets
271  and data ID keys which are used.
272  psfCache : `int`, optional
273  Size of PSF cache, or `None`. The size of the PSF cache can have
274  a significant effect upon the runtime for complicated PSF models.
275 
276  Notes
277  -----
278  Sources are generated with ``generateMeasCat`` in the ``measurement``
279  subtask. These are passed to ``measurement``'s ``run`` method, which
280  fills the source catalog with the forced measurement results. The
281  sources are then passed to the ``writeOutputs`` method (implemented by
282  derived classes) which writes the outputs.
283  """
284  refWcs = self.references.getWcs(dataRef)
285  exposure = self.getExposure(dataRef)
286  if psfCache is not None:
287  exposure.getPsf().setCacheCapacity(psfCache)
288  refCat = self.fetchReferences(dataRef, exposure)
289 
290  measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
291  idFactory=self.makeIdFactory(dataRef))
292  self.log.info("Performing forced measurement on %s", dataRef.dataId)
293  self.attachFootprints(measCat, refCat, exposure, refWcs, dataRef)
294 
295  exposureId = self.getExposureId(dataRef)
296 
297  forcedPhotResult = self.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
298 
299  self.writeOutput(dataRef, forcedPhotResult.measCat)
300 
301  def run(self, measCat, exposure, refCat, refWcs, exposureId=None):
302  """Perform forced measurement on a single exposure.
303 
304  Parameters
305  ----------
306  measCat : `lsst.afw.table.SourceCatalog`
307  The measurement catalog, based on the sources listed in the
308  reference catalog.
309  exposure : `lsst.afw.image.Exposure`
310  The measurement image upon which to perform forced detection.
311  refCat : `lsst.afw.table.SourceCatalog`
312  The reference catalog of sources to measure.
313  refWcs : `lsst.afw.image.SkyWcs`
314  The WCS for the references.
315  exposureId : `int`
316  Optional unique exposureId used for random seed in measurement
317  task.
318 
319  Returns
320  -------
321  result : ~`lsst.pipe.base.Struct`
322  Structure with fields:
323 
324  ``measCat``
325  Catalog of forced measurement results
326  (`lsst.afw.table.SourceCatalog`).
327  """
328  self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
329  if self.config.doApCorr:
330  self.applyApCorr.run(
331  catalog=measCat,
332  apCorrMap=exposure.getInfo().getApCorrMap()
333  )
334  self.catalogCalculation.run(measCat)
335 
336  return pipeBase.Struct(measCat=measCat)
337 
338  def makeIdFactory(self, dataRef):
339  """Create an object that generates globally unique source IDs.
340 
341  Source IDs are created based on a per-CCD ID and the ID of the CCD
342  itself.
343 
344  Parameters
345  ----------
346  dataRef : `lsst.daf.persistence.ButlerDataRef`
347  Butler data reference. The "CoaddId_bits" and "CoaddId" datasets
348  are accessed. The data ID must have tract and patch keys.
349  """
350  # With the default configuration, this IdFactory doesn't do anything,
351  # because the IDs it generates are immediately overwritten by the ID
352  # from the reference catalog (since that's in
353  # config.measurement.copyColumns). But we create one here anyway, to
354  # allow us to revert back to the old behavior of generating new forced
355  # source IDs, just by renaming the ID in config.copyColumns to
356  # "object_id".
357  exposureIdInfo = ExposureIdInfo(int(dataRef.get(self.config.coaddName + "CoaddId")),
358  dataRef.get(self.config.coaddName + "CoaddId_bits"))
359  return exposureIdInfo.makeSourceIdFactory()
360 
361  def getExposureId(self, dataRef):
362  return int(dataRef.get(self.config.coaddName + "CoaddId"))
363 
364  def fetchReferences(self, dataRef, exposure):
365  """Return an iterable of reference sources which overlap the exposure.
366 
367  Parameters
368  ----------
369  dataRef : `lsst.daf.persistence.ButlerDataRef`
370  Butler data reference corresponding to the image to be measured;
371  should have tract, patch, and filter keys.
372 
373  exposure : `lsst.afw.image.Exposure`
374  Unused.
375 
376  Notes
377  -----
378  All work is delegated to the references subtask; see
379  `CoaddSrcReferencesTask` for information about the default behavior.
380  """
381  skyMap = dataRef.get(self.dataPrefix + "skyMap", immediate=True)
382  tractInfo = skyMap[dataRef.dataId["tract"]]
383  patch = tuple(int(v) for v in dataRef.dataId["patch"].split(","))
384  patchInfo = tractInfo.getPatchInfo(patch)
385  references = lsst.afw.table.SourceCatalog(self.references.schema)
386  references.extend(self.references.fetchInPatches(dataRef, patchList=[patchInfo]))
387  return references
388 
389  def attachFootprints(self, sources, refCat, exposure, refWcs, dataRef):
390  r"""Attach Footprints to source records.
391 
392  For coadd forced photometry, we use the deblended "heavy"
393  `~lsst.afw.detection.Footprint`\ s from the single-band measurements
394  of the same band - because we've guaranteed that the peaks (and hence
395  child sources) will be consistent across all bands before we get to
396  measurement, this should yield reasonable deblending for most sources.
397  It's most likely limitation is that it will not provide good flux
398  upper limits for sources that were not detected in this band but were
399  blended with sources that were.
400  """
401  if self.config.footprintDatasetName is None:
402  return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs)
403 
404  self.log.info("Loading deblended footprints for sources from %s, %s",
405  self.config.footprintDatasetName, dataRef.dataId)
406  fpCat = dataRef.get("%sCoadd_%s" % (self.config.coaddName, self.config.footprintDatasetName),
407  immediate=True)
408  for refRecord, srcRecord in zip(refCat, sources):
409  fpRecord = fpCat.find(refRecord.getId())
410  if fpRecord is None:
411  raise LookupError("Cannot find Footprint for source %s; please check that %sCoadd_%s "
412  "IDs are compatible with reference source IDs" %
413  (srcRecord.getId(), self.config.coaddName,
414  self.config.footprintDatasetName))
415  srcRecord.setFootprint(fpRecord.getFootprint())
416 
417  def getExposure(self, dataRef):
418  """Read input exposure on which measurement will be performed.
419 
420  Parameters
421  ----------
422  dataRef : `lsst.daf.persistence.ButlerDataRef`
423  Butler data reference.
424  """
425  if self.config.hasFakes:
426  name = "fakes_" + self.config.coaddName + "Coadd_calexp"
427  else:
428  name = self.config.coaddName + "Coadd_calexp"
429 
430  return dataRef.get(name) if dataRef.datasetExists(name) else None
431 
432  def writeOutput(self, dataRef, sources):
433  """Write forced source table
434 
435  Parameters
436  ----------
437  dataRef : `lsst.daf.persistence.ButlerDataRef`
438  Butler data reference. The forced_src dataset (with
439  self.dataPrefix prepended) is all that will be modified.
440  sources : `lsst.afw.table.SourceCatalog`
441  Catalog of sources to save.
442  """
443  dataRef.put(sources, self.dataPrefix + "forced_src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
444 
445  def getSchemaCatalogs(self):
446  """The schema catalogs that will be used by this task.
447 
448  Returns
449  -------
450  schemaCatalogs : `dict`
451  Dictionary mapping dataset type to schema catalog.
452 
453  Notes
454  -----
455  There is only one schema for each type of forced measurement. The
456  dataset type for this measurement is defined in the mapper.
457  """
458  catalog = lsst.afw.table.SourceCatalog(self.measurement.schema)
459  catalog.getTable().setMetadata(self.measurement.algMetadata)
460  datasetType = self.dataPrefix + "forced_src"
461  return {datasetType: catalog}
462 
463  def _getConfigName(self):
464  # Documented in superclass
465  return self.dataPrefix + "forced_config"
466 
467  def _getMetadataName(self):
468  # Documented in superclass
469  return self.dataPrefix + "forced_metadata"
470 
471  @classmethod
472  def _makeArgumentParser(cls):
473  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
474  parser.add_id_argument("--id", "deepCoadd_forced_src", help="data ID, with raw CCD keys + tract",
476  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
477  return parser
def run(self, coaddExposures, bbox, wcs)
Definition: getTemplate.py:603
def attachFootprints(self, sources, refCat, exposure, refWcs, dataRef)
def writeOutput(self, dataRef, sources)
def fetchReferences(self, dataRef, exposure)
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174