LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
processCcdWithFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fake sources into calexps
24 """
25 from astropy.table import Table
26 import numpy as np
27 import pandas as pd
28 
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from .insertFakes import InsertFakesTask
33 from lsst.meas.base import PerTractCcdDataIdContainer
34 from lsst.afw.table import SourceTable
35 from lsst.obs.base import ExposureIdInfo
36 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
37 import lsst.pipe.base.connectionTypes as cT
38 import lsst.afw.table as afwTable
39 from lsst.pipe.tasks.calibrate import CalibrateTask
40 
41 __all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask",
42  "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"]
43 
44 
45 class ProcessCcdWithFakesConnections(PipelineTaskConnections,
46  dimensions=("skymap", "tract", "instrument", "visit", "detector"),
47  defaultTemplates={"coaddName": "deep",
48  "wcsName": "jointcal",
49  "photoCalibName": "jointcal",
50  "fakesType": "fakes_"}):
51 
52  exposure = cT.Input(
53  doc="Exposure into which fakes are to be added.",
54  name="calexp",
55  storageClass="ExposureF",
56  dimensions=("instrument", "visit", "detector")
57  )
58 
59  fakeCat = cT.Input(
60  doc="Catalog of fake sources to draw inputs from.",
61  name="{fakesType}fakeSourceCat",
62  storageClass="DataFrame",
63  dimensions=("tract", "skymap")
64  )
65 
66  externalSkyWcsTractCatalog = cT.Input(
67  doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
68  "id for the catalog id, sorted on id for fast lookup."),
69  name="{wcsName}SkyWcsCatalog",
70  storageClass="ExposureCatalog",
71  dimensions=("instrument", "visit", "tract", "skymap"),
72  )
73 
74  externalSkyWcsGlobalCatalog = cT.Input(
75  doc=("Per-visit wcs calibrations computed globally (with no tract information). "
76  "These catalogs use the detector id for the catalog id, sorted on id for "
77  "fast lookup."),
78  name="{wcsName}SkyWcsCatalog",
79  storageClass="ExposureCatalog",
80  dimensions=("instrument", "visit"),
81  )
82 
83  externalPhotoCalibTractCatalog = cT.Input(
84  doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
85  "detector id for the catalog id, sorted on id for fast lookup."),
86  name="{photoCalibName}PhotoCalibCatalog",
87  storageClass="ExposureCatalog",
88  dimensions=("instrument", "visit", "tract"),
89  )
90 
91  externalPhotoCalibGlobalCatalog = cT.Input(
92  doc=("Per-visit photometric calibrations. These catalogs use the "
93  "detector id for the catalog id, sorted on id for fast lookup."),
94  name="{photoCalibName}PhotoCalibCatalog",
95  storageClass="ExposureCatalog",
96  dimensions=("instrument", "visit"),
97  )
98 
99  icSourceCat = cT.Input(
100  doc="Catalog of calibration sources",
101  name="icSrc",
102  storageClass="SourceCatalog",
103  dimensions=("instrument", "visit", "detector")
104  )
105 
106  sfdSourceCat = cT.Input(
107  doc="Catalog of calibration sources",
108  name="src",
109  storageClass="SourceCatalog",
110  dimensions=("instrument", "visit", "detector")
111  )
112 
113  outputExposure = cT.Output(
114  doc="Exposure with fake sources added.",
115  name="{fakesType}calexp",
116  storageClass="ExposureF",
117  dimensions=("instrument", "visit", "detector")
118  )
119 
120  outputCat = cT.Output(
121  doc="Source catalog produced in calibrate task with fakes also measured.",
122  name="{fakesType}src",
123  storageClass="SourceCatalog",
124  dimensions=("instrument", "visit", "detector"),
125  )
126 
127  def __init__(self, *, config=None):
128  super().__init__(config=config)
129 
130  if not config.doApplyExternalGlobalPhotoCalib:
131  self.inputs.remove("externalPhotoCalibGlobalCatalog")
132  if not config.doApplyExternalTractPhotoCalib:
133  self.inputs.remove("externalPhotoCalibTractCatalog")
134 
135  if not config.doApplyExternalGlobalSkyWcs:
136  self.inputs.remove("externalSkyWcsGlobalCatalog")
137  if not config.doApplyExternalTractSkyWcs:
138  self.inputs.remove("externalSkyWcsTractCatalog")
139 
140 
141 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
142  pipelineConnections=ProcessCcdWithFakesConnections):
143  """Config for inserting fake sources
144 
145  Notes
146  -----
147  The default column names are those from the UW sims database.
148  """
149 
150  doApplyExternalGlobalPhotoCalib = pexConfig.Field(
151  dtype=bool,
152  default=False,
153  doc="Whether to apply an external photometric calibration via an "
154  "`lsst.afw.image.PhotoCalib` object. Uses the "
155  "`externalPhotoCalibName` config option to determine which "
156  "calibration to use. Uses a global calibration."
157  )
158 
159  doApplyExternalTractPhotoCalib = pexConfig.Field(
160  dtype=bool,
161  default=False,
162  doc="Whether to apply an external photometric calibration via an "
163  "`lsst.afw.image.PhotoCalib` object. Uses the "
164  "`externalPhotoCalibName` config option to determine which "
165  "calibration to use. Uses a per tract calibration."
166  )
167 
168  externalPhotoCalibName = pexConfig.ChoiceField(
169  doc="What type of external photo calib to use.",
170  dtype=str,
171  default="jointcal",
172  allowed={"jointcal": "Use jointcal_photoCalib",
173  "fgcm": "Use fgcm_photoCalib",
174  "fgcm_tract": "Use fgcm_tract_photoCalib"}
175  )
176 
177  doApplyExternalGlobalSkyWcs = pexConfig.Field(
178  dtype=bool,
179  default=False,
180  doc="Whether to apply an external astrometric calibration via an "
181  "`lsst.afw.geom.SkyWcs` object. Uses the "
182  "`externalSkyWcsName` config option to determine which "
183  "calibration to use. Uses a global calibration."
184  )
185 
186  doApplyExternalTractSkyWcs = pexConfig.Field(
187  dtype=bool,
188  default=False,
189  doc="Whether to apply an external astrometric calibration via an "
190  "`lsst.afw.geom.SkyWcs` object. Uses the "
191  "`externalSkyWcsName` config option to determine which "
192  "calibration to use. Uses a per tract calibration."
193  )
194 
195  externalSkyWcsName = pexConfig.ChoiceField(
196  doc="What type of updated WCS calib to use.",
197  dtype=str,
198  default="jointcal",
199  allowed={"jointcal": "Use jointcal_wcs"}
200  )
201 
202  coaddName = pexConfig.Field(
203  doc="The name of the type of coadd used",
204  dtype=str,
205  default="deep",
206  )
207 
208  srcFieldsToCopy = pexConfig.ListField(
209  dtype=str,
210  default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
211  "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
212  doc=("Fields to copy from the `src` catalog to the output catalog "
213  "for matching sources Any missing fields will trigger a "
214  "RuntimeError exception.")
215  )
216 
217  matchRadiusPix = pexConfig.Field(
218  dtype=float,
219  default=3,
220  doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
221  )
222 
223  calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
224  doc="The calibration task to use.")
225 
226  insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
227  doc="Configuration for the fake sources")
228 
229  def setDefaults(self):
230  super().setDefaults()
231  self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
232  self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
233  self.calibrate.doAstrometry = False
234  self.calibrate.doWriteMatches = False
235  self.calibrate.doPhotoCal = False
236  self.calibrate.detection.reEstimateBackground = False
237 
238 
239 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
240  """Insert fake objects into calexps.
241 
242  Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
243  from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
244  give a new background estimation and measurement of the calexp.
245 
246  `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
247  sources and then add them to the calexp.
248 
249  `addPixCoords`
250  Use the WCS information to add the pixel coordinates of each source
251  Adds an ``x`` and ``y`` column to the catalog of fake sources.
252  `trimFakeCat`
253  Trim the fake cat to about the size of the input image.
254  `mkFakeGalsimGalaxies`
255  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
256  `mkFakeStars`
257  Use the PSF information from the calexp to make a fake star using the magnitude information from the
258  input file.
259  `cleanCat`
260  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
261  that are 0.
262  `addFakeSources`
263  Add the fake sources to the calexp.
264 
265  Notes
266  -----
267  The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
268  """
269 
270  _DefaultName = "processCcdWithFakes"
271  ConfigClass = ProcessCcdWithFakesConfig
272 
273  def __init__(self, schema=None, butler=None, **kwargs):
274  """Initalize things! This should go above in the class docstring
275  """
276 
277  super().__init__(**kwargs)
278 
279  if schema is None:
280  schema = SourceTable.makeMinimalSchema()
281  self.schema = schema
282  self.makeSubtask("insertFakes")
283  self.makeSubtask("calibrate")
284 
285  def runDataRef(self, dataRef):
286  """Read in/write out the required data products and add fake sources to the calexp.
287 
288  Parameters
289  ----------
290  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
291  Data reference defining the ccd to have fakes added to it.
292  Used to access the following data products:
293  calexp
294  jointcal_wcs
295  jointcal_photoCalib
296 
297  Notes
298  -----
299  Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
300  of the sources unless the config option config.externalPhotoCalibName or config.externalSkyWcsName
301  are set then it uses the specified outputs. The config defualts for the column names in the catalog
302  of fakes are taken from the University of Washington simulations database.
303  Operates on one ccd at a time.
304  """
305  exposureIdInfo = dataRef.get("expIdInfo")
306 
307  if self.config.insertFakes.fakeType == "snapshot":
308  fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
309  elif self.config.insertFakes.fakeType == "static":
310  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
311  else:
312  fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
313 
314  calexp = dataRef.get("calexp")
315  if self.config.doApplyExternalGlobalSkyWcs or self.config.doApplyExternalTractSkyWcs:
316  self.log.info("Using external wcs from %s", self.config.externalSkyWcsName)
317  wcs = dataRef.get(self.config.externalSkyWcsName + "_wcs")
318  else:
319  wcs = calexp.getWcs()
320 
321  if self.config.doApplyExternalGlobalPhotoCalib or self.config.doApplyExternalTractPhotoCalib:
322  self.log.info("Using external photocalib from %s", self.config.externalPhotoCalibName)
323  photoCalib = dataRef.get(self.config.externalPhotoCalibName + "_photoCalib")
324  else:
325  photoCalib = calexp.getPhotoCalib()
326 
327  icSourceCat = dataRef.get("icSrc", immediate=True)
328  sfdSourceCat = dataRef.get("src", immediate=True)
329 
330  resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
331  exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
332  sfdSourceCat=sfdSourceCat)
333 
334  dataRef.put(resultStruct.outputExposure, "fakes_calexp")
335  dataRef.put(resultStruct.outputCat, "fakes_src")
336  return resultStruct
337 
338  def runQuantum(self, butlerQC, inputRefs, outputRefs):
339  inputs = butlerQC.get(inputRefs)
340  detectorId = inputs["exposure"].getInfo().getDetector().getId()
341 
342  if 'exposureIdInfo' not in inputs.keys():
343  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
344  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
345 
346  if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
347  inputs["wcs"] = inputs["exposure"].getWcs()
348 
349  elif self.config.doApplyExternalGlobalSkyWcs:
350  externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
351  row = externalSkyWcsCatalog.find(detectorId)
352  inputs["wcs"] = row.getWcs()
353 
354  elif self.config.doApplyExternalTractSkyWcs:
355  externalSkyWcsCatalog = inputs["externalSkyWcsTractCatalog"]
356  row = externalSkyWcsCatalog.find(detectorId)
357  inputs["wcs"] = row.getWcs()
358 
359  if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
360  inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
361 
362  elif self.config.doApplyExternalGlobalPhotoCalib:
363  externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
364  row = externalPhotoCalibCatalog.find(detectorId)
365  inputs["photoCalib"] = row.getPhotoCalib()
366 
367  elif self.config.doApplyExternalTractPhotoCalib:
368  externalPhotoCalibCatalog = inputs["externalPhotoCalibTractCatalog"]
369  row = externalPhotoCalibCatalog.find(detectorId)
370  inputs["photoCalib"] = row.getPhotoCalib()
371 
372  outputs = self.run(**inputs)
373  butlerQC.put(outputs, outputRefs)
374 
375  @classmethod
376  def _makeArgumentParser(cls):
377  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
378  parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
379  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
380  ContainerClass=PerTractCcdDataIdContainer)
381  return parser
382 
383  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
384  sfdSourceCat=None, externalSkyWcsGlobalCatalog=None, externalSkyWcsTractCatalog=None,
385  externalPhotoCalibGlobalCatalog=None, externalPhotoCalibTractCatalog=None):
386  """Add fake sources to a calexp and then run detection, deblending and measurement.
387 
388  Parameters
389  ----------
390  fakeCat : `pandas.core.frame.DataFrame`
391  The catalog of fake sources to add to the exposure
392  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
393  The exposure to add the fake sources to
394  wcs : `lsst.afw.geom.SkyWcs`
395  WCS to use to add fake sources
396  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
397  Photometric calibration to be used to calibrate the fake sources
398  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
399  icSourceCat : `lsst.afw.table.SourceCatalog`
400  Default : None
401  Catalog to take the information about which sources were used for calibration from.
402  sfdSourceCat : `lsst.afw.table.SourceCatalog`
403  Default : None
404  Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
405 
406  Returns
407  -------
408  resultStruct : `lsst.pipe.base.struct.Struct`
409  contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
410  outputCat : `lsst.afw.table.source.source.SourceCatalog`
411 
412  Notes
413  -----
414  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
415  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
416  pixels.
417 
418  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
419  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
420  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
421  the calexp and the calexp with fakes included returned.
422 
423  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
424  this is then convolved with the PSF at that point.
425 
426  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
427  """
428 
429  if wcs is None:
430  wcs = exposure.getWcs()
431 
432  if photoCalib is None:
433  photoCalib = exposure.getPhotoCalib()
434 
435  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
436 
437  # detect, deblend and measure sources
438  if exposureIdInfo is None:
439  exposureIdInfo = ExposureIdInfo()
440  returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
441  sourceCat = returnedStruct.sourceCat
442 
443  sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
444 
445  resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
446  return resultStruct
447 
448  def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
449  """Match sources in calibCat and sourceCat and copy the specified fields
450 
451  Parameters
452  ----------
453  calibCat : `lsst.afw.table.SourceCatalog`
454  Catalog from which to copy fields.
455  sourceCat : `lsst.afw.table.SourceCatalog`
456  Catalog to which to copy fields.
457  fieldsToCopy : `lsst.pex.config.listField.List`
458  Fields to copy from calibCat to SoourceCat.
459 
460  Returns
461  -------
462  newCat : `lsst.afw.table.SourceCatalog`
463  Catalog which includes the copied fields.
464 
465  The fields copied are those specified by `fieldsToCopy` that actually exist
466  in the schema of `calibCat`.
467 
468  This version was based on and adapted from the one in calibrateTask.
469  """
470 
471  # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
472  sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
473  sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
474 
475  calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
476 
477  # Add the desired columns from the option fieldsToCopy
478  missingFieldNames = []
479  for fieldName in fieldsToCopy:
480  if fieldName in calibCat.schema:
481  schemaItem = calibCat.schema.find(fieldName)
482  calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
483  schema = calibSchemaMapper.editOutputSchema()
484  calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
485  else:
486  missingFieldNames.append(fieldName)
487  if missingFieldNames:
488  raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
489  "fieldsToCopy")
490 
491  if "calib_detected" not in calibSchemaMapper.getOutputSchema():
492  self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
493  "Source was detected as an icSource"))
494  else:
495  self.calibSourceKey = None
496 
497  schema = calibSchemaMapper.getOutputSchema()
498  newCat = afwTable.SourceCatalog(schema)
499  newCat.reserve(len(sourceCat))
500  newCat.extend(sourceCat, sourceSchemaMapper)
501 
502  # Set the aliases so it doesn't complain.
503  for k, v in sourceCat.schema.getAliasMap().items():
504  newCat.schema.getAliasMap().set(k, v)
505 
506  select = newCat["deblend_nChild"] == 0
507  matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
508  # Check that no sourceCat sources are listed twice (we already know
509  # that each match has a unique calibCat source ID, due to using
510  # that ID as the key in bestMatches)
511  numMatches = len(matches)
512  numUniqueSources = len(set(m[1].getId() for m in matches))
513  if numUniqueSources != numMatches:
514  self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches,
515  numUniqueSources)
516 
517  self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
518 
519  # For each match: set the calibSourceKey flag and copy the desired
520  # fields
521  for src, calibSrc, d in matches:
522  if self.calibSourceKey:
523  src.setFlag(self.calibSourceKey, True)
524  # src.assign copies the footprint from calibSrc, which we don't want
525  # (DM-407)
526  # so set calibSrc's footprint to src's footprint before src.assign,
527  # then restore it
528  calibSrcFootprint = calibSrc.getFootprint()
529  try:
530  calibSrc.setFootprint(src.getFootprint())
531  src.assign(calibSrc, calibSchemaMapper)
532  finally:
533  calibSrc.setFootprint(calibSrcFootprint)
534 
535  return newCat
536 
537 
538 class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections):
539  ccdVisitFakeMagnitudes = cT.Output(
540  doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
541  name="{fakesType}ccdVisitFakeMagnitudes",
542  storageClass="DataFrame",
543  dimensions=("instrument", "visit", "detector"),
544  )
545 
546 
547 class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
548  pipelineConnections=ProcessCcdWithVariableFakesConnections):
549  scatterSize = pexConfig.RangeField(
550  dtype=float,
551  default=0.4,
552  min=0,
553  max=100,
554  doc="Amount of scatter to add to the visit magnitude for variable "
555  "sources."
556  )
557 
558 
559 class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
560  """As ProcessCcdWithFakes except add variablity to the fakes catalog
561  magnitude in the observed band for this ccdVisit.
562 
563  Additionally, write out the modified magnitudes to the Butler.
564  """
565 
566  _DefaultName = "processCcdWithVariableFakes"
567  ConfigClass = ProcessCcdWithVariableFakesConfig
568 
569  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
570  sfdSourceCat=None):
571  """Add fake sources to a calexp and then run detection, deblending and measurement.
572 
573  Parameters
574  ----------
575  fakeCat : `pandas.core.frame.DataFrame`
576  The catalog of fake sources to add to the exposure
577  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
578  The exposure to add the fake sources to
579  wcs : `lsst.afw.geom.SkyWcs`
580  WCS to use to add fake sources
581  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
582  Photometric calibration to be used to calibrate the fake sources
583  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
584  icSourceCat : `lsst.afw.table.SourceCatalog`
585  Default : None
586  Catalog to take the information about which sources were used for calibration from.
587  sfdSourceCat : `lsst.afw.table.SourceCatalog`
588  Default : None
589  Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
590 
591  Returns
592  -------
593  resultStruct : `lsst.pipe.base.struct.Struct`
594  Results Strcut containing:
595 
596  - outputExposure : Exposure with added fakes
597  (`lsst.afw.image.exposure.exposure.ExposureF`)
598  - outputCat : Catalog with detected fakes
599  (`lsst.afw.table.source.source.SourceCatalog`)
600  - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
601  inserted with after being scattered (`pandas.DataFrame`)
602 
603  Notes
604  -----
605  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
606  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
607  pixels.
608 
609  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
610  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
611  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
612  the calexp and the calexp with fakes included returned.
613 
614  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
615  this is then convolved with the PSF at that point.
616 
617  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
618  """
619  if wcs is None:
620  wcs = exposure.getWcs()
621 
622  if photoCalib is None:
623  photoCalib = exposure.getPhotoCalib()
624 
625  if exposureIdInfo is None:
626  exposureIdInfo = ExposureIdInfo()
627 
628  band = exposure.getFilterLabel().bandLabel
629  ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo)
630 
631  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
632 
633  # detect, deblend and measure sources
634  returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
635  sourceCat = returnedStruct.sourceCat
636 
637  sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
638 
639  resultStruct = pipeBase.Struct(outputExposure=exposure,
640  outputCat=sourceCat,
641  ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
642  return resultStruct
643 
644  def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo):
645  """Add scatter to the fake catalog visit magnitudes.
646 
647  Currently just adds a simple Gaussian scatter around the static fake
648  magnitude. This function could be modified to return any number of
649  fake variability.
650 
651  Parameters
652  ----------
653  fakeCat : `pandas.DataFrame`
654  Catalog of fakes to modify magnitudes of.
655  band : `str`
656  Current observing band to modify.
657  exposure : `lsst.afw.image.ExposureF`
658  Exposure fakes will be added to.
659  photoCalib : `lsst.afw.image.PhotoCalib`
660  Photometric calibration object of ``exposure``.
661  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
662  Exposure id information and metadata.
663 
664  Returns
665  -------
666  dataFrame : `pandas.DataFrame`
667  DataFrame containing the values of the magnitudes to that will
668  be inserted into this ccdVisit.
669  """
670  expId = exposureIdInfo.expId
671  rng = np.random.default_rng(expId)
672  magScatter = rng.normal(loc=0,
673  scale=self.config.scatterSize,
674  size=len(fakeCat))
675  visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
676  fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
677  return pd.DataFrame(data={"variableMag": visitMagnitudes})
std::vector< SchemaItem< Flag > > * items
A mapping between the keys of two Schemas, used to copy data between them.
Definition: SchemaMapper.h:21
daf::base::PropertySet * set
Definition: fits.cc:912
SourceMatchVector matchXy(SourceCatalog const &cat1, SourceCatalog const &cat2, double radius, MatchControl const &mc=MatchControl())
Compute all tuples (s1,s2,d) where s1 belings to cat1, s2 belongs to cat2 and d, the distance between...
Definition: Match.cc:305
def run(self, coaddExposures, bbox, wcs)
Definition: getTemplate.py:603
A description of a field in a table.
Definition: Field.h:24