LSSTApplications  19.0.0-14-gb0260a2+72efe9b372,20.0.0+7927753e06,20.0.0+8829bf0056,20.0.0+995114c5d2,20.0.0+b6f4b2abd1,20.0.0+bddc4f4cbe,20.0.0-1-g253301a+8829bf0056,20.0.0-1-g2b7511a+0d71a2d77f,20.0.0-1-g5b95a8c+7461dd0434,20.0.0-12-g321c96ea+23efe4bbff,20.0.0-16-gfab17e72e+fdf35455f6,20.0.0-2-g0070d88+ba3ffc8f0b,20.0.0-2-g4dae9ad+ee58a624b3,20.0.0-2-g61b8584+5d3db074ba,20.0.0-2-gb780d76+d529cf1a41,20.0.0-2-ged6426c+226a441f5f,20.0.0-2-gf072044+8829bf0056,20.0.0-2-gf1f7952+ee58a624b3,20.0.0-20-geae50cf+e37fec0aee,20.0.0-25-g3dcad98+544a109665,20.0.0-25-g5eafb0f+ee58a624b3,20.0.0-27-g64178ef+f1f297b00a,20.0.0-3-g4cc78c6+e0676b0dc8,20.0.0-3-g8f21e14+4fd2c12c9a,20.0.0-3-gbd60e8c+187b78b4b8,20.0.0-3-gbecbe05+48431fa087,20.0.0-38-ge4adf513+a12e1f8e37,20.0.0-4-g97dc21a+544a109665,20.0.0-4-gb4befbc+087873070b,20.0.0-4-gf910f65+5d3db074ba,20.0.0-5-gdfe0fee+199202a608,20.0.0-5-gfbfe500+d529cf1a41,20.0.0-6-g64f541c+d529cf1a41,20.0.0-6-g9a5b7a1+a1cd37312e,20.0.0-68-ga3f3dda+5fca18c6a4,20.0.0-9-g4aef684+e18322736b,w.2020.45
LSSTDataManagementBasePackage
processCcdWithFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fake sources into calexps
24 """
25 from astropy.table import Table
26 
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 
30 from .insertFakes import InsertFakesTask
31 from lsst.meas.base import PerTractCcdDataIdContainer
32 from lsst.afw.table import SourceTable
33 from lsst.obs.base import ExposureIdInfo
34 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
36 import lsst.afw.table as afwTable
37 from lsst.pipe.tasks.calibrate import CalibrateTask
38 
39 __all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask"]
40 
41 
43  dimensions=("skymap", "tract", "instrument", "visit", "detector"),
44  defaultTemplates={"CoaddName": "deep", "wcsName": "jointcal",
45  "photoCalibName": "jointcal"}):
46 
47  exposure = cT.Input(
48  doc="Exposure into which fakes are to be added.",
49  name="calexp",
50  storageClass="ExposureF",
51  dimensions=("instrument", "visit", "detector")
52  )
53 
54  fakeCat = cT.Input(
55  doc="Catalog of fake sources to draw inputs from.",
56  name="{CoaddName}Coadd_fakeSourceCat",
57  storageClass="DataFrame",
58  dimensions=("tract", "skymap")
59  )
60 
61  wcs = cT.Input(
62  doc="WCS information for the input exposure.",
63  name="{wcsName}_wcs",
64  storageClass="Wcs",
65  dimensions=("tract", "skymap", "instrument", "visit", "detector")
66  )
67 
68  photoCalib = cT.Input(
69  doc="Calib information for the input exposure.",
70  name="{photoCalibName}_photoCalib",
71  storageClass="PhotoCalib",
72  dimensions=("tract", "skymap", "instrument", "visit", "detector")
73  )
74 
75  icSourceCat = cT.Input(
76  doc="Catalog of calibration sources",
77  name="icSrc",
78  storageClass="SourceCatalog",
79  dimensions=("instrument", "visit", "detector")
80  )
81 
82  sfdSourceCat = cT.Input(
83  doc="Catalog of calibration sources",
84  name="src",
85  storageClass="SourceCatalog",
86  dimensions=("instrument", "visit", "detector")
87  )
88 
89  outputExposure = cT.Output(
90  doc="Exposure with fake sources added.",
91  name="fakes_calexp",
92  storageClass="ExposureF",
93  dimensions=("instrument", "visit", "detector")
94  )
95 
96  outputCat = cT.Output(
97  doc="Source catalog produced in calibrate task with fakes also measured.",
98  name="fakes_src",
99  storageClass="SourceCatalog",
100  dimensions=("instrument", "visit", "detector"),
101  )
102 
103  def __init__(self, *, config=None):
104  super().__init__(config=config)
105 
106  if config.doApplyExternalSkyWcs is False:
107  self.inputs.remove("wcs")
108  if config.doApplyExternalPhotoCalib is False:
109  self.inputs.remove("photoCalib")
110 
111 
112 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
113  pipelineConnections=ProcessCcdWithFakesConnections):
114  """Config for inserting fake sources
115 
116  Notes
117  -----
118  The default column names are those from the UW sims database.
119  """
120 
121  doApplyExternalPhotoCalib = pexConfig.Field(
122  dtype=bool,
123  default=False,
124  doc="Whether to apply an external photometric calibration via an "
125  "`lsst.afw.image.PhotoCalib` object. Uses the "
126  "`externalPhotoCalibName` config option to determine which "
127  "calibration to use."
128  )
129 
130  externalPhotoCalibName = pexConfig.ChoiceField(
131  doc="What type of external photo calib to use.",
132  dtype=str,
133  default="jointcal",
134  allowed={"jointcal": "Use jointcal_photoCalib",
135  "fgcm": "Use fgcm_photoCalib",
136  "fgcm_tract": "Use fgcm_tract_photoCalib"}
137  )
138 
139  doApplyExternalSkyWcs = pexConfig.Field(
140  dtype=bool,
141  default=False,
142  doc="Whether to apply an external astrometric calibration via an "
143  "`lsst.afw.geom.SkyWcs` object. Uses the "
144  "`externalSkyWcsName` config option to determine which "
145  "calibration to use."
146  )
147 
148  externalSkyWcsName = pexConfig.ChoiceField(
149  doc="What type of updated WCS calib to use.",
150  dtype=str,
151  default="jointcal",
152  allowed={"jointcal": "Use jointcal_wcs"}
153  )
154 
155  coaddName = pexConfig.Field(
156  doc="The name of the type of coadd used",
157  dtype=str,
158  default="deep",
159  )
160 
161  srcFieldsToCopy = pexConfig.ListField(
162  dtype=str,
163  default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
164  "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
165  doc=("Fields to copy from the `src` catalog to the output catalog "
166  "for matching sources Any missing fields will trigger a "
167  "RuntimeError exception.")
168  )
169 
170  matchRadiusPix = pexConfig.Field(
171  dtype=float,
172  default=3,
173  doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
174  )
175 
176  calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
177  doc="The calibration task to use.")
178 
179  insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
180  doc="Configuration for the fake sources")
181 
182  def setDefaults(self):
183  super().setDefaults()
184  self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
185  self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
186  self.calibrate.doAstrometry = False
187  self.calibrate.doWriteMatches = False
188  self.calibrate.doPhotoCal = False
189  self.calibrate.detection.reEstimateBackground = False
190 
191 
192 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
193  """Insert fake objects into calexps.
194 
195  Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
196  from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
197  give a new background estimation and measurement of the calexp.
198 
199  `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
200  sources and then add them to the calexp.
201 
202  `addPixCoords`
203  Use the WCS information to add the pixel coordinates of each source
204  Adds an ``x`` and ``y`` column to the catalog of fake sources.
205  `trimFakeCat`
206  Trim the fake cat to about the size of the input image.
207  `mkFakeGalsimGalaxies`
208  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
209  `mkFakeStars`
210  Use the PSF information from the calexp to make a fake star using the magnitude information from the
211  input file.
212  `cleanCat`
213  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
214  that are 0.
215  `addFakeSources`
216  Add the fake sources to the calexp.
217 
218  Notes
219  -----
220  The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
221  """
222 
223  _DefaultName = "processCcdWithFakes"
224  ConfigClass = ProcessCcdWithFakesConfig
225 
226  def __init__(self, schema=None, butler=None, **kwargs):
227  """Initalize things! This should go above in the class docstring
228  """
229 
230  super().__init__(**kwargs)
231 
232  if schema is None:
233  schema = SourceTable.makeMinimalSchema()
234  self.schema = schema
235  self.makeSubtask("insertFakes")
236  self.makeSubtask("calibrate")
237 
238  def runDataRef(self, dataRef):
239  """Read in/write out the required data products and add fake sources to the calexp.
240 
241  Parameters
242  ----------
243  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
244  Data reference defining the ccd to have fakes added to it.
245  Used to access the following data products:
246  calexp
247  jointcal_wcs
248  jointcal_photoCalib
249 
250  Notes
251  -----
252  Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
253  of the sources unless the config option config.externalPhotoCalibName or config.externalSkyWcsName
254  are set then it uses the specified outputs. The config defualts for the column names in the catalog
255  of fakes are taken from the University of Washington simulations database.
256  Operates on one ccd at a time.
257  """
258  exposureIdInfo = dataRef.get("expIdInfo")
259 
260  if self.config.insertFakes.fakeType == "snapshot":
261  fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
262  elif self.config.insertFakes.fakeType == "static":
263  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
264  else:
265  fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
266 
267  calexp = dataRef.get("calexp")
268  if self.config.doApplyExternalSkyWcs:
269  self.log.info("Using external wcs from " + self.config.externalSkyWcsName)
270  wcs = dataRef.get(self.config.externalSkyWcsName + "_wcs")
271  else:
272  wcs = calexp.getWcs()
273 
274  if self.config.doApplyExternalPhotoCalib:
275  self.log.info("Using external photocalib from " + self.config.externalPhotoCalibName)
276  photoCalib = dataRef.get(self.config.externalPhotoCalibName + "_photoCalib")
277  else:
278  photoCalib = calexp.getPhotoCalib()
279 
280  icSourceCat = dataRef.get("icSrc", immediate=True)
281  sfdSourceCat = dataRef.get("src", immediate=True)
282 
283  resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
284  exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
285  sfdSourceCat=sfdSourceCat)
286 
287  dataRef.put(resultStruct.outputExposure, "fakes_calexp")
288  dataRef.put(resultStruct.outputCat, "fakes_src")
289  return resultStruct
290 
291  def runQuantum(self, butlerQC, inputRefs, outputRefs):
292  inputs = butlerQC.get(inputRefs)
293  if 'exposureIdInfo' not in inputs.keys():
294  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
295  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
296 
297  if not self.config.doApplyExternalSkyWcs:
298  inputs["wcs"] = inputs["image"].getWcs()
299 
300  if not self.config.doApplyExternalPhotoCalib:
301  inputs["photoCalib"] = inputs["image"].getPhotoCalib()
302 
303  outputs = self.run(**inputs)
304  butlerQC.put(outputs, outputRefs)
305 
306  @classmethod
307  def _makeArgumentParser(cls):
308  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
309  parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
310  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
311  ContainerClass=PerTractCcdDataIdContainer)
312  return parser
313 
314  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
315  sfdSourceCat=None):
316  """Add fake sources to a calexp and then run detection, deblending and measurement.
317 
318  Parameters
319  ----------
320  fakeCat : `pandas.core.frame.DataFrame`
321  The catalog of fake sources to add to the exposure
322  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
323  The exposure to add the fake sources to
324  wcs : `lsst.afw.geom.SkyWcs`
325  WCS to use to add fake sources
326  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
327  Photometric calibration to be used to calibrate the fake sources
328  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
329  icSourceCat : `lsst.afw.table.SourceCatalog`
330  Default : None
331  Catalog to take the information about which sources were used for calibration from.
332  sfdSourceCat : `lsst.afw.table.SourceCatalog`
333  Default : None
334  Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
335 
336  Returns
337  -------
338  resultStruct : `lsst.pipe.base.struct.Struct`
339  contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
340  outputCat : `lsst.afw.table.source.source.SourceCatalog`
341 
342  Notes
343  -----
344  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
345  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
346  pixels.
347 
348  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
349  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
350  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
351  the calexp and the calexp with fakes included returned.
352 
353  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
354  this is then convolved with the PSF at that point.
355 
356  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
357  """
358 
359  if wcs is None:
360  wcs = exposure.getWcs()
361 
362  if photoCalib is None:
363  photoCalib = exposure.getPhotoCalib()
364 
365  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
366 
367  # detect, deblend and measure sources
368  if exposureIdInfo is None:
369  exposureIdInfo = ExposureIdInfo()
370  returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
371  sourceCat = returnedStruct.sourceCat
372 
373  sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
374 
375  resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
376  return resultStruct
377 
378  def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
379  """Match sources in calibCat and sourceCat and copy the specified fields
380 
381  Parameters
382  ----------
383  calibCat : `lsst.afw.table.SourceCatalog`
384  Catalog from which to copy fields.
385  sourceCat : `lsst.afw.table.SourceCatalog`
386  Catalog to which to copy fields.
387  fieldsToCopy : `lsst.pex.config.listField.List`
388  Fields to copy from calibCat to SoourceCat.
389 
390  Returns
391  -------
392  newCat : `lsst.afw.table.SourceCatalog`
393  Catalog which includes the copied fields.
394 
395  The fields copied are those specified by `fieldsToCopy` that actually exist
396  in the schema of `calibCat`.
397 
398  This version was based on and adapted from the one in calibrateTask.
399  """
400 
401  # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
402  sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
403  sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
404 
405  calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
406 
407  # Add the desired columns from the option fieldsToCopy
408  missingFieldNames = []
409  for fieldName in fieldsToCopy:
410  if fieldName in calibCat.schema:
411  schemaItem = calibCat.schema.find(fieldName)
412  calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
413  schema = calibSchemaMapper.editOutputSchema()
414  calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
415  else:
416  missingFieldNames.append(fieldName)
417  if missingFieldNames:
418  raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
419  "fieldsToCopy")
420 
421  if "calib_detected" not in calibSchemaMapper.getOutputSchema():
422  self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
423  "Source was detected as an icSource"))
424  else:
425  self.calibSourceKey = None
426 
427  schema = calibSchemaMapper.getOutputSchema()
428  newCat = afwTable.SourceCatalog(schema)
429  newCat.reserve(len(sourceCat))
430  newCat.extend(sourceCat, sourceSchemaMapper)
431 
432  # Set the aliases so it doesn't complain.
433  for k, v in sourceCat.schema.getAliasMap().items():
434  newCat.schema.getAliasMap().set(k, v)
435 
436  select = newCat["deblend_nChild"] == 0
437  matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
438  # Check that no sourceCat sources are listed twice (we already know
439  # that each match has a unique calibCat source ID, due to using
440  # that ID as the key in bestMatches)
441  numMatches = len(matches)
442  numUniqueSources = len(set(m[1].getId() for m in matches))
443  if numUniqueSources != numMatches:
444  self.log.warn("%d calibCat sources matched only %d sourceCat sources", numMatches,
445  numUniqueSources)
446 
447  self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
448 
449  # For each match: set the calibSourceKey flag and copy the desired
450  # fields
451  for src, calibSrc, d in matches:
452  if self.calibSourceKey:
453  src.setFlag(self.calibSourceKey, True)
454  # src.assign copies the footprint from calibSrc, which we don't want
455  # (DM-407)
456  # so set calibSrc's footprint to src's footprint before src.assign,
457  # then restore it
458  calibSrcFootprint = calibSrc.getFootprint()
459  try:
460  calibSrc.setFootprint(src.getFootprint())
461  src.assign(calibSrc, calibSchemaMapper)
462  finally:
463  calibSrc.setFootprint(calibSrcFootprint)
464 
465  return newCat
lsst::afw::table::matchXy
SourceMatchVector matchXy(SourceCatalog const &cat, double radius, bool symmetric)
Definition: Match.cc:383
lsst::log.log.logContinued.warn
def warn(fmt, *args)
Definition: logContinued.py:205
lsst::log.log.logContinued.info
def info(fmt, *args)
Definition: logContinued.py:201
lsst.pex.config.wrap.setDefaults
setDefaults
Definition: wrap.py:293
lsst.pipe.base.connections.PipelineTaskConnections.inputs
inputs
Definition: connections.py:365
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:720
lsst.pipe.base.pipelineTask.PipelineTask
Definition: pipelineTask.py:32
lsst.pex.config
Definition: __init__.py:1
lsst::meas::base
Definition: Algorithm.h:37
lsst::afw::table._source.SourceCatalog
Definition: _source.py:32
lsst::afw::table::SchemaMapper
A mapping between the keys of two Schemas, used to copy data between them.
Definition: SchemaMapper.h:21
lsst.pipe.base.connections.PipelineTaskConnections
Definition: connections.py:260
lsst::afw::table
Definition: table.dox:3
lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithFakesConnections
Definition: processCcdWithFakes.py:44
lsst.pipe.base.config.PipelineTaskConfig
Definition: config.py:115
lsst.obs.base.exposureIdInfo.ExposureIdInfo
Definition: exposureIdInfo.py:25
items
std::vector< SchemaItem< Flag > > * items
Definition: BaseColumnView.cc:142
lsst::afw::table::Field
A description of a field in a table.
Definition: Field.h:24
lsst.pipe.tasks.calibrate
Definition: calibrate.py:1
lsst.pipe.base
Definition: __init__.py:1
set
daf::base::PropertySet * set
Definition: fits.cc:912
lsst.pipe.base.connectionTypes
Definition: connectionTypes.py:1
lsst.pipe.base.connections.PipelineTaskConnections.__init__
def __init__(self, *'PipelineTaskConfig' config=None)
Definition: connections.py:364
lsst.obs.base
Definition: __init__.py:1
lsst.pipe.base.cmdLineTask.CmdLineTask
Definition: cmdLineTask.py:544