LSSTApplications  18.0.0+106,18.0.0+50,19.0.0,19.0.0+1,19.0.0+10,19.0.0+11,19.0.0+13,19.0.0+17,19.0.0+2,19.0.0-1-g20d9b18+6,19.0.0-1-g425ff20,19.0.0-1-g5549ca4,19.0.0-1-g580fafe+6,19.0.0-1-g6fe20d0+1,19.0.0-1-g7011481+9,19.0.0-1-g8c57eb9+6,19.0.0-1-gb5175dc+11,19.0.0-1-gdc0e4a7+9,19.0.0-1-ge272bc4+6,19.0.0-1-ge3aa853,19.0.0-10-g448f008b,19.0.0-12-g6990b2c,19.0.0-2-g0d9f9cd+11,19.0.0-2-g3d9e4fb2+11,19.0.0-2-g5037de4,19.0.0-2-gb96a1c4+3,19.0.0-2-gd955cfd+15,19.0.0-3-g2d13df8,19.0.0-3-g6f3c7dc,19.0.0-4-g725f80e+11,19.0.0-4-ga671dab3b+1,19.0.0-4-gad373c5+3,19.0.0-5-ga2acb9c+2,19.0.0-5-gfe96e6c+2,w.2020.01
LSSTDataManagementBasePackage
processCcdWithFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fake sources into calexps
24 """
25 from astropy.table import Table
26 
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 
31 from .insertFakes import InsertFakesTask
32 from lsst.meas.algorithms import SourceDetectionTask
33 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask,
34  PerTractCcdDataIdContainer)
35 from lsst.meas.deblender import SourceDeblendTask
36 from lsst.afw.table import SourceTable, IdFactory
37 from lsst.obs.base import ExposureIdInfo
38 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
40 
41 
42 __all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask"]
43 
44 
45 class ProcessCcdWithFakesConnections(PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
46  defaultTemplates={"CoaddName": "deep"}):
47 
48  exposure = cT.Input(
49  doc="Exposure into which fakes are to be added.",
50  name="calexp",
51  storageClass="ExposureF",
52  dimensions=("instrument", "visit", "detector")
53  )
54 
55  fakeCat = cT.Input(
56  doc="Catalog of fake sources to draw inputs from.",
57  name="{CoaddName}Coadd_fakeSourceCat",
58  storageClass="Parquet",
59  dimensions=("tract", "skymap")
60  )
61 
62  wcs = cT.Input(
63  doc="WCS information for the input exposure.",
64  name="jointcal_wcs",
65  storageClass="Wcs",
66  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
67  )
68 
69  photoCalib = cT.Input(
70  doc="Calib information for the input exposure.",
71  name="jointcal_photoCalib",
72  storageClass="PhotoCalib",
73  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
74  )
75 
76  outputExposure = cT.Output(
77  doc="Exposure with fake sources added.",
78  name="fakes_calexp",
79  storageClass="ExposureF",
80  dimensions=("instrument", "visit", "detector")
81  )
82 
83  outputCat = cT.Output(
84  doc="Source catalog produced in calibrate task with fakes also measured.",
85  name="fakes_src",
86  storageClass="SourceCatalog",
87  dimensions=("instrument", "visit", "detector"),
88  )
89 
90 
91 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
92  pipelineConnections=ProcessCcdWithFakesConnections):
93  """Config for inserting fake sources
94 
95  Notes
96  -----
97  The default column names are those from the UW sims database.
98  """
99 
100  useUpdatedCalibs = pexConfig.Field(
101  doc="Use updated calibs and wcs from jointcal?",
102  dtype=bool,
103  default=False,
104  )
105 
106  coaddName = pexConfig.Field(
107  doc="The name of the type of coadd used",
108  dtype=str,
109  default="deep",
110  )
111 
112  insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
113  doc="Configuration for the fake sources")
114 
115  detection = pexConfig.ConfigurableField(target=SourceDetectionTask,
116  doc="The detection task to use.")
117 
118  deblend = pexConfig.ConfigurableField(target=SourceDeblendTask, doc="The deblending task to use.")
119 
120  measurement = pexConfig.ConfigurableField(target=SingleFrameMeasurementTask,
121  doc="The measurement task to use")
122 
123  applyApCorr = pexConfig.ConfigurableField(target=ApplyApCorrTask,
124  doc="The apply aperture correction task to use.")
125 
126  catalogCalculation = pexConfig.ConfigurableField(target=CatalogCalculationTask,
127  doc="The catalog calculation ask to use.")
128 
129  def setDefaults(self):
130  self.detection.reEstimateBackground = False
131  super().setDefaults()
132  self.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
133  self.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
134 
135 
136 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
137  """Insert fake objects into calexps.
138 
139  Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
140  from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
141  give a new background estimation and measurement of the calexp.
142 
143  `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
144  sources and then add them to the calexp.
145 
146  `addPixCoords`
147  Use the WCS information to add the pixel coordinates of each source
148  Adds an ``x`` and ``y`` column to the catalog of fake sources.
149  `trimFakeCat`
150  Trim the fake cat to about the size of the input image.
151  `mkFakeGalsimGalaxies`
152  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
153  `mkFakeStars`
154  Use the PSF information from the calexp to make a fake star using the magnitude information from the
155  input file.
156  `cleanCat`
157  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
158  that are 0.
159  `addFakeSources`
160  Add the fake sources to the calexp.
161 
162  Notes
163  -----
164  The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
165  """
166 
167  _DefaultName = "processCcdWithFakes"
168  ConfigClass = ProcessCcdWithFakesConfig
169 
170  def __init__(self, schema=None, **kwargs):
171  """Initalize tings! This should go above in the class docstring
172  """
173 
174  super().__init__(**kwargs)
175 
176  if schema is None:
177  schema = SourceTable.makeMinimalSchema()
178  self.schema = schema
179  self.makeSubtask("insertFakes")
181  self.makeSubtask("detection", schema=self.schema)
182  self.makeSubtask("deblend", schema=self.schema)
183  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
184  self.makeSubtask("applyApCorr", schema=self.schema)
185  self.makeSubtask("catalogCalculation", schema=self.schema)
186 
187  def runDataRef(self, dataRef):
188  """Read in/write out the required data products and add fake sources to the calexp.
189 
190  Parameters
191  ----------
192  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
193  Data reference defining the ccd to have fakes added to it.
194  Used to access the following data products:
195  calexp
196  jointcal_wcs
197  jointcal_photoCalib
198 
199  Notes
200  -----
201  Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
202  of the sources unless the config option config.useUpdatedCalibs is set then it uses the
203  meas_mosaic/jointCal outputs. The config defualts for the column names in the catalog of fakes are
204  taken from the University of Washington simulations database. Operates on one ccd at a time.
205  """
206  exposureIdInfo = dataRef.get("expIdInfo")
207 
208  if self.config.insertFakes.fakeType == "snapshot":
209  fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
210  elif self.config.insertFakes.fakeType == "static":
211  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
212  else:
213  fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
214 
215  calexp = dataRef.get("calexp")
216  if self.config.useUpdatedCalibs:
217  self.log.info("Using updated calibs from meas_mosaic/jointCal")
218  wcs = dataRef.get("jointcal_wcs")
219  photoCalib = dataRef.get("jointcal_photoCalib")
220  else:
221  wcs = calexp.getWcs()
222  photoCalib = calexp.getPhotoCalib()
223 
224  resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
225  exposureIdInfo=exposureIdInfo)
226 
227  dataRef.put(resultStruct.outputExposure, "fakes_calexp")
228  dataRef.put(resultStruct.outputCat, "fakes_src")
229 
230  def runQuantum(self, butlerQC, inputRefs, outputRefs):
231  inputs = butlerQC.get(inputRefs)
232  if 'exposureIdInfo' not in inputs.keys():
233  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
234  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
235 
236  if inputs["wcs"] is None:
237  inputs["wcs"] = inputs["image"].getWcs()
238  if inputs["photoCalib"] is None:
239  inputs["photoCalib"] = inputs["image"].getPhotoCalib()
240 
241  outputs = self.run(**inputs)
242  butlerQC.put(outputs, outputRefs)
243 
244  @classmethod
245  def _makeArgumentParser(cls):
246  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
247  parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
248  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
249  ContainerClass=PerTractCcdDataIdContainer)
250  return parser
251 
252  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None):
253  """Add fake sources to a calexp and then run detection, deblending and measurement.
254 
255  Parameters
256  ----------
257  fakeCat : `pandas.core.frame.DataFrame`
258  The catalog of fake sources to add to the exposure
259  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
260  The exposure to add the fake sources to
261  wcs : `lsst.afw.geom.SkyWcs`
262  WCS to use to add fake sources
263  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
264  Photometric calibration to be used to calibrate the fake sources
265  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
266 
267  Returns
268  -------
269  resultStruct : `lsst.pipe.base.struct.Struct`
270  contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
271  outputCat : `lsst.afw.table.source.source.SourceCatalog`
272 
273  Notes
274  -----
275  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
276  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
277  pixels.
278 
279  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
280  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
281  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
282  the calexp and the calexp with fakes included returned.
283 
284  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
285  this is then convolved with the PSF at that point.
286 
287  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
288  """
289 
290  if wcs is None:
291  wcs = exposure.getWcs()
292 
293  if photoCalib is None:
294  photoCalib = exposure.getPhotoCalib()
295 
296  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
297 
298  # detect, deblend and measure sources
299  if exposureIdInfo is None:
300  exposureIdInfo = ExposureIdInfo()
301 
302  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
303  table = SourceTable.make(self.schema, sourceIdFactory)
304  table.setMetadata(self.algMetadata)
305 
306  detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
307  sourceCat = detRes.sources
308  self.deblend.run(exposure=exposure, sources=sourceCat)
309  self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
310  self.applyApCorr.run(catalog=sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
311  self.catalogCalculation.run(sourceCat)
312 
313  resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
314  return resultStruct
Class for storing ordered metadata with comments.
Definition: PropertyList.h:68
def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations...
afw::table::PointKey< int > dimensions
Definition: GaussianPsf.cc:49
def runQuantum(self, butlerQC, inputRefs, outputRefs)