LSST Applications 27.0.0,g0265f82a02+469cd937ee,g02d81e74bb+21ad69e7e1,g1470d8bcf6+cbe83ee85a,g2079a07aa2+e67c6346a6,g212a7c68fe+04a9158687,g2305ad1205+94392ce272,g295015adf3+81dd352a9d,g2bbee38e9b+469cd937ee,g337abbeb29+469cd937ee,g3939d97d7f+72a9f7b576,g487adcacf7+71499e7cba,g50ff169b8f+5929b3527e,g52b1c1532d+a6fc98d2e7,g591dd9f2cf+df404f777f,g5a732f18d5+be83d3ecdb,g64a986408d+21ad69e7e1,g858d7b2824+21ad69e7e1,g8a8a8dda67+a6fc98d2e7,g99cad8db69+f62e5b0af5,g9ddcbc5298+d4bad12328,ga1e77700b3+9c366c4306,ga8c6da7877+71e4819109,gb0e22166c9+25ba2f69a1,gb6a65358fc+469cd937ee,gbb8dafda3b+69d3c0e320,gc07e1c2157+a98bf949bb,gc120e1dc64+615ec43309,gc28159a63d+469cd937ee,gcf0d15dbbd+72a9f7b576,gdaeeff99f8+a38ce5ea23,ge6526c86ff+3a7c1ac5f1,ge79ae78c31+469cd937ee,gee10cc3b42+a6fc98d2e7,gf1cff7945b+21ad69e7e1,gfbcc870c63+9a11dc8c8f
LSST Data Management Base Package
Loading...
Searching...
No Matches
processCcdWithFakes.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22"""
23Insert fake sources into calexps
24"""
25
26__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask",
27 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"]
28
29import numpy as np
30import pandas as pd
31
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34
35from .insertFakes import InsertFakesTask
36from lsst.afw.table import SourceTable
37from lsst.meas.base import IdGenerator, DetectorVisitIdGeneratorConfig
38from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39import lsst.pipe.base.connectionTypes as cT
40import lsst.afw.table as afwTable
41from lsst.skymap import BaseSkyMap
42from lsst.pipe.tasks.calibrate import CalibrateTask
43
44
45class ProcessCcdWithFakesConnections(PipelineTaskConnections,
46 dimensions=("instrument", "visit", "detector"),
47 defaultTemplates={"coaddName": "deep",
48 "wcsName": "gbdesAstrometricFit",
49 "photoCalibName": "jointcal",
50 "fakesType": "fakes_"}):
51 skyMap = cT.Input(
52 doc="Input definition of geometry/bbox and projection/wcs for "
53 "template exposures. Needed to test which tract to generate ",
54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
55 dimensions=("skymap",),
56 storageClass="SkyMap",
57 )
58
59 exposure = cT.Input(
60 doc="Exposure into which fakes are to be added.",
61 name="calexp",
62 storageClass="ExposureF",
63 dimensions=("instrument", "visit", "detector")
64 )
65
66 fakeCats = cT.Input(
67 doc="Set of catalogs of fake sources to draw inputs from. We "
68 "concatenate the tract catalogs for detectorVisits that cover "
69 "multiple tracts.",
70 name="{fakesType}fakeSourceCat",
71 storageClass="DataFrame",
72 dimensions=("tract", "skymap"),
73 deferLoad=True,
74 multiple=True,
75 )
76
77 externalSkyWcsTractCatalog = cT.Input(
78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
79 "id for the catalog id, sorted on id for fast lookup."),
80 name="{wcsName}SkyWcsCatalog",
81 storageClass="ExposureCatalog",
82 dimensions=("instrument", "visit", "tract", "skymap"),
83 deferLoad=True,
84 multiple=True,
85 )
86
87 externalSkyWcsGlobalCatalog = cT.Input(
88 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
89 "These catalogs use the detector id for the catalog id, sorted on id for "
90 "fast lookup."),
91 name="finalVisitSummary",
92 storageClass="ExposureCatalog",
93 dimensions=("instrument", "visit"),
94 )
95
96 externalPhotoCalibTractCatalog = cT.Input(
97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
98 "detector id for the catalog id, sorted on id for fast lookup."),
99 name="{photoCalibName}PhotoCalibCatalog",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit", "tract"),
102 deferLoad=True,
103 multiple=True,
104 )
105
106 externalPhotoCalibGlobalCatalog = cT.Input(
107 doc=("Per-visit photometric calibrations. These catalogs use the "
108 "detector id for the catalog id, sorted on id for fast lookup."),
109 name="finalVisitSummary",
110 storageClass="ExposureCatalog",
111 dimensions=("instrument", "visit"),
112 )
113
114 icSourceCat = cT.Input(
115 doc="Catalog of calibration sources",
116 name="icSrc",
117 storageClass="SourceCatalog",
118 dimensions=("instrument", "visit", "detector")
119 )
120
121 sfdSourceCat = cT.Input(
122 doc="Catalog of calibration sources",
123 name="src",
124 storageClass="SourceCatalog",
125 dimensions=("instrument", "visit", "detector")
126 )
127
128 outputExposure = cT.Output(
129 doc="Exposure with fake sources added.",
130 name="{fakesType}calexp",
131 storageClass="ExposureF",
132 dimensions=("instrument", "visit", "detector")
133 )
134
135 outputCat = cT.Output(
136 doc="Source catalog produced in calibrate task with fakes also measured.",
137 name="{fakesType}src",
138 storageClass="SourceCatalog",
139 dimensions=("instrument", "visit", "detector"),
140 )
141
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
144
145 if not config.doApplyExternalGlobalPhotoCalib:
146 self.inputs.remove("externalPhotoCalibGlobalCatalog")
147 if not config.doApplyExternalTractPhotoCalib:
148 self.inputs.remove("externalPhotoCalibTractCatalog")
149
150 if not config.doApplyExternalGlobalSkyWcs:
151 self.inputs.remove("externalSkyWcsGlobalCatalog")
152 if not config.doApplyExternalTractSkyWcs:
153 self.inputs.remove("externalSkyWcsTractCatalog")
154
155
156class ProcessCcdWithFakesConfig(PipelineTaskConfig,
157 pipelineConnections=ProcessCcdWithFakesConnections):
158 """Config for inserting fake sources
159
160 Notes
161 -----
162 The default column names are those from the UW sims database.
163 """
164
165 doApplyExternalGlobalPhotoCalib = pexConfig.Field(
166 dtype=bool,
167 default=False,
168 doc="Whether to apply an external photometric calibration via an "
169 "`lsst.afw.image.PhotoCalib` object. Uses the "
170 "`externalPhotoCalibName` config option to determine which "
171 "calibration to use. Uses a global calibration."
172 )
173
174 doApplyExternalTractPhotoCalib = pexConfig.Field(
175 dtype=bool,
176 default=False,
177 doc="Whether to apply an external photometric calibration via an "
178 "`lsst.afw.image.PhotoCalib` object. Uses the "
179 "`externalPhotoCalibName` config option to determine which "
180 "calibration to use. Uses a per tract calibration."
181 )
182
183 externalPhotoCalibName = pexConfig.ChoiceField(
184 doc="What type of external photo calib to use.",
185 dtype=str,
186 default="jointcal",
187 allowed={"jointcal": "Use jointcal_photoCalib",
188 "fgcm": "Use fgcm_photoCalib",
189 "fgcm_tract": "Use fgcm_tract_photoCalib"}
190 )
191
192 doApplyExternalGlobalSkyWcs = pexConfig.Field(
193 dtype=bool,
194 default=False,
195 doc="Whether to apply an external astrometric calibration via an "
196 "`lsst.afw.geom.SkyWcs` object. Uses the "
197 "`externalSkyWcsName` config option to determine which "
198 "calibration to use. Uses a global calibration."
199 )
200
201 doApplyExternalTractSkyWcs = pexConfig.Field(
202 dtype=bool,
203 default=False,
204 doc="Whether to apply an external astrometric calibration via an "
205 "`lsst.afw.geom.SkyWcs` object. Uses the "
206 "`externalSkyWcsName` config option to determine which "
207 "calibration to use. Uses a per tract calibration."
208 )
209
210 externalSkyWcsName = pexConfig.ChoiceField(
211 doc="What type of updated WCS calib to use.",
212 dtype=str,
213 default="gbdesAstrometricFit",
214 allowed={"gbdesAstrometricFit": "Use gbdesAstrometricFit_wcs"}
215 )
216
217 coaddName = pexConfig.Field(
218 doc="The name of the type of coadd used",
219 dtype=str,
220 default="deep",
221 )
222
223 srcFieldsToCopy = pexConfig.ListField(
224 dtype=str,
225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
227 doc=("Fields to copy from the `src` catalog to the output catalog "
228 "for matching sources Any missing fields will trigger a "
229 "RuntimeError exception.")
230 )
231
232 matchRadiusPix = pexConfig.Field(
233 dtype=float,
234 default=3,
235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
236 )
237
238 doMatchVisit = pexConfig.Field(
239 dtype=bool,
240 default=False,
241 doc="Match visit to trim the fakeCat"
242 )
243
244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
245 doc="The calibration task to use.")
246
247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
248 doc="Configuration for the fake sources")
249
250 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
251
252 def setDefaults(self):
253 super().setDefaults()
254 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
255 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
256 self.calibrate.doAstrometry = False
257 self.calibrate.doWriteMatches = False
258 self.calibrate.doPhotoCal = False
259 self.calibrate.doComputeSummaryStats = False
260 self.calibrate.detection.reEstimateBackground = False
261
262
263class ProcessCcdWithFakesTask(PipelineTask):
264 """Insert fake objects into calexps.
265
266 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
267 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
268 give a new background estimation and measurement of the calexp.
269
270 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
271 sources and then add them to the calexp.
272
273 `addPixCoords`
274 Use the WCS information to add the pixel coordinates of each source
275 Adds an ``x`` and ``y`` column to the catalog of fake sources.
276 `trimFakeCat`
277 Trim the fake cat to about the size of the input image.
278 `mkFakeGalsimGalaxies`
279 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
280 `mkFakeStars`
281 Use the PSF information from the calexp to make a fake star using the magnitude information from the
282 input file.
283 `cleanCat`
284 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
285 that are 0.
286 `addFakeSources`
287 Add the fake sources to the calexp.
288
289 Notes
290 -----
291 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
292 """
293
294 _DefaultName = "processCcdWithFakes"
295 ConfigClass = ProcessCcdWithFakesConfig
296
297 def __init__(self, schema=None, **kwargs):
298 """Initalize things! This should go above in the class docstring
299 """
300
301 super().__init__(**kwargs)
302
303 if schema is None:
304 schema = SourceTable.makeMinimalSchema()
305 self.schema = schema
306 self.makeSubtask("insertFakes")
307 self.makeSubtask("calibrate")
308
309 def runQuantum(self, butlerQC, inputRefs, outputRefs):
310 inputs = butlerQC.get(inputRefs)
311 detectorId = inputs["exposure"].getInfo().getDetector().getId()
312
313 if 'idGenerator' not in inputs.keys():
314 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
315
316 expWcs = inputs["exposure"].getWcs()
317 tractId = None
318 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
319 if expWcs is None:
320 self.log.info("No WCS for exposure %s so cannot insert fake sources. Skipping detector.",
321 butlerQC.quantum.dataId)
322 return None
323 else:
324 inputs["wcs"] = expWcs
325 elif self.config.doApplyExternalGlobalSkyWcs:
326 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
327 row = externalSkyWcsCatalog.find(detectorId)
328 if row is None:
329 self.log.info("No %s external global sky WCS for exposure %s so cannot insert fake "
330 "sources. Skipping detector.", self.config.externalSkyWcsName,
331 butlerQC.quantum.dataId)
332 return None
333 inputs["wcs"] = row.getWcs()
334 elif self.config.doApplyExternalTractSkyWcs:
335 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
336 if tractId is None:
337 tractId = externalSkyWcsCatalogList[0].dataId["tract"]
338 externalSkyWcsCatalog = None
339 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
340 if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
341 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get()
342 break
343 if externalSkyWcsCatalog is None:
344 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
345 self.log.warn(
346 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
347 "instead.")
348 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get()
349 row = externalSkyWcsCatalog.find(detectorId)
350 if row is None:
351 self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake "
352 "sources. Skipping detector.", self.config.externalSkyWcsName,
353 butlerQC.quantum.dataId)
354 return None
355 inputs["wcs"] = row.getWcs()
356
357 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
358 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
359 elif self.config.doApplyExternalGlobalPhotoCalib:
360 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
361 row = externalPhotoCalibCatalog.find(detectorId)
362 if row is None:
363 self.log.info("No %s external global photoCalib for exposure %s so cannot insert fake "
364 "sources. Skipping detector.", self.config.externalPhotoCalibName,
365 butlerQC.quantum.dataId)
366 return None
367 inputs["photoCalib"] = row.getPhotoCalib()
368 elif self.config.doApplyExternalTractPhotoCalib:
369 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"]
370 if tractId is None:
371 tractId = externalPhotoCalibCatalogList[0].dataId["tract"]
372 externalPhotoCalibCatalog = None
373 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
374 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
375 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get()
376 break
377 if externalPhotoCalibCatalog is None:
378 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
379 self.log.warn(
380 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
381 "instead.")
382 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get()
383 row = externalPhotoCalibCatalog.find(detectorId)
384 if row is None:
385 self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake "
386 "sources. Skipping detector.", self.config.externalPhotoCalibName,
387 butlerQC.quantum.dataId)
388 return None
389 inputs["photoCalib"] = row.getPhotoCalib()
390
391 outputs = self.run(**inputs)
392 butlerQC.put(outputs, outputRefs)
393
394 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None,
395 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None,
396 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
397 externalPhotoCalibTractCatalog=None, idGenerator=None):
398 """Add fake sources to a calexp and then run detection, deblending and
399 measurement.
400
401 Parameters
402 ----------
403 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
404 Set of tract level fake catalogs that potentially cover this
405 detectorVisit.
406 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
407 The exposure to add the fake sources to.
408 skyMap : `lsst.skymap.SkyMap`
409 SkyMap defining the tracts and patches the fakes are stored over.
410 wcs : `lsst.afw.geom.SkyWcs`, optional
411 WCS to use to add fake sources.
412 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`, optional
413 Photometric calibration to be used to calibrate the fake sources.
414 icSourceCat : `lsst.afw.table.SourceCatalog`, optional
415 Catalog to take the information about which sources were used for
416 calibration from.
417 sfdSourceCat : `lsst.afw.table.SourceCatalog`, optional
418 Catalog produced by singleFrameDriver, needed to copy some
419 calibration flags from.
420 externalSkyWcsGlobalCatalog : `lsst.afw.table.ExposureCatalog`, \
421 optional
422 Exposure catalog with external skyWcs to be applied per config.
423 externalSkyWcsTractCatalog : `lsst.afw.table.ExposureCatalog`, optional
424 Exposure catalog with external skyWcs to be applied per config.
425 externalPhotoCalibGlobalCatalog : `lsst.afw.table.ExposureCatalog`, \
426 optional
427 Exposure catalog with external photoCalib to be applied per config
428 externalPhotoCalibTractCatalog : `lsst.afw.table.ExposureCatalog`, \
429 optional
430 Exposure catalog with external photoCalib to be applied per config.
431 idGenerator : `lsst.meas.base.IdGenerator`, optional
432 Object that generates Source IDs and random seeds.
433
434 Returns
435 -------
436 resultStruct : `lsst.pipe.base.struct.Struct`
437 Result struct containing:
438
439 - outputExposure: `lsst.afw.image.exposure.exposure.ExposureF`
440 - outputCat: `lsst.afw.table.source.source.SourceCatalog`
441
442 Notes
443 -----
444 Adds pixel coordinates for each source to the fakeCat and removes
445 objects with bulge or disk half light radius = 0 (if ``config.cleanCat
446 = True``). These columns are called ``x`` and ``y`` and are in pixels.
447
448 Adds the ``Fake`` mask plane to the exposure which is then set by
449 `addFakeSources` to mark where fake sources have been added. Uses the
450 information in the ``fakeCat`` to make fake galaxies (using galsim) and
451 fake stars, using the PSF models from the PSF information for the
452 calexp. These are then added to the calexp and the calexp with fakes
453 included returned.
454
455 The galsim galaxies are made using a double sersic profile, one for the
456 bulge and one for the disk, this is then convolved with the PSF at that
457 point.
458 """
459 fakeCat = self.composeFakeCat(fakeCats, skyMap)
460
461 if wcs is None:
462 wcs = exposure.getWcs()
463
464 if photoCalib is None:
465 photoCalib = exposure.getPhotoCalib()
466
467 if self.config.doMatchVisit:
468 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure)
469
470 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
471
472 # detect, deblend and measure sources
473 if idGenerator is None:
474 idGenerator = IdGenerator()
475 returnedStruct = self.calibrate.run(exposure, idGenerator=idGenerator)
476 sourceCat = returnedStruct.sourceCat
477
478 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
479
480 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
481 return resultStruct
482
483 def composeFakeCat(self, fakeCats, skyMap):
484 """Concatenate the fakeCats from tracts that may cover the exposure.
485
486 Parameters
487 ----------
488 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
489 Set of fake cats to concatenate.
490 skyMap : `lsst.skymap.SkyMap`
491 SkyMap defining the geometry of the tracts and patches.
492
493 Returns
494 -------
495 combinedFakeCat : `pandas.DataFrame`
496 All fakes that cover the inner polygon of the tracts in this
497 quantum.
498 """
499 if len(fakeCats) == 1:
500 return fakeCats[0].get()
501 outputCat = []
502 for fakeCatRef in fakeCats:
503 cat = fakeCatRef.get()
504 tractId = fakeCatRef.dataId["tract"]
505 # Make sure all data is within the inner part of the tract.
506 outputCat.append(cat[
507 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col],
508 cat[self.config.insertFakes.dec_col],
509 degrees=False)
510 == tractId])
511
512 return pd.concat(outputCat)
513
514 def getVisitMatchedFakeCat(self, fakeCat, exposure):
515 """Trim the fakeCat to select particular visit
516
517 Parameters
518 ----------
519 fakeCat : `pandas.core.frame.DataFrame`
520 The catalog of fake sources to add to the exposure
521 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
522 The exposure to add the fake sources to
523
524 Returns
525 -------
526 movingFakeCat : `pandas.DataFrame`
527 All fakes that belong to the visit
528 """
529 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"]
530
531 return fakeCat[selected]
532
533 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
534 """Match sources in calibCat and sourceCat and copy the specified fields
535
536 Parameters
537 ----------
538 calibCat : `lsst.afw.table.SourceCatalog`
539 Catalog from which to copy fields.
540 sourceCat : `lsst.afw.table.SourceCatalog`
541 Catalog to which to copy fields.
542 fieldsToCopy : `lsst.pex.config.listField.List`
543 Fields to copy from calibCat to SoourceCat.
544
545 Returns
546 -------
547 newCat : `lsst.afw.table.SourceCatalog`
548 Catalog which includes the copied fields.
549
550 The fields copied are those specified by `fieldsToCopy` that actually exist
551 in the schema of `calibCat`.
552
553 This version was based on and adapted from the one in calibrateTask.
554 """
555
556 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
557 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
558 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
559
560 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
561
562 # Add the desired columns from the option fieldsToCopy
563 missingFieldNames = []
564 for fieldName in fieldsToCopy:
565 if fieldName in calibCat.schema:
566 schemaItem = calibCat.schema.find(fieldName)
567 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
568 schema = calibSchemaMapper.editOutputSchema()
569 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
570 else:
571 missingFieldNames.append(fieldName)
572 if missingFieldNames:
573 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
574 "fieldsToCopy")
575
576 if "calib_detected" not in calibSchemaMapper.getOutputSchema():
577 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
578 "Source was detected as an icSource"))
579 else:
580 self.calibSourceKey = None
581
582 schema = calibSchemaMapper.getOutputSchema()
583 newCat = afwTable.SourceCatalog(schema)
584 newCat.reserve(len(sourceCat))
585 newCat.extend(sourceCat, sourceSchemaMapper)
586
587 # Set the aliases so it doesn't complain.
588 for k, v in sourceCat.schema.getAliasMap().items():
589 newCat.schema.getAliasMap().set(k, v)
590
591 select = newCat["deblend_nChild"] == 0
592 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
593 # Check that no sourceCat sources are listed twice (we already know
594 # that each match has a unique calibCat source ID, due to using
595 # that ID as the key in bestMatches)
596 numMatches = len(matches)
597 numUniqueSources = len(set(m[1].getId() for m in matches))
598 if numUniqueSources != numMatches:
599 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches,
600 numUniqueSources)
601
602 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
603
604 # For each match: set the calibSourceKey flag and copy the desired
605 # fields
606 for src, calibSrc, d in matches:
607 if self.calibSourceKey:
608 src.setFlag(self.calibSourceKey, True)
609 # src.assign copies the footprint from calibSrc, which we don't want
610 # (DM-407)
611 # so set calibSrc's footprint to src's footprint before src.assign,
612 # then restore it
613 calibSrcFootprint = calibSrc.getFootprint()
614 try:
615 calibSrc.setFootprint(src.getFootprint())
616 src.assign(calibSrc, calibSchemaMapper)
617 finally:
618 calibSrc.setFootprint(calibSrcFootprint)
619
620 return newCat
621
622
623class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections):
624 ccdVisitFakeMagnitudes = cT.Output(
625 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
626 name="{fakesType}ccdVisitFakeMagnitudes",
627 storageClass="DataFrame",
628 dimensions=("instrument", "visit", "detector"),
629 )
630
631
632class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
633 pipelineConnections=ProcessCcdWithVariableFakesConnections):
634 scatterSize = pexConfig.RangeField(
635 dtype=float,
636 default=0.4,
637 min=0,
638 max=100,
639 doc="Amount of scatter to add to the visit magnitude for variable "
640 "sources."
641 )
642
643
644class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
645 """As ProcessCcdWithFakes except add variablity to the fakes catalog
646 magnitude in the observed band for this ccdVisit.
647
648 Additionally, write out the modified magnitudes to the Butler.
649 """
650
651 _DefaultName = "processCcdWithVariableFakes"
652 ConfigClass = ProcessCcdWithVariableFakesConfig
653
654 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None,
655 icSourceCat=None, sfdSourceCat=None, idGenerator=None):
656 """Add fake sources to a calexp and then run detection, deblending and
657 measurement.
658
659 Parameters
660 ----------
661 fakeCat : `pandas.core.frame.DataFrame`
662 The catalog of fake sources to add to the exposure.
663 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
664 The exposure to add the fake sources to.
665 skyMap : `lsst.skymap.SkyMap`
666 SkyMap defining the tracts and patches the fakes are stored over.
667 wcs : `lsst.afw.geom.SkyWcs`, optional
668 WCS to use to add fake sources.
669 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`, optional
670 Photometric calibration to be used to calibrate the fake sources.
671 icSourceCat : `lsst.afw.table.SourceCatalog`, optional
672 Catalog to take the information about which sources were used for
673 calibration from.
674 sfdSourceCat : `lsst.afw.table.SourceCatalog`, optional
675 Catalog produced by singleFrameDriver, needed to copy some
676 calibration flags from.
677 idGenerator : `lsst.meas.base.IdGenerator`, optional
678 Object that generates Source IDs and random seeds.
679
680 Returns
681 -------
682 resultStruct : `lsst.pipe.base.struct.Struct`
683 Results struct containing:
684
685 - outputExposure : Exposure with added fakes
686 (`lsst.afw.image.exposure.exposure.ExposureF`)
687 - outputCat : Catalog with detected fakes
688 (`lsst.afw.table.source.source.SourceCatalog`)
689 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
690 inserted with after being scattered (`pandas.DataFrame`)
691
692 Notes
693 -----
694 Adds pixel coordinates for each source to the fakeCat and removes
695 objects with bulge or disk half light radius = 0 (if ``config.cleanCat
696 = True``). These columns are called ``x`` and ``y`` and are in pixels.
697
698 Adds the ``Fake`` mask plane to the exposure which is then set by
699 `addFakeSources` to mark where fake sources have been added. Uses the
700 information in the ``fakeCat`` to make fake galaxies (using galsim) and
701 fake stars, using the PSF models from the PSF information for the
702 calexp. These are then added to the calexp and the calexp with fakes
703 included returned.
704
705 The galsim galaxies are made using a double sersic profile, one for the
706 bulge and one for the disk, this is then convolved with the PSF at that
707 point.
708
709
710 """
711 fakeCat = self.composeFakeCat(fakeCats, skyMap)
712
713 if wcs is None:
714 wcs = exposure.getWcs()
715
716 if photoCalib is None:
717 photoCalib = exposure.getPhotoCalib()
718
719 if idGenerator is None:
720 idGenerator = IdGenerator()
721
722 band = exposure.getFilter().bandLabel
723 ccdVisitMagnitudes = self.addVariability(
724 fakeCat,
725 band,
726 exposure,
727 photoCalib,
728 idGenerator.catalog_id,
729 )
730
731 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
732
733 # detect, deblend and measure sources
734 returnedStruct = self.calibrate.run(exposure, idGenerator=idGenerator)
735 sourceCat = returnedStruct.sourceCat
736
737 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
738
739 resultStruct = pipeBase.Struct(outputExposure=exposure,
740 outputCat=sourceCat,
741 ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
742 return resultStruct
743
744 def addVariability(self, fakeCat, band, exposure, photoCalib, rngSeed):
745 """Add scatter to the fake catalog visit magnitudes.
746
747 Currently just adds a simple Gaussian scatter around the static fake
748 magnitude. This function could be modified to return any number of
749 fake variability.
750
751 Parameters
752 ----------
753 fakeCat : `pandas.DataFrame`
754 Catalog of fakes to modify magnitudes of.
755 band : `str`
756 Current observing band to modify.
757 exposure : `lsst.afw.image.ExposureF`
758 Exposure fakes will be added to.
759 photoCalib : `lsst.afw.image.PhotoCalib`
760 Photometric calibration object of ``exposure``.
761 rngSeed : `int`
762 Random number generator seed.
763
764 Returns
765 -------
766 dataFrame : `pandas.DataFrame`
767 DataFrame containing the values of the magnitudes to that will
768 be inserted into this ccdVisit.
769 """
770 rng = np.random.default_rng(rngSeed)
771 magScatter = rng.normal(loc=0,
772 scale=self.config.scatterSize,
773 size=len(fakeCat))
774 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
775 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
776 return pd.DataFrame(data={"variableMag": visitMagnitudes})
std::vector< SchemaItem< Flag > > * items
A mapping between the keys of two Schemas, used to copy data between them.
daf::base::PropertySet * set
Definition fits.cc:931
SourceMatchVector matchXy(SourceCatalog const &cat1, SourceCatalog const &cat2, double radius, MatchControl const &mc=MatchControl())
Compute all tuples (s1,s2,d) where s1 belings to cat1, s2 belongs to cat2 and d, the distance between...
Definition Match.cc:305
A description of a field in a table.
Definition Field.h:24