23 Insert fakes into deepCoadds
26 from astropy.table
import Table
35 from lsst.pipe.base import CmdLineTask, PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39 from lsst.geom import SpherePoint, radians, Box2D
42 __all__ = [
"InsertFakesConfig",
"InsertFakesTask"]
46 dimensions=(
"tract",
"patch",
"band",
"skymap")):
49 doc=
"Image into which fakes are to be added.",
50 name=
"{CoaddName}Coadd",
51 storageClass=
"ExposureF",
52 dimensions=(
"tract",
"patch",
"band",
"skymap")
56 doc=
"Catalog of fake sources to draw inputs from.",
57 name=
"{CoaddName}Coadd_fakeSourceCat",
58 storageClass=
"DataFrame",
59 dimensions=(
"tract",
"skymap")
62 imageWithFakes = cT.Output(
63 doc=
"Image with fake sources added.",
64 name=
"fakes_{CoaddName}Coadd",
65 storageClass=
"ExposureF",
66 dimensions=(
"tract",
"patch",
"band",
"skymap")
71 pipelineConnections=InsertFakesConnections):
72 """Config for inserting fake sources
76 The default column names are those from the University of Washington sims database.
79 raColName = pexConfig.Field(
80 doc=
"RA column name used in the fake source catalog.",
85 decColName = pexConfig.Field(
86 doc=
"Dec. column name used in the fake source catalog.",
91 doCleanCat = pexConfig.Field(
92 doc=
"If true removes bad sources from the catalog.",
97 diskHLR = pexConfig.Field(
98 doc=
"Column name for the disk half light radius used in the fake source catalog.",
100 default=
"DiskHalfLightRadius",
103 bulgeHLR = pexConfig.Field(
104 doc=
"Column name for the bulge half light radius used in the fake source catalog.",
106 default=
"BulgeHalfLightRadius",
109 magVar = pexConfig.Field(
110 doc=
"The column name for the magnitude calculated taking variability into account. In the format "
111 "``filter name``magVar, e.g. imagVar for the magnitude in the i band.",
116 nDisk = pexConfig.Field(
117 doc=
"The column name for the sersic index of the disk component used in the fake source catalog.",
122 nBulge = pexConfig.Field(
123 doc=
"The column name for the sersic index of the bulge component used in the fake source catalog.",
128 aDisk = pexConfig.Field(
129 doc=
"The column name for the semi major axis length of the disk component used in the fake source"
135 aBulge = pexConfig.Field(
136 doc=
"The column name for the semi major axis length of the bulge component.",
141 bDisk = pexConfig.Field(
142 doc=
"The column name for the semi minor axis length of the disk component.",
147 bBulge = pexConfig.Field(
148 doc=
"The column name for the semi minor axis length of the bulge component used in the fake source "
154 paDisk = pexConfig.Field(
155 doc=
"The column name for the PA of the disk component used in the fake source catalog.",
160 paBulge = pexConfig.Field(
161 doc=
"The column name for the PA of the bulge component used in the fake source catalog.",
166 sourceType = pexConfig.Field(
167 doc=
"The column name for the source type used in the fake source catalog.",
169 default=
"sourceType",
172 fakeType = pexConfig.Field(
173 doc=
"What type of fake catalog to use, snapshot (includes variability in the magnitudes calculated "
174 "from the MJD of the image), static (no variability) or filename for a user defined fits"
180 calibFluxRadius = pexConfig.Field(
181 doc=
"Aperture radius (in pixels) that was used to define the calibration for this image+catalog. "
182 "This will be used to produce the correct instrumental fluxes within the radius. "
183 "This value should match that of the field defined in slot_CalibFlux_instFlux.",
188 coaddName = pexConfig.Field(
189 doc=
"The name of the type of coadd used",
194 doSubSelectSources = pexConfig.Field(
195 doc=
"Set to True if you wish to sub select sources to be input based on the value in the column"
196 "set in the sourceSelectionColName config option.",
201 sourceSelectionColName = pexConfig.Field(
202 doc=
"The name of the column in the input fakes catalogue to be used to determine which sources to"
203 "add, default is none and when this is used all sources are added.",
205 default=
"templateSource"
208 insertImages = pexConfig.Field(
209 doc=
"Insert images directly? True or False.",
214 doProcessAllDataIds = pexConfig.Field(
215 doc=
"If True, all input data IDs will be processed, even those containing no fake sources.",
222 """Insert fake objects into images.
224 Add fake stars and galaxies to the given image, read in through the dataRef. Galaxy parameters are read in
225 from the specified file and then modelled using galsim.
227 `InsertFakesTask` has five functions that make images of the fake sources and then add them to the
231 Use the WCS information to add the pixel coordinates of each source.
232 `mkFakeGalsimGalaxies`
233 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
235 Use the PSF information from the image to make a fake star using the magnitude information from the
238 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
239 that are 0. Also removes rows that have Sersic index outside of galsim's allowed paramters. If
240 the config option sourceSelectionColName is set then this function limits the catalog of input fakes
241 to only those which are True in this column.
243 Add the fake sources to the image.
247 _DefaultName =
"insertFakes"
248 ConfigClass = InsertFakesConfig
250 def runDataRef(self, dataRef):
251 """Read in/write out the required data products and add fake sources to the deepCoadd.
255 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
256 Data reference defining the image to have fakes added to it
257 Used to access the following data products:
261 infoStr =
"Adding fakes to: tract: %d, patch: %s, filter: %s" % (dataRef.dataId[
"tract"],
262 dataRef.dataId[
"patch"],
263 dataRef.dataId[
"filter"])
264 self.log.
info(infoStr)
268 if self.
config.fakeType ==
"static":
269 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
272 self.fakeSourceCatType =
"deepCoadd_fakeSourceCat"
274 fakeCat = Table.read(self.
config.fakeType).to_pandas()
276 coadd = dataRef.get(
"deepCoadd")
278 photoCalib = coadd.getPhotoCalib()
280 imageWithFakes = self.run(fakeCat, coadd, wcs, photoCalib)
282 dataRef.put(imageWithFakes.imageWithFakes,
"fakes_deepCoadd")
284 def runQuantum(self, butlerQC, inputRefs, outputRefs):
285 inputs = butlerQC.get(inputRefs)
286 inputs[
"wcs"] = inputs[
"image"].getWcs()
287 inputs[
"photoCalib"] = inputs[
"image"].getPhotoCalib()
289 outputs = self.run(**inputs)
290 butlerQC.put(outputs, outputRefs)
293 def _makeArgumentParser(cls):
294 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
295 parser.add_id_argument(name=
"--id", datasetType=
"deepCoadd",
296 help=
"data IDs for the deepCoadd, e.g. --id tract=12345 patch=1,2 filter=r",
297 ContainerClass=ExistingCoaddDataIdContainer)
300 def run(self, fakeCat, image, wcs, photoCalib):
301 """Add fake sources to an image.
305 fakeCat : `pandas.core.frame.DataFrame`
306 The catalog of fake sources to be input
307 image : `lsst.afw.image.exposure.exposure.ExposureF`
308 The image into which the fake sources should be added
309 wcs : `lsst.afw.geom.SkyWcs`
310 WCS to use to add fake sources
311 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
312 Photometric calibration to be used to calibrate the fake sources
316 resultStruct : `lsst.pipe.base.struct.Struct`
317 contains : image : `lsst.afw.image.exposure.exposure.ExposureF`
321 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
322 light radius = 0 (if ``config.doCleanCat = True``).
324 Adds the ``Fake`` mask plane to the image which is then set by `addFakeSources` to mark where fake
325 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
326 and fake stars, using the PSF models from the PSF information for the image. These are then added to
327 the image and the image with fakes included returned.
329 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
330 this is then convolved with the PSF at that point.
333 image.mask.addMaskPlane(
"FAKE")
334 self.bitmask = image.mask.getPlaneBitMask(
"FAKE")
335 self.log.
info(
"Adding mask plane with bitmask %d" % self.bitmask)
337 fakeCat = self.addPixCoords(fakeCat, wcs)
338 fakeCat = self.trimFakeCat(fakeCat, image, wcs)
339 band = image.getFilter().getCanonicalName()[0]
341 pixelScale = wcs.getPixelScale().asArcseconds()
344 if isinstance(fakeCat[self.
config.sourceType].iloc[0], str):
345 galCheckVal =
"galaxy"
346 starCheckVal =
"star"
347 elif isinstance(fakeCat[self.
config.sourceType].iloc[0], bytes):
348 galCheckVal = b
"galaxy"
349 starCheckVal = b
"star"
350 elif isinstance(fakeCat[self.
config.sourceType].iloc[0], (int, float)):
354 raise TypeError(
"sourceType column does not have required type, should be str, bytes or int")
356 if not self.
config.insertImages:
357 if self.
config.doCleanCat:
358 fakeCat = self.cleanCat(fakeCat, starCheckVal)
360 galaxies = (fakeCat[self.
config.sourceType] == galCheckVal)
361 galImages = self.mkFakeGalsimGalaxies(fakeCat[galaxies], band, photoCalib, pixelScale, psf,
364 stars = (fakeCat[self.
config.sourceType] == starCheckVal)
365 starImages = self.mkFakeStars(fakeCat[stars], band, photoCalib, psf, image)
367 galImages, starImages = self.processImagesForInsertion(fakeCat, wcs, psf, photoCalib, band,
370 image = self.addFakeSources(image, galImages,
"galaxy")
371 image = self.addFakeSources(image, starImages,
"star")
372 elif len(fakeCat) == 0
and self.
config.doProcessAllDataIds:
373 self.log.
warn(
"No fakes found for this dataRef; processing anyway.")
375 raise RuntimeError(
"No fakes found for this dataRef.")
377 resultStruct = pipeBase.Struct(imageWithFakes=image)
381 def processImagesForInsertion(self, fakeCat, wcs, psf, photoCalib, band, pixelScale):
382 """Process images from files into the format needed for insertion.
386 fakeCat : `pandas.core.frame.DataFrame`
387 The catalog of fake sources to be input
388 wcs : `lsst.afw.geom.skyWcs.skyWcs.SkyWc`
389 WCS to use to add fake sources
390 psf : `lsst.meas.algorithms.coaddPsf.coaddPsf.CoaddPsf` or
391 `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
392 The PSF information to use to make the PSF images
393 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
394 Photometric calibration to be used to calibrate the fake sources
396 The filter band that the observation was taken in.
398 The pixel scale of the image the sources are to be added to.
403 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
404 `lsst.geom.Point2D` of their locations.
405 For sources labelled as galaxy.
407 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
408 `lsst.geom.Point2D` of their locations.
409 For sources labelled as star.
413 The input fakes catalog needs to contain the absolute path to the image in the
414 band that is being used to add images to. It also needs to have the R.A. and
415 declination of the fake source in radians and the sourceType of the object.
420 self.log.
info(
"Processing %d fake images" % len(fakeCat))
422 for (imFile, sourceType, mag, x, y)
in zip(fakeCat[band +
"imFilename"].array,
423 fakeCat[
"sourceType"].array,
424 fakeCat[self.
config.magVar % band].array,
425 fakeCat[
"x"].array, fakeCat[
"y"].array):
427 im = afwImage.ImageF.readFits(imFile)
434 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
435 psfKernel = psf.computeKernelImage(xy).getArray()
436 psfKernel /= correctedFlux
438 except InvalidParameterError:
439 self.log.
info(
"%s at %0.4f, %0.4f outside of image" % (sourceType, x, y))
442 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
443 galsimIm = galsim.InterpolatedImage(galsim.Image(im.array), scale=pixelScale)
444 convIm = galsim.Convolve([galsimIm, psfIm])
447 outIm = convIm.drawImage(scale=pixelScale, method=
"real_space").array
448 except (galsim.errors.GalSimFFTSizeError, MemoryError):
451 imSum = np.sum(outIm)
455 flux = photoCalib.magnitudeToInstFlux(mag, xy)
459 imWithFlux = flux*divIm
461 if sourceType == b
"galaxy":
462 galImages.append((afwImage.ImageF(imWithFlux), xy))
463 if sourceType == b
"star":
464 starImages.append((afwImage.ImageF(imWithFlux), xy))
466 return galImages, starImages
470 """Add pixel coordinates to the catalog of fakes.
474 fakeCat : `pandas.core.frame.DataFrame`
475 The catalog of fake sources to be input
476 wcs : `lsst.afw.geom.SkyWcs`
477 WCS to use to add fake sources
481 fakeCat : `pandas.core.frame.DataFrame`
485 The default option is to use the WCS information from the image. If the ``useUpdatedCalibs`` config
486 option is set then it will use the updated WCS from jointCal.
489 ras = fakeCat[self.config.raColName].values
490 decs = fakeCat[self.config.decColName].values
491 skyCoords = [
SpherePoint(ra, dec, radians)
for (ra, dec)
in zip(ras, decs)]
492 pixCoords = wcs.skyToPixel(skyCoords)
493 xs = [coord.getX()
for coord
in pixCoords]
494 ys = [coord.getY()
for coord
in pixCoords]
501 """Trim the fake cat to about the size of the input image.
505 fakeCat : `pandas.core.frame.DataFrame`
506 The catalog of fake sources to be input
507 image : `lsst.afw.image.exposure.exposure.ExposureF`
508 The image into which the fake sources should be added
509 wcs : `lsst.afw.geom.SkyWcs`
510 WCS to use to add fake sources
514 fakeCat : `pandas.core.frame.DataFrame`
515 The original fakeCat trimmed to the area of the image
518 bbox =
Box2D(image.getBBox())
519 corners = bbox.getCorners()
521 skyCorners = wcs.pixelToSky(corners)
525 coord =
SpherePoint(row[self.config.raColName], row[self.config.decColName], radians)
526 return region.contains(coord.getVector())
528 return fakeCat[fakeCat.apply(trim, axis=1)]
531 """Make images of fake galaxies using GalSim.
537 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
538 The PSF information to use to make the PSF images
539 fakeCat : `pandas.core.frame.DataFrame`
540 The catalog of fake sources to be input
541 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
542 Photometric calibration to be used to calibrate the fake sources
546 galImages : `generator`
547 A generator of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
548 `lsst.geom.Point2D` of their locations.
553 Fake galaxies are made by combining two sersic profiles, one for the bulge and one for the disk. Each
554 component has an individual sersic index (n), a, b and position angle (PA). The combined profile is
555 then convolved with the PSF at the specified x, y position on the image.
557 The names of the columns in the ``fakeCat`` are configurable and are the column names from the
558 University of Washington simulations database as default. For more information see the doc strings
559 attached to the config options.
561 See mkFakeStars doc string for an explanation of calibration to instrumental flux.
564 self.log.
info(
"Making %d fake galaxy images" % len(fakeCat))
566 for (index, row)
in fakeCat.iterrows():
572 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
573 psfKernel = psf.computeKernelImage(xy).getArray()
574 psfKernel /= correctedFlux
576 except InvalidParameterError:
577 self.log.
info(
"Galaxy at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
581 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
585 bulge = galsim.Sersic(row[self.config.nBulge], half_light_radius=row[self.config.bulgeHLR])
586 axisRatioBulge = row[self.config.bBulge]/row[self.config.aBulge]
587 bulge = bulge.shear(q=axisRatioBulge, beta=((90 - row[self.config.paBulge])*galsim.degrees))
589 disk = galsim.Sersic(row[self.config.nDisk], half_light_radius=row[self.config.diskHLR])
590 axisRatioDisk = row[self.config.bDisk]/row[self.config.aDisk]
591 disk = disk.shear(q=axisRatioDisk, beta=((90 - row[self.config.paDisk])*galsim.degrees))
594 gal = gal.withFlux(flux)
596 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
597 gal = galsim.Convolve([gal, psfIm])
599 galIm = gal.drawImage(scale=pixelScale, method=
"real_space").array
600 except (galsim.errors.GalSimFFTSizeError, MemoryError):
603 yield (afwImage.ImageF(galIm), xy)
607 """Make fake stars based off the properties in the fakeCat.
612 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
613 The PSF information to use to make the PSF images
614 fakeCat : `pandas.core.frame.DataFrame`
615 The catalog of fake sources to be input
616 image : `lsst.afw.image.exposure.exposure.ExposureF`
617 The image into which the fake sources should be added
618 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
619 Photometric calibration to be used to calibrate the fake sources
623 starImages : `generator`
624 A generator of tuples of `lsst.afw.image.ImageF` of fake stars and
625 `lsst.geom.Point2D` of their locations.
629 To take a given magnitude and translate to the number of counts in the image
630 we use photoCalib.magnitudeToInstFlux, which returns the instrumental flux for the
631 given calibration radius used in the photometric calibration step.
632 Thus `calibFluxRadius` should be set to this same radius so that we can normalize
633 the PSF model to the correct instrumental flux within calibFluxRadius.
636 self.log.
info(
"Making %d fake star images" % len(fakeCat))
638 for (index, row)
in fakeCat.iterrows():
644 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
645 starIm = psf.computeImage(xy)
646 starIm /= correctedFlux
648 except InvalidParameterError:
649 self.log.
info(
"Star at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
653 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
658 yield ((starIm.convertF(), xy))
661 """Remove rows from the fakes catalog which have HLR = 0 for either the buldge or disk component,
662 also remove galaxies that have Sersic index outside the galsim min and max
663 allowed (0.3 <= n <= 6.2).
667 fakeCat : `pandas.core.frame.DataFrame`
668 The catalog of fake sources to be input
669 starCheckVal : `str`, `bytes` or `int`
670 The value that is set in the sourceType column to specifiy an object is a star.
674 fakeCat : `pandas.core.frame.DataFrame`
675 The input catalog of fake sources but with the bad objects removed
679 If the config option sourceSelectionColName is set then only objects with this column set to True
683 rowsToKeep = (((fakeCat[self.config.bulgeHLR] != 0.0) & (fakeCat[self.config.diskHLR] != 0.0))
684 | (fakeCat[self.config.sourceType] == starCheckVal))
685 numRowsNotUsed = len(fakeCat) - len(np.where(rowsToKeep)[0])
686 self.log.
info(
"Removing %d rows with HLR = 0 for either the bulge or disk" % numRowsNotUsed)
687 fakeCat = fakeCat[rowsToKeep]
689 minN = galsim.Sersic._minimum_n
690 maxN = galsim.Sersic._maximum_n
691 rowsWithGoodSersic = (((fakeCat[self.config.nBulge] >= minN) & (fakeCat[self.config.nBulge] <= maxN)
692 & (fakeCat[self.config.nDisk] >= minN) & (fakeCat[self.config.nDisk] <= maxN))
693 | (fakeCat[self.config.sourceType] == starCheckVal))
694 numRowsNotUsed = len(fakeCat) - len(np.where(rowsWithGoodSersic)[0])
695 self.log.
info(
"Removing %d rows of galaxies with nBulge or nDisk outside of %0.2f <= n <= %0.2f" %
696 (numRowsNotUsed, minN, maxN))
697 fakeCat = fakeCat[rowsWithGoodSersic]
699 if self.config.doSubSelectSources:
701 rowsSelected = (fakeCat[self.config.sourceSelectionColName])
703 raise KeyError(
"Given column, %s, for source selection not found." %
704 self.config.sourceSelectionColName)
705 numRowsNotUsed = len(fakeCat) - len(rowsSelected)
706 self.log.
info(
"Removing %d rows which were not designated as template sources" % numRowsNotUsed)
707 fakeCat = fakeCat[rowsSelected]
712 """Add the fake sources to the given image
716 image : `lsst.afw.image.exposure.exposure.ExposureF`
717 The image into which the fake sources should be added
718 fakeImages : `typing.Iterator` [`tuple` ['lsst.afw.image.ImageF`, `lsst.geom.Point2d`]]
719 An iterator of tuples that contains (or generates) images of fake sources,
720 and the locations they are to be inserted at.
722 The type (star/galaxy) of fake sources input
726 image : `lsst.afw.image.exposure.exposure.ExposureF`
730 Uses the x, y information in the ``fakeCat`` to position an image of the fake interpolated onto the
731 pixel grid of the image. Sets the ``FAKE`` mask plane for the pixels added with the fake source.
734 imageBBox = image.getBBox()
735 imageMI = image.maskedImage
737 for (fakeImage, xy)
in fakeImages:
738 X0 = xy.getX() - fakeImage.getWidth()/2 + 0.5
739 Y0 = xy.getY() - fakeImage.getHeight()/2 + 0.5
740 self.log.
debug(
"Adding fake source at %d, %d" % (xy.getX(), xy.getY()))
741 if sourceType ==
"galaxy":
743 interpFakeImBBox = interpFakeImage.getBBox()
745 interpFakeImage = fakeImage
746 interpFakeImBBox = fakeImage.getBBox()
748 interpFakeImBBox.clip(imageBBox)
749 imageMIView = imageMI.Factory(imageMI, interpFakeImBBox)
751 if interpFakeImBBox.getArea() > 0:
752 clippedFakeImage = interpFakeImage.Factory(interpFakeImage, interpFakeImBBox)
753 clippedFakeImageMI = afwImage.MaskedImageF(clippedFakeImage)
754 clippedFakeImageMI.mask.set(self.bitmask)
755 imageMIView += clippedFakeImageMI
759 def _getMetadataName(self):
760 """Disable metadata writing"""