23 Insert fakes into deepCoadds
26 from astropy.table
import Table
35 from lsst.pipe.base import CmdLineTask, PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39 from lsst.geom import SpherePoint, radians, Box2D
41 __all__ = [
"InsertFakesConfig",
"InsertFakesTask"]
45 defaultTemplates={
"coaddName":
"deep",
46 "fakesType":
"fakes_"},
47 dimensions=(
"tract",
"patch",
"band",
"skymap")):
50 doc=
"Image into which fakes are to be added.",
51 name=
"{coaddName}Coadd",
52 storageClass=
"ExposureF",
53 dimensions=(
"tract",
"patch",
"band",
"skymap")
57 doc=
"Catalog of fake sources to draw inputs from.",
58 name=
"{fakesType}fakeSourceCat",
59 storageClass=
"DataFrame",
60 dimensions=(
"tract",
"skymap")
63 imageWithFakes = cT.Output(
64 doc=
"Image with fake sources added.",
65 name=
"{fakesType}{coaddName}Coadd",
66 storageClass=
"ExposureF",
67 dimensions=(
"tract",
"patch",
"band",
"skymap")
72 pipelineConnections=InsertFakesConnections):
73 """Config for inserting fake sources
77 The default column names are those from the University of Washington sims database.
80 raColName = pexConfig.Field(
81 doc=
"RA column name used in the fake source catalog.",
86 decColName = pexConfig.Field(
87 doc=
"Dec. column name used in the fake source catalog.",
92 doCleanCat = pexConfig.Field(
93 doc=
"If true removes bad sources from the catalog.",
98 diskHLR = pexConfig.Field(
99 doc=
"Column name for the disk half light radius used in the fake source catalog.",
101 default=
"DiskHalfLightRadius",
104 bulgeHLR = pexConfig.Field(
105 doc=
"Column name for the bulge half light radius used in the fake source catalog.",
107 default=
"BulgeHalfLightRadius",
110 magVar = pexConfig.Field(
111 doc=
"The column name for the magnitude calculated taking variability into account. In the format "
112 "``filter name``magVar, e.g. imagVar for the magnitude in the i band.",
117 nDisk = pexConfig.Field(
118 doc=
"The column name for the sersic index of the disk component used in the fake source catalog.",
123 nBulge = pexConfig.Field(
124 doc=
"The column name for the sersic index of the bulge component used in the fake source catalog.",
129 aDisk = pexConfig.Field(
130 doc=
"The column name for the semi major axis length of the disk component used in the fake source"
136 aBulge = pexConfig.Field(
137 doc=
"The column name for the semi major axis length of the bulge component.",
142 bDisk = pexConfig.Field(
143 doc=
"The column name for the semi minor axis length of the disk component.",
148 bBulge = pexConfig.Field(
149 doc=
"The column name for the semi minor axis length of the bulge component used in the fake source "
155 paDisk = pexConfig.Field(
156 doc=
"The column name for the PA of the disk component used in the fake source catalog.",
161 paBulge = pexConfig.Field(
162 doc=
"The column name for the PA of the bulge component used in the fake source catalog.",
167 sourceType = pexConfig.Field(
168 doc=
"The column name for the source type used in the fake source catalog.",
170 default=
"sourceType",
173 fakeType = pexConfig.Field(
174 doc=
"What type of fake catalog to use, snapshot (includes variability in the magnitudes calculated "
175 "from the MJD of the image), static (no variability) or filename for a user defined fits"
181 calibFluxRadius = pexConfig.Field(
182 doc=
"Aperture radius (in pixels) that was used to define the calibration for this image+catalog. "
183 "This will be used to produce the correct instrumental fluxes within the radius. "
184 "This value should match that of the field defined in slot_CalibFlux_instFlux.",
189 coaddName = pexConfig.Field(
190 doc=
"The name of the type of coadd used",
195 doSubSelectSources = pexConfig.Field(
196 doc=
"Set to True if you wish to sub select sources to be input based on the value in the column"
197 "set in the sourceSelectionColName config option.",
202 sourceSelectionColName = pexConfig.Field(
203 doc=
"The name of the column in the input fakes catalogue to be used to determine which sources to"
204 "add, default is none and when this is used all sources are added.",
206 default=
"templateSource"
209 insertImages = pexConfig.Field(
210 doc=
"Insert images directly? True or False.",
215 doProcessAllDataIds = pexConfig.Field(
216 doc=
"If True, all input data IDs will be processed, even those containing no fake sources.",
221 trimBuffer = pexConfig.Field(
222 doc=
"Size of the pixel buffer surrounding the image. Only those fake sources with a centroid"
223 "falling within the image+buffer region will be considered for fake source injection.",
230 """Insert fake objects into images.
232 Add fake stars and galaxies to the given image, read in through the dataRef. Galaxy parameters are read in
233 from the specified file and then modelled using galsim.
235 `InsertFakesTask` has five functions that make images of the fake sources and then add them to the
239 Use the WCS information to add the pixel coordinates of each source.
240 `mkFakeGalsimGalaxies`
241 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
243 Use the PSF information from the image to make a fake star using the magnitude information from the
246 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
247 that are 0. Also removes rows that have Sersic index outside of galsim's allowed paramters. If
248 the config option sourceSelectionColName is set then this function limits the catalog of input fakes
249 to only those which are True in this column.
251 Add the fake sources to the image.
255 _DefaultName =
"insertFakes"
256 ConfigClass = InsertFakesConfig
258 def runDataRef(self, dataRef):
259 """Read in/write out the required data products and add fake sources to the deepCoadd.
263 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
264 Data reference defining the image to have fakes added to it
265 Used to access the following data products:
269 infoStr =
"Adding fakes to: tract: %d, patch: %s, filter: %s" % (dataRef.dataId[
"tract"],
270 dataRef.dataId[
"patch"],
271 dataRef.dataId[
"filter"])
272 self.log.
info(infoStr)
276 if self.
configconfig.fakeType ==
"static":
277 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
280 self.fakeSourceCatType =
"deepCoadd_fakeSourceCat"
282 fakeCat = Table.read(self.
configconfig.fakeType).to_pandas()
284 coadd = dataRef.get(
"deepCoadd")
286 photoCalib = coadd.getPhotoCalib()
288 imageWithFakes = self.run(fakeCat, coadd, wcs, photoCalib)
290 dataRef.put(imageWithFakes.imageWithFakes,
"fakes_deepCoadd")
292 def runQuantum(self, butlerQC, inputRefs, outputRefs):
293 inputs = butlerQC.get(inputRefs)
294 inputs[
"wcs"] = inputs[
"image"].getWcs()
295 inputs[
"photoCalib"] = inputs[
"image"].getPhotoCalib()
297 outputs = self.run(**inputs)
298 butlerQC.put(outputs, outputRefs)
301 def _makeArgumentParser(cls):
302 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
303 parser.add_id_argument(name=
"--id", datasetType=
"deepCoadd",
304 help=
"data IDs for the deepCoadd, e.g. --id tract=12345 patch=1,2 filter=r",
305 ContainerClass=ExistingCoaddDataIdContainer)
308 def run(self, fakeCat, image, wcs, photoCalib):
309 """Add fake sources to an image.
313 fakeCat : `pandas.core.frame.DataFrame`
314 The catalog of fake sources to be input
315 image : `lsst.afw.image.exposure.exposure.ExposureF`
316 The image into which the fake sources should be added
317 wcs : `lsst.afw.geom.SkyWcs`
318 WCS to use to add fake sources
319 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
320 Photometric calibration to be used to calibrate the fake sources
324 resultStruct : `lsst.pipe.base.struct.Struct`
325 contains : image : `lsst.afw.image.exposure.exposure.ExposureF`
329 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
330 light radius = 0 (if ``config.doCleanCat = True``).
332 Adds the ``Fake`` mask plane to the image which is then set by `addFakeSources` to mark where fake
333 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
334 and fake stars, using the PSF models from the PSF information for the image. These are then added to
335 the image and the image with fakes included returned.
337 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
338 this is then convolved with the PSF at that point.
341 image.mask.addMaskPlane(
"FAKE")
342 self.bitmask = image.mask.getPlaneBitMask(
"FAKE")
343 self.log.
info(
"Adding mask plane with bitmask %d" % self.bitmask)
345 fakeCat = self.addPixCoords(fakeCat, wcs)
346 fakeCat = self.trimFakeCat(fakeCat, image, wcs)
347 band = image.getFilterLabel().bandLabel
349 pixelScale = wcs.getPixelScale().asArcseconds()
352 if isinstance(fakeCat[self.
configconfig.sourceType].iloc[0], str):
353 galCheckVal =
"galaxy"
354 starCheckVal =
"star"
355 elif isinstance(fakeCat[self.
configconfig.sourceType].iloc[0], bytes):
356 galCheckVal = b
"galaxy"
357 starCheckVal = b
"star"
358 elif isinstance(fakeCat[self.
configconfig.sourceType].iloc[0], (int, float)):
362 raise TypeError(
"sourceType column does not have required type, should be str, bytes or int")
364 if not self.
configconfig.insertImages:
365 if self.
configconfig.doCleanCat:
366 fakeCat = self.cleanCat(fakeCat, starCheckVal)
368 galaxies = (fakeCat[self.
configconfig.sourceType] == galCheckVal)
369 galImages = self.mkFakeGalsimGalaxies(fakeCat[galaxies], band, photoCalib, pixelScale, psf,
372 stars = (fakeCat[self.
configconfig.sourceType] == starCheckVal)
373 starImages = self.mkFakeStars(fakeCat[stars], band, photoCalib, psf, image)
375 galImages, starImages = self.processImagesForInsertion(fakeCat, wcs, psf, photoCalib, band,
378 image = self.addFakeSources(image, galImages,
"galaxy")
379 image = self.addFakeSources(image, starImages,
"star")
380 elif len(fakeCat) == 0
and self.
configconfig.doProcessAllDataIds:
381 self.log.
warn(
"No fakes found for this dataRef; processing anyway.")
383 raise RuntimeError(
"No fakes found for this dataRef.")
385 resultStruct = pipeBase.Struct(imageWithFakes=image)
389 def processImagesForInsertion(self, fakeCat, wcs, psf, photoCalib, band, pixelScale):
390 """Process images from files into the format needed for insertion.
394 fakeCat : `pandas.core.frame.DataFrame`
395 The catalog of fake sources to be input
396 wcs : `lsst.afw.geom.skyWcs.skyWcs.SkyWc`
397 WCS to use to add fake sources
398 psf : `lsst.meas.algorithms.coaddPsf.coaddPsf.CoaddPsf` or
399 `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
400 The PSF information to use to make the PSF images
401 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
402 Photometric calibration to be used to calibrate the fake sources
404 The filter band that the observation was taken in.
406 The pixel scale of the image the sources are to be added to.
411 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
412 `lsst.geom.Point2D` of their locations.
413 For sources labelled as galaxy.
415 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
416 `lsst.geom.Point2D` of their locations.
417 For sources labelled as star.
421 The input fakes catalog needs to contain the absolute path to the image in the
422 band that is being used to add images to. It also needs to have the R.A. and
423 declination of the fake source in radians and the sourceType of the object.
428 self.log.
info(
"Processing %d fake images" % len(fakeCat))
430 for (imFile, sourceType, mag, x, y)
in zip(fakeCat[band +
"imFilename"].array,
431 fakeCat[
"sourceType"].array,
432 fakeCat[self.
configconfig.magVar % band].array,
433 fakeCat[
"x"].array, fakeCat[
"y"].array):
435 im = afwImage.ImageF.readFits(imFile)
442 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
443 psfKernel = psf.computeKernelImage(xy).getArray()
444 psfKernel /= correctedFlux
446 except InvalidParameterError:
447 self.log.
info(
"%s at %0.4f, %0.4f outside of image" % (sourceType, x, y))
450 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
451 galsimIm = galsim.InterpolatedImage(galsim.Image(im.array), scale=pixelScale)
452 convIm = galsim.Convolve([galsimIm, psfIm])
455 outIm = convIm.drawImage(scale=pixelScale, method=
"real_space").array
456 except (galsim.errors.GalSimFFTSizeError, MemoryError):
459 imSum = np.sum(outIm)
463 flux = photoCalib.magnitudeToInstFlux(mag, xy)
467 imWithFlux = flux*divIm
469 if sourceType == b
"galaxy":
470 galImages.append((afwImage.ImageF(imWithFlux), xy))
471 if sourceType == b
"star":
472 starImages.append((afwImage.ImageF(imWithFlux), xy))
474 return galImages, starImages
478 """Add pixel coordinates to the catalog of fakes.
482 fakeCat : `pandas.core.frame.DataFrame`
483 The catalog of fake sources to be input
484 wcs : `lsst.afw.geom.SkyWcs`
485 WCS to use to add fake sources
489 fakeCat : `pandas.core.frame.DataFrame`
493 The default option is to use the WCS information from the image. If the ``useUpdatedCalibs`` config
494 option is set then it will use the updated WCS from jointCal.
497 ras = fakeCat[self.config.raColName].values
498 decs = fakeCat[self.config.decColName].values
499 skyCoords = [
SpherePoint(ra, dec, radians)
for (ra, dec)
in zip(ras, decs)]
500 pixCoords = wcs.skyToPixel(skyCoords)
501 xs = [coord.getX()
for coord
in pixCoords]
502 ys = [coord.getY()
for coord
in pixCoords]
509 """Trim the fake cat to about the size of the input image.
511 `fakeCat` must be processed with addPixCoords before using this method.
515 fakeCat : `pandas.core.frame.DataFrame`
516 The catalog of fake sources to be input
517 image : `lsst.afw.image.exposure.exposure.ExposureF`
518 The image into which the fake sources should be added
519 wcs : `lsst.afw.geom.SkyWcs`
520 WCS to use to add fake sources
524 fakeCat : `pandas.core.frame.DataFrame`
525 The original fakeCat trimmed to the area of the image
528 bbox =
Box2D(image.getBBox())
531 return bbox.dilatedBy(self.config.trimBuffer).
contains(row[
"x"], row[
"y"])
533 return fakeCat[fakeCat.apply(trim, axis=1)]
536 """Make images of fake galaxies using GalSim.
542 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
543 The PSF information to use to make the PSF images
544 fakeCat : `pandas.core.frame.DataFrame`
545 The catalog of fake sources to be input
546 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
547 Photometric calibration to be used to calibrate the fake sources
551 galImages : `generator`
552 A generator of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
553 `lsst.geom.Point2D` of their locations.
558 Fake galaxies are made by combining two sersic profiles, one for the bulge and one for the disk. Each
559 component has an individual sersic index (n), a, b and position angle (PA). The combined profile is
560 then convolved with the PSF at the specified x, y position on the image.
562 The names of the columns in the ``fakeCat`` are configurable and are the column names from the
563 University of Washington simulations database as default. For more information see the doc strings
564 attached to the config options.
566 See mkFakeStars doc string for an explanation of calibration to instrumental flux.
569 self.log.
info(
"Making %d fake galaxy images" % len(fakeCat))
571 for (index, row)
in fakeCat.iterrows():
577 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
578 psfKernel = psf.computeKernelImage(xy).getArray()
579 psfKernel /= correctedFlux
581 except InvalidParameterError:
582 self.log.
info(
"Galaxy at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
586 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
590 bulge = galsim.Sersic(row[self.config.nBulge], half_light_radius=row[self.config.bulgeHLR])
591 axisRatioBulge = row[self.config.bBulge]/row[self.config.aBulge]
592 bulge = bulge.shear(q=axisRatioBulge, beta=((90 - row[self.config.paBulge])*galsim.degrees))
594 disk = galsim.Sersic(row[self.config.nDisk], half_light_radius=row[self.config.diskHLR])
595 axisRatioDisk = row[self.config.bDisk]/row[self.config.aDisk]
596 disk = disk.shear(q=axisRatioDisk, beta=((90 - row[self.config.paDisk])*galsim.degrees))
599 gal = gal.withFlux(flux)
601 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
602 gal = galsim.Convolve([gal, psfIm])
604 galIm = gal.drawImage(scale=pixelScale, method=
"real_space").array
605 except (galsim.errors.GalSimFFTSizeError, MemoryError):
608 yield (afwImage.ImageF(galIm), xy)
612 """Make fake stars based off the properties in the fakeCat.
617 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
618 The PSF information to use to make the PSF images
619 fakeCat : `pandas.core.frame.DataFrame`
620 The catalog of fake sources to be input
621 image : `lsst.afw.image.exposure.exposure.ExposureF`
622 The image into which the fake sources should be added
623 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
624 Photometric calibration to be used to calibrate the fake sources
628 starImages : `generator`
629 A generator of tuples of `lsst.afw.image.ImageF` of fake stars and
630 `lsst.geom.Point2D` of their locations.
634 To take a given magnitude and translate to the number of counts in the image
635 we use photoCalib.magnitudeToInstFlux, which returns the instrumental flux for the
636 given calibration radius used in the photometric calibration step.
637 Thus `calibFluxRadius` should be set to this same radius so that we can normalize
638 the PSF model to the correct instrumental flux within calibFluxRadius.
641 self.log.
info(
"Making %d fake star images" % len(fakeCat))
643 for (index, row)
in fakeCat.iterrows():
649 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
650 starIm = psf.computeImage(xy)
651 starIm /= correctedFlux
653 except InvalidParameterError:
654 self.log.
info(
"Star at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
658 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
663 yield ((starIm.convertF(), xy))
666 """Remove rows from the fakes catalog which have HLR = 0 for either the buldge or disk component,
667 also remove galaxies that have Sersic index outside the galsim min and max
668 allowed (0.3 <= n <= 6.2).
672 fakeCat : `pandas.core.frame.DataFrame`
673 The catalog of fake sources to be input
674 starCheckVal : `str`, `bytes` or `int`
675 The value that is set in the sourceType column to specifiy an object is a star.
679 fakeCat : `pandas.core.frame.DataFrame`
680 The input catalog of fake sources but with the bad objects removed
684 If the config option sourceSelectionColName is set then only objects with this column set to True
688 rowsToKeep = (((fakeCat[self.config.bulgeHLR] != 0.0) & (fakeCat[self.config.diskHLR] != 0.0))
689 | (fakeCat[self.config.sourceType] == starCheckVal))
690 numRowsNotUsed = len(fakeCat) - len(np.where(rowsToKeep)[0])
691 self.log.
info(
"Removing %d rows with HLR = 0 for either the bulge or disk" % numRowsNotUsed)
692 fakeCat = fakeCat[rowsToKeep]
694 minN = galsim.Sersic._minimum_n
695 maxN = galsim.Sersic._maximum_n
696 rowsWithGoodSersic = (((fakeCat[self.config.nBulge] >= minN) & (fakeCat[self.config.nBulge] <= maxN)
697 & (fakeCat[self.config.nDisk] >= minN) & (fakeCat[self.config.nDisk] <= maxN))
698 | (fakeCat[self.config.sourceType] == starCheckVal))
699 numRowsNotUsed = len(fakeCat) - len(np.where(rowsWithGoodSersic)[0])
700 self.log.
info(
"Removing %d rows of galaxies with nBulge or nDisk outside of %0.2f <= n <= %0.2f" %
701 (numRowsNotUsed, minN, maxN))
702 fakeCat = fakeCat[rowsWithGoodSersic]
704 if self.config.doSubSelectSources:
706 rowsSelected = (fakeCat[self.config.sourceSelectionColName])
708 raise KeyError(
"Given column, %s, for source selection not found." %
709 self.config.sourceSelectionColName)
710 numRowsNotUsed = len(fakeCat) - len(rowsSelected)
711 self.log.
info(
"Removing %d rows which were not designated as template sources" % numRowsNotUsed)
712 fakeCat = fakeCat[rowsSelected]
717 """Add the fake sources to the given image
721 image : `lsst.afw.image.exposure.exposure.ExposureF`
722 The image into which the fake sources should be added
723 fakeImages : `typing.Iterator` [`tuple` ['lsst.afw.image.ImageF`, `lsst.geom.Point2d`]]
724 An iterator of tuples that contains (or generates) images of fake sources,
725 and the locations they are to be inserted at.
727 The type (star/galaxy) of fake sources input
731 image : `lsst.afw.image.exposure.exposure.ExposureF`
735 Uses the x, y information in the ``fakeCat`` to position an image of the fake interpolated onto the
736 pixel grid of the image. Sets the ``FAKE`` mask plane for the pixels added with the fake source.
739 imageBBox = image.getBBox()
740 imageMI = image.maskedImage
742 for (fakeImage, xy)
in fakeImages:
743 X0 = xy.getX() - fakeImage.getWidth()/2 + 0.5
744 Y0 = xy.getY() - fakeImage.getHeight()/2 + 0.5
745 self.log.
debug(
"Adding fake source at %d, %d" % (xy.getX(), xy.getY()))
746 if sourceType ==
"galaxy":
749 interpFakeImage = fakeImage
751 interpFakeImBBox = interpFakeImage.getBBox()
752 interpFakeImBBox.clip(imageBBox)
754 if interpFakeImBBox.getArea() > 0:
755 imageMIView = imageMI[interpFakeImBBox]
756 clippedFakeImage = interpFakeImage[interpFakeImBBox]
757 clippedFakeImageMI = afwImage.MaskedImageF(clippedFakeImage)
758 clippedFakeImageMI.mask.set(self.bitmask)
759 imageMIView += clippedFakeImageMI
763 def _getMetadataName(self):
764 """Disable metadata writing"""
A floating-point coordinate rectangle geometry.
Point in an unspecified spherical coordinate system.
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
std::shared_ptr< ImageT > offsetImage(ImageT const &image, float dx, float dy, std::string const &algorithmName="lanczos5", unsigned int buffer=0)
Return an image offset by (dx, dy) using the specified algorithm.
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def mkFakeStars(self, fakeCat, band, photoCalib, psf, image)
def trimFakeCat(self, fakeCat, image, wcs)
def addPixCoords(self, fakeCat, wcs)
def mkFakeGalsimGalaxies(self, fakeCat, band, photoCalib, pixelScale, psf, image)
def cleanCat(self, fakeCat, starCheckVal)
def addFakeSources(self, image, fakeImages, sourceType)