604 def __init__(self, initial_stars_schema=None, **kwargs):
606 self.makeSubtask(
"snap_combine")
609 self.makeSubtask(
"install_simple_psf")
610 self.makeSubtask(
"psf_repair")
611 self.makeSubtask(
"psf_subtract_background")
616 doc=
"PSF max value.",
618 afwTable.CoordKey.addErrorFields(self.
psf_schema)
619 self.makeSubtask(
"psf_detection", schema=self.
psf_schema)
620 self.makeSubtask(
"psf_source_measurement", schema=self.
psf_schema)
621 self.makeSubtask(
"psf_measure_psf", schema=self.
psf_schema)
622 self.makeSubtask(
"psf_normalized_calibration_flux", schema=self.
psf_schema)
624 self.makeSubtask(
"measure_aperture_correction", schema=self.
psf_schema)
625 self.makeSubtask(
"astrometry", schema=self.
psf_schema)
628 if initial_stars_schema
is None:
629 initial_stars_schema = afwTable.SourceTable.makeMinimalSchema()
633 self.
psf_fields = (
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved",
634 "calib_astrometry_used",
636 "apcorr_slot_CalibFlux_used",
"apcorr_base_GaussianFlux_used",
637 "apcorr_base_PsfFlux_used",)
640 initial_stars_schema.addField(item.getField())
641 id_type = self.
psf_schema[
"id"].asField().getTypeString()
642 psf_max_value_type = self.
psf_schema[
'psf_max_value'].asField().getTypeString()
643 initial_stars_schema.addField(
"psf_id",
645 doc=
"id of this source in psf_stars; 0 if there is no match.")
646 initial_stars_schema.addField(
"psf_max_value",
647 type=psf_max_value_type,
648 doc=
"Maximum value in the star image used to train PSF.")
650 afwTable.CoordKey.addErrorFields(initial_stars_schema)
651 self.makeSubtask(
"star_detection", schema=initial_stars_schema)
652 self.makeSubtask(
"star_sky_sources", schema=initial_stars_schema)
653 self.makeSubtask(
"star_deblend", schema=initial_stars_schema)
654 self.makeSubtask(
"star_measurement", schema=initial_stars_schema)
655 self.makeSubtask(
"star_normalized_calibration_flux", schema=initial_stars_schema)
657 self.makeSubtask(
"star_apply_aperture_correction", schema=initial_stars_schema)
658 self.makeSubtask(
"star_catalog_calculation", schema=initial_stars_schema)
659 self.makeSubtask(
"star_set_primary_flags", schema=initial_stars_schema, isSingleFrame=
True)
660 self.makeSubtask(
"star_selector")
661 self.makeSubtask(
"photometry", schema=initial_stars_schema)
662 self.makeSubtask(
"compute_summary_stats")
677 inputs = butlerQC.get(inputRefs)
678 exposures = inputs.pop(
"exposures")
680 id_generator = self.config.id_generator.apply(butlerQC.quantum.dataId)
683 dataIds=[ref.datasetRef.dataId
for ref
in inputRefs.astrometry_ref_cat],
684 refCats=inputs.pop(
"astrometry_ref_cat"),
685 name=self.config.connections.astrometry_ref_cat,
686 config=self.config.astrometry_ref_loader, log=self.log)
687 self.astrometry.setRefObjLoader(astrometry_loader)
690 dataIds=[ref.datasetRef.dataId
for ref
in inputRefs.photometry_ref_cat],
691 refCats=inputs.pop(
"photometry_ref_cat"),
692 name=self.config.connections.photometry_ref_cat,
693 config=self.config.photometry_ref_loader, log=self.log)
694 self.photometry.match.setRefObjLoader(photometry_loader)
696 if self.config.do_illumination_correction:
697 background_flat = inputs.pop(
"background_flat")
698 illumination_correction = inputs.pop(
"illumination_correction")
700 background_flat =
None
701 illumination_correction =
None
704 assert not inputs,
"runQuantum got more inputs than expected"
708 result = pipeBase.Struct(
710 stars_footprints=
None,
711 psf_stars_footprints=
None,
712 background_to_photometric_ratio=
None,
718 id_generator=id_generator,
719 background_flat=background_flat,
720 illumination_correction=illumination_correction,
722 except pipeBase.AlgorithmError
as e:
723 error = pipeBase.AnnotatedPartialOutputsError.annotate(
727 result.psf_stars_footprints,
728 result.stars_footprints,
731 butlerQC.put(result, outputRefs)
734 butlerQC.put(result, outputRefs)
743 background_flat=None,
744 illumination_correction=None,
746 """Find stars and perform psf measurement, then do a deeper detection
747 and measurement and calibrate astrometry and photometry from that.
751 exposures : `lsst.afw.image.Exposure` or `list` [`lsst.afw.image.Exposure`]
752 Post-ISR exposure(s), with an initial WCS, VisitInfo, and Filter.
753 Modified in-place during processing if only one is passed.
754 If two exposures are passed, treat them as snaps and combine
755 before doing further processing.
756 id_generator : `lsst.meas.base.IdGenerator`, optional
757 Object that generates source IDs and provides random seeds.
758 result : `lsst.pipe.base.Struct`, optional
759 Result struct that is modified to allow saving of partial outputs
760 for some failure conditions. If the task completes successfully,
761 this is also returned.
762 background_flat : `lsst.afw.image.Exposure`, optional
763 Background flat-field image.
764 illumination_correction : `lsst.afw.image.Exposure`, optional
765 Illumination correction image.
769 result : `lsst.pipe.base.Struct`
770 Results as a struct with attributes:
773 Calibrated exposure, with pixels in nJy units.
774 (`lsst.afw.image.Exposure`)
776 Stars that were used to calibrate the exposure, with
777 calibrated fluxes and magnitudes.
778 (`astropy.table.Table`)
780 Footprints of stars that were used to calibrate the exposure.
781 (`lsst.afw.table.SourceCatalog`)
783 Stars that were used to determine the image PSF.
784 (`astropy.table.Table`)
785 ``psf_stars_footprints``
786 Footprints of stars that were used to determine the image PSF.
787 (`lsst.afw.table.SourceCatalog`)
789 Background that was fit to the exposure when detecting
790 ``stars``. (`lsst.afw.math.BackgroundList`)
791 ``applied_photo_calib``
792 Photometric calibration that was fit to the star catalog and
793 applied to the exposure. (`lsst.afw.image.PhotoCalib`)
794 This is `None` if ``config.do_calibrate_pixels`` is `False`.
795 ``astrometry_matches``
796 Reference catalog stars matches used in the astrometric fit.
797 (`list` [`lsst.afw.table.ReferenceMatch`] or `lsst.afw.table.BaseCatalog`)
798 ``photometry_matches``
799 Reference catalog stars matches used in the photometric fit.
800 (`list` [`lsst.afw.table.ReferenceMatch`] or `lsst.afw.table.BaseCatalog`)
803 result = pipeBase.Struct()
804 if id_generator
is None:
807 result.exposure = self.snap_combine.run(exposures).exposure
809 self.log.info(
"Initial PhotoCalib: %s", result.exposure.getPhotoCalib())
811 result.exposure.metadata[
"LSST CALIB ILLUMCORR APPLIED"] =
False
814 if self.config.do_illumination_correction:
815 if not result.exposure.metadata.get(
"LSST ISR FLAT APPLIED",
False):
816 raise pipeBase.InvalidQuantumError(
817 "Cannot use do_illumination_correction with an image that has not had a flat applied",
820 result.background =
None
821 summary_stat_catalog =
None
827 have_fit_astrometry =
False
828 have_fit_photometry =
False
833 illumination_correction,
836 result.psf_stars_footprints, result.background, _ = self.
_compute_psf(
839 background_to_photometric_ratio=result.background_to_photometric_ratio,
846 if result.psf_stars_footprints[
"slot_Centroid_flag"].all():
847 psf_shape = result.exposure.psf.computeShape(result.exposure.psf.getAveragePosition())
849 n_sources=len(result.psf_stars_footprints),
850 psf_shape_ixx=psf_shape.getIxx(),
851 psf_shape_iyy=psf_shape.getIyy(),
852 psf_shape_ixy=psf_shape.getIxy(),
853 psf_size=psf_shape.getDeterminantRadius(),
857 result.psf_stars = result.psf_stars_footprints.asAstropy()
860 result.exposure, result.psf_stars_footprints
862 self.metadata[
"astrometry_matches_count"] = len(astrometry_matches)
863 if "astrometry_matches" in self.config.optional_outputs:
866 result.psf_stars = result.psf_stars_footprints.asAstropy()
873 background_to_photometric_ratio=result.background_to_photometric_ratio,
880 sourceList=result.stars_footprints,
881 include_covariance=self.config.do_include_astrometric_errors
884 summary_stat_catalog = result.stars_footprints
885 result.stars = result.stars_footprints.asAstropy()
886 self.metadata[
"star_count"] = np.sum(~result.stars[
"sky_source"])
891 self.astrometry.check(result.exposure, result.stars_footprints, len(astrometry_matches))
892 result.stars = result.stars_footprints.asAstropy()
893 have_fit_astrometry =
True
895 result.stars_footprints, photometry_matches, \
896 photometry_meta, photo_calib = self.
_fit_photometry(result.exposure, result.stars_footprints)
897 have_fit_photometry =
True
898 self.metadata[
"photometry_matches_count"] = len(photometry_matches)
900 result.stars = result.stars_footprints.asAstropy()
904 summary_stat_catalog = result.stars_footprints
905 if "photometry_matches" in self.config.optional_outputs:
908 except pipeBase.AlgorithmError:
910 result.exposure.setPsf(
None)
911 if not have_fit_astrometry:
912 result.exposure.setWcs(
None)
913 if not have_fit_photometry:
914 result.exposure.setPhotoCalib(
None)
921 self.
_summarize(result.exposure, summary_stat_catalog, result.background)
924 self.
_summarize(result.exposure, summary_stat_catalog, result.background)
926 if self.config.do_calibrate_pixels:
930 background_to_photometric_ratio=result.background_to_photometric_ratio,
932 result.applied_photo_calib = photo_calib
934 result.applied_photo_calib =
None
936 if self.config.run_sattle:
940 populate_sattle_visit_cache(result.exposure.getInfo().getVisitInfo(),
941 historical=self.config.sattle_historical)
942 self.log.info(
'Successfully triggered load of sattle visit cache')
943 except requests.exceptions.HTTPError:
944 self.log.exception(
"Sattle visit cache update failed; continuing with image processing")
992 def _compute_psf(self, exposure, id_generator, background_to_photometric_ratio=None):
993 """Find bright sources detected on an exposure and fit a PSF model to
994 them, repairing likely cosmic rays before detection.
996 Repair, detect, measure, and compute PSF twice, to ensure the PSF
997 model does not include contributions from cosmic rays.
1001 exposure : `lsst.afw.image.Exposure`
1002 Exposure to detect and measure bright stars on.
1003 id_generator : `lsst.meas.base.IdGenerator`
1004 Object that generates source IDs and provides random seeds.
1005 background_to_photometric_ratio : `lsst.afw.image.Image`, optional
1006 Image to convert photometric-flattened image to
1007 background-flattened image.
1011 sources : `lsst.afw.table.SourceCatalog`
1012 Catalog of detected bright sources.
1013 background : `lsst.afw.math.BackgroundList`
1014 Background that was fit to the exposure during detection.
1015 cell_set : `lsst.afw.math.SpatialCellSet`
1016 PSF candidates returned by the psf determiner.
1018 def log_psf(msg, addToMetadata=False):
1019 """Log the parameters of the psf and background, with a prepended
1020 message. There is also the option to add the PSF sigma to the task
1026 Message to prepend the log info with.
1027 addToMetadata : `bool`, optional
1028 Whether to add the final psf sigma value to the task metadata
1029 (the default is False).
1031 position = exposure.psf.getAveragePosition()
1032 sigma = exposure.psf.computeShape(position).getDeterminantRadius()
1033 dimensions = exposure.psf.computeImage(position).getDimensions()
1034 median_background = np.median(background.getImage().array)
1035 self.log.info(
"%s sigma=%0.4f, dimensions=%s; median background=%0.2f",
1036 msg, sigma, dimensions, median_background)
1038 self.metadata[
"final_psf_sigma"] = sigma
1040 self.log.info(
"First pass detection with Guassian PSF FWHM=%s pixels",
1041 self.config.install_simple_psf.fwhm)
1042 self.install_simple_psf.run(exposure=exposure)
1044 background = self.psf_subtract_background.run(
1046 backgroundToPhotometricRatio=background_to_photometric_ratio,
1048 log_psf(
"Initial PSF:")
1049 self.psf_repair.run(exposure=exposure, keepCRs=
True)
1051 table = afwTable.SourceTable.make(self.
psf_schema, id_generator.make_table_id_factory())
1054 detections = self.psf_detection.run(
1057 background=background,
1058 backgroundToPhotometricRatio=background_to_photometric_ratio,
1060 self.metadata[
"initial_psf_positive_footprint_count"] = detections.numPos
1061 self.metadata[
"initial_psf_negative_footprint_count"] = detections.numNeg
1062 self.metadata[
"initial_psf_positive_peak_count"] = detections.numPosPeaks
1063 self.metadata[
"initial_psf_negative_peak_count"] = detections.numNegPeaks
1064 self.psf_source_measurement.run(detections.sources, exposure)
1065 psf_result = self.psf_measure_psf.run(exposure=exposure, sources=detections.sources)
1068 self.install_simple_psf.run(exposure=exposure)
1070 log_psf(
"Rerunning with simple PSF:")
1078 self.psf_repair.run(exposure=exposure, keepCRs=
True)
1081 detections = self.psf_detection.run(
1084 background=background,
1085 backgroundToPhotometricRatio=background_to_photometric_ratio,
1087 self.metadata[
"simple_psf_positive_footprint_count"] = detections.numPos
1088 self.metadata[
"simple_psf_negative_footprint_count"] = detections.numNeg
1089 self.metadata[
"simple_psf_positive_peak_count"] = detections.numPosPeaks
1090 self.metadata[
"simple_psf_negative_peak_count"] = detections.numNegPeaks
1091 self.psf_source_measurement.run(detections.sources, exposure)
1092 psf_result = self.psf_measure_psf.run(exposure=exposure, sources=detections.sources)
1094 log_psf(
"Final PSF:", addToMetadata=
True)
1097 self.psf_repair.run(exposure=exposure)
1099 self.psf_source_measurement.run(detections.sources, exposure)
1103 return detections.sources, background, psf_result.cellSet
1135 def _find_stars(self, exposure, background, id_generator, background_to_photometric_ratio=None):
1136 """Detect stars on an exposure that has a PSF model, and measure their
1137 PSF, circular aperture, compensated gaussian fluxes.
1141 exposure : `lsst.afw.image.Exposure`
1142 Exposure to detect and measure stars on.
1143 background : `lsst.afw.math.BackgroundList`
1144 Background that was fit to the exposure during detection;
1145 modified in-place during subsequent detection.
1146 id_generator : `lsst.meas.base.IdGenerator`
1147 Object that generates source IDs and provides random seeds.
1148 background_to_photometric_ratio : `lsst.afw.image.Image`, optional
1149 Image to convert photometric-flattened image to
1150 background-flattened image.
1154 stars : `SourceCatalog`
1155 Sources that are very likely to be stars, with a limited set of
1156 measurements performed on them.
1159 id_generator.make_table_id_factory())
1162 detections = self.star_detection.run(
1165 background=background,
1166 backgroundToPhotometricRatio=background_to_photometric_ratio,
1168 sources = detections.sources
1169 self.star_sky_sources.run(exposure.mask, id_generator.catalog_id, sources)
1171 n_sky_sources = np.sum(sources[
"sky_source"])
1172 if (self.config.do_downsample_footprints
1173 and (len(sources) - n_sky_sources) > self.config.downsample_max_footprints):
1174 if exposure.info.id
is None:
1175 self.log.warning(
"Exposure does not have a proper id; using 0 seed for downsample.")
1178 seed = exposure.info.id & 0xFFFFFFFF
1180 gen = np.random.RandomState(seed)
1183 indices = np.arange(len(sources))[~sources[
"sky_source"]]
1184 indices = gen.choice(
1186 size=self.config.downsample_max_footprints,
1189 skyIndices, = np.where(sources[
"sky_source"])
1190 indices = np.concatenate((indices, skyIndices))
1192 self.log.info(
"Downsampling from %d to %d non-sky-source footprints.", len(sources), len(indices))
1194 sel = np.zeros(len(sources), dtype=bool)
1196 sources = sources[sel]
1199 self.star_deblend.run(exposure=exposure, sources=sources)
1202 if not sources.isContiguous():
1203 sources = sources.copy(deep=
True)
1206 self.star_measurement.run(sources, exposure)
1207 self.metadata[
"post_deblend_source_count"] = np.sum(~sources[
"sky_source"])
1208 self.metadata[
"saturated_source_count"] = np.sum(sources[
"base_PixelFlags_flag_saturated"])
1209 self.metadata[
"bad_source_count"] = np.sum(sources[
"base_PixelFlags_flag_bad"])
1214 self.star_normalized_calibration_flux.run(exposure=exposure, catalog=sources)
1215 self.star_apply_aperture_correction.run(sources, exposure.apCorrMap)
1216 self.star_catalog_calculation.run(sources)
1217 self.star_set_primary_flags.run(sources)
1219 result = self.star_selector.run(sources)
1221 if not result.sourceCat.isContiguous():
1222 return result.sourceCat.copy(deep=
True)
1224 return result.sourceCat
1227 """Match calibration stars to psf stars, to identify which were psf
1228 candidates, and which were used or reserved during psf measurement
1229 and the astrometric fit.
1233 psf_stars : `lsst.afw.table.SourceCatalog`
1234 PSF candidate stars that were sent to the psf determiner and
1235 used in the astrometric fit. Used to populate psf and astrometry
1236 related flag fields.
1237 stars : `lsst.afw.table.SourceCatalog`
1238 Stars that will be used for calibration; psf-related fields will
1239 be updated in-place.
1243 This code was adapted from CalibrateTask.copyIcSourceFields().
1247 control.findOnlyClosest =
False
1249 deblend_key = stars.schema[
"deblend_nChild"].asKey()
1250 matches = [m
for m
in matches
if m[1].get(deblend_key) == 0]
1257 for match_psf, match_stars, d
in matches:
1258 match = best.get(match_psf.getId())
1259 if match
is None or d <= match[2]:
1260 best[match_psf.getId()] = (match_psf, match_stars, d)
1261 matches = list(best.values())
1263 ids = np.array([(match_psf.getId(), match_stars.getId())
for match_psf, match_stars, d
in matches]).T
1265 if (n_matches := len(matches)) == 0:
1268 self.log.info(
"%d psf/astrometry stars out of %d matched %d calib stars",
1269 n_matches, len(psf_stars), len(stars))
1270 self.metadata[
"matched_psf_star_count"] = n_matches
1275 n_unique = len(set(m[1].getId()
for m
in matches))
1276 if n_unique != n_matches:
1277 self.log.warning(
"%d psf_stars matched only %d stars", n_matches, n_unique)
1280 idx_psf_stars = np.searchsorted(psf_stars[
"id"], ids[0])
1281 idx_stars = np.searchsorted(stars[
"id"], ids[1])
1283 result = np.zeros(len(stars), dtype=bool)
1284 result[idx_stars] = psf_stars[field][idx_psf_stars]
1285 stars[field] = result
1286 stars[
'psf_id'][idx_stars] = psf_stars[
'id'][idx_psf_stars]
1287 stars[
'psf_max_value'][idx_stars] = psf_stars[
'psf_max_value'][idx_psf_stars]
1344 """Apply the photometric model attached to the exposure to the
1345 exposure's pixels and an associated background model.
1349 exposure : `lsst.afw.image.Exposure`
1350 Exposure with the target `lsst.afw.image.PhotoCalib` attached.
1351 On return, pixel values will be calibrated and an identity
1352 photometric transform will be attached.
1353 background : `lsst.afw.math.BackgroundList`
1354 Background model to convert to nanojansky units in place.
1355 background_to_photometric_ratio : `lsst.afw.image.Image`, optional
1356 Image to convert photometric-flattened image to
1357 background-flattened image.
1359 photo_calib = exposure.getPhotoCalib()
1360 exposure.maskedImage = photo_calib.calibrateImage(exposure.maskedImage)
1362 photo_calib.getCalibrationErr(),
1363 bbox=exposure.getBBox())
1364 exposure.setPhotoCalib(identity)
1365 exposure.metadata[
"BUNIT"] =
"nJy"
1367 assert photo_calib._isConstant, \
1368 "Background calibration assumes a constant PhotoCalib; PhotoCalTask should always return that."
1370 for bg
in background:
1372 binned_image = bg[0].getStatsImage()
1373 binned_image *= photo_calib.getCalibrationMean()