LSST Applications g04e9c324dd+8c5ae1fdc5,g134cb467dc+1b3060144d,g18429d2f64+f642bf4753,g199a45376c+0ba108daf9,g1fd858c14a+2dcf163641,g262e1987ae+7b8c96d2ca,g29ae962dfc+3bd6ecb08a,g2cef7863aa+aef1011c0b,g35bb328faa+8c5ae1fdc5,g3fd5ace14f+53e1a9e7c5,g4595892280+fef73a337f,g47891489e3+2efcf17695,g4d44eb3520+642b70b07e,g53246c7159+8c5ae1fdc5,g67b6fd64d1+2efcf17695,g67fd3c3899+b70e05ef52,g74acd417e5+317eb4c7d4,g786e29fd12+668abc6043,g87389fa792+8856018cbb,g89139ef638+2efcf17695,g8d7436a09f+3be3c13596,g8ea07a8fe4+9f5ccc88ac,g90f42f885a+a4e7b16d9b,g97be763408+ad77d7208f,g9dd6db0277+b70e05ef52,ga681d05dcb+a3f46e7fff,gabf8522325+735880ea63,gac2eed3f23+2efcf17695,gb89ab40317+2efcf17695,gbf99507273+8c5ae1fdc5,gd8ff7fe66e+b70e05ef52,gdab6d2f7ff+317eb4c7d4,gdc713202bf+b70e05ef52,gdfd2d52018+b10e285e0f,ge365c994fd+310e8507c4,ge410e46f29+2efcf17695,geaed405ab2+562b3308c0,gffca2db377+8c5ae1fdc5,w.2025.35
LSST Data Management Base Package
Loading...
Searching...
No Matches
detectAndMeasure.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import numpy as np
23import requests
24import os
25
26import lsst.afw.detection as afwDetection
27import lsst.afw.image as afwImage
28import lsst.afw.math as afwMath
29import lsst.afw.table as afwTable
30import lsst.daf.base as dafBase
31import lsst.geom
32from lsst.ip.diffim.utils import (evaluateMaskFraction, computeDifferenceImageMetrics,
33 populate_sattle_visit_cache)
34from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask, SetPrimaryFlagsTask, MaskStreaksTask
35from lsst.meas.algorithms import FindGlintTrailsTask
36from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
39import lsst.meas.extensions.shapeHSM
40import lsst.pex.config as pexConfig
41from lsst.pex.exceptions import InvalidParameterError
42import lsst.pipe.base as pipeBase
43import lsst.utils
44from lsst.utils.timer import timeMethod
45
46from . import DipoleFitTask
47
48__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
49 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
50
51
52class BadSubtractionError(pipeBase.AlgorithmError):
53 """Raised when the residuals in footprints of stars used to compute the
54 psf-matching kernel exceeds the configured maximum.
55 """
56 def __init__(self, *, ratio, threshold):
57 msg = ("The ratio of residual power in source footprints on the"
58 " difference image to the power in the footprints on the"
59 f" science image was {ratio}, which exceeds the maximum"
60 f" threshold of {threshold}")
61 super().__init__(msg)
62 self.ratio = ratio
63 self.threshold = threshold
64
65 @property
66 def metadata(self):
67 return {"ratio": self.ratio,
68 "threshold": self.threshold
69 }
70
71
72class NoDiaSourcesError(pipeBase.AlgorithmError):
73 """Raised when there are no diaSources detected on an image difference.
74 """
75 def __init__(self):
76 msg = ("No diaSources detected!")
77 super().__init__(msg)
78
79 @property
80 def metadata(self):
81 return {}
82
83
84class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
85 dimensions=("instrument", "visit", "detector"),
86 defaultTemplates={"coaddName": "deep",
87 "warpTypeSuffix": "",
88 "fakesType": ""}):
89 science = pipeBase.connectionTypes.Input(
90 doc="Input science exposure.",
91 dimensions=("instrument", "visit", "detector"),
92 storageClass="ExposureF",
93 name="{fakesType}calexp"
94 )
95 matchedTemplate = pipeBase.connectionTypes.Input(
96 doc="Warped and PSF-matched template used to create the difference image.",
97 dimensions=("instrument", "visit", "detector"),
98 storageClass="ExposureF",
99 name="{fakesType}{coaddName}Diff_matchedExp",
100 )
101 difference = pipeBase.connectionTypes.Input(
102 doc="Result of subtracting template from science.",
103 dimensions=("instrument", "visit", "detector"),
104 storageClass="ExposureF",
105 name="{fakesType}{coaddName}Diff_differenceTempExp",
106 )
107 kernelSources = pipeBase.connectionTypes.Input(
108 doc="Final selection of sources used for psf matching.",
109 dimensions=("instrument", "visit", "detector"),
110 storageClass="SourceCatalog",
111 name="{fakesType}{coaddName}Diff_psfMatchSources"
112 )
113 outputSchema = pipeBase.connectionTypes.InitOutput(
114 doc="Schema (as an example catalog) for output DIASource catalog.",
115 storageClass="SourceCatalog",
116 name="{fakesType}{coaddName}Diff_diaSrc_schema",
117 )
118 diaSources = pipeBase.connectionTypes.Output(
119 doc="Detected diaSources on the difference image.",
120 dimensions=("instrument", "visit", "detector"),
121 storageClass="SourceCatalog",
122 name="{fakesType}{coaddName}Diff_diaSrc",
123 )
124 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
125 doc="Difference image with detection mask plane filled in.",
126 dimensions=("instrument", "visit", "detector"),
127 storageClass="ExposureF",
128 name="{fakesType}{coaddName}Diff_differenceExp",
129 )
130 differenceBackground = pipeBase.connectionTypes.Output(
131 doc="Background model that was subtracted from the difference image.",
132 dimensions=("instrument", "visit", "detector"),
133 storageClass="Background",
134 name="difference_background",
135 )
136 maskedStreaks = pipeBase.connectionTypes.Output(
137 doc='Catalog of streak fit parameters for the difference image.',
138 storageClass="ArrowNumpyDict",
139 dimensions=("instrument", "visit", "detector"),
140 name="{fakesType}{coaddName}Diff_streaks",
141 )
142 glintTrailInfo = pipeBase.connectionTypes.Output(
143 doc='Dict of fit parameters for glint trails in the catalog.',
144 storageClass="ArrowNumpyDict",
145 dimensions=("instrument", "visit", "detector"),
146 name="trailed_glints",
147 )
148
149 def __init__(self, *, config):
150 super().__init__(config=config)
151 if not (self.config.writeStreakInfo and self.config.doMaskStreaks):
152 self.outputs.remove("maskedStreaks")
153 if not (self.config.doSubtractBackground and self.config.doWriteBackground):
154 self.outputs.remove("differenceBackground")
155 if not (self.config.writeGlintInfo):
156 self.outputs.remove("glintTrailInfo")
157
158
159class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
160 pipelineConnections=DetectAndMeasureConnections):
161 """Config for DetectAndMeasureTask
162 """
163 doMerge = pexConfig.Field(
164 dtype=bool,
165 default=True,
166 doc="Merge positive and negative diaSources with grow radius "
167 "set by growFootprint"
168 )
169 doForcedMeasurement = pexConfig.Field(
170 dtype=bool,
171 default=True,
172 doc="Force photometer diaSource locations on PVI?"
173 )
174 doAddMetrics = pexConfig.Field(
175 dtype=bool,
176 default=False,
177 doc="Add columns to the source table to hold analysis metrics?"
178 )
179 doSubtractBackground = pexConfig.Field(
180 dtype=bool,
181 doc="Subtract a background model from the image before detection?",
182 default=True,
183 )
184 doWriteBackground = pexConfig.Field(
185 dtype=bool,
186 doc="Persist the fitted background model?",
187 default=False,
188 )
189 subtractInitialBackground = pexConfig.ConfigurableField(
191 doc="Task to perform intial background subtraction, before first detection pass.",
192 )
193 subtractFinalBackground = pexConfig.ConfigurableField(
195 doc="Task to perform final background subtraction, after first detection pass.",
196 )
197 detection = pexConfig.ConfigurableField(
198 target=SourceDetectionTask,
199 doc="Final source detection for diaSource measurement",
200 )
201 streakDetection = pexConfig.ConfigurableField(
202 target=SourceDetectionTask,
203 doc="Separate source detection used only for streak masking",
204 )
205 doDeblend = pexConfig.Field(
206 dtype=bool,
207 default=False,
208 doc="Deblend DIASources after detection?"
209 )
210 deblend = pexConfig.ConfigurableField(
212 doc="Task to split blended sources into their components."
213 )
214 measurement = pexConfig.ConfigurableField(
215 target=DipoleFitTask,
216 doc="Task to measure sources on the difference image.",
217 )
218 doApCorr = lsst.pex.config.Field(
219 dtype=bool,
220 default=True,
221 doc="Run subtask to apply aperture corrections"
222 )
224 target=ApplyApCorrTask,
225 doc="Task to apply aperture corrections"
226 )
227 forcedMeasurement = pexConfig.ConfigurableField(
228 target=ForcedMeasurementTask,
229 doc="Task to force photometer science image at diaSource locations.",
230 )
231 growFootprint = pexConfig.Field(
232 dtype=int,
233 default=2,
234 doc="Grow positive and negative footprints by this many pixels before merging"
235 )
236 diaSourceMatchRadius = pexConfig.Field(
237 dtype=float,
238 default=0.5,
239 doc="Match radius (in arcseconds) for DiaSource to Source association"
240 )
241 doSkySources = pexConfig.Field(
242 dtype=bool,
243 default=False,
244 doc="Generate sky sources?",
245 )
246 skySources = pexConfig.ConfigurableField(
247 target=SkyObjectsTask,
248 doc="Generate sky sources",
249 )
250 doMaskStreaks = pexConfig.Field(
251 dtype=bool,
252 default=True,
253 doc="Turn on streak masking",
254 )
255 maskStreaks = pexConfig.ConfigurableField(
256 target=MaskStreaksTask,
257 doc="Subtask for masking streaks. Only used if doMaskStreaks is True. "
258 "Adds a mask plane to an exposure, with the mask plane name set by streakMaskName.",
259 )
260 streakBinFactor = pexConfig.Field(
261 dtype=int,
262 default=4,
263 doc="Bin scale factor to use when rerunning detection for masking streaks. "
264 "Only used if doMaskStreaks is True.",
265 )
266 writeStreakInfo = pexConfig.Field(
267 dtype=bool,
268 default=False,
269 doc="Record the parameters of any detected streaks. For LSST, this should be turned off except for "
270 "development work."
271 )
272 findGlints = pexConfig.ConfigurableField(
273 target=FindGlintTrailsTask,
274 doc="Subtask for finding glint trails, usually caused by satellites or debris."
275 )
276 writeGlintInfo = pexConfig.Field(
277 dtype=bool,
278 default=True,
279 doc="Record the parameters of any detected glint trails."
280 )
281 setPrimaryFlags = pexConfig.ConfigurableField(
282 target=SetPrimaryFlagsTask,
283 doc="Task to add isPrimary and deblending-related flags to the catalog."
284 )
285 badSourceFlags = lsst.pex.config.ListField(
286 dtype=str,
287 doc="Sources with any of these flags set are removed before writing the output catalog.",
288 default=("base_PixelFlags_flag_offimage",
289 "base_PixelFlags_flag_interpolatedCenterAll",
290 "base_PixelFlags_flag_badCenterAll",
291 "base_PixelFlags_flag_edgeCenterAll",
292 "base_PixelFlags_flag_nodataCenterAll",
293 "base_PixelFlags_flag_saturatedCenterAll",
294 ),
295 )
296 clearMaskPlanes = lsst.pex.config.ListField(
297 dtype=str,
298 doc="Mask planes to clear before running detection.",
299 default=("DETECTED", "DETECTED_NEGATIVE", "NOT_DEBLENDED", "STREAK"),
300 )
301 raiseOnBadSubtractionRatio = pexConfig.Field(
302 dtype=bool,
303 default=True,
304 doc="Raise an error if the ratio of power in detected footprints"
305 " on the difference image to the power in footprints on the science"
306 " image exceeds ``badSubtractionRatioThreshold``",
307 )
308 badSubtractionRatioThreshold = pexConfig.Field(
309 dtype=float,
310 default=0.2,
311 doc="Maximum ratio of power in footprints on the difference image to"
312 " the same footprints on the science image."
313 "Only used if ``raiseOnBadSubtractionRatio`` is set",
314 )
315 badSubtractionVariationThreshold = pexConfig.Field(
316 dtype=float,
317 default=0.4,
318 doc="Maximum standard deviation of the ratio of power in footprints on"
319 " the difference image to the same footprints on the science image."
320 "Only used if ``raiseOnBadSubtractionRatio`` is set",
321 )
322 raiseOnNoDiaSources = pexConfig.Field(
323 dtype=bool,
324 default=True,
325 doc="Raise an algorithm error if no diaSources are detected.",
326 )
327 run_sattle = pexConfig.Field(
328 dtype=bool,
329 default=False,
330 doc="If true, dia source bounding boxes will be sent for verification"
331 "to the sattle service."
332 )
333 sattle_historical = pexConfig.Field(
334 dtype=bool,
335 default=False,
336 doc="If re-running a pipeline that requires sattle, this should be set "
337 "to True. This will populate sattle's cache with the historic data "
338 "closest in time to the exposure."
339 )
340 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
341
342 def setDefaults(self):
343 # Background subtraction
344 # Use a small binsize for the first pass to reduce detections on glints
345 # and extended structures. Should not affect the detectability of
346 # faint diaSources
347 self.subtractInitialBackground.binSize = 8
348 self.subtractInitialBackground.useApprox = False
349 self.subtractInitialBackground.statisticsProperty = "MEDIAN"
350 self.subtractInitialBackground.doFilterSuperPixels = True
351 self.subtractInitialBackground.ignoredPixelMask = ["BAD",
352 "EDGE",
353 "DETECTED",
354 "DETECTED_NEGATIVE",
355 "NO_DATA",
356 ]
357 # Use a larger binsize for the final background subtraction, to reduce
358 # over-subtraction of bright objects.
359 self.subtractFinalBackground.binSize = 40
360 self.subtractFinalBackground.useApprox = False
361 self.subtractFinalBackground.statisticsProperty = "MEDIAN"
362 self.subtractFinalBackground.doFilterSuperPixels = True
363 self.subtractFinalBackground.ignoredPixelMask = ["BAD",
364 "EDGE",
365 "DETECTED",
366 "DETECTED_NEGATIVE",
367 "NO_DATA",
368 ]
369 # DiaSource Detection
370 self.detection.thresholdPolarity = "both"
371 self.detection.thresholdValue = 5.0
372 self.detection.reEstimateBackground = False
373 self.detection.thresholdType = "pixel_stdev"
374 self.detection.excludeMaskPlanes = ["EDGE",
375 "BAD",
376 ]
377
378 # Copy configs for binned streak detection from the base detection task
379 self.streakDetection.thresholdType = self.detection.thresholdType
380 self.streakDetection.reEstimateBackground = False
381 self.streakDetection.excludeMaskPlanes = self.detection.excludeMaskPlanes
382 self.streakDetection.thresholdValue = self.detection.thresholdValue
383 # Only detect positive streaks
384 self.streakDetection.thresholdPolarity = "positive"
385 # Do not grow detected mask for streaks
386 self.streakDetection.nSigmaToGrow = 0
387 # Set the streak mask along the entire fit line, not only where the
388 # detected mask is set.
389 self.maskStreaks.onlyMaskDetected = False
390 # Restrict streak masking from growing too large
391 self.maskStreaks.maxStreakWidth = 100
392 # Restrict the number of iterations allowed for fitting streaks
393 # When the fit is good it should solve quickly, and exit a bad fit quickly
394 self.maskStreaks.maxFitIter = 10
395 # Only mask to 2 sigma in width
396 self.maskStreaks.nSigmaMask = 2
397 # Threshold for including streaks after the Hough Transform.
398 # A lower value will detect more features that are less linear.
399 self.maskStreaks.absMinimumKernelHeight = 2
400
401 self.measurement.plugins.names |= ["ext_trailedSources_Naive",
402 "base_LocalPhotoCalib",
403 "base_LocalWcs",
404 "ext_shapeHSM_HsmSourceMoments",
405 "ext_shapeHSM_HsmPsfMoments",
406 "base_ClassificationSizeExtendedness",
407 ]
408 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
409 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
410 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
411 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
412 self.forcedMeasurement.copyColumns = {
413 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
414 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
415 self.forcedMeasurement.slots.shape = None
416
417 # Keep track of which footprints contain streaks
418 self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [
419 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
420 self.measurement.plugins["base_PixelFlags"].masksFpCenter = [
421 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
422 self.skySources.avoidMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "NO_DATA", "EDGE"]
423
424 def validate(self):
425 super().validate()
426
427 if self.run_sattle:
428 if not os.getenv("SATTLE_URI_BASE"):
429 raise pexConfig.FieldValidationError(DetectAndMeasureConfig.run_sattle, self,
430 "Sattle requested but SATTLE_URI_BASE "
431 "environment variable not set.")
432
433
434class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
435 """Detect and measure sources on a difference image.
436 """
437 ConfigClass = DetectAndMeasureConfig
438 _DefaultName = "detectAndMeasure"
439
440 def __init__(self, **kwargs):
441 super().__init__(**kwargs)
442 self.schema = afwTable.SourceTable.makeMinimalSchema()
443
444 self.algMetadata = dafBase.PropertyList()
445 if self.config.doSubtractBackground:
446 self.makeSubtask("subtractInitialBackground")
447 self.makeSubtask("subtractFinalBackground")
448 self.makeSubtask("detection", schema=self.schema)
449 if self.config.doDeblend:
450 self.makeSubtask("deblend", schema=self.schema)
451 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True)
452 self.makeSubtask("measurement", schema=self.schema,
453 algMetadata=self.algMetadata)
454 if self.config.doApCorr:
455 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
456 if self.config.doForcedMeasurement:
457 self.schema.addField(
458 "ip_diffim_forced_PsfFlux_instFlux", "D",
459 "Forced PSF flux measured on the direct image.",
460 units="count")
461 self.schema.addField(
462 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
463 "Forced PSF flux error measured on the direct image.",
464 units="count")
465 self.schema.addField(
466 "ip_diffim_forced_PsfFlux_area", "F",
467 "Forced PSF flux effective area of PSF.",
468 units="pixel")
469 self.schema.addField(
470 "ip_diffim_forced_PsfFlux_flag", "Flag",
471 "Forced PSF flux general failure flag.")
472 self.schema.addField(
473 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
474 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
475 self.schema.addField(
476 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
477 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
478 self.schema.addField(
479 "ip_diffim_forced_template_PsfFlux_instFlux", "D",
480 "Forced PSF flux measured on the template image.",
481 units="count")
482 self.schema.addField(
483 "ip_diffim_forced_template_PsfFlux_instFluxErr", "D",
484 "Forced PSF flux error measured on the template image.",
485 units="count")
486 self.schema.addField(
487 "ip_diffim_forced_template_PsfFlux_area", "F",
488 "Forced template PSF flux effective area of PSF.",
489 units="pixel")
490 self.schema.addField(
491 "ip_diffim_forced_template_PsfFlux_flag", "Flag",
492 "Forced template PSF flux general failure flag.")
493 self.schema.addField(
494 "ip_diffim_forced_template_PsfFlux_flag_noGoodPixels", "Flag",
495 "Forced template PSF flux not enough non-rejected pixels in data to attempt the fit.")
496 self.schema.addField(
497 "ip_diffim_forced_template_PsfFlux_flag_edge", "Flag",
498 """Forced template PSF flux object was too close to the edge of the image """
499 """to use the full PSF model.""")
500 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
501
502 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
503 self.schema.addField("srcMatchId", "L", "unique id of source match")
504 # Create the sky source task for use by metrics,
505 # even if sky sources are not added to the diaSource catalog
506 self.makeSubtask("skySources", schema=self.schema)
507 if self.config.doMaskStreaks:
508 self.makeSubtask("maskStreaks")
509 self.makeSubtask("streakDetection")
510 self.makeSubtask("findGlints")
511 self.schema.addField("glint_trail", "Flag", "DiaSource is part of a glint trail.")
512 self.schema.addField("reliability", type="F", doc="Reliability score of the DiaSource")
513
514 # To get the "merge_*" fields in the schema; have to re-initialize
515 # this later, once we have a peak schema post-detection.
516 lsst.afw.detection.FootprintMergeList(self.schema, ["positive", "negative"])
517
518 # Check that the schema and config are consistent
519 for flag in self.config.badSourceFlags:
520 if flag not in self.schema:
521 raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag)
522
523 # initialize InitOutputs
524 self.outputSchema = afwTable.SourceCatalog(self.schema)
525 self.outputSchema.getTable().setMetadata(self.algMetadata)
526
527 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
528 inputRefs: pipeBase.InputQuantizedConnection,
529 outputRefs: pipeBase.OutputQuantizedConnection):
530 inputs = butlerQC.get(inputRefs)
531 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
532 idFactory = idGenerator.make_table_id_factory()
533 # Specify the fields that `annotate` needs below, to ensure they
534 # exist, even as None.
535 measurementResults = pipeBase.Struct(
536 subtractedMeasuredExposure=None,
537 diaSources=None,
538 maskedStreaks=None,
539 differenceBackground=None,
540 )
541 try:
542 self.run(**inputs, idFactory=idFactory, measurementResults=measurementResults)
543 except pipeBase.AlgorithmError as e:
544 error = pipeBase.AnnotatedPartialOutputsError.annotate(
545 e,
546 self,
547 measurementResults.subtractedMeasuredExposure,
548 measurementResults.diaSources,
549 measurementResults.maskedStreaks,
550 log=self.log
551 )
552 butlerQC.put(measurementResults, outputRefs)
553 raise error from e
554 butlerQC.put(measurementResults, outputRefs)
555
556 @timeMethod
557 def run(self, science, matchedTemplate, difference, kernelSources,
558 idFactory=None, measurementResults=None):
559 """Detect and measure sources on a difference image.
560
561 The difference image will be convolved with a gaussian approximation of
562 the PSF to form a maximum likelihood image for detection.
563 Close positive and negative detections will optionally be merged into
564 dipole diaSources.
565 Sky sources, or forced detections in background regions, will optionally
566 be added, and the configured measurement algorithm will be run on all
567 detections.
568
569 Parameters
570 ----------
571 science : `lsst.afw.image.ExposureF`
572 Science exposure that the template was subtracted from.
573 matchedTemplate : `lsst.afw.image.ExposureF`
574 Warped and PSF-matched template that was used produce the
575 difference image.
576 difference : `lsst.afw.image.ExposureF`
577 Result of subtracting template from the science image.
578 kernelSources : `lsst.afw.table.SourceCatalog`
579 Final selection of sources that was used for psf matching.
580 idFactory : `lsst.afw.table.IdFactory`, optional
581 Generator object used to assign ids to detected sources in the
582 difference image. Ids from this generator are not set until after
583 deblending and merging positive/negative peaks.
584 measurementResults : `lsst.pipe.base.Struct`, optional
585 Result struct that is modified to allow saving of partial outputs
586 for some failure conditions. If the task completes successfully,
587 this is also returned.
588
589 Returns
590 -------
591 measurementResults : `lsst.pipe.base.Struct`
592
593 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
594 Subtracted exposure with detection mask applied.
595 ``diaSources`` : `lsst.afw.table.SourceCatalog`
596 The catalog of detected sources.
597 ``differenceBackground`` : `lsst.afw.math.BackgroundList`
598 Background that was subtracted from the difference image.
599 """
600 if measurementResults is None:
601 measurementResults = pipeBase.Struct()
602 if idFactory is None:
603 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()
604
605 if self.config.doSubtractBackground:
606 # Run background subtraction before clearing the mask planes
607 detectionExposure = difference.clone()
608 background = self.subtractInitialBackground.run(detectionExposure).background
609 else:
610 detectionExposure = difference
611 background = afwMath.BackgroundList()
612
613 self._prepareInputs(detectionExposure)
614
615 # Don't use the idFactory until after deblend+merge, so that we aren't
616 # generating ids that just get thrown away (footprint merge doesn't
617 # know about past ids).
618 table = afwTable.SourceTable.make(self.schema)
619 results = self.detection.run(
620 table=table,
621 exposure=detectionExposure,
622 doSmooth=True,
623 background=background
624 )
625
626 if self.config.doSubtractBackground:
627 # Run background subtraction again after detecting peaks
628 # but before measurement
629 # First update the mask using the detection image
630 difference.setMask(detectionExposure.mask)
631 background = self.subtractFinalBackground.run(difference).background
632
633 # Re-run detection to get final footprints
634 table = afwTable.SourceTable.make(self.schema)
635 results = self.detection.run(
636 table=table,
637 exposure=difference,
638 doSmooth=True,
639 background=background
640 )
641 measurementResults.differenceBackground = background
642
643 if self.config.doDeblend:
644 sources, positives, negatives = self._deblend(difference,
645 results.positive,
646 results.negative)
647
648 else:
649 positives = afwTable.SourceCatalog(self.schema)
650 results.positive.makeSources(positives)
651 negatives = afwTable.SourceCatalog(self.schema)
652 results.negative.makeSources(negatives)
653 sources = results.sources
654
655 self.processResults(science, matchedTemplate, difference,
656 sources, idFactory, kernelSources,
657 positives=positives,
658 negatives=negatives,
659 measurementResults=measurementResults)
660 return measurementResults
661
662 def _prepareInputs(self, difference):
663 """Ensure that we start with an empty detection and deblended mask.
664
665 Parameters
666 ----------
667 difference : `lsst.afw.image.ExposureF`
668 The difference image that will be used for detecting diaSources.
669 The mask plane will be modified in place.
670
671 Raises
672 ------
673 lsst.pipe.base.UpstreamFailureNoWorkFound
674 If the PSF is not usable for measurement.
675 """
676 # Check that we have a valid PSF now before we do more work
677 sigma = difference.psf.computeShape(difference.psf.getAveragePosition()).getDeterminantRadius()
678 if np.isnan(sigma):
679 raise pipeBase.UpstreamFailureNoWorkFound("Invalid PSF detected! PSF width evaluates to NaN.")
680 # Ensure that we start with an empty detection and deblended mask.
681 mask = difference.mask
682 for mp in self.config.clearMaskPlanes:
683 if mp not in mask.getMaskPlaneDict():
684 mask.addMaskPlane(mp)
685 mask &= ~mask.getPlaneBitMask(self.config.clearMaskPlanes)
686
687 def processResults(self, science, matchedTemplate, difference, sources, idFactory, kernelSources,
688 positives=None, negatives=None, measurementResults=None):
689 """Measure and process the results of source detection.
690
691 Parameters
692 ----------
693 science : `lsst.afw.image.ExposureF`
694 Science exposure that the template was subtracted from.
695 matchedTemplate : `lsst.afw.image.ExposureF`
696 Warped and PSF-matched template that was used produce the
697 difference image.
698 difference : `lsst.afw.image.ExposureF`
699 Result of subtracting template from the science image.
700 sources : `lsst.afw.table.SourceCatalog`
701 Detected sources on the difference exposure.
702 idFactory : `lsst.afw.table.IdFactory`
703 Generator object used to assign ids to detected sources in the
704 difference image.
705 kernelSources : `lsst.afw.table.SourceCatalog`
706 Final selection of sources that was used for psf matching.
707 positives : `lsst.afw.table.SourceCatalog`, optional
708 Positive polarity footprints.
709 negatives : `lsst.afw.table.SourceCatalog`, optional
710 Negative polarity footprints.
711 measurementResults : `lsst.pipe.base.Struct`, optional
712 Result struct that is modified to allow saving of partial outputs
713 for some failure conditions. If the task completes successfully,
714 this is also returned.
715
716 Returns
717 -------
718 measurementResults : `lsst.pipe.base.Struct`
719
720 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
721 Subtracted exposure with detection mask applied.
722 ``diaSources`` : `lsst.afw.table.SourceCatalog`
723 The catalog of detected sources.
724 """
725 if measurementResults is None:
726 measurementResults = pipeBase.Struct()
727 self.metadata["nUnmergedDiaSources"] = len(sources)
728 if self.config.doMerge:
729 # preserve peak schema, if there are any footprints
730 if len(positives) > 0:
731 peakSchema = positives[0].getFootprint().peaks.schema
732 elif len(negatives) > 0:
733 peakSchema = negatives[0].getFootprint().peaks.schema
734 else:
735 peakSchema = afwDetection.PeakTable.makeMinimalSchema()
736 mergeList = afwDetection.FootprintMergeList(self.schema,
737 ["positive", "negative"], peakSchema)
738 initialDiaSources = afwTable.SourceCatalog(self.schema)
739 # Start with positive, as FootprintMergeList will self-merge the
740 # subsequent added catalogs, and we want to try to preserve
741 # deblended positive sources.
742 mergeList.addCatalog(initialDiaSources.table, positives, "positive", minNewPeakDist=0)
743 mergeList.addCatalog(initialDiaSources.table, negatives, "negative", minNewPeakDist=0)
744 mergeList.getFinalSources(initialDiaSources)
745 # Flag as negative those sources that *only* came from the negative
746 # footprint set.
747 initialDiaSources["is_negative"] = initialDiaSources["merge_footprint_negative"] & \
748 ~initialDiaSources["merge_footprint_positive"]
749 self.log.info("Merging detections into %d sources", len(initialDiaSources))
750 else:
751 initialDiaSources = sources
752
753 # Assign source ids at the end: deblend/merge mean that we don't keep
754 # track of parents and children, we only care about the final ids.
755 for source in initialDiaSources:
756 source.setId(idFactory())
757 # Ensure sources added after this get correct ids.
758 initialDiaSources.getTable().setIdFactory(idFactory)
759 initialDiaSources.setMetadata(self.algMetadata)
760
761 self.metadata["nMergedDiaSources"] = len(initialDiaSources)
762
763 if self.config.doMaskStreaks:
764 streakInfo = self._runStreakMasking(difference)
765
766 if self.config.doSkySources:
767 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
768
769 if not initialDiaSources.isContiguous():
770 initialDiaSources = initialDiaSources.copy(deep=True)
771
772 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
773
774 # Remove unphysical diaSources per config.badSourceFlags
775 diaSources = self._removeBadSources(initialDiaSources)
776
777 if self.config.run_sattle:
778 diaSources = self.filterSatellites(diaSources, science)
779
780 # Flag diaSources in glint trails, but do not remove them
781 diaSources, trail_parameters = self._find_glint_trails(diaSources)
782 if self.config.writeGlintInfo:
783 measurementResults.mergeItems(trail_parameters, 'glintTrailInfo')
784
785 if self.config.doForcedMeasurement:
786 self.measureForcedSources(diaSources, science, difference.getWcs())
787 self.measureForcedSources(diaSources, matchedTemplate, difference.getWcs(),
788 template=True)
789
790 # Clear the image plane for regions with NO_DATA.
791 # These regions are most often caused by insufficient template coverage.
792 # Do this for the final difference image after detection and measurement
793 # since the subtasks should all be configured to handle NO_DATA properly
794 difference.image.array[difference.mask.array & difference.mask.getPlaneBitMask('NO_DATA') > 0] = 0
795
796 measurementResults.subtractedMeasuredExposure = difference
797
798 if self.config.doMaskStreaks and self.config.writeStreakInfo:
799 measurementResults.mergeItems(streakInfo, 'maskedStreaks')
800
801 self.calculateMetrics(science, difference, diaSources, kernelSources)
802
803 if np.count_nonzero(~diaSources["sky_source"]) > 0:
804 measurementResults.diaSources = diaSources
805 elif self.config.raiseOnNoDiaSources:
806 raise NoDiaSourcesError()
807 elif len(diaSources) > 0:
808 # This option allows returning sky sources,
809 # even if there are no diaSources
810 measurementResults.diaSources = diaSources
811
812 return measurementResults
813
814 def _deblend(self, difference, positiveFootprints, negativeFootprints):
815 """Deblend the positive and negative footprints and return a catalog
816 containing just the children, and the deblended footprints.
817
818 Parameters
819 ----------
820 difference : `lsst.afw.image.Exposure`
821 Result of subtracting template from the science image.
822 positiveFootprints, negativeFootprints : `lsst.afw.detection.FootprintSet`
823 Positive and negative polarity footprints measured on
824 ``difference`` to be deblended separately.
825
826 Returns
827 -------
828 sources : `lsst.afw.table.SourceCatalog`
829 Positive and negative deblended children.
830 positives, negatives : `lsst.afw.table.SourceCatalog`
831 Deblended positive and negative polarity sources with footprints
832 detected on ``difference``.
833 """
834
835 def deblend(footprints, negative=False):
836 """Deblend a positive or negative footprint set,
837 and return the deblended children.
838
839 Parameters
840 ----------
841 footprints : `lsst.afw.detection.FootprintSet`
842 negative : `bool`
843 Set True if the footprints contain negative fluxes
844
845 Returns
846 -------
847 sources : `lsst.afw.table.SourceCatalog`
848 """
849 sources = afwTable.SourceCatalog(self.schema)
850 footprints.makeSources(sources)
851 if negative:
852 # Invert the image so the deblender can run on positive peaks
853 difference_inverted = difference.clone()
854 difference_inverted.image *= -1
855 self.deblend.run(exposure=difference_inverted, sources=sources)
856 children = sources[sources["parent"] != 0]
857 # Set the heavy footprint pixel values back to reality
858 for child in children:
859 footprint = child.getFootprint()
860 array = footprint.getImageArray()
861 array *= -1
862 else:
863 self.deblend.run(exposure=difference, sources=sources)
864 self.setPrimaryFlags.run(sources)
865 children = sources["detect_isDeblendedSource"] == 1
866 sources = sources[children].copy(deep=True)
867 # Clear parents, so that measurement plugins behave correctly.
868 sources['parent'] = 0
869 return sources.copy(deep=True)
870
871 positives = deblend(positiveFootprints)
872 negatives = deblend(negativeFootprints, negative=True)
873
874 sources = afwTable.SourceCatalog(self.schema)
875 sources.reserve(len(positives) + len(negatives))
876 sources.extend(positives, deep=True)
877 sources.extend(negatives, deep=True)
878 if len(negatives) > 0:
879 sources[-len(negatives):]["is_negative"] = True
880 return sources, positives, negatives
881
882 def _removeBadSources(self, diaSources):
883 """Remove unphysical diaSources from the catalog.
884
885 Parameters
886 ----------
887 diaSources : `lsst.afw.table.SourceCatalog`
888 The catalog of detected sources.
889
890 Returns
891 -------
892 diaSources : `lsst.afw.table.SourceCatalog`
893 The updated catalog of detected sources, with any source that has a
894 flag in ``config.badSourceFlags`` set removed.
895 """
896 selector = np.ones(len(diaSources), dtype=bool)
897 for flag in self.config.badSourceFlags:
898 flags = diaSources[flag]
899 nBad = np.count_nonzero(flags)
900 if nBad > 0:
901 self.log.debug("Found %d unphysical sources with flag %s.", nBad, flag)
902 selector &= ~flags
903 nBadTotal = np.count_nonzero(~selector)
904 self.metadata["nRemovedBadFlaggedSources"] = nBadTotal
905 self.log.info("Removed %d unphysical sources.", nBadTotal)
906 return diaSources[selector].copy(deep=True)
907
908 def _find_glint_trails(self, diaSources):
909 """Define a new flag column for diaSources that are in a glint trail.
910
911 Parameters
912 ----------
913 diaSources : `lsst.afw.table.SourceCatalog`
914 The catalog of detected sources.
915
916 Returns
917 -------
918 diaSources : `lsst.afw.table.SourceCatalog`
919 The updated catalog of detected sources, with a new bool column
920 called 'glint_trail' added.
921
922 trail_parameters : `dict`
923 Parameters of all the trails that were found.
924 """
925 if self.config.doSkySources:
926 # Do not include sky sources in glint detection
927 candidateDiaSources = diaSources[~diaSources["sky_source"]].copy(deep=True)
928 else:
929 candidateDiaSources = diaSources
930 trailed_glints = self.findGlints.run(candidateDiaSources)
931 glint_mask = [True if id in trailed_glints.trailed_ids else False for id in diaSources['id']]
932 if np.any(glint_mask):
933 diaSources['glint_trail'] = np.array(glint_mask)
934
935 slopes = np.array([trail.slope for trail in trailed_glints.parameters])
936 intercepts = np.array([trail.intercept for trail in trailed_glints.parameters])
937 stderrs = np.array([trail.stderr for trail in trailed_glints.parameters])
938 lengths = np.array([trail.length for trail in trailed_glints.parameters])
939 angles = np.array([trail.angle for trail in trailed_glints.parameters])
940 parameters = {'slopes': slopes, 'intercepts': intercepts, 'stderrs': stderrs, 'lengths': lengths,
941 'angles': angles}
942
943 trail_parameters = pipeBase.Struct(glintTrailInfo=parameters)
944
945 return diaSources, trail_parameters
946
947 def addSkySources(self, diaSources, mask, seed,
948 subtask=None):
949 """Add sources in empty regions of the difference image
950 for measuring the background.
951
952 Parameters
953 ----------
954 diaSources : `lsst.afw.table.SourceCatalog`
955 The catalog of detected sources.
956 mask : `lsst.afw.image.Mask`
957 Mask plane for determining regions where Sky sources can be added.
958 seed : `int`
959 Seed value to initialize the random number generator.
960 """
961 if subtask is None:
962 subtask = self.skySources
963 skySourceFootprints = subtask.run(mask=mask, seed=seed, catalog=diaSources)
964 self.metadata[f"n_{subtask.getName()}"] = len(skySourceFootprints)
965
966 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
967 """Use (matched) template and science image to constrain dipole fitting.
968
969 Parameters
970 ----------
971 diaSources : `lsst.afw.table.SourceCatalog`
972 The catalog of detected sources.
973 science : `lsst.afw.image.ExposureF`
974 Science exposure that the template was subtracted from.
975 difference : `lsst.afw.image.ExposureF`
976 Result of subtracting template from the science image.
977 matchedTemplate : `lsst.afw.image.ExposureF`
978 Warped and PSF-matched template that was used produce the
979 difference image.
980 """
981 # Ensure that the required mask planes are present
982 for mp in self.config.measurement.plugins["base_PixelFlags"].masksFpAnywhere:
983 difference.mask.addMaskPlane(mp)
984 # Note that this may not be correct if we convolved the science image.
985 # In the future we may wish to persist the matchedScience image.
986 self.measurement.run(diaSources, difference, science, matchedTemplate)
987 if self.config.doApCorr:
988 apCorrMap = difference.getInfo().getApCorrMap()
989 if apCorrMap is None:
990 self.log.warning("Difference image does not have valid aperture correction; skipping.")
991 else:
992 self.applyApCorr.run(
993 catalog=diaSources,
994 apCorrMap=apCorrMap,
995 )
996
997 def measureForcedSources(self, diaSources, image, wcs, template=False):
998 """Perform forced measurement of the diaSources on a direct image.
999
1000 Parameters
1001 ----------
1002 diaSources : `lsst.afw.table.SourceCatalog`
1003 The catalog of detected sources.
1004 image: `lsst.afw.image.ExposureF`
1005 Exposure that the forced measurement is being performed on
1006 wcs : `lsst.afw.geom.SkyWcs`
1007 Coordinate system definition (wcs) for the exposure.
1008 template : `bool`
1009 Is the forced measurement being performed on the template?
1010 """
1011 # Run forced psf photometry on the image at the diaSource locations.
1012 # Copy the measured flux and error into the diaSource.
1013 forcedSources = self.forcedMeasurement.generateMeasCat(image, diaSources, wcs)
1014 self.forcedMeasurement.run(forcedSources, image, diaSources, wcs)
1015
1016 if template:
1017 base_key = 'ip_diffim_forced_template_PsfFlux'
1018 else:
1019 base_key = 'ip_diffim_forced_PsfFlux'
1020 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1021 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1022 f"{base_key}_instFlux", True)
1023 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1024 f"{base_key}_instFluxErr", True)
1025 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1026 f"{base_key}_area", True)
1027 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1028 f"{base_key}_flag", True)
1029 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1030 f"{base_key}_flag_noGoodPixels", True)
1031 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1032 f"{base_key}_flag_edge", True)
1033 for diaSource, forcedSource in zip(diaSources, forcedSources):
1034 diaSource.assign(forcedSource, mapper)
1035
1036 def calculateMetrics(self, science, difference, diaSources, kernelSources):
1037 """Add difference image QA metrics to the Task metadata.
1038
1039 This may be used to produce corresponding metrics (see
1040 lsst.analysis.tools.tasks.diffimTaskDetectorVisitMetricAnalysis).
1041
1042 Parameters
1043 ----------
1044 science : `lsst.afw.image.ExposureF`
1045 Science exposure that was subtracted.
1046 difference : `lsst.afw.image.Exposure`
1047 The target difference image to calculate metrics for.
1048 diaSources : `lsst.afw.table.SourceCatalog`
1049 The catalog of detected sources.
1050 kernelSources : `lsst.afw.table.SourceCatalog`
1051 Final selection of sources that was used for psf matching.
1052 """
1053 mask = difference.mask
1054 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
1055 self.metadata["nGoodPixels"] = np.sum(~badPix)
1056 self.metadata["nBadPixels"] = np.sum(badPix)
1057 detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0
1058 detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0
1059 self.metadata["nPixelsDetectedPositive"] = np.sum(detPosPix)
1060 self.metadata["nPixelsDetectedNegative"] = np.sum(detNegPix)
1061 detPosPix &= badPix
1062 detNegPix &= badPix
1063 self.metadata["nBadPixelsDetectedPositive"] = np.sum(detPosPix)
1064 self.metadata["nBadPixelsDetectedNegative"] = np.sum(detNegPix)
1065
1066 metricsMaskPlanes = list(mask.getMaskPlaneDict().keys())
1067 for maskPlane in metricsMaskPlanes:
1068 try:
1069 self.metadata["%s_mask_fraction"%maskPlane.lower()] = evaluateMaskFraction(mask, maskPlane)
1070 except InvalidParameterError:
1071 self.metadata["%s_mask_fraction"%maskPlane.lower()] = -1
1072 self.log.info("Unable to calculate metrics for mask plane %s: not in image"%maskPlane)
1073
1074 if self.config.doSkySources:
1075 skySources = diaSources[diaSources["sky_source"]]
1076 else:
1077 skySources = None
1078 metrics = computeDifferenceImageMetrics(science, difference, kernelSources, sky_sources=skySources)
1079
1080 self.metadata["residualFootprintRatioMean"] = metrics.differenceFootprintRatioMean
1081 self.metadata["residualFootprintRatioStdev"] = metrics.differenceFootprintRatioStdev
1082 self.metadata["differenceFootprintSkyRatioMean"] = metrics.differenceFootprintSkyRatioMean
1083 self.metadata["differenceFootprintSkyRatioStdev"] = metrics.differenceFootprintSkyRatioStdev
1084 self.log.info("Mean, stdev of ratio of difference to science "
1085 "pixels in star footprints: %5.4f, %5.4f",
1086 self.metadata["residualFootprintRatioMean"],
1087 self.metadata["residualFootprintRatioStdev"])
1088 if self.config.raiseOnBadSubtractionRatio:
1089 if metrics.differenceFootprintRatioMean > self.config.badSubtractionRatioThreshold:
1090 raise BadSubtractionError(ratio=metrics.differenceFootprintRatioMean,
1091 threshold=self.config.badSubtractionRatioThreshold)
1092 if metrics.differenceFootprintRatioStdev > self.config.badSubtractionVariationThreshold:
1093 raise BadSubtractionError(ratio=metrics.differenceFootprintRatioStdev,
1094 threshold=self.config.badSubtractionVariationThreshold)
1095
1096 def getSattleDiaSourceAllowlist(self, diaSources, science):
1097 """Query the sattle service and determine which diaSources are allowed.
1098
1099 Parameters
1100 ----------
1101 diaSources : `lsst.afw.table.SourceCatalog`
1102 The catalog of detected sources.
1103 science : `lsst.afw.image.ExposureF`
1104 Science exposure that was subtracted.
1105
1106 Returns
1107 ----------
1108 allow_list : `list` of `int`
1109 diaSourceIds of diaSources that can be made public.
1110
1111 Raises
1112 ------
1113 requests.HTTPError
1114 Raised if sattle call does not return success.
1115 """
1116 wcs = science.getWcs()
1117 visit_info = science.getInfo().getVisitInfo()
1118 visit_id = visit_info.getId()
1119 sattle_uri_base = os.getenv('SATTLE_URI_BASE')
1120
1121 dia_sources_json = []
1122 for source in diaSources:
1123 source_bbox = source.getFootprint().getBBox()
1124 corners = wcs.pixelToSky([lsst.geom.Point2D(c) for c in source_bbox.getCorners()])
1125 bbox_radec = [[pt.getRa().asDegrees(), pt.getDec().asDegrees()] for pt in corners]
1126 dia_sources_json.append({"diasource_id": source["id"], "bbox": bbox_radec})
1127
1128 payload = {"visit_id": visit_id, "detector_id": science.getDetector().getId(),
1129 "diasources": dia_sources_json, "historical": self.config.sattle_historical}
1130
1131 sattle_output = requests.put(f'{sattle_uri_base}/diasource_allow_list',
1132 json=payload)
1133
1134 # retry once if visit cache is not populated
1135 if sattle_output.status_code == 404:
1136 self.log.warning(f'Visit {visit_id} not found in sattle cache, re-sending')
1137 populate_sattle_visit_cache(visit_info, historical=self.config.sattle_historical)
1138 sattle_output = requests.put(f'{sattle_uri_base}/diasource_allow_list', json=payload)
1139
1140 sattle_output.raise_for_status()
1141
1142 return sattle_output.json()['allow_list']
1143
1144 def filterSatellites(self, diaSources, science):
1145 """Remove diaSources overlapping predicted satellite positions.
1146
1147 Parameters
1148 ----------
1149 diaSources : `lsst.afw.table.SourceCatalog`
1150 The catalog of detected sources.
1151 science : `lsst.afw.image.ExposureF`
1152 Science exposure that was subtracted.
1153
1154 Returns
1155 ----------
1156 filterdDiaSources : `lsst.afw.table.SourceCatalog`
1157 Filtered catalog of diaSources
1158 """
1159
1160 allow_list = self.getSattleDiaSourceAllowlist(diaSources, science)
1161
1162 if allow_list:
1163 allow_set = set(allow_list)
1164 allowed_ids = [source['id'] in allow_set for source in diaSources]
1165 diaSources = diaSources[np.array(allowed_ids)].copy(deep=True)
1166 else:
1167 self.log.warning('Sattle allowlist is empty, all diaSources removed')
1168 diaSources = diaSources[0:0].copy(deep=True)
1169 return diaSources
1170
1171 def _runStreakMasking(self, difference):
1172 """Do streak masking and optionally save the resulting streak
1173 fit parameters in a catalog.
1174
1175 Only returns non-empty streakInfo if self.config.writeStreakInfo
1176 is set. The difference image is binned by self.config.streakBinFactor
1177 (and detection is run a second time) so that regions with lower
1178 surface brightness streaks are more likely to fall above the
1179 detection threshold.
1180
1181 Parameters
1182 ----------
1183 difference: `lsst.afw.image.Exposure`
1184 The exposure in which to search for streaks. Must have a detection
1185 mask.
1186
1187 Returns
1188 -------
1189 streakInfo: `lsst.pipe.base.Struct`
1190 ``rho`` : `np.ndarray`
1191 Angle of detected streak.
1192 ``theta`` : `np.ndarray`
1193 Distance from center of detected streak.
1194 ``sigma`` : `np.ndarray`
1195 Width of streak profile.
1196 ``reducedChi2`` : `np.ndarray`
1197 Reduced chi2 of the best-fit streak profile.
1198 ``modelMaximum`` : `np.ndarray`
1199 Peak value of the fit line profile.
1200 """
1201 maskedImage = difference.maskedImage
1202 # Bin the diffim to enhance low surface brightness streaks
1203 binnedMaskedImage = afwMath.binImage(maskedImage,
1204 self.config.streakBinFactor,
1205 self.config.streakBinFactor)
1206 binnedExposure = afwImage.ExposureF(binnedMaskedImage.getBBox())
1207 binnedExposure.setMaskedImage(binnedMaskedImage)
1208 # Clear the DETECTED mask plane before streak detection
1209 binnedExposure.mask &= ~binnedExposure.mask.getPlaneBitMask('DETECTED')
1210 # Rerun detection to set the DETECTED mask plane on binnedExposure
1211 sigma = difference.psf.computeShape(difference.psf.getAveragePosition()).getDeterminantRadius()
1212 _table = afwTable.SourceTable.make(afwTable.SourceTable.makeMinimalSchema())
1213 self.streakDetection.run(table=_table, exposure=binnedExposure, doSmooth=True,
1214 sigma=sigma/self.config.streakBinFactor)
1215 binnedDetectedMaskPlane = binnedExposure.mask.array & binnedExposure.mask.getPlaneBitMask('DETECTED')
1216 rescaledDetectedMaskPlane = binnedDetectedMaskPlane.repeat(self.config.streakBinFactor,
1217 axis=0).repeat(self.config.streakBinFactor,
1218 axis=1)
1219 # Create new version of a diffim with DETECTED based on binnedExposure
1220 streakMaskedImage = maskedImage.clone()
1221 ysize, xsize = rescaledDetectedMaskPlane.shape
1222 streakMaskedImage.mask.array[:ysize, :xsize] |= rescaledDetectedMaskPlane
1223 # Detect streaks on this new version of the diffim
1224 streaks = self.maskStreaks.run(streakMaskedImage)
1225 streakMaskPlane = streakMaskedImage.mask.array & streakMaskedImage.mask.getPlaneBitMask('STREAK')
1226 # Apply the new STREAK mask to the original diffim
1227 maskedImage.mask.array |= streakMaskPlane
1228
1229 if self.config.writeStreakInfo:
1230 rhos = np.array([line.rho for line in streaks.lines])
1231 thetas = np.array([line.theta for line in streaks.lines])
1232 sigmas = np.array([line.sigma for line in streaks.lines])
1233 chi2s = np.array([line.reducedChi2 for line in streaks.lines])
1234 modelMaximums = np.array([line.modelMaximum for line in streaks.lines])
1235 streakInfo = {'rho': rhos, 'theta': thetas, 'sigma': sigmas, 'reducedChi2': chi2s,
1236 'modelMaximum': modelMaximums}
1237 else:
1238 streakInfo = {'rho': np.array([]), 'theta': np.array([]), 'sigma': np.array([]),
1239 'reducedChi2': np.array([]), 'modelMaximum': np.array([])}
1240 return pipeBase.Struct(maskedStreaks=streakInfo)
1241
1242
1243class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
1244 scoreExposure = pipeBase.connectionTypes.Input(
1245 doc="Maximum likelihood image for detection.",
1246 dimensions=("instrument", "visit", "detector"),
1247 storageClass="ExposureF",
1248 name="{fakesType}{coaddName}Diff_scoreExp",
1249 )
1250
1251
1252class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
1253 pipelineConnections=DetectAndMeasureScoreConnections):
1254 pass
1255
1256
1257class DetectAndMeasureScoreTask(DetectAndMeasureTask):
1258 """Detect DIA sources using a score image,
1259 and measure the detections on the difference image.
1260
1261 Source detection is run on the supplied score, or maximum likelihood,
1262 image. Note that no additional convolution will be done in this case.
1263 Close positive and negative detections will optionally be merged into
1264 dipole diaSources.
1265 Sky sources, or forced detections in background regions, will optionally
1266 be added, and the configured measurement algorithm will be run on all
1267 detections.
1268 """
1269 ConfigClass = DetectAndMeasureScoreConfig
1270 _DefaultName = "detectAndMeasureScore"
1271
1272 @timeMethod
1273 def run(self, science, matchedTemplate, difference, scoreExposure, kernelSources,
1274 idFactory=None):
1275 """Detect and measure sources on a score image.
1276
1277 Parameters
1278 ----------
1279 science : `lsst.afw.image.ExposureF`
1280 Science exposure that the template was subtracted from.
1281 matchedTemplate : `lsst.afw.image.ExposureF`
1282 Warped and PSF-matched template that was used produce the
1283 difference image.
1284 difference : `lsst.afw.image.ExposureF`
1285 Result of subtracting template from the science image.
1286 scoreExposure : `lsst.afw.image.ExposureF`
1287 Score or maximum likelihood difference image
1288 kernelSources : `lsst.afw.table.SourceCatalog`
1289 Final selection of sources that was used for psf matching.
1290 idFactory : `lsst.afw.table.IdFactory`, optional
1291 Generator object used to assign ids to detected sources in the
1292 difference image. Ids from this generator are not set until after
1293 deblending and merging positive/negative peaks.
1294
1295 Returns
1296 -------
1297 measurementResults : `lsst.pipe.base.Struct`
1298
1299 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
1300 Subtracted exposure with detection mask applied.
1301 ``diaSources`` : `lsst.afw.table.SourceCatalog`
1302 The catalog of detected sources.
1303 """
1304 if idFactory is None:
1305 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()
1306
1307 self._prepareInputs(scoreExposure)
1308
1309 # Don't use the idFactory until after deblend+merge, so that we aren't
1310 # generating ids that just get thrown away (footprint merge doesn't
1311 # know about past ids).
1312 table = afwTable.SourceTable.make(self.schema)
1313 results = self.detection.run(
1314 table=table,
1315 exposure=scoreExposure,
1316 doSmooth=False,
1317 )
1318 # Copy the detection mask from the Score image to the difference image
1319 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
1320
1321 if self.config.doDeblend:
1322 sources, positives, negatives = self._deblend(difference,
1323 results.positive,
1324 results.negative)
1325
1326 return self.processResults(science, matchedTemplate, difference,
1327 sources, idFactory, kernelSources,
1328 positives=positives,
1329 negatives=negatives)
1330
1331 else:
1332 positives = afwTable.SourceCatalog(self.schema)
1333 results.positive.makeSources(positives)
1334 negatives = afwTable.SourceCatalog(self.schema)
1335 results.negative.makeSources(negatives)
1336 return self.processResults(science, matchedTemplate, difference,
1337 results.sources, idFactory, kernelSources,
1338 positives=positives,
1339 negatives=negatives)
A mapping between the keys of two Schemas, used to copy data between them.
Class for storing ordered metadata with comments.
std::shared_ptr< ImageT > binImage(ImageT const &inImage, int const binX, int const binY, lsst::afw::math::Property const flags=lsst::afw::math::MEAN)
Definition binImage.cc:44
run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter)