LSST Applications g04e9c324dd+8c5ae1fdc5,g134cb467dc+1b3060144d,g18429d2f64+f642bf4753,g199a45376c+0ba108daf9,g1fd858c14a+2dcf163641,g262e1987ae+7b8c96d2ca,g29ae962dfc+3bd6ecb08a,g2cef7863aa+aef1011c0b,g35bb328faa+8c5ae1fdc5,g3fd5ace14f+53e1a9e7c5,g4595892280+fef73a337f,g47891489e3+2efcf17695,g4d44eb3520+642b70b07e,g53246c7159+8c5ae1fdc5,g67b6fd64d1+2efcf17695,g67fd3c3899+b70e05ef52,g74acd417e5+317eb4c7d4,g786e29fd12+668abc6043,g87389fa792+8856018cbb,g89139ef638+2efcf17695,g8d7436a09f+3be3c13596,g8ea07a8fe4+9f5ccc88ac,g90f42f885a+a4e7b16d9b,g97be763408+ad77d7208f,g9dd6db0277+b70e05ef52,ga681d05dcb+a3f46e7fff,gabf8522325+735880ea63,gac2eed3f23+2efcf17695,gb89ab40317+2efcf17695,gbf99507273+8c5ae1fdc5,gd8ff7fe66e+b70e05ef52,gdab6d2f7ff+317eb4c7d4,gdc713202bf+b70e05ef52,gdfd2d52018+b10e285e0f,ge365c994fd+310e8507c4,ge410e46f29+2efcf17695,geaed405ab2+562b3308c0,gffca2db377+8c5ae1fdc5,w.2025.35
LSST Data Management Base Package
Loading...
Searching...
No Matches
dipoleFitTask.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22
23import math
24import logging
25import numpy as np
26import warnings
27
28import lsst.afw.image as afwImage
29import lsst.meas.base as measBase
30import lsst.afw.detection as afwDet
31import lsst.geom as geom
32import lsst.pex.exceptions as pexExcept
33import lsst.pex.config as pexConfig
34from lsst.pipe.base import Struct
35from lsst.utils.timer import timeMethod
36
37__all__ = ("DipoleFitTask", "DipoleFitPlugin", "DipoleFitTaskConfig", "DipoleFitPluginConfig",
38 "DipoleFitAlgorithm")
39
40
41# Create a new measurement task (`DipoleFitTask`) that can handle all other SFM tasks but can
42# pass a separate pos- and neg- exposure/image to the `DipoleFitPlugin`s `run()` method.
43
44
45class DipoleFitPluginConfig(measBase.SingleFramePluginConfig):
46 """Configuration for DipoleFitPlugin
47 """
48
49 fitAllDiaSources = pexConfig.Field(
50 dtype=bool, default=False,
51 doc="""Attempte dipole fit of all diaSources (otherwise just the ones consisting of overlapping
52 positive and negative footprints)""")
53
54 maxSeparation = pexConfig.Field(
55 dtype=float, default=5.,
56 doc="Assume dipole is not separated by more than maxSeparation * psfSigma")
57
58 relWeight = pexConfig.Field(
59 dtype=float, default=0.5,
60 doc="""Relative weighting of pre-subtraction images (higher -> greater influence of pre-sub.
61 images on fit)""")
62
63 tolerance = pexConfig.Field(
64 dtype=float, default=1e-7,
65 doc="Fit tolerance")
66
67 fitBackground = pexConfig.Field(
68 dtype=int, default=1,
69 doc="Set whether and how to fit for linear gradient in pre-sub. images. Possible values:"
70 "0: do not fit background at all"
71 "1 (default): pre-fit the background using linear least squares and then do not fit it as part"
72 "of the dipole fitting optimization"
73 "2: pre-fit the background using linear least squares (as in 1), and use the parameter"
74 "estimates from that fit as starting parameters for an integrated re-fit of the background")
75
76 fitSeparateNegParams = pexConfig.Field(
77 dtype=bool, default=False,
78 doc="Include parameters to fit for negative values (flux, gradient) separately from pos.")
79
80 # Config params for classification of detected diaSources as dipole or not
81 minSn = pexConfig.Field(
82 dtype=float, default=math.sqrt(2) * 5.0,
83 doc="Minimum quadrature sum of positive+negative lobe S/N to be considered a dipole")
84
85 maxFluxRatio = pexConfig.Field(
86 dtype=float, default=0.65,
87 doc="Maximum flux ratio in either lobe to be considered a dipole")
88
89 maxChi2DoF = pexConfig.Field(
90 dtype=float, default=0.05,
91 doc="""Maximum Chi2/DoF significance of fit to be considered a dipole.
92 Default value means \"Choose a chi2DoF corresponding to a significance level of at most 0.05\"
93 (note this is actually a significance, not a chi2 value).""")
94
95 maxFootprintArea = pexConfig.Field(
96 dtype=int, default=1_200,
97 doc=("Maximum area for footprints before they are ignored as large; "
98 "non-positive means no threshold applied"
99 "Threshold chosen for HSC and DECam data, see DM-38741 for details."))
100
101
102class DipoleFitTaskConfig(measBase.SingleFrameMeasurementConfig):
103
104 def setDefaults(self):
105 measBase.SingleFrameMeasurementConfig.setDefaults(self)
106
107 self.plugins.names = ["base_SdssCentroid",
108 "ip_diffim_DipoleFit",
109 "base_CircularApertureFlux",
110 "base_PixelFlags",
111 "base_SkyCoord",
112 "base_PsfFlux",
113 ]
114 # Only measure the apertures we need to report in the alert stream.
115 self.plugins["base_CircularApertureFlux"].radii = [12.0]
116
117 self.slots.calibFlux = None
118 self.slots.modelFlux = None
119 self.slots.gaussianFlux = None
120 self.slots.shape = None
121 # This will be switched to "ip_diffim_DipoleFit" as this task runs.
122 self.slots.centroid = "base_SdssCentroid"
124
125
126class DipoleFitTask(measBase.SingleFrameMeasurementTask):
127 """A task that fits a dipole to a difference image, with an optional
128 separate detection image.
129
130 Because it subclasses SingleFrameMeasurementTask, and calls
131 SingleFrameMeasurementTask.run() from its run() method, it still
132 can be used identically to a standard SingleFrameMeasurementTask.
133 """
134
135 ConfigClass = DipoleFitTaskConfig
136 _DefaultName = "dipoleFit"
137
138 def __init__(self, schema, algMetadata=None, **kwargs):
139 super().__init__(schema, algMetadata, **kwargs)
140
141 # Enforce a specific plugin order, so that DipoleFit can fall back on
142 # SdssCentroid for non-dipoles
143 self.plugins_pre = self.plugins.copy()
144 self.plugins_post = self.plugins.copy()
145 self.plugins_pre.clear()
146 self.plugins_pre["base_SdssCentroid"] = self.plugins["base_SdssCentroid"]
147 self.plugins_post.pop("base_SdssCentroid")
148 self.dipoleFit = self.plugins_post.pop("ip_diffim_DipoleFit")
149 del self.plugins
150
151 @timeMethod
152 def run(self, sources, exposure, posExp=None, negExp=None, **kwargs):
153 """Run dipole measurement and classification.
154
155 Run SdssCentroid first, then switch the centroid slot, then DipoleFit
156 then the rest; DipoleFit will fall back on SdssCentroid for sources
157 not containing positive+negative peaks.
158
159 Parameters
160 ----------
161 sources : `lsst.afw.table.SourceCatalog`
162 ``diaSources`` that will be measured using dipole measurement.
163 exposure : `lsst.afw.image.Exposure`
164 The difference exposure on which the ``sources`` were detected.
165 If neither ``posExp`` nor ``negExp`` are set, then the dipole is also
166 fitted directly to this difference image.
167 posExp : `lsst.afw.image.Exposure`, optional
168 "Positive" exposure, typically a science exposure, or None if unavailable
169 When `posExp` is `None`, will compute `posImage = exposure + negExp`.
170 negExp : `lsst.afw.image.Exposure`, optional
171 "Negative" exposure, typically a template exposure, or None if unavailable
172 When `negExp` is `None`, will compute `negImage = posExp - exposure`.
173 **kwargs
174 Additional keyword arguments for `lsst.meas.base.sfm.SingleFrameMeasurementTask`.
175 """
176 # Run plugins in a very specific order, so DipoleFitPlugin has a
177 # centroid to fall back on.
179 super().run(sources, exposure, **kwargs)
180
181 for source in sources:
182 self.dipoleFit.measureDipoles(source, exposure, posExp, negExp)
183 # Use the new DipoleFit outputs for subsequent measurements, now that
184 # non-dipoles have been filled in with the earlier centroid values.
185 sources.schema.getAliasMap().set("slot_Centroid", "ip_diffim_DipoleFit")
186
187 self.plugins = self.plugins_post
188 super().run(sources, exposure, **kwargs)
189
190
192 """Lightweight class containing methods for generating a dipole model for fitting
193 to sources in diffims, used by DipoleFitAlgorithm.
194
195 See also:
196 `DMTN-007: Dipole characterization for image differencing <https://dmtn-007.lsst.io>`_.
197 """
198
199 def __init__(self):
200 import lsstDebug
201 self.debug = lsstDebug.Info(__name__).debug
202 self.log = logging.getLogger(__name__)
203
204 def makeBackgroundModel(self, in_x, pars=None):
205 """Generate gradient model (2-d array) with up to 2nd-order polynomial
206
207 Parameters
208 ----------
209 in_x : `numpy.array`
210 (2, w, h)-dimensional `numpy.array`, containing the
211 input x,y meshgrid providing the coordinates upon which to
212 compute the gradient. This will typically be generated via
213 `_generateXYGrid()`. `w` and `h` correspond to the width and
214 height of the desired grid.
215 pars : `list` of `float`, optional
216 Up to 6 floats for up
217 to 6 2nd-order 2-d polynomial gradient parameters, in the
218 following order: (intercept, x, y, xy, x**2, y**2). If `pars`
219 is emtpy or `None`, do nothing and return `None` (for speed).
220
221 Returns
222 -------
223 result : `None` or `numpy.array`
224 return None, or 2-d numpy.array of width/height matching
225 input bbox, containing computed gradient values.
226 """
227
228 # Don't fit for other gradient parameters if the intercept is not included.
229 if (pars is None) or (len(pars) <= 0) or (pars[0] is None):
230 return
231
232 y, x = in_x[0, :], in_x[1, :]
233 gradient = np.full_like(x, pars[0], dtype='float64')
234 if len(pars) > 1 and pars[1] is not None:
235 gradient += pars[1] * x
236 if len(pars) > 2 and pars[2] is not None:
237 gradient += pars[2] * y
238 if len(pars) > 3 and pars[3] is not None:
239 gradient += pars[3] * (x * y)
240 if len(pars) > 4 and pars[4] is not None:
241 gradient += pars[4] * (x * x)
242 if len(pars) > 5 and pars[5] is not None:
243 gradient += pars[5] * (y * y)
244
245 return gradient
246
247 def _generateXYGrid(self, bbox):
248 """Generate a meshgrid covering the x,y coordinates bounded by bbox
249
250 Parameters
251 ----------
252 bbox : `lsst.geom.Box2I`
253 input Bounding Box defining the coordinate limits
254
255 Returns
256 -------
257 in_x : `numpy.array`
258 (2, w, h)-dimensional numpy array containing the grid indexing over x- and
259 y- coordinates
260 """
261
262 x, y = np.mgrid[bbox.getBeginY():bbox.getEndY(), bbox.getBeginX():bbox.getEndX()]
263 in_x = np.array([y, x]).astype(np.float64)
264 in_x[0, :] -= np.mean(in_x[0, :])
265 in_x[1, :] -= np.mean(in_x[1, :])
266 return in_x
267
268 def _getHeavyFootprintSubimage(self, fp, badfill=np.nan, grow=0):
269 """Extract the image from a ``~lsst.afw.detection.HeavyFootprint``
270 as an `lsst.afw.image.ImageF`.
271
272 Parameters
273 ----------
274 fp : `lsst.afw.detection.HeavyFootprint`
275 HeavyFootprint to use to generate the subimage
276 badfill : `float`, optional
277 Value to fill in pixels in extracted image that are outside the footprint
278 grow : `int`
279 Optionally grow the footprint by this amount before extraction
280
281 Returns
282 -------
283 subim2 : `lsst.afw.image.ImageF`
284 An `~lsst.afw.image.ImageF` containing the subimage.
285 """
286 bbox = fp.getBBox()
287 if grow > 0:
288 bbox.grow(grow)
289
290 subim2 = afwImage.ImageF(bbox, badfill)
291 fp.getSpans().unflatten(subim2.array, fp.getImageArray(), bbox.getCorners()[0])
292 return subim2
293
294 def fitFootprintBackground(self, source, posImage, order=1):
295 """Fit a linear (polynomial) model of given order (max 2) to the background of a footprint.
296
297 Only fit the pixels OUTSIDE of the footprint, but within its bounding box.
298
299 Parameters
300 ----------
301 source : `lsst.afw.table.SourceRecord`
302 SourceRecord, the footprint of which is to be fit
303 posImage : `lsst.afw.image.Exposure`
304 The exposure from which to extract the footprint subimage
305 order : `int`
306 Polynomial order of background gradient to fit.
307
308 Returns
309 -------
310 pars : `tuple` of `float`
311 `tuple` of length (1 if order==0; 3 if order==1; 6 if order == 2),
312 containing the resulting fit parameters
313 """
314
315 # TODO look into whether to use afwMath background methods -- see
316 # http://lsst-web.ncsa.illinois.edu/doxygen/x_masterDoxyDoc/_background_example.html
317 fp = source.getFootprint()
318 bbox = fp.getBBox()
319 bbox.grow(3)
320 posImg = afwImage.ImageF(posImage.image, bbox, afwImage.PARENT)
321
322 # This code constructs the footprint image so that we can identify the pixels that are
323 # outside the footprint (but within the bounding box). These are the pixels used for
324 # fitting the background.
325 posHfp = afwDet.HeavyFootprintF(fp, posImage.getMaskedImage())
326 posFpImg = self._getHeavyFootprintSubimage(posHfp, grow=3)
327
328 isBg = np.isnan(posFpImg.array).ravel()
329
330 data = posImg.array.ravel()
331 data = data[isBg]
332 B = data
333
334 x, y = np.mgrid[bbox.getBeginY():bbox.getEndY(), bbox.getBeginX():bbox.getEndX()]
335 x = x.astype(np.float64).ravel()
336 x -= np.mean(x)
337 x = x[isBg]
338 y = y.astype(np.float64).ravel()
339 y -= np.mean(y)
340 y = y[isBg]
341 b = np.ones_like(x, dtype=np.float64)
342
343 M = np.vstack([b]).T # order = 0
344 if order == 1:
345 M = np.vstack([b, x, y]).T
346 elif order == 2:
347 M = np.vstack([b, x, y, x**2., y**2., x*y]).T
348
349 pars = np.linalg.lstsq(M, B, rcond=-1)[0]
350 return pars
351
352 def makeStarModel(self, bbox, psf, xcen, ycen, flux):
353 """Generate a 2D image model of a single PDF centered at the given coordinates.
354
355 Parameters
356 ----------
357 bbox : `lsst.geom.Box`
358 Bounding box marking pixel coordinates for generated model
359 psf : TODO: DM-17458
360 Psf model used to generate the 'star'
361 xcen : `float`
362 Desired x-centroid of the 'star'
363 ycen : `float`
364 Desired y-centroid of the 'star'
365 flux : `float`
366 Desired flux of the 'star'
367
368 Returns
369 -------
370 p_Im : `lsst.afw.image.Image`
371 2-d stellar image of width/height matching input ``bbox``,
372 containing PSF with given centroid and flux
373 """
374
375 # Generate the psf image, normalize to flux
376 psf_img = psf.computeImage(geom.Point2D(xcen, ycen)).convertF()
377 psf_img_sum = np.nansum(psf_img.array)
378 psf_img *= (flux/psf_img_sum)
379
380 # Clip the PSF image bounding box to fall within the footprint bounding box
381 psf_box = psf_img.getBBox()
382 psf_box.clip(bbox)
383 psf_img = afwImage.ImageF(psf_img, psf_box, afwImage.PARENT)
384
385 # Then actually crop the psf image.
386 # Usually not necessary, but if the dipole is near the edge of the image...
387 # Would be nice if we could compare original pos_box with clipped pos_box and
388 # see if it actually was clipped.
389 p_Im = afwImage.ImageF(bbox)
390 tmpSubim = afwImage.ImageF(p_Im, psf_box, afwImage.PARENT)
391 tmpSubim += psf_img
392
393 return p_Im
394
395 def makeModel(self, x, flux, xcenPos, ycenPos, xcenNeg, ycenNeg, fluxNeg=None,
396 b=None, x1=None, y1=None, xy=None, x2=None, y2=None,
397 bNeg=None, x1Neg=None, y1Neg=None, xyNeg=None, x2Neg=None, y2Neg=None,
398 **kwargs):
399 """Generate dipole model with given parameters.
400
401 This is the function whose sum-of-squared difference from data
402 is minimized by `lmfit`.
403
404 x : TODO: DM-17458
405 Input independent variable. Used here as the grid on
406 which to compute the background gradient model.
407 flux : `float`
408 Desired flux of the positive lobe of the dipole
409 xcenPos, ycenPos : `float`
410 Desired x,y-centroid of the positive lobe of the dipole
411 xcenNeg, ycenNeg : `float`
412 Desired x,y-centroid of the negative lobe of the dipole
413 fluxNeg : `float`, optional
414 Desired flux of the negative lobe of the dipole, set to 'flux' if None
415 b, x1, y1, xy, x2, y2 : `float`
416 Gradient parameters for positive lobe.
417 bNeg, x1Neg, y1Neg, xyNeg, x2Neg, y2Neg : `float`, optional
418 Gradient parameters for negative lobe.
419 They are set to the corresponding positive values if None.
420
421 **kwargs : `dict` [`str`]
422 Keyword arguments passed through ``lmfit`` and
423 used by this function. These must include:
424
425 - ``psf`` Psf model used to generate the 'star'
426 - ``rel_weight`` Used to signify least-squares weighting of posImage/negImage
427 relative to diffim. If ``rel_weight == 0`` then posImage/negImage are ignored.
428 - ``bbox`` Bounding box containing region to be modelled
429
430 Returns
431 -------
432 zout : `numpy.array`
433 Has width and height matching the input bbox, and
434 contains the dipole model with given centroids and flux(es). If
435 ``rel_weight`` = 0, this is a 2-d array with dimensions matching
436 those of bbox; otherwise a stack of three such arrays,
437 representing the dipole (diffim), positive, and negative images
438 respectively.
439 """
440
441 psf = kwargs.get('psf')
442 rel_weight = kwargs.get('rel_weight') # if > 0, we're including pre-sub. images
443 fp = kwargs.get('footprint')
444 bbox = fp.getBBox()
445
446 if fluxNeg is None:
447 fluxNeg = flux
448
449 self.log.debug('flux: %.2f fluxNeg: %.2f x+: %.2f x-: %.2f y+: %.2f y-: %.2f ',
450 flux, fluxNeg, xcenPos, xcenNeg, ycenPos, ycenNeg)
451 if x1 is not None:
452 self.log.debug(' b: %.2f x1: %.2f y1: %.2f', b, x1, y1)
453 if xy is not None:
454 self.log.debug(' xy: %.2f x2: %.2f y2: %.2f', xy, x2, y2)
455
456 posIm = self.makeStarModel(bbox, psf, xcenPos, ycenPos, flux)
457 negIm = self.makeStarModel(bbox, psf, xcenNeg, ycenNeg, fluxNeg)
458
459 in_x = x
460 if in_x is None: # use the footprint to generate the input grid
461 y, x = np.mgrid[bbox.getBeginY():bbox.getEndY(), bbox.getBeginX():bbox.getEndX()]
462 in_x = np.array([x, y]) * 1.
463 in_x[0, :] -= in_x[0, :].mean() # center it!
464 in_x[1, :] -= in_x[1, :].mean()
465
466 if b is not None:
467 gradient = self.makeBackgroundModel(in_x, (b, x1, y1, xy, x2, y2))
468
469 # If bNeg is None, then don't fit the negative background separately
470 if bNeg is not None:
471 gradientNeg = self.makeBackgroundModel(in_x, (bNeg, x1Neg, y1Neg, xyNeg, x2Neg, y2Neg))
472 else:
473 gradientNeg = gradient
474
475 posIm.array[:, :] += gradient
476 negIm.array[:, :] += gradientNeg
477
478 # Generate the diffIm model
479 diffIm = afwImage.ImageF(bbox)
480 diffIm += posIm
481 diffIm -= negIm
482
483 zout = diffIm.array
484 if rel_weight > 0.:
485 zout = np.append([zout], [posIm.array, negIm.array], axis=0)
486
487 return zout
488
489
491 """Fit a dipole model using an image difference.
492
493 See also:
494 `DMTN-007: Dipole characterization for image differencing <https://dmtn-007.lsst.io>`_.
495 """
496
497 # This is just a private version number to sync with the ipython notebooks that I have been
498 # using for algorithm development.
499 _private_version_ = '0.0.5'
500
501 # Below is a (somewhat incomplete) list of improvements
502 # that would be worth investigating, given the time:
503
504 # todo 1. evaluate necessity for separate parameters for pos- and neg- images
505 # todo 2. only fit background OUTSIDE footprint (DONE) and dipole params INSIDE footprint (NOT DONE)?
506 # todo 3. correct normalization of least-squares weights based on variance planes
507 # todo 4. account for PSFs that vary across the exposures (should be happening by default?)
508 # todo 5. correctly account for NA/masks (i.e., ignore!)
509 # todo 6. better exception handling in the plugin
510 # todo 7. better classification of dipoles (e.g. by comparing chi2 fit vs. monopole?)
511 # todo 8. (DONE) Initial fast estimate of background gradient(s) params -- perhaps using numpy.lstsq
512 # todo 9. (NOT NEEDED - see (2)) Initial fast test whether a background gradient needs to be fit
513 # todo 10. (DONE) better initial estimate for flux when there's a strong gradient
514 # todo 11. (DONE) requires a new package `lmfit` -- investiate others? (astropy/scipy/iminuit?)
515
516 def __init__(self, diffim, posImage=None, negImage=None):
517 """Algorithm to run dipole measurement on a diaSource
518
519 Parameters
520 ----------
521 diffim : `lsst.afw.image.Exposure`
522 Exposure on which the diaSources were detected
523 posImage : `lsst.afw.image.Exposure`
524 "Positive" exposure from which the template was subtracted
525 negImage : `lsst.afw.image.Exposure`
526 "Negative" exposure which was subtracted from the posImage
527 """
528
529 self.diffim = diffim
530 self.posImage = posImage
531 self.negImage = negImage
532 self.psfSigma = None
533 if diffim is not None:
534 diffimPsf = diffim.getPsf()
535 diffimAvgPos = diffimPsf.getAveragePosition()
536 self.psfSigma = diffimPsf.computeShape(diffimAvgPos).getDeterminantRadius()
537
538 self.log = logging.getLogger(__name__)
539
540 import lsstDebug
541 self.debug = lsstDebug.Info(__name__).debug
542
543 def fitDipoleImpl(self, source, tol=1e-7, rel_weight=0.5,
544 fitBackground=1, bgGradientOrder=1, maxSepInSigma=5.,
545 separateNegParams=True, verbose=False):
546 """Fit a dipole model to an input difference image.
547
548 Actually, fits the subimage bounded by the input source's
549 footprint) and optionally constrain the fit using the
550 pre-subtraction images posImage and negImage.
551
552 Parameters
553 ----------
554 source : TODO: DM-17458
555 TODO: DM-17458
556 tol : float, optional
557 TODO: DM-17458
558 rel_weight : `float`, optional
559 TODO: DM-17458
560 fitBackground : `int`, optional
561 TODO: DM-17458
562 bgGradientOrder : `int`, optional
563 TODO: DM-17458
564 maxSepInSigma : `float`, optional
565 TODO: DM-17458
566 separateNegParams : `bool`, optional
567 TODO: DM-17458
568 verbose : `bool`, optional
569 TODO: DM-17458
570
571 Returns
572 -------
573 result : `lmfit.MinimizerResult`
574 return `lmfit.MinimizerResult` object containing the fit
575 parameters and other information.
576 """
577
578 # Only import lmfit if someone wants to use the new DipoleFitAlgorithm.
579 import lmfit
580
581 fp = source.getFootprint()
582 bbox = fp.getBBox()
583 subim = afwImage.MaskedImageF(self.diffim.getMaskedImage(), bbox=bbox, origin=afwImage.PARENT)
584
585 z = diArr = subim.image.array
586 # Make sure we don't overwrite buffers.
587 z = z.copy()
588 weights = 1. / subim.variance.array # get the weights (=1/variance)
589
590 if rel_weight > 0. and ((self.posImage is not None) or (self.negImage is not None)):
591 if self.negImage is not None:
592 negSubim = afwImage.MaskedImageF(self.negImage.getMaskedImage(), bbox, origin=afwImage.PARENT)
593 if self.posImage is not None:
594 posSubim = afwImage.MaskedImageF(self.posImage.getMaskedImage(), bbox, origin=afwImage.PARENT)
595 if self.posImage is None: # no science image provided; generate it from diffim + negImage
596 posSubim = subim.clone()
597 posSubim += negSubim
598 if self.negImage is None: # no template provided; generate it from the posImage - diffim
599 negSubim = posSubim.clone()
600 negSubim -= subim
601
602 z = np.append([z], [posSubim.image.array,
603 negSubim.image.array], axis=0)
604 # Weight the pos/neg images by rel_weight relative to the diffim
605 weights = np.append([weights], [1. / posSubim.variance.array * rel_weight,
606 1. / negSubim.variance.array * rel_weight], axis=0)
607 else:
608 rel_weight = 0. # a short-cut for "don't include the pre-subtraction data"
609
610 # It seems that `lmfit` requires a static functor as its optimized method, which eliminates
611 # the ability to pass a bound method or other class method. Here we write a wrapper which
612 # makes this possible.
613 def dipoleModelFunctor(x, flux, xcenPos, ycenPos, xcenNeg, ycenNeg, fluxNeg=None,
614 b=None, x1=None, y1=None, xy=None, x2=None, y2=None,
615 bNeg=None, x1Neg=None, y1Neg=None, xyNeg=None, x2Neg=None, y2Neg=None,
616 **kwargs):
617 """Generate dipole model with given parameters.
618
619 It simply defers to `modelObj.makeModel()`, where `modelObj` comes
620 out of `kwargs['modelObj']`.
621 """
622 modelObj = kwargs.pop('modelObj')
623 return modelObj.makeModel(x, flux, xcenPos, ycenPos, xcenNeg, ycenNeg, fluxNeg=fluxNeg,
624 b=b, x1=x1, y1=y1, xy=xy, x2=x2, y2=y2,
625 bNeg=bNeg, x1Neg=x1Neg, y1Neg=y1Neg, xyNeg=xyNeg,
626 x2Neg=x2Neg, y2Neg=y2Neg, **kwargs)
627
628 dipoleModel = DipoleModel()
629
630 modelFunctor = dipoleModelFunctor # dipoleModel.makeModel does not work for now.
631 # Create the lmfit model (lmfit uses scipy 'leastsq' option by default - Levenberg-Marquardt)
632 # We have to (later) filter out the nans by hand in our input to gmod.fit().
633 # The only independent variable in the model is "x"; lmfit tries to
634 # introspect variables and parameters from the function signature, but
635 # gets it wrong for the model signature above.
636 gmod = lmfit.Model(modelFunctor, independent_vars=["x"], verbose=verbose)
637
638 # Add the constraints for centroids, fluxes.
639 # starting constraint - near centroid of footprint
640 fpCentroid = np.array([fp.getCentroid().getX(), fp.getCentroid().getY()])
641 cenNeg = cenPos = fpCentroid
642
643 pks = fp.getPeaks()
644
645 if len(pks) >= 1:
646 cenPos = pks[0].getF() # if individual (merged) peaks were detected, use those
647 if len(pks) >= 2: # peaks are already sorted by centroid flux so take the most negative one
648 cenNeg = pks[-1].getF()
649
650 # For close/faint dipoles the starting locs (min/max) might be way off, let's help them a bit.
651 # First assume dipole is not separated by more than 5*psfSigma.
652 maxSep = self.psfSigma * maxSepInSigma
653
654 # As an initial guess -- assume the dipole is close to the center of the footprint.
655 if np.sum(np.sqrt((np.array(cenPos) - fpCentroid)**2.)) > maxSep:
656 cenPos = fpCentroid
657 if np.sum(np.sqrt((np.array(cenNeg) - fpCentroid)**2.)) > maxSep:
658 cenPos = fpCentroid
659
660 # parameter hints/constraints: https://lmfit.github.io/lmfit-py/model.html#model-param-hints-section
661 # might make sense to not use bounds -- see http://lmfit.github.io/lmfit-py/bounds.html
662 # also see this discussion -- https://github.com/scipy/scipy/issues/3129
663 gmod.set_param_hint('xcenPos', value=cenPos[0],
664 min=cenPos[0]-maxSep, max=cenPos[0]+maxSep)
665 gmod.set_param_hint('ycenPos', value=cenPos[1],
666 min=cenPos[1]-maxSep, max=cenPos[1]+maxSep)
667 gmod.set_param_hint('xcenNeg', value=cenNeg[0],
668 min=cenNeg[0]-maxSep, max=cenNeg[0]+maxSep)
669 gmod.set_param_hint('ycenNeg', value=cenNeg[1],
670 min=cenNeg[1]-maxSep, max=cenNeg[1]+maxSep)
671
672 # Use the (flux under the dipole)*5 for an estimate.
673 # Lots of testing showed that having startingFlux be too high was better than too low.
674 startingFlux = np.nansum(np.abs(diArr) - np.nanmedian(np.abs(diArr))) * 5.
675 posFlux = negFlux = startingFlux
676
677 # TBD: set max. flux limit?
678 gmod.set_param_hint('flux', value=posFlux, min=0.1)
679
680 if separateNegParams:
681 # TBD: set max negative lobe flux limit?
682 gmod.set_param_hint('fluxNeg', value=np.abs(negFlux), min=0.1)
683
684 # Fixed parameters (don't fit for them if there are no pre-sub images or no gradient fit requested):
685 # Right now (fitBackground == 1), we fit a linear model to the background and then subtract
686 # it from the data and then don't fit the background again (this is faster).
687 # A slower alternative (fitBackground == 2) is to use the estimated background parameters as
688 # starting points in the integrated model fit. That is currently not performed by default,
689 # but might be desirable in some cases.
690 bgParsPos = bgParsNeg = (0., 0., 0.)
691 if ((rel_weight > 0.) and (fitBackground != 0) and (bgGradientOrder >= 0)):
692 pbg = 0.
693 bgFitImage = self.posImage if self.posImage is not None else self.negImage
694 # Fit the gradient to the background (linear model)
695 bgParsPos = bgParsNeg = dipoleModel.fitFootprintBackground(source, bgFitImage,
696 order=bgGradientOrder)
697
698 # Generate the gradient and subtract it from the pre-subtraction image data
699 if fitBackground == 1:
700 in_x = dipoleModel._generateXYGrid(bbox)
701 pbg = dipoleModel.makeBackgroundModel(in_x, tuple(bgParsPos))
702 z[1, :] -= pbg
703 z[1, :] -= np.nanmedian(z[1, :])
704 posFlux = np.nansum(z[1, :])
705 gmod.set_param_hint('flux', value=posFlux*1.5, min=0.1)
706
707 if separateNegParams and self.negImage is not None:
708 bgParsNeg = dipoleModel.fitFootprintBackground(source, self.negImage,
709 order=bgGradientOrder)
710 pbg = dipoleModel.makeBackgroundModel(in_x, tuple(bgParsNeg))
711 z[2, :] -= pbg
712 z[2, :] -= np.nanmedian(z[2, :])
713 if separateNegParams:
714 negFlux = np.nansum(z[2, :])
715 gmod.set_param_hint('fluxNeg', value=negFlux*1.5, min=0.1)
716
717 # Do not subtract the background from the images but include the background parameters in the fit
718 if fitBackground == 2:
719 if bgGradientOrder >= 0:
720 gmod.set_param_hint('b', value=bgParsPos[0])
721 if separateNegParams:
722 gmod.set_param_hint('bNeg', value=bgParsNeg[0])
723 if bgGradientOrder >= 1:
724 gmod.set_param_hint('x1', value=bgParsPos[1])
725 gmod.set_param_hint('y1', value=bgParsPos[2])
726 if separateNegParams:
727 gmod.set_param_hint('x1Neg', value=bgParsNeg[1])
728 gmod.set_param_hint('y1Neg', value=bgParsNeg[2])
729 if bgGradientOrder >= 2:
730 gmod.set_param_hint('xy', value=bgParsPos[3])
731 gmod.set_param_hint('x2', value=bgParsPos[4])
732 gmod.set_param_hint('y2', value=bgParsPos[5])
733 if separateNegParams:
734 gmod.set_param_hint('xyNeg', value=bgParsNeg[3])
735 gmod.set_param_hint('x2Neg', value=bgParsNeg[4])
736 gmod.set_param_hint('y2Neg', value=bgParsNeg[5])
737
738 y, x = np.mgrid[bbox.getBeginY():bbox.getEndY(), bbox.getBeginX():bbox.getEndX()]
739 in_x = np.array([x, y]).astype(np.float64)
740 in_x[0, :] -= in_x[0, :].mean() # center it!
741 in_x[1, :] -= in_x[1, :].mean()
742
743 # Instead of explicitly using a mask to ignore flagged pixels, just set the ignored pixels'
744 # weights to 0 in the fit. TBD: need to inspect mask planes to set this mask.
745 mask = np.ones_like(z, dtype=bool) # TBD: set mask values to False if the pixels are to be ignored
746
747 # I'm not sure about the variance planes in the diffim (or convolved pre-sub. images
748 # for that matter) so for now, let's just do an un-weighted least-squares fit
749 # (override weights computed above).
750 weights = mask.astype(np.float64)
751 if self.posImage is not None and rel_weight > 0.:
752 weights = np.array([np.ones_like(diArr), np.ones_like(diArr)*rel_weight,
753 np.ones_like(diArr)*rel_weight])
754
755 # Set the weights to zero if mask is False
756 if np.any(~mask):
757 weights[~mask] = 0.
758
759 # Filter out any nans, and make the weights 0.
760 nans = (np.isnan(z) | np.isnan(weights))
761 nNans = nans.sum()
762 if nNans > 0:
763 if nNans < len(z):
764 z[nans] = np.nanmedian(z)
765 else:
766 z[nans] = 0
767 weights[nans] = 0
768
769 # Note that although we can, we're not required to set initial values for params here,
770 # since we set their param_hint's above.
771 # Can add "method" param to not use 'leastsq' (==levenberg-marquardt), e.g. "method='nelder'"
772 with warnings.catch_warnings():
773 # Ignore lmfit unknown argument warnings:
774 # "psf, rel_weight, footprint, modelObj" all become pass-through kwargs for makeModel.
775 warnings.filterwarnings("ignore", "The keyword argument .* does not match", UserWarning)
776 result = gmod.fit(z, weights=weights, x=in_x, max_nfev=250,
777 method="leastsq", # TODO: try using `least_squares` here for speed/robustness
778 verbose=verbose,
779 # see scipy docs for the meaning of these keywords
780 fit_kws={'ftol': tol, 'xtol': tol, 'gtol': tol,
781 # Our model is float32 internally, so we need a larger epsfcn.
782 'epsfcn': 1e-8},
783 psf=self.diffim.getPsf(), # hereon: kwargs that get passed to makeModel()
784 rel_weight=rel_weight,
785 footprint=fp,
786 modelObj=dipoleModel)
787
788 if verbose: # the ci_report() seems to fail if neg params are constrained -- TBD why.
789 # Never wanted in production - this takes a long time (longer than the fit!)
790 # This is how to get confidence intervals out:
791 # https://lmfit.github.io/lmfit-py/confidence.html and
792 # http://cars9.uchicago.edu/software/python/lmfit/model.html
793 print(result.fit_report(show_correl=False))
794 if separateNegParams:
795 print(result.ci_report())
796
797 return result
798
799 def fitDipole(self, source, tol=1e-7, rel_weight=0.1,
800 fitBackground=1, maxSepInSigma=5., separateNegParams=True,
801 bgGradientOrder=1, verbose=False, display=False):
802 """Fit a dipole model to an input ``diaSource`` (wraps `fitDipoleImpl`).
803
804 Actually, fits the subimage bounded by the input source's
805 footprint) and optionally constrain the fit using the
806 pre-subtraction images self.posImage (science) and
807 self.negImage (template). Wraps the output into a
808 `pipeBase.Struct` named tuple after computing additional
809 statistics such as orientation and SNR.
810
811 Parameters
812 ----------
813 source : `lsst.afw.table.SourceRecord`
814 Record containing the (merged) dipole source footprint detected on the diffim
815 tol : `float`, optional
816 Tolerance parameter for scipy.leastsq() optimization
817 rel_weight : `float`, optional
818 Weighting of posImage/negImage relative to the diffim in the fit
819 fitBackground : `int`, {0, 1, 2}, optional
820 How to fit linear background gradient in posImage/negImage
821
822 - 0: do not fit background at all
823 - 1 (default): pre-fit the background using linear least squares and then do not fit it
824 as part of the dipole fitting optimization
825 - 2: pre-fit the background using linear least squares (as in 1), and use the parameter
826 estimates from that fit as starting parameters for an integrated "re-fit" of the
827 background as part of the overall dipole fitting optimization.
828 maxSepInSigma : `float`, optional
829 Allowed window of centroid parameters relative to peak in input source footprint
830 separateNegParams : `bool`, optional
831 Fit separate parameters to the flux and background gradient in
832 bgGradientOrder : `int`, {0, 1, 2}, optional
833 Desired polynomial order of background gradient
834 verbose: `bool`, optional
835 Be verbose
836 display
837 Display input data, best fit model(s) and residuals in a matplotlib window.
838
839 Returns
840 -------
841 result : `struct`
842 `pipeBase.Struct` object containing the fit parameters and other information.
843
844 result : `callable`
845 `lmfit.MinimizerResult` object for debugging and error estimation, etc.
846
847 Notes
848 -----
849 Parameter `fitBackground` has three options, thus it is an integer:
850
851 """
852
853 fitResult = self.fitDipoleImpl(
854 source, tol=tol, rel_weight=rel_weight, fitBackground=fitBackground,
855 maxSepInSigma=maxSepInSigma, separateNegParams=separateNegParams,
856 bgGradientOrder=bgGradientOrder, verbose=verbose)
857
858 # Display images, model fits and residuals (currently uses matplotlib display functions)
859 if display:
860 fp = source.getFootprint()
861 self.displayFitResults(fp, fitResult)
862
863 # usually around 0.1 -- the minimum flux allowed -- i.e. bad fit.
864 if fitResult.params['flux'].value <= 1.:
865 self.log.debug("Fitted flux too small for id=%d; ModelResult.message='%s'",
866 source["id"], fitResult.message)
867 return None, fitResult
868 if not fitResult.result.errorbars:
869 self.log.debug("Could not estimate error bars for id=%d; ModelResult.message='%s'",
870 source["id"], fitResult.message)
871 return None, fitResult
872
873 # TODO: We could include covariances, which could be derived from
874 # `fitResult.params[name].correl`, but those are correlations.
875 posCentroid = measBase.CentroidResult(fitResult.params['xcenPos'].value,
876 fitResult.params['ycenPos'].value,
877 fitResult.params['xcenPos'].stderr,
878 fitResult.params['ycenPos'].stderr)
879 negCentroid = measBase.CentroidResult(fitResult.params['xcenNeg'].value,
880 fitResult.params['ycenNeg'].value,
881 fitResult.params['xcenNeg'].stderr,
882 fitResult.params['ycenNeg'].stderr)
883 xposIdx = fitResult.var_names.index("xcenPos")
884 yposIdx = fitResult.var_names.index("ycenPos")
885 xnegIdx = fitResult.var_names.index("xcenNeg")
886 ynegIdx = fitResult.var_names.index("ycenNeg")
887 centroid = measBase.CentroidResult((fitResult.params['xcenPos'] + fitResult.params['xcenNeg']) / 2,
888 (fitResult.params['ycenPos'] + fitResult.params['ycenNeg']) / 2.,
889 math.sqrt(posCentroid.xErr**2 + negCentroid.xErr**2
890 + 2*fitResult.covar[xposIdx, xnegIdx]) / 2.,
891 math.sqrt(posCentroid.yErr**2 + negCentroid.yErr**2
892 + 2*fitResult.covar[yposIdx, ynegIdx]) / 2.)
893 dx = fitResult.params['xcenPos'].value - fitResult.params['xcenNeg'].value
894 dy = fitResult.params['ycenPos'].value - fitResult.params['ycenNeg'].value
895 angle = np.arctan2(dy, dx)
896
897 # Extract flux value, compute signalToNoise from flux/variance_within_footprint
898 # Also extract the stderr of flux estimate.
899 # TODO: should this instead use the lmfit-computed uncertainty from
900 # `lmfitResult.result.uvars['flux'].std_dev`?
901 def computeSumVariance(exposure, footprint):
902 return math.sqrt(np.nansum(exposure[footprint.getBBox(), afwImage.PARENT].variance.array))
903
904 # NOTE: These will all be the same unless separateNegParams=True!
905 flux = measBase.FluxResult(fitResult.params["flux"].value, fitResult.params["flux"].stderr)
906 posFlux = measBase.FluxResult(fitResult.params["flux"].value, fitResult.params["flux"].stderr)
907 negFlux = measBase.FluxResult(fitResult.params["flux"].value, fitResult.params["flux"].stderr)
908 if self.posImage is not None:
909 fluxVar = computeSumVariance(self.posImage, source.getFootprint())
910 else:
911 fluxVar = computeSumVariance(self.diffim, source.getFootprint())
912 fluxVarNeg = fluxVar
913
914 if separateNegParams:
915 negFlux.instFlux = fitResult.params['fluxNeg'].value
916 negFlux.instFluxErr = fitResult.params['fluxNeg'].stderr
917 if self.negImage is not None:
918 fluxVarNeg = computeSumVariance(self.negImage, source.getFootprint())
919
920 try:
921 signalToNoise = math.sqrt((posFlux.instFlux/fluxVar)**2 + (negFlux.instFlux/fluxVarNeg)**2)
922 except ZeroDivisionError: # catch divide by zero - should never happen.
923 signalToNoise = np.nan
924
925 out = Struct(posCentroid=posCentroid, negCentroid=negCentroid, centroid=centroid,
926 posFlux=posFlux, negFlux=negFlux, flux=flux, orientation=angle,
927 signalToNoise=signalToNoise, chi2=fitResult.chisqr, redChi2=fitResult.redchi,
928 nData=fitResult.ndata)
929
930 # fitResult may be returned for debugging
931 return out, fitResult
932
933 def displayFitResults(self, footprint, result):
934 """Display data, model fits and residuals (currently uses matplotlib display functions).
935
936 Parameters
937 ----------
938 footprint : `lsst.afw.detection.Footprint`
939 Footprint containing the dipole that was fit
940 result : `lmfit.MinimizerResult`
941 `lmfit.MinimizerResult` object returned by `lmfit` optimizer
942 """
943 try:
944 import matplotlib.pyplot as plt
945 except ImportError as err:
946 self.log.warning('Unable to import matplotlib: %s', err)
947 raise err
948
949 def display2dArray(ax, arr, x, y, xErr, yErr, title, extent=None):
950 """Use `matplotlib.pyplot.imshow` to display a 2-D array with a given coordinate range.
951 """
952 fig = ax.imshow(arr, origin='lower', interpolation='none', cmap='gray', extent=extent)
953 ax.set_title(title)
954 ax.errorbar(x["total"], y["total"], xErr["total"], yErr["total"], c="cyan")
955 ax.errorbar(x["Pos"], y["Pos"], xErr["Pos"], yErr["Pos"], c="green")
956 ax.errorbar(x["Neg"], y["Neg"], xErr["Neg"], yErr["Neg"], c="red")
957 return fig
958
959 z = result.data
960 fit = result.best_fit
961 bbox = footprint.getBBox()
962 extent = (bbox.getBeginX(), bbox.getEndX(), bbox.getBeginY(), bbox.getEndY())
963
964 if z.shape[0] == 3:
965 x, y, xErr, yErr = {}, {}, {}, {}
966 for name in ("Pos", "Neg"):
967 x[name] = result.best_values[f"xcen{name}"]
968 y[name] = result.best_values[f"ycen{name}"]
969 xErr[name] = result.params[f"xcen{name}"].stderr
970 yErr[name] = result.params[f"ycen{name}"].stderr
971 x["total"] = (result.best_values["xcenPos"] + result.best_values["xcenNeg"])/2
972 y["total"] = (result.best_values["ycenPos"] + result.best_values["ycenNeg"])/2
973 xErr["total"] = math.sqrt(xErr["Pos"]**2 + xErr["Neg"]**2)
974 yErr["total"] = math.sqrt(yErr["Pos"]**2 + yErr["Neg"]**2)
975
976 fig, axes = plt.subplots(nrows=3, ncols=3, sharex=True, sharey=True, figsize=(8, 8))
977 for i, label in enumerate(("total", "Pos", "Neg")):
978 display2dArray(axes[i][0], z[i, :], x, y, xErr, yErr,
979 f'Data {label}', extent=extent)
980 display2dArray(axes[i][1], fit[i, :], x, y, xErr, yErr,
981 f'Model {label}', extent=extent)
982 display2dArray(axes[i][2], z[i, :] - fit[i, :], x, y, xErr, yErr,
983 f'Residual {label}', extent=extent)
984
985 plt.setp(axes[i][1].get_yticklabels(), visible=False)
986 plt.setp(axes[i][2].get_yticklabels(), visible=False)
987 if i != 2: # remove top two row x-axis labels
988 plt.setp(axes[i][0].get_xticklabels(), visible=False)
989 plt.setp(axes[i][1].get_xticklabels(), visible=False)
990 plt.setp(axes[i][2].get_xticklabels(), visible=False)
991 else:
992 fig, axes = plt.subplots(nrows=3, ncols=3, sharex=True, sharey=True, figsize=(10, 2.5))
993 display2dArray(axes[0], z, 'Data', extent=extent)
994 display2dArray(axes[1], 'Model', extent=extent)
995 display2dArray(axes[2], z - fit, 'Residual', extent=extent)
996
997 fig.tight_layout(pad=0, w_pad=0, h_pad=0)
998 plt.show()
999
1000
1001@measBase.register("ip_diffim_DipoleFit")
1002class DipoleFitPlugin(measBase.SingleFramePlugin):
1003 """A single frame measurement plugin that fits dipoles to all merged (two-peak) ``diaSources``.
1004
1005 This measurement plugin accepts up to three input images in
1006 its `measure` method. If these are provided, it includes data
1007 from the pre-subtraction posImage (science image) and optionally
1008 negImage (template image) to constrain the fit. The meat of the
1009 fitting routines are in the class `~lsst.module.name.DipoleFitAlgorithm`.
1010
1011 Notes
1012 -----
1013 The motivation behind this plugin and the necessity for including more than
1014 one exposure are documented in DMTN-007 (http://dmtn-007.lsst.io).
1015
1016 This class is named `ip_diffim_DipoleFit` so that it may be used alongside
1017 the existing `ip_diffim_DipoleMeasurement` classes until such a time as those
1018 are deemed to be replaceable by this.
1019 """
1020
1021 ConfigClass = DipoleFitPluginConfig
1022 DipoleFitAlgorithmClass = DipoleFitAlgorithm # Pointer to the class that performs the fit
1023
1024 FAILURE_EDGE = 1 # too close to the edge
1025 FAILURE_FIT = 2 # failure in the fitting
1026 FAILURE_NOT_DIPOLE = 4 # input source is not a putative dipole to begin with
1027 FAILURE_TOO_LARGE = 8 # input source is too large to be fit
1028
1029 @classmethod
1031 """This algorithm simultaneously fits the centroid and flux, and does
1032 not require any previous centroid fit.
1033 """
1034 return cls.CENTROID_ORDER
1035
1036 def __init__(self, config, name, schema, metadata, logName=None):
1037 if logName is None:
1038 logName = name
1039 measBase.SingleFramePlugin.__init__(self, config, name, schema, metadata, logName=logName)
1040
1041 self.log = logging.getLogger(logName)
1042
1043 self._setupSchema(config, name, schema, metadata)
1044
1045 def _setupSchema(self, config, name, schema, metadata):
1046 """Add fields for the outputs, and save the keys for fast assignment.
1047 """
1048 self.posFluxKey = measBase.FluxResultKey.addFields(schema,
1049 schema.join(name, "pos"),
1050 "Dipole positive lobe instrumental flux.")
1051 self.negFluxKey = measBase.FluxResultKey.addFields(schema,
1052 schema.join(name, "neg"),
1053 "Dipole negative lobe instrumental flux.")
1054 doc = "Dipole overall instrumental flux (mean of absolute value of positive and negative lobes)."
1055 self.fluxKey = measBase.FluxResultKey.addFields(schema, name, doc)
1056
1057 self.posCentroidKey = measBase.CentroidResultKey.addFields(schema,
1058 schema.join(name, "pos"),
1059 "Dipole positive lobe centroid position.",
1060 measBase.UncertaintyEnum.SIGMA_ONLY)
1061 self.negCentroidKey = measBase.CentroidResultKey.addFields(schema,
1062 schema.join(name, "neg"),
1063 "Dipole negative lobe centroid position.",
1064 measBase.UncertaintyEnum.SIGMA_ONLY)
1065 self.centroidKey = measBase.CentroidResultKey.addFields(schema,
1066 name,
1067 "Dipole centroid position.",
1068 measBase.UncertaintyEnum.SIGMA_ONLY)
1069
1070 self.orientationKey = schema.addField(
1071 schema.join(name, "orientation"), type=float, units="rad",
1072 doc="Dipole orientation. Convention is CCW from +x on image.")
1073
1074 self.separationKey = schema.addField(
1075 schema.join(name, "separation"), type=float, units="pixel",
1076 doc="Pixel separation between positive and negative lobes of dipole")
1077
1078 self.chi2dofKey = schema.addField(
1079 schema.join(name, "chi2dof"), type=float,
1080 doc="Chi2 per degree of freedom (chi2/(nData-nVariables)) of dipole fit")
1081
1082 self.nDataKey = schema.addField(
1083 schema.join(name, "nData"), type=np.int64,
1084 doc="Number of data points in the dipole fit")
1085
1086 self.signalToNoiseKey = schema.addField(
1087 schema.join(name, "signalToNoise"), type=float,
1088 doc="Estimated signal-to-noise of dipole fit")
1089
1090 self.classificationFlagKey = schema.addField(
1091 schema.join(name, "classification"), type="Flag",
1092 doc="Flag indicating diaSource is classified as a dipole")
1093
1094 self.classificationAttemptedFlagKey = schema.addField(
1095 schema.join(name, "classificationAttempted"), type="Flag",
1096 doc="Flag indicating diaSource was attempted to be classified as a dipole")
1097
1098 self.flagKey = schema.addField(
1099 schema.join(name, "flag"), type="Flag",
1100 doc="General failure flag for dipole fit")
1101
1102 self.edgeFlagKey = schema.addField(
1103 schema.join(name, "flag", "edge"), type="Flag",
1104 doc="Flag set when dipole is too close to edge of image")
1105
1106 def measureDipoles(self, measRecord, exposure, posExp=None, negExp=None):
1107 """Perform the non-linear least squares minimization on the putative dipole source.
1108
1109 Parameters
1110 ----------
1111 measRecord : `lsst.afw.table.SourceRecord`
1112 diaSources that will be measured using dipole measurement
1113 exposure : `lsst.afw.image.Exposure`
1114 Difference exposure on which the diaSources were detected; `exposure = posExp-negExp`
1115 If both `posExp` and `negExp` are `None`, will attempt to fit the
1116 dipole to just the `exposure` with no constraint.
1117 posExp : `lsst.afw.image.Exposure`, optional
1118 "Positive" exposure, typically a science exposure, or None if unavailable
1119 When `posExp` is `None`, will compute `posImage = exposure + negExp`.
1120 negExp : `lsst.afw.image.Exposure`, optional
1121 "Negative" exposure, typically a template exposure, or None if unavailable
1122 When `negExp` is `None`, will compute `negImage = posExp - exposure`.
1123
1124 Notes
1125 -----
1126 The main functionality of this routine was placed outside of
1127 this plugin (into `DipoleFitAlgorithm.fitDipole()`) so that
1128 `DipoleFitAlgorithm.fitDipole()` can be called separately for
1129 testing (@see `tests/testDipoleFitter.py`)
1130 """
1131 result = None
1132 pks = measRecord.getFootprint().getPeaks()
1133
1134 # Check if the footprint consists of a putative dipole - else don't fit it.
1135 if (
1136 # One peak in the footprint (not a dipole)
1137 ((nPeaks := len(pks)) <= 1)
1138 # Peaks are the same sign (not a dipole); peaks are ordered
1139 # from highest to lowest.
1140 or (nPeaks > 1 and (np.sign(pks[0].getPeakValue())
1141 == np.sign(pks[-1].getPeakValue())))
1142 ):
1143 if not self.config.fitAllDiaSources:
1144 # Non-dipoles fall back on the centroid slot for positions,
1145 # errors, and the failure flag, if we're not fitting them.
1146 measRecord[self.centroidKey.getX()] = measRecord.getX()
1147 measRecord[self.centroidKey.getY()] = measRecord.getY()
1148 self.centroidKey.getCentroidErr().set(measRecord, measRecord.getCentroidErr())
1149 measRecord[self.flagKey] = measRecord.getCentroidFlag()
1150 return
1151
1152 # Footprint is too large (not a dipole).
1153 if ((area := measRecord.getFootprint().getArea()) > self.config.maxFootprintArea):
1154 self.fail(measRecord, measBase.MeasurementError(f"{area} > {self.config.maxFootprintArea}",
1155 self.FAILURE_TOO_LARGE))
1156 return
1157
1158 try:
1159 alg = self.DipoleFitAlgorithmClass(exposure, posImage=posExp, negImage=negExp)
1160 result, _ = alg.fitDipole(
1161 measRecord, rel_weight=self.config.relWeight,
1162 tol=self.config.tolerance,
1163 maxSepInSigma=self.config.maxSeparation,
1164 fitBackground=self.config.fitBackground,
1165 separateNegParams=self.config.fitSeparateNegParams,
1166 verbose=False, display=False)
1167 except pexExcept.LengthError:
1168 self.fail(measRecord, measBase.MeasurementError('edge failure', self.FAILURE_EDGE))
1169 except Exception as e:
1170 errorMessage = f"Exception in dipole fit. {e.__class__.__name__}: {e}"
1171 self.fail(measRecord, measBase.MeasurementError(errorMessage, self.FAILURE_FIT))
1172
1173 self.log.debug("Dipole fit result: %d %s", measRecord.getId(), str(result))
1174
1175 if result is None:
1176 self.fail(measRecord, measBase.MeasurementError("bad dipole fit", self.FAILURE_FIT))
1177 return
1178
1179 # TODO: add chi2, dipole classification
1180 self.posFluxKey.set(measRecord, result.posFlux)
1181 self.posCentroidKey.set(measRecord, result.posCentroid)
1182
1183 self.negFluxKey.set(measRecord, result.negFlux)
1184 self.negCentroidKey.set(measRecord, result.negCentroid)
1185
1186 self.fluxKey.set(measRecord, result.flux)
1187 self.centroidKey.set(measRecord, result.centroid)
1188
1189 measRecord[self.orientationKey] = result.orientation
1190 measRecord[self.separationKey] = math.sqrt((result.posCentroid.x - result.negCentroid.x)**2
1191 + (result.posCentroid.y - result.negCentroid.y)**2)
1192
1193 measRecord[self.signalToNoiseKey] = result.signalToNoise
1194 measRecord[self.chi2dofKey] = result.redChi2
1195
1196 if result.nData >= 1:
1197 measRecord[self.nDataKey] = result.nData
1198 else:
1199 measRecord[self.nDataKey] = 0
1200
1201 self.doClassify(measRecord, result.chi2)
1202
1203 def doClassify(self, measRecord, chi2val):
1204 """Classify a source as a dipole.
1205
1206 Parameters
1207 ----------
1208 measRecord : TODO: DM-17458
1209 TODO: DM-17458
1210 chi2val : TODO: DM-17458
1211 TODO: DM-17458
1212
1213 Notes
1214 -----
1215 Sources are classified as dipoles, or not, according to three criteria:
1216
1217 1. Does the total signal-to-noise surpass the ``minSn``?
1218 2. Are the pos/neg fluxes greater than 1.0 and no more than 0.65 (``maxFluxRatio``)
1219 of the total flux? By default this will never happen since ``posFlux == negFlux``.
1220 3. Is it a good fit (``chi2dof`` < 1)? (Currently not used.)
1221 """
1222
1223 # First, does the total signal-to-noise surpass the minSn?
1224 passesSn = measRecord[self.signalToNoiseKey] > self.config.minSn
1225
1226 # Second, are the pos/neg fluxes greater than 1.0 and no more than 0.65 (param maxFluxRatio)
1227 # of the total flux? By default this will never happen since posFlux = negFlux.
1228 passesFluxPos = (abs(measRecord[self.posFluxKey.getInstFlux()])
1229 / (measRecord[self.fluxKey.getInstFlux()]*2.)) < self.config.maxFluxRatio
1230 passesFluxPos &= (abs(measRecord[self.posFluxKey.getInstFlux()]) >= 1.0)
1231 passesFluxNeg = (abs(measRecord[self.negFluxKey.getInstFlux()])
1232 / (measRecord[self.fluxKey.getInstFlux()]*2.)) < self.config.maxFluxRatio
1233 passesFluxNeg &= (abs(measRecord[self.negFluxKey.getInstFlux()]) >= 1.0)
1234 allPass = (passesSn and passesFluxPos and passesFluxNeg) # and passesChi2)
1235
1236 # Third, is it a good fit (chi2dof < 1)?
1237 # Use scipy's chi2 cumulative distrib to estimate significance
1238 # This doesn't really work since I don't trust the values in the variance plane (which
1239 # affects the least-sq weights, which affects the resulting chi2).
1240 # But I'm going to keep this here for future use.
1241 if False:
1242 from scipy.stats import chi2
1243 ndof = chi2val / measRecord[self.chi2dofKey]
1244 significance = chi2.cdf(chi2val, ndof)
1245 passesChi2 = significance < self.config.maxChi2DoF
1246 allPass = allPass and passesChi2
1247
1248 measRecord.set(self.classificationAttemptedFlagKey, True)
1249
1250 if allPass: # Note cannot pass `allPass` into the `measRecord.set()` call below...?
1251 measRecord.set(self.classificationFlagKey, True)
1252 else:
1253 measRecord.set(self.classificationFlagKey, False)
1254
1255 def fail(self, measRecord, error=None):
1256 """Catch failures and set the correct flags.
1257
1258 Fallback on the current slot centroid positions, but set the dipole
1259 failure flag, since we attempted to fit the source.
1260 """
1261 measRecord[self.centroidKey.getX()] = measRecord.getX()
1262 measRecord[self.centroidKey.getY()] = measRecord.getY()
1263 self.centroidKey.getCentroidErr().set(measRecord, measRecord.getCentroidErr())
1264
1265 measRecord.set(self.flagKey, True)
1266 if error is not None:
1267 if error.getFlagBit() == self.FAILURE_EDGE:
1268 self.log.debug('DipoleFitPlugin not run on record %d: %s', measRecord.getId(), str(error))
1269 measRecord.set(self.edgeFlagKey, True)
1270 if error.getFlagBit() == self.FAILURE_FIT:
1271 self.log.warning('DipoleFitPlugin failed on record %d: %s', measRecord.getId(), str(error))
1272 if error.getFlagBit() == self.FAILURE_TOO_LARGE:
1273 self.log.debug('DipoleFitPlugin not run on record with too large footprint %d: %s',
1274 measRecord.getId(), str(error))
1275 else:
1276 self.log.warning('DipoleFitPlugin failed on record %d', measRecord.getId())
fitDipole(self, source, tol=1e-7, rel_weight=0.1, fitBackground=1, maxSepInSigma=5., separateNegParams=True, bgGradientOrder=1, verbose=False, display=False)
fitDipoleImpl(self, source, tol=1e-7, rel_weight=0.5, fitBackground=1, bgGradientOrder=1, maxSepInSigma=5., separateNegParams=True, verbose=False)
__init__(self, diffim, posImage=None, negImage=None)
measureDipoles(self, measRecord, exposure, posExp=None, negExp=None)
_setupSchema(self, config, name, schema, metadata)
__init__(self, config, name, schema, metadata, logName=None)
__init__(self, schema, algMetadata=None, **kwargs)
run(self, sources, exposure, posExp=None, negExp=None, **kwargs)
makeStarModel(self, bbox, psf, xcen, ycen, flux)
_getHeavyFootprintSubimage(self, fp, badfill=np.nan, grow=0)
makeModel(self, x, flux, xcenPos, ycenPos, xcenNeg, ycenNeg, fluxNeg=None, b=None, x1=None, y1=None, xy=None, x2=None, y2=None, bNeg=None, x1Neg=None, y1Neg=None, xyNeg=None, x2Neg=None, y2Neg=None, **kwargs)
fitFootprintBackground(self, source, posImage, order=1)