LSST Applications 27.0.0,g0265f82a02+469cd937ee,g02d81e74bb+21ad69e7e1,g1470d8bcf6+cbe83ee85a,g2079a07aa2+e67c6346a6,g212a7c68fe+04a9158687,g2305ad1205+94392ce272,g295015adf3+81dd352a9d,g2bbee38e9b+469cd937ee,g337abbeb29+469cd937ee,g3939d97d7f+72a9f7b576,g487adcacf7+71499e7cba,g50ff169b8f+5929b3527e,g52b1c1532d+a6fc98d2e7,g591dd9f2cf+df404f777f,g5a732f18d5+be83d3ecdb,g64a986408d+21ad69e7e1,g858d7b2824+21ad69e7e1,g8a8a8dda67+a6fc98d2e7,g99cad8db69+f62e5b0af5,g9ddcbc5298+d4bad12328,ga1e77700b3+9c366c4306,ga8c6da7877+71e4819109,gb0e22166c9+25ba2f69a1,gb6a65358fc+469cd937ee,gbb8dafda3b+69d3c0e320,gc07e1c2157+a98bf949bb,gc120e1dc64+615ec43309,gc28159a63d+469cd937ee,gcf0d15dbbd+72a9f7b576,gdaeeff99f8+a38ce5ea23,ge6526c86ff+3a7c1ac5f1,ge79ae78c31+469cd937ee,gee10cc3b42+a6fc98d2e7,gf1cff7945b+21ad69e7e1,gfbcc870c63+9a11dc8c8f
LSST Data Management Base Package
Loading...
Searching...
No Matches
brighterFatterKernel.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Brighter Fatter Kernel calibration definition."""
23
24
25__all__ = ['BrighterFatterKernel']
26
27
28import numpy as np
29from astropy.table import Table
30import lsst.afw.math as afwMath
31from . import IsrCalib
32
33
34class BrighterFatterKernel(IsrCalib):
35 """Calibration of brighter-fatter kernels for an instrument.
36
37 ampKernels are the kernels for each amplifier in a detector, as
38 generated by having ``level == 'AMP'``.
39
40 detectorKernel is the kernel generated for a detector as a
41 whole, as generated by having ``level == 'DETECTOR'``.
42
43 makeDetectorKernelFromAmpwiseKernels is a method to generate the
44 kernel for a detector, constructed by averaging together the
45 ampwise kernels in the detector. The existing application code is
46 only defined for kernels with ``level == 'DETECTOR'``, so this method
47 is used if the supplied kernel was built with ``level == 'AMP'``.
48
49 Parameters
50 ----------
51 camera : `lsst.afw.cameraGeom.Camera`
52 Camera describing detector geometry.
53 level : `str`
54 Level the kernels will be generated for.
55 log : `logging.Logger`, optional
56 Log to write messages to.
57 **kwargs :
58 Parameters to pass to parent constructor.
59
60 Notes
61 -----
62 Version 1.1 adds the `expIdMask` property, and substitutes
63 `means` and `variances` for `rawMeans` and `rawVariances`
64 from the PTC dataset.
65
66 expIdMask : `dict`, [`str`,`numpy.ndarray`]
67 Dictionary keyed by amp names containing the mask produced after
68 outlier rejection.
69 rawMeans : `dict`, [`str`, `numpy.ndarray`]
70 Dictionary keyed by amp names containing the unmasked average of the
71 means of the exposures in each flat pair.
72 rawVariances : `dict`, [`str`, `numpy.ndarray`]
73 Dictionary keyed by amp names containing the variance of the
74 difference image of the exposures in each flat pair.
75 Corresponds to rawVars of PTC.
76 rawXcorrs : `dict`, [`str`, `numpy.ndarray`]
77 Dictionary keyed by amp names containing an array of measured
78 covariances per mean flux.
79 Corresponds to covariances of PTC.
80 badAmps : `list`
81 List of bad amplifiers names.
82 shape : `tuple`
83 Tuple of the shape of the BFK kernels.
84 gain : `dict`, [`str`,`float`]
85 Dictionary keyed by amp names containing the fitted gains.
86 noise : `dict`, [`str`,`float`]
87 Dictionary keyed by amp names containing the fitted noise.
88 meanXcorrs : `dict`, [`str`,`numpy.ndarray`]
89 Dictionary keyed by amp names containing the averaged
90 cross-correlations.
91 valid : `dict`, [`str`,`bool`]
92 Dictionary keyed by amp names containing validity of data.
93 ampKernels : `dict`, [`str`, `numpy.ndarray`]
94 Dictionary keyed by amp names containing the BF kernels.
95 detKernels : `dict`
96 Dictionary keyed by detector names containing the BF kernels.
97 """
98 _OBSTYPE = 'bfk'
99 _SCHEMA = 'Brighter-fatter kernel'
100 _VERSION = 1.1
101
102 def __init__(self, camera=None, level=None, **kwargs):
103 self.level = level
104
105 # Things inherited from the PTC
106 self.expIdMask = dict()
107 self.rawMeans = dict()
108 self.rawVariances = dict()
109 self.rawXcorrs = dict()
110 self.badAmps = list()
111 self.shape = (17, 17)
112 self.gain = dict()
113 self.noise = dict()
114
115 # Things calculated from the PTC
116 self.meanXcorrs = dict()
117 self.valid = dict()
118
119 # Things that are used downstream
120 self.ampKernels = dict()
121 self.detKernels = dict()
122
123 super().__init__(**kwargs)
124
125 if camera:
126 self.initFromCamera(camera, detectorId=kwargs.get('detectorId', None))
127
128 self.requiredAttributes.update(['level', 'expIdMask', 'rawMeans', 'rawVariances', 'rawXcorrs',
129 'badAmps', 'gain', 'noise', 'meanXcorrs', 'valid',
130 'ampKernels', 'detKernels'])
131
132 def updateMetadata(self, setDate=False, **kwargs):
133 """Update calibration metadata.
134
135 This calls the base class's method after ensuring the required
136 calibration keywords will be saved.
137
138 Parameters
139 ----------
140 setDate : `bool`, optional
141 Update the CALIBDATE fields in the metadata to the current
142 time. Defaults to False.
143 kwargs :
144 Other keyword parameters to set in the metadata.
145 """
146 kwargs['LEVEL'] = self.level
147 kwargs['KERNEL_DX'] = self.shape[0]
148 kwargs['KERNEL_DY'] = self.shape[1]
149
150 super().updateMetadata(setDate=setDate, **kwargs)
151
152 def initFromCamera(self, camera, detectorId=None):
153 """Initialize kernel structure from camera.
154
155 Parameters
156 ----------
157 camera : `lsst.afw.cameraGeom.Camera`
158 Camera to use to define geometry.
159 detectorId : `int`, optional
160 Index of the detector to generate.
161
162 Returns
163 -------
164 calib : `lsst.ip.isr.BrighterFatterKernel`
165 The initialized calibration.
166
167 Raises
168 ------
169 RuntimeError
170 Raised if no detectorId is supplied for a calibration with
171 ``level='AMP'``.
172 """
173 self._instrument = camera.getName()
174
175 if detectorId is not None:
176 detector = camera[detectorId]
177 self._detectorId = detectorId
178 self._detectorName = detector.getName()
179 self._detectorSerial = detector.getSerial()
180
181 if self.level == 'AMP':
182 if detectorId is None:
183 raise RuntimeError("A detectorId must be supplied if level='AMP'.")
184
185 self.badAmps = []
186
187 for amp in detector:
188 ampName = amp.getName()
189 self.expIdMask[ampName] = []
190 self.rawMeans[ampName] = []
191 self.rawVariances[ampName] = []
192 self.rawXcorrs[ampName] = []
193 self.gain[ampName] = amp.getGain()
194 self.noise[ampName] = amp.getReadNoise()
195 self.meanXcorrs[ampName] = []
196 self.ampKernels[ampName] = []
197 self.valid[ampName] = []
198 elif self.level == 'DETECTOR':
199 if detectorId is None:
200 for det in camera:
201 detName = det.getName()
202 self.detKernels[detName] = []
203 else:
204 self.detKernels[self._detectorName] = []
205
206 return self
207
208 def getLengths(self):
209 """Return the set of lengths needed for reshaping components.
210
211 Returns
212 -------
213 kernelLength : `int`
214 Product of the elements of self.shape.
215 smallLength : `int`
216 Size of an untiled covariance.
217 nObs : `int`
218 Number of observation pairs used in the kernel.
219 """
220 kernelLength = self.shape[0] * self.shape[1]
221 smallLength = int((self.shape[0] - 1)*(self.shape[1] - 1)/4)
222 if self.level == 'AMP':
223 nObservations = set([len(self.rawMeans[amp]) for amp in self.rawMeans])
224 if len(nObservations) != 1:
225 raise RuntimeError("Inconsistent number of observations found.")
226 nObs = nObservations.pop()
227 else:
228 nObs = 0
229
230 return (kernelLength, smallLength, nObs)
231
232 @classmethod
233 def fromDict(cls, dictionary):
234 """Construct a calibration from a dictionary of properties.
235
236 Parameters
237 ----------
238 dictionary : `dict`
239 Dictionary of properties.
240
241 Returns
242 -------
243 calib : `lsst.ip.isr.BrighterFatterKernel`
244 Constructed calibration.
245
246 Raises
247 ------
248 RuntimeError
249 Raised if the supplied dictionary is for a different
250 calibration.
251 Raised if the version of the supplied dictionary is 1.0.
252 """
253 calib = cls()
254
255 if calib._OBSTYPE != (found := dictionary['metadata']['OBSTYPE']):
256 raise RuntimeError(f"Incorrect brighter-fatter kernel supplied. Expected {calib._OBSTYPE}, "
257 f"found {found}")
258 calib.setMetadata(dictionary['metadata'])
259 calib.calibInfoFromDict(dictionary)
260
261 calib.level = dictionary['metadata'].get('LEVEL', 'AMP')
262 calib.shape = (dictionary['metadata'].get('KERNEL_DX', 0),
263 dictionary['metadata'].get('KERNEL_DY', 0))
264
265 calibVersion = dictionary['metadata']['bfk_VERSION']
266 if calibVersion == 1.0:
267 calib.log.debug("Old Version of brighter-fatter kernel found. Current version: "
268 f"{calib._VERSION}. The new attribute 'expIdMask' will be "
269 "populated with 'True' values, and the new attributes 'rawMeans' "
270 "and 'rawVariances' will be populated with the masked 'means' "
271 "and 'variances' values."
272 )
273 # use 'means', because 'expIdMask' does not exist.
274 calib.expIdMask = {amp: np.repeat(True, len(dictionary['means'][amp])) for amp in
275 dictionary['means']}
276 calib.rawMeans = {amp: np.array(dictionary['means'][amp]) for amp in dictionary['means']}
277 calib.rawVariances = {amp: np.array(dictionary['variances'][amp]) for amp in
278 dictionary['variances']}
279 elif calibVersion == 1.1:
280 calib.expIdMask = {amp: np.array(dictionary['expIdMask'][amp]) for amp in dictionary['expIdMask']}
281 calib.rawMeans = {amp: np.array(dictionary['rawMeans'][amp]) for amp in dictionary['rawMeans']}
282 calib.rawVariances = {amp: np.array(dictionary['rawVariances'][amp]) for amp in
283 dictionary['rawVariances']}
284 else:
285 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}")
286
287 # Lengths for reshape:
288 _, smallLength, nObs = calib.getLengths()
289 smallShapeSide = int(np.sqrt(smallLength))
290
291 calib.rawXcorrs = {amp: np.array(dictionary['rawXcorrs'][amp]).reshape((nObs,
292 smallShapeSide,
293 smallShapeSide))
294 for amp in dictionary['rawXcorrs']}
295
296 calib.gain = dictionary['gain']
297 calib.noise = dictionary['noise']
298
299 calib.meanXcorrs = {amp: np.array(dictionary['meanXcorrs'][amp]).reshape(calib.shape)
300 for amp in dictionary['rawXcorrs']}
301 calib.ampKernels = {amp: np.array(dictionary['ampKernels'][amp]).reshape(calib.shape)
302 for amp in dictionary['ampKernels']}
303 calib.valid = {amp: bool(value) for amp, value in dictionary['valid'].items()}
304 calib.badAmps = [amp for amp, valid in dictionary['valid'].items() if valid is False]
305
306 calib.detKernels = {det: np.array(dictionary['detKernels'][det]).reshape(calib.shape)
307 for det in dictionary['detKernels']}
308
309 calib.updateMetadata()
310 return calib
311
312 def toDict(self):
313 """Return a dictionary containing the calibration properties.
314
315 The dictionary should be able to be round-tripped through
316 `fromDict`.
317
318 Returns
319 -------
320 dictionary : `dict`
321 Dictionary of properties.
322 """
323 self.updateMetadata()
324
325 outDict = {}
326 metadata = self.getMetadata()
327 outDict['metadata'] = metadata
328
329 # Lengths for ravel:
330 kernelLength, smallLength, nObs = self.getLengths()
331
332 outDict['expIdMask'] = {amp: np.array(self.expIdMask[amp]).tolist() for amp in self.expIdMask}
333 outDict['rawMeans'] = {amp: np.array(self.rawMeans[amp]).tolist() for amp in self.rawMeans}
334 outDict['rawVariances'] = {amp: np.array(self.rawVariances[amp]).tolist() for amp in
335 self.rawVariances}
336
337 for amp in self.rawXcorrs.keys():
338 # Check to see if we need to repack the data.
339 correlationShape = np.array(self.rawXcorrs[amp]).shape
340 if nObs != correlationShape[0]:
341 if correlationShape[0] == np.sum(self.expIdMask[amp]):
342 # Repack data.
343 self.repackCorrelations(amp, correlationShape)
344 else:
345 raise ValueError("Could not coerce rawXcorrs into appropriate shape "
346 "(have %d correlations, but expect to see %d.",
347 correlationShape[0], np.sum(self.expIdMask[amp]))
348
349 outDict['rawXcorrs'] = {amp: np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist()
350 for amp in self.rawXcorrs}
351 outDict['badAmps'] = self.badAmps
352 outDict['gain'] = self.gain
353 outDict['noise'] = self.noise
354
355 outDict['meanXcorrs'] = {amp: self.meanXcorrs[amp].reshape(kernelLength).tolist()
356 for amp in self.meanXcorrs}
357 outDict['ampKernels'] = {amp: self.ampKernels[amp].reshape(kernelLength).tolist()
358 for amp in self.ampKernels}
359 outDict['valid'] = self.valid
360
361 outDict['detKernels'] = {det: self.detKernels[det].reshape(kernelLength).tolist()
362 for det in self.detKernels}
363 return outDict
364
365 @classmethod
366 def fromTable(cls, tableList):
367 """Construct calibration from a list of tables.
368
369 This method uses the `fromDict` method to create the
370 calibration, after constructing an appropriate dictionary from
371 the input tables.
372
373 Parameters
374 ----------
375 tableList : `list` [`astropy.table.Table`]
376 List of tables to use to construct the brighter-fatter
377 calibration.
378
379 Returns
380 -------
381 calib : `lsst.ip.isr.BrighterFatterKernel`
382 The calibration defined in the tables.
383 """
384 ampTable = tableList[0]
385
386 metadata = ampTable.meta
387 inDict = dict()
388 inDict['metadata'] = metadata
389
390 amps = ampTable['AMPLIFIER']
391
392 # Determine version for expected values. The ``fromDict``
393 # method can unpack either, but the appropriate fields need to
394 # be supplied.
395 calibVersion = metadata['bfk_VERSION']
396
397 if calibVersion == 1.0:
398 # We expect to find ``means`` and ``variances`` for this
399 # case, and will construct an ``expIdMask`` from these
400 # parameters in the ``fromDict`` method.
401 rawMeanList = ampTable['MEANS']
402 rawVarianceList = ampTable['VARIANCES']
403
404 inDict['means'] = {amp: mean for amp, mean in zip(amps, rawMeanList)}
405 inDict['variances'] = {amp: var for amp, var in zip(amps, rawVarianceList)}
406 elif calibVersion == 1.1:
407 # This will have ``rawMeans`` and ``rawVariances``, which
408 # are filtered via the ``expIdMask`` fields.
409 expIdMaskList = ampTable['EXP_ID_MASK']
410 rawMeanList = ampTable['RAW_MEANS']
411 rawVarianceList = ampTable['RAW_VARIANCES']
412
413 inDict['expIdMask'] = {amp: mask for amp, mask in zip(amps, expIdMaskList)}
414 inDict['rawMeans'] = {amp: mean for amp, mean in zip(amps, rawMeanList)}
415 inDict['rawVariances'] = {amp: var for amp, var in zip(amps, rawVarianceList)}
416 else:
417 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}")
418
419 rawXcorrs = ampTable['RAW_XCORRS']
420 gainList = ampTable['GAIN']
421 noiseList = ampTable['NOISE']
422
423 meanXcorrs = ampTable['MEAN_XCORRS']
424 ampKernels = ampTable['KERNEL']
425 validList = ampTable['VALID']
426
427 inDict['rawXcorrs'] = {amp: kernel for amp, kernel in zip(amps, rawXcorrs)}
428 inDict['gain'] = {amp: gain for amp, gain in zip(amps, gainList)}
429 inDict['noise'] = {amp: noise for amp, noise in zip(amps, noiseList)}
430 inDict['meanXcorrs'] = {amp: kernel for amp, kernel in zip(amps, meanXcorrs)}
431 inDict['ampKernels'] = {amp: kernel for amp, kernel in zip(amps, ampKernels)}
432 inDict['valid'] = {amp: bool(valid) for amp, valid in zip(amps, validList)}
433
434 inDict['badAmps'] = [amp for amp, valid in inDict['valid'].items() if valid is False]
435
436 if len(tableList) > 1:
437 detTable = tableList[1]
438 inDict['detKernels'] = {det: kernel for det, kernel
439 in zip(detTable['DETECTOR'], detTable['KERNEL'])}
440 else:
441 inDict['detKernels'] = {}
442
443 return cls.fromDict(inDict)
444
445 def toTable(self):
446 """Construct a list of tables containing the information in this
447 calibration.
448
449 The list of tables should create an identical calibration
450 after being passed to this class's fromTable method.
451
452 Returns
453 -------
454 tableList : `list` [`lsst.afw.table.Table`]
455 List of tables containing the crosstalk calibration
456 information.
457
458 """
459 tableList = []
460 self.updateMetadata()
461
462 # Lengths
463 kernelLength, smallLength, nObs = self.getLengths()
464
465 ampList = []
466 expIdMaskList = []
467 rawMeanList = []
468 rawVarianceList = []
469 rawXcorrs = []
470 gainList = []
471 noiseList = []
472
473 meanXcorrsList = []
474 kernelList = []
475 validList = []
476
477 if self.level == 'AMP':
478 for amp in self.rawMeans.keys():
479 ampList.append(amp)
480 expIdMaskList.append(self.expIdMask[amp])
481 rawMeanList.append(self.rawMeans[amp])
482 rawVarianceList.append(self.rawVariances[amp])
483
484 correlationShape = np.array(self.rawXcorrs[amp]).shape
485 if nObs != correlationShape[0]:
486 if correlationShape[0] == np.sum(self.expIdMask[amp]):
487 # Repack data.
488 self.repackCorrelations(amp, correlationShape)
489 else:
490 raise ValueError("Could not coerce rawXcorrs into appropriate shape "
491 "(have %d correlations, but expect to see %d.",
492 correlationShape[0], np.sum(self.expIdMask[amp]))
493
494 rawXcorrs.append(np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist())
495 gainList.append(self.gain[amp])
496 noiseList.append(self.noise[amp])
497
498 meanXcorrsList.append(self.meanXcorrs[amp].reshape(kernelLength).tolist())
499 kernelList.append(self.ampKernels[amp].reshape(kernelLength).tolist())
500 validList.append(int(self.valid[amp] and not (amp in self.badAmps)))
501
502 ampTable = Table({'AMPLIFIER': ampList,
503 'EXP_ID_MASK': expIdMaskList,
504 'RAW_MEANS': rawMeanList,
505 'RAW_VARIANCES': rawVarianceList,
506 'RAW_XCORRS': rawXcorrs,
507 'GAIN': gainList,
508 'NOISE': noiseList,
509 'MEAN_XCORRS': meanXcorrsList,
510 'KERNEL': kernelList,
511 'VALID': validList,
512 })
513
514 ampTable.meta = self.getMetadata().toDict()
515 tableList.append(ampTable)
516
517 if len(self.detKernels):
518 detList = []
519 kernelList = []
520 for det in self.detKernels.keys():
521 detList.append(det)
522 kernelList.append(self.detKernels[det].reshape(kernelLength).tolist())
523
524 detTable = Table({'DETECTOR': detList,
525 'KERNEL': kernelList})
526 detTable.meta = self.getMetadata().toDict()
527 tableList.append(detTable)
528
529 return tableList
530
531 def repackCorrelations(self, amp, correlationShape):
532 """If the correlations were masked, they need to be repacked into the
533 correct shape.
534
535 Parameters
536 ----------
537 amp : `str`
538 Amplifier needing repacked.
539 correlationShape : `tuple` [`int`], (3, )
540 Shape the correlations are expected to take.
541 """
542 repackedCorrelations = []
543 idx = 0
544 for maskValue in self.expIdMask[amp]:
545 if maskValue:
546 repackedCorrelations.append(self.rawXcorrs[amp][idx])
547 idx += 1
548 else:
549 repackedCorrelations.append(np.full((correlationShape[1], correlationShape[2]), np.nan))
550 self.rawXcorrs[amp] = repackedCorrelations
551
552 # Implementation methods
553 def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[]):
554 """Average the amplifier level kernels to create a detector level
555 kernel. There is no change in index ordering/orientation from
556 this averaging.
557
558 Parameters
559 ----------
560 detectorName : `str`
561 Detector for which the averaged kernel will be used.
562 ampsToExclude : `list` [`str`], optional
563 Amps that should not be included in the average.
564 """
565 inKernels = np.array([self.ampKernels[amp] for amp in
566 self.ampKernels if amp not in ampsToExclude])
567 avgKernel = np.zeros_like(inKernels[0])
569 sctrl.setNumSigmaClip(5.0)
570 for i in range(np.shape(avgKernel)[0]):
571 for j in range(np.shape(avgKernel)[1]):
572 avgKernel[i, j] = afwMath.makeStatistics(inKernels[:, i, j],
573 afwMath.MEANCLIP, sctrl).getValue()
574
575 self.detKernels[detectorName] = avgKernel
576
577 def replaceDetectorKernelWithAmpKernel(self, ampName, detectorName):
578 self.detKernel[detectorName] = self.ampKernel[ampName]
std::vector< SchemaItem< Flag > > * items
Pass parameters to a Statistics object.
Definition Statistics.h:83
makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[])
daf::base::PropertySet * set
Definition fits.cc:931
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Handle a watered-down front-end to the constructor (no variance)
Definition Statistics.h:361