LSST Applications 26.0.0,g0265f82a02+6660c170cc,g07994bdeae+30b05a742e,g0a0026dc87+17526d298f,g0a60f58ba1+17526d298f,g0e4bf8285c+96dd2c2ea9,g0ecae5effc+c266a536c8,g1e7d6db67d+6f7cb1f4bb,g26482f50c6+6346c0633c,g2bbee38e9b+6660c170cc,g2cc88a2952+0a4e78cd49,g3273194fdb+f6908454ef,g337abbeb29+6660c170cc,g337c41fc51+9a8f8f0815,g37c6e7c3d5+7bbafe9d37,g44018dc512+6660c170cc,g4a941329ef+4f7594a38e,g4c90b7bd52+5145c320d2,g58be5f913a+bea990ba40,g635b316a6c+8d6b3a3e56,g67924a670a+bfead8c487,g6ae5381d9b+81bc2a20b4,g93c4d6e787+26b17396bd,g98cecbdb62+ed2cb6d659,g98ffbb4407+81bc2a20b4,g9ddcbc5298+7f7571301f,ga1e77700b3+99e9273977,gae46bcf261+6660c170cc,gb2715bf1a1+17526d298f,gc86a011abf+17526d298f,gcf0d15dbbd+96dd2c2ea9,gdaeeff99f8+0d8dbea60f,gdb4ec4c597+6660c170cc,ge23793e450+96dd2c2ea9,gf041782ebf+171108ac67
LSST Data Management Base Package
Loading...
Searching...
No Matches
ptcDataset.py
Go to the documentation of this file.
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Define dataset class for MeasurePhotonTransferCurve task
24"""
25
26__all__ = ['PhotonTransferCurveDataset']
27
28import numpy as np
29from astropy.table import Table
30
31from lsst.ip.isr import IsrCalib
32
33
34class PhotonTransferCurveDataset(IsrCalib):
35 """A simple class to hold the output data from the PTC task.
36
37 The dataset is made up of a dictionary for each item, keyed by the
38 amplifiers' names, which much be supplied at construction time.
39 New items cannot be added to the class to save accidentally saving to the
40 wrong property, and the class can be frozen if desired.
41 inputExpIdPairs records the exposures used to produce the data.
42 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which
43 is by definition always the same length as inputExpIdPairs, rawExpTimes,
44 rawMeans and rawVars, and is a list of bools, which are incrementally set
45 to False as points are discarded from the fits.
46 PTC fit parameters for polynomials are stored in a list in ascending order
47 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
48 with the length of the list corresponding to the order of the polynomial
49 plus one.
50
51 Parameters
52 ----------
53 ampNames : `list`
54 List with the names of the amplifiers of the detector at hand.
55 ptcFitType : `str`
56 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION",
57 or "FULLCOVARIANCE".
58 covMatrixSide : `int`
59 Maximum lag of covariances (size of square covariance matrices).
60 kwargs : `dict`, optional
61 Other keyword arguments to pass to the parent init.
62
63 Notes
64 -----
65 The stored attributes are:
66
67 badAmps : `list` [`str`]
68 List with bad amplifiers names.
69 inputExpIdPairs : `dict`, [`str`, `list`]
70 Dictionary keyed by amp names containing the input exposures IDs.
71 expIdMask : `dict`, [`str`, `np.ndarray`]
72 Dictionary keyed by amp names containing the mask produced after
73 outlier rejection. The mask produced by the "FULLCOVARIANCE"
74 option may differ from the one produced in the other two PTC
75 fit types.
76 rawExpTimes : `dict`, [`str`, `np.ndarray`]
77 Dictionary keyed by amp names containing the unmasked exposure times.
78 rawMeans : `dict`, [`str`, `np.ndarray`]
79 Dictionary keyed by amp names containing the unmasked average of the
80 means of the exposures in each flat pair.
81 rawVars : `dict`, [`str`, `np.ndarray`]
82 Dictionary keyed by amp names containing the variance of the
83 difference image of the exposures in each flat pair.
84 histVars : `dict`, [`str`, `np.ndarray`]
85 Dictionary keyed by amp names containing the variance of the
86 difference image of the exposures in each flat pair estimated
87 by fitting a Gaussian model.
88 histChi2Dofs : `dict`, [`str`, `np.ndarray`]
89 Dictionary keyed by amp names containing the chi-squared per degree
90 of freedom fitting the difference image to a Gaussian model.
91 kspValues : `dict`, [`str`, `np.ndarray`]
92 Dictionary keyed by amp names containing the KS test p-value from
93 fitting the difference image to a Gaussian model.
94 gain : `dict`, [`str`, `float`]
95 Dictionary keyed by amp names containing the fitted gains.
96 gainErr : `dict`, [`str`, `float`]
97 Dictionary keyed by amp names containing the errors on the
98 fitted gains.
99 noise : `dict`, [`str`, `float`]
100 Dictionary keyed by amp names containing the fitted noise.
101 noiseErr : `dict`, [`str`, `float`]
102 Dictionary keyed by amp names containing the errors on the fitted
103 noise.
104 ptcFitPars : `dict`, [`str`, `np.ndarray`]
105 Dictionary keyed by amp names containing the fitted parameters of the
106 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
107 ptcFitParsError : `dict`, [`str`, `np.ndarray`]
108 Dictionary keyed by amp names containing the errors on the fitted
109 parameters of the PTC model for ptcFitTye in
110 ["POLYNOMIAL", "EXPAPPROXIMATION"].
111 ptcFitChiSq : `dict`, [`str`, `float`]
112 Dictionary keyed by amp names containing the reduced chi squared
113 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
114 ptcTurnoff : `dict` [`str, `float`]
115 Flux value (in ADU) where the variance of the PTC curve starts
116 decreasing consistently.
117 covariances : `dict`, [`str`, `np.ndarray`]
118 Dictionary keyed by amp names containing a list of measured
119 covariances per mean flux.
120 covariancesModel : `dict`, [`str`, `np.ndarray`]
121 Dictionary keyed by amp names containinging covariances model
122 (Eq. 20 of Astier+19) per mean flux.
123 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`]
124 Dictionary keyed by amp names containinging sqrt. of covariances
125 weights.
126 aMatrix : `dict`, [`str`, `np.ndarray`]
127 Dictionary keyed by amp names containing the "a" parameters from
128 the model in Eq. 20 of Astier+19.
129 bMatrix : `dict`, [`str`, `np.ndarray`]
130 Dictionary keyed by amp names containing the "b" parameters from
131 the model in Eq. 20 of Astier+19.
132 noiseMatrix : `dict`, [`str`, `np.ndarray`]
133 Dictionary keyed by amp names containing the "noise" parameters from
134 the model in Eq. 20 of Astier+19.
135 covariancesModelNoB : `dict`, [`str`, `np.ndarray`]
136 Dictionary keyed by amp names containing covariances model
137 (with 'b'=0 in Eq. 20 of Astier+19)
138 per mean flux.
139 aMatrixNoB : `dict`, [`str`, `np.ndarray`]
140 Dictionary keyed by amp names containing the "a" parameters from the
141 model in Eq. 20 of Astier+19
142 (and 'b' = 0).
143 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`]
144 Dictionary keyed by amp names containing the "noise" parameters from
145 the model in Eq. 20 of Astier+19, with 'b' = 0.
146 finalVars : `dict`, [`str`, `np.ndarray`]
147 Dictionary keyed by amp names containing the masked variance of the
148 difference image of each flat
149 pair. If needed, each array will be right-padded with
150 np.nan to match the length of rawExpTimes.
151 finalModelVars : `dict`, [`str`, `np.ndarray`]
152 Dictionary keyed by amp names containing the masked modeled
153 variance of the difference image of each flat pair. If needed, each
154 array will be right-padded with np.nan to match the length of
155 rawExpTimes.
156 finalMeans : `dict`, [`str`, `np.ndarray`]
157 Dictionary keyed by amp names containing the masked average of the
158 means of the exposures in each flat pair. If needed, each array
159 will be right-padded with np.nan to match the length of
160 rawExpTimes.
161 photoCharges : `dict`, [`str`, `np.ndarray`]
162 Dictionary keyed by amp names containing the integrated photocharge
163 for linearity calibration.
164 auxValues : `dict`, [`str`, `np.ndarray`]
165 Dictionary of per-detector auxiliary header values that can be used
166 for PTC, linearity computation.
167
168 Version 1.1 adds the `ptcTurnoff` attribute.
169 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues`
170 attributes.
171 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes.
172 Version 1.4 adds the `auxValues` attribute.
173 """
174
175 _OBSTYPE = 'PTC'
176 _SCHEMA = 'Gen3 Photon Transfer Curve'
177 _VERSION = 1.4
178
179 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
180 self.ptcFitType = ptcFitType
181 self.ampNames = ampNames
182 self.covMatrixSide = covMatrixSide
183
184 self.badAmps = []
185
186 self.inputExpIdPairs = {ampName: [] for ampName in ampNames}
187 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames}
188 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames}
189 self.rawMeans = {ampName: np.array([]) for ampName in ampNames}
190 self.rawVars = {ampName: np.array([]) for ampName in ampNames}
191 self.photoCharges = {ampName: np.array([]) for ampName in ampNames}
192
193 self.gain = {ampName: np.nan for ampName in ampNames}
194 self.gainErr = {ampName: np.nan for ampName in ampNames}
195 self.noise = {ampName: np.nan for ampName in ampNames}
196 self.noiseErr = {ampName: np.nan for ampName in ampNames}
197
198 self.histVars = {ampName: np.array([]) for ampName in ampNames}
199 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames}
200 self.kspValues = {ampName: np.array([]) for ampName in ampNames}
201
202 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames}
203 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames}
204 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames}
205 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames}
206
207 self.covariances = {ampName: np.array([]) for ampName in ampNames}
208 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames}
209 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames}
210 self.aMatrix = {ampName: np.array([]) for ampName in ampNames}
211 self.bMatrix = {ampName: np.array([]) for ampName in ampNames}
212 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames}
213 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames}
214 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
215 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
216
217 self.finalVars = {ampName: np.array([]) for ampName in ampNames}
218 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames}
219 self.finalMeans = {ampName: np.array([]) for ampName in ampNames}
220
221 # Try this as a dict of arrays.
222 self.auxValues = {}
223
224 super().__init__(**kwargs)
225 self.requiredAttributesrequiredAttributesrequiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes',
226 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr',
227 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff',
228 'aMatrixNoB', 'covariances', 'covariancesModel',
229 'covariancesSqrtWeights', 'covariancesModelNoB',
230 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars',
231 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars',
232 'histChi2Dofs', 'kspValues', 'auxValues'])
233
234 self.updateMetadataupdateMetadata(setCalibInfo=True, setCalibId=True, **kwargs)
235
236 def setAmpValuesPartialDataset(
237 self,
238 ampName,
239 inputExpIdPair=(-1, -1),
240 rawExpTime=np.nan,
241 rawMean=np.nan,
242 rawVar=np.nan,
243 photoCharge=np.nan,
244 expIdMask=False,
245 covariance=None,
246 covSqrtWeights=None,
247 gain=np.nan,
248 noise=np.nan,
249 histVar=np.nan,
250 histChi2Dof=np.nan,
251 kspValue=0.0,
252 auxValues=None,
253 ):
254 """
255 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask).
256
257 Parameters
258 ----------
259 ampName : `str`
260 Name of the amp to set the values.
261 inputExpIdPair : `tuple` [`int`]
262 Exposure IDs of input pair.
263 rawExpTime : `float`, optional
264 Exposure time for this exposure pair.
265 rawMean : `float`, optional
266 Average of the means of the exposures in this pair.
267 rawVar : `float`, optional
268 Variance of the difference of the exposures in this pair.
269 photoCharge : `float`, optional
270 Integrated photocharge for flat pair for linearity calibration.
271 expIdMask : `bool`, optional
272 Flag setting if this exposure pair should be used (True)
273 or not used (False).
274 covariance : `np.ndarray` or None, optional
275 Measured covariance for this exposure pair.
276 covSqrtWeights : `np.ndarray` or None, optional
277 Measured sqrt of covariance weights in this exposure pair.
278 gain : `float`, optional
279 Estimated gain for this exposure pair.
280 noise : `float`, optional
281 Estimated read noise for this exposure pair.
282 histVar : `float`, optional
283 Variance estimated from fitting a histogram with a Gaussian model.
284 histChi2Dof : `float`, optional
285 Chi-squared per degree of freedom from Gaussian histogram fit.
286 kspValue : `float`, optional
287 KS test p-value from the Gaussian histogram fit.
288 """
289 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan)
290 if covariance is None:
291 covariance = nanMatrix
292 if covSqrtWeights is None:
293 covSqrtWeights = nanMatrix
294
295 self.inputExpIdPairs[ampName] = [inputExpIdPair]
296 self.rawExpTimes[ampName] = np.array([rawExpTime])
297 self.rawMeans[ampName] = np.array([rawMean])
298 self.rawVars[ampName] = np.array([rawVar])
299 self.photoCharges[ampName] = np.array([photoCharge])
300 self.expIdMask[ampName] = np.array([expIdMask])
301 self.covariances[ampName] = np.array([covariance])
302 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights])
303 self.gain[ampName] = gain
304 self.noise[ampName] = noise
305 self.histVars[ampName] = np.array([histVar])
306 self.histChi2Dofs[ampName] = np.array([histChi2Dof])
307 self.kspValues[ampName] = np.array([kspValue])
308
309 self.covariancesModel[ampName] = np.array([nanMatrix])
310 self.covariancesModelNoB[ampName] = np.array([nanMatrix])
311 self.aMatrix[ampName] = nanMatrix
312 self.bMatrix[ampName] = nanMatrix
313 self.aMatrixNoB[ampName] = nanMatrix
314 self.noiseMatrix[ampName] = nanMatrix
315 self.noiseMatrixNoB[ampName] = nanMatrix
316
317 def setAuxValuesPartialDataset(self, auxDict):
318 """
319 Set a dictionary of auxiliary values for a partial dataset.
320
321 Parameters
322 ----------
323 auxDict : `dict` [`str`, `float`]
324 Dictionary of float values.
325 """
326 for key, value in auxDict.items():
327 self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))
328
329 def updateMetadata(self, **kwargs):
330 """Update calibration metadata.
331 This calls the base class's method after ensuring the required
332 calibration keywords will be saved.
333
334 Parameters
335 ----------
336 setDate : `bool`, optional
337 Update the CALIBDATE fields in the metadata to the current
338 time. Defaults to False.
339 kwargs :
340 Other keyword parameters to set in the metadata.
341 """
342 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs)
343
344 @classmethod
345 def fromDict(cls, dictionary):
346 """Construct a calibration from a dictionary of properties.
347 Must be implemented by the specific calibration subclasses.
348
349 Parameters
350 ----------
351 dictionary : `dict`
352 Dictionary of properties.
353
354 Returns
355 -------
356 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
357 Constructed calibration.
358
359 Raises
360 ------
361 RuntimeError
362 Raised if the supplied dictionary is for a different
363 calibration.
364 """
365 calib = cls()
366 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
367 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. "
368 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
369 calib.setMetadata(dictionary['metadata'])
370 calib.ptcFitType = dictionary['ptcFitType']
371 calib.covMatrixSide = dictionary['covMatrixSide']
372 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist()
373 calib.ampNames = []
374
375 # The cov matrices are square
376 covMatrixSide = calib.covMatrixSide
377 # Number of final signal levels
378 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel())
379 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide))
380
381 for ampName in dictionary['ampNames']:
382 calib.ampNames.append(ampName)
383 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName]
384 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName])
385 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64)
386 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64)
387 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64)
388 calib.gain[ampName] = float(dictionary['gain'][ampName])
389 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName])
390 calib.noise[ampName] = float(dictionary['noise'][ampName])
391 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName])
392 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64)
393 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64)
394 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64)
395 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64)
396 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName],
397 dtype=np.float64)
398 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName])
399 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName])
400 if nSignalPoints > 0:
401 # Regular dataset
402 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName],
403 dtype=np.float64).reshape(
404 (nSignalPoints, covMatrixSide, covMatrixSide))
405 calib.covariancesModel[ampName] = np.array(
406 dictionary['covariancesModel'][ampName],
407 dtype=np.float64).reshape(
408 (nSignalPoints, covMatrixSide, covMatrixSide))
409 calib.covariancesSqrtWeights[ampName] = np.array(
410 dictionary['covariancesSqrtWeights'][ampName],
411 dtype=np.float64).reshape(
412 (nSignalPoints, covMatrixSide, covMatrixSide))
413 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName],
414 dtype=np.float64).reshape(
415 (covMatrixSide, covMatrixSide))
416 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName],
417 dtype=np.float64).reshape(
418 (covMatrixSide, covMatrixSide))
419 calib.covariancesModelNoB[ampName] = np.array(
420 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape(
421 (nSignalPoints, covMatrixSide, covMatrixSide))
422 calib.aMatrixNoB[ampName] = np.array(
423 dictionary['aMatrixNoB'][ampName],
424 dtype=np.float64).reshape((covMatrixSide, covMatrixSide))
425 calib.noiseMatrix[ampName] = np.array(
426 dictionary['noiseMatrix'][ampName],
427 dtype=np.float64).reshape((covMatrixSide, covMatrixSide))
428 calib.noiseMatrixNoB[ampName] = np.array(
429 dictionary['noiseMatrixNoB'][ampName],
430 dtype=np.float64).reshape((covMatrixSide, covMatrixSide))
431 else:
432 # Empty dataset
433 calib.covariances[ampName] = np.array([], dtype=np.float64)
434 calib.covariancesModel[ampName] = np.array([], dtype=np.float64)
435 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64)
436 calib.aMatrix[ampName] = np.array([], dtype=np.float64)
437 calib.bMatrix[ampName] = np.array([], dtype=np.float64)
438 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64)
439 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64)
440 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64)
441 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64)
442
443 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64)
444 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64)
445 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64)
446 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64)
447
448 for key, value in dictionary['auxValues'].items():
449 calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))
450
451 calib.updateMetadata()
452 return calib
453
454 def toDict(self):
455 """Return a dictionary containing the calibration properties.
456 The dictionary should be able to be round-tripped through
457 `fromDict`.
458
459 Returns
460 -------
461 dictionary : `dict`
462 Dictionary of properties.
463 """
464 self.updateMetadataupdateMetadata()
465
466 outDict = dict()
467 metadata = self.getMetadata()
468 outDict['metadata'] = metadata
469
470 def _dictOfArraysToDictOfLists(dictOfArrays):
471 dictOfLists = {}
472 for key, value in dictOfArrays.items():
473 dictOfLists[key] = value.ravel().tolist()
474
475 return dictOfLists
476
477 outDict['ptcFitType'] = self.ptcFitType
478 outDict['covMatrixSide'] = self.covMatrixSide
479 outDict['ampNames'] = self.ampNames
480 outDict['badAmps'] = self.badAmps
481 outDict['inputExpIdPairs'] = self.inputExpIdPairs
482 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask)
483 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes)
484 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans)
485 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars)
486 outDict['gain'] = self.gain
487 outDict['gainErr'] = self.gainErr
488 outDict['noise'] = self.noise
489 outDict['noiseErr'] = self.noiseErr
490 outDict['histVars'] = self.histVars
491 outDict['histChi2Dofs'] = self.histChi2Dofs
492 outDict['kspValues'] = self.kspValues
493 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars)
494 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError)
495 outDict['ptcFitChiSq'] = self.ptcFitChiSq
496 outDict['ptcTurnoff'] = self.ptcTurnoff
497 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances)
498 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel)
499 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights)
500 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix)
501 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix)
502 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix)
503 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB)
504 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB)
505 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB)
506 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars)
507 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars)
508 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans)
509 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges)
510 outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues)
511
512 return outDict
513
514 @classmethod
515 def fromTable(cls, tableList):
516 """Construct calibration from a list of tables.
517 This method uses the `fromDict` method to create the
518 calibration, after constructing an appropriate dictionary from
519 the input tables.
520
521 Parameters
522 ----------
523 tableList : `list` [`lsst.afw.table.Table`]
524 List of tables to use to construct the datasetPtc.
525
526 Returns
527 -------
528 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
529 The calibration defined in the tables.
530 """
531 ptcTable = tableList[0]
532
533 metadata = ptcTable.meta
534 inDict = dict()
535 inDict['metadata'] = metadata
536 inDict['ampNames'] = []
537 inDict['ptcFitType'] = []
538 inDict['covMatrixSide'] = []
539 inDict['inputExpIdPairs'] = dict()
540 inDict['expIdMask'] = dict()
541 inDict['rawExpTimes'] = dict()
542 inDict['rawMeans'] = dict()
543 inDict['rawVars'] = dict()
544 inDict['gain'] = dict()
545 inDict['gainErr'] = dict()
546 inDict['noise'] = dict()
547 inDict['noiseErr'] = dict()
548 inDict['histVars'] = dict()
549 inDict['histChi2Dofs'] = dict()
550 inDict['kspValues'] = dict()
551 inDict['ptcFitPars'] = dict()
552 inDict['ptcFitParsError'] = dict()
553 inDict['ptcFitChiSq'] = dict()
554 inDict['ptcTurnoff'] = dict()
555 inDict['covariances'] = dict()
556 inDict['covariancesModel'] = dict()
557 inDict['covariancesSqrtWeights'] = dict()
558 inDict['aMatrix'] = dict()
559 inDict['bMatrix'] = dict()
560 inDict['noiseMatrix'] = dict()
561 inDict['covariancesModelNoB'] = dict()
562 inDict['aMatrixNoB'] = dict()
563 inDict['noiseMatrixNoB'] = dict()
564 inDict['finalVars'] = dict()
565 inDict['finalModelVars'] = dict()
566 inDict['finalMeans'] = dict()
567 inDict['badAmps'] = []
568 inDict['photoCharges'] = dict()
569
570 calibVersion = metadata['PTC_VERSION']
571 if calibVersion == 1.0:
572 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. "
573 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.")
574 for record in ptcTable:
575 ampName = record['AMPLIFIER_NAME']
576
577 inDict['ptcFitType'] = record['PTC_FIT_TYPE']
578 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE']
579 inDict['ampNames'].append(ampName)
580 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist()
581 inDict['expIdMask'][ampName] = record['EXP_ID_MASK']
582 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES']
583 inDict['rawMeans'][ampName] = record['RAW_MEANS']
584 inDict['rawVars'][ampName] = record['RAW_VARS']
585 inDict['gain'][ampName] = record['GAIN']
586 inDict['gainErr'][ampName] = record['GAIN_ERR']
587 inDict['noise'][ampName] = record['NOISE']
588 inDict['noiseErr'][ampName] = record['NOISE_ERR']
589 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS']
590 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR']
591 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ']
592 inDict['covariances'][ampName] = record['COVARIANCES']
593 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL']
594 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS']
595 inDict['aMatrix'][ampName] = record['A_MATRIX']
596 inDict['bMatrix'][ampName] = record['B_MATRIX']
597 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B']
598 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B']
599 inDict['finalVars'][ampName] = record['FINAL_VARS']
600 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS']
601 inDict['finalMeans'][ampName] = record['FINAL_MEANS']
602 inDict['badAmps'] = record['BAD_AMPS'].tolist()
603 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE']
604 if calibVersion == 1.0:
605 mask = record['FINAL_MEANS'].mask
606 array = record['FINAL_MEANS'][~mask]
607 if len(array) > 0:
608 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1]
609 else:
610 inDict['ptcTurnoff'][ampName] = np.nan
611 else:
612 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF']
613 if calibVersion < 1.2:
614 inDict['histVars'][ampName] = np.array([np.nan])
615 inDict['histChi2Dofs'][ampName] = np.array([np.nan])
616 inDict['kspValues'][ampName] = np.array([0.0])
617 else:
618 inDict['histVars'][ampName] = record['HIST_VARS']
619 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS']
620 inDict['kspValues'][ampName] = record['KS_PVALUES']
621 if calibVersion < 1.3:
622 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan)
623 inDict['noiseMatrix'][ampName] = nanMatrix
624 inDict['noiseMatrixNoB'][ampName] = nanMatrix
625 else:
626 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX']
627 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B']
628
629 inDict['auxValues'] = {}
630 record = ptcTable[0]
631 for col in record.columns.keys():
632 if col.startswith('PTCAUX_'):
633 parts = col.split('PTCAUX_')
634 inDict['auxValues'][parts[1]] = record[col]
635
636 return cls().fromDict(inDict)
637
638 def toTable(self):
639 """Construct a list of tables containing the information in this
640 calibration.
641
642 The list of tables should create an identical calibration
643 after being passed to this class's fromTable method.
644
645 Returns
646 -------
647 tableList : `list` [`astropy.table.Table`]
648 List of tables containing the linearity calibration
649 information.
650 """
651 tableList = []
652 self.updateMetadataupdateMetadata()
653
654 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3")
655
656 catalogList = []
657 for ampName in self.ampNames:
658 ampDict = {
659 'AMPLIFIER_NAME': ampName,
660 'PTC_FIT_TYPE': self.ptcFitType,
661 'COV_MATRIX_SIDE': self.covMatrixSide,
662 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName],
663 'EXP_ID_MASK': self.expIdMask[ampName],
664 'RAW_EXP_TIMES': self.rawExpTimes[ampName],
665 'RAW_MEANS': self.rawMeans[ampName],
666 'RAW_VARS': self.rawVars[ampName],
667 'GAIN': self.gain[ampName],
668 'GAIN_ERR': self.gainErr[ampName],
669 'NOISE': self.noise[ampName],
670 'NOISE_ERR': self.noiseErr[ampName],
671 'HIST_VARS': self.histVars[ampName],
672 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName],
673 'KS_PVALUES': self.kspValues[ampName],
674 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]),
675 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]),
676 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName],
677 'PTC_TURNOFF': self.ptcTurnoff[ampName],
678 'A_MATRIX': self.aMatrix[ampName].ravel(),
679 'B_MATRIX': self.bMatrix[ampName].ravel(),
680 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(),
681 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(),
682 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(),
683 'BAD_AMPS': badAmps,
684 'PHOTO_CHARGE': self.photoCharges[ampName],
685 'COVARIANCES': self.covariances[ampName].ravel(),
686 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(),
687 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(),
688 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(),
689 'FINAL_VARS': self.finalVars[ampName],
690 'FINAL_MODEL_VARS': self.finalModelVars[ampName],
691 'FINAL_MEANS': self.finalMeans[ampName],
692 }
693
694 if self.auxValues:
695 for key, value in self.auxValues.items():
696 ampDict[f"PTCAUX_{key}"] = value
697
698 catalogList.append(ampDict)
699
700 catalog = Table(catalogList)
701
702 inMeta = self.getMetadata().toDict()
703 outMeta = {k: v for k, v in inMeta.items() if v is not None}
704 outMeta.update({k: "" for k, v in inMeta.items() if v is None})
705 catalog.meta = outMeta
706 tableList.append(catalog)
707
708 return tableList
709
710 def fromDetector(self, detector):
711 """Read metadata parameters from a detector.
712
713 Parameters
714 ----------
715 detector : `lsst.afw.cameraGeom.detector`
716 Input detector with parameters to use.
717
718 Returns
719 -------
720 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
721 The calibration constructed from the detector.
722 """
723
724 pass
725
726 def getExpIdsUsed(self, ampName):
727 """Get the exposures used, i.e. not discarded, for a given amp.
728 If no mask has been created yet, all exposures are returned.
729
730 Parameters
731 ----------
732 ampName : `str`
733
734 Returns
735 -------
736 expIdsUsed : `list` [`tuple`]
737 List of pairs of exposure ids used in PTC.
738 """
739 if len(self.expIdMask[ampName]) == 0:
740 return self.inputExpIdPairs[ampName]
741
742 # if the mask exists it had better be the same length as the expIdPairs
743 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName])
744
745 pairs = self.inputExpIdPairs[ampName]
746 mask = self.expIdMask[ampName]
747 # cast to bool required because numpy
748 try:
749 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m]
750 except ValueError:
751 self.log.warning("The PTC file was written incorrectly; you should rerun the "
752 "PTC solve task if possible.")
753 expIdsUsed = []
754 for pairList, m in zip(pairs, mask):
755 if m:
756 expIdsUsed.append(pairList[0])
757
758 return expIdsUsed
759
760 def getGoodAmps(self):
761 """Get the good amps from this PTC."""
762 return [amp for amp in self.ampNames if amp not in self.badAmps]
763
764 def getGoodPoints(self, ampName):
765 """Get the good points used for a given amp in the PTC.
766
767 Parameters
768 ----------
769 ampName : `str`
770
771 Returns
772 -------
773 goodPoints : `np.ndarray`
774 Boolean array of good points used in PTC.
775 """
776 return self.expIdMask[ampName]
items
std::vector< SchemaItem< Flag > > * items
Definition BaseColumnView.cc:152
to
table::Key< int > to
Definition TransformMap.cc:348
lsst::ip::isr.calibType.IsrCalib.updateMetadata
updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
Definition calibType.py:197
lsst::ip::isr.ptcDataset.PhotonTransferCurveDataset.__init__
__init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs)
Definition ptcDataset.py:179
lsst::ip::isr.ptcDataset.PhotonTransferCurveDataset.setAmpValuesPartialDataset
setAmpValuesPartialDataset(self, ampName, inputExpIdPair=(-1, -1), rawExpTime=np.nan, rawMean=np.nan, rawVar=np.nan, photoCharge=np.nan, expIdMask=False, covariance=None, covSqrtWeights=None, gain=np.nan, noise=np.nan, histVar=np.nan, histChi2Dof=np.nan, kspValue=0.0, auxValues=None)
Definition ptcDataset.py:253
list
daf::base::PropertyList * list
Definition fits.cc:928
Generated on Thu Dec 14 2023 04:18:46 for LSST Applications by doxygen 1.9.7