LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
ptcDataset.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2017 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 """
23 Define dataset class for MeasurePhotonTransferCurve task
24 """
25 import numpy as np
26 from astropy.table import Table
27 
28 from lsst.ip.isr import IsrCalib
29 
30 __all__ = ['PhotonTransferCurveDataset']
31 
32 
34  """A simple class to hold the output data from the PTC task.
35  The dataset is made up of a dictionary for each item, keyed by the
36  amplifiers' names, which much be supplied at construction time.
37  New items cannot be added to the class to save accidentally saving to the
38  wrong property, and the class can be frozen if desired.
39  inputExpIdPairs records the exposures used to produce the data.
40  When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which
41  is by definition always the same length as inputExpIdPairs, rawExpTimes,
42  rawMeans and rawVars, and is a list of bools, which are incrementally set
43  to False as points are discarded from the fits.
44  PTC fit parameters for polynomials are stored in a list in ascending order
45  of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
46  with the length of the list corresponding to the order of the polynomial
47  plus one.
48 
49  Parameters
50  ----------
51  ampNames : `list`
52  List with the names of the amplifiers of the detector at hand.
53 
54  ptcFitType : `str`
55  Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION",
56  or "FULLCOVARIANCE".
57 
58  covMatrixSide : `int`
59  Maximum lag of covariances (size of square covariance matrices).
60 
61  kwargs : `dict`, optional
62  Other keyword arguments to pass to the parent init.
63 
64  Notes
65  -----
66  The stored attributes are:
67  badAmps : `list`
68  List with bad amplifiers names.
69  inputExpIdPairs : `dict`, [`str`, `list`]
70  Dictionary keyed by amp names containing the input exposures IDs.
71  expIdMask : `dict`, [`str`, `list`]
72  Dictionary keyed by amp names containing the mask produced after
73  outlier rejection. The mask produced by the "FULLCOVARIANCE"
74  option may differ from the one produced in the other two PTC
75  fit types.
76  rawExpTimes : `dict`, [`str`, `list`]
77  Dictionary keyed by amp names containing the unmasked exposure times.
78  rawMeans : `dict`, [`str`, `list`]
79  Dictionary keyed by amp namescontaining the unmasked average of the
80  means of the exposures in each flat pair.
81  rawVars : `dict`, [`str`, `list`]
82  Dictionary keyed by amp names containing the variance of the
83  difference image of the exposures in each flat pair.
84  gain : `dict`, [`str`, `list`]
85  Dictionary keyed by amp names containing the fitted gains.
86  gainErr : `dict`, [`str`, `list`]
87  Dictionary keyed by amp names containing the errors on the
88  fitted gains.
89  noise : `dict`, [`str`, `list`]
90  Dictionary keyed by amp names containing the fitted noise.
91  noiseErr : `dict`, [`str`, `list`]
92  Dictionary keyed by amp names containing the errors on the fitted noise.
93  ptcFitPars : `dict`, [`str`, `list`]
94  Dictionary keyed by amp names containing the fitted parameters of the
95  PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
96  ptcFitParsError : `dict`, [`str`, `list`]
97  Dictionary keyed by amp names containing the errors on the fitted
98  parameters of the PTC model for ptcFitTye in
99  ["POLYNOMIAL", "EXPAPPROXIMATION"].
100  ptcFitChiSq : `dict`, [`str`, `list`]
101  Dictionary keyed by amp names containing the reduced chi squared
102  of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
103  covariances : `dict`, [`str`, `list`]
104  Dictionary keyed by amp names containing a list of measured
105  covariances per mean flux.
106  covariancesModel : `dict`, [`str`, `list`]
107  Dictionary keyed by amp names containinging covariances model
108  (Eq. 20 of Astier+19) per mean flux.
109  covariancesSqrtWeights : `dict`, [`str`, `list`]
110  Dictionary keyed by amp names containinging sqrt. of covariances
111  weights.
112  aMatrix : `dict`, [`str`, `list`]
113  Dictionary keyed by amp names containing the "a" parameters from
114  the model in Eq. 20 of Astier+19.
115  bMatrix : `dict`, [`str`, `list`]
116  Dictionary keyed by amp names containing the "b" parameters from
117  the model in Eq. 20 of Astier+19.
118  covariancesModelNoB : `dict`, [`str`, `list`]
119  Dictionary keyed by amp names containing covariances model
120  (with 'b'=0 in Eq. 20 of Astier+19)
121  per mean flux.
122  aMatrixNoB : `dict`, [`str`, `list`]
123  Dictionary keyed by amp names containing the "a" parameters from the
124  model in Eq. 20 of Astier+19
125  (and 'b' = 0).
126  finalVars : `dict`, [`str`, `list`]
127  Dictionary keyed by amp names containing the masked variance of the
128  difference image of each flat
129  pair. If needed, each array will be right-padded with
130  np.nan to match the length of rawExpTimes.
131  finalModelVars : `dict`, [`str`, `list`]
132  Dictionary keyed by amp names containing the masked modeled
133  variance of the difference image of each flat pair. If needed, each
134  array will be right-padded with np.nan to match the length of
135  rawExpTimes.
136  finalMeans : `dict`, [`str`, `list`]
137  Dictionary keyed by amp names containing the masked average of the
138  means of the exposures in each flat pair. If needed, each array
139  will be right-padded with np.nan to match the length of
140  rawExpTimes.
141  photoCharge : `dict`, [`str`, `list`]
142  Dictionary keyed by amp names containing the integrated photocharge
143  for linearity calibration.
144 
145  Returns
146  -------
147  `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
148  Output dataset from MeasurePhotonTransferCurveTask.
149  """
150 
151  _OBSTYPE = 'PTC'
152  _SCHEMA = 'Gen3 Photon Transfer Curve'
153  _VERSION = 1.0
154 
155  def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
156 
157  self.ptcFitTypeptcFitType = ptcFitType
158  self.ampNamesampNames = ampNames
159  self.covMatrixSidecovMatrixSide = covMatrixSide
160 
161  self.badAmpsbadAmps = [np.nan]
162 
163  self.inputExpIdPairsinputExpIdPairs = {ampName: [] for ampName in ampNames}
164  self.expIdMaskexpIdMask = {ampName: [] for ampName in ampNames}
165  self.rawExpTimesrawExpTimes = {ampName: [] for ampName in ampNames}
166  self.rawMeansrawMeans = {ampName: [] for ampName in ampNames}
167  self.rawVarsrawVars = {ampName: [] for ampName in ampNames}
168  self.photoChargephotoCharge = {ampName: [] for ampName in ampNames}
169 
170  self.gaingain = {ampName: np.nan for ampName in ampNames}
171  self.gainErrgainErr = {ampName: np.nan for ampName in ampNames}
172  self.noisenoise = {ampName: np.nan for ampName in ampNames}
173  self.noiseErrnoiseErr = {ampName: np.nan for ampName in ampNames}
174 
175  self.ptcFitParsptcFitPars = {ampName: [] for ampName in ampNames}
176  self.ptcFitParsErrorptcFitParsError = {ampName: [] for ampName in ampNames}
177  self.ptcFitChiSqptcFitChiSq = {ampName: np.nan for ampName in ampNames}
178 
179  self.covariancescovariances = {ampName: [] for ampName in ampNames}
180  self.covariancesModelcovariancesModel = {ampName: [] for ampName in ampNames}
181  self.covariancesSqrtWeightscovariancesSqrtWeights = {ampName: [] for ampName in ampNames}
182  self.aMatrixaMatrix = {ampName: np.nan for ampName in ampNames}
183  self.bMatrixbMatrix = {ampName: np.nan for ampName in ampNames}
184  self.covariancesModelNoBcovariancesModelNoB = {ampName: [] for ampName in ampNames}
185  self.aMatrixNoBaMatrixNoB = {ampName: np.nan for ampName in ampNames}
186 
187  self.finalVarsfinalVars = {ampName: [] for ampName in ampNames}
188  self.finalModelVarsfinalModelVars = {ampName: [] for ampName in ampNames}
189  self.finalMeansfinalMeans = {ampName: [] for ampName in ampNames}
190 
191  super().__init__(**kwargs)
192  self.requiredAttributesrequiredAttributesrequiredAttributesrequiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes',
193  'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr',
194  'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'aMatrixNoB',
195  'covariances', 'covariancesModel', 'covariancesSqrtWeights',
196  'covariancesModelNoB',
197  'aMatrix', 'bMatrix', 'finalVars', 'finalModelVars', 'finalMeans',
198  'photoCharge'])
199 
200  def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np.nan],
201  rawExpTime=[np.nan], rawMean=[np.nan], rawVar=[np.nan], photoCharge=[np.nan],
202  gain=np.nan, gainErr=np.nan, noise=np.nan, noiseErr=np.nan, ptcFitPars=[np.nan],
203  ptcFitParsError=[np.nan], ptcFitChiSq=np.nan, covArray=[], covArrayModel=[],
204  covSqrtWeights=[], aMatrix=[], bMatrix=[], covArrayModelNoB=[], aMatrixNoB=[],
205  finalVar=[np.nan], finalModelVar=[np.nan], finalMean=[np.nan]):
206  """Function to initialize an amp of a PhotonTransferCurveDataset.
207 
208  Notes
209  -----
210  The parameters are all documented in `init`.
211  """
212  nanMatrix = np.full((self.covMatrixSidecovMatrixSide, self.covMatrixSidecovMatrixSide), np.nan)
213  if len(covArray) == 0:
214  covArray = [nanMatrix]
215  if len(covArrayModel) == 0:
216  covArrayModel = [nanMatrix]
217  if len(covSqrtWeights) == 0:
218  covSqrtWeights = [nanMatrix]
219  if len(covArrayModelNoB) == 0:
220  covArrayModelNoB = [nanMatrix]
221  if len(aMatrix) == 0:
222  aMatrix = nanMatrix
223  if len(bMatrix) == 0:
224  bMatrix = nanMatrix
225  if len(aMatrixNoB) == 0:
226  aMatrixNoB = nanMatrix
227 
228  self.inputExpIdPairsinputExpIdPairs[ampName] = inputExpIdPair
229  self.expIdMaskexpIdMask[ampName] = expIdMask
230  self.rawExpTimesrawExpTimes[ampName] = rawExpTime
231  self.rawMeansrawMeans[ampName] = rawMean
232  self.rawVarsrawVars[ampName] = rawVar
233  self.photoChargephotoCharge[ampName] = photoCharge
234  self.gaingain[ampName] = gain
235  self.gainErrgainErr[ampName] = gainErr
236  self.noisenoise[ampName] = noise
237  self.noiseErrnoiseErr[ampName] = noiseErr
238  self.ptcFitParsptcFitPars[ampName] = ptcFitPars
239  self.ptcFitParsErrorptcFitParsError[ampName] = ptcFitParsError
240  self.ptcFitChiSqptcFitChiSq[ampName]
241  self.covariancescovariances[ampName] = covArray
242  self.covariancesSqrtWeightscovariancesSqrtWeights[ampName] = covSqrtWeights
243  self.covariancesModelcovariancesModel[ampName] = covArrayModel
244  self.covariancesModelNoBcovariancesModelNoB[ampName] = covArrayModelNoB
245  self.aMatrixaMatrix[ampName] = aMatrix
246  self.bMatrixbMatrix[ampName] = bMatrix
247  self.aMatrixNoBaMatrixNoB[ampName] = aMatrixNoB
248  self.ptcFitParsptcFitPars[ampName] = ptcFitPars
249  self.ptcFitParsErrorptcFitParsError[ampName] = ptcFitParsError
250  self.ptcFitChiSqptcFitChiSq[ampName] = ptcFitChiSq
251  self.finalVarsfinalVars[ampName] = finalVar
252  self.finalModelVarsfinalModelVars[ampName] = finalModelVar
253  self.finalMeansfinalMeans[ampName] = finalMean
254 
255  def updateMetadata(self, setDate=False, **kwargs):
256  """Update calibration metadata.
257  This calls the base class's method after ensuring the required
258  calibration keywords will be saved.
259  Parameters
260  ----------
261  setDate : `bool`, optional
262  Update the CALIBDATE fields in the metadata to the current
263  time. Defaults to False.
264  kwargs :
265  Other keyword parameters to set in the metadata.
266  """
267  kwargs['PTC_FIT_TYPE'] = self.ptcFitTypeptcFitType
268 
269  super().updateMetadata(setDate=setDate, **kwargs)
270 
271  @classmethod
272  def fromDict(cls, dictionary):
273  """Construct a calibration from a dictionary of properties.
274  Must be implemented by the specific calibration subclasses.
275  Parameters
276  ----------
277  dictionary : `dict`
278  Dictionary of properties.
279  Returns
280  -------
281  calib : `lsst.ip.isr.CalibType`
282  Constructed calibration.
283  Raises
284  ------
285  RuntimeError :
286  Raised if the supplied dictionary is for a different
287  calibration.
288  """
289  calib = cls()
290  if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
291  raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. "
292  f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
293  calib.setMetadata(dictionary['metadata'])
294  calib.ptcFitType = dictionary['ptcFitType']
295  calib.covMatrixSide = dictionary['covMatrixSide']
296  calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist()
297  # The cov matrices are square
298  covMatrixSide = calib.covMatrixSide
299  # Number of final signal levels
300  covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel())
301  nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide))
302 
303  for ampName in dictionary['ampNames']:
304  calib.ampNames.append(ampName)
305  calib.inputExpIdPairs[ampName] = np.array(dictionary['inputExpIdPairs'][ampName]).tolist()
306  calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]).tolist()
307  calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName]).tolist()
308  calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName]).tolist()
309  calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName]).tolist()
310  calib.gain[ampName] = np.array(dictionary['gain'][ampName]).tolist()
311  calib.gainErr[ampName] = np.array(dictionary['gainErr'][ampName]).tolist()
312  calib.noise[ampName] = np.array(dictionary['noise'][ampName]).tolist()
313  calib.noiseErr[ampName] = np.array(dictionary['noiseErr'][ampName]).tolist()
314  calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName]).tolist()
315  calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName]).tolist()
316  calib.ptcFitChiSq[ampName] = np.array(dictionary['ptcFitChiSq'][ampName]).tolist()
317  calib.covariances[ampName] = np.array(dictionary['covariances'][ampName]).reshape(
318  (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
319  calib.covariancesModel[ampName] = np.array(
320  dictionary['covariancesModel'][ampName]).reshape(
321  (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
322  calib.covariancesSqrtWeights[ampName] = np.array(
323  dictionary['covariancesSqrtWeights'][ampName]).reshape(
324  (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
325  calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName]).reshape(
326  (covMatrixSide, covMatrixSide)).tolist()
327  calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName]).reshape(
328  (covMatrixSide, covMatrixSide)).tolist()
329  calib.covariancesModelNoB[ampName] = np.array(
330  dictionary['covariancesModelNoB'][ampName]).reshape(
331  (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
332  calib.aMatrixNoB[ampName] = np.array(
333  dictionary['aMatrixNoB'][ampName]).reshape((covMatrixSide, covMatrixSide)).tolist()
334  calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName]).tolist()
335  calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName]).tolist()
336  calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName]).tolist()
337  calib.photoCharge[ampName] = np.array(dictionary['photoCharge'][ampName]).tolist()
338  calib.updateMetadata()
339  return calib
340 
341  def toDict(self):
342  """Return a dictionary containing the calibration properties.
343  The dictionary should be able to be round-tripped through
344  `fromDict`.
345  Returns
346  -------
347  dictionary : `dict`
348  Dictionary of properties.
349  """
350  self.updateMetadataupdateMetadataupdateMetadata()
351 
352  outDict = dict()
353  metadata = self.getMetadatagetMetadata()
354  outDict['metadata'] = metadata
355 
356  outDict['ptcFitType'] = self.ptcFitTypeptcFitType
357  outDict['covMatrixSide'] = self.covMatrixSidecovMatrixSide
358  outDict['ampNames'] = self.ampNamesampNames
359  outDict['badAmps'] = self.badAmpsbadAmps
360  outDict['inputExpIdPairs'] = self.inputExpIdPairsinputExpIdPairs
361  outDict['expIdMask'] = self.expIdMaskexpIdMask
362  outDict['rawExpTimes'] = self.rawExpTimesrawExpTimes
363  outDict['rawMeans'] = self.rawMeansrawMeans
364  outDict['rawVars'] = self.rawVarsrawVars
365  outDict['gain'] = self.gaingain
366  outDict['gainErr'] = self.gainErrgainErr
367  outDict['noise'] = self.noisenoise
368  outDict['noiseErr'] = self.noiseErrnoiseErr
369  outDict['ptcFitPars'] = self.ptcFitParsptcFitPars
370  outDict['ptcFitParsError'] = self.ptcFitParsErrorptcFitParsError
371  outDict['ptcFitChiSq'] = self.ptcFitChiSqptcFitChiSq
372  outDict['covariances'] = self.covariancescovariances
373  outDict['covariancesModel'] = self.covariancesModelcovariancesModel
374  outDict['covariancesSqrtWeights'] = self.covariancesSqrtWeightscovariancesSqrtWeights
375  outDict['aMatrix'] = self.aMatrixaMatrix
376  outDict['bMatrix'] = self.bMatrixbMatrix
377  outDict['covariancesModelNoB'] = self.covariancesModelNoBcovariancesModelNoB
378  outDict['aMatrixNoB'] = self.aMatrixNoBaMatrixNoB
379  outDict['finalVars'] = self.finalVarsfinalVars
380  outDict['finalModelVars'] = self.finalModelVarsfinalModelVars
381  outDict['finalMeans'] = self.finalMeansfinalMeans
382  outDict['photoCharge'] = self.photoChargephotoCharge
383 
384  return outDict
385 
386  @classmethod
387  def fromTable(cls, tableList):
388  """Construct calibration from a list of tables.
389  This method uses the `fromDict` method to create the
390  calibration, after constructing an appropriate dictionary from
391  the input tables.
392  Parameters
393  ----------
394  tableList : `list` [`lsst.afw.table.Table`]
395  List of tables to use to construct the datasetPtc.
396  Returns
397  -------
398  calib : `lsst.cp.pipe.`
399  The calibration defined in the tables.
400  """
401  ptcTable = tableList[0]
402 
403  metadata = ptcTable.meta
404  inDict = dict()
405  inDict['metadata'] = metadata
406  inDict['ampNames'] = []
407  inDict['ptcFitType'] = []
408  inDict['covMatrixSide'] = []
409  inDict['inputExpIdPairs'] = dict()
410  inDict['expIdMask'] = dict()
411  inDict['rawExpTimes'] = dict()
412  inDict['rawMeans'] = dict()
413  inDict['rawVars'] = dict()
414  inDict['gain'] = dict()
415  inDict['gainErr'] = dict()
416  inDict['noise'] = dict()
417  inDict['noiseErr'] = dict()
418  inDict['ptcFitPars'] = dict()
419  inDict['ptcFitParsError'] = dict()
420  inDict['ptcFitChiSq'] = dict()
421  inDict['covariances'] = dict()
422  inDict['covariancesModel'] = dict()
423  inDict['covariancesSqrtWeights'] = dict()
424  inDict['aMatrix'] = dict()
425  inDict['bMatrix'] = dict()
426  inDict['covariancesModelNoB'] = dict()
427  inDict['aMatrixNoB'] = dict()
428  inDict['finalVars'] = dict()
429  inDict['finalModelVars'] = dict()
430  inDict['finalMeans'] = dict()
431  inDict['badAmps'] = []
432  inDict['photoCharge'] = dict()
433 
434  for record in ptcTable:
435  ampName = record['AMPLIFIER_NAME']
436 
437  inDict['ptcFitType'] = record['PTC_FIT_TYPE']
438  inDict['covMatrixSide'] = record['COV_MATRIX_SIDE']
439  inDict['ampNames'].append(ampName)
440  inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS']
441  inDict['expIdMask'][ampName] = record['EXP_ID_MASK']
442  inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES']
443  inDict['rawMeans'][ampName] = record['RAW_MEANS']
444  inDict['rawVars'][ampName] = record['RAW_VARS']
445  inDict['gain'][ampName] = record['GAIN']
446  inDict['gainErr'][ampName] = record['GAIN_ERR']
447  inDict['noise'][ampName] = record['NOISE']
448  inDict['noiseErr'][ampName] = record['NOISE_ERR']
449  inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS']
450  inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR']
451  inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ']
452  inDict['covariances'][ampName] = record['COVARIANCES']
453  inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL']
454  inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS']
455  inDict['aMatrix'][ampName] = record['A_MATRIX']
456  inDict['bMatrix'][ampName] = record['B_MATRIX']
457  inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B']
458  inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B']
459  inDict['finalVars'][ampName] = record['FINAL_VARS']
460  inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS']
461  inDict['finalMeans'][ampName] = record['FINAL_MEANS']
462  inDict['badAmps'] = record['BAD_AMPS']
463  inDict['photoCharge'][ampName] = record['PHOTO_CHARGE']
464  return cls().fromDict(inDict)
465 
466  def toTable(self):
467  """Construct a list of tables containing the information in this
468  calibration.
469 
470  The list of tables should create an identical calibration
471  after being passed to this class's fromTable method.
472  Returns
473  -------
474  tableList : `list` [`astropy.table.Table`]
475  List of tables containing the linearity calibration
476  information.
477  """
478  tableList = []
479  self.updateMetadataupdateMetadataupdateMetadata()
480  nPoints = []
481  for i, ampName in enumerate(self.ampNamesampNames):
482  nPoints.append(len(list(self.covariancescovariances.values())[i]))
483  nSignalPoints = max(nPoints)
484  nPadPoints = {}
485  for i, ampName in enumerate(self.ampNamesampNames):
486  nPadPoints[ampName] = nSignalPoints - len(list(self.covariancescovariances.values())[i])
487  covMatrixSide = self.covMatrixSidecovMatrixSide
488 
489  catalog = Table([{'AMPLIFIER_NAME': ampName,
490  'PTC_FIT_TYPE': self.ptcFitTypeptcFitType,
491  'COV_MATRIX_SIDE': self.covMatrixSidecovMatrixSide,
492  'INPUT_EXP_ID_PAIRS': self.inputExpIdPairsinputExpIdPairs[ampName]
493  if len(self.expIdMaskexpIdMask[ampName]) else np.nan,
494  'EXP_ID_MASK': self.expIdMaskexpIdMask[ampName]
495  if len(self.expIdMaskexpIdMask[ampName]) else np.nan,
496  'RAW_EXP_TIMES': np.array(self.rawExpTimesrawExpTimes[ampName]).tolist()
497  if len(self.rawExpTimesrawExpTimes[ampName]) else np.nan,
498  'RAW_MEANS': np.array(self.rawMeansrawMeans[ampName]).tolist()
499  if len(self.rawMeansrawMeans[ampName]) else np.nan,
500  'RAW_VARS': np.array(self.rawVarsrawVars[ampName]).tolist()
501  if len(self.rawVarsrawVars[ampName]) else np.nan,
502  'GAIN': self.gaingain[ampName],
503  'GAIN_ERR': self.gainErrgainErr[ampName],
504  'NOISE': self.noisenoise[ampName],
505  'NOISE_ERR': self.noiseErrnoiseErr[ampName],
506  'PTC_FIT_PARS': np.array(self.ptcFitParsptcFitPars[ampName]).tolist(),
507  'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsErrorptcFitParsError[ampName]).tolist(),
508  'PTC_FIT_CHI_SQ': self.ptcFitChiSqptcFitChiSq[ampName],
509  'COVARIANCES': np.pad(np.array(self.covariancescovariances[ampName]),
510  ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
511  'constant', constant_values=np.nan).reshape(
512  nSignalPoints*covMatrixSide**2).tolist(),
513  'COVARIANCES_MODEL': np.pad(np.array(self.covariancesModelcovariancesModel[ampName]),
514  ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
515  'constant', constant_values=np.nan).reshape(
516  nSignalPoints*covMatrixSide**2).tolist(),
517  'COVARIANCES_SQRT_WEIGHTS': np.pad(np.array(self.covariancesSqrtWeightscovariancesSqrtWeights[ampName]),
518  ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
519  'constant', constant_values=0.0).reshape(
520  nSignalPoints*covMatrixSide**2).tolist(),
521  'A_MATRIX': np.array(self.aMatrixaMatrix[ampName]).reshape(covMatrixSide**2).tolist(),
522  'B_MATRIX': np.array(self.bMatrixbMatrix[ampName]).reshape(covMatrixSide**2).tolist(),
523  'COVARIANCES_MODEL_NO_B':
524  np.pad(np.array(self.covariancesModelNoBcovariancesModelNoB[ampName]),
525  ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
526  'constant', constant_values=np.nan).reshape(
527  nSignalPoints*covMatrixSide**2).tolist(),
528  'A_MATRIX_NO_B': np.array(self.aMatrixNoBaMatrixNoB[ampName]).reshape(
529  covMatrixSide**2).tolist(),
530  'FINAL_VARS': np.pad(np.array(self.finalVarsfinalVars[ampName]), (0, nPadPoints[ampName]),
531  'constant', constant_values=np.nan).tolist(),
532  'FINAL_MODEL_VARS': np.pad(np.array(self.finalModelVarsfinalModelVars[ampName]),
533  (0, nPadPoints[ampName]),
534  'constant', constant_values=np.nan).tolist(),
535  'FINAL_MEANS': np.pad(np.array(self.finalMeansfinalMeans[ampName]),
536  (0, nPadPoints[ampName]),
537  'constant', constant_values=np.nan).tolist(),
538  'BAD_AMPS': np.array(self.badAmpsbadAmps).tolist() if len(self.badAmpsbadAmps) else np.nan,
539  'PHOTO_CHARGE': np.array(self.photoChargephotoCharge[ampName]).tolist(),
540  } for ampName in self.ampNamesampNames])
541  inMeta = self.getMetadatagetMetadata().toDict()
542  outMeta = {k: v for k, v in inMeta.items() if v is not None}
543  outMeta.update({k: "" for k, v in inMeta.items() if v is None})
544  catalog.meta = outMeta
545  tableList.append(catalog)
546 
547  return(tableList)
548 
549  def getExpIdsUsed(self, ampName):
550  """Get the exposures used, i.e. not discarded, for a given amp.
551  If no mask has been created yet, all exposures are returned.
552  """
553  if len(self.expIdMaskexpIdMask[ampName]) == 0:
554  return self.inputExpIdPairsinputExpIdPairs[ampName]
555 
556  # if the mask exists it had better be the same length as the expIdPairs
557  assert len(self.expIdMaskexpIdMask[ampName]) == len(self.inputExpIdPairsinputExpIdPairs[ampName])
558 
559  pairs = self.inputExpIdPairsinputExpIdPairs[ampName]
560  mask = self.expIdMaskexpIdMask[ampName]
561  # cast to bool required because numpy
562  return [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if bool(m) is True]
563 
564  def getGoodAmps(self):
565  return [amp for amp in self.ampNamesampNames if amp not in self.badAmpsbadAmps]
int max
def requiredAttributes(self, value)
Definition: calibType.py:142
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
Definition: calibType.py:181
def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs)
Definition: ptcDataset.py:155
def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np.nan], rawExpTime=[np.nan], rawMean=[np.nan], rawVar=[np.nan], photoCharge=[np.nan], gain=np.nan, gainErr=np.nan, noise=np.nan, noiseErr=np.nan, ptcFitPars=[np.nan], ptcFitParsError=[np.nan], ptcFitChiSq=np.nan, covArray=[], covArrayModel=[], covSqrtWeights=[], aMatrix=[], bMatrix=[], covArrayModelNoB=[], aMatrixNoB=[], finalVar=[np.nan], finalModelVar=[np.nan], finalMean=[np.nan])
Definition: ptcDataset.py:205
def updateMetadata(self, setDate=False, **kwargs)
Definition: ptcDataset.py:255
daf::base::PropertyList * list
Definition: fits.cc:913
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
Definition: functional.cc:33