LSSTApplications  20.0.0
LSSTDataManagementBasePackage
runEotestTask.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 """Calibration products production task code."""
23 from __future__ import absolute_import, division, print_function
24 
25 import os
26 import glob
27 import sys
28 
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 import lsst.log as lsstLog
32 import lsst.eotest.sensor as sensorTest
33 
34 
35 class RunEotestConfig(pexConfig.Config):
36  """Config class for the calibration products production (CP) task."""
37 
38  ccdKey = pexConfig.Field(
39  dtype=str,
40  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
41  default='ccd',
42  )
43  fe55 = pexConfig.ConfigurableField(
44  target=sensorTest.Fe55Task,
45  doc="The Fe55 analysis task.",
46  )
47  doFe55 = pexConfig.Field(
48  dtype=bool,
49  doc="Measure gains using Fe55?",
50  default=True,
51  )
52  readNoise = pexConfig.ConfigurableField(
53  target=sensorTest.ReadNoiseTask,
54  doc="The read noise task.",
55  )
56  doReadNoise = pexConfig.Field(
57  dtype=bool,
58  doc="Measure the read-noise?",
59  default=True,
60  )
61  brightPixels = pexConfig.ConfigurableField(
62  target=sensorTest.BrightPixelsTask,
63  doc="The bright pixel/column finding task.",
64  )
65  doBrightPixels = pexConfig.Field(
66  dtype=bool,
67  doc="Find bright pixels?",
68  default=True,
69  )
70  darkPixels = pexConfig.ConfigurableField(
71  target=sensorTest.DarkPixelsTask,
72  doc="The dark pixel/column finding task.",
73  )
74  doDarkPixels = pexConfig.Field(
75  dtype=bool,
76  doc="Find dark pixels?",
77  default=True,
78  )
79  traps = pexConfig.ConfigurableField(
80  target=sensorTest.TrapTask,
81  doc="The trap-finding task.",
82  )
83  doTraps = pexConfig.Field(
84  dtype=bool,
85  doc="Find traps using pocket-pumping exposures?",
86  default=True,
87  )
88  cte = pexConfig.ConfigurableField(
89  target=sensorTest.CteTask,
90  doc="The CTE analysis task.",
91  )
92  doCTE = pexConfig.Field(
93  dtype=bool,
94  doc="Measure the charge transfer efficiency?",
95  default=True,
96  )
97  ptc = pexConfig.ConfigurableField(
98  target=sensorTest.PtcTask,
99  doc="The PTC analysis task.",
100  )
101  doPTC = pexConfig.Field(
102  dtype=bool,
103  doc="Measure the photon transfer curve?",
104  default=True,
105  )
106  flatPair = pexConfig.ConfigurableField(
107  target=sensorTest.FlatPairTask,
108  doc="The flat-pair analysis task.",
109  )
110  doFlatPair = pexConfig.Field(
111  dtype=bool,
112  doc="Measure the detector response vs incident flux using flat pairs?",
113  default=True,
114  )
115  eotestOutputPath = pexConfig.Field(
116  dtype=str,
117  doc="Path to which to write the eotest output results. Madatory runtime arg for running eotest.",
118  default='',
119  )
120  requireAllEOTests = pexConfig.Field(
121  dtype=bool,
122  doc="If True, all tests are required to be runnable, and will Raise if data is missing. If False, "
123  "processing will continue if a previous part failed due to the input dataset being incomplete.",
124  default=True,
125  )
126  flatPairMaxPdFracDev = pexConfig.Field(
127  dtype=float,
128  doc="Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. "
129  "This value is passed to the task's run() method at runtime rather than being stored in the task's"
130  "own pexConfig field.",
131  default=0.05,
132  )
133 
134  def setDefaults(self):
135  """Set default config options for the subTasks."""
136  # TODO: Set to proper values - DM-12939
137  self.fe55.temp_set_point = -100
138  self.fe55.temp_set_point_tol = 20
139 
140  # TODO: Set to proper values - DM-12939
141  self.readNoise.temp_set_point = -100
142  self.readNoise.temp_set_point_tol = 20
143 
144  # TODO: make this work - DM-12939
145  self.brightPixels.temp_set_point = -100
146  self.brightPixels.temp_set_point_tol = 20
147 
148  # TODO: find the proper settings for flatPairTask to work. This will mean either using the expTime,
149  # or working out what's wrong with the MONDIODE values (debug that anyway as a large difference might
150  # indicate something else going on). Note that this task doesn't really use much in its config class,
151  # but passes in things at runtime param to its run() method, hence putting in a slot for them here.
152  # DM-12939
153  self.flatPairMaxPdFracDev = 0.99
154 
155  def validate(self):
156  """Override of the valiate() method.
157 
158  The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs
159  for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we
160  override the validate() method here, and use it to set the output directory for each of the tasks
161  based on the legal pexConfig parameter for the main task.
162  """
163  log = lsstLog.Log.getLogger("cp.pipe.runEotestConfig")
164  if not self.eotestOutputPath:
165  raise RuntimeError("Must supply an output path for eotest data. "
166  "Please set config.eotestOutputPath.")
167 
168  taskList = ['fe55', 'brightPixels', 'darkPixels', 'readNoise', 'traps', 'cte', 'flatPair', 'ptc']
169  for task in taskList:
170  if getattr(self, task).output_dir != '.':
171  # Being thorough here: '.' is the eotest default. If this is not the value then the user has
172  # specified something, and we're going to clobber it, so raise a warning. Unlike to happen.
173  log.warn("OVERWRITING: Found a user defined output path of %s for %sTask. "
174  "This has been overwritten with %s, as individually specified output paths for "
175  "subTasks are not supported at present" % (getattr(self, task).output_dir,
176  task, self.eotestOutputPath))
177  getattr(self, task).output_dir = self.eotestOutputPath
178 
179 
180 class RunEotestTask(pipeBase.CmdLineTask):
181  """
182  Task to run test stand data through eotest using a butler.
183 
184  This task is used to produce an eotest report (the project's sensor
185  acceptance testing package)
186  Examples of some of its operations are as follows:
187  * Given a set of flat-field images, find the dark pixels and columns.
188  * Given a set of darks, find the bright pixels and columns.
189  * Given a set of Fe55 exposures, calulate the gain of the readout chain,
190  in e-/ADU
191  * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon,
192  and the degradation of
193  * the PSF due to CTE.
194  * Given a set of flat-pairs, measure the photon transfer curve (PTC).
195  * Given a set of bias frames, calculate the read noise of the system in e-.
196  * Given a set of pocket-pumping exposures, find charge-traps in the silicon.
197 
198  The RunEotestTask.runEotestDirect() is only applicable to LSST sensors, and
199  only for a specific type of dataset. This method takes a
200  dafPersistance.Butler corresponding to a repository in which a full eotest
201  run has been taken and ingested, and runs each of the tasks in eotest
202  directly, allowing for bitwise comparison with results given by the camera
203  team.
204 
205  See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for
206  further details regarding the inputs and outputs.
207  """
208 
209  ConfigClass = RunEotestConfig
210  _DefaultName = "runEotest"
211 
212  def __init__(self, *args, **kwargs):
213  """Constructor for the RunEotestTask."""
214  if 'lsst.eotest.sensor' not in sys.modules: # check we have eotest before going further
215  raise RuntimeError('eotest failed to import')
216 
217  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
218 
219  # Note - we can't currently call validate on the subTask configs, as they are NOT valid
220  # due to state of eotest. However, we override validate() and call it here
221  # and use it to set the output dir config parameter in the subTasks.
222  self.config.validate()
223  self.config.freeze()
224 
225  self.makeSubtask("fe55")
226  self.makeSubtask("readNoise")
227  self.makeSubtask("brightPixels")
228  self.makeSubtask("darkPixels")
229  self.makeSubtask("traps")
230  self.makeSubtask("cte")
231  self.makeSubtask("flatPair")
232  self.makeSubtask("ptc")
233 
234  def _getMaskFiles(self, path, ccd):
235  """Get all available eotest mask files for a given ccd.
236 
237  Each stage of the processing generates more mask files, so this allows each to be picked up
238  as more and more tests run, and saves having to have clever logic for if some tasks fail.
239 
240  Parameters
241  ----------
242  path : `str`
243  Path on which to find the mask files
244  ccd : `string` or `int`
245  Name/identifier of the CCD
246 
247  Returns
248  -------
249  maskFiles : iterable of `str`
250  List of mask files, or an empty tuple if none are found
251  """
252  pattern = '*' + str(ccd) + '*mask*' # the cast to str supports obs_auxTel
253  maskFiles = glob.glob(os.path.join(path, pattern))
254  return maskFiles if len(maskFiles) > 0 else () # eotest wants an empty tuple here
255 
256  def _cleanupEotest(self, path):
257  """Delete all the medianed files left behind after eotest has run.
258 
259  Running eotest generates a lot of interim medianed files, so this just cleans them up.
260 
261  Parameters
262  ----------
263  path : `str`
264  Path on which to delete all the eotest medianed files.
265  """
266  for filename in glob.glob(os.path.join(path, '*_median_*.fits')):
267  os.remove(filename)
268 
269  def makeEotestReport(self, butler):
270  """After running eotest, generate pdf(s) of the results.
271 
272  Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD.
273  The pdf file(s), along with the .tex file(s) and the individual plots are written
274  to the eotestOutputPath.
275  .pdf generation requires a TeX distro including pdflatex to be installed.
276  """
277  ccds = butler.queryMetadata('raw', self.config.ccdKey)
278  for ccd in ccds:
279  self.log.info("Starting test report generation for %s"%ccd)
280  try:
281  plotPath = os.path.join(self.config.eotestOutputPath, 'plots')
282  if not os.path.exists(plotPath):
283  os.makedirs(plotPath)
284  plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath)
285  eoTestReport = sensorTest.EOTestReport(plots, wl_dir='')
286  eoTestReport.make_figures()
287  eoTestReport.make_pdf()
288  except Exception as e:
289  self.log.warn("Failed to make eotest report for %s: %s"%(ccd, e))
290  self.log.info("Finished test report generation.")
291 
292  @pipeBase.timeMethod
293  def runEotestDirect(self, butler, run=None):
294  """
295  Generate calibration products using eotest algorithms.
296 
297  Generate all calibration products possible using the vanilla eotest implementation,
298  given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to
299  only a single raft/RTM.
300 
301  - Run all eotest tasks possible, using the butler to gather the data
302  - Write outputs in eotest format
303 
304  In order to replicate the canonical eotest analysis, the tasks should be run in a specific order.
305  This is given/defined in the "Steps" section here:
306  http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179
307 
308  But is replicated here for conveniece:
309  * 55Fe Analysis
310  * CCD Read Noise Analysis
311  * Bright Defects Analysis
312  * Dark Defects Analysis
313  * Traps Finding
314  * Dark Current X - will not be implemented here
315  * Charge Transfer Efficiencies
316  * Photo-response analysis X - will not be implemented here
317  * Flat Pairs Analysis
318  * Photon Transfer Curve
319  * Quantum Efficiency X - will not be implemented here
320 
321  List of tasks that exist in the eotest package but aren't mentioned on the above link:
322  * linearityTask()
323  * fe55CteTask()
324  * eperTask()
325  * crosstalkTask()
326  * persistenceTask()
327 
328  # TODO: For each eotest task, find out what the standard raft testing does for the optional params.
329  i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we
330  need to know what is typically provided to these tasks when the camera team runs this code.
331  This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness
332  but it sounds like Jim Chiang doesn't recommend trying to do that.
333  DM-12939
334 
335  Parameters
336  ----------
337  butler : `lsst.daf.persistence.butler`
338  Butler for the repo containg the eotest data to be used
339  run : `str` or `int`
340  Optional run number, to be used for repos containing multiple runs
341  """
342  self.log.info("Running eotest routines direct")
343 
344  # Input testing to check that run is in the repo
345  runs = butler.queryMetadata('raw', ['run'])
346  if run is None:
347  if len(runs) == 1:
348  run = runs[0]
349  else:
350  raise RuntimeError("Butler query found %s for runs. eotest datasets must have a run number,"
351  "and you must specify which run to use if a respoitory contains several."
352  % runs)
353  else:
354  run = str(run)
355  if run not in runs:
356  raise RuntimeError("Butler query found %s for runs, but the run specified (%s) "
357  "was not among them." % (runs, run))
358  del runs # we have run defined now, so remove this to avoid potential confusion later
359 
360  if not os.path.exists(self.config.eotestOutputPath):
361  os.makedirs(self.config.eotestOutputPath)
362 
363  ccds = butler.queryMetadata('raw', self.config.ccdKey)
364  imTypes = butler.queryMetadata('raw', ['imageType'])
365  testTypes = butler.queryMetadata('raw', ['testType'])
366 
367 
370  if self.config.doFe55:
371  fe55TaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'FE55'}
372  self.log.info("Starting Fe55 pixel task")
373  for ccd in ccds:
374  if 'FE55' not in testTypes:
375  msg = "No Fe55 tests found. Available data: %s" % testTypes
376  if self.config.requireAllEOTests:
377  raise RuntimeError(msg)
378  else:
379  self.log.warn(msg + "\nSkipping Fe55 task")
380  break
381  fe55Filenames = [butler.get('raw_filename', dataId={'visit': visit,
382  self.config.ccdKey: ccd})[0][:-3]
383  for visit in butler.queryMetadata('raw', ['visit'], dataId=fe55TaskDataId)]
384  self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames)))
385  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
386  gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles)
387  # gainsPropSet = dafBase.PropertySet()
388  # for amp, gain in gains.items(): # there is no propSet.fromDict() method so make like this
389  # gainsPropSet.addDouble(str(amp), gain)
390  butler.put(gains, 'eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
391  del fe55TaskDataId
392 
393  # TODO: validate the results above, and/or change code to (be able to) always run
394  # over all files instead of stopping at the "required accuracy"
395  # This will require making changes to the eotest code.
396  # DM-12939
397 
398 
401  if self.config.doReadNoise:
402  # note that LCA-10103 defines the Fe55 bias frames as the ones to use here
403  self.log.info("Starting readNoise task")
404  noiseTaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'BIAS'}
405  for ccd in ccds:
406  if ('FE55' not in testTypes) or ('BIAS' not in imTypes):
407  msg = "Required data for readNoise unavailable. Available data:\
408  \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes)
409  if self.config.requireAllEOTests:
410  raise RuntimeError(msg)
411  else:
412  self.log.warn(msg + "\nSkipping noise task")
413  noiseFilenames = [butler.get('raw_filename', dataId={'visit': visit,
414  self.config.ccdKey: ccd})[0][:-3]
415  for visit in butler.queryMetadata('raw', ['visit'],
416  dataId=noiseTaskDataId)]
417  self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames)))
418  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
419  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
420  self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames,
421  gains=gains, mask_files=maskFiles)
422  del noiseTaskDataId
423 
424 
427  if self.config.doBrightPixels:
428  self.log.info("Starting bright pixel task")
429  brightTaskDataId = {'run': run, 'testType': 'DARK', 'imageType': 'DARK'}
430  for ccd in ccds:
431  if 'DARK' not in testTypes:
432  msg = "No dark tests found. Available data: %s" % testTypes
433  if self.config.requireAllEOTests:
434  raise RuntimeError(msg)
435  else:
436  self.log.warn(msg + "\nSkipping bright pixel task")
437  break
438  darkFilenames = [butler.get('raw_filename', dataId={'visit': visit,
439  self.config.ccdKey: ccd})[0][:-3]
440  for visit in butler.queryMetadata('raw', ['visit'],
441  dataId=brightTaskDataId)]
442  self.log.trace("BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames)))
443  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
444  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
445  self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames,
446  mask_files=maskFiles, gains=gains)
447  del brightTaskDataId
448 
449 
452  if self.config.doDarkPixels:
453  self.log.info("Starting dark pixel task")
454  darkTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
455  for ccd in ccds:
456  if 'SFLAT_500' not in testTypes:
457  msg = "No superflats found. Available data: %s" % testTypes
458  if self.config.requireAllEOTests:
459  raise RuntimeError(msg)
460  else:
461  self.log.warn(msg + "\nSkipping dark pixel task")
462  break
463  sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
464  self.config.ccdKey: ccd})[0][:-3]
465  for visit in butler.queryMetadata('raw', ['visit'],
466  dataId=darkTaskDataId)]
467  self.log.trace("DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
468  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
469  self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles)
470  del darkTaskDataId
471 
472 
475  if self.config.doTraps:
476  self.log.info("Starting trap task")
477  trapTaskDataId = {'run': run, 'testType': 'TRAP', 'imageType': 'PPUMP'}
478  for ccd in ccds:
479  if ('TRAP' not in testTypes) and ('PPUMP' not in imTypes):
480  msg = "No pocket pumping exposures found. Available data: %s" % testTypes
481  if self.config.requireAllEOTests:
482  raise RuntimeError(msg)
483  else:
484  self.log.warn(msg + "\nSkipping trap task")
485  break
486  trapFilenames = [butler.get('raw_filename', dataId={'visit': visit,
487  self.config.ccdKey: ccd})[0][:-3]
488  for visit in butler.queryMetadata('raw', ['visit'], dataId=trapTaskDataId)]
489  if len(trapFilenames) != 1: # eotest can't handle more than one
490  msg = "Trap Task: Found more than one ppump trap file: %s" % trapFilenames
491  msg += " Running using only the first one found."
492  self.log.warn(msg)
493  self.log.trace("Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames)))
494  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
495  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
496  self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0],
497  mask_files=maskFiles, gains=gains)
498  del trapTaskDataId
499 
500 
503  if self.config.doCTE:
504  self.log.info("Starting CTE task")
505  cteTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
506  for ccd in ccds:
507  if 'SFLAT_500' not in testTypes:
508  msg = "No superflats found. Available data: %s" % testTypes
509  if self.config.requireAllEOTests:
510  raise RuntimeError(msg)
511  else:
512  self.log.warn(msg + "\nSkipping CTE task")
513  break
514  sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
515  self.config.ccdKey: ccd})[0][:-3]
516  for visit in butler.queryMetadata('raw', ['visit'], dataId=cteTaskDataId)]
517  self.log.trace("CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
518  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
519  self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles)
520  del cteTaskDataId
521 
522 
525  if self.config.doFlatPair:
526  self.log.info("Starting flatPair task")
527  flatPairDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
528  for ccd in ccds:
529  if 'FLAT' not in testTypes:
530  msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
531  if self.config.requireAllEOTests:
532  raise RuntimeError(msg)
533  else:
534  self.log.warn(msg + "\nSkipping flatPair task")
535  break
536  flatPairFilenames = [butler.get('raw_filename', dataId={'visit': visit,
537  self.config.ccdKey: ccd})[0][:-3]
538  for visit in butler.queryMetadata('raw', ['visit'],
539  dataId=flatPairDataId)]
540  # Note that eotest needs the original filename as written by the test-stand data acquisition
541  # system, as that is the only place the flat pair-number is recorded, so we have to resolve
542  # sym-links and pass in the *original* paths/filenames here :(
543  # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
544  # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
545  # isn't totally thorough on rejecting the wrong types of data here)
546  # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
547  # by the butler instead of here. DM-12939
548  flatPairFilenames = [os.path.realpath(f) for f in flatPairFilenames if
549  os.path.realpath(f).find('flat1') != -1 or
550  os.path.realpath(f).find('flat2') != -1]
551  if not flatPairFilenames:
552  raise RuntimeError("No flatPair files found.")
553  self.log.trace("FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames)))
554  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
555  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
556  self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles,
557  gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev)
558  del flatPairDataId
559 
560 
563  if self.config.doPTC:
564  self.log.info("Starting PTC task")
565  ptcDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
566  for ccd in ccds:
567  if 'FLAT' not in testTypes:
568  msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
569  if self.config.requireAllEOTests:
570  raise RuntimeError(msg)
571  else:
572  self.log.warn(msg + "\nSkipping PTC task")
573  break
574  ptcFilenames = [butler.get('raw_filename', dataId={'visit': visit,
575  self.config.ccdKey: ccd})[0][:-3]
576  for visit in butler.queryMetadata('raw', ['visit'], dataId=ptcDataId)]
577  # Note that eotest needs the original filename as written by the test-stand data acquisition
578  # system, as that is the only place the flat pair-number is recorded, so we have to resolve
579  # sym-links and pass in the *original* paths/filenames here :(
580  # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
581  # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
582  # isn't totally thorough on rejecting the wrong types of data here)
583  # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
584  # by the butler instead of here. DM-12939
585  ptcFilenames = [os.path.realpath(f) for f in ptcFilenames if
586  os.path.realpath(f).find('flat1') != -1 or
587  os.path.realpath(f).find('flat2') != -1]
588  if not ptcFilenames:
589  raise RuntimeError("No flatPair files found")
590  self.log.trace("PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames)))
591  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
592  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
593  self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains)
594  del ptcDataId
595 
596  self._cleanupEotest(self.config.eotestOutputPath)
597  self.log.info("Finished running EOTest")
lsst.cp.pipe.runEotestTask.RunEotestConfig
Definition: runEotestTask.py:35
lsst::log.log.logContinued.warn
def warn(fmt, *args)
Definition: logContinued.py:202
lsst::log.log.logContinued.info
def info(fmt, *args)
Definition: logContinued.py:198
lsst.cp.pipe.runEotestTask.RunEotestConfig.readNoise
readNoise
Definition: runEotestTask.py:52
lsst.cp.pipe.runEotestTask.RunEotestTask
Definition: runEotestTask.py:180
lsst.cp.pipe.runEotestTask.RunEotestTask._getMaskFiles
def _getMaskFiles(self, path, ccd)
Definition: runEotestTask.py:234
lsst.cp.pipe.runEotestTask.RunEotestTask.runEotestDirect
def runEotestDirect(self, butler, run=None)
Definition: runEotestTask.py:293
lsst.cp.pipe.runEotestTask.RunEotestConfig.flatPairMaxPdFracDev
flatPairMaxPdFracDev
Definition: runEotestTask.py:126
lsst.cp.pipe.runEotestTask.RunEotestTask._cleanupEotest
def _cleanupEotest(self, path)
Definition: runEotestTask.py:256
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsst::log
Definition: Log.h:706
lsst.cp.pipe.runEotestTask.RunEotestConfig.validate
def validate(self)
Definition: runEotestTask.py:155
lsst.cp.pipe.runEotestTask.RunEotestConfig.setDefaults
def setDefaults(self)
Definition: runEotestTask.py:134
lsst.cp.pipe.runEotestTask.RunEotestConfig.brightPixels
brightPixels
Definition: runEotestTask.py:61
lsst.cp.pipe.runEotestTask.RunEotestTask.__init__
def __init__(self, *args, **kwargs)
Definition: runEotestTask.py:212
lsst::log.log.logContinued.trace
def trace(fmt, *args)
Definition: logContinued.py:190
lsst.cp.pipe.runEotestTask.RunEotestConfig.fe55
fe55
Definition: runEotestTask.py:43
lsst.cp.pipe.runEotestTask.RunEotestTask.makeEotestReport
def makeEotestReport(self, butler)
Definition: runEotestTask.py:269
lsst.pipe.base
Definition: __init__.py:1
pex.config.wrap.validate
validate
Definition: wrap.py:295
lsst.cp.pipe.runEotestTask.RunEotestConfig.eotestOutputPath
eotestOutputPath
Definition: runEotestTask.py:115