LSST Applications  21.0.0+04719a4bac,21.0.0-1-ga51b5d4+f5e6047307,21.0.0-11-g2b59f77+a9c1acf22d,21.0.0-11-ga42c5b2+86977b0b17,21.0.0-12-gf4ce030+76814010d2,21.0.0-13-g1721dae+760e7a6536,21.0.0-13-g3a573fe+768d78a30a,21.0.0-15-g5a7caf0+f21cbc5713,21.0.0-16-g0fb55c1+b60e2d390c,21.0.0-19-g4cded4ca+71a93a33c0,21.0.0-2-g103fe59+bb20972958,21.0.0-2-g45278ab+04719a4bac,21.0.0-2-g5242d73+3ad5d60fb1,21.0.0-2-g7f82c8f+8babb168e8,21.0.0-2-g8f08a60+06509c8b61,21.0.0-2-g8faa9b5+616205b9df,21.0.0-2-ga326454+8babb168e8,21.0.0-2-gde069b7+5e4aea9c2f,21.0.0-2-gecfae73+1d3a86e577,21.0.0-2-gfc62afb+3ad5d60fb1,21.0.0-25-g1d57be3cd+e73869a214,21.0.0-3-g357aad2+ed88757d29,21.0.0-3-g4a4ce7f+3ad5d60fb1,21.0.0-3-g4be5c26+3ad5d60fb1,21.0.0-3-g65f322c+e0b24896a3,21.0.0-3-g7d9da8d+616205b9df,21.0.0-3-ge02ed75+a9c1acf22d,21.0.0-4-g591bb35+a9c1acf22d,21.0.0-4-g65b4814+b60e2d390c,21.0.0-4-gccdca77+0de219a2bc,21.0.0-4-ge8a399c+6c55c39e83,21.0.0-5-gd00fb1e+05fce91b99,21.0.0-6-gc675373+3ad5d60fb1,21.0.0-64-g1122c245+4fb2b8f86e,21.0.0-7-g04766d7+cd19d05db2,21.0.0-7-gdf92d54+04719a4bac,21.0.0-8-g5674e7b+d1bd76f71f,master-gac4afde19b+a9c1acf22d,w.2021.13
LSST Data Management Base Package
fgcmCalibrateTractBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
23 """
24 
25 import sys
26 import traceback
27 import abc
28 
29 import numpy as np
30 
31 import lsst.daf.persistence as dafPersist
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 
35 from .fgcmBuildStars import FgcmBuildStarsTask, FgcmBuildStarsConfig
36 from .fgcmFitCycle import FgcmFitCycleConfig
37 from .fgcmOutputProducts import FgcmOutputProductsTask
38 from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
39 from .utilities import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
40 from .utilities import makeZptSchema, makeZptCat
41 from .utilities import makeAtmSchema, makeAtmCat
42 from .utilities import makeStdSchema, makeStdCat
43 
44 import fgcm
45 
46 __all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask', 'FgcmCalibrateTractRunner']
47 
48 
49 class FgcmCalibrateTractConfigBase(pexConfig.Config):
50  """Config for FgcmCalibrateTract"""
51 
52  fgcmBuildStars = pexConfig.ConfigurableField(
53  target=FgcmBuildStarsTask,
54  doc="Task to load and match stars for fgcm",
55  )
56  fgcmFitCycle = pexConfig.ConfigField(
57  dtype=FgcmFitCycleConfig,
58  doc="Config to run a single fgcm fit cycle",
59  )
60  fgcmOutputProducts = pexConfig.ConfigurableField(
61  target=FgcmOutputProductsTask,
62  doc="Task to output fgcm products",
63  )
64  convergenceTolerance = pexConfig.Field(
65  doc="Tolerance on repeatability convergence (per band)",
66  dtype=float,
67  default=0.005,
68  )
69  maxFitCycles = pexConfig.Field(
70  doc="Maximum number of fit cycles",
71  dtype=int,
72  default=5,
73  )
74  doDebuggingPlots = pexConfig.Field(
75  doc="Make plots for debugging purposes?",
76  dtype=bool,
77  default=False,
78  )
79 
80  def setDefaults(self):
81  pexConfig.Config.setDefaults(self)
82 
83  self.fgcmFitCyclefgcmFitCycle.quietMode = True
84  self.fgcmOutputProductsfgcmOutputProducts.doReferenceCalibration = False
85  self.fgcmOutputProductsfgcmOutputProducts.doRefcatOutput = False
86  self.fgcmOutputProductsfgcmOutputProducts.cycleNumber = 0
87  self.fgcmOutputProductsfgcmOutputProducts.photoCal.applyColorTerms = False
88 
89  def validate(self):
90  super().validate()
91 
92  for band in self.fgcmFitCyclefgcmFitCycle.bands:
93  if not self.fgcmFitCyclefgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
94  msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
95  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
96  self, msg)
97 
98 
99 class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner):
100  """Subclass of TaskRunner for FgcmCalibrateTractTask
101 
102  fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
103  the butler (for persistence and mapper data), and a list of dataRefs
104  extracted from the command line. This task runs on a constrained set
105  of dataRefs, typically a single tract.
106  This class transforms the process arguments generated by the ArgumentParser
107  into the arguments expected by FgcmCalibrateTractTask.run().
108  This runner does not use any parallelization.
109  """
110 
111  @staticmethod
112  def getTargetList(parsedCmd):
113  """
114  Return a list with one element: a tuple with the butler and
115  list of dataRefs.
116  """
117  return [(parsedCmd.butler, parsedCmd.id.refList)]
118 
119  def __call__(self, args):
120  """
121  Parameters
122  ----------
123  args: `tuple` with (butler, dataRefList)
124 
125  Returns
126  -------
127  exitStatus: `list` with `lsst.pipe.base.Struct`
128  exitStatus (0: success; 1: failure)
129  May also contain results if `self.doReturnResults` is `True`.
130  """
131  butler, dataRefList = args
132 
133  task = self.TaskClass(config=self.config, log=self.log)
134 
135  exitStatus = 0
136  if self.doRaise:
137  results = task.runDataRef(butler, dataRefList)
138  else:
139  try:
140  results = task.runDataRef(butler, dataRefList)
141  except Exception as e:
142  exitStatus = 1
143  task.log.fatal("Failed: %s" % e)
144  if not isinstance(e, pipeBase.TaskError):
145  traceback.print_exc(file=sys.stderr)
146 
147  task.writeMetadata(butler)
148 
149  if self.doReturnResults:
150  return [pipeBase.Struct(exitStatus=exitStatus,
151  results=results)]
152  else:
153  return [pipeBase.Struct(exitStatus=exitStatus)]
154 
155  def run(self, parsedCmd):
156  """
157  Run the task, with no multiprocessing
158 
159  Parameters
160  ----------
161  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
162  """
163 
164  resultList = []
165 
166  if self.precall(parsedCmd):
167  targetList = self.getTargetListgetTargetList(parsedCmd)
168  resultList = self(targetList[0])
169 
170  return resultList
171 
172 
173 class FgcmCalibrateTractBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
174  """Base class to calibrate a single tract using fgcmcal
175  """
176  def __init__(self, butler=None, **kwargs):
177  """
178  Instantiate an `FgcmCalibrateTractTask`.
179 
180  Parameters
181  ----------
182  butler : `lsst.daf.persistence.Butler`, optional
183  """
184  super().__init__(**kwargs)
185  self.makeSubtask("fgcmBuildStars", butler=butler)
186  self.makeSubtask("fgcmOutputProducts", butler=butler)
187 
188  # no saving of metadata for now
189  def _getMetadataName(self):
190  return None
191 
192  @pipeBase.timeMethod
193  def runDataRef(self, butler, dataRefs):
194  """
195  Run full FGCM calibration on a single tract, including building star list,
196  fitting multiple cycles, and making outputs.
197 
198  Parameters
199  ----------
200  butler: `lsst.daf.persistence.Butler`
201  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
202  Data references for the input visits.
203  These may be either per-ccd "src" or per-visit"sourceTable_visit"
204  references.
205 
206  Raises
207  ------
208  RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
209  not True, or if fgcmLookUpTable is not available, or if
210  doSubtractLocalBackground is True and aperture radius cannot be
211  determined.
212  """
213  datasetType = dataRefs[0].butlerSubset.datasetType
214  self.log.info("Running with %d %s dataRefs" % (len(dataRefs), datasetType))
215 
216  if not butler.datasetExists('fgcmLookUpTable'):
217  raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable")
218 
219  if not self.config.fgcmBuildStars.doReferenceMatches:
220  raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
221  if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
222  if self.config.fgcmBuildStars.checkAllCcds:
223  raise RuntimeError("Cannot run FgcmCalibrateTract with "
224  "fgcmBuildStars.checkAllCcds set to True")
225 
226  tract = int(dataRefs[0].dataId['tract'])
227  camera = butler.get('camera')
228 
229  dataRefDict = {}
230  dataRefDict['camera'] = camera
231  dataRefDict['source_catalogs'] = dataRefs
232  dataRefDict['sourceSchema'] = butler.dataRef('src_schema')
233  dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
234 
235  struct = self.runrun(dataRefDict, tract, butler=butler, returnCatalogs=False)
236 
237  visitDataRefName = self.config.fgcmBuildStars.visitDataRefName
238  ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName
239 
240  if struct.photoCalibs is not None:
241  self.log.info("Outputting photoCalib files.")
242 
243  for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
244  butler.put(photoCalib, 'fgcm_tract_photoCalib',
245  dataId={visitDataRefName: visit,
246  ccdDataRefName: detector,
247  'filter': physicalFilter,
248  'tract': tract})
249 
250  self.log.info("Done outputting photoCalib files.")
251 
252  if struct.atmospheres is not None:
253  self.log.info("Outputting atmosphere files.")
254  for visit, atm in struct.atmospheres:
255  butler.put(atm, "transmission_atmosphere_fgcm_tract",
256  dataId={visitDataRefName: visit,
257  'tract': tract})
258  self.log.info("Done outputting atmosphere transmissions.")
259 
260  return pipeBase.Struct(repeatability=struct.repeatability)
261 
262  def run(self, dataRefDict, tract,
263  buildStarsRefObjLoader=None, returnCatalogs=True, butler=None):
264  """Run the calibrations for a single tract with fgcm.
265 
266  Parameters
267  ----------
268  dataRefDict : `dict`
269  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
270  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
271  dataRef dictionary with the following keys. Note that all
272  keys need not be set based on config parameters.
273 
274  ``"camera"``
275  Camera object (`lsst.afw.cameraGeom.Camera`)
276  ``"source_catalogs"``
277  `list` of dataRefs for input source catalogs.
278  ``"sourceSchema"``
279  Schema for the source catalogs.
280  ``"fgcmLookUpTable"``
281  dataRef for the FGCM look-up table.
282  ``"calexps"``
283  `list` of dataRefs for the input calexps (Gen3 only)
284  ``"fgcmPhotoCalibs"``
285  `dict` of output photoCalib dataRefs. Key is
286  (tract, visit, detector). (Gen3 only)
287  Present if doZeropointOutput is True.
288  ``"fgcmTransmissionAtmospheres"``
289  `dict` of output atmosphere transmission dataRefs.
290  Key is (tract, visit). (Gen3 only)
291  Present if doAtmosphereOutput is True.
292  tract : `int`
293  Tract number
294  buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
295  Reference object loader object for fgcmBuildStars.
296  returnCatalogs : `bool`, optional
297  Return photoCalibs as per-visit exposure catalogs.
298  butler : `lsst.daf.persistence.Butler`, optional
299  Gen2 butler used for reference star outputs
300 
301  Returns
302  -------
303  outstruct : `lsst.pipe.base.Struct`
304  Output structure with keys:
305 
306  offsets : `np.ndarray`
307  Final reference offsets, per band.
308  repeatability : `np.ndarray`
309  Raw fgcm repeatability for bright stars, per band.
310  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
311  Generator that returns (visit, transmissionCurve) tuples.
312  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
313  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
314  (returned if returnCatalogs is False).
315  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
316  Generator that returns (visit, exposureCatalog) tuples.
317  (returned if returnCatalogs is True).
318  """
319  self.log.info("Running on tract %d", (tract))
320 
321  # Compute the aperture radius if necessary. This is useful to do now before
322  # any heavy lifting has happened (fail early).
323  calibFluxApertureRadius = None
324  if self.config.fgcmBuildStars.doSubtractLocalBackground:
325  try:
326  field = self.config.fgcmBuildStars.instFluxField
327  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefDict['source_catalogs'][0],
328  field)
329  except RuntimeError:
330  raise RuntimeError("Could not determine aperture radius from %s. "
331  "Cannot use doSubtractLocalBackground." %
332  (field))
333 
334  # Run the build stars tasks
335 
336  # Note that we will need visitCat at the end of the procedure for the outputs
337  if isinstance(butler, dafPersist.Butler):
338  # Gen2
339  groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefsGen2(butler, dataRefDict['camera'],
340  dataRefDict['source_catalogs'])
341  else:
342  # Gen3
343  groupedDataRefs = self.fgcmBuildStars._groupDataRefs(dataRefDict['sourceTableDataRefDict'],
344  dataRefDict['visitSummaryDataRefDict'])
345  visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict['camera'], groupedDataRefs)
346  rad = calibFluxApertureRadius
347  fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
348  visitCat,
349  dataRefDict['sourceSchema'],
350  dataRefDict['camera'],
351  calibFluxApertureRadius=rad)
352 
353  if self.fgcmBuildStars.config.doReferenceMatches:
354  lutDataRef = dataRefDict['fgcmLookUpTable']
355  if buildStarsRefObjLoader is not None:
356  self.fgcmBuildStars.makeSubtask("fgcmLoadReferenceCatalog",
357  refObjLoader=buildStarsRefObjLoader)
358  else:
359  self.fgcmBuildStars.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
360  else:
361  lutDataRef = None
362 
363  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
364  self.fgcmBuildStars.fgcmMatchStars(visitCat,
365  fgcmStarObservationCat,
366  lutDataRef=lutDataRef)
367 
368  # Load the LUT
369  lutCat = dataRefDict['fgcmLookUpTable'].get()
370  fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
371  dict(self.config.fgcmFitCycle.physicalFilterMap))
372  del lutCat
373 
374  # Translate the visit catalog into fgcm format
375  fgcmExpInfo = translateVisitCatalog(visitCat)
376 
377  configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict['camera'],
378  self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
379  True, False, lutIndexVals[0]['FILTERNAMES'],
380  tract=tract)
381 
382  # Turn off plotting in tract mode
383  configDict['doPlots'] = False
384 
385  # Use the first orientation.
386  # TODO: DM-21215 will generalize to arbitrary camera orientations
387  ccdOffsets = computeCcdOffsets(dataRefDict['camera'], fgcmExpInfo['TELROT'][0])
388 
389  # Set up the fit cycle task
390 
391  noFitsDict = {'lutIndex': lutIndexVals,
392  'lutStd': lutStd,
393  'expInfo': fgcmExpInfo,
394  'ccdOffsets': ccdOffsets}
395 
396  fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
397  noFitsDict=noFitsDict, noOutput=True)
398 
399  # We determine the conversion from the native units (typically radians) to
400  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
401  # numpy arrays rather than Angles, which would we approximately 600x slower.
402  conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra'])
403 
404  # To load the stars, we need an initial parameter object
405  fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
406  fgcmLut,
407  fgcmExpInfo)
408 
409  # Match star observations to visits
410  # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
411  # actually be transferred into fgcm using the indexing below.
412 
413  obsIndex = fgcmStarIndicesCat['obsIndex']
414  visitIndex = np.searchsorted(fgcmExpInfo['VISIT'],
415  fgcmStarObservationCat['visit'][obsIndex])
416 
417  refMag, refMagErr = extractReferenceMags(fgcmRefCat,
418  self.config.fgcmFitCycle.bands,
419  self.config.fgcmFitCycle.physicalFilterMap)
420  refId = fgcmRefCat['fgcm_id'][:]
421 
422  fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
423  fgcmStars.loadStars(fgcmPars,
424  fgcmStarObservationCat['visit'][obsIndex],
425  fgcmStarObservationCat['ccd'][obsIndex],
426  fgcmStarObservationCat['ra'][obsIndex] * conv,
427  fgcmStarObservationCat['dec'][obsIndex] * conv,
428  fgcmStarObservationCat['instMag'][obsIndex],
429  fgcmStarObservationCat['instMagErr'][obsIndex],
430  fgcmExpInfo['FILTERNAME'][visitIndex],
431  fgcmStarIdCat['fgcm_id'][:],
432  fgcmStarIdCat['ra'][:],
433  fgcmStarIdCat['dec'][:],
434  fgcmStarIdCat['obsArrIndex'][:],
435  fgcmStarIdCat['nObs'][:],
436  obsX=fgcmStarObservationCat['x'][obsIndex],
437  obsY=fgcmStarObservationCat['y'][obsIndex],
438  obsDeltaMagBkg=fgcmStarObservationCat['deltaMagBkg'][obsIndex],
439  psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex],
440  refID=refId,
441  refMag=refMag,
442  refMagErr=refMagErr,
443  flagID=None,
444  flagFlag=None,
445  computeNobs=True)
446 
447  # Clear out some memory
448  del fgcmStarIdCat
449  del fgcmStarIndicesCat
450  del fgcmRefCat
451 
452  fgcmFitCycle.setLUT(fgcmLut)
453  fgcmFitCycle.setStars(fgcmStars, fgcmPars)
454 
455  converged = False
456  cycleNumber = 0
457 
458  previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
459  previousParInfo = None
460  previousParams = None
461  previousSuperStar = None
462 
463  while (not converged and cycleNumber < self.config.maxFitCycles):
464 
465  fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
466 
467  if cycleNumber > 0:
468  # Use parameters from previous cycle
469  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
470  fgcmExpInfo,
471  previousParInfo,
472  previousParams,
473  previousSuperStar)
474  # We need to reset the star magnitudes and errors for the next
475  # cycle
476  fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
477  fgcmStarObservationCat['instMagErr'][obsIndex])
478  fgcmFitCycle.initialCycle = False
479 
480  fgcmFitCycle.setPars(fgcmPars)
481  fgcmFitCycle.finishSetup()
482 
483  fgcmFitCycle.run()
484 
485  # Grab the parameters for the next cycle
486  previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
487  previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
488 
489  self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber))
490  for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
491  if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
492  continue
493  rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
494  self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
495 
496  # Check for convergence
497  if np.alltrue((previousReservedRawRepeatability
498  - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
499  < self.config.convergenceTolerance):
500  self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber))
501  converged = True
502  else:
503  fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
504  fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
505  fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False
506  fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
507  previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
508  self.log.info("Setting exposure gray photometricity cuts to:")
509  for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
510  if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
511  continue
512  cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
513  self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut))
514 
515  cycleNumber += 1
516 
517  # Log warning if not converged
518  if not converged:
519  self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
520 
521  # Do final clean-up iteration
522  fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
523  fgcmFitCycle.fgcmConfig.resetParameters = False
524  fgcmFitCycle.fgcmConfig.maxIter = 0
525  fgcmFitCycle.fgcmConfig.outputZeropoints = True
526  fgcmFitCycle.fgcmConfig.outputStandards = True
527  fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
528  fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
529  fgcmFitCycle.initialCycle = False
530 
531  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
532  fgcmExpInfo,
533  previousParInfo,
534  previousParams,
535  previousSuperStar)
536  fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
537  fgcmStarObservationCat['instMagErr'][obsIndex])
538  fgcmFitCycle.setPars(fgcmPars)
539  fgcmFitCycle.finishSetup()
540 
541  self.log.info("Running final clean-up fit cycle...")
542  fgcmFitCycle.run()
543 
544  self.log.info("Raw repeatability after clean-up cycle is:")
545  for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
546  if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
547  continue
548  rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
549  self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
550 
551  # Do the outputs. Need to keep track of tract.
552 
553  superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
554  zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
555 
556  zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
557  zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
558 
559  atmSchema = makeAtmSchema()
560  atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
561 
562  stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
563  stdSchema = makeStdSchema(len(goodBands))
564  stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
565 
566  outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
567  tract,
568  visitCat,
569  zptCat, atmCat, stdCat,
570  self.config.fgcmBuildStars,
571  returnCatalogs=returnCatalogs,
572  butler=butler)
573 
574  outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
575 
576  return outStruct
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:880
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:805
def makeStdSchema(nBands)
Definition: utilities.py:735
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:702
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition: utilities.py:50
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:215
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:623
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:767
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:537
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:396
def translateVisitCatalog(visitCat)
Definition: utilities.py:345