21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
35 from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
36 from .fgcmFitCycle
import FgcmFitCycleConfig
37 from .fgcmOutputProducts
import FgcmOutputProductsTask
38 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
39 from .utilities
import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
40 from .utilities
import makeZptSchema, makeZptCat
41 from .utilities
import makeAtmSchema, makeAtmCat
42 from .utilities
import makeStdSchema, makeStdCat
46 __all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
50 """Config for FgcmCalibrateTract"""
52 fgcmBuildStars = pexConfig.ConfigurableField(
53 target=FgcmBuildStarsTask,
54 doc=
"Task to load and match stars for fgcm",
56 fgcmFitCycle = pexConfig.ConfigField(
57 dtype=FgcmFitCycleConfig,
58 doc=
"Config to run a single fgcm fit cycle",
60 fgcmOutputProducts = pexConfig.ConfigurableField(
61 target=FgcmOutputProductsTask,
62 doc=
"Task to output fgcm products",
64 convergenceTolerance = pexConfig.Field(
65 doc=
"Tolerance on repeatability convergence (per band)",
69 maxFitCycles = pexConfig.Field(
70 doc=
"Maximum number of fit cycles",
74 doDebuggingPlots = pexConfig.Field(
75 doc=
"Make plots for debugging purposes?",
81 pexConfig.Config.setDefaults(self)
94 if not self.
fgcmFitCyclefgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
95 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
96 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
101 """Subclass of TaskRunner for FgcmCalibrateTractTask
103 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
104 the butler (for persistence and mapper data), and a list of dataRefs
105 extracted from the command line. This task runs on a constrained set
106 of dataRefs, typically a single tract.
107 This class transforms the process arguments generated by the ArgumentParser
108 into the arguments expected by FgcmCalibrateTractTask.run().
109 This runner does not use any parallelization.
115 Return a list with one element: a tuple with the butler and
118 return [(parsedCmd.butler, parsedCmd.id.refList)]
124 args: `tuple` with (butler, dataRefList)
128 exitStatus: `list` with `lsst.pipe.base.Struct`
129 exitStatus (0: success; 1: failure)
130 May also contain results if `self.doReturnResults` is `True`.
132 butler, dataRefList = args
134 task = self.TaskClass(config=self.config, log=self.log)
138 results = task.runDataRef(butler, dataRefList)
141 results = task.runDataRef(butler, dataRefList)
142 except Exception
as e:
144 task.log.fatal(
"Failed: %s" % e)
145 if not isinstance(e, pipeBase.TaskError):
146 traceback.print_exc(file=sys.stderr)
148 task.writeMetadata(butler)
150 if self.doReturnResults:
151 return [pipeBase.Struct(exitStatus=exitStatus,
154 return [pipeBase.Struct(exitStatus=exitStatus)]
158 Run the task, with no multiprocessing
162 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
167 if self.precall(parsedCmd):
169 resultList = self(targetList[0])
175 """Base class to calibrate a single tract using fgcmcal
177 def __init__(self, initInputs=None, butler=None, **kwargs):
179 Instantiate an `FgcmCalibrateTractTask`.
183 butler : `lsst.daf.persistence.Butler`, optional
186 self.makeSubtask(
"fgcmBuildStars", initInputs=initInputs, butler=butler)
187 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
190 def _getMetadataName(self):
196 Run full FGCM calibration on a single tract, including building star list,
197 fitting multiple cycles, and making outputs.
201 butler: `lsst.daf.persistence.Butler`
202 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
203 Data references for the input visits.
204 These may be either per-ccd "src" or per-visit"sourceTable_visit"
209 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
210 not True, or if fgcmLookUpTable is not available, or if
211 doSubtractLocalBackground is True and aperture radius cannot be
214 datasetType = dataRefs[0].butlerSubset.datasetType
215 self.log.
info(
"Running with %d %s dataRefs" % (len(dataRefs), datasetType))
217 if not butler.datasetExists(
'fgcmLookUpTable'):
218 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
220 if not self.config.fgcmBuildStars.doReferenceMatches:
221 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
222 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
223 if self.config.fgcmBuildStars.checkAllCcds:
224 raise RuntimeError(
"Cannot run FgcmCalibrateTract with "
225 "fgcmBuildStars.checkAllCcds set to True")
227 tract = int(dataRefs[0].dataId[
'tract'])
228 camera = butler.get(
'camera')
231 dataRefDict[
'camera'] = camera
232 dataRefDict[
'source_catalogs'] = dataRefs
233 dataRefDict[
'sourceSchema'] = butler.get(
'src_schema', immediate=
True).schema
234 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
236 struct = self.
runrun(dataRefDict, tract, butler=butler, returnCatalogs=
False)
238 visitDataRefName = self.config.fgcmBuildStars.visitDataRefName
239 ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName
241 if struct.photoCalibs
is not None:
242 self.log.
info(
"Outputting photoCalib files.")
244 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
245 butler.put(photoCalib,
'fgcm_tract_photoCalib',
246 dataId={visitDataRefName: visit,
247 ccdDataRefName: detector,
248 'filter': physicalFilter,
251 self.log.
info(
"Done outputting photoCalib files.")
253 if struct.atmospheres
is not None:
254 self.log.
info(
"Outputting atmosphere files.")
255 for visit, atm
in struct.atmospheres:
256 butler.put(atm,
"transmission_atmosphere_fgcm_tract",
257 dataId={visitDataRefName: visit,
259 self.log.
info(
"Done outputting atmosphere transmissions.")
261 return pipeBase.Struct(repeatability=struct.repeatability)
263 def run(self, dataRefDict, tract,
264 buildStarsRefObjLoader=None, returnCatalogs=True, butler=None):
265 """Run the calibrations for a single tract with fgcm.
270 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
271 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
272 dataRef dictionary with the following keys. Note that all
273 keys need not be set based on config parameters.
276 Camera object (`lsst.afw.cameraGeom.Camera`)
277 ``"source_catalogs"``
278 `list` of dataRefs for input source catalogs.
280 Schema for the source catalogs.
281 ``"fgcmLookUpTable"``
282 dataRef for the FGCM look-up table.
284 `list` of dataRefs for the input calexps (Gen3 only)
285 ``"fgcmPhotoCalibs"``
286 `dict` of output photoCalib dataRefs. Key is
287 (tract, visit, detector). (Gen3 only)
288 Present if doZeropointOutput is True.
289 ``"fgcmTransmissionAtmospheres"``
290 `dict` of output atmosphere transmission dataRefs.
291 Key is (tract, visit). (Gen3 only)
292 Present if doAtmosphereOutput is True.
295 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
296 Reference object loader object for fgcmBuildStars.
297 returnCatalogs : `bool`, optional
298 Return photoCalibs as per-visit exposure catalogs.
299 butler : `lsst.daf.persistence.Butler`, optional
300 Gen2 butler used for reference star outputs
304 outstruct : `lsst.pipe.base.Struct`
305 Output structure with keys:
307 offsets : `np.ndarray`
308 Final reference offsets, per band.
309 repeatability : `np.ndarray`
310 Raw fgcm repeatability for bright stars, per band.
311 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
312 Generator that returns (visit, transmissionCurve) tuples.
313 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
314 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
315 (returned if returnCatalogs is False).
316 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
317 Generator that returns (visit, exposureCatalog) tuples.
318 (returned if returnCatalogs is True).
320 self.log.
info(
"Running on tract %d", (tract))
324 calibFluxApertureRadius =
None
325 if self.config.fgcmBuildStars.doSubtractLocalBackground:
327 field = self.config.fgcmBuildStars.instFluxField
331 raise RuntimeError(
"Could not determine aperture radius from %s. "
332 "Cannot use doSubtractLocalBackground." %
340 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefsGen2(butler, dataRefDict[
'camera'],
341 dataRefDict[
'source_catalogs'])
344 groupedDataRefs = self.fgcmBuildStars._groupDataRefs(dataRefDict[
'sourceTableDataRefDict'],
345 dataRefDict[
'visitSummaryDataRefDict'])
346 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict[
'camera'], groupedDataRefs)
347 rad = calibFluxApertureRadius
348 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
350 dataRefDict[
'sourceSchema'],
351 dataRefDict[
'camera'],
352 calibFluxApertureRadius=rad)
354 if self.fgcmBuildStars.config.doReferenceMatches:
355 lutDataRef = dataRefDict[
'fgcmLookUpTable']
356 if buildStarsRefObjLoader
is not None:
357 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
358 refObjLoader=buildStarsRefObjLoader)
360 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog", butler=butler)
364 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
365 self.fgcmBuildStars.fgcmMatchStars(visitCat,
366 fgcmStarObservationCat,
367 lutDataRef=lutDataRef)
370 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
372 dict(self.config.fgcmFitCycle.physicalFilterMap))
378 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict[
'camera'],
379 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
380 True,
False, lutIndexVals[0][
'FILTERNAMES'],
389 noFitsDict = {
'lutIndex': lutIndexVals,
391 'expInfo': fgcmExpInfo,
392 'ccdOffsets': ccdOffsets}
394 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
395 noFitsDict=noFitsDict, noOutput=
True)
400 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
403 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
411 obsIndex = fgcmStarIndicesCat[
'obsIndex']
412 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
413 fgcmStarObservationCat[
'visit'][obsIndex])
416 self.config.fgcmFitCycle.bands,
417 self.config.fgcmFitCycle.physicalFilterMap)
418 refId = fgcmRefCat[
'fgcm_id'][:]
420 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
421 fgcmStars.loadStars(fgcmPars,
422 fgcmStarObservationCat[
'visit'][obsIndex],
423 fgcmStarObservationCat[
'ccd'][obsIndex],
424 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
425 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
426 fgcmStarObservationCat[
'instMag'][obsIndex],
427 fgcmStarObservationCat[
'instMagErr'][obsIndex],
428 fgcmExpInfo[
'FILTERNAME'][visitIndex],
429 fgcmStarIdCat[
'fgcm_id'][:],
430 fgcmStarIdCat[
'ra'][:],
431 fgcmStarIdCat[
'dec'][:],
432 fgcmStarIdCat[
'obsArrIndex'][:],
433 fgcmStarIdCat[
'nObs'][:],
434 obsX=fgcmStarObservationCat[
'x'][obsIndex],
435 obsY=fgcmStarObservationCat[
'y'][obsIndex],
436 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
437 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
447 del fgcmStarIndicesCat
450 fgcmFitCycle.setLUT(fgcmLut)
451 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
456 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
457 previousParInfo =
None
458 previousParams =
None
459 previousSuperStar =
None
461 while (
not converged
and cycleNumber < self.config.maxFitCycles):
463 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
467 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
474 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
475 fgcmStarObservationCat[
'instMagErr'][obsIndex])
476 fgcmFitCycle.initialCycle =
False
478 fgcmFitCycle.setPars(fgcmPars)
479 fgcmFitCycle.finishSetup()
484 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
485 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
487 self.log.
info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
488 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
489 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
491 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
492 self.log.
info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
495 if np.alltrue((previousReservedRawRepeatability
496 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
497 < self.config.convergenceTolerance):
498 self.log.
info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
501 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
502 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
503 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
504 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
505 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
506 self.log.
info(
"Setting exposure gray photometricity cuts to:")
507 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
508 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
510 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
511 self.log.
info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
517 self.log.
warning(
"Maximum number of fit cycles exceeded (%d) without convergence.", cycleNumber)
520 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
521 fgcmFitCycle.fgcmConfig.resetParameters =
False
522 fgcmFitCycle.fgcmConfig.maxIter = 0
523 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
524 fgcmFitCycle.fgcmConfig.outputStandards =
True
525 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
526 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
527 fgcmFitCycle.initialCycle =
False
529 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
534 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
535 fgcmStarObservationCat[
'instMagErr'][obsIndex])
536 fgcmFitCycle.setPars(fgcmPars)
537 fgcmFitCycle.finishSetup()
539 self.log.
info(
"Running final clean-up fit cycle...")
542 self.log.
info(
"Raw repeatability after clean-up cycle is:")
543 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
544 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
546 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
547 self.log.
info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
551 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
552 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
555 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
558 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
560 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
562 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
564 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
567 zptCat, atmCat, stdCat,
568 self.config.fgcmBuildStars,
569 returnCatalogs=returnCatalogs,
572 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
574 fgcmFitCycle.freeSharedMemory()
def __init__(self, initInputs=None, butler=None, **kwargs)
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
def runDataRef(self, butler, dataRefs)
def getTargetList(parsedCmd)
def extractReferenceMags(refStars, bands, filterMap)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def makeStdSchema(nBands)
def makeAtmCat(atmSchema, atmStruct)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def computeCcdOffsets(camera, defaultOrientation)
def translateVisitCatalog(visitCat)