21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
35 from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
36 from .fgcmFitCycle
import FgcmFitCycleConfig
37 from .fgcmOutputProducts
import FgcmOutputProductsTask
38 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
39 from .utilities
import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
40 from .utilities
import makeZptSchema, makeZptCat
41 from .utilities
import makeAtmSchema, makeAtmCat
42 from .utilities
import makeStdSchema, makeStdCat
46 __all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
50 """Config for FgcmCalibrateTract"""
52 fgcmBuildStars = pexConfig.ConfigurableField(
53 target=FgcmBuildStarsTask,
54 doc=
"Task to load and match stars for fgcm",
56 fgcmFitCycle = pexConfig.ConfigField(
57 dtype=FgcmFitCycleConfig,
58 doc=
"Config to run a single fgcm fit cycle",
60 fgcmOutputProducts = pexConfig.ConfigurableField(
61 target=FgcmOutputProductsTask,
62 doc=
"Task to output fgcm products",
64 convergenceTolerance = pexConfig.Field(
65 doc=
"Tolerance on repeatability convergence (per band)",
69 maxFitCycles = pexConfig.Field(
70 doc=
"Maximum number of fit cycles",
74 doDebuggingPlots = pexConfig.Field(
75 doc=
"Make plots for debugging purposes?",
81 pexConfig.Config.setDefaults(self)
93 if not self.
fgcmFitCyclefgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
94 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
95 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
100 """Subclass of TaskRunner for FgcmCalibrateTractTask
102 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
103 the butler (for persistence and mapper data), and a list of dataRefs
104 extracted from the command line. This task runs on a constrained set
105 of dataRefs, typically a single tract.
106 This class transforms the process arguments generated by the ArgumentParser
107 into the arguments expected by FgcmCalibrateTractTask.run().
108 This runner does not use any parallelization.
114 Return a list with one element: a tuple with the butler and
117 return [(parsedCmd.butler, parsedCmd.id.refList)]
123 args: `tuple` with (butler, dataRefList)
127 exitStatus: `list` with `lsst.pipe.base.Struct`
128 exitStatus (0: success; 1: failure)
129 May also contain results if `self.doReturnResults` is `True`.
131 butler, dataRefList = args
133 task = self.TaskClass(config=self.config, log=self.log)
137 results = task.runDataRef(butler, dataRefList)
140 results = task.runDataRef(butler, dataRefList)
141 except Exception
as e:
143 task.log.fatal(
"Failed: %s" % e)
144 if not isinstance(e, pipeBase.TaskError):
145 traceback.print_exc(file=sys.stderr)
147 task.writeMetadata(butler)
149 if self.doReturnResults:
150 return [pipeBase.Struct(exitStatus=exitStatus,
153 return [pipeBase.Struct(exitStatus=exitStatus)]
157 Run the task, with no multiprocessing
161 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
166 if self.precall(parsedCmd):
168 resultList = self(targetList[0])
174 """Base class to calibrate a single tract using fgcmcal
178 Instantiate an `FgcmCalibrateTractTask`.
182 butler : `lsst.daf.persistence.Butler`, optional
185 self.makeSubtask(
"fgcmBuildStars", butler=butler)
186 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
189 def _getMetadataName(self):
195 Run full FGCM calibration on a single tract, including building star list,
196 fitting multiple cycles, and making outputs.
200 butler: `lsst.daf.persistence.Butler`
201 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
202 Data references for the input visits.
203 These may be either per-ccd "src" or per-visit"sourceTable_visit"
208 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
209 not True, or if fgcmLookUpTable is not available, or if
210 doSubtractLocalBackground is True and aperture radius cannot be
213 datasetType = dataRefs[0].butlerSubset.datasetType
214 self.log.
info(
"Running with %d %s dataRefs" % (len(dataRefs), datasetType))
216 if not butler.datasetExists(
'fgcmLookUpTable'):
217 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
219 if not self.config.fgcmBuildStars.doReferenceMatches:
220 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
221 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
222 if self.config.fgcmBuildStars.checkAllCcds:
223 raise RuntimeError(
"Cannot run FgcmCalibrateTract with "
224 "fgcmBuildStars.checkAllCcds set to True")
226 tract = int(dataRefs[0].dataId[
'tract'])
227 camera = butler.get(
'camera')
230 dataRefDict[
'camera'] = camera
231 dataRefDict[
'source_catalogs'] = dataRefs
232 dataRefDict[
'sourceSchema'] = butler.dataRef(
'src_schema')
233 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
235 struct = self.
runrun(dataRefDict, tract, butler=butler, returnCatalogs=
False)
237 visitDataRefName = self.config.fgcmBuildStars.visitDataRefName
238 ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName
240 if struct.photoCalibs
is not None:
241 self.log.
info(
"Outputting photoCalib files.")
243 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
244 butler.put(photoCalib,
'fgcm_tract_photoCalib',
245 dataId={visitDataRefName: visit,
246 ccdDataRefName: detector,
247 'filter': physicalFilter,
250 self.log.
info(
"Done outputting photoCalib files.")
252 if struct.atmospheres
is not None:
253 self.log.
info(
"Outputting atmosphere files.")
254 for visit, atm
in struct.atmospheres:
255 butler.put(atm,
"transmission_atmosphere_fgcm_tract",
256 dataId={visitDataRefName: visit,
258 self.log.
info(
"Done outputting atmosphere transmissions.")
260 return pipeBase.Struct(repeatability=struct.repeatability)
262 def run(self, dataRefDict, tract,
263 buildStarsRefObjLoader=None, returnCatalogs=True, butler=None):
264 """Run the calibrations for a single tract with fgcm.
269 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
270 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
271 dataRef dictionary with the following keys. Note that all
272 keys need not be set based on config parameters.
275 Camera object (`lsst.afw.cameraGeom.Camera`)
276 ``"source_catalogs"``
277 `list` of dataRefs for input source catalogs.
279 Schema for the source catalogs.
280 ``"fgcmLookUpTable"``
281 dataRef for the FGCM look-up table.
283 `list` of dataRefs for the input calexps (Gen3 only)
284 ``"fgcmPhotoCalibs"``
285 `dict` of output photoCalib dataRefs. Key is
286 (tract, visit, detector). (Gen3 only)
287 Present if doZeropointOutput is True.
288 ``"fgcmTransmissionAtmospheres"``
289 `dict` of output atmosphere transmission dataRefs.
290 Key is (tract, visit). (Gen3 only)
291 Present if doAtmosphereOutput is True.
294 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
295 Reference object loader object for fgcmBuildStars.
296 returnCatalogs : `bool`, optional
297 Return photoCalibs as per-visit exposure catalogs.
298 butler : `lsst.daf.persistence.Butler`, optional
299 Gen2 butler used for reference star outputs
303 outstruct : `lsst.pipe.base.Struct`
304 Output structure with keys:
306 offsets : `np.ndarray`
307 Final reference offsets, per band.
308 repeatability : `np.ndarray`
309 Raw fgcm repeatability for bright stars, per band.
310 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
311 Generator that returns (visit, transmissionCurve) tuples.
312 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
313 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
314 (returned if returnCatalogs is False).
315 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
316 Generator that returns (visit, exposureCatalog) tuples.
317 (returned if returnCatalogs is True).
319 self.log.
info(
"Running on tract %d", (tract))
323 calibFluxApertureRadius =
None
324 if self.config.fgcmBuildStars.doSubtractLocalBackground:
326 field = self.config.fgcmBuildStars.instFluxField
330 raise RuntimeError(
"Could not determine aperture radius from %s. "
331 "Cannot use doSubtractLocalBackground." %
339 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefsGen2(butler, dataRefDict[
'camera'],
340 dataRefDict[
'source_catalogs'])
343 groupedDataRefs = self.fgcmBuildStars._groupDataRefs(dataRefDict[
'sourceTableDataRefDict'],
344 dataRefDict[
'visitSummaryDataRefDict'])
345 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict[
'camera'], groupedDataRefs)
346 rad = calibFluxApertureRadius
347 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
349 dataRefDict[
'sourceSchema'],
350 dataRefDict[
'camera'],
351 calibFluxApertureRadius=rad)
353 if self.fgcmBuildStars.config.doReferenceMatches:
354 lutDataRef = dataRefDict[
'fgcmLookUpTable']
355 if buildStarsRefObjLoader
is not None:
356 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
357 refObjLoader=buildStarsRefObjLoader)
359 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog", butler=butler)
363 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
364 self.fgcmBuildStars.fgcmMatchStars(visitCat,
365 fgcmStarObservationCat,
366 lutDataRef=lutDataRef)
369 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
371 dict(self.config.fgcmFitCycle.physicalFilterMap))
377 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict[
'camera'],
378 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
379 True,
False, lutIndexVals[0][
'FILTERNAMES'],
383 configDict[
'doPlots'] =
False
391 noFitsDict = {
'lutIndex': lutIndexVals,
393 'expInfo': fgcmExpInfo,
394 'ccdOffsets': ccdOffsets}
396 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
397 noFitsDict=noFitsDict, noOutput=
True)
402 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
405 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
413 obsIndex = fgcmStarIndicesCat[
'obsIndex']
414 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
415 fgcmStarObservationCat[
'visit'][obsIndex])
418 self.config.fgcmFitCycle.bands,
419 self.config.fgcmFitCycle.physicalFilterMap)
420 refId = fgcmRefCat[
'fgcm_id'][:]
422 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
423 fgcmStars.loadStars(fgcmPars,
424 fgcmStarObservationCat[
'visit'][obsIndex],
425 fgcmStarObservationCat[
'ccd'][obsIndex],
426 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
427 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
428 fgcmStarObservationCat[
'instMag'][obsIndex],
429 fgcmStarObservationCat[
'instMagErr'][obsIndex],
430 fgcmExpInfo[
'FILTERNAME'][visitIndex],
431 fgcmStarIdCat[
'fgcm_id'][:],
432 fgcmStarIdCat[
'ra'][:],
433 fgcmStarIdCat[
'dec'][:],
434 fgcmStarIdCat[
'obsArrIndex'][:],
435 fgcmStarIdCat[
'nObs'][:],
436 obsX=fgcmStarObservationCat[
'x'][obsIndex],
437 obsY=fgcmStarObservationCat[
'y'][obsIndex],
438 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
439 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
449 del fgcmStarIndicesCat
452 fgcmFitCycle.setLUT(fgcmLut)
453 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
458 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
459 previousParInfo =
None
460 previousParams =
None
461 previousSuperStar =
None
463 while (
not converged
and cycleNumber < self.config.maxFitCycles):
465 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
469 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
476 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
477 fgcmStarObservationCat[
'instMagErr'][obsIndex])
478 fgcmFitCycle.initialCycle =
False
480 fgcmFitCycle.setPars(fgcmPars)
481 fgcmFitCycle.finishSetup()
486 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
487 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
489 self.log.
info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
490 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
491 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
493 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
494 self.log.
info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
497 if np.alltrue((previousReservedRawRepeatability
498 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
499 < self.config.convergenceTolerance):
500 self.log.
info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
503 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
504 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
505 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
506 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
507 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
508 self.log.
info(
"Setting exposure gray photometricity cuts to:")
509 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
510 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
512 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
513 self.log.
info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
519 self.log.
warn(
"Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
522 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
523 fgcmFitCycle.fgcmConfig.resetParameters =
False
524 fgcmFitCycle.fgcmConfig.maxIter = 0
525 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
526 fgcmFitCycle.fgcmConfig.outputStandards =
True
527 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
528 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
529 fgcmFitCycle.initialCycle =
False
531 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
536 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
537 fgcmStarObservationCat[
'instMagErr'][obsIndex])
538 fgcmFitCycle.setPars(fgcmPars)
539 fgcmFitCycle.finishSetup()
541 self.log.
info(
"Running final clean-up fit cycle...")
544 self.log.
info(
"Raw repeatability after clean-up cycle is:")
545 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
546 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
548 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
549 self.log.
info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
553 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
554 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
557 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
560 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
562 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
564 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
566 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
569 zptCat, atmCat, stdCat,
570 self.config.fgcmBuildStars,
571 returnCatalogs=returnCatalogs,
574 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
def runDataRef(self, butler, dataRefs)
def __init__(self, butler=None, **kwargs)
def getTargetList(parsedCmd)
def extractReferenceMags(refStars, bands, filterMap)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def makeStdSchema(nBands)
def makeAtmCat(atmSchema, atmStruct)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def computeCcdOffsets(camera, defaultOrientation)
def translateVisitCatalog(visitCat)