104 def run(self, handleDict, tract,
105 buildStarsRefObjLoader=None, returnCatalogs=True):
106 """Run the calibrations for a single tract with fgcm.
111 All handles are `lsst.daf.butler.DeferredDatasetHandle`
112 handle dictionary with the following keys. Note that all
113 keys need not be set based on config parameters.
116 Camera object (`lsst.afw.cameraGeom.Camera`)
117 ``"source_catalogs"``
118 `list` of handles for input source catalogs.
120 Schema for the source catalogs.
121 ``"fgcmLookUpTable"``
122 handle for the FGCM look-up table.
124 `list` of handles for the input calexps
125 ``"fgcmPhotoCalibs"``
126 `dict` of output photoCalib handles. Key is
127 (tract, visit, detector).
128 Present if doZeropointOutput is True.
129 ``"fgcmTransmissionAtmospheres"``
130 `dict` of output atmosphere transmission handles.
131 Key is (tract, visit).
132 Present if doAtmosphereOutput is True.
135 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
136 Reference object loader object for fgcmBuildStars.
137 returnCatalogs : `bool`, optional
138 Return photoCalibs as per-visit exposure catalogs.
142 outstruct : `lsst.pipe.base.Struct`
143 Output structure with keys:
145 offsets : `np.ndarray`
146 Final reference offsets, per band.
147 repeatability : `np.ndarray`
148 Raw fgcm repeatability for bright stars, per band.
149 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
150 Generator that returns (visit, transmissionCurve) tuples.
151 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
152 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
153 (returned if returnCatalogs is False).
154 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
155 Generator that returns (visit, exposureCatalog) tuples.
156 (returned if returnCatalogs is True).
158 self.log.info(
"Running on tract %d", (tract))
162 calibFluxApertureRadius =
None
163 if self.config.fgcmBuildStars.doSubtractLocalBackground:
165 field = self.config.fgcmBuildStars.instFluxField
166 calibFluxApertureRadius = computeApertureRadiusFromName(field)
168 raise RuntimeError(
"Could not determine aperture radius from %s. "
169 "Cannot use doSubtractLocalBackground." %
175 groupedHandles = self.fgcmBuildStars._groupHandles(handleDict[
'sourceTableHandleDict'],
176 handleDict[
'visitSummaryHandleDict'])
178 lutCat = handleDict[
"fgcmLookUpTable"].get()
179 camera = handleDict[
"camera"]
180 if len(camera) == lutCat[0][
"nCcd"]:
181 useScienceDetectors =
False
186 useScienceDetectors =
True
189 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(
192 useScienceDetectors=useScienceDetectors,
194 rad = calibFluxApertureRadius
195 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedHandles,
197 handleDict[
'sourceSchema'],
198 handleDict[
'camera'],
199 calibFluxApertureRadius=rad)
201 if self.fgcmBuildStars.config.doReferenceMatches:
202 lutHandle = handleDict[
'fgcmLookUpTable']
203 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
204 refObjLoader=buildStarsRefObjLoader,
205 refCatName=self.fgcmBuildStars.config.connections.refCat)
209 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
210 self.fgcmBuildStars.fgcmMatchStars(visitCat,
211 fgcmStarObservationCat,
215 lutCat = handleDict[
'fgcmLookUpTable'].get()
216 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
217 dict(self.config.fgcmFitCycle.physicalFilterMap))
221 fgcmExpInfo = translateVisitCatalog(visitCat)
223 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, handleDict[
'camera'],
224 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
225 True,
False, lutIndexVals[0][
'FILTERNAMES'],
229 self.config.fgcmFitCycle.defaultCameraOrientation)
233 noFitsDict = {
'lutIndex': lutIndexVals,
235 'expInfo': fgcmExpInfo,
236 'focalPlaneProjector': focalPlaneProjector}
238 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
239 noFitsDict=noFitsDict, noOutput=
True)
244 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
247 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
255 obsIndex = fgcmStarIndicesCat[
'obsIndex']
256 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
257 fgcmStarObservationCat[
'visit'][obsIndex])
259 refMag, refMagErr = extractReferenceMags(fgcmRefCat,
260 self.config.fgcmFitCycle.bands,
261 self.config.fgcmFitCycle.physicalFilterMap)
262 refId = fgcmRefCat[
'fgcm_id'][:]
264 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
265 fgcmStars.loadStars(fgcmPars,
266 fgcmStarObservationCat[
'visit'][obsIndex],
267 fgcmStarObservationCat[
'ccd'][obsIndex],
268 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
269 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
270 fgcmStarObservationCat[
'instMag'][obsIndex],
271 fgcmStarObservationCat[
'instMagErr'][obsIndex],
272 fgcmExpInfo[
'FILTERNAME'][visitIndex],
273 fgcmStarIdCat[
'fgcm_id'][:],
274 fgcmStarIdCat[
'ra'][:],
275 fgcmStarIdCat[
'dec'][:],
276 fgcmStarIdCat[
'obsArrIndex'][:],
277 fgcmStarIdCat[
'nObs'][:],
278 obsX=fgcmStarObservationCat[
'x'][obsIndex],
279 obsY=fgcmStarObservationCat[
'y'][obsIndex],
280 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
281 obsDeltaAper=fgcmStarObservationCat[
'deltaMagAper'][obsIndex],
282 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
292 del fgcmStarIndicesCat
295 fgcmFitCycle.setLUT(fgcmLut)
296 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
301 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
302 previousParInfo =
None
303 previousParams =
None
304 previousSuperStar =
None
306 while (
not converged
and cycleNumber < self.config.maxFitCycles):
308 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
312 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
319 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
320 fgcmStarObservationCat[
'instMagErr'][obsIndex])
321 fgcmFitCycle.initialCycle =
False
323 fgcmFitCycle.setPars(fgcmPars)
324 fgcmFitCycle.finishSetup()
329 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
330 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
332 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
333 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
334 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
336 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
337 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
340 if np.all((previousReservedRawRepeatability
341 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
342 < self.config.convergenceTolerance):
343 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
346 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
347 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
348 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
349 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
350 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
351 self.log.info(
"Setting exposure gray photometricity cuts to:")
352 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
353 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
355 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
356 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
362 self.log.warning(
"Maximum number of fit cycles exceeded (%d) without convergence.", cycleNumber)
365 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
366 fgcmFitCycle.fgcmConfig.resetParameters =
False
367 fgcmFitCycle.fgcmConfig.maxIter = 0
368 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
369 fgcmFitCycle.fgcmConfig.outputStandards =
True
370 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
371 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
372 fgcmFitCycle.initialCycle =
False
374 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
379 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
380 fgcmStarObservationCat[
'instMagErr'][obsIndex])
381 fgcmFitCycle.setPars(fgcmPars)
382 fgcmFitCycle.finishSetup()
384 self.log.info(
"Running final clean-up fit cycle...")
387 self.log.info(
"Raw repeatability after clean-up cycle is:")
388 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
389 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
391 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
392 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
396 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
397 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
399 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
400 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
402 atmSchema = makeAtmSchema()
403 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
405 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
406 stdSchema = makeStdSchema(len(goodBands))
407 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
409 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(handleDict,
412 zptCat, atmCat, stdCat,
413 self.config.fgcmBuildStars)
415 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
417 fgcmFitCycle.freeSharedMemory()