266 buildStarsRefObjLoader=
None, returnCatalogs=
True, butler=
None):
267 """Run the calibrations for a single tract with fgcm.
272 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
273 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
274 dataRef dictionary with the following keys. Note that all
275 keys need not be set based on config parameters.
278 Camera object (`lsst.afw.cameraGeom.Camera`)
279 ``"source_catalogs"``
280 `list` of dataRefs for input source catalogs.
282 Schema for the source catalogs.
283 ``"fgcmLookUpTable"``
284 dataRef for the FGCM look-up table.
286 `list` of dataRefs for the input calexps (Gen3 only)
287 ``"fgcmPhotoCalibs"``
288 `dict` of output photoCalib dataRefs. Key is
289 (tract, visit, detector). (Gen3 only)
290 Present if doZeropointOutput is True.
291 ``"fgcmTransmissionAtmospheres"``
292 `dict` of output atmosphere transmission dataRefs.
293 Key is (tract, visit). (Gen3 only)
294 Present if doAtmosphereOutput is True.
297 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
298 Reference object loader object for fgcmBuildStars.
299 returnCatalogs : `bool`, optional
300 Return photoCalibs as per-visit exposure catalogs.
301 butler : `lsst.daf.persistence.Butler`, optional
302 Gen2 butler used for reference star outputs
306 outstruct : `lsst.pipe.base.Struct`
307 Output structure with keys:
309 offsets : `np.ndarray`
310 Final reference offsets, per band.
311 repeatability : `np.ndarray`
312 Raw fgcm repeatability for bright stars, per band.
313 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
314 Generator that returns (visit, transmissionCurve) tuples.
315 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
316 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
317 (returned if returnCatalogs is False).
318 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
319 Generator that returns (visit, exposureCatalog) tuples.
320 (returned if returnCatalogs is True).
322 self.log.
info(
"Running on tract %d", (tract))
326 calibFluxApertureRadius =
None
327 if self.config.fgcmBuildStars.doSubtractLocalBackground:
329 field = self.config.fgcmBuildStars.instFluxField
333 raise RuntimeError(
"Could not determine aperture radius from %s. "
334 "Cannot use doSubtractLocalBackground." %
342 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefsGen2(butler, dataRefDict[
'camera'],
343 dataRefDict[
'source_catalogs'])
346 groupedDataRefs = self.fgcmBuildStars._groupDataRefs(dataRefDict[
'sourceTableDataRefDict'],
347 dataRefDict[
'visitSummaryDataRefDict'])
348 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict[
'camera'], groupedDataRefs)
349 rad = calibFluxApertureRadius
350 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
352 dataRefDict[
'sourceSchema'],
353 dataRefDict[
'camera'],
354 calibFluxApertureRadius=rad)
356 if self.fgcmBuildStars.config.doReferenceMatches:
357 lutDataRef = dataRefDict[
'fgcmLookUpTable']
358 if buildStarsRefObjLoader
is not None:
359 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
360 refObjLoader=buildStarsRefObjLoader)
362 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog", butler=butler)
366 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
367 self.fgcmBuildStars.fgcmMatchStars(visitCat,
368 fgcmStarObservationCat,
369 lutDataRef=lutDataRef)
372 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
374 dict(self.config.fgcmFitCycle.physicalFilterMap))
380 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict[
'camera'],
381 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
382 True,
False, lutIndexVals[0][
'FILTERNAMES'],
385 focalPlaneProjector = FocalPlaneProjector(dataRefDict[
'camera'],
386 self.config.fgcmFitCycle.defaultCameraOrientation)
390 noFitsDict = {
'lutIndex': lutIndexVals,
392 'expInfo': fgcmExpInfo,
393 'focalPlaneProjector': focalPlaneProjector}
395 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
396 noFitsDict=noFitsDict, noOutput=
True)
401 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
404 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
412 obsIndex = fgcmStarIndicesCat[
'obsIndex']
413 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
414 fgcmStarObservationCat[
'visit'][obsIndex])
417 self.config.fgcmFitCycle.bands,
418 self.config.fgcmFitCycle.physicalFilterMap)
419 refId = fgcmRefCat[
'fgcm_id'][:]
421 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
422 fgcmStars.loadStars(fgcmPars,
423 fgcmStarObservationCat[
'visit'][obsIndex],
424 fgcmStarObservationCat[
'ccd'][obsIndex],
425 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
426 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
427 fgcmStarObservationCat[
'instMag'][obsIndex],
428 fgcmStarObservationCat[
'instMagErr'][obsIndex],
429 fgcmExpInfo[
'FILTERNAME'][visitIndex],
430 fgcmStarIdCat[
'fgcm_id'][:],
431 fgcmStarIdCat[
'ra'][:],
432 fgcmStarIdCat[
'dec'][:],
433 fgcmStarIdCat[
'obsArrIndex'][:],
434 fgcmStarIdCat[
'nObs'][:],
435 obsX=fgcmStarObservationCat[
'x'][obsIndex],
436 obsY=fgcmStarObservationCat[
'y'][obsIndex],
437 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
438 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
448 del fgcmStarIndicesCat
451 fgcmFitCycle.setLUT(fgcmLut)
452 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
457 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
458 previousParInfo =
None
459 previousParams =
None
460 previousSuperStar =
None
462 while (
not converged
and cycleNumber < self.config.maxFitCycles):
464 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
468 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
475 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
476 fgcmStarObservationCat[
'instMagErr'][obsIndex])
477 fgcmFitCycle.initialCycle =
False
479 fgcmFitCycle.setPars(fgcmPars)
480 fgcmFitCycle.finishSetup()
485 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
486 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
488 self.log.
info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
489 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
490 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
492 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
493 self.log.
info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
496 if np.alltrue((previousReservedRawRepeatability
497 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
498 < self.config.convergenceTolerance):
499 self.log.
info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
502 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
503 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
504 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
505 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
506 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
507 self.log.
info(
"Setting exposure gray photometricity cuts to:")
508 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
509 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
511 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
512 self.log.
info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
518 self.log.
warning(
"Maximum number of fit cycles exceeded (%d) without convergence.", cycleNumber)
521 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
522 fgcmFitCycle.fgcmConfig.resetParameters =
False
523 fgcmFitCycle.fgcmConfig.maxIter = 0
524 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
525 fgcmFitCycle.fgcmConfig.outputStandards =
True
526 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
527 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
528 fgcmFitCycle.initialCycle =
False
530 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
535 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
536 fgcmStarObservationCat[
'instMagErr'][obsIndex])
537 fgcmFitCycle.setPars(fgcmPars)
538 fgcmFitCycle.finishSetup()
540 self.log.
info(
"Running final clean-up fit cycle...")
543 self.log.
info(
"Raw repeatability after clean-up cycle is:")
544 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
545 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
547 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
548 self.log.
info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
552 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
553 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
556 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
559 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
561 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
563 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
565 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
568 zptCat, atmCat, stdCat,
569 self.config.fgcmBuildStars,
570 returnCatalogs=returnCatalogs,
573 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
575 fgcmFitCycle.freeSharedMemory()
def extractReferenceMags(refStars, bands, filterMap)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def makeStdSchema(nBands)
def makeAtmCat(atmSchema, atmStruct)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def translateVisitCatalog(visitCat)