LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
Public Member Functions | Public Attributes | Static Public Attributes | List of all members
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask Class Reference
Inheritance diagram for lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask:
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask

Public Member Functions

def __init__ (self, initInputs=None, **kwargs)
 
def runQuantum (self, butlerQC, inputRefs, outputRefs)
 
def fgcmMakeAllStarObservations (self, groupedDataRefs, visitCat, sourceSchema, camera, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
 
def runDataRef (self, butler, dataRefs)
 
def fgcmMakeVisitCatalog (self, camera, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None, inVisitCat=None)
 
def fgcmMatchStars (self, visitCat, obsCat, lutDataRef=None)
 

Public Attributes

 sourceSchema
 

Static Public Attributes

 ConfigClass = FgcmBuildStarsTableConfig
 
 RunnerClass = FgcmBuildStarsRunner
 
bool canMultiprocess = False
 

Detailed Description

Build stars for the FGCM global calibration, using sourceTable_visit catalogs.

Definition at line 215 of file fgcmBuildStarsTable.py.

Constructor & Destructor Documentation

◆ __init__()

def lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.__init__ (   self,
  initInputs = None,
**  kwargs 
)

Definition at line 225 of file fgcmBuildStarsTable.py.

225  def __init__(self, initInputs=None, **kwargs):
226  super().__init__(initInputs=initInputs, **kwargs)
227  if initInputs is not None:
228  self.sourceSchema = initInputs["sourceSchema"].schema
229 

Member Function Documentation

◆ fgcmMakeAllStarObservations()

def lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.fgcmMakeAllStarObservations (   self,
  groupedDataRefs,
  visitCat,
  sourceSchema,
  camera,
  calibFluxApertureRadius = None,
  visitCatDataRef = None,
  starObsDataRef = None,
  inStarObsCat = None 
)
Compile all good star observations from visits in visitCat.  Checkpoint files
will be stored if both visitCatDataRef and starObsDataRef are not None.

Parameters
----------
groupedDataRefs : `dict` of `list`s
    Lists of `~lsst.daf.persistence.ButlerDataRef` or
    `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
visitCat : `~afw.table.BaseCatalog`
    Catalog with visit data for FGCM
sourceSchema : `~lsst.afw.table.Schema`
    Schema for the input src catalogs.
camera : `~lsst.afw.cameraGeom.Camera`
calibFluxApertureRadius : `float`, optional
    Aperture radius for calibration flux.
visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
    Dataref to write visitCat for checkpoints
starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
    Dataref to write the star observation catalog for checkpoints.
inStarObsCat : `~afw.table.BaseCatalog`
    Input observation catalog.  If this is incomplete, observations
    will be appended from when it was cut off.

Returns
-------
fgcmStarObservations : `afw.table.BaseCatalog`
    Full catalog of good observations.

Raises
------
RuntimeError: Raised if doSubtractLocalBackground is True and
   calibFluxApertureRadius is not set.

Reimplemented from lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.

Definition at line 394 of file fgcmBuildStarsTable.py.

400  inStarObsCat=None):
401  startTime = time.time()
402 
403  # If both dataRefs are None, then we assume the caller does not
404  # want to store checkpoint files. If both are set, we will
405  # do checkpoint files. And if only one is set, this is potentially
406  # unintentional and we will warn.
407  if (visitCatDataRef is not None and starObsDataRef is None
408  or visitCatDataRef is None and starObsDataRef is not None):
409  self.log.warning("Only one of visitCatDataRef and starObsDataRef are set, so "
410  "no checkpoint files will be persisted.")
411 
412  if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
413  raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
414 
415  # To get the correct output schema, we use similar code as fgcmBuildStarsTask
416  # We are not actually using this mapper, except to grab the outputSchema
417  sourceMapper = self._makeSourceMapper(sourceSchema)
418  outputSchema = sourceMapper.getOutputSchema()
419 
420  # Construct mapping from ccd number to index
421  ccdMapping = {}
422  for ccdIndex, detector in enumerate(camera):
423  ccdMapping[detector.getId()] = ccdIndex
424 
425  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
426 
427  if inStarObsCat is not None:
428  fullCatalog = inStarObsCat
429  comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
430  comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
431  if not comp1 or not comp2:
432  raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
433  else:
434  fullCatalog = afwTable.BaseCatalog(outputSchema)
435 
436  visitKey = outputSchema['visit'].asKey()
437  ccdKey = outputSchema['ccd'].asKey()
438  instMagKey = outputSchema['instMag'].asKey()
439  instMagErrKey = outputSchema['instMagErr'].asKey()
440 
441  # Prepare local background if desired
442  if self.config.doSubtractLocalBackground:
443  localBackgroundArea = np.pi*calibFluxApertureRadius**2.
444 
445  columns = None
446 
447  k = 2.5/np.log(10.)
448 
449  for counter, visit in enumerate(visitCat):
450  # Check if these sources have already been read and stored in the checkpoint file
451  if visit['sources_read']:
452  continue
453 
454  expTime = visit['exptime']
455 
456  dataRef = groupedDataRefs[visit['visit']][-1]
457 
458  if isinstance(dataRef, dafPersist.ButlerDataRef):
459  srcTable = dataRef.get()
460  if columns is None:
461  columns, detColumn = self._get_sourceTable_visit_columns(srcTable.columns)
462  df = srcTable.toDataFrame(columns)
463  else:
464  if columns is None:
465  inColumns = dataRef.get(component='columns')
466  columns, detColumn = self._get_sourceTable_visit_columns(inColumns)
467  df = dataRef.get(parameters={'columns': columns})
468 
469  goodSrc = self.sourceSelector.selectSources(df)
470 
471  # Need to add a selection based on the local background correction
472  # if necessary
473  if self.config.doSubtractLocalBackground:
474  localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
475  use, = np.where((goodSrc.selected)
476  & ((df[self.config.instFluxField].values - localBackground) > 0.0))
477  else:
478  use, = np.where(goodSrc.selected)
479 
480  tempCat = afwTable.BaseCatalog(fullCatalog.schema)
481  tempCat.resize(use.size)
482 
483  tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
484  tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
485  tempCat['x'][:] = df['x'].values[use]
486  tempCat['y'][:] = df['y'].values[use]
487  # The "visit" name in the parquet table is hard-coded.
488  tempCat[visitKey][:] = df['visit'].values[use]
489  tempCat[ccdKey][:] = df[detColumn].values[use]
490  tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
491 
492  if self.config.doSubtractLocalBackground:
493  # At the moment we only adjust the flux and not the flux
494  # error by the background because the error on
495  # base_LocalBackground_instFlux is the rms error in the
496  # background annulus, not the error on the mean in the
497  # background estimate (which is much smaller, by sqrt(n)
498  # pixels used to estimate the background, which we do not
499  # have access to in this task). In the default settings,
500  # the annulus is sufficiently large such that these
501  # additional errors are are negligibly small (much less
502  # than a mmag in quadrature).
503 
504  # This is the difference between the mag with local background correction
505  # and the mag without local background correction.
506  tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
507  - localBackground[use]) -
508  -2.5*np.log10(df[self.config.instFluxField].values[use]))
509  else:
510  tempCat['deltaMagBkg'][:] = 0.0
511 
512  # Need to loop over ccds here
513  for detector in camera:
514  ccdId = detector.getId()
515  # used index for all observations with a given ccd
516  use2 = (tempCat[ccdKey] == ccdId)
517  tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
518  tempCat['y'][use2])
519  scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
520  * visit['scaling'][ccdMapping[ccdId]])
521  tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
522 
523  # Compute instMagErr from instFluxErr/instFlux, any scaling
524  # will cancel out.
525  tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
526  / df[self.config.instFluxField].values[use])
527 
528  # Apply the jacobian if configured
529  if self.config.doApplyWcsJacobian:
530  tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
531 
532  fullCatalog.extend(tempCat)
533 
534  # Now do the aperture information
535  with np.warnings.catch_warnings():
536  # Ignore warnings, we will filter infinites and nans below
537  np.warnings.simplefilter("ignore")
538 
539  instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
540  instMagErrIn = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
541  / df[self.config.apertureInnerInstFluxField].values[use])
542  instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
543  instMagErrOut = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
544  / df[self.config.apertureOuterInstFluxField].values[use])
545 
546  ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
547  & np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
548 
549  visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
550  visit['sources_read'] = True
551 
552  self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
553  use.size, visit['visit'], visit['deltaAper'])
554 
555  if ((counter % self.config.nVisitsPerCheckpoint) == 0
556  and starObsDataRef is not None and visitCatDataRef is not None):
557  # We need to persist both the stars and the visit catalog which gets
558  # additional metadata from each visit.
559  starObsDataRef.put(fullCatalog)
560  visitCatDataRef.put(visitCat)
561 
562  self.log.info("Found all good star observations in %.2f s" %
563  (time.time() - startTime))
564 
565  return fullCatalog
566 
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:499

◆ fgcmMakeVisitCatalog()

def lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.fgcmMakeVisitCatalog (   self,
  camera,
  groupedDataRefs,
  bkgDataRefDict = None,
  visitCatDataRef = None,
  inVisitCat = None 
)
inherited
Make a visit catalog with all the keys from each visit

Parameters
----------
camera: `lsst.afw.cameraGeom.Camera`
   Camera from the butler
groupedDataRefs: `dict`
   Dictionary with visit keys, and `list`s of
   `lsst.daf.persistence.ButlerDataRef`
bkgDataRefDict: `dict`, optional
   Dictionary of gen3 dataRefHandles for background info.
visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
   Dataref to write visitCat for checkpoints
inVisitCat: `afw.table.BaseCatalog`, optional
   Input (possibly incomplete) visit catalog

Returns
-------
visitCat: `afw.table.BaseCatalog`

Definition at line 460 of file fgcmBuildStarsBase.py.

461  visitCatDataRef=None, inVisitCat=None):
462  """
463  Make a visit catalog with all the keys from each visit
464 
465  Parameters
466  ----------
467  camera: `lsst.afw.cameraGeom.Camera`
468  Camera from the butler
469  groupedDataRefs: `dict`
470  Dictionary with visit keys, and `list`s of
471  `lsst.daf.persistence.ButlerDataRef`
472  bkgDataRefDict: `dict`, optional
473  Dictionary of gen3 dataRefHandles for background info.
474  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
475  Dataref to write visitCat for checkpoints
476  inVisitCat: `afw.table.BaseCatalog`, optional
477  Input (possibly incomplete) visit catalog
478 
479  Returns
480  -------
481  visitCat: `afw.table.BaseCatalog`
482  """
483 
484  self.log.info("Assembling visitCatalog from %d %ss" %
485  (len(groupedDataRefs), self.config.visitDataRefName))
486 
487  nCcd = len(camera)
488 
489  if inVisitCat is None:
490  schema = self._makeFgcmVisitSchema(nCcd)
491 
492  visitCat = afwTable.BaseCatalog(schema)
493  visitCat.reserve(len(groupedDataRefs))
494  visitCat.resize(len(groupedDataRefs))
495 
496  visitCat['visit'] = list(groupedDataRefs.keys())
497  visitCat['used'] = 0
498  visitCat['sources_read'] = False
499  else:
500  visitCat = inVisitCat
501 
502  # No matter what, fill the catalog. This will check if it was
503  # already read.
504  self._fillVisitCatalog(visitCat, groupedDataRefs,
505  bkgDataRefDict=bkgDataRefDict,
506  visitCatDataRef=visitCatDataRef)
507 
508  return visitCat
509 
daf::base::PropertyList * list
Definition: fits.cc:913

◆ fgcmMatchStars()

def lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.fgcmMatchStars (   self,
  visitCat,
  obsCat,
  lutDataRef = None 
)
inherited
Use FGCM code to match observations into unique stars.

Parameters
----------
visitCat: `afw.table.BaseCatalog`
   Catalog with visit data for fgcm
obsCat: `afw.table.BaseCatalog`
   Full catalog of star observations for fgcm
lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
            `lsst.daf.butler.DeferredDatasetHandle`, optional
   Data reference to fgcm look-up table (used if matching reference stars).

Returns
-------
fgcmStarIdCat: `afw.table.BaseCatalog`
   Catalog of unique star identifiers and index keys
fgcmStarIndicesCat: `afwTable.BaseCatalog`
   Catalog of unique star indices
fgcmRefCat: `afw.table.BaseCatalog`
   Catalog of matched reference stars.
   Will be None if `config.doReferenceMatches` is False.

Definition at line 687 of file fgcmBuildStarsBase.py.

687  def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
688  """
689  Use FGCM code to match observations into unique stars.
690 
691  Parameters
692  ----------
693  visitCat: `afw.table.BaseCatalog`
694  Catalog with visit data for fgcm
695  obsCat: `afw.table.BaseCatalog`
696  Full catalog of star observations for fgcm
697  lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
698  `lsst.daf.butler.DeferredDatasetHandle`, optional
699  Data reference to fgcm look-up table (used if matching reference stars).
700 
701  Returns
702  -------
703  fgcmStarIdCat: `afw.table.BaseCatalog`
704  Catalog of unique star identifiers and index keys
705  fgcmStarIndicesCat: `afwTable.BaseCatalog`
706  Catalog of unique star indices
707  fgcmRefCat: `afw.table.BaseCatalog`
708  Catalog of matched reference stars.
709  Will be None if `config.doReferenceMatches` is False.
710  """
711  # get filter names into a numpy array...
712  # This is the type that is expected by the fgcm code
713  visitFilterNames = np.zeros(len(visitCat), dtype='a30')
714  for i in range(len(visitCat)):
715  visitFilterNames[i] = visitCat[i]['physicalFilter']
716 
717  # match to put filterNames with observations
718  visitIndex = np.searchsorted(visitCat['visit'],
719  obsCat['visit'])
720 
721  obsFilterNames = visitFilterNames[visitIndex]
722 
723  if self.config.doReferenceMatches:
724  # Get the reference filter names, using the LUT
725  lutCat = lutDataRef.get()
726 
727  stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
728  zip(lutCat[0]['physicalFilters'].split(','),
729  lutCat[0]['stdPhysicalFilters'].split(','))}
730  stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
731  zip(lutCat[0]['stdPhysicalFilters'].split(','),
732  lutCat[0]['lambdaStdFilter'])}
733 
734  del lutCat
735 
736  referenceFilterNames = self._getReferenceFilterNames(visitCat,
737  stdFilterDict,
738  stdLambdaDict)
739  self.log.info("Using the following reference filters: %s" %
740  (', '.join(referenceFilterNames)))
741 
742  else:
743  # This should be an empty list
744  referenceFilterNames = []
745 
746  # make the fgcm starConfig dict
747  starConfig = {'logger': self.log,
748  'filterToBand': self.config.physicalFilterMap,
749  'requiredBands': self.config.requiredBands,
750  'minPerBand': self.config.minPerBand,
751  'matchRadius': self.config.matchRadius,
752  'isolationRadius': self.config.isolationRadius,
753  'matchNSide': self.config.matchNside,
754  'coarseNSide': self.config.coarseNside,
755  'densNSide': self.config.densityCutNside,
756  'densMaxPerPixel': self.config.densityCutMaxPerPixel,
757  'randomSeed': self.config.randomSeed,
758  'primaryBands': self.config.primaryBands,
759  'referenceFilterNames': referenceFilterNames}
760 
761  # initialize the FgcmMakeStars object
762  fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
763 
764  # make the primary stars
765  # note that the ra/dec native Angle format is radians
766  # We determine the conversion from the native units (typically
767  # radians) to degrees for the first observation. This allows us
768  # to treate ra/dec as numpy arrays rather than Angles, which would
769  # be approximately 600x slower.
770  conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
771  fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
772  obsCat['dec'] * conv,
773  filterNameArray=obsFilterNames,
774  bandSelected=False)
775 
776  # and match all the stars
777  fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
778  obsCat['dec'] * conv,
779  obsFilterNames)
780 
781  if self.config.doReferenceMatches:
782  fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
783 
784  # now persist
785 
786  objSchema = self._makeFgcmObjSchema()
787 
788  # make catalog and records
789  fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
790  fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
791  for i in range(fgcmMakeStars.objIndexCat.size):
792  fgcmStarIdCat.addNew()
793 
794  # fill the catalog
795  fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
796  fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
797  fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
798  fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
799  fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
800 
801  obsSchema = self._makeFgcmObsSchema()
802 
803  fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
804  fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
805  for i in range(fgcmMakeStars.obsIndexCat.size):
806  fgcmStarIndicesCat.addNew()
807 
808  fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
809 
810  if self.config.doReferenceMatches:
811  refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
812 
813  fgcmRefCat = afwTable.BaseCatalog(refSchema)
814  fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
815 
816  for i in range(fgcmMakeStars.referenceCat.size):
817  fgcmRefCat.addNew()
818 
819  fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
820  fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
821  fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
822 
823  md = PropertyList()
824  md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
825  md.set("FILTERNAMES", referenceFilterNames)
826  fgcmRefCat.setMetadata(md)
827 
828  else:
829  fgcmRefCat = None
830 
831  return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
832 

◆ runDataRef()

def lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.runDataRef (   self,
  butler,
  dataRefs 
)
inherited
Cross-match and make star list for FGCM Input

Parameters
----------
butler:  `lsst.daf.persistence.Butler`
dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
   Source data references for the input visits.

Raises
------
RuntimeErrror: Raised if `config.doReferenceMatches` is set and
   an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
   fails if the calibFlux is not a CircularAperture flux.

Definition at line 296 of file fgcmBuildStarsBase.py.

296  def runDataRef(self, butler, dataRefs):
297  """
298  Cross-match and make star list for FGCM Input
299 
300  Parameters
301  ----------
302  butler: `lsst.daf.persistence.Butler`
303  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
304  Source data references for the input visits.
305 
306  Raises
307  ------
308  RuntimeErrror: Raised if `config.doReferenceMatches` is set and
309  an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
310  fails if the calibFlux is not a CircularAperture flux.
311  """
312  datasetType = dataRefs[0].butlerSubset.datasetType
313  self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
314 
315  if self.config.doReferenceMatches:
316  self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
317  # Ensure that we have a LUT
318  if not butler.datasetExists('fgcmLookUpTable'):
319  raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
320  # Compute aperture radius if necessary. This is useful to do now before
321  # any heavy lifting has happened (fail early).
322  calibFluxApertureRadius = None
323  if self.config.doSubtractLocalBackground:
324  try:
325  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
326  self.config.instFluxField)
327  except RuntimeError as e:
328  raise RuntimeError("Could not determine aperture radius from %s. "
329  "Cannot use doSubtractLocalBackground." %
330  (self.config.instFluxField)) from e
331 
332  camera = butler.get('camera')
333  groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs)
334 
335  # Make the visit catalog if necessary
336  # First check if the visit catalog is in the _current_ path
337  # We cannot use Gen2 datasetExists() because that checks all parent
338  # directories as well, which would make recovering from faults
339  # and fgcmcal reruns impossible.
340  visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
341  filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
342  if os.path.exists(filename):
343  # This file exists and we should continue processing
344  inVisitCat = visitCatDataRef.get()
345  if len(inVisitCat) != len(groupedDataRefs):
346  raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
347  "number of visits. Cannot continue.")
348  else:
349  inVisitCat = None
350 
351  visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
352  visitCatDataRef=visitCatDataRef,
353  inVisitCat=inVisitCat)
354 
355  # Persist the visitCat as a checkpoint file.
356  visitCatDataRef.put(visitCat)
357 
358  starObsDataRef = butler.dataRef('fgcmStarObservations')
359  filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
360  if os.path.exists(filename):
361  inStarObsCat = starObsDataRef.get()
362  else:
363  inStarObsCat = None
364 
365  rad = calibFluxApertureRadius
366  sourceSchemaDataRef = butler.dataRef('src_schema')
367  sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema
368  fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
369  visitCat,
370  sourceSchema,
371  camera,
372  calibFluxApertureRadius=rad,
373  starObsDataRef=starObsDataRef,
374  visitCatDataRef=visitCatDataRef,
375  inStarObsCat=inStarObsCat)
376  visitCatDataRef.put(visitCat)
377  starObsDataRef.put(fgcmStarObservationCat)
378 
379  # Always do the matching.
380  if self.config.doReferenceMatches:
381  lutDataRef = butler.dataRef('fgcmLookUpTable')
382  else:
383  lutDataRef = None
384  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
385  fgcmStarObservationCat,
386  lutDataRef=lutDataRef)
387 
388  # Persist catalogs via the butler
389  butler.put(fgcmStarIdCat, 'fgcmStarIds')
390  butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
391  if fgcmRefCat is not None:
392  butler.put(fgcmRefCat, 'fgcmReferenceStars')
393 
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:812

◆ runQuantum()

def lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.runQuantum (   self,
  butlerQC,
  inputRefs,
  outputRefs 
)

Definition at line 230 of file fgcmBuildStarsTable.py.

230  def runQuantum(self, butlerQC, inputRefs, outputRefs):
231  inputRefDict = butlerQC.get(inputRefs)
232 
233  sourceTableRefs = inputRefDict['sourceTable_visit']
234 
235  self.log.info("Running with %d sourceTable_visit dataRefs",
236  len(sourceTableRefs))
237 
238  sourceTableDataRefDict = {sourceTableRef.dataId['visit']: sourceTableRef for
239  sourceTableRef in sourceTableRefs}
240 
241  if self.config.doReferenceMatches:
242  # Get the LUT dataRef
243  lutDataRef = inputRefDict['fgcmLookUpTable']
244 
245  # Prepare the refCat loader
246  refConfig = self.config.fgcmLoadReferenceCatalog.refObjLoader
247  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
248  for ref in inputRefs.refCat],
249  refCats=butlerQC.get(inputRefs.refCat),
250  config=refConfig,
251  log=self.log)
252  self.makeSubtask('fgcmLoadReferenceCatalog', refObjLoader=refObjLoader)
253  else:
254  lutDataRef = None
255 
256  # Compute aperture radius if necessary. This is useful to do now before
257  # any heave lifting has happened (fail early).
258  calibFluxApertureRadius = None
259  if self.config.doSubtractLocalBackground:
260  try:
261  calibFluxApertureRadius = computeApertureRadiusFromDataRef(sourceTableRefs[0],
262  self.config.instFluxField)
263  except RuntimeError as e:
264  raise RuntimeError("Could not determine aperture radius from %s. "
265  "Cannot use doSubtractLocalBackground." %
266  (self.config.instFluxField)) from e
267 
268  visitSummaryRefs = inputRefDict['visitSummary']
269  visitSummaryDataRefDict = {visitSummaryRef.dataId['visit']: visitSummaryRef for
270  visitSummaryRef in visitSummaryRefs}
271 
272  camera = inputRefDict['camera']
273  groupedDataRefs = self._groupDataRefs(sourceTableDataRefDict,
274  visitSummaryDataRefDict)
275 
276  if self.config.doModelErrorsWithBackground:
277  bkgRefs = inputRefDict['background']
278  bkgDataRefDict = {(bkgRef.dataId.byName()['visit'],
279  bkgRef.dataId.byName()['detector']): bkgRef for
280  bkgRef in bkgRefs}
281  else:
282  bkgDataRefDict = None
283 
284  # Gen3 does not currently allow "checkpoint" saving of datasets,
285  # so we need to have this all in one go.
286  visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
287  bkgDataRefDict=bkgDataRefDict,
288  visitCatDataRef=None,
289  inVisitCat=None)
290 
291  rad = calibFluxApertureRadius
292  # sourceSchemaDataRef = inputRefDict['sourceSchema']
293  fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
294  visitCat,
295  self.sourceSchema,
296  camera,
297  calibFluxApertureRadius=rad,
298  starObsDataRef=None,
299  visitCatDataRef=None,
300  inStarObsCat=None)
301 
302  butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
303  butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
304 
305  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
306  fgcmStarObservationCat,
307  lutDataRef=lutDataRef)
308 
309  butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
310  butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
311  if fgcmRefCat is not None:
312  butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
313 

Member Data Documentation

◆ canMultiprocess

bool lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.canMultiprocess = False
static

Definition at line 223 of file fgcmBuildStarsTable.py.

◆ ConfigClass

lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.ConfigClass = FgcmBuildStarsTableConfig
static

Definition at line 219 of file fgcmBuildStarsTable.py.

◆ RunnerClass

lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.RunnerClass = FgcmBuildStarsRunner
static

Definition at line 220 of file fgcmBuildStarsTable.py.

◆ sourceSchema

lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.sourceSchema

Definition at line 228 of file fgcmBuildStarsTable.py.


The documentation for this class was generated from the following file: