LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
fgcmCalibrateTractTable.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Class for running fgcmcal on a single tract using sourceTable_visit tables.
24 """
25 import numpy as np
26 
27 import lsst.pipe.base as pipeBase
28 from lsst.pipe.base import connectionTypes
29 from lsst.meas.algorithms import ReferenceObjectLoader
30 import lsst.afw.table as afwTable
31 
32 from .dataIds import TractCheckDataIdContainer
33 from .fgcmBuildStarsTable import FgcmBuildStarsTableTask
34 from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, FgcmCalibrateTractRunner,
35  FgcmCalibrateTractBaseTask)
36 from .utilities import lookupStaticCalibrations
37 
38 __all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask']
39 
40 
41 class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections,
42  dimensions=("instrument",
43  "tract",)):
44  camera = connectionTypes.PrerequisiteInput(
45  doc="Camera instrument",
46  name="camera",
47  storageClass="Camera",
48  dimensions=("instrument",),
49  lookupFunction=lookupStaticCalibrations,
50  isCalibration=True,
51  )
52 
53  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
54  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
55  "chromatic corrections."),
56  name="fgcmLookUpTable",
57  storageClass="Catalog",
58  dimensions=("instrument",),
59  deferLoad=True,
60  )
61 
62  sourceSchema = connectionTypes.InitInput(
63  doc="Schema for source catalogs",
64  name="src_schema",
65  storageClass="SourceCatalog",
66  )
67 
68  refCat = connectionTypes.PrerequisiteInput(
69  doc="Reference catalog to use for photometric calibration",
70  name="cal_ref_cat",
71  storageClass="SimpleCatalog",
72  dimensions=("skypix",),
73  deferLoad=True,
74  multiple=True,
75  )
76 
77  source_catalogs = connectionTypes.Input(
78  doc="Source table in parquet format, per visit",
79  name="sourceTable_visit",
80  storageClass="DataFrame",
81  dimensions=("instrument", "visit"),
82  deferLoad=True,
83  multiple=True,
84  )
85 
86  visitSummary = connectionTypes.Input(
87  doc="Per-visit summary statistics table",
88  name="visitSummary",
89  storageClass="ExposureCatalog",
90  dimensions=("instrument", "visit"),
91  deferLoad=True,
92  multiple=True,
93  )
94 
95  background = connectionTypes.Input(
96  doc="Calexp background model",
97  name="calexpBackground",
98  storageClass="Background",
99  dimensions=("instrument", "visit", "detector"),
100  deferLoad=True,
101  multiple=True,
102  )
103 
104  fgcmPhotoCalib = connectionTypes.Output(
105  doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration",
106  name="fgcmPhotoCalibTractCatalog",
107  storageClass="ExposureCatalog",
108  dimensions=("instrument", "tract", "visit",),
109  multiple=True,
110  )
111 
112  fgcmTransmissionAtmosphere = connectionTypes.Output(
113  doc="Per-visit atmosphere transmission files produced from fgcm calibration",
114  name="transmission_atmosphere_fgcm_tract",
115  storageClass="TransmissionCurve",
116  dimensions=("instrument", "tract", "visit",),
117  multiple=True,
118  )
119 
120  fgcmRepeatability = connectionTypes.Output(
121  doc="Per-band raw repeatability numbers in the fgcm tract calibration",
122  name="fgcmRawRepeatability",
123  storageClass="Catalog",
124  dimensions=("instrument", "tract",),
125  multiple=False,
126  )
127 
128  def __init__(self, *, config=None):
129  super().__init__(config=config)
130 
131  # The ref_dataset_name will be deprecated with Gen2
132  loaderName = config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name
133  if config.connections.refCat != loaderName:
134  raise ValueError("connections.refCat must be the same as "
135  "config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name")
136  if config.fgcmOutputProducts.doReferenceCalibration:
137  loaderName = config.fgcmOutputProducts.refObjLoader.ref_dataset_name
138  if config.connections.refCat != loaderName:
139  raise ValueError("connections.refCat must be the same as "
140  "config.fgcmOutputProducts.refObjLoader.ref_dataset_name")
141 
142  if not config.fgcmBuildStars.doModelErrorsWithBackground:
143  self.inputs.remove("background")
144 
145  if config.fgcmOutputProducts.doRefcatOutput:
146  raise ValueError("FgcmCalibrateTractTableTask (Gen3) does not support doRefcatOutput")
147  if not config.fgcmOutputProducts.doAtmosphereOutput:
148  self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
149  if not config.fgcmOutputProducts.doZeropointOutput:
150  self.prerequisiteInputs.remove("fgcmZeropoints")
151 
152 
154  pipelineConnections=FgcmCalibrateTractTableConnections):
155  """Config for FgcmCalibrateTractTable task"""
156  def setDefaults(self):
157  super().setDefaults()
158 
159  # For the Table version of CalibrateTract, use the associated
160  # Table version of the BuildStars task.
161  self.fgcmBuildStarsfgcmBuildStars.retarget(FgcmBuildStarsTableTask)
162  # For tract mode, we set a very high effective density cut.
163  self.fgcmBuildStarsfgcmBuildStars.densityCutMaxPerPixel = 10000
164 
165 
167  """
168  Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet)
169  input catalogs.
170  """
171  ConfigClass = FgcmCalibrateTractTableConfig
172  RunnerClass = FgcmCalibrateTractRunner
173  _DefaultName = "fgcmCalibrateTractTable"
174 
175  canMultiprocess = False
176 
177  def __init__(self, initInputs=None, **kwargs):
178  super().__init__(initInputs=initInputs, **kwargs)
179  if initInputs is not None:
180  self.sourceSchemasourceSchema = initInputs["sourceSchema"].schema
181 
182  def runQuantum(self, butlerQC, inputRefs, outputRefs):
183  dataRefDict = butlerQC.get(inputRefs)
184 
185  self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefDict['source_catalogs'])))
186 
187  # Run the build stars tasks
188  tract = butlerQC.quantum.dataId['tract']
189 
190  dataRefDict['sourceSchema'] = self.sourceSchemasourceSchema
191 
192  sourceTableRefs = dataRefDict['source_catalogs']
193  sourceTableDataRefDict = {sourceTableRef.dataId['visit']: sourceTableRef for
194  sourceTableRef in sourceTableRefs}
195 
196  visitSummaryRefs = dataRefDict['visitSummary']
197  visitSummaryDataRefDict = {visitSummaryRef.dataId['visit']: visitSummaryRef for
198  visitSummaryRef in visitSummaryRefs}
199 
200  dataRefDict['sourceTableDataRefDict'] = sourceTableDataRefDict
201  dataRefDict['visitSummaryDataRefDict'] = visitSummaryDataRefDict
202 
203  # And the outputs
204  if self.config.fgcmOutputProducts.doZeropointOutput:
205  photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
206  photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
207  dataRefDict['fgcmPhotoCalibs'] = photoCalibRefDict
208 
209  if self.config.fgcmOutputProducts.doAtmosphereOutput:
210  atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
211  atmRef in outputRefs.fgcmTransmissionAtmosphere}
212  dataRefDict['fgcmTransmissionAtmospheres'] = atmRefDict
213 
214  if self.config.fgcmBuildStars.doReferenceMatches:
215  refConfig = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader
216  loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
217  for ref in inputRefs.refCat],
218  refCats=butlerQC.get(inputRefs.refCat),
219  config=refConfig,
220  log=self.log)
221  buildStarsRefObjLoader = loader
222  else:
223  buildStarsRefObjLoader = None
224 
225  if self.config.fgcmOutputProducts.doReferenceCalibration:
226  refConfig = self.config.fgcmOutputProducts.refObjLoader
227  loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
228  for ref in inputRefs.refCat],
229  refCats=butlerQC.get(inputRefs.refCat),
230  config=refConfig,
231  log=self.log)
232  self.fgcmOutputProducts.refObjLoader = loader
233 
234  struct = self.runrun(dataRefDict, tract,
235  buildStarsRefObjLoader=buildStarsRefObjLoader)
236 
237  if struct.photoCalibCatalogs is not None:
238  self.log.info("Outputting photoCalib catalogs.")
239  for visit, expCatalog in struct.photoCalibCatalogs:
240  butlerQC.put(expCatalog, photoCalibRefDict[visit])
241  self.log.info("Done outputting photoCalib catalogs.")
242 
243  if struct.atmospheres is not None:
244  self.log.info("Outputting atmosphere transmission files.")
245  for visit, atm in struct.atmospheres:
246  butlerQC.put(atm, atmRefDict[visit])
247  self.log.info("Done outputting atmosphere files.")
248 
249  # Turn raw repeatability into simple catalog for persistence
250  schema = afwTable.Schema()
251  schema.addField('rawRepeatability', type=np.float64,
252  doc="Per-band raw repeatability in FGCM calibration.")
253  repeatabilityCat = afwTable.BaseCatalog(schema)
254  repeatabilityCat.resize(len(struct.repeatability))
255  repeatabilityCat['rawRepeatability'][:] = struct.repeatability
256 
257  butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
258 
259  return
260 
261  @classmethod
262  def _makeArgumentParser(cls):
263  parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
264  parser.add_id_argument("--id", "sourceTable_visit",
265  help="Data ID, e.g. --id visit=6789 tract=9617",
266  ContainerClass=TractCheckDataIdContainer)
267 
268  return parser
Defines the fields and offsets for a table.
Definition: Schema.h:51
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations.