LSST Applications  21.0.0+04719a4bac,21.0.0-1-ga51b5d4+f5e6047307,21.0.0-11-g2b59f77+a9c1acf22d,21.0.0-11-ga42c5b2+86977b0b17,21.0.0-12-gf4ce030+76814010d2,21.0.0-13-g1721dae+760e7a6536,21.0.0-13-g3a573fe+768d78a30a,21.0.0-15-g5a7caf0+f21cbc5713,21.0.0-16-g0fb55c1+b60e2d390c,21.0.0-19-g4cded4ca+71a93a33c0,21.0.0-2-g103fe59+bb20972958,21.0.0-2-g45278ab+04719a4bac,21.0.0-2-g5242d73+3ad5d60fb1,21.0.0-2-g7f82c8f+8babb168e8,21.0.0-2-g8f08a60+06509c8b61,21.0.0-2-g8faa9b5+616205b9df,21.0.0-2-ga326454+8babb168e8,21.0.0-2-gde069b7+5e4aea9c2f,21.0.0-2-gecfae73+1d3a86e577,21.0.0-2-gfc62afb+3ad5d60fb1,21.0.0-25-g1d57be3cd+e73869a214,21.0.0-3-g357aad2+ed88757d29,21.0.0-3-g4a4ce7f+3ad5d60fb1,21.0.0-3-g4be5c26+3ad5d60fb1,21.0.0-3-g65f322c+e0b24896a3,21.0.0-3-g7d9da8d+616205b9df,21.0.0-3-ge02ed75+a9c1acf22d,21.0.0-4-g591bb35+a9c1acf22d,21.0.0-4-g65b4814+b60e2d390c,21.0.0-4-gccdca77+0de219a2bc,21.0.0-4-ge8a399c+6c55c39e83,21.0.0-5-gd00fb1e+05fce91b99,21.0.0-6-gc675373+3ad5d60fb1,21.0.0-64-g1122c245+4fb2b8f86e,21.0.0-7-g04766d7+cd19d05db2,21.0.0-7-gdf92d54+04719a4bac,21.0.0-8-g5674e7b+d1bd76f71f,master-gac4afde19b+a9c1acf22d,w.2021.13
LSST Data Management Base Package
coaddBase.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import lsst.pex.config as pexConfig
23 import lsst.geom as geom
24 import lsst.afw.geom as afwGeom
25 import lsst.afw.image as afwImage
26 import lsst.pipe.base as pipeBase
27 import lsst.meas.algorithms as measAlg
28 
29 from lsst.afw.fits import FitsError
30 from lsst.coadd.utils import CoaddDataIdContainer
31 from .selectImages import WcsSelectImagesTask, SelectStruct
32 from .coaddInputRecorder import CoaddInputRecorderTask
33 from .scaleVariance import ScaleVarianceTask
34 
35 __all__ = ["CoaddBaseTask", "getSkyInfo", "makeSkyInfo", "makeCoaddSuffix"]
36 
37 
38 class CoaddBaseConfig(pexConfig.Config):
39  """!Configuration parameters for CoaddBaseTask
40 
41  @anchor CoaddBaseConfig_
42 
43  @brief Configuration parameters shared between MakeCoaddTempExp and AssembleCoadd
44  """
45  coaddName = pexConfig.Field(
46  doc="Coadd name: typically one of deep or goodSeeing.",
47  dtype=str,
48  default="deep",
49  )
50  select = pexConfig.ConfigurableField(
51  doc="Image selection subtask.",
52  target=WcsSelectImagesTask,
53  )
54  badMaskPlanes = pexConfig.ListField(
55  dtype=str,
56  doc="Mask planes that, if set, the associated pixel should not be included in the coaddTempExp.",
57  default=("NO_DATA",),
58  )
59  inputRecorder = pexConfig.ConfigurableField(
60  doc="Subtask that helps fill CoaddInputs catalogs added to the final Exposure",
61  target=CoaddInputRecorderTask
62  )
63  doPsfMatch = pexConfig.Field(
64  dtype=bool,
65  doc="Match to modelPsf? Deprecated. Sets makePsfMatched=True, makeDirect=False",
66  default=False
67  )
68  modelPsf = measAlg.GaussianPsfFactory.makeField(doc="Model Psf factory")
69  doApplyExternalPhotoCalib = pexConfig.Field(
70  dtype=bool,
71  default=False,
72  doc=("Whether to apply external photometric calibration via an "
73  "`lsst.afw.image.PhotoCalib` object. Uses the "
74  "`externalPhotoCalibName` field to determine which calibration "
75  "to load.")
76  )
77  useGlobalExternalPhotoCalib = pexConfig.Field(
78  dtype=bool,
79  default=True,
80  doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations "
81  "that are not run per-tract. When False, use per-tract photometric "
82  "calibration files.")
83  )
84  externalPhotoCalibName = pexConfig.ChoiceField(
85  # TODO: Remove this config with the removal of Gen2 in DM-20572.
86  dtype=str,
87  doc=("Type of external PhotoCalib if `doApplyExternalPhotoCalib` is True. "
88  "This field is only used for Gen2 middleware."),
89  default="jointcal",
90  allowed={
91  "jointcal": "Use jointcal_photoCalib",
92  "fgcm": "Use fgcm_photoCalib",
93  "fgcm_tract": "Use fgcm_tract_photoCalib"
94  }
95  )
96  doApplyExternalSkyWcs = pexConfig.Field(
97  dtype=bool,
98  default=False,
99  doc=("Whether to apply external astrometric calibration via an "
100  "`lsst.afw.geom.SkyWcs` object. Uses `externalSkyWcsName` "
101  "field to determine which calibration to load.")
102  )
103  useGlobalExternalSkyWcs = pexConfig.Field(
104  dtype=bool,
105  default=False,
106  doc=("When using doApplyExternalSkyWcs, use 'global' calibrations "
107  "that are not run per-tract. When False, use per-tract wcs "
108  "files.")
109  )
110  externalSkyWcsName = pexConfig.ChoiceField(
111  # TODO: Remove this config with the removal of Gen2 in DM-20572.
112  dtype=str,
113  doc=("Type of external SkyWcs if `doApplyExternalSkyWcs` is True. "
114  "This field is only used for Gen2 middleware."),
115  default="jointcal",
116  allowed={
117  "jointcal": "Use jointcal_wcs"
118  }
119  )
120  includeCalibVar = pexConfig.Field(
121  dtype=bool,
122  doc="Add photometric calibration variance to warp variance plane.",
123  default=False
124  )
125  matchingKernelSize = pexConfig.Field(
126  dtype=int,
127  doc="Size in pixels of matching kernel. Must be odd.",
128  default=21,
129  check=lambda x: x % 2 == 1
130  )
131 
132 
133 class CoaddTaskRunner(pipeBase.TaskRunner):
134 
135  @staticmethod
136  def getTargetList(parsedCmd, **kwargs):
137  return pipeBase.TaskRunner.getTargetList(parsedCmd, selectDataList=parsedCmd.selectId.dataList,
138  **kwargs)
139 
140 
141 class CoaddBaseTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
142  """!Base class for coaddition.
143 
144  Subclasses must specify _DefaultName
145  """
146  ConfigClass = CoaddBaseConfig
147  RunnerClass = CoaddTaskRunner
148 
149  def __init__(self, **kwargs):
150  super().__init__(**kwargs)
151  self.makeSubtask("select")
152  self.makeSubtask("inputRecorder")
153 
154  def selectExposures(self, patchRef, skyInfo=None, selectDataList=[]):
155  """!
156  @brief Select exposures to coadd
157 
158  Get the corners of the bbox supplied in skyInfo using @ref geom.Box2D and convert the pixel
159  positions of the bbox corners to sky coordinates using @ref skyInfo.wcs.pixelToSky. Use the
160  @ref WcsSelectImagesTask_ "WcsSelectImagesTask" to select exposures that lie inside the patch
161  indicated by the dataRef.
162 
163  @param[in] patchRef data reference for sky map patch. Must include keys "tract", "patch",
164  plus the camera-specific filter key (e.g. "filter" or "band")
165  @param[in] skyInfo geometry for the patch; output from getSkyInfo
166  @return a list of science exposures to coadd, as butler data references
167  """
168  if skyInfo is None:
169  skyInfo = self.getSkyInfogetSkyInfo(patchRef)
170  cornerPosList = geom.Box2D(skyInfo.bbox).getCorners()
171  coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
172  return self.select.runDataRef(patchRef, coordList, selectDataList=selectDataList).dataRefList
173 
174  def getSkyInfo(self, patchRef):
175  """!
176  @brief Use @ref getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox
177  of the patch.
178 
179  @param[in] patchRef data reference for sky map. Must include keys "tract" and "patch"
180 
181  @return pipe_base Struct containing:
182  - skyMap: sky map
183  - tractInfo: information for chosen tract of sky map
184  - patchInfo: information about chosen patch of tract
185  - wcs: WCS of tract
186  - bbox: outer bbox of patch, as an geom Box2I
187  """
188  return getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
189 
190  def getCoaddDatasetName(self, warpType="direct"):
191  """Return coadd name for given warpType and task config
192 
193  Parameters
194  ----------
195  warpType : string
196  Either 'direct' or 'psfMatched'
197 
198  Returns
199  -------
200  CoaddDatasetName : `string`
201  """
202  return self.config.coaddName + "Coadd" + makeCoaddSuffix(warpType)
203 
204  def getTempExpDatasetName(self, warpType="direct"):
205  """Return warp name for given warpType and task config
206 
207  Parameters
208  ----------
209  warpType : string
210  Either 'direct' or 'psfMatched'
211 
212  Returns
213  -------
214  WarpDatasetName : `string`
215  """
216  return self.config.coaddName + "Coadd_" + warpType + "Warp"
217 
218  @classmethod
219  def _makeArgumentParser(cls):
220  """Create an argument parser
221  """
222  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
223  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2",
224  ContainerClass=CoaddDataIdContainer)
225  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
226  ContainerClass=SelectDataIdContainer)
227  return parser
228 
229  def _getConfigName(self):
230  """Return the name of the config dataset
231  """
232  return "%s_%s_config" % (self.config.coaddName, self._DefaultName)
233 
234  def _getMetadataName(self):
235  """Return the name of the metadata dataset
236  """
237  return "%s_%s_metadata" % (self.config.coaddName, self._DefaultName)
238 
239  def getBadPixelMask(self):
240  """!
241  @brief Convenience method to provide the bitmask from the mask plane names
242  """
243  return afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes)
244 
245 
246 class SelectDataIdContainer(pipeBase.DataIdContainer):
247  """!
248  @brief A dataId container for inputs to be selected.
249 
250  Read the header (including the size and Wcs) for all specified
251  inputs and pass those along, ultimately for the SelectImagesTask.
252  This is most useful when used with multiprocessing, as input headers are
253  only read once.
254  """
255 
256  def makeDataRefList(self, namespace):
257  """Add a dataList containing useful information for selecting images"""
258  super(SelectDataIdContainer, self).makeDataRefList(namespace)
259  self.dataListdataList = []
260  for ref in self.refList:
261  try:
262  md = ref.get("calexp_md", immediate=True)
263  wcs = afwGeom.makeSkyWcs(md)
264  data = SelectStruct(dataRef=ref, wcs=wcs, bbox=afwImage.bboxFromMetadata(md))
265  except FitsError:
266  namespace.log.warn("Unable to construct Wcs from %s" % (ref.dataId))
267  continue
268  self.dataListdataList.append(data)
269 
270 
271 def getSkyInfo(coaddName, patchRef):
272  """!
273  @brief Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded.
274 
275  @param[in] coaddName coadd name; typically one of deep or goodSeeing
276  @param[in] patchRef data reference for sky map. Must include keys "tract" and "patch"
277 
278  @return pipe_base Struct containing:
279  - skyMap: sky map
280  - tractInfo: information for chosen tract of sky map
281  - patchInfo: information about chosen patch of tract
282  - wcs: WCS of tract
283  - bbox: outer bbox of patch, as an geom Box2I
284  """
285  skyMap = patchRef.get(coaddName + "Coadd_skyMap")
286  return makeSkyInfo(skyMap, patchRef.dataId["tract"], patchRef.dataId["patch"])
287 
288 
289 def makeSkyInfo(skyMap, tractId, patchId):
290  """Return SkyInfo Struct
291 
292  Constructs SkyInfo used by coaddition tasks for multiple
293  patchId formats.
294 
295  Parameters
296  ----------
297  skyMap : `lsst.skyMap.SkyMap`
298  tractId : int
299  patchId : str or int or tuple of int
300  Either Gen2-style comma delimited string (e.g. '4,5'),
301  tuple of integers (e.g (4, 5), Gen3-style integer.
302  """
303  tractInfo = skyMap[tractId]
304 
305  if isinstance(patchId, str) and ',' in patchId:
306  # patch format is "xIndex,yIndex"
307  patchIndex = tuple(int(i) for i in patchId.split(","))
308  else:
309  patchIndex = patchId
310 
311  patchInfo = tractInfo.getPatchInfo(patchIndex)
312 
313  return pipeBase.Struct(
314  skyMap=skyMap,
315  tractInfo=tractInfo,
316  patchInfo=patchInfo,
317  wcs=tractInfo.getWcs(),
318  bbox=patchInfo.getOuterBBox(),
319  )
320 
321 
322 def scaleVariance(maskedImage, maskPlanes, log=None):
323  """!
324  @brief Scale the variance in a maskedImage
325 
326  The variance plane in a convolved or warped image (or a coadd derived
327  from warped images) does not accurately reflect the noise properties of
328  the image because variance has been lost to covariance. This function
329  attempts to correct for this by scaling the variance plane to match
330  the observed variance in the image. This is not perfect (because we're
331  not tracking the covariance) but it's simple and is often good enough.
332 
333  @deprecated Use the ScaleVarianceTask instead.
334 
335  @param maskedImage MaskedImage to operate on; variance will be scaled
336  @param maskPlanes List of mask planes for pixels to reject
337  @param log Log for reporting the renormalization factor; or None
338  @return renormalisation factor
339  """
340  config = ScaleVarianceTask.ConfigClass()
341  config.maskPlanes = maskPlanes
342  task = ScaleVarianceTask(config=config, name="scaleVariance", log=log)
343  return task.run(maskedImage)
344 
345 
346 def makeCoaddSuffix(warpType="direct"):
347  """Return coadd suffix for warpType
348 
349  Parameters
350  ----------
351  warpType : string
352  Either 'direct' or 'psfMatched'
353 
354  Returns
355  -------
356  CoaddSuffix : `string`
357  """
358  suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
359  return suffix
360 
361 
362 def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
363  """Match the order of one list to another, padding if necessary
364 
365  Parameters
366  ----------
367  inputList : list
368  List to be reordered and padded. Elements can be any type.
369  inputKeys : iterable
370  Iterable of values to be compared with outputKeys.
371  Length must match `inputList`
372  outputKeys : iterable
373  Iterable of values to be compared with inputKeys.
374  padWith :
375  Any value to be inserted where inputKey not in outputKeys
376 
377  Returns
378  -------
379  list
380  Copy of inputList reordered per outputKeys and padded with `padWith`
381  so that the length matches length of outputKeys.
382  """
383  outputList = []
384  for d in outputKeys:
385  if d in inputKeys:
386  outputList.append(inputList[inputKeys.index(d)])
387  else:
388  outputList.append(padWith)
389  return outputList
A floating-point coordinate rectangle geometry.
Definition: Box.h:413
Configuration parameters for CoaddBaseTask.
Definition: coaddBase.py:38
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
def getTargetList(parsedCmd, **kwargs)
Definition: coaddBase.py:136
A dataId container for inputs to be selected.
Definition: coaddBase.py:246
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
Definition: functional.cc:33
std::shared_ptr< SkyWcs > makeSkyWcs(daf::base::PropertySet &metadata, bool strip=false)
Construct a SkyWcs from FITS keywords.
Definition: SkyWcs.cc:526
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
lsst::geom::Box2I bboxFromMetadata(daf::base::PropertySet &metadata)
Determine the image bounding box from its metadata (FITS header)
Definition: Image.cc:694
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations.
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:346
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:322
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded.
Definition: coaddBase.py:271