LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
decamMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012-2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import re
24 import traceback
25 
26 import numpy as np
27 
28 from lsst.utils import getPackageDir
29 from astro_metadata_translator import fix_header, DecamTranslator
30 import lsst.afw.image as afwImage
31 from lsst.afw.fits import readMetadata
32 from lsst.afw.geom import makeSkyWcs
33 from lsst.obs.base import CameraMapper
34 from lsst.obs.base.utils import createInitialSkyWcs, InitialSkyWcsError
35 from lsst.daf.persistence import ButlerLocation, Storage, Policy
36 from .makeDecamRawVisitInfo import MakeDecamRawVisitInfo
37 from .decamFilters import DECAM_FILTER_DEFINITIONS
38 from ._instrument import DarkEnergyCamera
39 
40 np.seterr(divide="ignore")
41 
42 __all__ = ["DecamMapper"]
43 
44 
45 class DecamMapper(CameraMapper):
46  packageName = 'obs_decam'
47  _gen3instrument = DarkEnergyCamera
48 
49  MakeRawVisitInfoClass = MakeDecamRawVisitInfo
50 
51  detectorNames = {
52  1: 'S29', 2: 'S30', 3: 'S31', 4: 'S25', 5: 'S26', 6: 'S27', 7: 'S28', 8: 'S20', 9: 'S21',
53  10: 'S22', 11: 'S23', 12: 'S24', 13: 'S14', 14: 'S15', 15: 'S16', 16: 'S17', 17: 'S18',
54  18: 'S19', 19: 'S8', 20: 'S9', 21: 'S10', 22: 'S11', 23: 'S12', 24: 'S13', 25: 'S1', 26: 'S2',
55  27: 'S3', 28: 'S4', 29: 'S5', 30: 'S6', 31: 'S7', 32: 'N1', 33: 'N2', 34: 'N3', 35: 'N4',
56  36: 'N5', 37: 'N6', 38: 'N7', 39: 'N8', 40: 'N9', 41: 'N10', 42: 'N11', 43: 'N12', 44: 'N13',
57  45: 'N14', 46: 'N15', 47: 'N16', 48: 'N17', 49: 'N18', 50: 'N19', 51: 'N20', 52: 'N21',
58  53: 'N22', 54: 'N23', 55: 'N24', 56: 'N25', 57: 'N26', 58: 'N27', 59: 'N28', 60: 'N29',
59  62: 'N31'}
60 
61  def __init__(self, inputPolicy=None, **kwargs):
62  policyFile = Policy.defaultPolicyFile(self.packageNamepackageName, "DecamMapper.yaml", "policy")
63  policy = Policy(policyFile)
64 
65  super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
66 
67  DECAM_FILTER_DEFINITIONS.defineFilters()
68 
69  # The data ID key ccdnum is not directly used in the current policy
70  # template of the raw and instcal et al. datasets, so is not in its
71  # keyDict automatically. Add it so the butler know about the data ID key
72  # ccdnum.
73  # Similarly, add "object" for raws.
74  for datasetType in ("raw", "instcal", "dqmask", "wtmap", "cpIllumcor"):
75  self.mappings[datasetType].keyDict.update({'ccdnum': int})
76  self.mappings["raw"].keyDict.update({'object': str})
77 
78  # The number of bits allocated for fields in object IDs
79  # TODO: This needs to be updated; also see Trac #2797
80  DecamMapper._nbit_tract = 10
81  DecamMapper._nbit_patch = 10
82  DecamMapper._nbit_filter = 4
83  DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract
84  + 2*DecamMapper._nbit_patch
85  + DecamMapper._nbit_filter)
86 
87  def _extractDetectorName(self, dataId):
88  copyId = self._transformId_transformId(dataId)
89  try:
90  return DecamMapper.detectorNames[copyId['ccdnum']]
91  except KeyError:
92  raise RuntimeError("No name found for dataId: %s"%(dataId))
93 
94  def _transformId(self, dataId):
95  copyId = CameraMapper._transformId(self, dataId)
96  if "ccd" in copyId:
97  copyId.setdefault("ccdnum", copyId["ccd"])
98  return copyId
99 
100  def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
101  return self._computeCcdExposureId_computeCcdExposureId(dataId)
102 
103  def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
104  return 32 # not really, but this leaves plenty of space for sources
105 
106  def _computeCcdExposureId(self, dataId):
107  """Compute the 64-bit (long) identifier for a CCD exposure.
108 
109  Parameters
110  ----------
111  dataId : `dict`
112  Data identifier with visit, ccd.
113 
114  Returns
115  -------
116  result : `int`
117  Integer identifier for a CCD exposure.
118  """
119  copyId = self._transformId_transformId(dataId)
120  visit = copyId['visit']
121  ccdnum = copyId['ccdnum']
122  return int("%07d%02d" % (visit, ccdnum))
123 
124  def _computeCoaddExposureId(self, dataId, singleFilter):
125  """Compute the 64-bit (long) identifier for a coadd.
126 
127  Parameters
128  ----------
129  dataId : `dict`
130  Data identifier with tract and patch.
131  singleFilter : `bool`
132  True means the desired ID is for a single-filter coadd,
133  in which case the dataId must contain filter.
134 
135  Returns
136  -------
137  oid : `int`
138  Unique integer identifier.
139  """
140  tract = int(dataId['tract'])
141  if tract < 0 or tract >= 2**DecamMapper._nbit_tract:
142  raise RuntimeError('tract not in range [0,%d)' % (2**DecamMapper._nbit_tract))
143  patchX, patchY = [int(x) for x in dataId['patch'].split(',')]
144  for p in (patchX, patchY):
145  if p < 0 or p >= 2**DecamMapper._nbit_patch:
146  raise RuntimeError('patch component not in range [0, %d)' % 2**DecamMapper._nbit_patch)
147  oid = (((tract << DecamMapper._nbit_patch) + patchX) << DecamMapper._nbit_patch) + patchY
148  if singleFilter:
149  return (oid << DecamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
150  return oid
151 
152  def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
153  return self._computeCoaddExposureId_computeCoaddExposureId(dataId, True)
154 
155  def bypass_deepCoaddId_bits(self, *args, **kwargs):
156  return 64 - DecamMapper._nbit_id
157 
158  def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
159  return self._computeCoaddExposureId_computeCoaddExposureId(dataId, False)
160 
161  def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
162  return 64 - DecamMapper._nbit_id
163 
164  def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId):
165  return self.bypass_deepCoaddIdbypass_deepCoaddId(datasetType, pythonType, location, dataId)
166 
167  def bypass_dcrCoaddId_bits(self, *args, **kwargs):
168  return self.bypass_deepCoaddId_bitsbypass_deepCoaddId_bits(*args, **kwargs)
169 
170  def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId):
171  return self.bypass_deepMergedCoaddIdbypass_deepMergedCoaddId(datasetType, pythonType, location, dataId)
172 
173  def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs):
174  return self.bypass_deepMergedCoaddId_bitsbypass_deepMergedCoaddId_bits(*args, **kwargs)
175 
176  def translate_dqmask(self, dqmask):
177  # TODO: make a class member variable that knows the mappings
178  # below instead of hard-coding them
179  dqmArr = dqmask.getArray()
180  mask = afwImage.Mask(dqmask.getDimensions())
181  mArr = mask.getArray()
182  idxBad = np.where(dqmArr & 1)
183  idxSat = np.where(dqmArr & 2)
184  idxIntrp = np.where(dqmArr & 4)
185  idxCr = np.where(dqmArr & 16)
186  idxBleed = np.where(dqmArr & 64)
187  idxEdge = np.where(dqmArr & 512)
188  mArr[idxBad] |= mask.getPlaneBitMask("BAD")
189  mArr[idxSat] |= mask.getPlaneBitMask("SAT")
190  mArr[idxIntrp] |= mask.getPlaneBitMask("INTRP")
191  mArr[idxCr] |= mask.getPlaneBitMask("CR")
192  mArr[idxBleed] |= mask.getPlaneBitMask("SAT")
193  mArr[idxEdge] |= mask.getPlaneBitMask("EDGE")
194  return mask
195 
196  def translate_wtmap(self, wtmap):
197  wtmArr = wtmap.getArray()
198  idxUndefWeight = np.where(wtmArr <= 0)
199  # Reassign weights to be finite but small:
200  wtmArr[idxUndefWeight] = min(1e-14, np.min(wtmArr[np.where(wtmArr > 0)]))
201  var = 1.0 / wtmArr
202  varim = afwImage.ImageF(var)
203  return varim
204 
205  def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId):
206  # Workaround until I can access the butler
207  instcalMap = self.map_instcal(dataId)
208  dqmaskMap = self.map_dqmask(dataId)
209  wtmapMap = self.map_wtmap(dataId)
210  instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1])
211  dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1])
212  wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1])
213  instcal = instcalType(instcalMap.getLocationsWithRoot()[0])
214  dqmask = dqmaskType(dqmaskMap.getLocationsWithRoot()[0])
215  wtmap = wtmapType(wtmapMap.getLocationsWithRoot()[0])
216 
217  mask = self.translate_dqmasktranslate_dqmask(dqmask)
218  variance = self.translate_wtmaptranslate_wtmap(wtmap)
219 
220  mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance)
221  md = readMetadata(instcalMap.getLocationsWithRoot()[0])
222  fix_header(md, translator_class=DecamTranslator)
223  wcs = makeSkyWcs(md, strip=True)
224  exp = afwImage.ExposureF(mi, wcs)
225 
226  exp.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(10**(0.4 * md.getScalar("MAGZERO")), 0))
227  visitInfo = self.makeRawVisitInfo(md=md)
228  exp.getInfo().setVisitInfo(visitInfo)
229  if 'EXPID' in md:
230  exp.info.id = md['EXPID']
231 
232  for kw in ('LTV1', 'LTV2'):
233  md.remove(kw)
234 
235  exp.setMetadata(md)
236  return exp
237 
238  def std_raw(self, item, dataId):
239  """Standardize a raw dataset by converting it to an Exposure.
240 
241  Raw images are MEF files with one HDU for each detector.
242 
243  Parameters
244  ----------
245  item : `lsst.afw.image.DecoratedImage`
246  The image read by the butler.
247  dataId : data ID
248  Data identifier.
249 
250  Returns
251  -------
252  result : `lsst.afw.image.Exposure`
253  The standardized Exposure.
254  """
255  return self._standardizeExposure(self.exposures['raw'], item, dataId,
256  trimmed=False, setExposureId=True)
257 
258  def _createInitialSkyWcs(self, exposure):
259  # DECam has a coordinate system flipped on X with respect to our
260  # VisitInfo definition of the field angle orientation.
261  # We have to override this method until RFC-605 is implemented, to pass
262  # `flipX=True` to createInitialSkyWcs below.
263  self._createSkyWcsFromMetadata(exposure)
264 
265  if exposure.getInfo().getVisitInfo() is None:
266  msg = "No VisitInfo; cannot access boresight information. Defaulting to metadata-based SkyWcs."
267  self.log.warn(msg)
268  return
269  try:
270  newSkyWcs = createInitialSkyWcs(exposure.getInfo().getVisitInfo(), exposure.getDetector(),
271  flipX=True)
272  exposure.setWcs(newSkyWcs)
273  except InitialSkyWcsError as e:
274  msg = "Cannot create SkyWcs using VisitInfo and Detector, using metadata-based SkyWcs: %s"
275  self.log.warn(msg, e)
276  self.log.debug("Exception was: %s", traceback.TracebackException.from_exception(e))
277  if e.__context__ is not None:
278  self.log.debug("Root-cause Exception was: %s",
279  traceback.TracebackException.from_exception(e.__context__))
280 
281  def std_dark(self, item, dataId):
283  rawPath = self.map_raw(dataId).getLocations()[0]
284  headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath)
285  md0 = readMetadata(headerPath)
286  fix_header(md0, translator_class=DecamTranslator)
287  visitInfo = self.makeRawVisitInfo(md0)
288  exp.getInfo().setVisitInfo(visitInfo)
289  return self._standardizeExposure(self.calibrations["dark"], exp, dataId, filter=False)
290 
291  def std_bias(self, item, dataId):
293  return self._standardizeExposure(self.calibrations["bias"], exp, dataId, filter=False)
294 
295  def std_flat(self, item, dataId):
297  return self._standardizeExposure(self.calibrations["flat"], exp, dataId, filter=True)
298 
299  def std_illumcor(self, item, dataId):
301  return self._standardizeExposure(self.calibrations["illumcor"], exp, dataId, filter=True)
302 
303  def _standardizeCpMasterCal(self, datasetType, item, dataId, setFilter=False):
304  """Standardize a MasterCal image obtained from NOAO archive into Exposure
305 
306  These MasterCal images are MEF files with one HDU for each detector.
307  Some WCS header, eg CTYPE1, exists only in the zeroth extensionr,
308  so info in the zeroth header need to be copied over to metadata.
309 
310  Parameters
311  ----------
312  datasetType : `str`
313  Dataset type ("bias", "flat", or "illumcor").
314  item : `lsst.afw.image.DecoratedImage`
315  The image read by the butler.
316  dataId : data ID
317  Data identifier.
318  setFilter : `bool`
319  Whether to set the filter in the Exposure.
320 
321  Returns
322  -------
323  result : `lsst.afw.image.Exposure`
324  The standardized Exposure.
325  """
326  mi = afwImage.makeMaskedImage(item.getImage())
327  md = item.getMetadata()
328  masterCalMap = getattr(self, "map_" + datasetType)
329  masterCalPath = masterCalMap(dataId).getLocationsWithRoot()[0]
330  headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath)
331  md0 = readMetadata(headerPath)
332  fix_header(md0, translator_class=DecamTranslator)
333  for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2',
334  'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'):
335  if kw in md0.paramNames() and kw not in md.paramNames():
336  md.add(kw, md0.getScalar(kw))
337  wcs = makeSkyWcs(md, strip=True)
338  exp = afwImage.makeExposure(mi, wcs)
339  exp.setMetadata(md)
340  return self._standardizeExposure(self.calibrations[datasetType], exp, dataId, filter=setFilter)
341 
342  def std_cpBias(self, item, dataId):
343  return self._standardizeCpMasterCal_standardizeCpMasterCal("cpBias", item, dataId, setFilter=False)
344 
345  def std_cpFlat(self, item, dataId):
346  return self._standardizeCpMasterCal_standardizeCpMasterCal("cpFlat", item, dataId, setFilter=True)
347 
348  def std_fringe(self, item, dataId):
350  return self._standardizeExposure(self.calibrations["fringe"], exp, dataId)
351 
352  def std_cpIllumcor(self, item, dataId):
353  return self._standardizeCpMasterCal_standardizeCpMasterCal("cpIllumcor", item, dataId, setFilter=True)
354 
355  @classmethod
357  """Directory containing linearizers"""
358  packageName = cls.getPackageName()
359  packageDir = getPackageDir(packageName)
360  return os.path.join(packageDir, "decam", "linearizer")
361 
362  def map_linearizer(self, dataId, write=False):
363  """Map a linearizer"""
364  actualId = self._transformId_transformId(dataId)
365  location = "%02d.fits" % (dataId["ccdnum"])
366  return ButlerLocation(
367  pythonType="lsst.ip.isr.LinearizeSquared",
368  cppType="Config",
369  storageName="PickleStorage",
370  locationList=[location],
371  dataId=actualId,
372  mapper=self,
373  storage=Storage.makeFromURI(self.getLinearizerDirgetLinearizerDir())
374  )
375 
376  @classmethod
377  def getCrosstalkDir(cls):
378  """Directory containing crosstalk tables.
379  """
380  packageName = cls.getPackageName()
381  packageDir = getPackageDir(packageName)
382  return os.path.join(packageDir, "decam", "crosstalk")
int min
Represent a 2-dimensional array of bitmask pixels.
Definition: Mask.h:77
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
Definition: decamMapper.py:158
def bypass_deepMergedCoaddId_bits(self, *args, **kwargs)
Definition: decamMapper.py:161
def std_cpIllumcor(self, item, dataId)
Definition: decamMapper.py:352
def std_dark(self, item, dataId)
Definition: decamMapper.py:281
def std_fringe(self, item, dataId)
Definition: decamMapper.py:348
def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs)
Definition: decamMapper.py:173
def bypass_dcrCoaddId_bits(self, *args, **kwargs)
Definition: decamMapper.py:167
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
Definition: decamMapper.py:100
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
Definition: decamMapper.py:152
def std_illumcor(self, item, dataId)
Definition: decamMapper.py:299
def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId)
Definition: decamMapper.py:164
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
Definition: decamMapper.py:103
def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId)
Definition: decamMapper.py:170
def std_raw(self, item, dataId)
Definition: decamMapper.py:238
def map_linearizer(self, dataId, write=False)
Definition: decamMapper.py:362
def _standardizeCpMasterCal(self, datasetType, item, dataId, setFilter=False)
Definition: decamMapper.py:303
def __init__(self, inputPolicy=None, **kwargs)
Definition: decamMapper.py:61
def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId)
Definition: decamMapper.py:205
def std_bias(self, item, dataId)
Definition: decamMapper.py:291
def _computeCoaddExposureId(self, dataId, singleFilter)
Definition: decamMapper.py:124
def std_flat(self, item, dataId)
Definition: decamMapper.py:295
def std_cpBias(self, item, dataId)
Definition: decamMapper.py:342
def std_cpFlat(self, item, dataId)
Definition: decamMapper.py:345
def bypass_deepCoaddId_bits(self, *args, **kwargs)
Definition: decamMapper.py:155
std::shared_ptr< daf::base::PropertyList > readMetadata(std::string const &fileName, int hdu=DEFAULT_HDU, bool strip=false)
Read FITS header.
Definition: fits.cc:1657
std::shared_ptr< SkyWcs > makeSkyWcs(daf::base::PropertySet &metadata, bool strip=false)
Construct a SkyWcs from FITS keywords.
Definition: SkyWcs.cc:521
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
std::shared_ptr< PhotoCalib > makePhotoCalibFromCalibZeroPoint(double instFluxMag0, double instFluxMag0Err)
Construct a PhotoCalib from the deprecated Calib-style instFluxMag0/instFluxMag0Err values.
Definition: PhotoCalib.cc:613
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< geom::SkyWcs const > wcs=std::shared_ptr< geom::SkyWcs const >())
A function to return an Exposure of the correct type (cf.
Definition: Exposure.h:462
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT >> image, typename std::shared_ptr< Mask< MaskPixelT >> mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT >> variance=Image< VariancePixelT >())
A function to return a MaskedImage of the correct type (cf.
Definition: MaskedImage.h:1240
std::string getPackageDir(std::string const &packageName)
return the root directory of a setup package
Definition: packaging.cc:33