22 from collections
import OrderedDict
28 from lsst.afw.image import Exposure, MaskedImage, Image, DecoratedImage
30 __all__ = [
"Mapping",
"ImageMapping",
"ExposureMapping",
"CalibrationMapping",
"DatasetMapping"]
35 """Mapping is a base class for all mappings. Mappings are used by
36 the Mapper to map (determine a path to some data given some
37 identifiers) and standardize (convert data into some standard
38 format or type) data, and to query the associated registry to see
39 what data is available.
41 Subclasses must specify self.storage or else override self.map().
43 Public methods: lookup, have, need, getKeys, map
45 Mappings are specified mainly by policy. A Mapping policy should
48 template (string): a Python string providing the filename for that
49 particular dataset type based on some data identifiers. In the
50 case of redundancy in the path (e.g., file uniquely specified by
51 the exposure number, but filter in the path), the
52 redundant/dependent identifiers can be looked up in the registry.
54 python (string): the Python type for the retrieved data (e.g.
55 lsst.afw.image.ExposureF)
57 persistable (string): the Persistable registration for the on-disk data
60 storage (string, optional): Storage type for this dataset type (e.g.
63 level (string, optional): the level in the camera hierarchy at which the
64 data is stored (Amp, Ccd or skyTile), if relevant
66 tables (string, optional): a whitespace-delimited list of tables in the
67 registry that can be NATURAL JOIN-ed to look up additional
73 Butler dataset type to be mapped.
74 policy : `daf_persistence.Policy`
76 registry : `lsst.obs.base.Registry`
77 Registry for metadata lookups.
78 rootStorage : Storage subclass instance
79 Interface to persisted repository data.
80 provided : `list` of `str`
81 Keys provided by the mapper.
84 def __init__(self, datasetType, policy, registry, rootStorage, provided=None):
87 raise RuntimeError(
"No policy provided for mapping")
103 (k, _formatMap(v, k, datasetType))
105 re.findall(
r'\%\((\w+)\).*?([diouxXeEfFgGcrs])', self.
template)
109 if provided
is not None:
116 if 'level' in policy:
118 if 'tables' in policy:
124 self.
obsTimeName = policy[
'obsTimeName']
if 'obsTimeName' in policy
else None
125 self.
recipe = policy[
'recipe']
if 'recipe' in policy
else 'default'
132 raise RuntimeError(f
"Template is not defined for the {self.datasetType} dataset type, "
133 "it must be set before it can be used.")
136 """Return the dict of keys and value types required for this mapping.
140 def map(self, mapper, dataId, write=False):
141 """Standard implementation of map function.
145 mapper: `lsst.daf.persistence.Mapper`
152 lsst.daf.persistence.ButlerLocation
153 Location of object that was mapped.
156 usedDataId = {key: actualId[key]
for key
in self.
keyDict.
keys()}
157 path = mapper._mapActualToPath(self.
template, actualId)
158 if os.path.isabs(path):
159 raise RuntimeError(
"Mapped path should not be absolute.")
168 for ext
in (
None,
'.gz',
'.fz'):
169 if ext
and path.endswith(ext):
171 extPath = path + ext
if ext
else path
176 assert path,
"Fully-qualified filename is empty."
179 if hasattr(mapper, addFunc):
180 addFunc = getattr(mapper, addFunc)
181 additionalData = addFunc(self.
datasetType, actualId)
182 assert isinstance(additionalData, PropertySet), \
183 "Bad type for returned data: %s" (
type(additionalData),)
185 additionalData =
None
188 locationList=path, dataId=actualId.copy(), mapper=mapper,
190 additionalData=additionalData)
193 """Look up properties for in a metadata registry given a partial
198 properties : `list` of `str`
206 Values of properties.
209 raise RuntimeError(
"No registry for lookup")
211 skyMapKeys = (
"tract",
"patch")
223 substitutions = OrderedDict()
225 properties =
list(properties)
229 substitutions[p] = dataId[p]
233 "Cannot look up skymap key '%s'; it must be explicitly included in the data ID" % p
236 substitutions[p] = index
244 if p
not in (
'filter',
'expTime',
'taiObs'):
247 if fastPath
and 'visit' in dataId
and "raw" in self.
tables:
248 lookupDataId = {
'visit': dataId[
'visit']}
251 if dataId
is not None:
252 for k, v
in dataId.items():
259 where.append((k,
'?'))
261 lookupDataId = {k[0]: v
for k, v
in zip(where, values)}
271 result = [tuple(v
if k
in removed
else item[v]
for k, v
in substitutions.items())
275 def have(self, properties, dataId):
276 """Returns whether the provided data identifier has all
277 the properties in the provided list.
281 properties : `list of `str`
289 True if all properties are present.
291 for prop
in properties:
292 if prop
not in dataId:
296 def need(self, properties, dataId):
297 """Ensures all properties in the provided list are present in
298 the data identifier, looking them up as needed. This is only
299 possible for the case where the data identifies a single
304 properties : `list` of `str`
307 Partial dataset identifier
312 Copy of dataset identifier with enhanced values.
314 newId = dataId.copy()
316 for prop
in properties:
317 if prop
not in newId:
318 newProps.append(prop)
319 if len(newProps) == 0:
322 lookups = self.
lookup(newProps, newId)
323 if len(lookups) != 1:
324 raise NoResults(
"No unique lookup for %s from %s: %d matches" %
325 (newProps, newId, len(lookups)),
327 for i, prop
in enumerate(newProps):
328 newId[prop] = lookups[0][i]
332 def _formatMap(ch, k, datasetType):
333 """Convert a format character into a Python type."""
341 raise RuntimeError(
"Unexpected format specifier %s"
342 " for field %s in template for dataset %s" %
343 (ch, k, datasetType))
347 """ImageMapping is a Mapping subclass for non-camera images.
352 Butler dataset type to be mapped.
353 policy : `daf_persistence.Policy`
355 registry : `lsst.obs.base.Registry`
356 Registry for metadata lookups
358 Path of root directory
361 def __init__(self, datasetType, policy, registry, root, **kwargs):
362 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
363 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None
367 """ExposureMapping is a Mapping subclass for normal exposures.
372 Butler dataset type to be mapped.
373 policy : `daf_persistence.Policy`
375 registry : `lsst.obs.base.Registry`
376 Registry for metadata lookups
378 Path of root directory
381 def __init__(self, datasetType, policy, registry, root, **kwargs):
382 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
383 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None
386 return mapper._standardizeExposure(self, item, dataId)
390 """CalibrationMapping is a Mapping subclass for calibration-type products.
392 The difference is that data properties in the query or template
393 can be looked up using a reference Mapping in addition to this one.
395 CalibrationMapping Policies can contain the following:
397 reference (string, optional)
398 a list of tables for finding missing dataset
399 identifier components (including the observation time, if a validity
400 range is required) in the exposure registry; note that the "tables"
401 entry refers to the calibration registry
403 refCols (string, optional)
404 a list of dataset properties required from the
405 reference tables for lookups in the calibration registry
408 true if the calibration dataset has a validity range
409 specified by a column in the tables of the reference dataset in the
410 exposure registry) and two columns in the tables of this calibration
411 dataset in the calibration registry)
413 obsTimeName (string, optional)
414 the name of the column in the reference
415 dataset tables containing the observation time (default "taiObs")
417 validStartName (string, optional)
418 the name of the column in the
419 calibration dataset tables containing the start of the validity range
420 (default "validStart")
422 validEndName (string, optional)
423 the name of the column in the
424 calibration dataset tables containing the end of the validity range
430 Butler dataset type to be mapped.
431 policy : `daf_persistence.Policy`
433 registry : `lsst.obs.base.Registry`
434 Registry for metadata lookups
435 calibRegistry : `lsst.obs.base.Registry`
436 Registry for calibration metadata lookups.
438 Path of calibration root directory.
440 Path of data root directory; used for outputs only.
443 def __init__(self, datasetType, policy, registry, calibRegistry, calibRoot, dataRoot=None, **kwargs):
444 Mapping.__init__(self, datasetType, policy, calibRegistry, calibRoot, **kwargs)
445 self.
reference = policy.asArray(
"reference")
if "reference" in policy
else None
446 self.
refCols = policy.asArray(
"refCols")
if "refCols" in policy
else None
449 if "validRange" in policy
and policy[
"validRange"]:
450 self.
range = (
"?", policy[
"validStartName"], policy[
"validEndName"])
451 if "columns" in policy:
453 if "filter" in policy:
456 if "metadataKey" in policy:
459 def map(self, mapper, dataId, write=False):
460 location = Mapping.map(self, mapper, dataId, write=write)
467 """Look up properties for in a metadata registry given a partial
472 properties : `list` of `str`
473 Properties to look up.
480 Values of properties.
486 newId = dataId.copy()
490 for k, v
in dataId.items():
499 for k
in dataId.keys():
502 columns =
set(properties)
507 return Mapping.lookup(self, properties, newId)
509 lookupDataId = dict(zip(where, values))
511 if len(lookups) != 1:
512 raise RuntimeError(
"No unique lookup for %s from %s: %d matches" %
513 (columns, dataId, len(lookups)))
514 if columns ==
set(properties):
517 for i, prop
in enumerate(columns):
518 newId[prop] = lookups[0][i]
519 return Mapping.lookup(self, properties, newId)
522 """Default standardization function for calibration datasets.
524 If the item is of a type that should be standardized, the base class
525 ``standardizeExposure`` method is called, otherwise the item is
530 mapping : `lsst.obs.base.Mapping`
531 Mapping object to pass through.
533 Will be standardized if of type lsst.afw.image.Exposure,
534 lsst.afw.image.DecoratedImage, lsst.afw.image.Image
535 or lsst.afw.image.MaskedImage
542 `lsst.afw.image.Exposure` or item
543 The standardized object.
545 if issubclass(
doImport(self.
python), (Exposure, MaskedImage, Image, DecoratedImage)):
546 return mapper._standardizeExposure(self, item, dataId, filter=self.
setFilter)
551 """DatasetMapping is a Mapping subclass for non-Exposure datasets that can
552 be retrieved by the standard daf_persistence mechanism.
554 The differences are that the Storage type must be specified and no
555 Exposure standardization is performed.
557 The "storage" entry in the Policy is mandatory; the "tables" entry is
558 optional; no "level" entry is allowed.
563 Butler dataset type to be mapped.
564 policy : `daf_persistence.Policy`
566 registry : `lsst.obs.base.Registry`
567 Registry for metadata lookups
569 Path of root directory
572 def __init__(self, datasetType, policy, registry, root, **kwargs):
573 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)