25 import lsst.daf.butlerUtils
26 from functools
import reduce
28 __all__ = [
'getMapperClass',
36 'lsstsim':
'lsst.obs.lsstSim.LsstSimMapper',
37 'sdss':
'lsst.obs.sdss.SdssMapper',
38 'cfht':
'lsst.obs.cfht.CfhtMapper',
43 """Return the subclass of lsst.daf.persistence.Mapper 44 to use for the camera with the given name (case-insensitive). 46 camera = camera.lower()
47 if camera
not in _mapperClassName:
48 raise RuntimeError(str.format(
"{} is not a valid camera name", camera))
49 name = _mapperClassName[camera]
51 pieces = name.split(
'.')
52 cls = reduce(getattr, pieces[1:], __import__(
'.'.join(pieces[:-1])))
55 raise RuntimeError(str.format(
"Failed to import {}", name))
106 """A rule is a string in the following format: 108 'key=value1[^value2[^value3...]' 110 The values may either be strings, or of the form 'int...int' 111 (e.g. '1..3') which is interpreted as '1^2^3' (inclusive, unlike a python 112 range). So '0^2..4^7..9' is equivalent to '0^2^3^4^7^8^9'. 114 This function parses a list of such strings, and returns a dict mapping 115 keys to sets of legal values. 120 Camera the rule list applies to (e.g. 'lsstSim' or 'sdss') 122 camera = camera.lower()
123 if camera
not in _keyTypes:
124 raise RuntimeError(
'{} is not a recognized camera name'.
format(camera))
126 for rule
in ruleList:
128 key, _, pattern = rule.partition(
'=')
129 if key
not in _keyTypes[camera]:
130 raise RuntimeError(
'{} is not a valid dataId key for camera {}'.
format(key, camera))
131 if len(pattern) == 0:
135 for p
in pattern.split(
'^'):
136 if _keyTypes[camera][key] == int:
138 m = re.search(
r'^(\d+)\.\.(\d+)$', p)
140 values.update(xrange(int(m.group(1)), int(m.group(2)) + 1))
146 kvs[key].update(values)
153 """A key in a path template. Three attributes are provided: 156 Formatting spec for the key, e.g. '%(filter)s'. 159 key value type; int or str 162 A function that takes a key name, key value string and a dictionary. 163 This function should return a fresh dictionary including new entries 164 derived from the given key, value, and existing entries. The 165 _mungeStr and _mungeInt functions are examples. 175 """Munger for keys with string formats.""" 182 """Munger for keys with integer formats.""" 189 """A single component (directory or file) of a path template. The 190 following attributes are provided: 193 List of key names first occurring in this path component. 196 Compiled regular expression identifying matches to this path 197 component unless simple is True; in that case, regex is just 201 True if regex is a simple string literal rather than a pattern. 202 In this case, keys will always by None or []. 212 """A hierarchical scanner for paths matching a template, optionally 213 also restricting visited paths to those matching a list of dataId rules. 217 """Build an FsScanner for given a path template. The path template 218 should be a Python string with named format substitution 219 specifications, as used in mapper policy files. For example: 221 deepCoadd-results/%(filter)s/%(tract)d/%(patch)s/calexp-%(filter)s-%(tract)d-%(patch)s.fits 223 Note that a key may appear multiple times. If it does, 224 the value for each occurrence should be identical (the formatting 225 specs must be identical). Octal, binary, hexadecimal, and floating 226 point formats are not supported. 228 template = os.path.normpath(template)
229 if (len(template) == 0
or 230 template == os.curdir
or 231 template[0] == os.sep
or 232 template[-1] == os.sep):
234 'Path template is empty, absolute, or identifies a directory')
237 fmt = re.compile(
r'%\((\w+)\).*?([diucrs])')
240 for component
in template.split(os.sep):
246 for m
in fmt.finditer(component):
252 regex += re.escape(component[last:m.start(0)])
257 if m.group(2)
in 'crs':
264 regex +=
r'[+-]?\d+)' 269 'Path template contains inconsistent format type-codes ' 274 regex += re.escape(component[last:])
278 regex = re.compile(
'^' + regex +
'$')
281 def walk(self, root, rules=None):
282 """Generator that descends the given root directory in top-down 283 fashion, matching paths corresponding to the template and satisfying 284 the given rule list. The generator yields tuples of the form 285 (path, dataId), where path is a dataset file name relative to root, 286 and dataId is a key value dictionary identifying the file. 289 while os.path.exists(root)
and not oneFound:
290 stack = [(0, root, rules, {})]
292 depth, path, rules, dataId = stack.pop()
293 if os.path.isfile(path):
299 if not os.path.exists(os.path.join(path, pc.regex)):
302 entries = os.listdir(path)
309 m = pc.regex.match(e)
314 for i, k
in enumerate(pc.keys):
315 subDataId = self.
_formatKeys[k].munge(k, m.group(i + 1), subDataId)
320 if subRules
and pc.keys:
325 if k
not in r
or subDataId[k]
in r[k]:
331 p = os.path.join(path, e)
334 stack.append((depth, p, subRules, subDataId))
336 if os.path.isfile(p):
338 yield os.path.relpath(p, root), subDataId
341 root = os.path.join(root,
"_parent")
347 dataId = dataId.copy()
350 dataId[
'raft'] = r1 +
',' + r2
351 dataId[
'raftId'] = int(r1) * 5 + int(r2)
352 elif k
in (
'sensor',
'ccd'):
354 dataId[
'sensor'] = s1 +
',' + s2
355 dataId[
'sensorNum'] = int(s1) * 3 + int(s2)
356 elif k
in (
'channel',
'amp'):
358 dataId[
'channel'] = c1 +
',' + c2
359 dataId[
'channelNum'] = int(c1) * 8 + int(c2)
360 elif k
in (
'snap',
'exposure'):
361 dataId[
'snap'] = int(v)
362 elif _keyTypes[
'lsstsim'][k] == int:
370 dataId = dataId.copy()
371 if _keyTypes[
'sdss'][k] == int:
379 dataId = dataId.copy()
381 dataId[
'ccd'] = int(v)
382 dataId[
'ccdName'] = v
384 dataId[
'amp'] = int(v)
385 dataId[
'ampName'] = v
386 elif _keyTypes[
'sdss'][k] == int:
393 'lsstsim': _mungeLsstSim,
400 """File system scanner for a dataset known to a camera mapper. 404 if not isinstance(cameraMapper, lsst.daf.butlerUtils.CameraMapper):
405 raise TypeError(
'Expecting a lsst.daf.butlerUtils.CameraMapper!')
406 if dataset
not in cameraMapper.mappings:
407 raise NotFoundError(
'Unknown dataset ' + str(dataset))
408 HfsScanner.__init__(self, cameraMapper.mappings[dataset].template)
409 camera = camera.lower()
410 if camera
not in _keyTypes:
411 raise RuntimeError(
'{} camera not supported yet'.
format(camera))
413 if k
not in _keyTypes[camera]:
414 raise RuntimeError(
'{} is not a valid dataId key for camera {}'.
format(k, camera))
415 self.
_formatKeys[k].munge = _mungeFunctions[camera]
def getMapperClass(camera)
def parseDataIdRules(ruleList, camera)
def _mungeCfht(k, v, dataId)
def _mungeInt(k, v, dataId)
def __init__(self, dataset, camera, cameraMapper)
def __init__(self, template)
def _mungeStr(k, v, dataId)
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
def _mungeLsstSim(k, v, dataId)
def walk(self, root, rules=None)
def _mungeSdss(k, v, dataId)
def __init__(self, keys, regex, simple)