LSSTApplications  19.0.0-14-gb0260a2+72efe9b372,20.0.0+7927753e06,20.0.0+8829bf0056,20.0.0+995114c5d2,20.0.0+b6f4b2abd1,20.0.0+bddc4f4cbe,20.0.0-1-g253301a+8829bf0056,20.0.0-1-g2b7511a+0d71a2d77f,20.0.0-1-g5b95a8c+7461dd0434,20.0.0-12-g321c96ea+23efe4bbff,20.0.0-16-gfab17e72e+fdf35455f6,20.0.0-2-g0070d88+ba3ffc8f0b,20.0.0-2-g4dae9ad+ee58a624b3,20.0.0-2-g61b8584+5d3db074ba,20.0.0-2-gb780d76+d529cf1a41,20.0.0-2-ged6426c+226a441f5f,20.0.0-2-gf072044+8829bf0056,20.0.0-2-gf1f7952+ee58a624b3,20.0.0-20-geae50cf+e37fec0aee,20.0.0-25-g3dcad98+544a109665,20.0.0-25-g5eafb0f+ee58a624b3,20.0.0-27-g64178ef+f1f297b00a,20.0.0-3-g4cc78c6+e0676b0dc8,20.0.0-3-g8f21e14+4fd2c12c9a,20.0.0-3-gbd60e8c+187b78b4b8,20.0.0-3-gbecbe05+48431fa087,20.0.0-38-ge4adf513+a12e1f8e37,20.0.0-4-g97dc21a+544a109665,20.0.0-4-gb4befbc+087873070b,20.0.0-4-gf910f65+5d3db074ba,20.0.0-5-gdfe0fee+199202a608,20.0.0-5-gfbfe500+d529cf1a41,20.0.0-6-g64f541c+d529cf1a41,20.0.0-6-g9a5b7a1+a1cd37312e,20.0.0-68-ga3f3dda+5fca18c6a4,20.0.0-9-g4aef684+e18322736b,w.2020.45
LSSTDataManagementBasePackage
fsScanner.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 
24 """This module provides the FsScanner class."""
25 
26 import glob
27 import os
28 import re
29 import sys
30 
31 
32 class FsScanner:
33  """Class to scan a filesystem location for paths matching a template.
34 
35  Decomposes the resulting paths into fields and passes them to a callback
36  function.
37  """
38 
39  def __init__(self, pathTemplate):
40  """Constructor. Takes the path template, which should be in the form
41  of a Python string with named format substitution specifications.
42  Such a template would be suitable for generating a path given a set of
43  fields in a dictionary. Does not handle hex (%x or %X).
44 
45  Example:
46  %(field)s/%(visit)d/%(exposure)d/raw-%(visit)d-e%(exposure)03d-c%(ccd)03d-a%(amp)03d.fits
47 
48  Note that fields may appear multiple times; the second and subsequent
49  appearances of such fields will have "_{number}" appended to them to
50  disambiguate, although it is typically assumed that they will all be
51  identical.
52 
53  Trailing brackets (and their contents) can be used to indicate which HDU from a file should
54  be used. They will not be included in the filename search.
55  """
56 
57  # Trim any trailing braces off the end of the path template.
58  if pathTemplate.endswith(']'):
59  pathTemplate = pathTemplate[0:pathTemplate.rfind('[')]
60 
61  # Change template into a globbable path specification.
62  fmt = re.compile(r'%\‍((\w+)\‍).*?([dioueEfFgGcrs])')
63 
64  self.globString = fmt.sub('*', pathTemplate)
65 
66  # Change template into a regular expression.
67  last = 0
68  self.fields = {}
69  self.reString = ""
70  n = 0
71  pos = 0
72  for m in fmt.finditer(pathTemplate):
73  fieldName = m.group(1)
74  if fieldName in self.fields:
75  fieldName += "_%d" % (n,)
76  n += 1
77 
78  prefix = pathTemplate[last:m.start(0)]
79  last = m.end(0)
80  self.reString += prefix
81 
82  if m.group(2) in 'crs':
83  fieldType = str
84  self.reString += r'(?P<' + fieldName + '>.+)'
85  elif m.group(2) in 'eEfFgG':
86  fieldType = float
87  self.reString += r'(?P<' + fieldName + r'>[\d.eE+-]+)'
88  else:
89  fieldType = int
90  self.reString += r'(?P<' + fieldName + r'>[\d+-]+)'
91 
92  self.fields[fieldName] = dict(pos=pos, fieldType=fieldType)
93  pos += 1
94 
95  self.reString += pathTemplate[last:]
96 
97  def getFields(self):
98  """Return the list of fields that will be returned from matched
99  paths, in order."""
100 
101  fieldList = ["" for i in range(len(self.fields))]
102  for f in list(self.fields.keys()):
103  fieldList[self.fields[f]['pos']] = f
104  return fieldList
105 
106  def isNumeric(self, name):
107  """Return true if the given field contains a number."""
108 
109  return self.fields[name]['fieldType'] in (float, int)
110 
111  def isInt(self, name):
112  """Return true if the given field contains an integer."""
113 
114  return self.fields[name]['fieldType'] == int
115 
116  def isFloat(self, name):
117  """Return true if the given field contains an float."""
118 
119  return self.fields[name]['fieldType'] == float
120 
121  def processPath(self, location):
122  """
123  Scan a given path location. Return info about paths that conform to the path template:
124  :param location:
125  :return: Path info: {path: {key:value ...}, ...} e.g.:
126  {'0239622/instcal0239622.fits.fz': {'visit_0': 239622, 'visit': 239622}}
127  """
128  ret = {}
129  curdir = os.getcwd()
130  os.chdir(location)
131  pathList = glob.glob(self.globString)
132  for path in pathList:
133  m = re.search(self.reString, path)
134  if m:
135  dataId = m.groupdict()
136  for f in self.fields:
137  if self.isInt(f):
138  dataId[f] = int(dataId[f])
139  elif self.isFloat(f):
140  dataId[f] = float(dataId[f])
141  ret[path] = dataId
142  else:
143  print("Warning: unmatched path: %s" % (path,), file=sys.stderr)
144  os.chdir(curdir)
145  return ret
lsst::daf::persistence.fsScanner.FsScanner.__init__
def __init__(self, pathTemplate)
Definition: fsScanner.py:39
lsst::daf::persistence.fsScanner.FsScanner.isFloat
def isFloat(self, name)
Definition: fsScanner.py:116
lsst::daf::persistence.fsScanner.FsScanner.isInt
def isInt(self, name)
Definition: fsScanner.py:111
lsst::daf::persistence.fsScanner.FsScanner.reString
reString
Definition: fsScanner.py:69
lsst::daf::persistence.fsScanner.FsScanner.getFields
def getFields(self)
Definition: fsScanner.py:97
astshim.keyMap.keyMapContinued.keys
def keys(self)
Definition: keyMapContinued.py:6
lsst::daf::persistence.fsScanner.FsScanner.fields
fields
Definition: fsScanner.py:68
list
daf::base::PropertyList * list
Definition: fits.cc:913
lsst::daf::persistence.fsScanner.FsScanner
Definition: fsScanner.py:32
lsst::daf::persistence.fsScanner.FsScanner.processPath
def processPath(self, location)
Definition: fsScanner.py:121
lsst::daf::persistence.fsScanner.FsScanner.isNumeric
def isNumeric(self, name)
Definition: fsScanner.py:106
lsst::daf::persistence.fsScanner.FsScanner.globString
globString
Definition: fsScanner.py:64