21 """Classes used in `RepoWalker` construction.
23 The objects here form a temporary tree that is pruned and then transformed
24 into a similar tree of `PathElementHandler` instances. See `BuilderNode`
25 method documentation for more information.
27 from __future__
import annotations
29 __all__ = [
"BuilderSkipInput",
"BuilderTargetInput",
"BuilderTree"]
31 from abc
import ABC, abstractmethod
42 from lsst.daf.butler
import DatasetType, DimensionUniverse, StorageClass, FormatterParameter
43 from ..translators
import TranslatorFactory
44 from .parser
import PathElementParser
45 from .scanner
import PathElementHandler, DirectoryScanner
46 from .handlers
import (IgnoreHandler, SubdirectoryHandler, SkipHandler,
51 """Abstract interface for nodes in the temporary tree that is used to
52 construct a `RepoWalker`.
56 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
57 """Attempt to prune this node and its children from the tree.
61 replacement : `BuilderNode`
62 The result of recursively pruning child nodes; often just ``self``.
63 messages : `list` [`str`]
64 Warning messages that should be logged by a parent node when a
65 matching path element is encountered, if this node is pruned.
67 If `True`, this node may be pruned from the tree (but will not
68 necessarily be - it may correspond to a path element that should
69 be skipped with siblings that should not be).
71 raise NotImplementedError()
74 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
75 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
76 ) -> PathElementHandler:
77 """Transform this node in the build tree into a corresponding
78 `PathElementHandler`, recursing to any children.
80 Must be called after `prune`.
84 parser : `PathElementParser`
85 An object that matches the path element the new handler is
86 responsible for and extracts a (partial) Gen2 data ID from it.
87 allKeys : `dict` [`str`, `type`]
88 A mapping from Gen2 data ID key to the type of its value. Will
89 contain all keys that may be extracted by the given parser, and
91 cumulativeKeys : `dict` [`str`, `type`], optional
92 A dictionary containing key strings and types for Gen2 data ID keys
93 that have been extracted from previous path elements for this
94 template, including those extracted by ``parser``.
98 handler : `PathElementHandler`
101 raise NotImplementedError()
105 """An intermediate base for `BuilderNode` classes that are provided as
106 direct inputs to a `RepoWalker`, and generally correspond to exactly one
112 The complete Gen2 template to be matched (not just the template for
114 keys : `dict` [`str`, `type`]
115 A mapping from Gen2 data ID key to the type of its value.
117 def __init__(self, template: str, keys: Dict[str, type]):
123 """The complete Gen2 template to be matched (`str`).
126 keys: Dict[str, type]
127 """A mapping from Gen2 data ID key to the type of its value
128 (`dict` [`str`, `type`]).
132 """The path elements (file or directory levels) of `template`
138 """An input to a `RepoWalker` that indicates that matched files should be
139 skipped, possibly with a warning message.
141 BuilderSkipInputs can be pruned. When they are not pruned, they build
142 `SkipHandler` instances.
147 The complete Gen2 template to be matched (not just the template for
149 keys : `dict` [`str`, `type`]
150 A mapping from Gen2 data ID key to the type of its value.
151 message : `str`, optional
152 If not `None`, a warning message that should be printed either when a
153 matching file is enountered or a directory that may contain such files
155 isForFiles : `bool`, optional
156 If `True` (default), this handler should be run on files. Otherwise it
157 should be run on directories.
159 def __init__(self, template: str, keys: Dict[str, type], message: Optional[str] =
None, *,
160 isForFiles: bool =
True):
161 super().
__init__(template=template, keys=keys)
165 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
166 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
167 ) -> PathElementHandler:
171 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
177 """An input to a `RepoWalker` that matches files that correspond to
178 datasets that we want to extract.
180 BuilderTargetInputs can never be pruned, and always build
181 `TargetFileHandler` instances.
185 datasetTypeName : `str`
186 Name of the dataset type.
188 Full Gen2 filename template.
189 keys : `dict` [`str`, `type`]
190 Dictionary that maps Gen2 data ID key to the type of its value.
191 storageClass : `StorageClass`
192 `StorageClass` for the Gen3 dataset type.
193 universe : `DimensionUniverse`
194 All candidate dimensions for the Gen3 dataset type.
195 formatter : `lsst.daf.butler.Formatter` or `str`, optional
196 A Gen 3 formatter class or fully-qualified name.
197 translatorFactory : `TranslatorFactory`
198 Object that can be used to construct data ID translators.
199 targetHandler : `PathElementHandler`, optional
200 Override target handler for this dataset type.
202 Additional keyword arguments are passed to `Translator.makeMatching`,
203 in along with ``datasetTypeName`` and ``keys``.
205 def __init__(self, *, datasetTypeName: str, template: str, keys: Dict[str, type],
206 storageClass: StorageClass, universe: DimensionUniverse,
207 formatter: FormatterParameter, translatorFactory: TranslatorFactory,
208 targetHandler: Optional[PathElementHandler] =
None,
211 template = template.split(
'[%(')[0]
212 super().
__init__(template=template, keys=keys)
213 self.
_translator = translatorFactory.makeMatching(datasetTypeName, keys, **kwargs)
215 storageClass=storageClass, universe=universe)
217 if targetHandler
is None:
218 targetHandler = TargetFileHandler
221 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
222 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
223 ) -> PathElementHandler:
228 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
230 return self, [],
False
232 datasetType: DatasetType
233 """The Gen3 dataset type extracted by the handler this object builds
234 (`lsst.daf.butler.DatasetType`).
239 """A `BuilderNode` that represents a subdirectory to be skipped,
240 created by pruning `BuilderTree` that contained only `BuilderSkipInput`
243 BuilderPrunedTrees can be pruned. When they are not pruned, they
244 build `SkipHandler` instances.
248 messages : `list` [`str`]
249 A list of warning messages to be printed when the handler produced by
250 this builder matches a subdirectory.
256 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
257 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
258 ) -> PathElementHandler:
261 return SkipHandler(parser=parser, isForFiles=
False, message=message)
263 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
269 """A `BuilderNode` that represents a collection of `BuilderInput` instances
270 that all have the same template.
272 def __init__(self, old: BuilderInput, new: BuilderInput):
274 if isinstance(old, BuilderDuplicateInputs):
281 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
282 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
283 ) -> PathElementHandler:
286 return SkipHandler(parser=parser, isForFiles=
False, message=message)
288 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
293 newChild, childMessages, toPruneChild = child.prune()
297 unprunable.append(newChild)
298 newChildren.append(newChildren)
300 if len(unprunable) == 0:
304 elif len(unprunable) == 1
and not self.
_messages:
309 return unprunable[0], [],
False
315 nested = [f
"{c.datasetType.name} (target)" for c
in unprunable]
317 self.
_messages = [f
"ambiguous match: [{', '.join(nested)}]"]
322 """A `BuilderNode` that represents a directory.
324 This is the only `BuilderNode` class that is not a leaf node. If all
325 of its children can be pruned, it is replaced by a `BuilderPrunedTree`
326 (which can then be pruned itself). It builds `SubdirectoryHandler`
327 instances when not pruned.
332 def insert(self, level: int, leaf: BuilderInput):
333 """Insert an input leaf node into the tree, recursively constructing
334 intermediate parents in order to put it at the right level.
339 The level ``self``is at in the larger tree, with zero the
340 repository root. The right level for the leaf is given by the
341 length of ``leaf.elements``.
342 leaf : `BuilderInput`
343 The leaf node to insert.
345 nextLevel = level + 1
346 element = leaf.elements[level]
347 if nextLevel == len(leaf.elements):
349 if conflict
is not None:
356 child.insert(nextLevel, leaf)
358 def fill(self, scanner: DirectoryScanner, allKeys: Dict[str, type], previousKeys: Dict[str, type], *,
359 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]):
360 """Fill a `DirectoryScanner` instance by recursively building all
365 scanner : `DirectoryScanner`
367 allKeys : `dict` [`str`, `type`]
368 Mapping from Gen2 data ID key to its value type, covering all keys
369 that could be used in any child template.
370 previousKeys : `dict` [`str`, `type`], optional
371 A dictionary containing key strings and types for Gen2 data ID keys
372 that have been extracted from previous path elements of the same
374 fileIgnoreRegEx : `re.Pattern`, optional
375 A regular expression pattern that identifies non-dataset files that
376 can be ignored, to be applied at all levels of the directory tree.
377 dirIgnoreRegEx : `re.Pattern`, optional
378 A regular expression pattern that identifies non-dataset
379 subdirectories that can be ignored, to be applied at all levels of
382 if fileIgnoreRegEx
is not None:
384 if dirIgnoreRegEx
is not None:
388 cumulativeKeys = previousKeys.copy()
389 cumulativeKeys.update(parser.keys)
390 scanner.add(child.build(parser, allKeys, cumulativeKeys, fileIgnoreRegEx=fileIgnoreRegEx,
391 dirIgnoreRegEx=dirIgnoreRegEx))
393 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
400 newChild, childMessages, toPruneChild = child.prune()
401 newChildren[template] = newChild
402 messages.extend(childMessages)
409 return self, [],
False
411 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
412 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
413 ) -> PathElementHandler:
416 self.
fill(built.scanner, allKeys, cumulativeKeys, fileIgnoreRegEx=fileIgnoreRegEx,
417 dirIgnoreRegEx=dirIgnoreRegEx)