LSSTApplications  17.0+11,17.0+34,17.0+56,17.0+57,17.0+59,17.0+7,17.0-1-g377950a+33,17.0.1-1-g114240f+2,17.0.1-1-g4d4fbc4+28,17.0.1-1-g55520dc+49,17.0.1-1-g5f4ed7e+52,17.0.1-1-g6dd7d69+17,17.0.1-1-g8de6c91+11,17.0.1-1-gb9095d2+7,17.0.1-1-ge9fec5e+5,17.0.1-1-gf4e0155+55,17.0.1-1-gfc65f5f+50,17.0.1-1-gfc6fb1f+20,17.0.1-10-g87f9f3f+1,17.0.1-11-ge9de802+16,17.0.1-16-ga14f7d5c+4,17.0.1-17-gc79d625+1,17.0.1-17-gdae4c4a+8,17.0.1-2-g26618f5+29,17.0.1-2-g54f2ebc+9,17.0.1-2-gf403422+1,17.0.1-20-g2ca2f74+6,17.0.1-23-gf3eadeb7+1,17.0.1-3-g7e86b59+39,17.0.1-3-gb5ca14a,17.0.1-3-gd08d533+40,17.0.1-30-g596af8797,17.0.1-4-g59d126d+4,17.0.1-4-gc69c472+5,17.0.1-6-g5afd9b9+4,17.0.1-7-g35889ee+1,17.0.1-7-gc7c8782+18,17.0.1-9-gc4bbfb2+3,w.2019.22
LSSTDataManagementBasePackage
butler.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2008-2015 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 
25 # -*- python -*-
26 
27 """This module defines the Butler class."""
28 from builtins import str, super
29 from past.builtins import basestring
30 from builtins import object
31 
32 import copy
33 import inspect
34 
35 import yaml
36 
37 from lsst.log import Log
38 from . import ReadProxy, ButlerSubset, ButlerDataRef, \
39  Storage, Policy, NoResults, Repository, DataId, RepositoryCfg, \
40  RepositoryArgs, listify, setify, sequencify, doImport, ButlerComposite, genericAssembler, \
41  genericDisassembler, PosixStorage, ParentsMismatch
42 
43 preinitedMapperWarning = ("Passing an instantiated mapper into " +
44  "Butler.__init__ will prevent Butler from passing " +
45  "parentRegistry or repositoryCfg information to " +
46  "the mapper, which is done only at init time. " +
47  "It is better to pass a importable string or " +
48  "class object.")
49 
50 
51 class ButlerCfg(Policy, yaml.YAMLObject):
52  """Represents a Butler configuration.
53 
54  .. warning::
55 
56  cfg is 'wet paint' and very likely to change. Use of it in production
57  code other than via the 'old butler' API is strongly discouraged.
58  """
59  yaml_tag = u"!ButlerCfg"
60 
61  def __init__(self, cls, repoCfg):
62  super().__init__({'repoCfg': repoCfg, 'cls': cls})
63 
64 
66  """Container object for repository data used by Butler
67 
68  Parameters
69  ----------
70  args : RepositoryArgs
71  The arguments that are used to find or create the RepositoryCfg.
72  role : string
73  "input", "output", or "parent", indicating why Butler loaded this repository.
74  * input: the Repository was passed as a Butler input.
75  * output: the Repository was passed as a Butler output.
76  * parent: the Repository was specified in the RepositoryCfg parents list of a readable repository.
77 
78  Attributes
79  ----------
80  cfg: RepositoryCfg
81  The configuration for the Repository.
82 
83  _cfgOrigin : string
84  "new", "existing", or "nested". Indicates the origin of the repository and its RepositoryCfg:
85  * new: it was created by this instance of Butler, and this instance of Butler will generate the
86  RepositoryCfg file.
87  * existing: it was found (via the root or cfgRoot argument)
88  * nested: the full RepositoryCfg was nested in another RepositoryCfg's parents list (this can happen
89  if parameters of an input specified by RepositoryArgs or dict does not entirely match an existing
90  RepositoryCfg).
91 
92  cfgRoot : string
93  Path or URI to the location of the RepositoryCfg file.
94 
95  repo : lsst.daf.persistence.Repository
96  The Repository class instance.
97 
98  parentRepoDatas : list of RepoData
99  The parents of this Repository, as indicated this Repository's RepositoryCfg. If this is a new
100  Repository then these are the inputs to this Butler (and will be saved in the RepositoryCfg). These
101  RepoData objects are not owned by this RepoData, these are references to peer RepoData objects in the
102  Butler's RepoDataContainer.
103 
104  isV1Repository : bool
105  True if this is an Old Butler repository. In this case the repository does not have a RepositoryCfg
106  file. It may have a _mapper file and may have a _parent symlink. It will never be treated as a "new"
107  repository, i.e. even though there is not a RepositoryCfg file, one will not be generated.
108  If False, this is a New Butler repository and is specified by RepositoryCfg file.
109 
110  tags : set
111  These are values that may be used to restrict the search of input repositories. Details are available
112  in the RepositoryArgs and DataId classes.
113 
114  role : string
115  "input", "output", or "parent", indicating why Butler loaded this repository.
116  * input: the Repository was passed as a Butler input.
117  * output: the Repository was passed as a Butler output.
118  * parent: the Repository was specified in the RepositoryCfg parents list of a readable repository.
119 
120  _repoArgs : RepositoryArgs
121  Contains the arguments that were used to specify this Repository.
122  """
123 
124  def __init__(self, args, role):
125  self.cfg = None
126  self._cfgOrigin = None
127  self.cfgRoot = None
128  self.repo = None
129  self.parentRepoDatas = []
130  self.isV1Repository = False
131  self.tags = set()
132  self.role = role
133  self.parentRegistry = None
134  self._repoArgs = args
135 
136  @property
137  def repoArgs(self):
138  return self._repoArgs
139 
140  @property
141  def repoData(self):
142  return self
143 
144  def __repr__(self):
145  return ("{}(id={},"
146  "repoArgs={}"
147  "cfg={!r},"
148  "cfgOrigin={},"
149  "cfgRoot={}," +
150  "repo={},"
151  "parentRepoDatas={}," +
152  "isV1Repository={},"
153  "role={}," +
154  "parentRegistry={})").format(
155  self.__class__.__name__,
156  id(self),
157  self.repoArgs,
158  self.cfg,
159  self.cfgOrigin,
160  self.cfgRoot,
161  self.repo,
162  [id(p) for p in self.parentRepoDatas],
163  self.isV1Repository,
164  self.role,
165  self.parentRegistry)
166 
167  def setCfg(self, cfg, origin, root, isV1Repository):
168  """Set information about the cfg into the RepoData
169 
170  Parameters
171  ----------
172  cfg : RepositoryCfg
173  The RepositoryCfg for the repo.
174  origin : string
175  'new', 'existing', or 'nested'
176  root : string
177  URI or absolute path to the location of the RepositoryCfg.yaml file.
178 
179  Returns
180  -------
181  None
182  """
183  if origin not in ('new', 'existing', 'nested'):
184  raise RuntimeError("Invalid value for origin:{}".format(origin))
185  self.cfg = cfg
186  self._cfgOrigin = origin
187  self.cfgRoot = root
188  self.isV1Repository = isV1Repository
189 
190  @property
191  def cfgOrigin(self):
192  return self._cfgOrigin
193 
194  @property
195  def isNewRepository(self):
196  return self.cfgOrigin == 'new'
197 
198  @property
199  def role(self):
200  return self._role
201 
202  @role.setter
203  def role(self, val):
204  if val not in ('input', 'output', 'parent'):
205  raise RuntimeError("Invalid value for role: {}".format(val))
206  self._role = val
207 
208  def getParentRepoDatas(self, context=None):
209  """Get the parents & grandparents etc of this repo data, in depth-first search order.
210 
211  Duplicate entries will be removed in cases where the same parent appears more than once in the parent
212  graph.
213 
214  Parameters
215  ----------
216  context : set, optional
217  Users should typically omit context and accept the default argument. Context is used to keep a set
218  of known RepoDatas when calling this function recursively, for duplicate elimination.
219 
220  Returns
221  -------
222  list of RepoData
223  A list of the parents & grandparents etc of a given repo data, in depth-first search order.
224  """
225  if context is None:
226  context = set()
227  parents = []
228  if id(self) in context:
229  return parents
230  context.add(id(self))
231  for parent in self.parentRepoDatas:
232  parents.append(parent)
233  parents += parent.getParentRepoDatas(context)
234  return parents
235 
236  def addParentRepoData(self, parentRepoData):
237  self.parentRepoDatas.append(parentRepoData)
238 
239  def addTags(self, tags):
240  self.tags = self.tags.union(tags)
241 
242 
244  """Container object for RepoData instances owned by a Butler instance.
245 
246  Parameters
247  ----------
248  repoDataList : list of RepoData
249  repoData - RepoData instance to add
250  """
251 
252  def __init__(self, repoDataList):
253  self._inputs = None
254  self._outputs = None
255  self._all = repoDataList
256  self._buildLookupLists()
257 
258  def inputs(self):
259  """Get a list of RepoData that are used to as inputs to the Butler.
260  The list is created lazily as needed, and cached.
261 
262  Returns
263  -------
264  A list of RepoData with readable repositories, in the order to be used when searching.
265  """
266  if self._inputs is None:
267  raise RuntimeError("Inputs not yet initialized.")
268  return self._inputs
269 
270  def outputs(self):
271  """Get a list of RepoData that are used to as outputs to the Butler.
272  The list is created lazily as needed, and cached.
273 
274  Returns
275  -------
276  A list of RepoData with writable repositories, in the order to be use when searching.
277  """
278  if self._outputs is None:
279  raise RuntimeError("Outputs not yet initialized.")
280  return self._outputs
281 
282  def all(self):
283  """Get a list of all RepoData that are used to as by the Butler.
284  The list is created lazily as needed, and cached.
285 
286  Returns
287  -------
288  A list of RepoData with writable repositories, in the order to be use when searching.
289  """
290  return self._all
291 
292  def __repr__(self):
293  return "%s(_inputs=%r, \n_outputs=%s, \n_all=%s)" % (
294  self.__class__.__name__,
295  self._inputs,
296  self._outputs,
297  self._all)
298 
299  def _buildLookupLists(self):
300  """Build the inputs and outputs lists based on the order of self.all()."""
301 
302  def addToList(repoData, lst):
303  """Add a repoData and each of its parents (depth first) to a list"""
304  if id(repoData) in alreadyAdded:
305  return
306  lst.append(repoData)
307  alreadyAdded.add(id(repoData))
308  for parent in repoData.parentRepoDatas:
309  addToList(parent, lst)
310 
311  if self._inputs is not None or self._outputs is not None:
312  raise RuntimeError("Lookup lists are already built.")
313  inputs = [repoData for repoData in self.all() if repoData.role == 'input']
314  outputs = [repoData for repoData in self.all() if repoData.role == 'output']
315  self._inputs = []
316  alreadyAdded = set()
317  for repoData in outputs:
318  if 'r' in repoData.repoArgs.mode:
319  addToList(repoData.repoData, self._inputs)
320  for repoData in inputs:
321  addToList(repoData.repoData, self._inputs)
322  self._outputs = [repoData.repoData for repoData in outputs]
323 
324 
325 class Butler(object):
326  """Butler provides a generic mechanism for persisting and retrieving data using mappers.
327 
328  A Butler manages a collection of datasets known as a repository. Each dataset has a type representing its
329  intended usage and a location. Note that the dataset type is not the same as the C++ or Python type of the
330  object containing the data. For example, an ExposureF object might be used to hold the data for a raw
331  image, a post-ISR image, a calibrated science image, or a difference image. These would all be different
332  dataset types.
333 
334  A Butler can produce a collection of possible values for a key (or tuples of values for multiple keys) if
335  given a partial data identifier. It can check for the existence of a file containing a dataset given its
336  type and data identifier. The Butler can then retrieve the dataset. Similarly, it can persist an object to
337  an appropriate location when given its associated data identifier.
338 
339  Note that the Butler has two more advanced features when retrieving a data set. First, the retrieval is
340  lazy. Input does not occur until the data set is actually accessed. This allows datasets to be retrieved
341  and placed on a clipboard prospectively with little cost, even if the algorithm of a stage ends up not
342  using them. Second, the Butler will call a standardization hook upon retrieval of the dataset. This
343  function, contained in the input mapper object, must perform any necessary manipulations to force the
344  retrieved object to conform to standards, including translating metadata.
345 
346  Public methods:
347 
348  __init__(self, root, mapper=None, **mapperArgs)
349 
350  defineAlias(self, alias, datasetType)
351 
352  getKeys(self, datasetType=None, level=None)
353 
354  queryMetadata(self, datasetType, format=None, dataId={}, **rest)
355 
356  datasetExists(self, datasetType, dataId={}, **rest)
357 
358  get(self, datasetType, dataId={}, immediate=False, **rest)
359 
360  put(self, obj, datasetType, dataId={}, **rest)
361 
362  subset(self, datasetType, level=None, dataId={}, **rest)
363 
364  dataRef(self, datasetType, level=None, dataId={}, **rest)
365 
366  Initialization:
367 
368  The preferred method of initialization is to use the `inputs` and `outputs` __init__ parameters. These
369  are described in the parameters section, below.
370 
371  For backward compatibility: this initialization method signature can take a posix root path, and
372  optionally a mapper class instance or class type that will be instantiated using the mapperArgs input
373  argument. However, for this to work in a backward compatible way it creates a single repository that is
374  used as both an input and an output repository. This is NOT preferred, and will likely break any
375  provenance system we have in place.
376 
377  Parameters
378  ----------
379  root : string
380  .. note:: Deprecated in 12_0
381  `root` will be removed in TBD, it is replaced by `inputs` and `outputs` for
382  multiple-repository support.
383  A file system path. Will only work with a PosixRepository.
384  mapper : string or instance
385  .. note:: Deprecated in 12_0
386  `mapper` will be removed in TBD, it is replaced by `inputs` and `outputs` for
387  multiple-repository support.
388  Provides a mapper to be used with Butler.
389  mapperArgs : dict
390  .. note:: Deprecated in 12_0
391  `mapperArgs` will be removed in TBD, it is replaced by `inputs` and `outputs` for
392  multiple-repository support.
393  Provides arguments to be passed to the mapper if the mapper input argument is a class type to be
394  instantiated by Butler.
395  inputs : RepositoryArgs, dict, or string
396  Can be a single item or a list. Provides arguments to load an existing repository (or repositories).
397  String is assumed to be a URI and is used as the cfgRoot (URI to the location of the cfg file). (Local
398  file system URI does not have to start with 'file://' and in this way can be a relative path). The
399  `RepositoryArgs` class can be used to provide more parameters with which to initialize a repository
400  (such as `mapper`, `mapperArgs`, `tags`, etc. See the `RepositoryArgs` documentation for more
401  details). A dict may be used as shorthand for a `RepositoryArgs` class instance. The dict keys must
402  match parameters to the `RepositoryArgs.__init__` function.
403  outputs : RepositoryArgs, dict, or string
404  Provides arguments to load one or more existing repositories or create new ones. The different types
405  are handled the same as for `inputs`.
406 
407  The Butler init sequence loads all of the input and output repositories.
408  This creates the object hierarchy to read from and write to them. Each
409  repository can have 0 or more parents, which also get loaded as inputs.
410  This becomes a DAG of repositories. Ultimately, Butler creates a list of
411  these Repositories in the order that they are used.
412 
413  Initialization Sequence
414  =======================
415 
416  During initialization Butler creates a Repository class instance & support structure for each object
417  passed to `inputs` and `outputs` as well as the parent repositories recorded in the `RepositoryCfg` of
418  each existing readable repository.
419 
420  This process is complex. It is explained below to shed some light on the intent of each step.
421 
422  1. Input Argument Standardization
423  ---------------------------------
424 
425  In `Butler._processInputArguments` the input arguments are verified to be legal (and a RuntimeError is
426  raised if not), and they are converted into an expected format that is used for the rest of the Butler
427  init sequence. See the docstring for `_processInputArguments`.
428 
429  2. Create RepoData Objects
430  --------------------------
431 
432  Butler uses an object, called `RepoData`, to keep track of information about each repository; each
433  repository is contained in a single `RepoData`. The attributes are explained in its docstring.
434 
435  After `_processInputArguments`, a RepoData is instantiated and put in a list for each repository in
436  `outputs` and `inputs`. This list of RepoData, the `repoDataList`, now represents all the output and input
437  repositories (but not parent repositories) that this Butler instance will use.
438 
439  3. Get `RepositoryCfg`s
440  -----------------------
441 
442  `Butler._getCfgs` gets the `RepositoryCfg` for each repository the `repoDataList`. The behavior is
443  described in the docstring.
444 
445  4. Add Parents
446  --------------
447 
448  `Butler._addParents` then considers the parents list in the `RepositoryCfg` of each `RepoData` in the
449  `repoDataList` and inserts new `RepoData` objects for each parent not represented in the proper location
450  in the `repoDataList`. Ultimately a flat list is built to represent the DAG of readable repositories
451  represented in depth-first order.
452 
453  5. Set and Verify Parents of Outputs
454  ------------------------------------
455 
456  To be able to load parent repositories when output repositories are used as inputs, the input repositories
457  are recorded as parents in the `RepositoryCfg` file of new output repositories. When an output repository
458  already exists, for consistency the Butler's inputs must match the list of parents specified the already-
459  existing output repository's `RepositoryCfg` file.
460 
461  In `Butler._setAndVerifyParentsLists`, the list of parents is recorded in the `RepositoryCfg` of new
462  repositories. For existing repositories the list of parents is compared with the `RepositoryCfg`'s parents
463  list, and if they do not match a `RuntimeError` is raised.
464 
465  6. Set the Default Mapper
466  -------------------------
467 
468  If all the input repositories use the same mapper then we can assume that mapper to be the
469  "default mapper". If there are new output repositories whose `RepositoryArgs` do not specify a mapper and
470  there is a default mapper then the new output repository will be set to use that default mapper.
471 
472  This is handled in `Butler._setDefaultMapper`.
473 
474  7. Cache References to Parent RepoDatas
475  ---------------------------------------
476 
477  In `Butler._connectParentRepoDatas`, in each `RepoData` in `repoDataList`, a list of `RepoData` object
478  references is built that matches the parents specified in that `RepoData`'s `RepositoryCfg`.
479 
480  This list is used later to find things in that repository's parents, without considering peer repository's
481  parents. (e.g. finding the registry of a parent)
482 
483  8. Set Tags
484  -----------
485 
486  Tags are described at https://ldm-463.lsst.io/v/draft/#tagging
487 
488  In `Butler._setRepoDataTags`, for each `RepoData`, the tags specified by its `RepositoryArgs` are recorded
489  in a set, and added to the tags set in each of its parents, for ease of lookup when mapping.
490 
491  9. Find Parent Registry and Instantiate RepoData
492  ------------------------------------------------
493 
494  At this point there is enough information to instantiate the `Repository` instances. There is one final
495  step before instantiating the Repository, which is to try to get a parent registry that can be used by the
496  child repository. The criteria for "can be used" is spelled out in `Butler._setParentRegistry`. However,
497  to get the registry from the parent, the parent must be instantiated. The `repoDataList`, in depth-first
498  search order, is built so that the most-dependent repositories are first, and the least dependent
499  repositories are last. So the `repoDataList` is reversed and the Repositories are instantiated in that
500  order; for each RepoData a parent registry is searched for, and then the Repository is instantiated with
501  whatever registry could be found."""
502 
503  GENERATION = 2
504  """This is a Generation 2 Butler.
505  """
506 
507  def __init__(self, root=None, mapper=None, inputs=None, outputs=None, **mapperArgs):
508  self._initArgs = {'root': root, 'mapper': mapper, 'inputs': inputs, 'outputs': outputs,
509  'mapperArgs': mapperArgs}
510 
511  self.log = Log.getLogger("daf.persistence.butler")
512 
513  inputs, outputs = self._processInputArguments(
514  root=root, mapper=mapper, inputs=inputs, outputs=outputs, **mapperArgs)
515 
516  # convert the RepoArgs into RepoData
517  inputs = [RepoData(args, 'input') for args in inputs]
518  outputs = [RepoData(args, 'output') for args in outputs]
519  repoDataList = outputs + inputs
520 
521  self._getCfgs(repoDataList)
522 
523  self._addParents(repoDataList)
524 
525  self._setAndVerifyParentsLists(repoDataList)
526 
527  self._setDefaultMapper(repoDataList)
528 
529  self._connectParentRepoDatas(repoDataList)
530 
531  self._repos = RepoDataContainer(repoDataList)
532 
533  self._setRepoDataTags()
534 
535  for repoData in repoDataList:
536  self._initRepo(repoData)
537 
538  def _initRepo(self, repoData):
539  if repoData.repo is not None:
540  # this repository may have already been initialized by its children, in which case there is
541  # nothing more to do.
542  return
543  for parentRepoData in repoData.parentRepoDatas:
544  if parentRepoData.cfg.mapper != repoData.cfg.mapper:
545  continue
546  if parentRepoData.repo is None:
547  self._initRepo(parentRepoData)
548  parentRegistry = parentRepoData.repo.getRegistry()
549  repoData.parentRegistry = parentRegistry if parentRegistry else parentRepoData.parentRegistry
550  if repoData.parentRegistry:
551  break
552  repoData.repo = Repository(repoData)
553 
554  def _processInputArguments(self, root=None, mapper=None, inputs=None, outputs=None, **mapperArgs):
555  """Process, verify, and standardize the input arguments.
556  * Inputs can not be for Old Butler (root, mapper, mapperArgs) AND New Butler (inputs, outputs)
557  `root`, `mapper`, and `mapperArgs` are Old Butler init API.
558  `inputs` and `outputs` are New Butler init API.
559  Old Butler and New Butler init API may not be mixed, Butler may be initialized with only the Old
560  arguments or the New arguments.
561  * Verify that if there is a readable output that there is exactly one output. (This restriction is in
562  place because all readable repositories must be parents of writable repositories, and for
563  consistency the DAG of readable repositories must always be the same. Keeping the list of parents
564  becomes very complicated in the presence of multiple readable output repositories. It is better to
565  only write to output repositories, and then create a new Butler instance and use the outputs as
566  inputs, and write to new output repositories.)
567  * Make a copy of inputs & outputs so they may be modified without changing the passed-in arguments.
568  * Convert any input/output values that are URI strings to RepositoryArgs.
569  * Listify inputs & outputs.
570  * Set default RW mode on inputs & outputs as needed.
571 
572  Parameters
573  ----------
574  Same as Butler.__init__
575 
576  Returns
577  -------
578  (list of RepositoryArgs, list of RepositoryArgs)
579  First item is a list to use as inputs.
580  Second item is a list to use as outputs.
581 
582  Raises
583  ------
584  RuntimeError
585  If Old Butler and New Butler arguments are both used this will raise.
586  If an output is readable there is more than one output this will raise.
587  """
588  # inputs and outputs may be modified, do not change the external value.
589  inputs = copy.deepcopy(inputs)
590  outputs = copy.deepcopy(outputs)
591 
592  isV1Args = inputs is None and outputs is None
593  if isV1Args:
594  inputs, outputs = self._convertV1Args(root=root,
595  mapper=mapper,
596  mapperArgs=mapperArgs or None)
597  elif root or mapper or mapperArgs:
598  raise RuntimeError(
599  'Butler version 1 API (root, mapper, **mapperArgs) may ' +
600  'not be used with version 2 API (inputs, outputs)')
602 
603  self.storage = Storage()
604 
605  # make sure inputs and outputs are lists, and if list items are a string convert it RepositoryArgs.
606  inputs = listify(inputs)
607  outputs = listify(outputs)
608  inputs = [RepositoryArgs(cfgRoot=args)
609  if not isinstance(args, RepositoryArgs) else args for args in inputs]
610  outputs = [RepositoryArgs(cfgRoot=args)
611  if not isinstance(args, RepositoryArgs) else args for args in outputs]
612  # Set the default value of inputs & outputs, verify the required values ('r' for inputs, 'w' for
613  # outputs) and remove the 'w' from inputs if needed.
614  for args in inputs:
615  if args.mode is None:
616  args.mode = 'r'
617  elif 'rw' == args.mode:
618  args.mode = 'r'
619  elif 'r' != args.mode:
620  raise RuntimeError("The mode of an input should be readable.")
621  for args in outputs:
622  if args.mode is None:
623  args.mode = 'w'
624  elif 'w' not in args.mode:
625  raise RuntimeError("The mode of an output should be writable.")
626  # check for class instances in args.mapper (not allowed)
627  for args in inputs + outputs:
628  if (args.mapper and not isinstance(args.mapper, basestring) and
629  not inspect.isclass(args.mapper)):
630  self.log.warn(preinitedMapperWarning)
631  # if the output is readable, there must be only one output:
632  for o in outputs:
633  if 'r' in o.mode:
634  if len(outputs) > 1:
635  raise RuntimeError("Butler does not support multiple output repositories if any of the "
636  "outputs are readable.")
637 
638  # Handle the case where the output is readable and is also passed in as one of the inputs by removing
639  # the input. This supports a legacy use case in pipe_tasks where the input is also passed as the
640  # output, to the command line parser.
641  def inputIsInOutputs(inputArgs, outputArgsList):
642  for o in outputArgsList:
643  if ('r' in o.mode and
644  o.root == inputArgs.root and
645  o.mapper == inputArgs.mapper and
646  o.mapperArgs == inputArgs.mapperArgs and
647  o.tags == inputArgs.tags and
648  o.policy == inputArgs.policy):
649  self.log.debug(("Input repositoryArgs {} is also listed in outputs as readable; " +
650  "throwing away the input.").format(inputArgs))
651  return True
652  return False
653 
654  inputs = [args for args in inputs if not inputIsInOutputs(args, outputs)]
655  return inputs, outputs
656 
657  @staticmethod
658  def _getParentVal(repoData):
659  """Get the value of this repoData as it should appear in the parents
660  list of other repositories"""
661  if repoData.isV1Repository:
662  return repoData.cfg
663  if repoData.cfgOrigin == 'nested':
664  return repoData.cfg
665  else:
666  return repoData.cfg.root
667 
668  @staticmethod
669  def _getParents(ofRepoData, repoInfo):
670  """Create a parents list of repoData from inputs and (readable) outputs."""
671  parents = []
672  # get the parents list of repoData:
673  for repoData in repoInfo:
674  if repoData is ofRepoData:
675  continue
676  if 'r' not in repoData.repoArgs.mode:
677  continue
678  parents.append(Butler._getParentVal(repoData))
679  return parents
680 
681  @staticmethod
682  def _getOldButlerRepositoryCfg(repositoryArgs):
683  if not Storage.isPosix(repositoryArgs.cfgRoot):
684  return None
685  if not PosixStorage.v1RepoExists(repositoryArgs.cfgRoot):
686  return None
687  if not repositoryArgs.mapper:
688  repositoryArgs.mapper = PosixStorage.getMapperClass(repositoryArgs.cfgRoot)
689  cfg = RepositoryCfg.makeFromArgs(repositoryArgs)
690  parent = PosixStorage.getParentSymlinkPath(repositoryArgs.cfgRoot)
691  if parent:
692  parent = Butler._getOldButlerRepositoryCfg(RepositoryArgs(cfgRoot=parent, mode='r'))
693  if parent is not None:
694  cfg.addParents([parent])
695  return cfg
696 
697  def _getRepositoryCfg(self, repositoryArgs):
698  """Try to get a repository from the location described by cfgRoot.
699 
700  Parameters
701  ----------
702  repositoryArgs : RepositoryArgs or string
703  Provides arguments to load an existing repository (or repositories). String is assumed to be a URI
704  and is used as the cfgRoot (URI to the location of the cfg file).
705 
706  Returned
707  --------
708  (RepositoryCfg or None, bool)
709  The RepositoryCfg, or None if one cannot be found, and True if the RepositoryCfg was created by
710  reading an Old Butler repository, or False if it is a New Butler Repository.
711  """
712  if not isinstance(repositoryArgs, RepositoryArgs):
713  repositoryArgs = RepositoryArgs(cfgRoot=repositoryArgs, mode='r')
714 
715  cfg = self.storage.getRepositoryCfg(repositoryArgs.cfgRoot)
716  isOldButlerRepository = False
717  if cfg is None:
718  cfg = Butler._getOldButlerRepositoryCfg(repositoryArgs)
719  if cfg is not None:
720  isOldButlerRepository = True
721  return cfg, isOldButlerRepository
722 
723  def _getCfgs(self, repoDataList):
724  """Get or make a RepositoryCfg for each RepoData, and add the cfg to the RepoData.
725  If the cfg exists, compare values. If values match then use the cfg as an "existing" cfg. If the
726  values do not match, use the cfg as a "nested" cfg.
727  If the cfg does not exist, the RepositoryArgs must be for a writable repository.
728 
729  Parameters
730  ----------
731  repoDataList : list of RepoData
732  The RepoData that are output and inputs of this Butler
733 
734  Raises
735  ------
736  RuntimeError
737  If the passed-in RepositoryArgs indicate an existing repository but other cfg parameters in those
738  RepositoryArgs don't
739  match the existing repository's cfg a RuntimeError will be raised.
740  """
741  def cfgMatchesArgs(args, cfg):
742  """Test if there are any values in an RepositoryArgs that conflict with the values in a cfg"""
743  if args.mapper is not None and cfg.mapper != args.mapper:
744  return False
745  if args.mapperArgs is not None and cfg.mapperArgs != args.mapperArgs:
746  return False
747  if args.policy is not None and cfg.policy != args.policy:
748  return False
749  return True
750 
751  for repoData in repoDataList:
752  cfg, isOldButlerRepository = self._getRepositoryCfg(repoData.repoArgs)
753  if cfg is None:
754  if 'w' not in repoData.repoArgs.mode:
755  raise RuntimeError(
756  "No cfg found for read-only input repository at {}".format(repoData.repoArgs.cfgRoot))
757  repoData.setCfg(cfg=RepositoryCfg.makeFromArgs(repoData.repoArgs),
758  origin='new',
759  root=repoData.repoArgs.cfgRoot,
760  isV1Repository=isOldButlerRepository)
761  else:
762 
763  # This is a hack fix for an issue introduced by DM-11284; Old Butler parent repositories used
764  # to be stored as a path to the repository in the parents list and it was changed so that the
765  # whole RepositoryCfg, that described the Old Butler repository (including the mapperArgs that
766  # were used with it), was recorded as a "nested" repository cfg. That checkin did not account
767  # for the fact that there were repositoryCfg.yaml files in the world with only the path to
768  # Old Butler repositories in the parents list.
769  if cfg.parents:
770  for i, parent in enumerate(cfg.parents):
771  if isinstance(parent, RepositoryCfg):
772  continue
773  parentCfg, parentIsOldButlerRepository = self._getRepositoryCfg(parent)
774  if parentIsOldButlerRepository:
775  parentCfg.mapperArgs = cfg.mapperArgs
776  self.log.info(("Butler is replacing an Old Butler parent repository path '{}' "
777  "found in the parents list of a New Butler repositoryCfg: {} "
778  "with a repositoryCfg that includes the child repository's "
779  "mapperArgs: {}. This affects the instantiated RepositoryCfg "
780  "but does not change the persisted child repositoryCfg.yaml file."
781  ).format(parent, cfg, parentCfg))
782  cfg._parents[i] = cfg._normalizeParents(cfg.root, [parentCfg])[0]
783 
784  if 'w' in repoData.repoArgs.mode:
785  # if it's an output repository, the RepositoryArgs must match the existing cfg.
786  if not cfgMatchesArgs(repoData.repoArgs, cfg):
787  raise RuntimeError(("The RepositoryArgs and RepositoryCfg must match for writable " +
788  "repositories, RepositoryCfg:{}, RepositoryArgs:{}").format(
789  cfg, repoData.repoArgs))
790  repoData.setCfg(cfg=cfg, origin='existing', root=repoData.repoArgs.cfgRoot,
791  isV1Repository=isOldButlerRepository)
792  else:
793  # if it's an input repository, the cfg can overwrite the in-repo cfg.
794  if cfgMatchesArgs(repoData.repoArgs, cfg):
795  repoData.setCfg(cfg=cfg, origin='existing', root=repoData.repoArgs.cfgRoot,
796  isV1Repository=isOldButlerRepository)
797  else:
798  repoData.setCfg(cfg=cfg, origin='nested', root=None,
799  isV1Repository=isOldButlerRepository)
800 
801  def _addParents(self, repoDataList):
802  """For each repoData in the input list, see if its parents are the next items in the list, and if not
803  add the parent, so that the repoDataList includes parents and is in order to operate depth-first 0..n.
804 
805  Parameters
806  ----------
807  repoDataList : list of RepoData
808  The RepoData for the Butler outputs + inputs.
809 
810  Raises
811  ------
812  RuntimeError
813  Raised if a RepositoryCfg can not be found at a location where a parent repository should be.
814  """
815  repoDataIdx = 0
816  while True:
817  if repoDataIdx == len(repoDataList):
818  break
819  repoData = repoDataList[repoDataIdx]
820  if 'r' not in repoData.repoArgs.mode:
821  repoDataIdx += 1
822  continue # the repoData only needs parents if it's readable.
823  if repoData.isNewRepository:
824  repoDataIdx += 1
825  continue # if it's new the parents will be the inputs of this butler.
826  if repoData.cfg.parents is None:
827  repoDataIdx += 1
828  continue # if there are no parents then there's nothing to do.
829  for repoParentIdx, repoParent in enumerate(repoData.cfg.parents):
830  parentIdxInRepoDataList = repoDataIdx + repoParentIdx + 1
831  if not isinstance(repoParent, RepositoryCfg):
832  repoParentCfg, isOldButlerRepository = self._getRepositoryCfg(repoParent)
833  if repoParentCfg is not None:
834  cfgOrigin = 'existing'
835  else:
836  isOldButlerRepository = False
837  repoParentCfg = repoParent
838  cfgOrigin = 'nested'
839  if (parentIdxInRepoDataList < len(repoDataList) and
840  repoDataList[parentIdxInRepoDataList].cfg == repoParentCfg):
841  continue
842  args = RepositoryArgs(cfgRoot=repoParentCfg.root, mode='r')
843  role = 'input' if repoData.role == 'output' else 'parent'
844  newRepoInfo = RepoData(args, role)
845  newRepoInfo.repoData.setCfg(cfg=repoParentCfg, origin=cfgOrigin, root=args.cfgRoot,
846  isV1Repository=isOldButlerRepository)
847  repoDataList.insert(parentIdxInRepoDataList, newRepoInfo)
848  repoDataIdx += 1
849 
850  def _setAndVerifyParentsLists(self, repoDataList):
851  """Make a list of all the input repositories of this Butler, these are the parents of the outputs.
852  For new output repositories, set the parents in the RepositoryCfg. For existing output repositories
853  verify that the RepositoryCfg's parents match the parents list.
854 
855  Parameters
856  ----------
857  repoDataList : list of RepoData
858  All the RepoDatas loaded by this butler, in search order.
859 
860  Raises
861  ------
862  RuntimeError
863  If an existing output repository is loaded and its parents do not match the parents of this Butler
864  an error will be raised.
865  """
866  def getIOParents(ofRepoData, repoDataList):
867  """make a parents list for repo in `ofRepoData` that is comprised of inputs and readable
868  outputs (not parents-of-parents) of this butler"""
869  parents = []
870  for repoData in repoDataList:
871  if repoData.role == 'parent':
872  continue
873  if repoData is ofRepoData:
874  continue
875  if repoData.role == 'output':
876  if 'r' in repoData.repoArgs.mode:
877  raise RuntimeError("If an output is readable it must be the only output.")
878  # and if this is the only output, this should have continued in
879  # "if repoData is ofRepoData"
880  continue
881  parents.append(self._getParentVal(repoData))
882  return parents
883 
884  for repoData in repoDataList:
885  if repoData.role != 'output':
886  continue
887  parents = getIOParents(repoData, repoDataList)
888  # if repoData is new, add the parent RepositoryCfgs to it.
889  if repoData.cfgOrigin == 'new':
890  repoData.cfg.addParents(parents)
891  elif repoData.cfgOrigin in ('existing', 'nested'):
892  if repoData.cfg.parents != parents:
893  try:
894  repoData.cfg.extendParents(parents)
895  except ParentsMismatch as e:
896  raise RuntimeError(("Inputs of this Butler:{} do not match parents of existing " +
897  "writable cfg:{} (ParentMismatch exception: {}").format(
898  parents, repoData.cfg.parents, e))
899 
900  def _setDefaultMapper(self, repoDataList):
901  """Establish a default mapper if there is one and assign it to outputs that do not have a mapper
902  assigned.
903 
904  If all inputs have the same mapper it will be used as the default mapper.
905 
906  Parameters
907  ----------
908  repoDataList : list of RepoData
909  All the RepoDatas loaded by this butler, in search order.
910 
911  Raises
912  ------
913  RuntimeError
914  If a default mapper can not be established and there is an output that does not have a mapper.
915  """
916  needyOutputs = [rd for rd in repoDataList if rd.role == 'output' and rd.cfg.mapper is None]
917  if len(needyOutputs) == 0:
918  return
919  mappers = set([rd.cfg.mapper for rd in repoDataList if rd.role == 'input'])
920  if len(mappers) != 1:
921  inputs = [rd for rd in repoDataList if rd.role == 'input']
922  raise RuntimeError(
923  ("No default mapper could be established from inputs:{} and no mapper specified " +
924  "for outputs:{}").format(inputs, needyOutputs))
925  defaultMapper = mappers.pop()
926  for repoData in needyOutputs:
927  repoData.cfg.mapper = defaultMapper
928 
929  def _connectParentRepoDatas(self, repoDataList):
930  """For each RepoData in repoDataList, find its parent in the repoDataList and cache a reference to it.
931 
932  Parameters
933  ----------
934  repoDataList : list of RepoData
935  All the RepoDatas loaded by this butler, in search order.
936 
937  Raises
938  ------
939  RuntimeError
940  When a parent is listed in the parents list but not found in the repoDataList. This is not
941  expected to ever happen and would indicate an internal Butler error.
942  """
943  for repoData in repoDataList:
944  for parent in repoData.cfg.parents:
945  parentToAdd = None
946  for otherRepoData in repoDataList:
947  if isinstance(parent, RepositoryCfg):
948  if otherRepoData.repoData.repoData.cfg == parent:
949  parentToAdd = otherRepoData.repoData
950  break
951  elif otherRepoData.repoData.cfg.root == parent:
952  parentToAdd = otherRepoData.repoData
953  break
954  if parentToAdd is None:
955  raise RuntimeError(
956  "Could not find a parent matching {} to add to {}".format(parent, repoData))
957  repoData.addParentRepoData(parentToAdd)
958 
959  @staticmethod
960  def _getParentRepoData(parent, repoDataList):
961  """get a parent RepoData from a cfg from a list of RepoData
962 
963  Parameters
964  ----------
965  parent : string or RepositoryCfg
966  cfgRoot of a repo or a cfg that describes the repo
967  repoDataList : list of RepoData
968  list to search in
969 
970  Returns
971  -------
972  RepoData or None
973  A RepoData if one can be found, else None
974  """
975  repoData = None
976  for otherRepoData in repoDataList:
977  if isinstance(parent, RepositoryCfg):
978  if otherRepoData.cfg == parent:
979  repoData = otherRepoData
980  break
981  elif otherRepoData.cfg.root == parent:
982  repoData = otherRepoData
983  break
984  return repoData
985 
986  def _setRepoDataTags(self):
987  """Set the tags from each repoArgs into all its parent repoArgs so that they can be included in tagged
988  searches."""
989  def setTags(repoData, tags, context):
990  if id(repoData) in context:
991  return
992  repoData.addTags(tags)
993  context.add(id(repoData))
994  for parentRepoData in repoData.parentRepoDatas:
995  setTags(parentRepoData, tags, context)
996  for repoData in self._repos.outputs() + self._repos.inputs():
997  setTags(repoData.repoData, repoData.repoArgs.tags, set())
998 
999  def _convertV1Args(self, root, mapper, mapperArgs):
1000  """Convert Old Butler RepositoryArgs (root, mapper, mapperArgs) to New Butler RepositoryArgs
1001  (inputs, outputs)
1002 
1003  Parameters
1004  ----------
1005  root : string
1006  Posix path to repository root
1007  mapper : class, class instance, or string
1008  Instantiated class, a class object to be instantiated, or a string that refers to a class that
1009  can be imported & used as the mapper.
1010  mapperArgs : dict
1011  RepositoryArgs & their values used when instantiating the mapper.
1012 
1013  Returns
1014  -------
1015  tuple
1016  (inputs, outputs) - values to be used for inputs and outputs in Butler.__init__
1017  """
1018  if (mapper and not isinstance(mapper, basestring) and
1019  not inspect.isclass(mapper)):
1020  self.log.warn(preinitedMapperWarning)
1021  inputs = None
1022  if root is None:
1023  if hasattr(mapper, 'root'):
1024  # in legacy repositories, the mapper may be given the root directly.
1025  root = mapper.root
1026  else:
1027  # in the past root="None" could be used to mean root='.'
1028  root = '.'
1029  outputs = RepositoryArgs(mode='rw',
1030  root=root,
1031  mapper=mapper,
1032  mapperArgs=mapperArgs)
1033  return inputs, outputs
1034 
1035  def __repr__(self):
1036  return 'Butler(datasetTypeAliasDict=%s, repos=%s)' % (
1037  self.datasetTypeAliasDict, self._repos)
1038 
1039  def _getDefaultMapper(self):
1040 
1041  """Get the default mapper. Currently this means if all the repositories use exactly the same mapper,
1042  that mapper may be considered the default.
1043 
1044  This definition may be changing; mappers may be able to exclude themselves as candidates for default,
1045  and they may nominate a different mapper instead. Also, we may not want to look at *all* the
1046  repositories, but only a depth-first search on each of the input & output repositories, and use the
1047  first-found mapper for each of those. TBD.
1048 
1049  Parameters
1050  ----------
1051  inputs : TYPE
1052  Description
1053 
1054  Returns
1055  -------
1056  Mapper class or None
1057  Returns the class type of the default mapper, or None if a default
1058  mapper can not be determined.
1059  """
1060  defaultMapper = None
1061 
1062  for inputRepoData in self._repos.inputs():
1063  mapper = None
1064  if inputRepoData.cfg.mapper is not None:
1065  mapper = inputRepoData.cfg.mapper
1066  # if the mapper is:
1067  # * a string, import it.
1068  # * a class instance, get its class type
1069  # * a class, do nothing; use it
1070  if isinstance(mapper, basestring):
1071  mapper = doImport(mapper)
1072  elif not inspect.isclass(mapper):
1073  mapper = mapper.__class__
1074  # If no mapper has been found, note the first found mapper.
1075  # Then, if a mapper has been found and each next mapper matches it,
1076  # continue looking for mappers.
1077  # If a mapper has been found and another non-matching mapper is
1078  # found then we have no default, return None.
1079  if defaultMapper is None:
1080  defaultMapper = mapper
1081  elif mapper == defaultMapper:
1082  continue
1083  elif mapper is not None:
1084  return None
1085  return defaultMapper
1086 
1087  def _assignDefaultMapper(self, defaultMapper):
1088  for repoData in self._repos.all().values():
1089  if repoData.cfg.mapper is None and (repoData.isNewRepository or repoData.isV1Repository):
1090  if defaultMapper is None:
1091  raise RuntimeError(
1092  "No mapper specified for %s and no default mapper could be determined." %
1093  repoData.args)
1094  repoData.cfg.mapper = defaultMapper
1095 
1096  @staticmethod
1097  def getMapperClass(root):
1098  """posix-only; gets the mapper class at the path specified by root (if a file _mapper can be found at
1099  that location or in a parent location.
1100 
1101  As we abstract the storage and support different types of storage locations this method will be
1102  moved entirely into Butler Access, or made more dynamic, and the API will very likely change."""
1103  return Storage.getMapperClass(root)
1104 
1105  def defineAlias(self, alias, datasetType):
1106  """Register an alias that will be substituted in datasetTypes.
1107 
1108  Parameters
1109  ----------
1110  alias - string
1111  The alias keyword. It may start with @ or not. It may not contain @ except as the first character.
1112  datasetType - string
1113  The string that will be substituted when @alias is passed into datasetType. It may not contain '@'
1114  """
1115  # verify formatting of alias:
1116  # it can have '@' as the first character (if not it's okay, we will add it) or not at all.
1117  atLoc = alias.rfind('@')
1118  if atLoc == -1:
1119  alias = "@" + str(alias)
1120  elif atLoc > 0:
1121  raise RuntimeError("Badly formatted alias string: %s" % (alias,))
1122 
1123  # verify that datasetType does not contain '@'
1124  if datasetType.count('@') != 0:
1125  raise RuntimeError("Badly formatted type string: %s" % (datasetType))
1126 
1127  # verify that the alias keyword does not start with another alias keyword,
1128  # and vice versa
1129  for key in self.datasetTypeAliasDict:
1130  if key.startswith(alias) or alias.startswith(key):
1131  raise RuntimeError("Alias: %s overlaps with existing alias: %s" % (alias, key))
1132 
1133  self.datasetTypeAliasDict[alias] = datasetType
1134 
1135  def getKeys(self, datasetType=None, level=None, tag=None):
1136  """Get the valid data id keys at or above the given level of hierarchy for the dataset type or the
1137  entire collection if None. The dict values are the basic Python types corresponding to the keys (int,
1138  float, string).
1139 
1140  Parameters
1141  ----------
1142  datasetType - string
1143  The type of dataset to get keys for, entire collection if None.
1144  level - string
1145  The hierarchy level to descend to. None if it should not be restricted. Use an empty string if the
1146  mapper should lookup the default level.
1147  tags - any, or list of any
1148  Any object that can be tested to be the same as the tag in a dataId passed into butler input
1149  functions. Applies only to input repositories: If tag is specified by the dataId then the repo
1150  will only be read from used if the tag in the dataId matches a tag used for that repository.
1151 
1152  Returns
1153  -------
1154  Returns a dict. The dict keys are the valid data id keys at or above the given level of hierarchy for
1155  the dataset type or the entire collection if None. The dict values are the basic Python types
1156  corresponding to the keys (int, float, string).
1157  """
1158  datasetType = self._resolveDatasetTypeAlias(datasetType)
1159 
1160  keys = None
1161  tag = setify(tag)
1162  for repoData in self._repos.inputs():
1163  if not tag or len(tag.intersection(repoData.tags)) > 0:
1164  keys = repoData.repo.getKeys(datasetType, level)
1165  # An empty dict is a valid "found" condition for keys. The only value for keys that should
1166  # cause the search to continue is None
1167  if keys is not None:
1168  break
1169  return keys
1170 
1171  def queryMetadata(self, datasetType, format, dataId={}, **rest):
1172  """Returns the valid values for one or more keys when given a partial
1173  input collection data id.
1174 
1175  Parameters
1176  ----------
1177  datasetType - string
1178  The type of dataset to inquire about.
1179  format - str, tuple
1180  Key or tuple of keys to be returned.
1181  dataId - DataId, dict
1182  The partial data id.
1183  **rest -
1184  Keyword arguments for the partial data id.
1185 
1186  Returns
1187  -------
1188  A list of valid values or tuples of valid values as specified by the
1189  format.
1190  """
1191 
1192  datasetType = self._resolveDatasetTypeAlias(datasetType)
1193  dataId = DataId(dataId)
1194  dataId.update(**rest)
1195  format = sequencify(format)
1196 
1197  tuples = None
1198  for repoData in self._repos.inputs():
1199  if not dataId.tag or len(dataId.tag.intersection(repoData.tags)) > 0:
1200  tuples = repoData.repo.queryMetadata(datasetType, format, dataId)
1201  if tuples:
1202  break
1203 
1204  if not tuples:
1205  return []
1206 
1207  if len(format) == 1:
1208  ret = []
1209  for x in tuples:
1210  try:
1211  ret.append(x[0])
1212  except TypeError:
1213  ret.append(x)
1214  return ret
1215 
1216  return tuples
1217 
1218  def datasetExists(self, datasetType, dataId={}, write=False, **rest):
1219  """Determines if a dataset file exists.
1220 
1221  Parameters
1222  ----------
1223  datasetType - string
1224  The type of dataset to inquire about.
1225  dataId - DataId, dict
1226  The data id of the dataset.
1227  write - bool
1228  If True, look only in locations where the dataset could be written,
1229  and return True only if it is present in all of them.
1230  **rest keyword arguments for the data id.
1231 
1232  Returns
1233  -------
1234  exists - bool
1235  True if the dataset exists or is non-file-based.
1236  """
1237  datasetType = self._resolveDatasetTypeAlias(datasetType)
1238  dataId = DataId(dataId)
1239  dataId.update(**rest)
1240  locations = self._locate(datasetType, dataId, write=write)
1241  if not write: # when write=False, locations is not a sequence
1242  if locations is None:
1243  return False
1244  locations = [locations]
1245 
1246  if not locations: # empty list
1247  return False
1248 
1249  for location in locations:
1250  # If the location is a ButlerComposite (as opposed to a ButlerLocation),
1251  # verify the component objects exist.
1252  if isinstance(location, ButlerComposite):
1253  for name, componentInfo in location.componentInfo.items():
1254  if componentInfo.subset:
1255  subset = self.subset(datasetType=componentInfo.datasetType, dataId=location.dataId)
1256  exists = all([obj.datasetExists() for obj in subset])
1257  else:
1258  exists = self.datasetExists(componentInfo.datasetType, location.dataId)
1259  if exists is False:
1260  return False
1261  else:
1262  if not location.repository.exists(location):
1263  return False
1264  return True
1265 
1266  def _locate(self, datasetType, dataId, write):
1267  """Get one or more ButlerLocations and/or ButlercComposites.
1268 
1269  Parameters
1270  ----------
1271  datasetType : string
1272  The datasetType that is being searched for. The datasetType may be followed by a dot and
1273  a component name (component names are specified in the policy). IE datasetType.componentName
1274 
1275  dataId : dict or DataId class instance
1276  The dataId
1277 
1278  write : bool
1279  True if this is a search to write an object. False if it is a search to read an object. This
1280  affects what type (an object or a container) is returned.
1281 
1282  Returns
1283  -------
1284  If write is False, will return either a single object or None. If write is True, will return a list
1285  (which may be empty)
1286  """
1287  repos = self._repos.outputs() if write else self._repos.inputs()
1288  locations = []
1289  for repoData in repos:
1290  # enforce dataId & repository tags when reading:
1291  if not write and dataId.tag and len(dataId.tag.intersection(repoData.tags)) == 0:
1292  continue
1293  components = datasetType.split('.')
1294  datasetType = components[0]
1295  components = components[1:]
1296  try:
1297  location = repoData.repo.map(datasetType, dataId, write=write)
1298  except NoResults:
1299  continue
1300  if location is None:
1301  continue
1302  location.datasetType = datasetType # todo is there a better way than monkey patching here?
1303  if len(components) > 0:
1304  if not isinstance(location, ButlerComposite):
1305  raise RuntimeError("The location for a dotted datasetType must be a composite.")
1306  # replace the first component name with the datasetType
1307  components[0] = location.componentInfo[components[0]].datasetType
1308  # join components back into a dot-delimited string
1309  datasetType = '.'.join(components)
1310  location = self._locate(datasetType, dataId, write)
1311  # if a component location is not found, we can not continue with this repo, move to next repo.
1312  if location is None:
1313  break
1314  # if reading, only one location is desired.
1315  if location:
1316  if not write:
1317  # If there is a bypass function for this dataset type, we can't test to see if the object
1318  # exists in storage, because the bypass function may not actually use the location
1319  # according to the template. Instead, execute the bypass function and include its results
1320  # in the bypass attribute of the location. The bypass function may fail for any reason,
1321  # the most common case being that a file does not exist. If it raises an exception
1322  # indicating such, we ignore the bypass function and proceed as though it does not exist.
1323  if hasattr(location.mapper, "bypass_" + location.datasetType):
1324  bypass = self._getBypassFunc(location, dataId)
1325  try:
1326  bypass = bypass()
1327  location.bypass = bypass
1328  except (NoResults, IOError):
1329  self.log.debug("Continuing dataset search while evaluating "
1330  "bypass function for Dataset type:{} Data ID:{} at "
1331  "location {}".format(datasetType, dataId, location))
1332  # If a location was found but the location does not exist, keep looking in input
1333  # repositories (the registry may have had enough data for a lookup even thought the object
1334  # exists in a different repository.)
1335  if (isinstance(location, ButlerComposite) or hasattr(location, 'bypass') or
1336  location.repository.exists(location)):
1337  return location
1338  else:
1339  try:
1340  locations.extend(location)
1341  except TypeError:
1342  locations.append(location)
1343  if not write:
1344  return None
1345  return locations
1346 
1347  @staticmethod
1348  def _getBypassFunc(location, dataId):
1349  pythonType = location.getPythonType()
1350  if pythonType is not None:
1351  if isinstance(pythonType, basestring):
1352  pythonType = doImport(pythonType)
1353  bypassFunc = getattr(location.mapper, "bypass_" + location.datasetType)
1354  return lambda: bypassFunc(location.datasetType, pythonType, location, dataId)
1355 
1356  def get(self, datasetType, dataId=None, immediate=True, **rest):
1357  """Retrieves a dataset given an input collection data id.
1358 
1359  Parameters
1360  ----------
1361  datasetType - string
1362  The type of dataset to retrieve.
1363  dataId - dict
1364  The data id.
1365  immediate - bool
1366  If False use a proxy for delayed loading.
1367  **rest
1368  keyword arguments for the data id.
1369 
1370  Returns
1371  -------
1372  An object retrieved from the dataset (or a proxy for one).
1373  """
1374  datasetType = self._resolveDatasetTypeAlias(datasetType)
1375  dataId = DataId(dataId)
1376  dataId.update(**rest)
1377 
1378  location = self._locate(datasetType, dataId, write=False)
1379  if location is None:
1380  raise NoResults("No locations for get:", datasetType, dataId)
1381  self.log.debug("Get type=%s keys=%s from %s", datasetType, dataId, str(location))
1382 
1383  if hasattr(location, 'bypass'):
1384  # this type loader block should get moved into a helper someplace, and duplications removed.
1385  def callback():
1386  return location.bypass
1387  else:
1388  def callback():
1389  return self._read(location)
1390  if location.mapper.canStandardize(location.datasetType):
1391  innerCallback = callback
1392 
1393  def callback():
1394  return location.mapper.standardize(location.datasetType, innerCallback(), dataId)
1395  if immediate:
1396  return callback()
1397  return ReadProxy(callback)
1398 
1399  def put(self, obj, datasetType, dataId={}, doBackup=False, **rest):
1400  """Persists a dataset given an output collection data id.
1401 
1402  Parameters
1403  ----------
1404  obj -
1405  The object to persist.
1406  datasetType - string
1407  The type of dataset to persist.
1408  dataId - dict
1409  The data id.
1410  doBackup - bool
1411  If True, rename existing instead of overwriting.
1412  WARNING: Setting doBackup=True is not safe for parallel processing, as it may be subject to race
1413  conditions.
1414  **rest
1415  Keyword arguments for the data id.
1416  """
1417  datasetType = self._resolveDatasetTypeAlias(datasetType)
1418  dataId = DataId(dataId)
1419  dataId.update(**rest)
1420 
1421  locations = self._locate(datasetType, dataId, write=True)
1422  if not locations:
1423  raise NoResults("No locations for put:", datasetType, dataId)
1424  for location in locations:
1425  if isinstance(location, ButlerComposite):
1426  disassembler = location.disassembler if location.disassembler else genericDisassembler
1427  disassembler(obj=obj, dataId=location.dataId, componentInfo=location.componentInfo)
1428  for name, info in location.componentInfo.items():
1429  if not info.inputOnly:
1430  self.put(info.obj, info.datasetType, location.dataId, doBackup=doBackup)
1431  else:
1432  if doBackup:
1433  location.getRepository().backup(location.datasetType, dataId)
1434  location.getRepository().write(location, obj)
1435 
1436  def subset(self, datasetType, level=None, dataId={}, **rest):
1437  """Return complete dataIds for a dataset type that match a partial (or empty) dataId.
1438 
1439  Given a partial (or empty) dataId specified in dataId and **rest, find all datasets that match the
1440  dataId. Optionally restrict the results to a given level specified by a dataId key (e.g. visit or
1441  sensor or amp for a camera). Return an iterable collection of complete dataIds as ButlerDataRefs.
1442  Datasets with the resulting dataIds may not exist; that needs to be tested with datasetExists().
1443 
1444  Parameters
1445  ----------
1446  datasetType - string
1447  The type of dataset collection to subset
1448  level - string
1449  The level of dataId at which to subset. Use an empty string if the mapper should look up the
1450  default level.
1451  dataId - dict
1452  The data id.
1453  **rest
1454  Keyword arguments for the data id.
1455 
1456  Returns
1457  -------
1458  subset - ButlerSubset
1459  Collection of ButlerDataRefs for datasets matching the data id.
1460 
1461  Examples
1462  -----------
1463  To print the full dataIds for all r-band measurements in a source catalog
1464  (note that the subset call is equivalent to: `butler.subset('src', dataId={'filter':'r'})`):
1465 
1466  >>> subset = butler.subset('src', filter='r')
1467  >>> for data_ref in subset: print(data_ref.dataId)
1468  """
1469  datasetType = self._resolveDatasetTypeAlias(datasetType)
1470 
1471  # Currently expected behavior of subset is that if specified level is None then the mapper's default
1472  # level should be used. Convention for level within Butler is that an empty string is used to indicate
1473  # 'get default'.
1474  if level is None:
1475  level = ''
1476 
1477  dataId = DataId(dataId)
1478  dataId.update(**rest)
1479  return ButlerSubset(self, datasetType, level, dataId)
1480 
1481  def dataRef(self, datasetType, level=None, dataId={}, **rest):
1482  """Returns a single ButlerDataRef.
1483 
1484  Given a complete dataId specified in dataId and **rest, find the unique dataset at the given level
1485  specified by a dataId key (e.g. visit or sensor or amp for a camera) and return a ButlerDataRef.
1486 
1487  Parameters
1488  ----------
1489  datasetType - string
1490  The type of dataset collection to reference
1491  level - string
1492  The level of dataId at which to reference
1493  dataId - dict
1494  The data id.
1495  **rest
1496  Keyword arguments for the data id.
1497 
1498  Returns
1499  -------
1500  dataRef - ButlerDataRef
1501  ButlerDataRef for dataset matching the data id
1502  """
1503 
1504  datasetType = self._resolveDatasetTypeAlias(datasetType)
1505  dataId = DataId(dataId)
1506  subset = self.subset(datasetType, level, dataId, **rest)
1507  if len(subset) != 1:
1508  raise RuntimeError("No unique dataset for: Dataset type:%s Level:%s Data ID:%s Keywords:%s" %
1509  (str(datasetType), str(level), str(dataId), str(rest)))
1510  return ButlerDataRef(subset, subset.cache[0])
1511 
1512  def getUri(self, datasetType, dataId=None, write=False, **rest):
1513  """Return the URI for a dataset
1514 
1515  .. warning:: This is intended only for debugging. The URI should
1516  never be used for anything other than printing.
1517 
1518  .. note:: In the event there are multiple URIs for read, we return only
1519  the first.
1520 
1521  .. note:: getUri() does not currently support composite datasets.
1522 
1523  Parameters
1524  ----------
1525  datasetType : `str`
1526  The dataset type of interest.
1527  dataId : `dict`, optional
1528  The data identifier.
1529  write : `bool`, optional
1530  Return the URI for writing?
1531  rest : `dict`, optional
1532  Keyword arguments for the data id.
1533 
1534  Returns
1535  -------
1536  uri : `str`
1537  URI for dataset.
1538  """
1539  datasetType = self._resolveDatasetTypeAlias(datasetType)
1540  dataId = DataId(dataId)
1541  dataId.update(**rest)
1542  locations = self._locate(datasetType, dataId, write=write)
1543  if locations is None:
1544  raise NoResults("No locations for getUri: ", datasetType, dataId)
1545 
1546  if write:
1547  # Follow the write path
1548  # Return the first valid write location.
1549  for location in locations:
1550  if isinstance(location, ButlerComposite):
1551  for name, info in location.componentInfo.items():
1552  if not info.inputOnly:
1553  return self.getUri(info.datasetType, location.dataId, write=True)
1554  else:
1555  return location.getLocationsWithRoot()[0]
1556  # fall back to raise
1557  raise NoResults("No locations for getUri(write=True): ", datasetType, dataId)
1558  else:
1559  # Follow the read path, only return the first valid read
1560  return locations.getLocationsWithRoot()[0]
1561 
1562  def _read(self, location):
1563  """Unpersist an object using data inside a ButlerLocation or ButlerComposite object.
1564 
1565  Parameters
1566  ----------
1567  location : ButlerLocation or ButlerComposite
1568  A ButlerLocation or ButlerComposite instance populated with data needed to read the object.
1569 
1570  Returns
1571  -------
1572  object
1573  An instance of the object specified by the location.
1574  """
1575  self.log.debug("Starting read from %s", location)
1576 
1577  if isinstance(location, ButlerComposite):
1578  for name, componentInfo in location.componentInfo.items():
1579  if componentInfo.subset:
1580  subset = self.subset(datasetType=componentInfo.datasetType, dataId=location.dataId)
1581  componentInfo.obj = [obj.get() for obj in subset]
1582  else:
1583  obj = self.get(componentInfo.datasetType, location.dataId, immediate=True)
1584  componentInfo.obj = obj
1585  assembler = location.assembler or genericAssembler
1586  results = assembler(dataId=location.dataId, componentInfo=location.componentInfo,
1587  cls=location.python)
1588  return results
1589  else:
1590  results = location.repository.read(location)
1591  if len(results) == 1:
1592  results = results[0]
1593  self.log.debug("Ending read from %s", location)
1594  return results
1595 
1596  def __reduce__(self):
1597  ret = (_unreduce, (self._initArgs, self.datasetTypeAliasDict))
1598  return ret
1599 
1600  def _resolveDatasetTypeAlias(self, datasetType):
1601  """Replaces all the known alias keywords in the given string with the alias value.
1602 
1603  Parameters
1604  ----------
1605  datasetType - string
1606  A datasetType string to search & replace on
1607 
1608  Returns
1609  -------
1610  datasetType - string
1611  The de-aliased string
1612  """
1613  for key in self.datasetTypeAliasDict:
1614  # if all aliases have been replaced, bail out
1615  if datasetType.find('@') == -1:
1616  break
1617  datasetType = datasetType.replace(key, self.datasetTypeAliasDict[key])
1618 
1619  # If an alias specifier can not be resolved then throw.
1620  if datasetType.find('@') != -1:
1621  raise RuntimeError("Unresolvable alias specifier in datasetType: %s" % (datasetType))
1622 
1623  return datasetType
1624 
1625 
1626 def _unreduce(initArgs, datasetTypeAliasDict):
1627  mapperArgs = initArgs.pop('mapperArgs')
1628  initArgs.update(mapperArgs)
1629  butler = Butler(**initArgs)
1630  butler.datasetTypeAliasDict = datasetTypeAliasDict
1631  return butler
def _resolveDatasetTypeAlias(self, datasetType)
Definition: butler.py:1600
def datasetExists(self, datasetType, dataId={}, write=False, rest)
Definition: butler.py:1218
def _convertV1Args(self, root, mapper, mapperArgs)
Definition: butler.py:999
def __init__(self, root=None, mapper=None, inputs=None, outputs=None, mapperArgs)
Definition: butler.py:507
def setCfg(self, cfg, origin, root, isV1Repository)
Definition: butler.py:167
def _getRepositoryCfg(self, repositoryArgs)
Definition: butler.py:697
def getParentRepoDatas(self, context=None)
Definition: butler.py:208
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
Definition: functional.cc:33
table::Key< int > id
Definition: Detector.cc:166
daf::base::PropertySet * set
Definition: fits.cc:884
def _getCfgs(self, repoDataList)
Definition: butler.py:723
def subset(self, datasetType, level=None, dataId={}, rest)
Definition: butler.py:1436
def __init__(self, cls, repoCfg)
Definition: butler.py:61
Definition: Log.h:691
bool all(CoordinateExpr< N > const &expr) noexcept
Return true if all elements are true.
def _initRepo(self, repoData)
Definition: butler.py:538
def _setDefaultMapper(self, repoDataList)
Definition: butler.py:900
def getUri(self, datasetType, dataId=None, write=False, rest)
Definition: butler.py:1512
def defineAlias(self, alias, datasetType)
Definition: butler.py:1105
def _connectParentRepoDatas(self, repoDataList)
Definition: butler.py:929
def _addParents(self, repoDataList)
Definition: butler.py:801
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168
def getKeys(self, datasetType=None, level=None, tag=None)
Definition: butler.py:1135
def doImport(pythonType)
Definition: utils.py:106
def _getBypassFunc(location, dataId)
Definition: butler.py:1348
def put(self, obj, datasetType, dataId={}, doBackup=False, rest)
Definition: butler.py:1399
def queryMetadata(self, datasetType, format, dataId={}, rest)
Definition: butler.py:1171
def _processInputArguments(self, root=None, mapper=None, inputs=None, outputs=None, mapperArgs)
Definition: butler.py:554
def addParentRepoData(self, parentRepoData)
Definition: butler.py:236
def _locate(self, datasetType, dataId, write)
Definition: butler.py:1266
def _setAndVerifyParentsLists(self, repoDataList)
Definition: butler.py:850
def get(self, datasetType, dataId=None, immediate=True, rest)
Definition: butler.py:1356
def __init__(self, args, role)
Definition: butler.py:124