LSST Applications  21.0.0+04719a4bac,21.0.0-1-ga51b5d4+f5e6047307,21.0.0-11-g2b59f77+a9c1acf22d,21.0.0-11-ga42c5b2+86977b0b17,21.0.0-12-gf4ce030+76814010d2,21.0.0-13-g1721dae+760e7a6536,21.0.0-13-g3a573fe+768d78a30a,21.0.0-15-g5a7caf0+f21cbc5713,21.0.0-16-g0fb55c1+b60e2d390c,21.0.0-19-g4cded4ca+71a93a33c0,21.0.0-2-g103fe59+bb20972958,21.0.0-2-g45278ab+04719a4bac,21.0.0-2-g5242d73+3ad5d60fb1,21.0.0-2-g7f82c8f+8babb168e8,21.0.0-2-g8f08a60+06509c8b61,21.0.0-2-g8faa9b5+616205b9df,21.0.0-2-ga326454+8babb168e8,21.0.0-2-gde069b7+5e4aea9c2f,21.0.0-2-gecfae73+1d3a86e577,21.0.0-2-gfc62afb+3ad5d60fb1,21.0.0-25-g1d57be3cd+e73869a214,21.0.0-3-g357aad2+ed88757d29,21.0.0-3-g4a4ce7f+3ad5d60fb1,21.0.0-3-g4be5c26+3ad5d60fb1,21.0.0-3-g65f322c+e0b24896a3,21.0.0-3-g7d9da8d+616205b9df,21.0.0-3-ge02ed75+a9c1acf22d,21.0.0-4-g591bb35+a9c1acf22d,21.0.0-4-g65b4814+b60e2d390c,21.0.0-4-gccdca77+0de219a2bc,21.0.0-4-ge8a399c+6c55c39e83,21.0.0-5-gd00fb1e+05fce91b99,21.0.0-6-gc675373+3ad5d60fb1,21.0.0-64-g1122c245+4fb2b8f86e,21.0.0-7-g04766d7+cd19d05db2,21.0.0-7-gdf92d54+04719a4bac,21.0.0-8-g5674e7b+d1bd76f71f,master-gac4afde19b+a9c1acf22d,w.2021.13
LSST Data Management Base Package
getRepositoryData.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 """Retrieve collections of metadata or data based on a set of data references
23 
24 Use this as a base task for creating graphs and reports for a set of data.
25 """
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 
29 __all__ = ["DataRefListRunner", "GetRepositoryDataTask"]
30 
31 
32 class DataRefListRunner(pipeBase.TaskRunner):
33  """A task runner that calls run with a list of data references
34 
35  Differs from the default TaskRunner by providing all data references at once,
36  instead of iterating over them one at a time.
37  """
38  @staticmethod
39  def getTargetList(parsedCmd):
40  """Return a list of targets (arguments for __call__); one entry per invocation
41  """
42  return [parsedCmd.id.refList] # one argument consisting of a list of dataRefs
43 
44  def __call__(self, dataRefList):
45  """Run GetRepositoryDataTask.run on a single target
46 
47  @param dataRefList: argument dict for run; contains one key: dataRefList
48 
49  @return:
50  - None if doReturnResults false
51  - A pipe_base Struct containing these fields if doReturnResults true:
52  - dataRefList: the argument dict sent to runDataRef
53  - metadata: task metadata after execution of runDataRef
54  - result: result returned by task runDataRef
55  """
56  task = self.TaskClass(config=self.config, log=self.log)
57  result = task.runDataRef(dataRefList)
58 
59  if self.doReturnResults:
60  return pipeBase.Struct(
61  dataRefList=dataRefList,
62  metadata=task.metadata,
63  result=result,
64  )
65 
66 
67 class GetRepositoryDataTask(pipeBase.CmdLineTask):
68  """Retrieve data from a repository, e.g. for plotting or analysis purposes
69  """
70  ConfigClass = pexConfig.Config # nothing to configure
71  RunnerClass = DataRefListRunner
72  _DefaultName = "getTaskData"
73 
74  def __init__(self, *args, **kwargs):
75  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
76 
77  @pipeBase.timeMethod
78  def runDataRef(self, dataRefList):
79  """Get data from a repository for a collection of data references
80 
81  @param dataRefList: a list of data references
82  """
83  raise NotImplementedError("subclass must specify a run method")
84 
85  def getIdList(self, dataRefList):
86  """Get a list of data IDs in a form that can be used as dictionary keys
87 
88  @param dataRefList: a list of data references
89  @return a pipe_base Struct with fields:
90  - idKeyTuple: a tuple of dataRef data ID keys
91  - idValList: a list of data ID value tuples, each tuple contains values in the order in idKeyTuple
92  """
93  if not dataRefList:
94  raise RuntimeError("No data refs")
95  idKeyTuple = tuple(sorted(dataRefList[0].dataId.keys()))
96 
97  idValList = []
98  for dataRef in dataRefList:
99  idValTuple = tuple(dataRef.dataId[key] for key in idKeyTuple)
100  idValList.append(idValTuple)
101 
102  return pipeBase.Struct(
103  idKeyTuple=idKeyTuple,
104  idValList=idValList,
105  )
106 
107  def getDataList(self, dataRefList, datasetType):
108  """Retrieve a list of data
109 
110  @param dataRefList: a list of data references
111  @param datasetType: datasetType of data to be retrieved
112  @return a list of data, one entry per dataRef in dataRefList (in order)
113  """
114  return [dataRef.get(datasetType=datasetType) for dataRef in dataRefList]
115 
116  def getMetadataItems(self, dataRefList, datasetType, nameList):
117  """Retrieve a list of dictionaries of metadata
118 
119  @param dataRefList: a list of data references
120  @param datasetType: datasetType of metadata (or any object that supports get(name))
121  @return a list of dicts of metadata:
122  - each entry in the list corresponds to a dataRef in dataRefList
123  - each dict contains name: item of metadata, for each name in nameList;
124  numeric and string values will be returned as arrays
125  """
126  valList = []
127  for dataRef in dataRefList:
128  metadata = dataRef.get(datasetType=datasetType)
129  valList.append(dict((name, metadata.getArray(name)) for name in nameList))
130  return valList
def getMetadataItems(self, dataRefList, datasetType, nameList)