LSSTApplications  18.1.0
LSSTDataManagementBasePackage
pipeline.py
Go to the documentation of this file.
1 # This file is part of pipe_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Module defining Pipeline class and related methods.
23 """
24 
25 __all__ = ["Pipeline", "TaskDef"]
26 
27 # -------------------------------
28 # Imports of standard modules --
29 # -------------------------------
30 
31 # -----------------------------
32 # Imports for other modules --
33 # -----------------------------
34 
35 # ----------------------------------
36 # Local non-exported definitions --
37 # ----------------------------------
38 
39 # ------------------------
40 # Exported definitions --
41 # ------------------------
42 
43 
44 class TaskDef:
45  """TaskDef is a collection of information about task needed by Pipeline.
46 
47  The information includes task name, configuration object and optional
48  task class. This class is just a collection of attributes and it exposes
49  all of them so that attributes could potentially be modified in place
50  (e.g. if configuration needs extra overrides).
51 
52  Attributes
53  ----------
54  taskName : `str`
55  `PipelineTask` class name, currently it is not specified whether this
56  is a fully-qualified name or partial name (e.g. ``module.TaskClass``).
57  Framework should be prepared to handle all cases.
58  config : `lsst.pex.config.Config`
59  Instance of the configuration class corresponding to this task class,
60  usually with all overrides applied.
61  taskClass : `type` or ``None``
62  `PipelineTask` class object, can be ``None``. If ``None`` then
63  framework will have to locate and load class.
64  label : `str`, optional
65  Task label, usually a short string unique in a pipeline.
66  """
67  def __init__(self, taskName, config, taskClass=None, label=""):
68  self.taskName = taskName
69  self.config = config
70  self.taskClass = taskClass
71  self.label = label
72 
73  def __str__(self):
74  rep = "TaskDef(" + self.taskName
75  if self.label:
76  rep += ", label=" + self.label
77  rep += ")"
78  return rep
79 
80 
81 class Pipeline(list):
82  """Pipeline is a sequence of `TaskDef` objects.
83 
84  Pipeline is given as one of the inputs to a supervising framework
85  which builds execution graph out of it. Pipeline contains a sequence
86  of `TaskDef` instances.
87 
88  Main purpose of this class is to provide a mechanism to pass pipeline
89  definition from users to supervising framework. That mechanism is
90  implemented using simple serialization and de-serialization via
91  `pickle`. Note that pipeline serialization is not guaranteed to be
92  compatible between different versions or releases.
93 
94  In current implementation Pipeline is a list (it inherits from `list`)
95  and one can use all list methods on pipeline. Content of the pipeline
96  can be modified, it is up to the client to verify that modifications
97  leave pipeline in a consistent state. One could modify container
98  directly by adding or removing its elements.
99 
100  Parameters
101  ----------
102  pipeline : iterable of `TaskDef` instances, optional
103  Initial sequence of tasks.
104  """
105  def __init__(self, iterable=None):
106  list.__init__(self, iterable or [])
107 
108  def labelIndex(self, label):
109  """Return task index given its label.
110 
111  Parameters
112  ----------
113  label : `str`
114  Task label.
115 
116  Returns
117  -------
118  index : `int`
119  Task index, or -1 if label is not found.
120  """
121  for idx, taskDef in enumerate(self):
122  if taskDef.label == label:
123  return idx
124  return -1
125 
126  def __str__(self):
127  infos = [str(tdef) for tdef in self]
128  return "Pipeline({})".format(", ".join(infos))
def __init__(self, taskName, config, taskClass=None, label="")
Definition: pipeline.py:67
def __init__(self, iterable=None)
Definition: pipeline.py:105
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168
def labelIndex(self, label)
Definition: pipeline.py:108
daf::base::PropertyList * list
Definition: fits.cc:885