LSST Applications  21.0.0-147-g0e635eb1+1acddb5be5,22.0.0+052faf71bd,22.0.0+1ea9a8b2b2,22.0.0+6312710a6c,22.0.0+729191ecac,22.0.0+7589c3a021,22.0.0+9f079a9461,22.0.1-1-g7d6de66+b8044ec9de,22.0.1-1-g87000a6+536b1ee016,22.0.1-1-g8e32f31+6312710a6c,22.0.1-10-gd060f87+016f7cdc03,22.0.1-12-g9c3108e+df145f6f68,22.0.1-16-g314fa6d+c825727ab8,22.0.1-19-g93a5c75+d23f2fb6d8,22.0.1-19-gb93eaa13+aab3ef7709,22.0.1-2-g8ef0a89+b8044ec9de,22.0.1-2-g92698f7+9f079a9461,22.0.1-2-ga9b0f51+052faf71bd,22.0.1-2-gac51dbf+052faf71bd,22.0.1-2-gb66926d+6312710a6c,22.0.1-2-gcb770ba+09e3807989,22.0.1-20-g32debb5+b8044ec9de,22.0.1-23-gc2439a9a+fb0756638e,22.0.1-3-g496fd5d+09117f784f,22.0.1-3-g59f966b+1e6ba2c031,22.0.1-3-g849a1b8+f8b568069f,22.0.1-3-gaaec9c0+c5c846a8b1,22.0.1-32-g5ddfab5d3+60ce4897b0,22.0.1-4-g037fbe1+64e601228d,22.0.1-4-g8623105+b8044ec9de,22.0.1-5-g096abc9+d18c45d440,22.0.1-5-g15c806e+57f5c03693,22.0.1-7-gba73697+57f5c03693,master-g6e05de7fdc+c1283a92b8,master-g72cdda8301+729191ecac,w.2021.39
LSST Data Management Base Package
singleFrameDriver.py
Go to the documentation of this file.
1 from lsst.pipe.base import ArgumentParser, ButlerInitializedTaskRunner, ConfigDatasetType
2 from lsst.pipe.tasks.processCcd import ProcessCcdTask
3 from lsst.pipe.tasks.postprocess import WriteSourceTableTask, TransformSourceTableTask
4 from lsst.pex.config import Config, Field, ConfigurableField, ListField
5 from lsst.ctrl.pool.parallel import BatchParallelTask, BatchTaskRunner
6 
7 
9  processCcd = ConfigurableField(
10  target=ProcessCcdTask, doc="CCD processing task")
11  doMakeSourceTable = Field(dtype=bool, default=False,
12  doc="Do postprocessing tasks to write parquet Source Table?")
13  doSaveWideSourceTable = Field(dtype=bool, default=False,
14  doc=("Save the parquet version of the full src catalog?",
15  "Only respected if doMakeSourceTable"))
16  writeSourceTable = ConfigurableField(
17  target=WriteSourceTableTask, doc="Task to make parquet table for full src catalog")
18  transformSourceTable = ConfigurableField(
19  target=TransformSourceTableTask, doc="Transform Source Table to DPDD specification")
20  ignoreCcdList = ListField(dtype=int, default=[],
21  doc="List of CCDs to ignore when processing")
22  ccdKey = Field(dtype=str, default="ccd",
23  doc="DataId key corresponding to a single sensor")
24 
25 
26 class SingleFrameTaskRunner(BatchTaskRunner, ButlerInitializedTaskRunner):
27  """Run batches, and initialize Task using a butler"""
28  pass
29 
30 
31 class SingleFrameDriverTask(BatchParallelTask):
32  """Process CCDs in parallel
33  """
34  ConfigClass = SingleFrameDriverConfig
35  _DefaultName = "singleFrameDriver"
36  RunnerClass = SingleFrameTaskRunner
37 
38  def __init__(self, butler=None, psfRefObjLoader=None, astromRefObjLoader=None, photoRefObjLoader=None,
39  *args, **kwargs):
40  """!
41  Constructor
42 
43  The psfRefObjLoader, astromRefObjLoader, photoRefObjLoader should
44  be an instance of LoadReferenceObjectsTasks that supplies an external
45  reference catalog. They may be None if the butler argument is
46  provided or the particular reference catalog is not required.
47 
48  @param[in] butler The butler is passed to the refObjLoader constructor in case it is
49  needed. Ignored if the refObjLoader argument provides a loader directly.
50  @param[in] psfRefObjLoader Reference catalog loader for PSF determination.
51  @param[in] astromRefObjLoader Reference catalog loader for astrometric calibration.
52  @param[in] photoRefObjLoader Reference catalog loader for photometric calibration.
53  @param[in,out] kwargs other keyword arguments for lsst.ctrl.pool.BatchParallelTask
54  """
55  BatchParallelTask.__init__(self, *args, **kwargs)
56  self.ignoreCcdsignoreCcds = set(self.config.ignoreCcdList)
57  self.makeSubtask("processCcd", butler=butler, psfRefObjLoader=psfRefObjLoader,
58  astromRefObjLoader=astromRefObjLoader, photoRefObjLoader=photoRefObjLoader)
59  if self.config.doMakeSourceTable:
60  self.makeSubtask("writeSourceTable")
61  self.makeSubtask("transformSourceTable")
62 
63  @classmethod
64  def _makeArgumentParser(cls, *args, **kwargs):
65  kwargs.pop("doBatch", False)
66  parser = ArgumentParser(name="singleFrameDriver", *args, **kwargs)
67  parser.add_id_argument("--id",
68  datasetType=ConfigDatasetType(
69  name="processCcd.isr.datasetType"),
70  level="sensor",
71  help="data ID, e.g. --id visit=12345 ccd=67")
72  return parser
73 
74  def runDataRef(self, sensorRef):
75  """Process a single CCD, with scatter-gather-scatter using MPI.
76  """
77  if sensorRef.dataId[self.config.ccdKey] in self.ignoreCcdsignoreCcds:
78  self.log.warn("Ignoring %s: CCD in ignoreCcdList" %
79  (sensorRef.dataId))
80  return None
81 
82  with self.logOperationlogOperation("processing %s" % (sensorRef.dataId,)):
83  result = self.processCcd.runDataRef(sensorRef)
84  if self.config.doMakeSourceTable:
85  parquet = self.writeSourceTable.run(result.calibRes.sourceCat,
86  ccdVisitId=sensorRef.get('ccdExposureId'))
87  if self.config.doSaveWideSourceTable:
88  sensorRef.put(parquet.table, 'source')
89 
90  df = self.transformSourceTable.run(parquet.table,
91  funcs=self.transformSourceTable.getFunctors(),
92  dataId=sensorRef.dataId)
93  self.transformSourceTable.write(df, sensorRef)
94 
95  return result
def logOperation(self, operation, catch=False, trace=True)
Provide a context manager for logging an operation.
Definition: parallel.py:502
def __init__(self, butler=None, psfRefObjLoader=None, astromRefObjLoader=None, photoRefObjLoader=None, *args, **kwargs)
Constructor.
daf::base::PropertySet * set
Definition: fits.cc:912
void write(OutputArchiveHandle &handle) const override
def run(self, coaddExposures, bbox, wcs)
Definition: getTemplate.py:603