24 from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
25 PipelineTask, PipelineTaskConfig, InitInputDatasetField,
26 InitOutputDatasetField, InputDatasetField, OutputDatasetField)
29 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
42 from .mergeDetections
import MergeDetectionsConfig, MergeDetectionsTask
43 from .mergeMeasurements
import MergeMeasurementsConfig, MergeMeasurementsTask
44 from .multiBandUtils
import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs
45 from .multiBandUtils
import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory
46 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleConfig
47 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleTask
48 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiConfig
49 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiTask
54 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) 55 * deepCoadd_mergeDet: merged detections (tract, patch) 56 * deepCoadd_meas: measurements of merged detections (tract, patch, filter) 57 * deepCoadd_ref: reference sources (tract, patch) 58 All of these have associated *_schema catalogs that require no data ID and hold no records. 60 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in 61 the mergeDet, meas, and ref dataset Footprints: 62 * deepCoadd_peak_schema 70 @anchor DetectCoaddSourcesConfig_ 72 @brief Configuration parameters for the DetectCoaddSourcesTask 74 doScaleVariance =
Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
77 coaddName =
Field(dtype=str, default=
"deep", doc=
"Name of coadd")
78 doInsertFakes =
Field(dtype=bool, default=
False,
79 doc=
"Run fake sources injection task")
81 doc=
"Injection of fake sources for testing " 82 "purposes (must be retargeted)")
84 doc=
"Schema of the detection catalog",
85 nameTemplate=
"{outputCoaddName}Coadd_det_schema",
86 storageClass=
"SourceCatalog",
89 doc=
"Exposure on which detections are to be performed",
90 nameTemplate=
"{inputCoaddName}Coadd",
92 storageClass=
"ExposureF",
93 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
96 doc=
"Output Backgrounds used in detection",
97 nameTemplate=
"{outputCoaddName}Coadd_calexp_background",
99 storageClass=
"Background",
100 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
103 doc=
"Detected sources catalog",
104 nameTemplate=
"{outputCoaddName}Coadd_det",
106 storageClass=
"SourceCatalog",
107 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
110 doc=
"Exposure post detection",
111 nameTemplate=
"{outputCoaddName}Coadd_calexp",
113 storageClass=
"ExposureF",
114 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
119 self.quantum.dimensions = (
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
120 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep"})
121 self.
detection.thresholdType =
"pixel_stdev" 124 self.
detection.reEstimateBackground =
False 125 self.
detection.background.useApprox =
False 127 self.
detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER' 128 self.
detection.doTempWideBackground =
True 140 @anchor DetectCoaddSourcesTask_ 142 @brief Detect sources on a coadd 144 @section pipe_tasks_multiBand_Contents Contents 146 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose 147 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize 148 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run 149 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config 150 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug 151 - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example 153 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description 155 Command-line task that detects sources on a coadd of exposures obtained with a single filter. 157 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise 158 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane 159 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should 160 propagate the full covariance matrix -- but it is simple and works well in practice. 162 After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref 163 SourceDetectionTask_ "detection" subtask. 166 deepCoadd{tract,patch,filter}: ExposureF 168 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 169 @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input 171 @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList 175 DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask. 176 You can retarget this subtask if you wish. 178 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization 180 @copydoc \_\_init\_\_ 182 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task 186 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters 188 See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig" 190 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables 192 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 193 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 196 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to 197 @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for 198 @ref SourceDetectionTask_ "SourceDetectionTask" for further information. 200 @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example 201 of using DetectCoaddSourcesTask 203 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of 204 the task is to update the background, detect all sources in a single band and generate a set of parent 205 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and, 206 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data 207 reference to the coadd to be processed. A list of the available optional arguments can be obtained by 208 calling detectCoaddSources.py with the `--help` command line argument: 210 detectCoaddSources.py --help 213 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 214 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed 215 steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows: 217 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 219 that will process the HSC-I band data. The results are written to 220 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 222 It is also necessary to run: 224 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 226 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 227 processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask". 229 _DefaultName =
"detectCoaddSources" 230 ConfigClass = DetectCoaddSourcesConfig
231 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
232 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
235 def _makeArgumentParser(cls):
237 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
238 ContainerClass=ExistingCoaddDataIdContainer)
243 @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask. 245 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 247 @param[in] schema: initial schema for the output catalog, modified-in place to include all 248 fields set by this task. If None, the source minimal schema will be used. 249 @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ 255 schema = afwTable.SourceTable.makeMinimalSchema()
256 if self.config.doInsertFakes:
257 self.makeSubtask(
"insertFakes")
259 self.makeSubtask(
"detection", schema=self.
schema)
260 if self.config.doScaleVariance:
261 self.makeSubtask(
"scaleVariance")
268 @brief Run detection on a coadd. 270 Invokes @ref run and then uses @ref write to output the 273 @param[in] patchRef: data reference for patch 275 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
276 expId =
int(patchRef.get(self.config.coaddName +
"CoaddId"))
278 self.
write(results, patchRef)
282 packedId, maxBits = butler.registry.packDataId(
"TractPatchAbstractFilter",
283 inputDataIds[
"exposure"],
285 inputData[
"idFactory"] = afwTable.IdFactory.makeSource(packedId, 64 - maxBits)
286 inputData[
"expId"] = packedId
287 return self.
run(**inputData)
289 def run(self, exposure, idFactory, expId):
291 @brief Run detection on an exposure. 293 First scale the variance plane to match the observed variance 294 using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to 297 @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled, 298 depending on configuration). 299 @param[in] idFactory: IdFactory to set source identifiers 300 @param[in] expId: Exposure identifier (integer) for RNG seed 302 @return a pipe.base.Struct with fields 303 - sources: catalog of detections 304 - backgrounds: list of backgrounds 306 if self.config.doScaleVariance:
307 varScale = self.scaleVariance.
run(exposure.maskedImage)
308 exposure.getMetadata().add(
"variance_scale", varScale)
310 if self.config.doInsertFakes:
311 self.insertFakes.
run(exposure, background=backgrounds)
312 table = afwTable.SourceTable.make(self.
schema, idFactory)
313 detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
314 sources = detections.sources
315 fpSets = detections.fpSets
316 if hasattr(fpSets,
"background")
and fpSets.background:
317 for bg
in fpSets.background:
318 backgrounds.append(bg)
319 return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
323 @brief Write out results from runDetection. 325 @param[in] exposure: Exposure to write out 326 @param[in] results: Struct returned from runDetection 327 @param[in] patchRef: data reference for patch 329 coaddName = self.config.coaddName +
"Coadd" 330 patchRef.put(results.outputBackgrounds, coaddName +
"_calexp_background")
331 patchRef.put(results.outputSources, coaddName +
"_det")
332 patchRef.put(results.outputExposure, coaddName +
"_calexp")
338 """DeblendCoaddSourcesConfig 340 Configuration parameters for the `DeblendCoaddSourcesTask`. 343 doc=
"Deblend sources separately in each band")
345 doc=
"Deblend sources simultaneously across bands")
346 simultaneous =
Field(dtype=bool, default=
False, doc=
"Simultaneously deblend all bands?")
347 coaddName =
Field(dtype=str, default=
"deep", doc=
"Name of coadd")
350 Config.setDefaults(self)
355 """Task runner for the `MergeSourcesTask` 357 Required because the run method requires a list of 358 dataRefs rather than a single dataRef. 362 """Provide a list of patch references for each patch, tract, filter combo. 369 Keyword arguments passed to the task 374 List of tuples, where each tuple is a (dataRef, kwargs) pair. 376 refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
377 kwargs[
"psfCache"] = parsedCmd.psfCache
378 return [(
list(p.values()), kwargs)
for t
in refDict.values()
for p
in t.values()]
382 """Deblend the sources in a merged catalog 384 Deblend sources from master catalog in each coadd. 385 This can either be done separately in each band using the HSC-SDSS deblender 386 (`DeblendCoaddSourcesTask.config.simultaneous==False`) 387 or use SCARLET to simultaneously fit the blend in all bands 388 (`DeblendCoaddSourcesTask.config.simultaneous==True`). 389 The task will set its own `self.schema` atribute to the `Schema` of the 390 output deblended catalog. 391 This will include all fields from the input `Schema`, as well as additional fields 394 `pipe.tasks.multiband.DeblendCoaddSourcesTask Description 395 --------------------------------------------------------- 401 Butler used to read the input schemas from disk or 402 construct the reference catalog loader, if `schema` or `peakSchema` or 404 The schema of the merged detection catalog as an input to this task. 406 The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog 408 ConfigClass = DeblendCoaddSourcesConfig
409 RunnerClass = DeblendCoaddSourcesRunner
410 _DefaultName =
"deblendCoaddSources" 411 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
414 def _makeArgumentParser(cls):
416 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
417 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
418 ContainerClass=ExistingCoaddDataIdContainer)
419 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
422 def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
423 CmdLineTask.__init__(self, **kwargs)
425 assert butler
is not None,
"Neither butler nor schema is defined" 426 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
430 if peakSchema
is None:
431 assert butler
is not None,
"Neither butler nor peakSchema is defined" 432 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
434 if self.config.simultaneous:
435 self.makeSubtask(
"multiBandDeblend", schema=self.
schema, peakSchema=peakSchema)
437 self.makeSubtask(
"singleBandDeblend", schema=self.
schema, peakSchema=peakSchema)
440 """Return a dict of empty catalogs for each catalog dataset produced by this task. 445 Dictionary of empty catalogs, with catalog names as keys. 448 return {self.config.coaddName +
"Coadd_deblendedFlux": catalog,
449 self.config.coaddName +
"Coadd_deblendedModel": catalog}
454 Deblend each source simultaneously or separately 455 (depending on `DeblendCoaddSourcesTask.config.simultaneous`). 456 Set `is-primary` and related flags. 457 Propagate flags from individual visits. 458 Write the deblended sources out. 463 List of data references for each filter 465 if self.config.simultaneous:
469 for patchRef
in patchRefList:
470 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
471 filters.append(patchRef.dataId[
"filter"])
472 exposures.append(exposure)
475 exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
476 fluxCatalogs, templateCatalogs = self.multiBandDeblend.
run(exposure, sources)
477 for n
in range(len(patchRefList)):
478 self.
write(patchRefList[n], fluxCatalogs[filters[n]], templateCatalogs[filters[n]])
481 for patchRef
in patchRefList:
482 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
483 exposure.getPsf().setCacheCapacity(psfCache)
485 self.singleBandDeblend.
run(exposure, sources)
486 self.
write(patchRef, sources)
489 """Read merged catalog 491 Read the catalog of merged detections and create a catalog 496 dataRef: data reference 497 Data reference for catalog of merged detections 501 sources: `SourceCatalog` 502 List of sources in merged catalog 504 We also need to add columns to hold the measurements we're about to make 505 so we can measure in-place. 507 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
508 self.log.
info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
511 idFactory.notify(s.getId())
512 table = afwTable.SourceTable.make(self.
schema, idFactory)
517 def write(self, dataRef, flux_sources, template_sources=None):
518 """Write the source catalog(s) 522 dataRef: Data Reference 523 Reference to the output catalog. 524 flux_sources: `SourceCatalog` 525 Flux conserved sources to write to file. 526 If using the single band deblender, this is the catalog 528 template_sources: `SourceCatalog` 529 Source catalog using the multiband template models 534 if flux_sources
is not None:
535 assert not self.config.simultaneous
or self.config.multiBandDeblend.conserveFlux
536 dataRef.put(flux_sources, self.config.coaddName +
"Coadd_deblendedFlux")
540 if template_sources
is not None:
541 assert self.config.multiBandDeblend.saveTemplates
542 dataRef.put(template_sources, self.config.coaddName +
"Coadd_deblendedModel")
543 self.log.
info(
"Wrote %d sources: %s" % (len(flux_sources), dataRef.dataId))
546 """Write the metadata produced from processing the data. 550 List of Butler data references used to write the metadata. 551 The metadata is written to dataset type `CmdLineTask._getMetadataName`. 553 for dataRef
in dataRefList:
555 metadataName = self._getMetadataName()
556 if metadataName
is not None:
557 dataRef.put(self.getFullMetadata(), metadataName)
558 except Exception
as e:
559 self.log.
warn(
"Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
562 """Get the ExposureId from a data reference 564 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
569 @anchor MeasureMergedCoaddSourcesConfig_ 571 @brief Configuration parameters for the MeasureMergedCoaddSourcesTask 573 inputCatalog =
Field(dtype=str, default=
"deblendedFlux",
574 doc=(
"Name of the input catalog to use." 575 "If the single band deblender was used this should be 'deblendedFlux." 576 "If the multi-band deblender was used this should be 'deblendedModel." 577 "If no deblending was performed this should be 'mergeDet'"))
579 setPrimaryFlags =
ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
581 dtype=bool, default=
True,
582 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" 584 propagateFlags =
ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
585 doMatchSources =
Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
590 doc=(
"Write reference matches in denormalized format? " 591 "This format uses more disk space, but is more convenient to read."),
593 coaddName =
Field(dtype=str, default=
"deep", doc=
"Name of coadd")
594 psfCache =
Field(dtype=int, default=100, doc=
"Size of psfCache")
596 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
603 doc=
"Apply aperture corrections" 606 target=ApplyApCorrTask,
607 doc=
"Subtask to apply aperture corrections" 612 doc=
'Run catalogCalculation task' 615 target=CatalogCalculationTask,
616 doc=
"Subtask to run catalogCalculation plugins on catalog" 619 doc=
"Input schema for measure merged task produced by a deblender or detection task",
620 nameTemplate=
"{inputCoaddName}Coadd_deblendedFlux_schema",
621 storageClass=
"SourceCatalog" 624 doc=
"Output schema after all new fields are added by task",
625 nameTemplate=
"{inputCoaddName}Coadd_meas_schema",
626 storageClass=
"SourceCatalog" 629 doc=
"Reference catalog used to match measured sources against known sources",
631 storageClass=
"SimpleCatalog",
632 dimensions=(
"SkyPix",),
636 doc=
"Input coadd image",
637 nameTemplate=
"{inputCoaddName}Coadd_calexp",
639 storageClass=
"ExposureF",
640 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
643 doc=
"SkyMap to use in processing",
644 nameTemplate=
"{inputCoaddName}Coadd_skyMap",
645 storageClass=
"SkyMap",
646 dimensions=(
"SkyMap",),
650 doc=
"Source catalogs for visits which overlap input tract, patch, abstract_filter. Will be " 651 "further filtered in the task for the purpose of propagating flags from image calibration " 652 "and characterization to codd objects",
654 dimensions=(
"Instrument",
"Visit",
"Detector"),
655 storageClass=
"SourceCatalog" 658 doc=(
"Name of the input catalog to use." 659 "If the single band deblender was used this should be 'deblendedFlux." 660 "If the multi-band deblender was used this should be 'deblendedModel, " 661 "or deblendedFlux if the multiband deblender was configured to output " 662 "deblended flux catalogs. If no deblending was performed this should " 664 nameTemplate=
"{inputCoaddName}Coadd_deblendedFlux",
665 storageClass=
"SourceCatalog",
666 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
670 doc=
"Source catalog containing all the measurement information generated in this task",
671 nameTemplate=
"{outputCoaddName}Coadd_meas",
672 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
673 storageClass=
"SourceCatalog",
677 doc=
"Match catalog produced by configured matcher, optional on doMatchSources",
678 nameTemplate=
"{outputCoaddName}Coadd_measMatch",
679 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
680 storageClass=
"Catalog",
684 doc=
"Denormalized Match catalog produced by configured matcher, optional on " 685 "doWriteMatchesDenormalized",
686 nameTemplate=
"{outputCoaddName}Coadd_measMatchFull",
687 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
688 storageClass=
"Catalog",
694 return self.
match.refObjLoader
698 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep"})
699 self.quantum.dimensions = (
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
700 self.
measurement.plugins.names |= [
'base_InputCount',
'base_Variance']
701 self.
measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED',
'SENSOR_EDGE',
703 self.
measurement.plugins[
'base_PixelFlags'].masksFpCenter = [
'CLIPPED',
'SENSOR_EDGE',
715 """Get the psfCache setting into MeasureMergedCoaddSourcesTask""" 718 return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
723 @anchor MeasureMergedCoaddSourcesTask_ 725 @brief Deblend sources from master catalog in each coadd seperately and measure. 727 @section pipe_tasks_multiBand_Contents Contents 729 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose 730 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize 731 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run 732 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config 733 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug 734 - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example 736 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description 738 Command-line task that uses peaks and footprints from a master catalog to perform deblending and 739 measurement in each coadd. 741 Given a master input catalog of sources (peaks and footprints) or deblender outputs 742 (including a HeavyFootprint in each band), measure each source on the 743 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a 744 consistent set of child sources. 746 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source 747 properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit 748 flags are propagated to the coadd sources. 750 Optionally, we can match the coadd sources to an external reference catalog. 753 deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog 754 @n deepCoadd_calexp{tract,patch,filter}: ExposureF 756 deepCoadd_meas{tract,patch,filter}: SourceCatalog 760 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks: 763 <DT> @ref SingleFrameMeasurementTask_ "measurement" 764 <DD> Measure source properties of deblended sources.</DD> 765 <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags" 766 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are 767 not at the edge of the field and that have either not been deblended or are the children of deblended 769 <DT> @ref PropagateVisitFlagsTask_ "propagateFlags" 770 <DD> Propagate flags set in individual visits to the coadd.</DD> 771 <DT> @ref DirectMatchTask_ "match" 772 <DD> Match input sources to a reference catalog (optional). 775 These subtasks may be retargeted as required. 777 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization 779 @copydoc \_\_init\_\_ 781 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task 785 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters 787 See @ref MeasureMergedCoaddSourcesConfig_ 789 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables 791 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 792 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 795 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to 796 the various sub-tasks. See the documetation for individual sub-tasks for more information. 798 @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using 799 MeasureMergedCoaddSourcesTask 801 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. 802 The next stage in the multi-band processing procedure will merge these measurements into a suitable 803 catalog for driving forced photometry. 805 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds 807 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the 808 `--help` command line argument: 810 measureCoaddSources.py --help 813 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 814 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 815 step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band 818 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 820 This will process the HSC-I band data. The results are written in 821 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits 823 It is also necessary to run 825 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 827 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 828 procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask". 830 _DefaultName =
"measureCoaddSources" 831 ConfigClass = MeasureMergedCoaddSourcesConfig
832 RunnerClass = MeasureMergedCoaddSourcesRunner
833 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
834 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
837 def _makeArgumentParser(cls):
839 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
840 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
841 ContainerClass=ExistingCoaddDataIdContainer)
842 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
845 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
848 @brief Initialize the task. 850 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 851 @param[in] schema: the schema of the merged detection catalog used as input to this one 852 @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog 853 @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference 854 catalog. May be None if the loader can be constructed from the butler argument or all steps 855 requiring a reference catalog are disabled. 856 @param[in] butler: a butler used to read the input schemas from disk or construct the reference 857 catalog loader, if schema or peakSchema or refObjLoader is None 859 The task will set its own self.schema attribute to the schema of the output measurement catalog. 860 This will include all fields from the input schema, as well as additional fields for all the 864 self.
deblended = self.config.inputCatalog.startswith(
"deblended")
866 if initInputs
is not None:
867 schema = initInputs[
'inputSchema'].schema
869 assert butler
is not None,
"Neither butler nor schema is defined" 870 schema = butler.get(self.config.coaddName + self.
inputCatalog +
"_schema", immediate=
True).schema
875 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
876 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
877 if self.config.doMatchSources:
878 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
879 if self.config.doPropagateFlags:
880 self.makeSubtask(
"propagateFlags", schema=self.
schema)
881 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
882 if self.config.doApCorr:
883 self.makeSubtask(
"applyApCorr", schema=self.
schema)
884 if self.config.doRunCatalogCalculation:
885 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
890 if not config.doPropagateFlags:
891 inputDatasetTypes.pop(
"visitCatalogs")
892 return inputDatasetTypes
897 if config.doMatchSources
is False:
898 outputDatasetTypes.pop(
"matchResult")
899 if config.doWriteMatchesDenormalized
is False:
900 outputDatasetTypes.pop(
"denormMatches")
901 return outputDatasetTypes
905 return frozenset([
"refCat"])
912 config=self.config.refObjLoader, log=self.log)
913 self.match.setRefObjLoader(refObjLoader)
917 inputData[
'exposure'].getPsf().setCacheCapacity(self.config.psfCache)
920 packedId, maxBits = butler.registry.packDataId(
"TractPatch", outputDataIds[
"outputSources"],
922 inputData[
'exposureId'] = packedId
923 idFactory = afwTable.IdFactory.makeSource(packedId, 64 - maxBits)
925 table = afwTable.SourceTable.make(self.
schema, idFactory)
927 sources.extend(inputData.pop(
'intakeCatalog'), self.
schemaMapper)
928 table = sources.getTable()
930 inputData[
'sources'] = sources
932 skyMap = inputData.pop(
'skyMap')
933 tractNumber = inputDataIds[
'intakeCatalog'][
'tract']
934 tractInfo = skyMap[tractNumber]
935 patchInfo = tractInfo.getPatchInfo(inputDataIds[
'intakeCatalog'][
'patch'])
940 wcs=tractInfo.getWcs(),
941 bbox=patchInfo.getOuterBBox()
943 inputData[
'skyInfo'] = skyInfo
945 if self.config.doPropagateFlags:
947 ccdInputs = inputData[
'exposure'].
getInfo().getCoaddInputs().ccds
948 visitKey = ccdInputs.schema.find(
"visit").key
949 ccdKey = ccdInputs.schema.find(
"ccd").key
950 inputVisitIds =
set()
952 for ccdRecord
in ccdInputs:
953 visit = ccdRecord.get(visitKey)
954 ccd = ccdRecord.get(ccdKey)
955 inputVisitIds.add((visit, ccd))
956 ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs()
958 inputCatalogsToKeep = []
959 inputCatalogWcsUpdate = []
960 for i, dataId
in enumerate(inputDataIds[
'visitCatalogs']):
961 key = (dataId[
'visit'], dataId[
'detector'])
962 if key
in inputVisitIds:
963 inputCatalogsToKeep.append(inputData[
'visitCatalogs'][i])
964 inputCatalogWcsUpdate.append(ccdRecordsWcs[key])
965 inputData[
'visitCatalogs'] = inputCatalogsToKeep
966 inputData[
'wcsUpdates'] = inputCatalogWcsUpdate
967 inputData[
'ccdInputs'] = ccdInputs
969 return self.
run(**inputData)
973 @brief Deblend and measure. 975 @param[in] patchRef: Patch reference. 977 Set 'is-primary' and related flags. Propagate flags 978 from individual visits. Optionally match the sources to a reference catalog and write the matches. 979 Finally, write the deblended sources and measurements out. 981 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
982 exposure.getPsf().setCacheCapacity(psfCache)
984 table = sources.getTable()
986 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
988 if self.config.doPropagateFlags:
989 ccdInputs = self.propagateFlags.getCcdInputs(exposure)
993 results = self.
run(exposure=exposure, sources=sources,
995 skyInfo=skyInfo, butler=patchRef.getButler(),
998 if self.config.doMatchSources:
1000 self.
write(patchRef, results.outputSources)
1002 def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None,
1004 """Run measurement algorithms on the input exposure, and optionally populate the 1005 resulting catalog with extra information. 1009 exposure : `lsst.afw.exposure.Exposure` 1010 The input exposure on which measurements are to be performed 1011 sources : `lsst.afw.table.SourceCatalog` 1012 A catalog built from the results of merged detections, or 1014 skyInfo : `lsst.pipe.base.Struct` 1015 A struct containing information about the position of the input exposure within 1016 a `SkyMap`, the `SkyMap`, its `Wcs`, and its bounding box 1017 exposureId : `int` or `bytes` 1018 packed unique number or bytes unique to the input exposure 1019 ccdInputs : `lsst.afw.table.ExposureCatalog` 1020 Catalog containing information on the individual visits which went into making 1022 visitCatalogs : list of `lsst.afw.table.SourceCatalogs` or `None` 1023 A list of source catalogs corresponding to measurements made on the individual 1024 visits which went into the input exposure. If None and butler is `None` then 1025 the task cannot propagate visit flags to the output catalog. 1026 wcsUpdates : list of `lsst.afw.geom.SkyWcs` or `None` 1027 If visitCatalogs is not `None` this should be a list of wcs objects which correspond 1028 to the input visits. Used to put all coordinates to common system. If `None` and 1029 butler is `None` then the task cannot propagate visit flags to the output catalog. 1030 butler : `lsst.daf.butler.Butler` or `lsst.daf.persistence.Butler` 1031 Either a gen2 or gen3 butler used to load visit catalogs 1035 results : `lsst.pipe.base.Struct` 1036 Results of running measurement task. Will contain the catalog in the 1037 sources attribute. Optionally will have results of matching to a 1038 reference catalog in the matchResults attribute, and denormalized 1039 matches in the denormMatches attribute. 1041 self.measurement.
run(sources, exposure, exposureId=exposureId)
1043 if self.config.doApCorr:
1044 self.applyApCorr.
run(
1046 apCorrMap=exposure.getInfo().getApCorrMap()
1053 if not sources.isContiguous():
1054 sources = sources.copy(deep=
True)
1056 if self.config.doRunCatalogCalculation:
1057 self.catalogCalculation.
run(sources)
1059 self.setPrimaryFlags.
run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1061 if self.config.doPropagateFlags:
1062 self.propagateFlags.
run(butler, sources, ccdInputs, exposure.getWcs(), visitCatalogs, wcsUpdates)
1066 if self.config.doMatchSources:
1067 matchResult = self.match.
run(sources, exposure.getInfo().getFilter().getName())
1069 matches.table.setMetadata(matchResult.matchMeta)
1070 results.matchResult = matches
1071 if self.config.doWriteMatchesDenormalized:
1073 matchResult.matchMeta)
1075 results.outputSources = sources
1080 @brief Read input sources. 1082 @param[in] dataRef: Data reference for catalog of merged detections 1083 @return List of sources in merged catalog 1085 We also need to add columns to hold the measurements we're about to make 1086 so we can measure in-place. 1088 merged = dataRef.get(self.config.coaddName + self.
inputCatalog, immediate=
True)
1089 self.log.
info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
1092 idFactory.notify(s.getId())
1093 table = afwTable.SourceTable.make(self.
schema, idFactory)
1100 @brief Write matches of the sources to the astrometric reference catalog. 1102 @param[in] dataRef: data reference 1103 @param[in] results: results struct from run method 1105 if hasattr(results,
"matchResult"):
1106 dataRef.put(results.matchResult, self.config.coaddName +
"Coadd_measMatch")
1107 if hasattr(results,
"denormMatches"):
1108 dataRef.put(results.denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1112 @brief Write the source catalog. 1114 @param[in] dataRef: data reference 1115 @param[in] sources: source catalog 1117 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
1118 self.log.
info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1121 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
def InitOutputDatasetField
def readSources(self, dataRef)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def getTargetList(parsedCmd, kwargs)
def runDataRef(self, patchRef)
Run detection on a coadd.
def getPrerequisiteDatasetTypes(cls, config)
Class for storing ordered metadata with comments.
A mapping between the keys of two Schemas, used to copy data between them.
Configuration parameters for the DetectCoaddSourcesTask.
def denormalizeMatches(matches, matchMeta=None)
def runDataRef(self, patchRefList, psfCache=100)
def __init__(self, schema=None, kwargs)
Initialize the task.
def getInitOutputDatasets(self)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Deblend sources from master catalog in each coadd seperately and measure.
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations...
daf::base::PropertySet * set
def __init__(self, butler=None, schema=None, peakSchema=None, kwargs)
def getInitOutputDatasets(self)
def getOutputDatasetTypes(cls, config)
def writeMatches(self, dataRef, results)
Write matches of the sources to the astrometric reference catalog.
def getExposureId(self, dataRef)
def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None, butler=None)
def readSources(self, dataRef)
Read input sources.
def getSchemaCatalogs(self)
def write(self, results, patchRef)
Write out results from runDetection.
template BaseCatalog packMatches(SourceMatchVector const &)
def getInputDatasetTypes(cls, config)
def getExposureId(self, dataRef)
def write(self, dataRef, flux_sources, template_sources=None)
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None, kwargs)
Initialize the task.
def write(self, dataRef, sources)
Write the source catalog.
def InitInputDatasetField
def getTargetList(parsedCmd, kwargs)
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
def run(self, exposure, idFactory, expId)
Run detection on an exposure.
Detect sources on a coadd.
daf::base::PropertyList * list
def runDataRef(self, patchRef, psfCache=100)
Deblend and measure.
def writeMetadata(self, dataRefList)