24 from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
25 PipelineTask, PipelineTaskConfig, InitInputDatasetField,
26 InitOutputDatasetField, InputDatasetField, OutputDatasetField)
29 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
42 from .mergeDetections
import MergeDetectionsConfig, MergeDetectionsTask
43 from .mergeMeasurements
import MergeMeasurementsConfig, MergeMeasurementsTask
44 from .multiBandUtils
import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs
45 from .multiBandUtils
import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory
46 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleConfig
47 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleTask
48 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiConfig
49 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiTask
54 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) 55 * deepCoadd_mergeDet: merged detections (tract, patch) 56 * deepCoadd_meas: measurements of merged detections (tract, patch, filter) 57 * deepCoadd_ref: reference sources (tract, patch) 58 All of these have associated *_schema catalogs that require no data ID and hold no records. 60 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in 61 the mergeDet, meas, and ref dataset Footprints: 62 * deepCoadd_peak_schema 70 @anchor DetectCoaddSourcesConfig_ 72 @brief Configuration parameters for the DetectCoaddSourcesTask 74 doScaleVariance =
Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
77 coaddName =
Field(dtype=str, default=
"deep", doc=
"Name of coadd")
78 doInsertFakes =
Field(dtype=bool, default=
False,
79 doc=
"Run fake sources injection task")
81 doc=
"Injection of fake sources for testing " 82 "purposes (must be retargeted)")
84 doc=
"Schema of the detection catalog",
85 nameTemplate=
"{outputCoaddName}Coadd_det_schema",
86 storageClass=
"SourceCatalog",
89 doc=
"Exposure on which detections are to be performed",
90 nameTemplate=
"{inputCoaddName}Coadd",
92 storageClass=
"ExposureF",
93 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
96 doc=
"Output Backgrounds used in detection",
97 nameTemplate=
"{outputCoaddName}Coadd_calexp_background",
99 storageClass=
"Background",
100 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
103 doc=
"Detected sources catalog",
104 nameTemplate=
"{outputCoaddName}Coadd_det",
106 storageClass=
"SourceCatalog",
107 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
110 doc=
"Exposure post detection",
111 nameTemplate=
"{outputCoaddName}Coadd_calexp",
113 storageClass=
"ExposureF",
114 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
120 doc=
"Should be set to True if fake sources have been inserted into the input data." 125 self.quantum.dimensions = (
"tract",
"patch",
"abstract_filter",
"skymap")
126 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep"})
127 self.
detection.thresholdType =
"pixel_stdev" 130 self.
detection.reEstimateBackground =
False 131 self.
detection.background.useApprox =
False 133 self.
detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER' 134 self.
detection.doTempWideBackground =
True 146 @anchor DetectCoaddSourcesTask_ 148 @brief Detect sources on a coadd 150 @section pipe_tasks_multiBand_Contents Contents 152 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose 153 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize 154 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run 155 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config 156 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug 157 - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example 159 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description 161 Command-line task that detects sources on a coadd of exposures obtained with a single filter. 163 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise 164 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane 165 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should 166 propagate the full covariance matrix -- but it is simple and works well in practice. 168 After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref 169 SourceDetectionTask_ "detection" subtask. 172 deepCoadd{tract,patch,filter}: ExposureF 174 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 175 @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input 177 @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList 181 DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask. 182 You can retarget this subtask if you wish. 184 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization 186 @copydoc \_\_init\_\_ 188 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task 192 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters 194 See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig" 196 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables 198 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 199 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 202 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to 203 @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for 204 @ref SourceDetectionTask_ "SourceDetectionTask" for further information. 206 @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example 207 of using DetectCoaddSourcesTask 209 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of 210 the task is to update the background, detect all sources in a single band and generate a set of parent 211 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and, 212 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data 213 reference to the coadd to be processed. A list of the available optional arguments can be obtained by 214 calling detectCoaddSources.py with the `--help` command line argument: 216 detectCoaddSources.py --help 219 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 220 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed 221 steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows: 223 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 225 that will process the HSC-I band data. The results are written to 226 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 228 It is also necessary to run: 230 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 232 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 233 processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask". 235 _DefaultName =
"detectCoaddSources" 236 ConfigClass = DetectCoaddSourcesConfig
237 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
238 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
241 def _makeArgumentParser(cls):
243 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
244 ContainerClass=ExistingCoaddDataIdContainer)
249 @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask. 251 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 253 @param[in] schema: initial schema for the output catalog, modified-in place to include all 254 fields set by this task. If None, the source minimal schema will be used. 255 @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ 261 schema = afwTable.SourceTable.makeMinimalSchema()
262 if self.config.doInsertFakes:
263 self.makeSubtask(
"insertFakes")
265 self.makeSubtask(
"detection", schema=self.
schema)
266 if self.config.doScaleVariance:
267 self.makeSubtask(
"scaleVariance")
274 @brief Run detection on a coadd. 276 Invokes @ref run and then uses @ref write to output the 279 @param[in] patchRef: data reference for patch 281 if self.config.hasFakes:
282 exposure = patchRef.get(
"fakes_" + self.config.coaddName +
"Coadd", immediate=
True)
284 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
285 expId =
int(patchRef.get(self.config.coaddName +
"CoaddId"))
287 self.
write(results, patchRef)
291 packedId, maxBits = butler.registry.packDataId(
"tract_patch_abstract_filter",
292 inputDataIds[
"exposure"],
294 inputData[
"idFactory"] = afwTable.IdFactory.makeSource(packedId, 64 - maxBits)
295 inputData[
"expId"] = packedId
296 return self.
run(**inputData)
298 def run(self, exposure, idFactory, expId):
300 @brief Run detection on an exposure. 302 First scale the variance plane to match the observed variance 303 using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to 306 @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled, 307 depending on configuration). 308 @param[in] idFactory: IdFactory to set source identifiers 309 @param[in] expId: Exposure identifier (integer) for RNG seed 311 @return a pipe.base.Struct with fields 312 - sources: catalog of detections 313 - backgrounds: list of backgrounds 315 if self.config.doScaleVariance:
316 varScale = self.scaleVariance.
run(exposure.maskedImage)
317 exposure.getMetadata().add(
"variance_scale", varScale)
319 if self.config.doInsertFakes:
320 self.insertFakes.
run(exposure, background=backgrounds)
321 table = afwTable.SourceTable.make(self.
schema, idFactory)
322 detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
323 sources = detections.sources
324 fpSets = detections.fpSets
325 if hasattr(fpSets,
"background")
and fpSets.background:
326 for bg
in fpSets.background:
327 backgrounds.append(bg)
328 return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
332 @brief Write out results from runDetection. 334 @param[in] exposure: Exposure to write out 335 @param[in] results: Struct returned from runDetection 336 @param[in] patchRef: data reference for patch 338 coaddName = self.config.coaddName +
"Coadd" 339 patchRef.put(results.outputBackgrounds, coaddName +
"_calexp_background")
340 patchRef.put(results.outputSources, coaddName +
"_det")
341 if self.config.hasFakes:
342 patchRef.put(results.outputExposure,
"fakes_" + coaddName +
"_calexp")
344 patchRef.put(results.outputExposure, coaddName +
"_calexp")
350 """DeblendCoaddSourcesConfig 352 Configuration parameters for the `DeblendCoaddSourcesTask`. 355 doc=
"Deblend sources separately in each band")
357 doc=
"Deblend sources simultaneously across bands")
358 simultaneous =
Field(dtype=bool, default=
False, doc=
"Simultaneously deblend all bands?")
359 coaddName =
Field(dtype=str, default=
"deep", doc=
"Name of coadd")
362 doc=
"Should be set to True if fake sources have been inserted into the input data.")
365 Config.setDefaults(self)
370 """Task runner for the `MergeSourcesTask` 372 Required because the run method requires a list of 373 dataRefs rather than a single dataRef. 377 """Provide a list of patch references for each patch, tract, filter combo. 384 Keyword arguments passed to the task 389 List of tuples, where each tuple is a (dataRef, kwargs) pair. 391 refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
392 kwargs[
"psfCache"] = parsedCmd.psfCache
393 return [(
list(p.values()), kwargs)
for t
in refDict.values()
for p
in t.values()]
397 """Deblend the sources in a merged catalog 399 Deblend sources from master catalog in each coadd. 400 This can either be done separately in each band using the HSC-SDSS deblender 401 (`DeblendCoaddSourcesTask.config.simultaneous==False`) 402 or use SCARLET to simultaneously fit the blend in all bands 403 (`DeblendCoaddSourcesTask.config.simultaneous==True`). 404 The task will set its own `self.schema` atribute to the `Schema` of the 405 output deblended catalog. 406 This will include all fields from the input `Schema`, as well as additional fields 409 `pipe.tasks.multiband.DeblendCoaddSourcesTask Description 410 --------------------------------------------------------- 416 Butler used to read the input schemas from disk or 417 construct the reference catalog loader, if `schema` or `peakSchema` or 419 The schema of the merged detection catalog as an input to this task. 421 The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog 423 ConfigClass = DeblendCoaddSourcesConfig
424 RunnerClass = DeblendCoaddSourcesRunner
425 _DefaultName =
"deblendCoaddSources" 426 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
429 def _makeArgumentParser(cls):
431 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
432 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
433 ContainerClass=ExistingCoaddDataIdContainer)
434 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
437 def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
438 CmdLineTask.__init__(self, **kwargs)
440 assert butler
is not None,
"Neither butler nor schema is defined" 441 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
445 if peakSchema
is None:
446 assert butler
is not None,
"Neither butler nor peakSchema is defined" 447 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
449 if self.config.simultaneous:
450 self.makeSubtask(
"multiBandDeblend", schema=self.
schema, peakSchema=peakSchema)
452 self.makeSubtask(
"singleBandDeblend", schema=self.
schema, peakSchema=peakSchema)
455 """Return a dict of empty catalogs for each catalog dataset produced by this task. 460 Dictionary of empty catalogs, with catalog names as keys. 463 return {self.config.coaddName +
"Coadd_deblendedFlux": catalog,
464 self.config.coaddName +
"Coadd_deblendedModel": catalog}
469 Deblend each source simultaneously or separately 470 (depending on `DeblendCoaddSourcesTask.config.simultaneous`). 471 Set `is-primary` and related flags. 472 Propagate flags from individual visits. 473 Write the deblended sources out. 478 List of data references for each filter 481 if self.config.hasFakes:
482 coaddType =
"fakes_" + self.config.coaddName
484 coaddType = self.config.coaddName
486 if self.config.simultaneous:
490 for patchRef
in patchRefList:
491 exposure = patchRef.get(coaddType +
"Coadd_calexp", immediate=
True)
492 filters.append(patchRef.dataId[
"filter"])
493 exposures.append(exposure)
496 exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
497 fluxCatalogs, templateCatalogs = self.multiBandDeblend.
run(exposure, sources)
498 for n
in range(len(patchRefList)):
499 self.
write(patchRefList[n], fluxCatalogs[filters[n]], templateCatalogs[filters[n]])
502 for patchRef
in patchRefList:
503 exposure = patchRef.get(coaddType +
"Coadd_calexp", immediate=
True)
504 exposure.getPsf().setCacheCapacity(psfCache)
506 self.singleBandDeblend.
run(exposure, sources)
507 self.
write(patchRef, sources)
510 """Read merged catalog 512 Read the catalog of merged detections and create a catalog 517 dataRef: data reference 518 Data reference for catalog of merged detections 522 sources: `SourceCatalog` 523 List of sources in merged catalog 525 We also need to add columns to hold the measurements we're about to make 526 so we can measure in-place. 528 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
529 self.log.
info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
532 idFactory.notify(s.getId())
533 table = afwTable.SourceTable.make(self.
schema, idFactory)
538 def write(self, dataRef, flux_sources, template_sources=None):
539 """Write the source catalog(s) 543 dataRef: Data Reference 544 Reference to the output catalog. 545 flux_sources: `SourceCatalog` 546 Flux conserved sources to write to file. 547 If using the single band deblender, this is the catalog 549 template_sources: `SourceCatalog` 550 Source catalog using the multiband template models 555 if flux_sources
is not None:
556 assert not self.config.simultaneous
or self.config.multiBandDeblend.conserveFlux
557 dataRef.put(flux_sources, self.config.coaddName +
"Coadd_deblendedFlux")
561 if template_sources
is not None:
562 assert self.config.multiBandDeblend.saveTemplates
563 dataRef.put(template_sources, self.config.coaddName +
"Coadd_deblendedModel")
564 self.log.
info(
"Wrote %d sources: %s" % (len(flux_sources), dataRef.dataId))
567 """Write the metadata produced from processing the data. 571 List of Butler data references used to write the metadata. 572 The metadata is written to dataset type `CmdLineTask._getMetadataName`. 574 for dataRef
in dataRefList:
576 metadataName = self._getMetadataName()
577 if metadataName
is not None:
578 dataRef.put(self.getFullMetadata(), metadataName)
579 except Exception
as e:
580 self.log.
warn(
"Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
583 """Get the ExposureId from a data reference 585 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
590 @anchor MeasureMergedCoaddSourcesConfig_ 592 @brief Configuration parameters for the MeasureMergedCoaddSourcesTask 594 inputCatalog =
Field(dtype=str, default=
"deblendedFlux",
595 doc=(
"Name of the input catalog to use." 596 "If the single band deblender was used this should be 'deblendedFlux." 597 "If the multi-band deblender was used this should be 'deblendedModel." 598 "If no deblending was performed this should be 'mergeDet'"))
600 setPrimaryFlags =
ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
602 dtype=bool, default=
True,
603 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" 605 propagateFlags =
ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
606 doMatchSources =
Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
611 doc=(
"Write reference matches in denormalized format? " 612 "This format uses more disk space, but is more convenient to read."),
614 coaddName =
Field(dtype=str, default=
"deep", doc=
"Name of coadd")
615 psfCache =
Field(dtype=int, default=100, doc=
"Size of psfCache")
617 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
624 doc=
"Apply aperture corrections" 627 target=ApplyApCorrTask,
628 doc=
"Subtask to apply aperture corrections" 633 doc=
'Run catalogCalculation task' 636 target=CatalogCalculationTask,
637 doc=
"Subtask to run catalogCalculation plugins on catalog" 640 doc=
"Input schema for measure merged task produced by a deblender or detection task",
641 nameTemplate=
"{inputCoaddName}Coadd_deblendedFlux_schema",
642 storageClass=
"SourceCatalog" 645 doc=
"Output schema after all new fields are added by task",
646 nameTemplate=
"{inputCoaddName}Coadd_meas_schema",
647 storageClass=
"SourceCatalog" 650 doc=
"Reference catalog used to match measured sources against known sources",
652 storageClass=
"SimpleCatalog",
653 dimensions=(
"skypix",),
657 doc=
"Input coadd image",
658 nameTemplate=
"{inputCoaddName}Coadd_calexp",
660 storageClass=
"ExposureF",
661 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
664 doc=
"SkyMap to use in processing",
665 nameTemplate=
"{inputCoaddName}Coadd_skyMap",
666 storageClass=
"SkyMap",
667 dimensions=(
"skymap",),
671 doc=
"Source catalogs for visits which overlap input tract, patch, abstract_filter. Will be " 672 "further filtered in the task for the purpose of propagating flags from image calibration " 673 "and characterization to codd objects",
675 dimensions=(
"instrument",
"visit",
"detector"),
676 storageClass=
"SourceCatalog" 679 doc=(
"Name of the input catalog to use." 680 "If the single band deblender was used this should be 'deblendedFlux." 681 "If the multi-band deblender was used this should be 'deblendedModel, " 682 "or deblendedFlux if the multiband deblender was configured to output " 683 "deblended flux catalogs. If no deblending was performed this should " 685 nameTemplate=
"{inputCoaddName}Coadd_deblendedFlux",
686 storageClass=
"SourceCatalog",
687 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap"),
691 doc=
"Source catalog containing all the measurement information generated in this task",
692 nameTemplate=
"{outputCoaddName}Coadd_meas",
693 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap"),
694 storageClass=
"SourceCatalog",
698 doc=
"Match catalog produced by configured matcher, optional on doMatchSources",
699 nameTemplate=
"{outputCoaddName}Coadd_measMatch",
700 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap"),
701 storageClass=
"Catalog",
705 doc=
"Denormalized Match catalog produced by configured matcher, optional on " 706 "doWriteMatchesDenormalized",
707 nameTemplate=
"{outputCoaddName}Coadd_measMatchFull",
708 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap"),
709 storageClass=
"Catalog",
716 doc=
"Should be set to True if fake sources have been inserted into the input data." 721 return self.
match.refObjLoader
725 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep"})
726 self.quantum.dimensions = (
"tract",
"patch",
"abstract_filter",
"skymap")
727 self.
measurement.plugins.names |= [
'base_InputCount',
'base_Variance']
728 self.
measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED',
'SENSOR_EDGE',
730 self.
measurement.plugins[
'base_PixelFlags'].masksFpCenter = [
'CLIPPED',
'SENSOR_EDGE',
742 """Get the psfCache setting into MeasureMergedCoaddSourcesTask""" 745 return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
750 @anchor MeasureMergedCoaddSourcesTask_ 752 @brief Deblend sources from master catalog in each coadd seperately and measure. 754 @section pipe_tasks_multiBand_Contents Contents 756 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose 757 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize 758 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run 759 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config 760 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug 761 - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example 763 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description 765 Command-line task that uses peaks and footprints from a master catalog to perform deblending and 766 measurement in each coadd. 768 Given a master input catalog of sources (peaks and footprints) or deblender outputs 769 (including a HeavyFootprint in each band), measure each source on the 770 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a 771 consistent set of child sources. 773 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source 774 properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit 775 flags are propagated to the coadd sources. 777 Optionally, we can match the coadd sources to an external reference catalog. 780 deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog 781 @n deepCoadd_calexp{tract,patch,filter}: ExposureF 783 deepCoadd_meas{tract,patch,filter}: SourceCatalog 787 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks: 790 <DT> @ref SingleFrameMeasurementTask_ "measurement" 791 <DD> Measure source properties of deblended sources.</DD> 792 <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags" 793 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are 794 not at the edge of the field and that have either not been deblended or are the children of deblended 796 <DT> @ref PropagateVisitFlagsTask_ "propagateFlags" 797 <DD> Propagate flags set in individual visits to the coadd.</DD> 798 <DT> @ref DirectMatchTask_ "match" 799 <DD> Match input sources to a reference catalog (optional). 802 These subtasks may be retargeted as required. 804 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization 806 @copydoc \_\_init\_\_ 808 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task 812 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters 814 See @ref MeasureMergedCoaddSourcesConfig_ 816 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables 818 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 819 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 822 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to 823 the various sub-tasks. See the documetation for individual sub-tasks for more information. 825 @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using 826 MeasureMergedCoaddSourcesTask 828 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. 829 The next stage in the multi-band processing procedure will merge these measurements into a suitable 830 catalog for driving forced photometry. 832 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds 834 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the 835 `--help` command line argument: 837 measureCoaddSources.py --help 840 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 841 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 842 step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band 845 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 847 This will process the HSC-I band data. The results are written in 848 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits 850 It is also necessary to run 852 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 854 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 855 procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask". 857 _DefaultName =
"measureCoaddSources" 858 ConfigClass = MeasureMergedCoaddSourcesConfig
859 RunnerClass = MeasureMergedCoaddSourcesRunner
860 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
861 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
864 def _makeArgumentParser(cls):
866 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
867 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
868 ContainerClass=ExistingCoaddDataIdContainer)
869 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
872 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
875 @brief Initialize the task. 877 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 878 @param[in] schema: the schema of the merged detection catalog used as input to this one 879 @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog 880 @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference 881 catalog. May be None if the loader can be constructed from the butler argument or all steps 882 requiring a reference catalog are disabled. 883 @param[in] butler: a butler used to read the input schemas from disk or construct the reference 884 catalog loader, if schema or peakSchema or refObjLoader is None 886 The task will set its own self.schema attribute to the schema of the output measurement catalog. 887 This will include all fields from the input schema, as well as additional fields for all the 891 self.
deblended = self.config.inputCatalog.startswith(
"deblended")
893 if initInputs
is not None:
894 schema = initInputs[
'inputSchema'].schema
896 assert butler
is not None,
"Neither butler nor schema is defined" 897 schema = butler.get(self.config.coaddName + self.
inputCatalog +
"_schema", immediate=
True).schema
902 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
903 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
904 if self.config.doMatchSources:
905 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
906 if self.config.doPropagateFlags:
907 self.makeSubtask(
"propagateFlags", schema=self.
schema)
908 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
909 if self.config.doApCorr:
910 self.makeSubtask(
"applyApCorr", schema=self.
schema)
911 if self.config.doRunCatalogCalculation:
912 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
917 if not config.doPropagateFlags:
918 inputDatasetTypes.pop(
"visitCatalogs")
919 return inputDatasetTypes
924 if config.doMatchSources
is False:
925 outputDatasetTypes.pop(
"matchResult")
926 if config.doWriteMatchesDenormalized
is False:
927 outputDatasetTypes.pop(
"denormMatches")
928 return outputDatasetTypes
932 return frozenset([
"refCat"])
939 config=self.config.refObjLoader, log=self.log)
940 self.match.setRefObjLoader(refObjLoader)
944 inputData[
'exposure'].getPsf().setCacheCapacity(self.config.psfCache)
947 packedId, maxBits = butler.registry.packDataId(
"tract_patch", outputDataIds[
"outputSources"],
949 inputData[
'exposureId'] = packedId
950 idFactory = afwTable.IdFactory.makeSource(packedId, 64 - maxBits)
952 table = afwTable.SourceTable.make(self.
schema, idFactory)
954 sources.extend(inputData.pop(
'intakeCatalog'), self.
schemaMapper)
955 table = sources.getTable()
957 inputData[
'sources'] = sources
959 skyMap = inputData.pop(
'skyMap')
960 tractNumber = inputDataIds[
'intakeCatalog'][
'tract']
961 tractInfo = skyMap[tractNumber]
962 patchInfo = tractInfo.getPatchInfo(inputDataIds[
'intakeCatalog'][
'patch'])
967 wcs=tractInfo.getWcs(),
968 bbox=patchInfo.getOuterBBox()
970 inputData[
'skyInfo'] = skyInfo
972 if self.config.doPropagateFlags:
974 ccdInputs = inputData[
'exposure'].
getInfo().getCoaddInputs().ccds
975 visitKey = ccdInputs.schema.find(
"visit").key
976 ccdKey = ccdInputs.schema.find(
"ccd").key
977 inputVisitIds =
set()
979 for ccdRecord
in ccdInputs:
980 visit = ccdRecord.get(visitKey)
981 ccd = ccdRecord.get(ccdKey)
982 inputVisitIds.add((visit, ccd))
983 ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs()
985 inputCatalogsToKeep = []
986 inputCatalogWcsUpdate = []
987 for i, dataId
in enumerate(inputDataIds[
'visitCatalogs']):
988 key = (dataId[
'visit'], dataId[
'detector'])
989 if key
in inputVisitIds:
990 inputCatalogsToKeep.append(inputData[
'visitCatalogs'][i])
991 inputCatalogWcsUpdate.append(ccdRecordsWcs[key])
992 inputData[
'visitCatalogs'] = inputCatalogsToKeep
993 inputData[
'wcsUpdates'] = inputCatalogWcsUpdate
994 inputData[
'ccdInputs'] = ccdInputs
996 return self.
run(**inputData)
1000 @brief Deblend and measure. 1002 @param[in] patchRef: Patch reference. 1004 Set 'is-primary' and related flags. Propagate flags 1005 from individual visits. Optionally match the sources to a reference catalog and write the matches. 1006 Finally, write the deblended sources and measurements out. 1008 if self.config.hasFakes:
1009 coaddType =
"fakes_" + self.config.coaddName
1011 coaddType = self.config.coaddName
1012 exposure = patchRef.get(coaddType +
"Coadd_calexp", immediate=
True)
1013 exposure.getPsf().setCacheCapacity(psfCache)
1015 table = sources.getTable()
1017 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1019 if self.config.doPropagateFlags:
1020 ccdInputs = self.propagateFlags.getCcdInputs(exposure)
1024 results = self.
run(exposure=exposure, sources=sources,
1025 ccdInputs=ccdInputs,
1026 skyInfo=skyInfo, butler=patchRef.getButler(),
1029 if self.config.doMatchSources:
1031 self.
write(patchRef, results.outputSources)
1033 def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None,
1035 """Run measurement algorithms on the input exposure, and optionally populate the 1036 resulting catalog with extra information. 1040 exposure : `lsst.afw.exposure.Exposure` 1041 The input exposure on which measurements are to be performed 1042 sources : `lsst.afw.table.SourceCatalog` 1043 A catalog built from the results of merged detections, or 1045 skyInfo : `lsst.pipe.base.Struct` 1046 A struct containing information about the position of the input exposure within 1047 a `SkyMap`, the `SkyMap`, its `Wcs`, and its bounding box 1048 exposureId : `int` or `bytes` 1049 packed unique number or bytes unique to the input exposure 1050 ccdInputs : `lsst.afw.table.ExposureCatalog` 1051 Catalog containing information on the individual visits which went into making 1053 visitCatalogs : list of `lsst.afw.table.SourceCatalogs` or `None` 1054 A list of source catalogs corresponding to measurements made on the individual 1055 visits which went into the input exposure. If None and butler is `None` then 1056 the task cannot propagate visit flags to the output catalog. 1057 wcsUpdates : list of `lsst.afw.geom.SkyWcs` or `None` 1058 If visitCatalogs is not `None` this should be a list of wcs objects which correspond 1059 to the input visits. Used to put all coordinates to common system. If `None` and 1060 butler is `None` then the task cannot propagate visit flags to the output catalog. 1061 butler : `lsst.daf.butler.Butler` or `lsst.daf.persistence.Butler` 1062 Either a gen2 or gen3 butler used to load visit catalogs 1066 results : `lsst.pipe.base.Struct` 1067 Results of running measurement task. Will contain the catalog in the 1068 sources attribute. Optionally will have results of matching to a 1069 reference catalog in the matchResults attribute, and denormalized 1070 matches in the denormMatches attribute. 1072 self.measurement.
run(sources, exposure, exposureId=exposureId)
1074 if self.config.doApCorr:
1075 self.applyApCorr.
run(
1077 apCorrMap=exposure.getInfo().getApCorrMap()
1084 if not sources.isContiguous():
1085 sources = sources.copy(deep=
True)
1087 if self.config.doRunCatalogCalculation:
1088 self.catalogCalculation.
run(sources)
1090 self.setPrimaryFlags.
run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1092 if self.config.doPropagateFlags:
1093 self.propagateFlags.
run(butler, sources, ccdInputs, exposure.getWcs(), visitCatalogs, wcsUpdates)
1097 if self.config.doMatchSources:
1098 matchResult = self.match.
run(sources, exposure.getInfo().getFilter().getName())
1100 matches.table.setMetadata(matchResult.matchMeta)
1101 results.matchResult = matches
1102 if self.config.doWriteMatchesDenormalized:
1103 if matchResult.matches:
1106 self.log.
warn(
"No matches, so generating dummy denormalized matches file")
1109 denormMatches.getMetadata().add(
"COMMENT",
1110 "This catalog is empty because no matches were found.")
1111 results.denormMatches = denormMatches
1112 results.denormMatches = denormMatches
1114 results.outputSources = sources
1119 @brief Read input sources. 1121 @param[in] dataRef: Data reference for catalog of merged detections 1122 @return List of sources in merged catalog 1124 We also need to add columns to hold the measurements we're about to make 1125 so we can measure in-place. 1127 merged = dataRef.get(self.config.coaddName + self.
inputCatalog, immediate=
True)
1128 self.log.
info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
1131 idFactory.notify(s.getId())
1132 table = afwTable.SourceTable.make(self.
schema, idFactory)
1139 @brief Write matches of the sources to the astrometric reference catalog. 1141 @param[in] dataRef: data reference 1142 @param[in] results: results struct from run method 1144 if hasattr(results,
"matchResult"):
1145 dataRef.put(results.matchResult, self.config.coaddName +
"Coadd_measMatch")
1146 if hasattr(results,
"denormMatches"):
1147 dataRef.put(results.denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1151 @brief Write the source catalog. 1153 @param[in] dataRef: data reference 1154 @param[in] sources: source catalog 1156 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
1157 self.log.
info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1160 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
Defines the fields and offsets for a table.
def InitOutputDatasetField
def readSources(self, dataRef)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def getTargetList(parsedCmd, kwargs)
def runDataRef(self, patchRef)
Run detection on a coadd.
def getPrerequisiteDatasetTypes(cls, config)
Class for storing ordered metadata with comments.
A mapping between the keys of two Schemas, used to copy data between them.
Configuration parameters for the DetectCoaddSourcesTask.
def denormalizeMatches(matches, matchMeta=None)
def runDataRef(self, patchRefList, psfCache=100)
def __init__(self, schema=None, kwargs)
Initialize the task.
def getInitOutputDatasets(self)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Deblend sources from master catalog in each coadd seperately and measure.
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations...
daf::base::PropertySet * set
def __init__(self, butler=None, schema=None, peakSchema=None, kwargs)
def getInitOutputDatasets(self)
def getOutputDatasetTypes(cls, config)
def writeMatches(self, dataRef, results)
Write matches of the sources to the astrometric reference catalog.
def getExposureId(self, dataRef)
def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None, butler=None)
def readSources(self, dataRef)
Read input sources.
def getSchemaCatalogs(self)
def write(self, results, patchRef)
Write out results from runDetection.
template BaseCatalog packMatches(SourceMatchVector const &)
def getInputDatasetTypes(cls, config)
def getExposureId(self, dataRef)
def write(self, dataRef, flux_sources, template_sources=None)
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None, kwargs)
Initialize the task.
def write(self, dataRef, sources)
Write the source catalog.
def InitInputDatasetField
def getTargetList(parsedCmd, kwargs)
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...
def run(self, exposure, idFactory, expId)
Run detection on an exposure.
Detect sources on a coadd.
daf::base::PropertyList * list
def runDataRef(self, patchRef, psfCache=100)
Deblend and measure.
def writeMetadata(self, dataRefList)