22__all__ = [
"BaseSelectImagesTask",
"BaseExposureInfo",
"WcsSelectImagesTask",
"PsfWcsSelectImagesTask",
23 "DatabaseSelectImagesConfig",
"BestSeeingSelectVisitsTask",
24 "BestSeeingQuantileSelectVisitsTask"]
34from lsst.utils.timer
import timeMethod
38 """Base configuration for subclasses of BaseSelectImagesTask that use a
42 host = pexConfig.Field(
43 doc=
"Database server host name",
46 port = pexConfig.Field(
47 doc=
"Database server port",
50 database = pexConfig.Field(
51 doc=
"Name of database",
54 maxExposures = pexConfig.Field(
55 doc=
"maximum exposures to select; intended for debugging; ignored if None",
62 """Data about a selected exposure.
67 Data ID keys of exposure.
68 coordList : `list` [`lsst.afw.geom.SpherePoint`]
69 ICRS coordinates of the corners of the exposure
70 plus any others items that are desired.
74 super(BaseExposureInfo, self).
__init__(dataId=dataId, coordList=coordList)
78 """Base task for selecting images suitable for coaddition.
81 ConfigClass = pexConfig.Config
82 _DefaultName =
"selectImages"
85 def run(self, coordList):
86 """Select images suitable for coaddition in a particular region.
90 coordList : `list` [`lsst.geom.SpherePoint`] or `None`
91 List of coordinates defining region of interest; if `None`, then
92 select all images subclasses may add additional keyword arguments,
97 result : `pipeBase.Struct`
98 Results as a struct with attributes:
101 A list of exposure information objects (subclasses of
102 BaseExposureInfo), which have at least the following fields:
103 - dataId: Data ID dictionary (`dict`).
104 - coordList: ICRS coordinates of the corners of the exposure.
105 (`list` [`lsst.geom.SpherePoint`])
107 raise NotImplementedError()
85 def run(self, coordList):
…
111 """Extract the keys and values from a list of dataIds.
113 The input dataList is a list of objects that have 'dataId' members.
114 This allows it to be used for both a list of data references and a
115 list of ExposureInfo.
130 Raised if DataId keys are inconsistent.
132 assert len(dataList) > 0
134 keys = sorted(dataList[0].dataId.keys())
137 for data
in dataList:
138 thisKeys = set(data.dataId.keys())
139 if thisKeys != keySet:
140 raise RuntimeError(
"DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
141 values.append(tuple(data.dataId[k]
for k
in keys))
146 """A container for data to be passed to the WcsSelectImagesTask.
152 wcs : `lsst.afw.geom.SkyWcs`
153 Coordinate system definition (wcs).
154 bbox : `lsst.geom.box.Box2I`
155 Integer bounding box for image.
159 super(SelectStruct, self).
__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
163 """Select images using their Wcs.
165 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
166 polygons on the celestial sphere, and test the polygon of the
167 patch for overlap with the polygon of the image.
169 We use "convexHull" instead of generating a ConvexPolygon
170 directly because the standard for the inputs to ConvexPolygon
171 are pretty high and we don't want to be responsible for reaching them.
174 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
175 """Return indices of provided lists that meet the selection criteria.
179 wcsList : `list` [`lsst.afw.geom.SkyWcs`]
180 Specifying the WCS's of the input ccds to be selected.
181 bboxList : `list` [`lsst.geom.Box2I`]
182 Specifying the bounding boxes of the input ccds to be selected.
183 coordList : `list` [`lsst.geom.SpherePoint`]
184 ICRS coordinates specifying boundary of the patch.
185 dataIds : iterable [`lsst.daf.butler.dataId`] or `None`, optional
186 An iterable object of dataIds which point to reference catalogs.
188 Additional keyword arguments.
192 result : `list` [`int`]
193 The indices of selected ccds.
196 dataIds = [
None] * len(wcsList)
197 patchVertices = [coord.getVector()
for coord
in coordList]
200 for i, (imageWcs, imageBox, dataId)
in enumerate(zip(wcsList, bboxList, dataIds)):
202 self.log.info(
"De-selecting exposure %s: Exposure has no WCS.", dataId)
174 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
…
210 """Return corners or `None` if bad.
216 patchPoly : `Unknown`
220 imageCorners = [imageWcs.pixelToSky(pix)
for pix
in geom.Box2D(imageBox).getCorners()]
221 except (pexExceptions.DomainError, pexExceptions.RuntimeError)
as e:
223 self.log.debug(
"WCS error in testing calexp %s (%s): deselecting", dataId, e)
227 if imagePoly
is None:
228 self.log.debug(
"Unable to create polygon from image %s: deselecting", dataId)
231 if patchPoly.intersects(imagePoly):
233 self.log.info(
"Selecting calexp %s", dataId)
240 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
241 defaultTemplates={
"coaddName":
"deep"}):
245class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
246 pipelineConnections=PsfWcsSelectImagesConnections):
247 maxEllipResidual = pexConfig.Field(
248 doc=
"Maximum median ellipticity residual",
253 maxSizeScatter = pexConfig.Field(
254 doc=
"Maximum scatter in the size residuals",
258 maxScaledSizeScatter = pexConfig.Field(
259 doc=
"Maximum scatter in the size residuals, scaled by the median size",
264 maxPsfTraceRadiusDelta = pexConfig.Field(
265 doc=
"Maximum delta (max - min) of model PSF trace radius values evaluated on a grid on "
266 "the unmasked detector pixels (pixel).",
271 maxPsfApFluxDelta = pexConfig.Field(
272 doc=
"Maximum delta (max - min) of model PSF aperture flux (with aperture radius of "
273 "max(2, 3*psfSigma)) values evaluated on a grid on the unmasked detector pixels (based "
274 "on a normalized-to-one flux).",
279 maxPsfApCorrSigmaScaledDelta = pexConfig.Field(
280 doc=
"Maximum delta (max - min) of model PSF aperture correction values evaluated on a grid "
281 "on the unmasked detector pixels scaled (divided) by the measured model psfSigma.",
286 minNPsfStarPerBand = pexConfig.DictField(
298 doc=
"Minimum number of PSF stars for the final PSF model to be considered "
299 "well-constrained and suitible for inclusion in the coadd. This number should "
300 "take into consideration the spatial order used for the PSF model. If the current "
301 "band for the exposure is not included as a key in this dict, the value associated "
302 "with the \"fallback\" key will be used.",
307 if "fallback" not in self.minNPsfStarPerBand:
308 msg = (
"Must include a \"fallback\" key in the config.minNPsfStarPerBand config dict. "
309 f
"It is currenly: {self.minNPsfStarPerBand}.")
310 raise pexConfig.FieldValidationError(self.__class__.minNPsfStarPerBand, self, msg)
314 """Select images using their Wcs and cuts on the PSF properties.
316 The PSF quality criteria are based on the size and ellipticity
317 residuals from the adaptive second moments of the star and the PSF.
320 - the median of the ellipticty residuals.
321 - the robust scatter of the size residuals (using the median absolute
323 - the robust scatter of the size residuals scaled by the square of
327 ConfigClass = PsfWcsSelectImagesConfig
328 _DefaultName =
"PsfWcsSelectImages"
330 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs):
331 """Return indices of provided lists that meet the selection criteria.
335 wcsList : `list` [`lsst.afw.geom.SkyWcs`]
336 Specifying the WCS's of the input ccds to be selected.
337 bboxList : `list` [`lsst.geom.Box2I`]
338 Specifying the bounding boxes of the input ccds to be selected.
339 coordList : `list` [`lsst.geom.SpherePoint`]
340 ICRS coordinates specifying boundary of the patch.
341 visitSummary : `list` [`lsst.afw.table.ExposureCatalog`]
342 containing the PSF shape information for the input ccds to be
344 dataIds : iterable [`lsst.daf.butler.dataId`] or `None`, optional
345 An iterable object of dataIds which point to reference catalogs.
347 Additional keyword arguments.
351 goodPsf : `list` [`int`]
352 The indices of selected ccds.
354 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
355 coordList=coordList, dataIds=dataIds)
359 for i, dataId
in enumerate(dataIds):
362 if self.
isValid(visitSummary, dataId[
"detector"]):
330 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs):
…
368 """Should this ccd be selected based on its PSF shape information.
372 visitSummary : `lsst.afw.table.ExposureCatalog`
373 Exposure catalog with per-detector summary information.
382 row = visitSummary.find(detectorId)
385 self.log.warning(
"Removing detector %d because summary stats not available.", detectorId)
389 if band
in self.config.minNPsfStarPerBand:
390 minNPsfStar = self.config.minNPsfStarPerBand[band]
392 minNPsfStar = self.config.minNPsfStarPerBand[
"fallback"]
393 nPsfStar = row[
"nPsfStar"]
394 medianE = np.sqrt(row[
"psfStarDeltaE1Median"]**2. + row[
"psfStarDeltaE2Median"]**2.)
395 scatterSize = row[
"psfStarDeltaSizeScatter"]
396 scaledScatterSize = row[
"psfStarScaledDeltaSizeScatter"]
397 psfTraceRadiusDelta = row[
"psfTraceRadiusDelta"]
398 psfApFluxDelta = row[
"psfApFluxDelta"]
399 psfApCorrSigmaScaledDelta = row[
"psfApCorrSigmaScaledDelta"]
402 if self.config.maxEllipResidual
and not (medianE <= self.config.maxEllipResidual):
403 self.log.info(
"Removing visit %d detector %d because median e residual too large: %f vs %f",
404 row[
"visit"], detectorId, medianE, self.config.maxEllipResidual)
406 elif self.config.maxSizeScatter
and not (scatterSize <= self.config.maxSizeScatter):
407 self.log.info(
"Removing visit %d detector %d because size scatter too large: %f vs %f",
408 row[
"visit"], detectorId, scatterSize, self.config.maxSizeScatter)
410 elif self.config.maxScaledSizeScatter
and not (scaledScatterSize <= self.config.maxScaledSizeScatter):
411 self.log.info(
"Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
412 row[
"visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
415 self.config.maxPsfTraceRadiusDelta
is not None
416 and not (psfTraceRadiusDelta <= self.config.maxPsfTraceRadiusDelta)
419 "Removing visit %d detector %d because max-min delta of model PSF trace radius values "
420 "across the unmasked detector pixels is not finite or too large: %.3f vs %.3f (pixels)",
421 row[
"visit"], detectorId, psfTraceRadiusDelta, self.config.maxPsfTraceRadiusDelta
425 self.config.maxPsfApFluxDelta
is not None
426 and not (psfApFluxDelta <= self.config.maxPsfApFluxDelta)
429 "Removing visit %d detector %d because max-min delta of model PSF aperture flux values "
430 "across the unmasked detector pixels is not finite or too large: %.3f vs %.3f (pixels)",
431 row[
"visit"], detectorId, psfApFluxDelta, self.config.maxPsfApFluxDelta
435 self.config.maxPsfApCorrSigmaScaledDelta
is not None
436 and not (psfApCorrSigmaScaledDelta <= self.config.maxPsfApCorrSigmaScaledDelta)
439 "Removing visit %d detector %d because max-min delta of the model PSF apterture correction "
440 "values scaled (divided) by the psfSigma across the unmasked detector pixels is not "
441 "finite or too large: %.3f vs %.3f (fatcor)",
442 row[
"visit"], detectorId, psfApCorrSigmaScaledDelta, self.config.maxPsfApCorrSigmaScaledDelta
445 elif minNPsfStar
and (nPsfStar < minNPsfStar):
447 "Removing visit %d detector %d because the PSF model had too few stars: %d vs %d",
448 row[
"visit"], detectorId, nPsfStar, minNPsfStar
456 dimensions=(
"tract",
"patch",
"skymap",
"band",
"instrument"),
457 defaultTemplates={
"coaddName":
"goodSeeing",
459 skyMap = pipeBase.connectionTypes.Input(
460 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
461 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
462 storageClass=
"SkyMap",
463 dimensions=(
"skymap",),
465 visitSummaries = pipeBase.connectionTypes.Input(
466 doc=
"Per-visit consolidated exposure metadata",
467 name=
"finalVisitSummary",
468 storageClass=
"ExposureCatalog",
469 dimensions=(
"instrument",
"visit",),
473 goodVisits = pipeBase.connectionTypes.Output(
474 doc=
"Selected visits to be coadded.",
475 name=
"{coaddName}Visits",
476 storageClass=
"StructuredDataDict",
477 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band"),
481class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
482 pipelineConnections=BestSeeingSelectVisitsConnections):
483 nVisitsMax = pexConfig.RangeField(
485 doc=
"Maximum number of visits to select; use -1 to select all.",
489 maxPsfFwhm = pexConfig.Field(
491 doc=
"Maximum PSF FWHM (in arcseconds) to select",
495 minPsfFwhm = pexConfig.Field(
497 doc=
"Minimum PSF FWHM (in arcseconds) to select",
501 doConfirmOverlap = pexConfig.Field(
503 doc=
"Do remove visits that do not actually overlap the patch?",
506 minMJD = pexConfig.Field(
508 doc=
"Minimum visit MJD to select",
512 maxMJD = pexConfig.Field(
514 doc=
"Maximum visit MJD to select",
520class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
521 """Select up to a maximum number of the best-seeing visits.
523 Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
524 This Task is a port of the Gen2 image-selector used in the AP pipeline:
525 BestSeeingSelectImagesTask. This Task selects full visits based on the
526 average PSF of the entire visit.
529 ConfigClass = BestSeeingSelectVisitsConfig
530 _DefaultName =
'bestSeeingSelectVisits'
532 def runQuantum(self, butlerQC, inputRefs, outputRefs):
533 inputs = butlerQC.get(inputRefs)
534 quantumDataId = butlerQC.quantum.dataId
535 outputs = self.run(**inputs, dataId=quantumDataId)
536 butlerQC.put(outputs, outputRefs)
538 def run(self, visitSummaries, skyMap, dataId):
543 visitSummary : `list` [`lsst.pipe.base.connections.DeferredDatasetRef`]
544 List of `lsst.pipe.base.connections.DeferredDatasetRef` of
545 visitSummary tables of type `lsst.afw.table.ExposureCatalog`.
546 skyMap : `lsst.skyMap.SkyMap`
547 SkyMap for checking visits overlap patch.
548 dataId : `dict` of dataId keys
549 For retrieving patch info for checking visits overlap patch.
553 result : `lsst.pipe.base.Struct`
554 Results as a struct with attributes:
557 A `dict` with selected visit ids as keys,
558 so that it can be be saved as a StructuredDataDict.
559 StructuredDataList's are currently limited.
561 if self.config.doConfirmOverlap:
562 patchPolygon = self.makePatchPolygon(skyMap, dataId)
564 inputVisits = [visitSummary.ref.dataId[
'visit']
for visitSummary
in visitSummaries]
567 for visit, visitSummary
in zip(inputVisits, visitSummaries):
569 visitSummary = visitSummary.get()
573 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
575 pixelScales = np.array([vs[
'pixelScale']
for vs
in visitSummary
if vs.getWcs()])
577 psfSigmas = np.array([vs[
'psfSigma']
for vs
in visitSummary
if vs.getWcs()])
578 fwhm = np.nanmean(psfSigmas * pixelScales) * np.sqrt(8.*np.log(2.))
580 if self.config.maxPsfFwhm
and fwhm > self.config.maxPsfFwhm:
582 if self.config.minPsfFwhm
and fwhm < self.config.minPsfFwhm:
584 if self.config.minMJD
and mjd < self.config.minMJD:
585 self.log.debug(
'MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
587 if self.config.maxMJD
and mjd > self.config.maxMJD:
588 self.log.debug(
'MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
590 if self.config.doConfirmOverlap
and not self.doesIntersectPolygon(visitSummary, patchPolygon):
593 fwhmSizes.append(fwhm)
596 sortedVisits = [ind
for (_, ind)
in sorted(zip(fwhmSizes, visits))]
597 if self.config.nVisitsMax < 0:
598 output = sortedVisits
600 output = sortedVisits[:self.config.nVisitsMax]
603 self.log.info(
"All images rejected in BestSeeingSelectVisitsTask.")
604 raise pipeBase.NoWorkFound(f
"No good images found for {dataId}")
607 "%d images selected with FWHM range of %f--%f arcseconds",
609 fwhmSizes[visits.index(output[0])],
610 fwhmSizes[visits.index(output[-1])],
614 goodVisits = {key:
True for key
in output}
615 return pipeBase.Struct(goodVisits=goodVisits)
617 def makePatchPolygon(self, skyMap, dataId):
618 """Return True if sky polygon overlaps visit.
622 skyMap : `lsst.afw.table.ExposureCatalog`
623 Exposure catalog with per-detector geometry.
624 dataId : `dict` of dataId keys
625 For retrieving patch info.
629 result : `lsst.sphgeom.ConvexPolygon.convexHull`
630 Polygon of patch's outer bbox.
632 wcs = skyMap[dataId[
'tract']].getWcs()
633 bbox = skyMap[dataId[
'tract']][dataId[
'patch']].getOuterBBox()
638 def doesIntersectPolygon(self, visitSummary, polygon):
639 """Return True if sky polygon overlaps visit.
643 visitSummary : `lsst.afw.table.ExposureCatalog`
644 Exposure catalog with per-detector geometry.
645 polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
646 Polygon to check overlap.
650 doesIntersect : `bool`
651 True if the visit overlaps the polygon.
653 doesIntersect =
False
654 for detectorSummary
in visitSummary:
655 if (np.all(np.isfinite(detectorSummary[
'raCorners']))
656 and np.all(np.isfinite(detectorSummary[
'decCorners']))):
659 zip(detectorSummary[
'raCorners'], detectorSummary[
'decCorners'])]
661 if detectorPolygon.intersects(polygon):
667class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
668 pipelineConnections=BestSeeingSelectVisitsConnections):
669 qMin = pexConfig.RangeField(
670 doc=
"Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
671 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
672 "This config should be changed from zero only for exploratory diffIm testing.",
678 qMax = pexConfig.RangeField(
679 doc=
"Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
680 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
686 nVisitsMin = pexConfig.Field(
687 doc=
"The minimum number of visits to select, if qMin and qMax alone would have "
688 "selected fewer. In regimes with many visits, at least this number of visits will be "
689 "selected, superceding quantile when necessary. "
690 "For example, if 10 visits cover this patch, qMin=0, qMax=0.33, and nVisitsMin=5, "
691 "the best 5 visits will be selected, even though 5 > 0.33*10. "
692 "In regimes with few visits, all available visits will be selected. "
693 "For example, if 2 visits cover this patch and nVisitsMin=12, "
694 "both visits will be selected regardless of qMin and qMax.",
698 doConfirmOverlap = pexConfig.Field(
700 doc=
"Do remove visits that do not actually overlap the patch?",
703 minMJD = pexConfig.Field(
705 doc=
"Minimum visit MJD to select",
709 maxMJD = pexConfig.Field(
711 doc=
"Maximum visit MJD to select",
717class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
718 """Select a quantile of the best-seeing visits.
720 Selects the best (for example, third) full visits based on the average
721 PSF width in the entire visit. It can also be used for difference imaging
722 experiments that require templates with the worst seeing visits.
723 For example, selecting the worst third can be acheived by
724 changing the config parameters qMin to 0.66 and qMax to 1.
726 ConfigClass = BestSeeingQuantileSelectVisitsConfig
727 _DefaultName =
'bestSeeingQuantileSelectVisits'
729 def run(self, visitSummaries, skyMap, dataId):
730 if self.config.doConfirmOverlap:
731 patchPolygon = self.makePatchPolygon(skyMap, dataId)
732 visits = np.array([visitSummary.ref.dataId[
'visit']
for visitSummary
in visitSummaries])
733 radius = np.empty(len(visits))
734 intersects = np.full(len(visits),
True)
735 for i, visitSummary
in enumerate(visitSummaries):
737 visitSummary = visitSummary.get()
739 psfSigma = np.nanmedian([vs[
'psfSigma']
for vs
in visitSummary])
741 if self.config.doConfirmOverlap:
742 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
743 if self.config.minMJD
or self.config.maxMJD:
746 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
747 aboveMin = mjd > self.config.minMJD
if self.config.minMJD
else True
748 belowMax = mjd < self.config.maxMJD
if self.config.maxMJD
else True
749 intersects[i] = intersects[i]
and aboveMin
and belowMax
751 sortedVisits = [v
for rad, v
in sorted(zip(radius[intersects], visits[intersects]))]
752 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
753 max(0, len(visits[intersects]) - self.config.nVisitsMin))
754 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
757 goodVisits = {int(visit):
True for visit
in sortedVisits[lowerBound:upperBound]}
758 return pipeBase.Struct(goodVisits=goodVisits)
A floating-point coordinate rectangle geometry.
Point in an unspecified spherical coordinate system.
__init__(self, dataId, coordList)
run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs)
isValid(self, visitSummary, detectorId)
__init__(self, dataRef, wcs, bbox)
run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs)
getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)
convexHull returns the convex hull of the given set of points if it exists and throws an exception ot...
_extractKeyValue(dataList, keys=None)