158 isTransposed=True, leadingToSkip=0, trailingToSkip=0,
159 overscanFraction=1.0, imageThreshold=np.inf,
160 maskedRowColumnGrowSize=0):
161 """Trim the exposure, fit the overscan, subtract the fit, and
162 calculate statistics.
166 exposure : `lsst.afw.image.Exposure`
167 Exposure containing the data.
168 amp : `lsst.afw.cameraGeom.Amplifier`
169 The amplifier that is to be corrected.
170 imageBBox: `lsst.geom.Box2I`
171 Bounding box of the image data that will have the overscan
172 subtracted. If parallel overscan will be performed, that
173 area is added to the image bounding box during serial
175 overscanBBox: `lsst.geom.Box2I`
176 Bounding box for the overscan data.
178 If true, then the data will be transposed before fitting
180 leadingToSkip : `int`, optional
181 Leading rows/columns to skip.
182 trailingToSkip : `int`, optional
183 Leading rows/columns to skip.
184 overscanFraction : `float`, optional
185 If the overscan region median is greater than overscanFraction
186 and the imaging region median is greater than imageThreshold
187 then overscan correction will be skipped.
188 maxLevel : `float`, optional
189 If the overscan region median is greater than overscanFraction
190 and the imaging region median is greater than imageThreshold
191 then overscan correction will be skipped.
192 maskedRowColumnGrowSize : `int`, optional
193 If a column (parallel overscan) or row (serial overscan) is
194 completely masked, then grow the mask by this radius. If the
195 value is <=0 then this will not be checked.
199 results : `lsst.pipe.base.Struct`
201 Overscan model broadcast to the full image size.
202 (`lsst.afw.image.Exposure`)
203 ``overscanOverscanModel``
204 Overscan model broadcast to the full overscan image
205 size. (`lsst.afw.image.Exposure`)
207 Overscan image with the overscan fit subtracted.
208 (`lsst.afw.image.Exposure`)
210 Overscan model. (`float` or `np.array`)
212 Mean value of the overscan fit. (`float`)
214 Median value of the overscan fit. (`float`)
216 Standard deviation of the overscan fit. (`float`)
217 ``overscanMeanResidual``
218 Mean value of the overscan region after overscan
219 subtraction. (`float`)
220 ``overscanMedianResidual``
221 Median value of the overscan region after overscan
222 subtraction. (`float`)
223 ``overscanSigmaResidual``
224 Standard deviation of the overscan region after
225 overscan subtraction. (`float`)
227 overscanBox = self.
trimOverscan(exposure, amp, overscanBBox,
230 transpose=isTransposed)
231 overscanImage = exposure[overscanBox].getMaskedImage()
232 overscanArray = overscanImage.image.array
235 maskVal = overscanImage.mask.getPlaneBitMask(self.config.maskPlanes)
236 overscanMask = ~((overscanImage.mask.array & maskVal) == 0)
239 overscanMedian = np.nanmedian(overscanImage.image.array)
240 imageMedian = np.nanmedian(exposure[imageBBox].image.array)
242 if np.all(overscanMask):
244 "All overscan pixels masked when attempting overscan correction for %s",
248 elif overscanMedian/imageMedian > overscanFraction
and imageMedian > imageThreshold:
250 "The level in the overscan region (%.2f) compared to the image region (%.2f) is "
251 "greater than the maximum fraction (%.2f) for %s",
261 overscanResults = pipeBase.Struct(
268 median = np.ma.median(np.ma.masked_where(overscanMask, overscanArray))
269 bad = np.where((np.abs(overscanArray - median) > self.config.maxDeviation) & (~overscanMask))
271 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"BAD")
274 if maskedRowColumnGrowSize > 0
and len(bad) > 0:
277 nComp = overscanArray.shape[0]
280 nComp = overscanArray.shape[1]
284 overscanMaskTemp = np.zeros_like(overscanMask)
285 overscanMaskTemp[bad] =
True
287 nMaskedArray = np.sum(overscanMaskTemp, axis=axis, dtype=np.int32)
288 badRowsColumns, = np.where(nMaskedArray == nComp)
289 if len(badRowsColumns) > 0:
290 dataView = afwImage.MaskedImageF(exposure.maskedImage,
294 pixelsCopy = dataView.image.array[:, badRowsColumns].copy()
295 dataView.image.array[:, badRowsColumns] = 1e30
297 pixelsCopy = dataView.image.array[badRowsColumns, :].copy()
298 dataView.image.array[badRowsColumns, :] = 1e30
301 maskedImage=dataView,
303 growFootprints=maskedRowColumnGrowSize,
308 dataView.image.array[:, badRowsColumns] = pixelsCopy
310 dataView.image.array[badRowsColumns, :] = 1e30
314 overscanResults = self.
fitOverscan(overscanImage, isTransposed=isTransposed)
317 ampImage = exposure[imageBBox]
319 ampImage.image.array,
320 transpose=isTransposed)
321 ampImage.image.array -= ampOverscanModel
325 overscanImage = exposure[overscanBBox]
328 overscanImage.image.array)
329 self.
debugView(overscanImage, overscanResults.overscanValue, amp, isTransposed=isTransposed)
330 overscanImage.image.array -= overscanOverscanModel
334 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
335 residualMean = stats.getValue(afwMath.MEAN)
336 residualMedian = stats.getValue(afwMath.MEDIAN)
337 residualSigma = stats.getValue(afwMath.STDEVCLIP)
339 return pipeBase.Struct(ampOverscanModel=ampOverscanModel,
340 overscanOverscanModel=overscanOverscanModel,
341 overscanImage=overscanImage,
342 overscanValue=overscanResults.overscanValue,
344 overscanMean=overscanResults.overscanMean,
345 overscanMedian=overscanResults.overscanMedian,
346 overscanSigma=overscanResults.overscanSigma,
347 overscanMeanResidual=residualMean,
348 overscanMedianResidual=residualMedian,
349 overscanSigmaResidual=residualSigma
353 """Broadcast 0 or 1 dimension fit to appropriate shape.
357 overscanValue : `numpy.ndarray`, (Nrows, ) or scalar
358 Overscan fit to broadcast.
359 imageArray : `numpy.ndarray`, (Nrows, Ncols)
360 Image array that we want to match.
361 transpose : `bool`, optional
362 Switch order to broadcast along the other axis.
366 overscanModel : `numpy.ndarray`, (Nrows, Ncols) or scalar
367 Expanded overscan fit.
372 Raised if no axis has the appropriate dimension.
374 if isinstance(overscanValue, np.ndarray):
375 overscanModel = np.zeros_like(imageArray)
377 if transpose
is False:
378 if imageArray.shape[0] == overscanValue.shape[0]:
379 overscanModel[:, :] = overscanValue[:, np.newaxis]
380 elif imageArray.shape[1] == overscanValue.shape[0]:
381 overscanModel[:, :] = overscanValue[np.newaxis, :]
382 elif imageArray.shape[0] == overscanValue.shape[1]:
383 overscanModel[:, :] = overscanValue[np.newaxis, :]
385 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
386 f
"match {imageArray.shape}")
388 if imageArray.shape[1] == overscanValue.shape[0]:
389 overscanModel[:, :] = overscanValue[np.newaxis, :]
390 elif imageArray.shape[0] == overscanValue.shape[0]:
391 overscanModel[:, :] = overscanValue[:, np.newaxis]
392 elif imageArray.shape[1] == overscanValue.shape[1]:
393 overscanModel[:, :] = overscanValue[:, np.newaxis]
395 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
396 f
"match {imageArray.shape}")
398 overscanModel = overscanValue
402 def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False):
403 """Trim overscan region to remove edges.
407 exposure : `lsst.afw.image.Exposure`
408 Exposure containing data.
409 amp : `lsst.afw.cameraGeom.Amplifier`
410 Amplifier containing geometry information.
411 bbox : `lsst.geom.Box2I`
412 Bounding box of the overscan region.
414 Number of leading (towards data region) rows/columns to skip.
416 Number of trailing (away from data region) rows/columns to skip.
417 transpose : `bool`, optional
418 Operate on the transposed array.
422 overscanArray : `numpy.array`, (N, M)
424 overscanMask : `numpy.array`, (N, M)
427 dx0, dy0, dx1, dy1 = (0, 0, 0, 0)
428 dataBBox = amp.getRawDataBBox()
430 if dataBBox.getBeginY() < bbox.getBeginY():
437 if dataBBox.getBeginX() < bbox.getBeginX():
446 bbox.getHeight() - dy0 + dy1))
450 if self.config.fitType
in (
'MEAN',
'MEANCLIP',
'MEDIAN'):
453 overscanValue = overscanResult.overscanValue
454 overscanMean = overscanValue
455 overscanMedian = overscanValue
457 elif self.config.fitType
in (
'MEDIAN_PER_ROW',
'MEAN_PER_ROW',
'POLY',
'CHEB',
'LEG',
458 'NATURAL_SPLINE',
'CUBIC_SPLINE',
'AKIMA_SPLINE'):
461 overscanValue = overscanResult.overscanValue
464 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP,
466 overscanMean = stats.getValue(afwMath.MEAN)
467 overscanMedian = stats.getValue(afwMath.MEDIAN)
468 overscanSigma = stats.getValue(afwMath.STDEVCLIP)
470 raise ValueError(
'%s : %s an invalid overscan type' %
471 (
"overscanCorrection", self.config.fitType))
473 return pipeBase.Struct(overscanValue=overscanValue,
474 overscanMean=overscanMean,
475 overscanMedian=overscanMedian,
476 overscanSigma=overscanSigma,
480 """Mask the union of high values on all amplifiers in the parallel
483 This operates on the image in-place.
487 exposure : `lsst.afw.image.Exposure`
488 An untrimmed raw exposure.
489 detector : `lsst.afw.cameraGeom.Detector`
490 The detetor to use for amplifier geometry.
495 dataView = afwImage.MaskedImageF(exposure.getMaskedImage(),
496 amp.getRawParallelOverscanBBox(),
500 maskedImage=dataView,
501 threshold=self.config.parallelOverscanMaskThreshold,
502 growFootprints=self.config.parallelOverscanMaskGrowSize,
505 if parallelMask
is None:
506 parallelMask = dataView.mask.array
508 parallelMask |= dataView.mask.array
510 dataView = afwImage.MaskedImageF(exposure.getMaskedImage(),
511 amp.getRawParallelOverscanBBox(),
513 dataView.mask.array |= parallelMask
558 """Mask outliers in a row of overscan data from a robust sigma
563 imageArray : `numpy.ndarray`
564 Image to filter along numpy axis=1.
568 maskedArray : `numpy.ma.masked_array`
569 Masked image marking outliers.
571 lq, median, uq = np.percentile(np.ma.getdata(imageArray),
572 [25.0, 50.0, 75.0], axis=1)
574 axisStdev = 0.74*(uq - lq)
579 axisStdev = np.where(axisStdev > 2.0 * np.median(axisStdev),
580 np.median(axisStdev), axisStdev)
583 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
584 masked = np.ma.masked_where(diff > self.
statControl.getNumSigmaClip()
585 * axisStdev[:, np.newaxis], imageArray)
590 """Fill masked/NaN pixels in the overscan.
594 overscanVector : `np.array` or `np.ma.masked_array`
595 Overscan vector to fill.
599 overscanVector : `np.ma.masked_array`
604 Each maskSlice is a section of overscan with contiguous masks.
605 Ideally this adds 5 pixels from the left and right of that
606 mask slice, and takes the median of those values to fill the
607 slice. If this isn't possible, the median of all non-masked
608 values is used. The mask is removed for the pixels filled.
610 workingCopy = overscanVector
611 if not isinstance(overscanVector, np.ma.MaskedArray):
612 workingCopy = np.ma.masked_array(overscanVector,
613 mask=~np.isfinite(overscanVector))
615 defaultValue = np.median(workingCopy.data[~workingCopy.mask])
616 for maskSlice
in np.ma.clump_masked(workingCopy):
618 if maskSlice.start > 5:
619 neighborhood.extend(workingCopy[maskSlice.start - 5:maskSlice.start].data)
620 if maskSlice.stop < workingCopy.size - 5:
621 neighborhood.extend(workingCopy[maskSlice.stop:maskSlice.stop+5].data)
622 if len(neighborhood) > 0:
623 workingCopy.data[maskSlice] = np.nanmedian(neighborhood)
624 workingCopy.mask[maskSlice] =
False
626 workingCopy.data[maskSlice] = defaultValue
627 workingCopy.mask[maskSlice] =
False
654 """Wrapper function to match spline fit API to polynomial fit API.
658 indices : `numpy.ndarray`
659 Locations to evaluate the spline.
660 collapsed : `numpy.ndarray`
661 Collapsed overscan values corresponding to the spline
664 Number of bins to use in constructing the spline.
668 interp : `lsst.afw.math.Interpolate`
669 Interpolation object for later evaluation.
671 if not np.ma.is_masked(collapsed):
672 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
674 numPerBin, binEdges = np.histogram(indices, bins=numBins,
675 weights=1 - collapsed.mask.astype(int))
676 with np.errstate(invalid=
"ignore"):
677 values = np.histogram(indices, bins=numBins,
678 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
679 binCenters = np.histogram(indices, bins=numBins,
680 weights=indices*~collapsed.mask)[0]/numPerBin
682 if len(binCenters[numPerBin > 0]) < 5:
683 self.log.warning(
"Cannot do spline fitting for overscan: %s valid points.",
684 len(binCenters[numPerBin > 0]))
688 if len(values[numPerBin > 0]) != 0:
689 return float(values[numPerBin > 0][0])
694 values.astype(float)[numPerBin > 0],
748 """Calculate the 1-d vector overscan from the input overscan image.
752 image : `lsst.afw.image.MaskedImage`
753 Image containing the overscan data.
754 isTransposed : `bool`
755 If true, the image has been transposed.
759 results : `lsst.pipe.base.Struct`
760 Overscan result with entries:
763 Overscan value to subtract (`float`)
765 List of rows that should be masked as ``SUSPECT`` when the
766 overscan solution is applied. (`list` [ `bool` ])
768 Indicates if the overscan data was transposed during
769 calcuation, noting along which axis the overscan should be
776 calcImage = np.transpose(calcImage)
779 if self.config.fitType
in (
'MEDIAN_PER_ROW',
"MEAN_PER_ROW"):
780 if self.config.overscanIsInt:
781 mi = afwImage.MaskedImageI(image.getBBox())
782 masked = masked.astype(int)
787 masked = masked.transpose()
789 mi.image.array[:, :] = masked.data[:, :]
790 if bool(masked.mask.shape):
791 mi.mask.array[:, :] = masked.mask[:, :]
793 if self.config.fitType ==
"MEDIAN_PER_ROW":
794 overscanVector = fitOverscanImage(mi, self.config.maskPlanes, isTransposed)
796 overscanVector = fitOverscanImageMean(mi, self.config.maskPlanes, isTransposed)
804 indices = 2.0*np.arange(num)/float(num) - 1.0
808 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
809 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
810 'LEG': (poly.legendre.legfit, poly.legendre.legval),
814 }[self.config.fitType]
818 coeffs = fitter(indices, collapsed, self.config.order)
820 if isinstance(coeffs, float):
821 self.log.warning(
"Using fallback value %f due to fitter failure. Amplifier will be masked.",
823 overscanVector = np.full_like(indices, coeffs)
824 maskArray = np.full_like(collapsed,
True, dtype=bool)
827 overscanVector = evaler(indices, coeffs)
830 return pipeBase.Struct(overscanValue=np.array(overscanVector),
832 isTransposed=isTransposed)
834 def debugView(self, image, model, amp=None, isTransposed=True):
835 """Debug display for the final overscan solution.
839 image : `lsst.afw.image.Image`
840 Input image the overscan solution was determined from.
841 model : `numpy.ndarray` or `float`
842 Overscan model determined for the image.
843 amp : `lsst.afw.cameraGeom.Amplifier`, optional
844 Amplifier to extract diagnostic information.
845 isTransposed : `bool`, optional
846 Does the data need to be transposed before display?
857 calcImage = np.transpose(calcImage)
862 indices = 2.0 * np.arange(num)/float(num) - 1.0
863 indices = np.arange(num)
865 if np.ma.is_masked(collapsed):
866 collapsedMask = collapsed.mask
868 collapsedMask = np.array(num*[np.ma.nomask])
870 import matplotlib.pyplot
as plot
871 figure = plot.figure(1)
873 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
874 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask],
'k+')
875 if collapsedMask.sum() > 0:
876 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask],
'b+')
877 if isinstance(model, np.ndarray):
880 plotModel = np.zeros_like(indices)
883 axes.plot(indices, plotModel,
'r-')
884 plot.xlabel(
"position along overscan region")
885 plot.ylabel(
"pixel value/fit value")
887 plot.title(f
"{amp.getName()} DataX: "
888 f
"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]"
889 f
"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:"
890 f
"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}")
892 plot.title(
"No amp supplied.")
894 prompt =
"Press Enter or c to continue [chp]..."
896 ans = input(prompt).lower()
897 if ans
in (
"",
" ",
"c",):
906 print(
"[h]elp [c]ontinue [p]db e[x]itDebug")