LSSTApplications  10.0+286,10.0+36,10.0+46,10.0-2-g4f67435,10.1+152,10.1+37,11.0,11.0+1,11.0-1-g47edd16,11.0-1-g60db491,11.0-1-g7418c06,11.0-2-g04d2804,11.0-2-g68503cd,11.0-2-g818369d,11.0-2-gb8b8ce7
LSSTDataManagementBasePackage
Public Member Functions | Public Attributes | Private Member Functions | List of all members
lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa Class Reference
Inheritance diagram for lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa:

Public Member Functions

def __init__
 
def makeMetricMap
 
def addToSchema
 
def apply
 
def aggregate
 

Public Attributes

 fields
 

Private Member Functions

def _calculateStats
 

Detailed Description

Definition at line 33 of file kernelCandidateQa.py.

Constructor & Destructor Documentation

def lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa.__init__ (   self,
  nKernelSpatial 
) 
Class to undertake QA of KernelCandidates after modeling of
the Psf-matching kernel.  Both directly--fitted diffim (LOCAL)
and spatially--interpolated kernel diffim (SPATIAL) metrics
are calculated, based on the distribution of residuals in the
KernelCandidates stamp.

@param nKernelSpatial : Number of terms in the spatial model; needed to initialize per-basis QA arrays

Definition at line 35 of file kernelCandidateQa.py.

35 
36  def __init__(self, nKernelSpatial):
37  """Class to undertake QA of KernelCandidates after modeling of
38  the Psf-matching kernel. Both directly--fitted diffim (LOCAL)
39  and spatially--interpolated kernel diffim (SPATIAL) metrics
40  are calculated, based on the distribution of residuals in the
41  KernelCandidates stamp.
42 
43  @param nKernelSpatial : Number of terms in the spatial model; needed to initialize per-basis QA arrays
44  """
45  self.fields = []
46  self.fields.append(afwTable.Field["PointD"]("RegisterRefPosition",
47  "Position of reference object for registration (radians)."))
48  #TODO check units of the following angles
49  self.fields.append(afwTable.Field["Angle"]("RegisterResidualBearing",
50  "Angle of residual wrt declination parallel in radians"))
51 
52  self.fields.append(afwTable.Field["Angle"]("RegisterResidualDistance",
53  "Offset of residual in radians"))
54  metricMap = self.makeMetricMap()
55 
56  for kType in ("LOCAL", "SPATIAL"):
57  for k in metricMap:
58  commentAndUnit = metricMap[k]['comment']
59  self.fields.append(afwTable.Field[metricMap[k]['type']](k%(kType), *commentAndUnit))
60 
61  self.fields.append(afwTable.Field["I"]("KCKernelStatus_LOCAL",
62  "Status of the KernelCandidate"))
63 
64  self.fields.append(afwTable.Field["ArrayD"]("KernelCoeffValues_LOCAL",
65  "Original basis coefficients",
66  nKernelSpatial))
67 
68  self.fields.append(afwTable.Field["F"]("BackgroundValue_LOCAL",
69  "Evaluation of background model at this point"))
70 
71  self.fields.append(afwTable.Field["F"]("KCDiffimMseKernel_SPATIAL",
72  "Mean squared error of spatial kernel estimate"))
lsst::afw::table::Field
A description of a field in a table.
Definition: Field.h:22

Member Function Documentation

def lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa._calculateStats (   self,
  di,
  dof = 0. 
)
private 
Calculate the core QA statistics on a difference image

Definition at line 131 of file kernelCandidateQa.py.

131 
132  def _calculateStats(self, di, dof=0.):
133  """Calculate the core QA statistics on a difference image"""
134  mask = di.getMask()
135  maskArr = di.getMask().getArray()
136 
137  # Create a mask using BAD, SAT, NO_DATA, EDGE bits. Keep detections
138  maskArr &= mask.getPlaneBitMask(["BAD", "SAT", "NO_DATA", "EDGE"])
139 
140  # Mask out values based on maskArr
141  diArr = ma.array(di.getImage().getArray(), mask=maskArr)
142  varArr = ma.array(di.getVariance().getArray(), mask=maskArr)
143 
144  # Normalize by sqrt variance, units are in sigma
145  diArr /= np.sqrt(varArr)
146  mean = diArr.mean()
147 
148  # This is the maximum-likelihood extimate of the variance stdev**2
149  stdev = diArr.std()
150  median = ma.extras.median(diArr)
151 
152  # Compute IQR of just un-masked data
153  data = ma.getdata(diArr[~diArr.mask])
154  iqr = np.percentile(data, 75.) - np.percentile(data, 25.)
155 
156  #Calculte chisquare of the residual
157  chisq=np.sum(np.power(data, 2.))
158 
159  # Mean squared error: variance + bias**2
160  # Bias = |data - model| = mean of diffim
161  # Variance = |(data - model)**2| = mean of diffim**2
162  bias = mean
163  variance = np.power(data, 2.).mean()
164  mseResids = bias**2 + variance
165 
166  # If scipy is not set up, return zero for the stats
167  try:
168  #In try block because of risk of divide by zero
169  rchisq=chisq/(len(data)-1-dof)
170  # K-S test on the diffim to a Normal distribution
171  import scipy.stats
172  D, prob = scipy.stats.kstest(data, 'norm')
173 
174  A2, crit, sig = scipy.stats.anderson(data, 'norm')
175  # Anderson Darling statistic cand be inf for really non-Gaussian distributions.
176  if np.isinf(A2) or np.isnan(A2):
177  A2 = 9999.
178  except ZeroDivisionError:
179  D = 0.
180  prob = 0.
181  A2 = 0.
182  crit = np.zeros(5)
183  sig = np.zeros(5)
184  rchisq = 0
185 
186  return {"mean": mean, "stdev": stdev, "median": median, "iqr": iqr,
187  "D": D, "prob": prob, "A2": A2, "crit": crit, "sig": sig,
188  "rchisq": rchisq, "mseResids": mseResids}
189 
def lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa.addToSchema (   self,
  inSourceCatalog 
) 
Add the to-be-generated QA keys to the Source schema

Definition at line 104 of file kernelCandidateQa.py.

104 
105  def addToSchema(self, inSourceCatalog):
106  """Add the to-be-generated QA keys to the Source schema"""
107  schema = inSourceCatalog.getSchema()
108  inKeys = []
109  psfDef = inSourceCatalog.getPsfFluxDefinition()
110  centroidDef = inSourceCatalog.getCentroidDefinition()
111  shapeDef = inSourceCatalog.getShapeDefinition()
112  for n in schema.getNames():
113  inKeys.append(schema[n].asKey())
114 
115  for field in self.fields:
116  schema.addField(field)
117  outSourceCatalog = afwTable.SourceCatalog(schema)
118  for source in inSourceCatalog:
119  rec = outSourceCatalog.addNew()
120  for k in inKeys:
121  if k.getTypeString() == 'Coord':
122  rec.setCoord(source.getCoord())
123  else:
124  setter = getattr(rec, "set"+k.getTypeString())
125  getter = getattr(source, "get"+k.getTypeString())
126  setter(k, getter(k))
127  outSourceCatalog.definePsfFlux(psfDef)
128  outSourceCatalog.defineCentroid(centroidDef)
129  outSourceCatalog.defineShape(shapeDef)
130  return outSourceCatalog
lsst::afw::table::SortedCatalogT
Custom catalog class for record/table subclasses that are guaranteed to have an ID, and should generally be sorted by that ID.
Definition: fwd.h:55
def lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa.aggregate (   self,
  sourceCatalog,
  metadata,
  wcsresids,
  diaSources = None 
) 
Generate aggregate metrics (e.g. total numbers of false
positives) from all the Sources in the sourceCatalog

Definition at line 285 of file kernelCandidateQa.py.

285 
286  def aggregate(self, sourceCatalog, metadata, wcsresids, diaSources=None):
287  """Generate aggregate metrics (e.g. total numbers of false
288  positives) from all the Sources in the sourceCatalog"""
289  for source in sourceCatalog:
290  sourceId = source.getId()
291  if sourceId in wcsresids.keys():
292  #Note that the residuals are not delta RA, delta Dec
293  #From the source code "bearing (angle wrt a declination parallel) and distance
294  coord, resids = wcsresids[sourceId]
295  key = source.schema["RegisterResidualBearing"].asKey()
296  setter = getattr(source, "set"+key.getTypeString())
297  setter(key, resids[0])
298  key = source.schema["RegisterResidualDistance"].asKey()
299  setter = getattr(source, "set"+key.getTypeString())
300  setter(key, resids[1])
301  key = source.schema["RegisterRefPosition"].asKey()
302  setter = getattr(source, "set"+key.getTypeString())
303  setter(key, afwGeom.Point2D(coord.getRa().asRadians(),
304  coord.getDec().asRadians()))
305  if diaSources:
306  metadata.add("NFalsePositivesTotal", len(diaSources))
307  nRefMatch = 0
308  nSrcMatch = 0
309  nunmatched = 0
310  for source in diaSources:
311  refId = source.get("refMatchId")
312  srcId = source.get("srcMatchId")
313  if refId > 0:
314  nRefMatch += 1
315  if srcId > 0:
316  nSrcMatch += 1
317  if refId == 0 and srcId == 0:
318  nunmatched += 1
319  metadata.add("NFalsePositivesRefAssociated", nRefMatch)
320  metadata.add("NFalsePositivesSrcAssociated", nSrcMatch)
321  metadata.add("NFalsePositivesUnassociated", nunmatched)
322  for kType in ("LOCAL", "SPATIAL"):
323  for sName in ("KCDiffimMean", "KCDiffimMedian", "KCDiffimIQR", "KCDiffimStDev",
324  "KCDiffimKSProb", "KCDiffimADSig", "KCDiffimChiSq",
325  "KCDiffimMseResids", "KCDiffimMseKernel"):
326  if sName == "KCDiffimMseKernel" and kType == "LOCAL":
327  continue
328  kName = "%s_%s" % (sName, kType)
329  vals = np.array([s.get(kName) for s in sourceCatalog])
330  idx = np.isfinite(vals)
331  metadata.add("%s_MEAN" % (kName), np.mean(vals[idx]))
332  metadata.add("%s_MEDIAN" % (kName), np.median(vals[idx]))
333  metadata.add("%s_STDEV" % (kName), np.std(vals[idx]))
334 
def lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa.apply (   self,
  candidateList,
  spatialKernel,
  spatialBackground,
  dof = 0 
) 
Evaluate the QA metrics for all KernelCandidates in the
candidateList; set the values of the metrics in their
associated Sources

Definition at line 190 of file kernelCandidateQa.py.

190 
191  def apply(self, candidateList, spatialKernel, spatialBackground, dof=0):
192  """Evaluate the QA metrics for all KernelCandidates in the
193  candidateList; set the values of the metrics in their
194  associated Sources"""
195  for kernelCandidate in candidateList:
196  source = kernelCandidate.getSource()
197  schema = source.schema
198 
199  # Calculate ORIG stats (original basis fit)
200  if kernelCandidate.getStatus() != afwMath.SpatialCellCandidate.UNKNOWN:
201  kType = getattr(diffimLib.KernelCandidateF, "ORIG")
202  di = kernelCandidate.getDifferenceImage(kType)
203  kernelValues = kernelCandidate.getKernel(kType).getKernelParameters()
204  kernelValues = np.asarray(kernelValues)
205 
206  lkim = kernelCandidate.getKernelImage(kType)
207  centx, centy = calcCentroid(lkim.getArray())
208  stdx, stdy = calcWidth(lkim.getArray(), centx, centy)
209  # NOTE
210  # What is the difference between kernelValues and solution?
211 
212  localResults = self._calculateStats(di, dof=dof)
213 
214  metrics = {"KCDiffimMean_LOCAL":localResults["mean"],
215  "KCDiffimMedian_LOCAL":localResults["median"],
216  "KCDiffimIQR_LOCAL":localResults["iqr"],
217  "KCDiffimStDev_LOCAL":localResults["stdev"],
218  "KCDiffimKSD_LOCAL":localResults["D"],
219  "KCDiffimKSProb_LOCAL":localResults["prob"],
220  "KCDiffimADA2_LOCAL":localResults["A2"],
221  "KCDiffimADCrit_LOCAL":localResults["crit"],
222  "KCDiffimADSig_LOCAL":localResults["sig"],
223  "KCDiffimChiSq_LOCAL":localResults["rchisq"],
224  "KCDiffimMseResids_LOCAL":localResults["mseResids"],
225  "KCKernelCentX_LOCAL":centx,
226  "KCKernelCentY_LOCAL":centy,
227  "KCKernelStdX_LOCAL":stdx,
228  "KCKernelStdY_LOCAL":stdy,
229  "KernelCandidateId_LOCAL":kernelCandidate.getId(),
230  "KernelCoeffValues_LOCAL":kernelValues}
231  for k in metrics.keys():
232  key = schema[k].asKey()
233  setter = getattr(source, "set"+key.getTypeString())
234  setter(key, metrics[k])
235  else:
236  try:
237  kType = getattr(diffimLib.KernelCandidateF, "ORIG")
238  lkim = kernelCandidate.getKernelImage(kType)
239  except Exception:
240  lkim = None
241 
242  # Calculate spatial model evaluated at each position, for
243  # all candidates
244  skim = afwImage.ImageD(spatialKernel.getDimensions())
245  spatialKernel.computeImage(skim, False, kernelCandidate.getXCenter(),
246  kernelCandidate.getYCenter())
247  centx, centy = calcCentroid(skim.getArray())
248  stdx, stdy = calcWidth(skim.getArray(), centx, centy)
249 
250  sk = afwMath.FixedKernel(skim)
251  sbg = spatialBackground(kernelCandidate.getXCenter(), kernelCandidate.getYCenter())
252  di = kernelCandidate.getDifferenceImage(sk, sbg)
253  spatialResults = self._calculateStats(di, dof=dof)
254 
255  # Kernel mse
256  if lkim is not None:
257  skim -= lkim
258  bias = np.mean(skim.getArray())
259  variance = np.mean(np.power(skim.getArray(), 2.))
260  mseKernel = bias**2 + variance
261  else:
262  mseKernel = -99.999
263 
264  metrics = {"KCDiffimMean_SPATIAL":spatialResults["mean"],
265  "KCDiffimMedian_SPATIAL":spatialResults["median"],
266  "KCDiffimIQR_SPATIAL":spatialResults["iqr"],
267  "KCDiffimStDev_SPATIAL":spatialResults["stdev"],
268  "KCDiffimKSD_SPATIAL":spatialResults["D"],
269  "KCDiffimKSProb_SPATIAL":spatialResults["prob"],
270  "KCDiffimADA2_SPATIAL":spatialResults["A2"],
271  "KCDiffimADCrit_SPATIAL":spatialResults["crit"],
272  "KCDiffimADSig_SPATIAL":spatialResults["sig"],
273  "KCDiffimChiSq_SPATIAL":spatialResults["rchisq"],
274  "KCDiffimMseResids_SPATIAL":spatialResults["mseResids"],
275  "KCDiffimMseKernel_SPATIAL":mseKernel,
276  "KCKernelCentX_SPATIAL":centx,
277  "KCKernelCentY_SPATIAL":centy,
278  "KCKernelStdX_SPATIAL":stdx,
279  "KCKernelStdY_SPATIAL":stdy,
280  "KernelCandidateId_SPATIAL":kernelCandidate.getId()}
281  for k in metrics.keys():
282  key = schema[k].asKey()
283  setter = getattr(source, "set"+key.getTypeString())
284  setter(key, metrics[k])
lsst::afw::math::FixedKernel
A kernel created from an Image.
Definition: Kernel.h:551
def lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa.makeMetricMap (   self)

Definition at line 73 of file kernelCandidateQa.py.

73 
74  def makeMetricMap(self):
75  nameList = ['KCDiffimMean_%s', 'KCDiffimMedian_%s', 'KCDiffimIQR_%s', 'KCDiffimStDev_%s',
76  'KCDiffimKSD_%s', 'KCDiffimKSProb_%s', 'KCDiffimADA2_%s', 'KCDiffimADCrit_%s',
77  'KCDiffimADSig_%s', 'KCDiffimChiSq_%s', 'KCDiffimMseResids_%s', 'KCKernelCentX_%s',
78  'KCKernelCentY_%s', 'KCKernelStdX_%s', 'KCKernelStdY_%s', 'KernelCandidateId_%s']
79  typeList = ['F', 'F', 'F', 'F', 'F', 'F', 'F', 'ArrayD', 'ArrayD', 'F', 'F', 'F',
80  'F', 'F', 'F', 'I']
81  commentList = [("Mean of KernelCandidate diffim", "sigma"),
82  ("Median of KernelCandidate diffim", "sigma"),
83  ("Inner quartile range of KernelCandidate diffim", "sigma"),
84  ("Standard deviation of KernelCandidate diffim","sigma"),
85  ("D from K-S test of diffim pixels relative to Normal", ),
86  ("Prob from K-S test of diffim pixels relative to Normal", "likelihood"),
87  ("Anderson-Darling test statistic of diffim pixels relative to Normal", ),
88  ("Critical values for the significance levels in KCDiffimADSig. If A2 is greater "+\
89  "than this number, hypothesis that the distributions are similar can be rejected.", 5),
90  ("Anderson-Darling significance levels for the Normal distribution", 5),
91  ("Reduced chi^2 of the residual.", "likelihood"),
92  ("Mean squared error in diffim : Variance + Bias**2",),
93  ("Centroid in X for this Kernel", "pixels"),
94  ("Centroid in Y for this Kernel", "pixels"),
95  ("Standard deviation in X for this Kernel", "pixels"),
96  ("Standard deviation in Y for this Kernel", "pixels"),
97  ("Id for this KernelCandidate",)]
98  metricMap = {}
99  for name, mtype, comment in zip(nameList, typeList, commentList):
100  metricMap[name] = {'type':mtype, 'comment':comment}
101 
102  return metricMap
103 

Member Data Documentation

lsst.ip.diffim.kernelCandidateQa.KernelCandidateQa.fields

Definition at line 44 of file kernelCandidateQa.py.

The documentation for this class was generated from the following file:
Generated on Thu Sep 24 2015 02:29:36 for LSSTApplications by   doxygen 1.8.5