24 from itertools
import product
29 import astropy.units
as u
32 from lsst.daf.butler
import DeferredDatasetHandle
36 from .parquetTable
import ParquetTable, MultilevelParquetTable
40 typeKey='functor', name=None):
41 """Initialize an object defined in a dictionary
43 The object needs to be importable as
44 f'{basePath}.{initDict[typeKey]}'
45 The positional and keyword arguments (if any) are contained in
46 "args" and "kwargs" entries in the dictionary, respectively.
47 This is used in `functors.CompositeFunctor.from_yaml` to initialize
48 a composite functor from a specification in a YAML file.
53 Dictionary describing object's initialization. Must contain
54 an entry keyed by ``typeKey`` that is the name of the object,
55 relative to ``basePath``.
57 Path relative to module in which ``initDict[typeKey]`` is defined.
59 Key of ``initDict`` that is the name of the object
60 (relative to `basePath`).
62 initDict = initDict.copy()
64 pythonType =
doImport(f
'{basePath}.{initDict.pop(typeKey)}')
66 if 'args' in initDict:
67 args = initDict.pop(
'args')
68 if isinstance(args, str):
71 element = pythonType(*args, **initDict)
72 except Exception
as e:
73 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
74 raise type(e)(message, e.args)
79 """Define and execute a calculation on a ParquetTable
81 The `__call__` method accepts either a `ParquetTable` object or a
82 `DeferredDatasetHandle`, and returns the
83 result of the calculation as a single column. Each functor defines what
84 columns are needed for the calculation, and only these columns are read
85 from the `ParquetTable`.
87 The action of `__call__` consists of two steps: first, loading the
88 necessary columns from disk into memory as a `pandas.DataFrame` object;
89 and second, performing the computation on this dataframe and returning the
93 To define a new `Functor`, a subclass must define a `_func` method,
94 that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
95 In addition, it must define the following attributes
97 * `_columns`: The columns necessary to perform the calculation
98 * `name`: A name appropriate for a figure axis label
99 * `shortname`: A name appropriate for use as a dictionary key
101 On initialization, a `Functor` should declare what band (`filt` kwarg)
102 and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
103 applied to. This enables the `_get_data` method to extract the proper
104 columns from the parquet file. If not specified, the dataset will fall back
105 on the `_defaultDataset`attribute. If band is not specified and `dataset`
106 is anything other than `'ref'`, then an error will be raised when trying to
107 perform the calculation.
109 Originally, `Functor` was set up to expect
110 datasets formatted like the `deepCoadd_obj` dataset; that is, a
111 dataframe with a multi-level column index, with the levels of the
112 column index being `band`, `dataset`, and `column`.
113 It has since been generalized to apply to dataframes without mutli-level
114 indices and multi-level indices with just `dataset` and `column` levels.
115 In addition, the `_get_data` method that reads
116 the dataframe from the `ParquetTable` will return a dataframe with column
117 index levels defined by the `_dfLevels` attribute; by default, this is
120 The `_dfLevels` attributes should generally not need to
121 be changed, unless `_func` needs columns from multiple filters or datasets
122 to do the calculation.
123 An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
124 which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
125 it gets to have those levels in the column index.
130 Filter upon which to do the calculation
133 Dataset upon which to do the calculation
134 (e.g., 'ref', 'meas', 'forced_src').
138 _defaultDataset =
'ref'
139 _dfLevels = (
'column',)
140 _defaultNoDup =
False
142 def __init__(self, filt=None, dataset=None, noDup=None):
149 if self.
_noDup_noDup
is not None:
156 """Columns required to perform calculation
158 if not hasattr(self,
'_columns'):
159 raise NotImplementedError(
'Must define columns property or _columns attribute')
162 def _get_data_columnLevels(self, data, columnIndex=None):
163 """Gets the names of the column index levels
165 This should only be called in the context of a multilevel table.
166 The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
167 and with the gen3 `DeferredDatasetHandle`.
171 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
173 columnnIndex (optional): pandas `Index` object
174 if not passed, then it is read from the `DeferredDatasetHandle`
176 if isinstance(data, DeferredDatasetHandle):
177 if columnIndex
is None:
178 columnIndex = data.get(component=
"columns")
179 if columnIndex
is not None:
180 return columnIndex.names
181 if isinstance(data, MultilevelParquetTable):
182 return data.columnLevels
184 raise TypeError(f
"Unknown type for data: {type(data)}!")
186 def _get_data_columnLevelNames(self, data, columnIndex=None):
187 """Gets the content of each of the column levels for a multilevel table
189 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
191 Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
193 if isinstance(data, DeferredDatasetHandle):
194 if columnIndex
is None:
195 columnIndex = data.get(component=
"columns")
196 if columnIndex
is not None:
197 columnLevels = columnIndex.names
199 level:
list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
200 for i, level
in enumerate(columnLevels)
202 return columnLevelNames
203 if isinstance(data, MultilevelParquetTable):
204 return data.columnLevelNames
206 raise TypeError(f
"Unknown type for data: {type(data)}!")
208 def _colsFromDict(self, colDict, columnIndex=None):
209 """Converts dictionary column specficiation to a list of columns
211 This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
216 for i, lev
in enumerate(columnLevels):
218 if isinstance(colDict[lev], str):
219 new_colDict[lev] = [colDict[lev]]
221 new_colDict[lev] = colDict[lev]
223 new_colDict[lev] = columnIndex.levels[i]
225 levelCols = [new_colDict[lev]
for lev
in columnLevels]
226 cols = product(*levelCols)
230 """Returns columns needed by functor from multilevel dataset
232 To access tables with multilevel column structure, the `MultilevelParquetTable`
233 or `DeferredDatasetHandle` need to be passed either a list of tuples or a
238 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
240 columnIndex (optional): pandas `Index` object
241 either passed or read in from `DeferredDatasetHandle`.
244 If true, then return a list of tuples rather than the column dictionary
245 specification. This is set to `True` by `CompositeFunctor` in order to be able to
246 combine columns from the various component functors.
249 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
250 columnIndex = data.get(component=
"columns")
255 columnDict = {
'column': self.
columnscolumns,
256 'dataset': self.
datasetdataset}
257 if self.
filtfilt
is None:
259 if "band" in columnLevels:
260 if self.
datasetdataset ==
"ref":
261 columnDict[
"band"] = columnLevelNames[
"band"][0]
263 raise ValueError(f
"'filt' not set for functor {self.name}"
264 f
"(dataset {self.dataset}) "
266 "contains multiple filters in column index. "
267 "Set 'filt' or set 'dataset' to 'ref'.")
269 columnDict[
'band'] = self.
filtfilt
271 if isinstance(data, MultilevelParquetTable):
272 return data._colsFromDict(columnDict)
273 elif isinstance(data, DeferredDatasetHandle):
275 return self.
_colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
279 def _func(self, df, dropna=True):
280 raise NotImplementedError(
'Must define calculation on dataframe')
282 def _get_columnIndex(self, data):
283 """Return columnIndex
286 if isinstance(data, DeferredDatasetHandle):
287 return data.get(component=
"columns")
291 def _get_data(self, data):
292 """Retrieve dataframe necessary for calculation.
294 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
296 Returns dataframe upon which `self._func` can act.
298 N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
300 if isinstance(data, pd.DataFrame):
305 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
308 if isinstance(data, ParquetTable)
and not is_multiLevel:
310 df = data.toDataFrame(columns=columns)
319 if isinstance(data, MultilevelParquetTable):
321 df = data.toDataFrame(columns=columns, droplevels=
False)
322 elif isinstance(data, DeferredDatasetHandle):
324 df = data.get(parameters={
"columns": columns})
332 def _setLevels(self, df):
333 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels_dfLevels]
334 df.columns = df.columns.droplevel(levelsToDrop)
337 def _dropna(self, vals):
343 vals = self.
_func_func(df)
345 vals = self.
failfail(df)
347 vals = self.
_dropna_dropna(vals)
352 """Computes difference between functor called on two different ParquetTable objects
354 return self(data1, **kwargs) - self(data2, **kwargs)
357 return pd.Series(np.full(len(df), np.nan), index=df.index)
361 """Full name of functor (suitable for figure labels)
363 return NotImplementedError
367 """Short name of functor (suitable for column name/dict key)
373 """Perform multiple calculations at once on a catalog
375 The role of a `CompositeFunctor` is to group together computations from
376 multiple functors. Instead of returning `pandas.Series` a
377 `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
378 being the keys of `funcDict`.
380 The `columns` attribute of a `CompositeFunctor` is the union of all columns
381 in all the component functors.
383 A `CompositeFunctor` does not use a `_func` method itself; rather,
384 when a `CompositeFunctor` is called, all its columns are loaded
385 at once, and the resulting dataframe is passed to the `_func` method of each component
386 functor. This has the advantage of only doing I/O (reading from parquet file) once,
387 and works because each individual `_func` method of each component functor does not
388 care if there are *extra* columns in the dataframe being passed; only that it must contain
389 *at least* the `columns` it expects.
391 An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
392 file specifying a collection of functors.
396 funcs : `dict` or `list`
397 Dictionary or list of functors. If a list, then it will be converted
398 into a dictonary according to the `.shortname` attribute of each functor.
405 if type(funcs) == dict:
408 self.
funcDictfuncDict = {f.shortname: f
for f
in funcs}
410 self.
_filt_filt =
None
416 return self.
_filt_filt
423 self.
_filt_filt = filt
426 if isinstance(new, dict):
428 elif isinstance(new, CompositeFunctor):
431 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
439 return list(
set([x
for y
in [f.columns
for f
in self.
funcDictfuncDict.values()]
for x
in y]))
448 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDictfuncDict.values()
456 """Apply the functor to the data table
460 data : `lsst.daf.butler.DeferredDatasetHandle`,
461 `lsst.pipe.tasks.parquetTable.MultilevelParquetTable`,
462 `lsst.pipe.tasks.parquetTable.ParquetTable`,
463 or `pandas.DataFrame`.
464 The table or a pointer to a table on disk from which columns can
470 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
476 if isinstance(data, MultilevelParquetTable):
478 df = data.toDataFrame(columns=columns, droplevels=
False)
479 elif isinstance(data, DeferredDatasetHandle):
481 df = data.get(parameters={
"columns": columns})
486 subdf = f._setLevels(
487 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
489 valDict[k] = f._func(subdf)
490 except Exception
as e:
492 valDict[k] = f.fail(subdf)
497 if isinstance(data, DeferredDatasetHandle):
500 elif isinstance(data, pd.DataFrame):
507 valDict = {k: f._func(df)
for k, f
in self.
funcDictfuncDict.
items()}
510 for name, colVal
in valDict.items():
511 if len(colVal.shape) != 1:
512 raise RuntimeError(
"Transformed column '%s' is not the shape of a column. "
513 "It is shaped %s and type %s." % (name, colVal.shape,
type(colVal)))
516 valDf = pd.concat(valDict, axis=1)
518 print([(k,
type(v))
for k, v
in valDict.items()])
521 if kwargs.get(
'dropna',
False):
522 valDf = valDf.dropna(how=
'any')
528 if renameRules
is None:
530 for old, new
in renameRules:
531 if col.startswith(old):
532 col = col.replace(old, new)
538 filename = os.path.expandvars(filename)
539 with open(filename)
as f:
540 translationDefinition = yaml.safe_load(f)
542 return cls.
from_yamlfrom_yaml(translationDefinition, **kwargs)
547 for func, val
in translationDefinition[
'funcs'].
items():
550 if 'flag_rename_rules' in translationDefinition:
551 renameRules = translationDefinition[
'flag_rename_rules']
555 if 'calexpFlags' in translationDefinition:
556 for flag
in translationDefinition[
'calexpFlags']:
557 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
559 if 'refFlags' in translationDefinition:
560 for flag
in translationDefinition[
'refFlags']:
561 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'ref')
563 if 'forcedFlags' in translationDefinition:
564 for flag
in translationDefinition[
'forcedFlags']:
565 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
567 if 'flags' in translationDefinition:
568 for flag
in translationDefinition[
'flags']:
569 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'meas')
571 return cls(funcs, **kwargs)
575 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
577 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
581 df : pandas.DataFrame
582 Dataframe on which to evaluate expression.
588 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
589 val = df.eval(expr_new, truediv=
True)
591 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
592 val = df.eval(expr_new, truediv=
True)
597 """Arbitrary computation on a catalog
599 Column names (and thus the columns to be loaded from catalog) are found
600 by finding all words and trying to ignore all "math-y" words.
605 Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
607 _ignore_words = (
'mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
619 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
exprexpr)
621 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
exprexpr)
if c
not in self.
_ignore_words_ignore_words]
624 if not re.search(
'_instFlux$', c):
625 cols.append(f
'{c}_instFlux')
630 return list(
set([c
for c
in cols
if c
not in not_a_col]))
637 """Get column with specified name
653 return df[self.
colcol]
657 """Return the value of the index for each object
660 columns = [
'coord_ra']
661 _defaultDataset =
'ref'
665 return pd.Series(df.index, index=df.index)
670 _allow_difference =
False
674 return pd.Series(df.index, index=df.index)
678 col =
'base_Footprint_nPix'
682 """Base class for coordinate column, in degrees
691 output = df[self.
colcol] * 180 / np.pi
if self.
_radians_radians
else df[self.
colcol]
696 """Right Ascension, in degrees
702 super().
__init__(
'coord_ra', **kwargs)
705 return super().
__call__(catalog, **kwargs)
709 """Declination, in degrees
715 super().
__init__(
'coord_dec', **kwargs)
718 return super().
__call__(catalog, **kwargs)
722 """Compute the level 20 HtmIndex for the catalog.
726 This functor was implemented to satisfy requirements of old APDB interface
727 which required ``pixelId`` column in DiaObject with HTM20 index. APDB
728 interface had migrated to not need that information, but we keep this
729 class in case it may be useful for something else.
744 def computePixel(row):
753 return self.
pixelatorpixelator.index(sphPoint.getVector())
755 return df.apply(computePixel, axis=1, result_type=
'reduce').astype(
'int64')
759 if not col.endswith(
'_instFlux'):
765 if not col.endswith(
'_instFluxErr'):
766 col +=
'_instFluxErr'
771 """Compute calibrated magnitude
773 Takes a `calib` argument, which returns the flux at mag=0
774 as `calib.getFluxMag0()`. If not provided, then the default
775 `fluxMag0` is 63095734448.0194, which is default for HSC.
776 This default should be removed in DM-21955
778 This calculation hides warnings about invalid values and dividing by zero.
780 As for all functors, a `dataset` and `filt` kwarg should be provided upon
781 initialization. Unlike the default `Functor`, however, the default dataset
782 for a `Mag` is `'meas'`, rather than `'ref'`.
787 Name of flux column from which to compute magnitude. Can be parseable
788 by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
789 `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
791 calib : `lsst.afw.image.calib.Calib` (optional)
792 Object that knows zero point.
794 _defaultDataset =
'meas'
799 if calib
is not None:
803 self.
fluxMag0fluxMag0 = 63095734448.0194
812 with np.warnings.catch_warnings():
813 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
814 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
815 return -2.5*np.log10(df[self.
colcol] / self.
fluxMag0fluxMag0)
819 return f
'mag_{self.col}'
823 """Compute calibrated magnitude uncertainty
825 Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
830 calib : `lsst.afw.image.calib.Calib` (optional)
831 Object that knows zero point.
836 if self.
calibcalib
is not None:
843 return [self.
colcol, self.
colcol +
'Err']
846 with np.warnings.catch_warnings():
847 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
848 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
850 x = df[fluxErrCol] / df[fluxCol]
852 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
857 return super().name +
'_err'
865 return (df[self.
colcol] / self.
fluxMag0fluxMag0) * 1e9
869 _defaultDataset =
'meas'
871 """Functor to calculate magnitude difference"""
880 return [self.
col1col1, self.
col2col2]
883 with np.warnings.catch_warnings():
884 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
885 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
886 return -2.5*np.log10(df[self.
col1col1]/df[self.
col2col2])
890 return f
'(mag_{self.col1} - mag_{self.col2})'
894 return f
'magDiff_{self.col1}_{self.col2}'
898 """Compute the color between two filters
900 Computes color by initializing two different `Mag`
901 functors based on the `col` and filters provided, and
902 then returning the difference.
904 This is enabled by the `_func` expecting a dataframe with a
905 multilevel column index, with both `'band'` and `'column'`,
906 instead of just `'column'`, which is the `Functor` default.
907 This is controlled by the `_dfLevels` attribute.
909 Also of note, the default dataset for `Color` is `forced_src'`,
910 whereas for `Mag` it is `'meas'`.
915 Name of flux column from which to compute; same as would be passed to
916 `lsst.pipe.tasks.functors.Mag`.
919 Filters from which to compute magnitude difference.
920 Color computed is `Mag(filt2) - Mag(filt1)`.
922 _defaultDataset =
'forced_src'
923 _dfLevels = (
'band',
'column')
929 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
933 self.
mag2mag2 =
Mag(col, filt=filt2, **kwargs)
934 self.
mag1mag1 =
Mag(col, filt=filt1, **kwargs)
947 mag2 = self.mag2._func(df[self.filt2])
948 mag1 = self.mag1._func(df[self.filt1])
953 return [self.
mag1mag1.col, self.
mag2mag2.col]
960 return f
'{self.filt2} - {self.filt1} ({self.col})'
964 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
968 """Main function of this subclass is to override the dropna=True
971 _allow_difference =
False
976 return super().
__call__(parq, dropna=
False, **kwargs)
980 _columns = [
"base_ClassificationExtendedness_value"]
981 _column =
"base_ClassificationExtendedness_value"
986 test = (x < 0.5).astype(int)
987 test = test.mask(mask, 2)
991 categories = [
'galaxy',
'star', self.
_null_label_null_label]
992 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
993 index=x.index, name=
'label')
995 label = label.astype(str)
1000 _columns = [
'numStarFlags']
1001 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
1003 def _func(self, df):
1007 n = len(x.unique()) - 1
1009 labels = [
'noStar',
'maybe',
'star']
1010 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
1011 index=x.index, name=
'label')
1014 label = label.astype(str)
1020 name =
'Deconvolved Moments'
1021 shortname =
'deconvolvedMoments'
1022 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1023 "ext_shapeHSM_HsmSourceMoments_yy",
1024 "base_SdssShape_xx",
"base_SdssShape_yy",
1025 "ext_shapeHSM_HsmPsfMoments_xx",
1026 "ext_shapeHSM_HsmPsfMoments_yy")
1028 def _func(self, df):
1029 """Calculate deconvolved moments"""
1030 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
1031 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
1033 hsm = np.ones(len(df))*np.nan
1034 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
1035 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
1036 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
1041 raise RuntimeError(
'No psf shape parameter found in catalog')
1043 return hsm.where(np.isfinite(hsm), sdss) - psf
1047 """Functor to calculate SDSS trace radius size for sources"""
1048 name =
"SDSS Trace Size"
1049 shortname =
'sdssTrace'
1050 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1052 def _func(self, df):
1053 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1058 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1059 name =
"PSF - SDSS Trace Size"
1060 shortname =
'psf_sdssTrace'
1061 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1062 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1064 def _func(self, df):
1065 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1066 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1067 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1072 """Functor to calculate HSM trace radius size for sources"""
1073 name =
'HSM Trace Size'
1074 shortname =
'hsmTrace'
1075 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1076 "ext_shapeHSM_HsmSourceMoments_yy")
1078 def _func(self, df):
1079 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1080 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1085 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1086 name =
'PSF - HSM Trace Size'
1087 shortname =
'psf_HsmTrace'
1088 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1089 "ext_shapeHSM_HsmSourceMoments_yy",
1090 "ext_shapeHSM_HsmPsfMoments_xx",
1091 "ext_shapeHSM_HsmPsfMoments_yy")
1093 def _func(self, df):
1094 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1095 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1096 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1097 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1098 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1103 name =
'HSM Psf FWHM'
1104 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1107 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1109 def _func(self, df):
1111 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1115 name =
"Distortion Ellipticity (e1)"
1116 shortname =
"Distortion"
1129 def _func(self, df):
1134 name =
"Ellipticity e2"
1146 def _func(self, df):
1147 return 2*df[self.
colXYcolXY] / (df[self.
colXXcolXX] + df[self.
colYYcolYY])
1162 def _func(self, df):
1163 return (df[self.
colXXcolXX]*df[self.
colYYcolYY] - df[self.
colXYcolXY]**2)**0.25
1167 """Computations using the stored localWcs.
1169 name =
"LocalWcsOperations"
1184 """Compute the distance on the sphere from x2, y1 to x1, y1.
1192 cd11 : `pandas.Series`
1193 [1, 1] element of the local Wcs affine transform.
1194 cd11 : `pandas.Series`
1195 [1, 1] element of the local Wcs affine transform.
1196 cd12 : `pandas.Series`
1197 [1, 2] element of the local Wcs affine transform.
1198 cd21 : `pandas.Series`
1199 [2, 1] element of the local Wcs affine transform.
1200 cd22 : `pandas.Series`
1201 [2, 2] element of the local Wcs affine transform.
1206 RA and dec conversion of x and y given the local Wcs. Returned
1207 units are in radians.
1210 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1213 """Compute the local pixel scale conversion.
1217 ra1 : `pandas.Series`
1218 Ra of the first coordinate in radians.
1219 dec1 : `pandas.Series`
1220 Dec of the first coordinate in radians.
1221 ra2 : `pandas.Series`
1222 Ra of the second coordinate in radians.
1223 dec2 : `pandas.Series`
1224 Dec of the second coordinate in radians.
1228 dist : `pandas.Series`
1229 Distance on the sphere in radians.
1231 deltaDec = dec2 - dec1
1233 return 2 * np.arcsin(
1235 np.sin(deltaDec / 2) ** 2
1236 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1239 """Compute the distance on the sphere from x2, y1 to x1, y1.
1243 x1 : `pandas.Series`
1245 y1 : `pandas.Series`
1247 x2 : `pandas.Series`
1249 y2 : `pandas.Series`
1251 cd11 : `pandas.Series`
1252 [1, 1] element of the local Wcs affine transform.
1253 cd11 : `pandas.Series`
1254 [1, 1] element of the local Wcs affine transform.
1255 cd12 : `pandas.Series`
1256 [1, 2] element of the local Wcs affine transform.
1257 cd21 : `pandas.Series`
1258 [2, 1] element of the local Wcs affine transform.
1259 cd22 : `pandas.Series`
1260 [2, 2] element of the local Wcs affine transform.
1264 Distance : `pandas.Series`
1265 Arcseconds per pixel at the location of the local WC
1267 ra1, dec1 = self.
computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1268 ra2, dec2 = self.
computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1274 """Compute the local pixel scale from the stored CDMatrix.
1286 """Compute the local pixel to scale conversion in arcseconds.
1290 cd11 : `pandas.Series`
1291 [1, 1] element of the local Wcs affine transform in radians.
1292 cd11 : `pandas.Series`
1293 [1, 1] element of the local Wcs affine transform in radians.
1294 cd12 : `pandas.Series`
1295 [1, 2] element of the local Wcs affine transform in radians.
1296 cd21 : `pandas.Series`
1297 [2, 1] element of the local Wcs affine transform in radians.
1298 cd22 : `pandas.Series`
1299 [2, 2] element of the local Wcs affine transform in radians.
1303 pixScale : `pandas.Series`
1304 Arcseconds per pixel at the location of the local WC
1306 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1308 def _func(self, df):
1316 """Convert a value in units pixels squared to units arcseconds squared.
1335 return f
"{self.col}_asArcseconds"
1339 return [self.
colcol,
1345 def _func(self, df):
1353 """Convert a value in units pixels to units arcseconds.
1372 return f
"{self.col}_asArcsecondsSq"
1376 return [self.
colcol,
1382 def _func(self, df):
1387 return df[self.
colcol] * pixScale * pixScale
1391 name =
'Reference Band'
1392 shortname =
'refBand'
1396 return [
"merge_measurement_i",
1397 "merge_measurement_r",
1398 "merge_measurement_z",
1399 "merge_measurement_y",
1400 "merge_measurement_g",
1401 "merge_measurement_u"]
1403 def _func(self, df: pd.DataFrame) -> pd.Series:
1404 def getFilterAliasName(row):
1406 colName = row.idxmax()
1407 return colName.replace(
'merge_measurement_',
'')
1410 return df[self.
columnscolumnscolumns].apply(getFilterAliasName, axis=1,
1411 result_type=
'reduce').astype(
'object')
1416 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1417 LOG_AB_FLUX_SCALE = 12.56
1418 FIVE_OVER_2LOG10 = 1.085736204758129569
1422 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1428 if calib
is not None:
1438 return [self.
colcol]
1442 return f
'mag_{self.col}'
1446 if np.abs(a) < np.abs(b):
1451 return np.abs(a) * np.sqrt(1. + q*q)
1457 with np.warnings.catch_warnings():
1458 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1459 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1460 return -2.5 * np.log10(dn/fluxMag0)
1463 retVal = self.
vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1464 retVal *= self.
AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1468 retVal = self.
dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2fluxdn2flux(dn, fluxMag0)
1473 def _func(self, df):
1482 def _func(self, df):
1484 return pd.Series(retArr, index=df.index)
1488 def _func(self, df):
1497 def _func(self, df):
1499 return pd.Series(retArr, index=df.index)
1503 """Base class for calibrating the specified instrument flux column using
1504 the local photometric calibration.
1509 Name of the instrument flux column.
1510 instFluxErrCol : `str`
1511 Name of the assocated error columns for ``instFluxCol``.
1512 photoCalibCol : `str`
1513 Name of local calibration column.
1514 photoCalibErrCol : `str`
1515 Error associated with ``photoCalibCol``
1525 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1540 """Convert instrument flux to nanojanskys.
1544 instFlux : `numpy.ndarray` or `pandas.Series`
1545 Array of instrument flux measurements
1546 localCalib : `numpy.ndarray` or `pandas.Series`
1547 Array of local photometric calibration estimates.
1551 calibFlux : `numpy.ndarray` or `pandas.Series`
1552 Array of calibrated flux measurements.
1554 return instFlux * localCalib
1557 """Convert instrument flux to nanojanskys.
1561 instFlux : `numpy.ndarray` or `pandas.Series`
1562 Array of instrument flux measurements
1563 instFluxErr : `numpy.ndarray` or `pandas.Series`
1564 Errors on associated ``instFlux`` values
1565 localCalib : `numpy.ndarray` or `pandas.Series`
1566 Array of local photometric calibration estimates.
1567 localCalibErr : `numpy.ndarray` or `pandas.Series`
1568 Errors on associated ``localCalib`` values
1572 calibFluxErr : `numpy.ndarray` or `pandas.Series`
1573 Errors on calibrated flux measurements.
1575 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1578 """Convert instrument flux to nanojanskys.
1582 instFlux : `numpy.ndarray` or `pandas.Series`
1583 Array of instrument flux measurements
1584 localCalib : `numpy.ndarray` or `pandas.Series`
1585 Array of local photometric calibration estimates.
1589 calibMag : `numpy.ndarray` or `pandas.Series`
1590 Array of calibrated AB magnitudes.
1595 """Convert instrument flux err to nanojanskys.
1599 instFlux : `numpy.ndarray` or `pandas.Series`
1600 Array of instrument flux measurements
1601 instFluxErr : `numpy.ndarray` or `pandas.Series`
1602 Errors on associated ``instFlux`` values
1603 localCalib : `numpy.ndarray` or `pandas.Series`
1604 Array of local photometric calibration estimates.
1605 localCalibErr : `numpy.ndarray` or `pandas.Series`
1606 Errors on associated ``localCalib`` values
1610 calibMagErr: `numpy.ndarray` or `pandas.Series`
1611 Error on calibrated AB magnitudes.
1614 return 2.5 / np.log(10) * err / self.
instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1618 """Compute calibrated fluxes using the local calibration value.
1634 return f
'flux_{self.instFluxCol}'
1636 def _func(self, df):
1641 """Compute calibrated flux errors using the local calibration value.
1658 return f
'fluxErr_{self.instFluxCol}'
1660 def _func(self, df):
1666 """Compute calibrated AB magnitudes using the local calibration value.
1682 return f
'mag_{self.instFluxCol}'
1684 def _func(self, df):
1690 """Compute calibrated AB magnitude errors using the local calibration value.
1707 return f
'magErr_{self.instFluxCol}'
1709 def _func(self, df):
1717 """Compute absolute mean of dipole fluxes.
1726 LocalDipoleMeanFluxErr
1728 LocalDipoleDiffFluxErr
1758 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1760 def _func(self, df):
1766 """Compute the error on the absolute mean of dipole fluxes.
1775 LocalDipoleMeanFluxErr
1777 LocalDipoleDiffFluxErr
1791 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1793 def _func(self, df):
1802 """Compute the absolute difference of dipole fluxes.
1804 Value is (abs(pos) - abs(neg))
1813 LocalDipoleMeanFluxErr
1815 LocalDipoleDiffFluxErr
1826 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1828 def _func(self, df):
1834 """Compute the error on the absolute difference of dipole fluxes.
1843 LocalDipoleMeanFluxErr
1845 LocalDipoleDiffFluxErr
1859 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1861 def _func(self, df):
1870 """Base class for returning the ratio of 2 columns.
1872 Can be used to compute a Signal to Noise ratio for any input flux.
1877 Name of the column to use at the numerator in the ratio
1879 Name of the column to use as the denominator in the ratio.
1895 return f
'ratio_{self.numerator}_{self.denominator}'
1897 def _func(self, df):
1898 with np.warnings.catch_warnings():
1899 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1900 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
std::vector< SchemaItem< Flag > > * items
Point in an unspecified spherical coordinate system.
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __init__(self, ra, decl, **kwargs)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
HtmPixelization provides HTM indexing of points and regions.
daf::base::PropertyList * list
daf::base::PropertySet * set
def mag_aware_eval(df, expr)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)