LSST Applications 28.0.2,g0fba68d861+5b923b673a,g1fd858c14a+7a7b9dd5ed,g2c84ff76c0+5548bfee71,g30358e5240+f0e04ebe90,g35bb328faa+fcb1d3bbc8,g436fd98eb5+bdc6fcdd04,g4af146b050+742274f7cd,g4d2262a081+3efd3f8190,g4e0f332c67+cb09b8a5b6,g53246c7159+fcb1d3bbc8,g5a012ec0e7+477f9c599b,g5edb6fd927+826dfcb47f,g60b5630c4e+bdc6fcdd04,g67b6fd64d1+2218407a0c,g78460c75b0+2f9a1b4bcd,g786e29fd12+cf7ec2a62a,g7b71ed6315+fcb1d3bbc8,g87b7deb4dc+f9ac2ab1bd,g8852436030+ebf28f0d95,g89139ef638+2218407a0c,g9125e01d80+fcb1d3bbc8,g989de1cb63+2218407a0c,g9f33ca652e+42fb53f4c8,g9f7030ddb1+11b9b6f027,ga2b97cdc51+bdc6fcdd04,gab72ac2889+bdc6fcdd04,gabe3b4be73+1e0a283bba,gabf8522325+3210f02652,gb1101e3267+9c79701da9,gb58c049af0+f03b321e39,gb89ab40317+2218407a0c,gcf25f946ba+ebf28f0d95,gd6cbbdb0b4+e8f9c9c900,gd9a9a58781+fcb1d3bbc8,gde0f65d7ad+a08f294619,ge278dab8ac+3ef3db156b,ge410e46f29+2218407a0c,gf67bdafdda+2218407a0c
LSST Data Management Base Package
Loading...
Searching...
No Matches
_base.py
Go to the documentation of this file.
1# This file is part of afw.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import numpy as np
22
23from lsst.utils import continueClass, TemplateMeta
24from ._table import BaseRecord, BaseCatalog
25from ._schema import Key
26
27
28__all__ = ["Catalog"]
29
30
31@continueClass
32class BaseRecord: # noqa: F811
33
34 def extract(self, *patterns, **kwargs):
35 """Extract a dictionary of {<name>: <field-value>} in which the field
36 names match the given shell-style glob pattern(s).
37
38 Any number of glob patterns may be passed; the result will be the union
39 of all the result of each glob considered separately.
40
41 Parameters
42 ----------
43 items : `dict`
44 The result of a call to self.schema.extract(); this will be used
45 instead of doing any new matching, and allows the pattern matching
46 to be reused to extract values from multiple records. This
47 keyword is incompatible with any position arguments and the regex,
48 sub, and ordered keyword arguments.
49 regex : `str` or `re` pattern object
50 A regular expression to be used in addition to any glob patterns
51 passed as positional arguments. Note that this will be compared
52 with re.match, not re.search.
53 sub : `str`
54 A replacement string (see `re.MatchObject.expand`) used to set the
55 dictionary keys of any fields matched by regex.
56 ordered : `bool`
57 If `True`, a `collections.OrderedDict` will be returned instead of
58 a standard dict, with the order corresponding to the definition
59 order of the `Schema`. Default is `False`.
60 """
61 d = kwargs.pop("items", None)
62 if d is None:
63 d = self.schema.extract(*patterns, **kwargs).copy()
64 elif kwargs:
65 kwargsStr = ", ".join(kwargs.keys())
66 raise ValueError(f"Unrecognized keyword arguments for extract: {kwargsStr}")
67 return {name: self.get(schemaItem.key) for name, schemaItem in d.items()}
68
69 def __repr__(self):
70 return f"{type(self)}\n{self}"
71
72
73class Catalog(metaclass=TemplateMeta):
74
75 def getColumnView(self):
76 self._columns = self._getColumnView()
77 return self._columns
78
79 def __getColumns(self):
80 if not hasattr(self, "_columns") or self._columns is None:
81 self._columns = self._getColumnView()
82 return self._columns
83 columns = property(__getColumns, doc="a column view of the catalog")
84
85 def __getitem__(self, key):
86 """Return the record at index key if key is an integer,
87 return a column if `key` is a string field name or Key,
88 or return a subset of the catalog if key is a slice
89 or boolean NumPy array.
90 """
91 if type(key) is slice:
92 (start, stop, step) = (key.start, key.stop, key.step)
93 if step is None:
94 step = 1
95 if start is None:
96 start = 0
97 if stop is None:
98 stop = len(self)
99 return self.subset(start, stop, step)
100 elif isinstance(key, np.ndarray):
101 if key.dtype == bool:
102 return self.subset(key)
103 raise RuntimeError("Unsupported array type for indexing a Catalog, "
104 f"only boolean arrays are supported: {key.dtype}")
105 elif isinstance(key, str):
106 key = self.schema.find(key).key
107 result, self._columns = self._get_column_from_key(key, self._columns)
108 return result
109 elif isinstance(key, Key):
110 result, self._columns = self._get_column_from_key(key, self._columns)
111 return result
112 else:
113 return self._getitem_(key)
114
115 def __setitem__(self, key, value):
116 """If ``key`` is an integer, set ``catalog[key]`` to
117 ``value``. Otherwise select column ``key`` and set it to
118 ``value``.
119 """
120 self._columns = None
121 if isinstance(key, str):
122 key = self.schema[key].asKey()
123 if isinstance(key, Key):
124 if isinstance(key, Key["Flag"]):
125 self._set_flag(key, value)
126 else:
127 self.columns[key] = value
128 else:
129 return self.set(key, value)
130
131 def __delitem__(self, key):
132 self._columns = None
133 if isinstance(key, slice):
134 self._delslice_(key)
135 else:
136 self._delitem_(key)
137
138 def append(self, record):
139 self._columns = None
140 self._append(record)
141
142 def insert(self, key, value):
143 self._columns = None
144 self._insert(key, value)
145
146 def clear(self):
147 self._columns = None
148 self._clear()
149
150 def addNew(self):
151 self._columns = None
152 return self._addNew()
153
154 def cast(self, type_, deep=False):
155 """Return a copy of the catalog with the given type.
156
157 Parameters
158 ----------
159 type_ :
160 Type of catalog to return.
161 deep : `bool`, optional
162 If `True`, clone the table and deep copy all records.
163
164 Returns
165 -------
166 copy :
167 Copy of catalog with the requested type.
168 """
169 if deep:
170 table = self.table.clone()
171 table.preallocate(len(self))
172 else:
173 table = self.table
174 copy = type_(table)
175 copy.extend(self, deep=deep)
176 return copy
177
178 def copy(self, deep=False):
179 """
180 Copy a catalog (default is not a deep copy).
181 """
182 return self.cast(type(self), deep)
183
184 def extend(self, iterable, deep=False, mapper=None):
185 """Append all records in the given iterable to the catalog.
186
187 Parameters
188 ----------
189 iterable :
190 Any Python iterable containing records.
191 deep : `bool`, optional
192 If `True`, the records will be deep-copied; ignored if
193 mapper is not `None` (that always implies `True`).
194 mapper : `lsst.afw.table.schemaMapper.SchemaMapper`, optional
195 Used to translate records.
196 """
197 self._columns = None
198 # We can't use isinstance here, because the SchemaMapper symbol isn't available
199 # when this code is part of a subclass of Catalog in another package.
200 if type(deep).__name__ == "SchemaMapper":
201 mapper = deep
202 deep = None
203 if isinstance(iterable, type(self)):
204 if mapper is not None:
205 self._extend(iterable, mapper)
206 else:
207 self._extend(iterable, deep)
208 else:
209 for record in iterable:
210 if mapper is not None:
211 self._append(self.table.copyRecord(record, mapper))
212 elif deep:
213 self._append(self.table.copyRecord(record))
214 else:
215 self._append(record)
216
217 def __reduce__(self):
218 import lsst.afw.fits
219 return lsst.afw.fits.reduceToFits(self)
220
221 def asAstropy(self, cls=None, copy=False, unviewable="copy"):
222 """Return an astropy.table.Table (or subclass thereof) view into this catalog.
223
224 Parameters
225 ----------
226 cls :
227 Table subclass to use; `None` implies `astropy.table.Table`
228 itself. Use `astropy.table.QTable` to get Quantity columns.
229 copy : bool, optional
230 If `True`, copy data from the LSST catalog to the astropy
231 table. Not copying is usually faster, but can keep memory
232 from being freed if columns are later removed from the
233 Astropy view.
234 unviewable : `str`, optional
235 One of the following options (which is ignored if
236 copy=`True` ), indicating how to handle field types (`str`
237 and `Flag`) for which views cannot be constructed:
238
239 - 'copy' (default): copy only the unviewable fields.
240 - 'raise': raise ValueError if unviewable fields are present.
241 - 'skip': do not include unviewable fields in the Astropy Table.
242
243 Returns
244 -------
245 cls : `astropy.table.Table`
246 Astropy view into the catalog.
247
248 Raises
249 ------
250 ValueError
251 Raised if the `unviewable` option is not a known value, or
252 if the option is 'raise' and an uncopyable field is found.
253
254 """
255 import astropy.table
256 if cls is None:
257 cls = astropy.table.Table
258 if unviewable not in ("copy", "raise", "skip"):
259 raise ValueError(
260 f"'unviewable'={unviewable!r} must be one of 'copy', 'raise', or 'skip'")
261 ps = self.getMetadata()
262 meta = ps.toOrderedDict() if ps is not None else None
263 columns = []
264 items = self.schema.extract("*", ordered=True)
265 for name, item in items.items():
266 key = item.key
267 unit = item.field.getUnits() or None # use None instead of "" when empty
268 if key.getTypeString() == "String":
269 if not copy:
270 if unviewable == "raise":
271 raise ValueError("Cannot extract string "
272 "unless copy=True or unviewable='copy' or 'skip'.")
273 elif unviewable == "skip":
274 continue
275 data = np.zeros(
276 len(self), dtype=np.dtype((str, key.getSize())))
277 for i, record in enumerate(self):
278 data[i] = record.get(key)
279 elif key.getTypeString() == "Flag":
280 if not copy:
281 if unviewable == "raise":
282 raise ValueError("Cannot extract packed bit columns "
283 "unless copy=True or unviewable='copy' or 'skip'.")
284 elif unviewable == "skip":
285 continue
286 data = self[key]
287 elif key.getTypeString() == "Angle":
288 data = self.columns.get(key)
289 unit = "radian"
290 if copy:
291 data = data.copy()
292 elif "Array" in key.getTypeString() and key.isVariableLength():
293 # Can't get columns for variable-length array fields.
294 if unviewable == "raise":
295 raise ValueError("Cannot extract variable-length array fields unless unviewable='skip'.")
296 elif unviewable == "skip" or unviewable == "copy":
297 continue
298 else:
299 data = self.columns.get(key)
300 if copy:
301 data = data.copy()
302 columns.append(
303 astropy.table.Column(
304 data,
305 name=name,
306 unit=unit,
307 description=item.field.getDoc()
308 )
309 )
310 return cls(columns, meta=meta, copy=False)
311
312 def __dir__(self):
313 """
314 This custom dir is necessary due to the custom getattr below.
315 Without it, not all of the methods available are returned with dir.
316 See DM-7199.
317 """
318 def recursive_get_class_dir(cls):
319 """
320 Return a set containing the names of all methods
321 for a given class *and* all of its subclasses.
322 """
323 result = set()
324 if cls.__bases__:
325 for subcls in cls.__bases__:
326 result |= recursive_get_class_dir(subcls)
327 result |= set(cls.__dict__.keys())
328 return result
329 return sorted(set(dir(self.columns)) | set(dir(self.table))
330 | recursive_get_class_dir(type(self)) | set(self.__dict__.keys()))
331
332 def __getattr__(self, name):
333 # Catalog forwards unknown method calls to its table and column view
334 # for convenience. (Feature requested by RHL; complaints about magic
335 # should be directed to him.)
336 if name == "_columns":
337 self._columns = None
338 return None
339 try:
340 return getattr(self.table, name)
341 except AttributeError:
342 # Special case __ properties as they are never going to be column
343 # names.
344 if name.startswith("__"):
345 raise
346 # This can fail if the table is non-contiguous
347 try:
348 attr = getattr(self.columns, name)
349 except Exception as e:
350 e.add_note(f"Error retrieving column attribute '{name}' from {type(self)}")
351 raise
352 return attr
353
354 def __str__(self):
355 if self.isContiguous():
356 return str(self.asAstropy())
357 else:
358 fields = ' '.join(x.field.getName() for x in self.schema)
359 return f"Non-contiguous afw.Catalog of {len(self)} rows.\ncolumns: {fields}"
360
361 def __repr__(self):
362 return "%s\n%s" % (type(self), self)
363
364 def extract(self, *patterns, **kwds):
365 """Extract a dictionary of {<name>: <column-array>} in which the field
366 names match the given shell-style glob pattern(s).
367
368 Any number of glob patterns may be passed (including none); the result
369 will be the union of all the result of each glob considered separately.
370
371 Note that extract("*", copy=True) provides an easy way to transform a
372 catalog into a set of writeable contiguous NumPy arrays.
373
374 This routines unpacks `Flag` columns into full boolean arrays. String
375 fields are silently ignored.
376
377 Parameters
378 ----------
379 patterns : Array of `str`
380 List of glob patterns to use to select field names.
381 kwds : `dict`
382 Dictionary of additional keyword arguments. May contain:
383
384 ``items`` : `list`
385 The result of a call to self.schema.extract(); this will be
386 used instead of doing any new matching, and allows the pattern
387 matching to be reused to extract values from multiple records.
388 This keyword is incompatible with any position arguments and
389 the regex, sub, and ordered keyword arguments.
390 ``where`` : array index expression
391 Any expression that can be passed as indices to a NumPy array,
392 including slices, boolean arrays, and index arrays, that will
393 be used to index each column array. This is applied before
394 arrays are copied when copy is True, so if the indexing results
395 in an implicit copy no unnecessary second copy is performed.
396 ``copy`` : `bool`
397 If True, the returned arrays will be contiguous copies rather
398 than strided views into the catalog. This ensures that the
399 lifetime of the catalog is not tied to the lifetime of a
400 particular catalog, and it also may improve the performance if
401 the array is used repeatedly. Default is False. Copies are
402 always made if the catalog is noncontiguous, but if
403 ``copy=False`` these set as read-only to ensure code does not
404 assume they are views that could modify the original catalog.
405 ``regex`` : `str` or `re` pattern
406 A regular expression to be used in addition to any glob
407 patterns passed as positional arguments. Note that this will
408 be compared with re.match, not re.search.
409 ``sub`` : `str`
410 A replacement string (see re.MatchObject.expand) used to set
411 the dictionary keys of any fields matched by regex.
412 ``ordered`` : `bool`
413 If True, a collections.OrderedDict will be returned instead of
414 a standard dict, with the order corresponding to the definition
415 order of the Schema. Default is False.
416
417 Returns
418 -------
419 d : `dict`
420 Dictionary of extracted name-column array sets.
421
422 Raises
423 ------
424 ValueError
425 Raised if a list of ``items`` is supplied with additional keywords.
426 """
427 copy = kwds.pop("copy", False)
428 where = kwds.pop("where", None)
429 d = kwds.pop("items", None)
430 # If ``items`` is given as a kwd, an extraction has already been
431 # performed and there shouldn't be any additional keywords. Otherwise
432 # call schema.extract to load the dictionary.
433 if d is None:
434 d = self.schema.extract(*patterns, **kwds).copy()
435 elif kwds:
436 raise ValueError(
437 "kwd 'items' was specified, which is not compatible with additional keywords")
438
439 def processArray(a):
440 if where is not None:
441 a = a[where]
442 if copy:
443 a = a.copy()
444 return a
445
446 # must use list because we might be adding/deleting elements
447 for name, schemaItem in list(d.items()):
448 key = schemaItem.key
449 if key.getTypeString() == "String":
450 del d[name]
451 else:
452 d[name] = processArray(self[schemaItem.key])
453 return d
454
455
456Catalog.register("Base", BaseCatalog)
extract(self, *patterns, **kwargs)
Definition _base.py:34
asAstropy(self, cls=None, copy=False, unviewable="copy")
Definition _base.py:221
insert(self, key, value)
Definition _base.py:142
extend(self, iterable, deep=False, mapper=None)
Definition _base.py:184
copy(self, deep=False)
Definition _base.py:178
cast(self, type_, deep=False)
Definition _base.py:154
extract(self, *patterns, **kwds)
Definition _base.py:364