LSSTApplications  1.1.2+25,10.0+13,10.0+132,10.0+133,10.0+224,10.0+41,10.0+8,10.0-1-g0f53050+14,10.0-1-g4b7b172+19,10.0-1-g61a5bae+98,10.0-1-g7408a83+3,10.0-1-gc1e0f5a+19,10.0-1-gdb4482e+14,10.0-11-g3947115+2,10.0-12-g8719d8b+2,10.0-15-ga3f480f+1,10.0-2-g4f67435,10.0-2-gcb4bc6c+26,10.0-28-gf7f57a9+1,10.0-3-g1bbe32c+14,10.0-3-g5b46d21,10.0-4-g027f45f+5,10.0-4-g86f66b5+2,10.0-4-gc4fccf3+24,10.0-40-g4349866+2,10.0-5-g766159b,10.0-5-gca2295e+25,10.0-6-g462a451+1
LSSTDataManagementBasePackage
Functions | Variables
lsst.datarel.utils Namespace Reference

Functions

def runStage
 
def cfhtMain
 
def lsstSimMain
 
def cfhtSetup
 
def getDataset
 
def getPsf
 
def lsstSimSetup
 

Variables

 haveCfht = True
 
 haveLsstSim = True
 

Function Documentation

def lsst.datarel.utils.cfhtMain (   processFunction,
  outDatasetType,
  need = (),
  defaultRoot = "." 
)

Definition at line 51 of file utils.py.

51 
52 def cfhtMain(processFunction, outDatasetType, need=(), defaultRoot="."):
53  parser = OptionParser()
54  parser.add_option("-i", "--input", dest="root",
55  default=defaultRoot, help="input root")
56  parser.add_option("-o", "--output", dest="outRoot", default=".",
57  help="output root")
58  parser.add_option("-f", "--force", action="store_true", default=False,
59  help="execute even if output dataset exists")
60  if "calib" in need:
61  parser.add_option("-C", "--calibRoot", dest="calibRoot",
62  help="calibration root")
63  parser.add_option("-R", "--registry", help="registry")
64  if "skyTile" in need:
65  parser.add_option("-t", "--skyTile", action="append", type="int",
66  help="sky tile numbers (can be repeated)")
67  else:
68  parser.add_option("-v", "--visit", action="append", type="int",
69  help="visit numbers (can be repeated)")
70  if "ccd" in need or "amp" in need:
71  parser.add_option("-c", "--ccd", action="append", type="int",
72  help="ccd number (can be repeated)")
73  if "amp" in need:
74  parser.add_option("-a", "--amp", action="append", type="int",
75  help="amp number (can be repeated)")
76  (options, args) = parser.parse_args()
77 
78  if options.registry is None:
79  if os.path.exists(os.path.join(options.root, "registry.sqlite3")):
80  options.registry = os.path.join(options.root, "registry.sqlite3")
81  if options.registry is None:
82  if os.path.exists("/lsst/DC3/data/obstest/CFHTLS/registry.sqlite3"):
83  options.registry = "/lsst/DC3/data/obstest/CFHTLS/registry.sqlite3"
84  if "calib" in need:
85  if options.calibRoot is None:
86  if os.path.exists("/lsst/DC3/data/obstest/CFHTLS/calib"):
87  options.calibRoot = "/lsst/DC3/data/obstest/CFHTLS/calib"
88  bf = dafPersist.ButlerFactory(mapper=CfhtMapper(
89  root=options.root, calibRoot=options.calibRoot,
90  registry=options.registry))
91  else:
92  bf = dafPersist.ButlerFactory(mapper=CfhtMapper(
93  root=options.root, registry=options.registry))
94  inButler = bf.create()
95  obf = dafPersist.ButlerFactory(mapper=CfhtMapper(
96  root=options.outRoot, registry=options.registry))
97  outButler = obf.create()
98 
99  if "skyTile" in need:
100  if options.skyTile is None:
101  print >>sys.stderr, "Running over all sky tiles"
102  options.skyTile = inButler.queryMetadata("raw", "skyTile")
103  elif not hasattr(options.skyTile, "__iter__"):
104  options.skyTile = [options.skyTile]
105  for skyTile in options.skyTile:
106  if options.force or not outButler.datasetExists(outDatasetType,
107  skyTile=skyTile):
108  print >>sys.stderr, \
109  "***** Processing skyTile %d" % (skyTile,)
110  processFunction(inButler=inButler, outButler=outButler,
111  skyTile=skyTile)
112  return
113 
114  if options.visit is None:
115  print >>sys.stderr, "Running over all input visits"
116  options.visit = inButler.queryMetadata("raw", "visit")
117  elif not hasattr(options.visit, "__iter__"):
118  options.visit = [options.visit]
119  if "ccd" in need or "amp" in need:
120  if options.ccd is None:
121  print >>sys.stderr, "Running over all CCDs"
122  options.ccd = inButler.queryMetadata("raw", "ccd")
123  elif not hasattr(options.ccd, "__iter__"):
124  options.ccd = [options.ccd]
125  if "amp" in need:
126  if options.amp is None:
127  print >>sys.stderr, "Running over all amps"
128  options.amp = inButler.queryMetadata("raw", "amp")
129  elif not hasattr(options.amp, "__iter__"):
130  options.amp = [options.amp]
131 
132  for visit in options.visit:
133  if "ccd" in need or "amp" in need:
134  for ccd in options.ccd:
135  if "amp" in need:
136  for amp in options.amp:
137  if options.force or \
138  not outButler.datasetExists(outDatasetType,
139  visit=visit, ccd=ccd, amp=amp):
140  print >>sys.stderr, \
141  "***** Processing visit %d ccd %d amp %d" % \
142  (visit, ccd, amp)
143  processFunction(inButler=inButler,
144  outButler=outButler,
145  visit=visit, ccd=ccd, amp=amp)
146  else:
147  if options.force or \
148  not outButler.datasetExists(outDatasetType,
149  visit=visit, ccd=ccd):
150  print >>sys.stderr, \
151  "***** Processing visit %d ccd %d" % \
152  (visit, ccd)
153  processFunction(inButler=inButler, outButler=outButler,
154  visit=visit, ccd=ccd)
155  else:
156  if options.force or \
157  not outButler.datasetExists(outDatasetType, visit=visit):
158  print >>sys.stderr, "***** Processing visit %d" % (visit,)
159  processFunction(inButler=inButler, outButler=outButler,
160  visit=visit)
161 
def lsst.datarel.utils.cfhtSetup (   root,
  outRoot,
  registry,
  calibRoot,
  inButler,
  outButler 
)

Definition at line 338 of file utils.py.

338 
339 def cfhtSetup(root, outRoot, registry, calibRoot, inButler, outButler):
340  if inButler is None:
341  if calibRoot is None:
342  if os.path.exists("/lsst/DC3/data/obstest/CFHTLS/calib"):
343  calibRoot = "/lsst/DC3/data/obstest/CFHTLS/calib"
344  if registry is None and root is not None:
345  if os.path.exists(os.path.join(root, "registry.sqlite3")):
346  registry = os.path.join(root, "registry.sqlite3")
347  bf = dafPersist.ButlerFactory(mapper=CfhtMapper(
348  root=root, calibRoot=calibRoot, registry=registry))
349  inButler = bf.create()
350  if outButler is None:
351  if outRoot is None:
352  outRoot = root
353  obf = dafPersist.ButlerFactory(mapper=CfhtMapper(
354  root=outRoot, registry=registry))
355  outButler = obf.create()
356  return (inButler, outButler)
def lsst.datarel.utils.getDataset (   butler,
  dataset,
  dataId,
  strict,
  warn 
) 
Get a dataset from a repository with an optional exception or warning if not found

@param[in] butler: data butler
@param[in] dataset: name of desired dataset
@param[in] dataId: data ID dict
@param[in] strict: if True then raise RuntimeError if dataset not found
@param[in] warn: if True and strict False then print a warning to stderr if dataset not found

@raise RuntimeError if dataset not found and strict true

Definition at line 357 of file utils.py.

357 
358 def getDataset(butler, dataset, dataId, strict, warn):
359  """Get a dataset from a repository with an optional exception or warning if not found
360 
361  @param[in] butler: data butler
362  @param[in] dataset: name of desired dataset
363  @param[in] dataId: data ID dict
364  @param[in] strict: if True then raise RuntimeError if dataset not found
365  @param[in] warn: if True and strict False then print a warning to stderr if dataset not found
366 
367  @raise RuntimeError if dataset not found and strict true
368  """
369  try:
370  ds = butler.get(dataset, dataId=dataId, immediate=True)
371  except:
372  ds = None
373  if ds == None:
374  msg = '{} : Failed to retrieve {} dataset'.format(dataId, dataset)
375  if strict:
376  raise RuntimeError(msg)
377  elif warn:
378  print >>sys.stderr, '*** Skipping ' + msg
379  return ds
def lsst.datarel.utils.getPsf (   butler,
  dataset,
  dataId,
  strict,
  warn 
) 
Get the PSF from a repository without reading (very much of) the exposure

@param[in] butler: data butler
@param[in] dataset: name of desired dataset
@param[in] dataId: data ID dict of exposure containing desired PSF
@param[in] strict: if True then raise RuntimeError if psf not found
@param[in] warn: if True and strict False then print a warning to stderr if psf not found

@raise RuntimeError if exposure not found (regardless of strict)
@raise RuntimeError if exposure has no PSF and strict true

Definition at line 380 of file utils.py.

380 
381 def getPsf(butler, dataset, dataId, strict, warn):
382  """Get the PSF from a repository without reading (very much of) the exposure
383 
384  @param[in] butler: data butler
385  @param[in] dataset: name of desired dataset
386  @param[in] dataId: data ID dict of exposure containing desired PSF
387  @param[in] strict: if True then raise RuntimeError if psf not found
388  @param[in] warn: if True and strict False then print a warning to stderr if psf not found
389 
390  @raise RuntimeError if exposure not found (regardless of strict)
391  @raise RuntimeError if exposure has no PSF and strict true
392  """
393  # there is not yet a way to read just the PSF, so read a 1x1 subregion of the exposure
394  tinyBBox = afwGeom.Box2I(afwGeom.Point2I(0,0), afwGeom.Extent2I(1,1))
395  tinyExposure = butler.get(dataset + "_sub", dataId=dataId, bbox=tinyBBox, imageOrigin="LOCAL", immediate=True)
396  psf = tinyExposure.getPsf()
397  if psf is None:
398  msg = '%s : %s exposure had no PSF' % (dataId, dataset)
399  psf = None
400  if strict:
401  raise RuntimeError(msg)
402  elif warn:
403  print >>sys.stderr, '*** Skipping ' + msg
404  return psf
lsst.afw.geom::Box2I
An integer coordinate rectangle.
Definition: Box.h:53
def lsst.datarel.utils.lsstSimMain (   processFunction,
  outDatasetType,
  need = (),
  defaultRoot = "." 
)

Definition at line 162 of file utils.py.

162 
163 def lsstSimMain(processFunction, outDatasetType, need=(), defaultRoot="."):
164  parser = OptionParser()
165  parser.add_option("-i", "--input", dest="root",
166  default=defaultRoot, help="input root")
167  parser.add_option("-o", "--output", dest="outRoot", default=".",
168  help="output root")
169  parser.add_option("-f", "--force", action="store_true", default=False,
170  help="execute even if output dataset exists")
171  if "calib" in need:
172  parser.add_option("-C", "--calibRoot", dest="calibRoot",
173  help="calibration root")
174  parser.add_option("-R", "--registry", help="registry")
175  if "skyTile" in need:
176  parser.add_option("-t", "--skyTile", action="append", type="int",
177  help="sky tile numbers (can be repeated)")
178  else:
179  parser.add_option("-v", "--visit", action="append", type="int",
180  help="visit number (can be repeated)")
181  if "snap" in need:
182  parser.add_option("-S", "--snap", action="append", type="int",
183  help="snap number (can be repeated)")
184  if "sensor" in need or "channel" in need:
185  parser.add_option("-r", "--raft", action="append",
186  help="raft coords (can be repeated)")
187  parser.add_option("-s", "--sensor", action="append",
188  help="sensor coords (can be repeated)")
189  if "channel" in need:
190  parser.add_option("-a", "--channel", action="append",
191  help="channel coords (can be repeated)")
192  (options, args) = parser.parse_args()
193 
194  if options.registry is None:
195  if os.path.exists(os.path.join(options.root, "registry.sqlite3")):
196  options.registry = os.path.join(options.root, "registry.sqlite3")
197  elif os.path.exists("/lsst/DC3/data/obstest/ImSim/registry.sqlite3"):
198  options.registry = "/lsst/DC3/data/obstest/ImSim/registry.sqlite3"
199  if "calib" in need:
200  if os.path.exists("/lsst/DC3/data/obstest/ImSim"):
201  options.calibRoot = "/lsst/DC3/data/obstest/ImSim"
202  bf = dafPersist.ButlerFactory(mapper=LsstSimMapper(
203  root=options.root, calibRoot=options.calibRoot,
204  registry=options.registry))
205  else:
206  bf = dafPersist.ButlerFactory(mapper=LsstSimMapper(
207  root=options.root, registry=options.registry))
208  inButler = bf.create()
209  obf = dafPersist.ButlerFactory(mapper=LsstSimMapper(
210  root=options.outRoot, registry=options.registry))
211  outButler = obf.create()
212 
213  if "skyTile" in need:
214  if options.skyTile is None:
215  print >>sys.stderr, "Running over all sky tiles"
216  options.skyTile = inButler.queryMetadata("raw", "skyTile")
217  elif not hasattr(options.skyTile, "__iter__"):
218  options.skyTile = [options.skyTile]
219  for skyTile in options.skyTile:
220  if options.force or not outButler.datasetExists(outDatasetType,
221  skyTile=skyTile):
222  print >>sys.stderr, \
223  "***** Processing skyTile %d" % (skyTile,)
224  processFunction(inButler=inButler, outButler=outButler,
225  skyTile=skyTile)
226  return
227 
228  if options.visit is None:
229  print >>sys.stderr, "Running over all input visits"
230  options.visit = inButler.queryMetadata("raw", "visit")
231  elif not hasattr(options.visit, "__iter__"):
232  options.visit = [options.visit]
233 
234  if "snap" in need:
235  if options.snap is None:
236  print >>sys.stderr, "Running over all snaps"
237  options.snap = inButler.queryMetadata("raw", "snap")
238  elif not hasattr(options.snap, "__iter__"):
239  options.snap = [options.snap]
240  else:
241  setattr(options, "snap", [0])
242 
243  if "sensor" in need or "channel" in need:
244  if options.raft is None:
245  print >>sys.stderr, "Running over all rafts"
246  options.raft = inButler.queryMetadata("raw", "raft")
247  elif not hasattr(options.raft, "__iter__"):
248  options.raft = [options.raft]
249 
250  if "sensor" in need or "channel" in need:
251  if options.sensor is None:
252  print >>sys.stderr, "Running over all sensors"
253  options.sensor = inButler.queryMetadata("raw", "sensor")
254  elif not hasattr(options.sensor, "__iter__"):
255  options.sensor = [options.sensor]
256 
257  if "channel" in need:
258  if options.channel is None:
259  print >>sys.stderr, "Running over all channels"
260  options.channel = inButler.queryMetadata("raw", "channel")
261  elif not hasattr(options.channel, "__iter__"):
262  options.channel = [options.channel]
263 
264  for visit in options.visit:
265  if "sensor" in need or "channel" in need:
266  if "snap" in need:
267  for snap in options.snap:
268  for raft in options.raft:
269  for sensor in options.sensor:
270  if "channel" in need:
271  for channel in options.channel:
272  if options.force or \
273  not outButler.datasetExists(
274  outDatasetType,
275  visit=visit, snap=snap,
276  raft=raft, sensor=sensor,
277  channel=channel):
278  print >>sys.stderr, \
279  ("***** Processing " + \
280  "visit %d snap %d raft %s " + \
281  "sensor %s channel %s") % \
282  (visit, snap, raft, sensor,
283  channel)
284  processFunction(inButler=inButler,
285  outButler=outButler,
286  visit=visit, snap=snap,
287  raft=raft, sensor=sensor,
288  channel=channel)
289  else:
290  if options.force or \
291  not outButler.datasetExists(
292  outDatasetType,
293  visit=visit, snap=snap,
294  raft=raft, sensor=sensor):
295  print >>sys.stderr, \
296  ("***** Processing visit %d " + \
297  "snap %d raft %s sensor %s") % \
298  (visit, snap, raft, sensor)
299  processFunction(inButler=inButler,
300  outButler=outButler, visit=visit,
301  snap=snap, raft=raft, sensor=sensor)
302  else: # snap
303  for raft in options.raft:
304  for sensor in options.sensor:
305  if "channel" in need:
306  for channel in options.channel:
307  if options.force or \
308  not outButler.datasetExists(
309  outDatasetType, visit=visit,
310  raft=raft, sensor=sensor,
311  channel=channel):
312  print >>sys.stderr, \
313  ("***** Processing visit %d " + \
314  "raft %s sensor %s channel %s") % \
315  (visit, raft, sensor, channel)
316  processFunction(inButler=inButler,
317  outButler=outButler,
318  visit=visit, raft=raft,
319  sensor=sensor, channel=channel)
320  else:
321  if options.force or \
322  not outButler.datasetExists(outDatasetType,
323  visit=visit, raft=raft,
324  sensor=sensor):
325  print >>sys.stderr, \
326  ("***** Processing visit %d " + \
327  "raft %s sensor %s") % \
328  (visit, raft, sensor)
329  processFunction(inButler=inButler,
330  outButler=outButler, visit=visit,
331  raft=raft, sensor=sensor)
332  else: # raft, sensor
333  if options.force or \
334  not outButler.datasetExists(outDatasetType, visit=visit):
335  print >>sys.stderr, "***** Processing visit %d" % (visit,)
336  processFunction(inButler=inButler, outButler=outButler,
337  visit=visit)
def lsst.datarel.utils.lsstSimSetup (   root,
  outRoot,
  registry,
  calibRoot,
  inButler,
  outButler 
)

Definition at line 405 of file utils.py.

405 
406 def lsstSimSetup(root, outRoot, registry, calibRoot, inButler, outButler):
407  if inButler is None:
408  if calibRoot is None:
409  if os.path.exists("/lsst/DC3/data/obstest/ImSim"):
410  calibRoot = "/lsst/DC3/data/obstest/ImSim"
411  if registry is None and root is not None:
412  if os.path.exists(os.path.join(root, "registry.sqlite3")):
413  registry = os.path.join(root, "registry.sqlite3")
414  bf = dafPersist.ButlerFactory(mapper=LsstSimMapper(
415  root=root, calibRoot=calibRoot, registry=registry))
416  inButler = bf.create()
417  if outButler is None:
418  if outRoot is None:
419  outRoot = root
420  obf = dafPersist.ButlerFactory(mapper=LsstSimMapper(
421  root=outRoot, registry=registry))
422  outButler = obf.create()
423  return (inButler, outButler)
def lsst.datarel.utils.runStage (   stage,
  policyString,
  clip 
)

Definition at line 44 of file utils.py.

44 
45 def runStage(stage, policyString, clip):
46  if policyString.startswith("#<?cfg "):
47  policyString = pexPolicy.PolicyString(policyString)
48  pol = pexPolicy.Policy.createPolicy(policyString)
49  sst = SimpleStageTester(stage(pol))
50  return sst.runWorker(clip)
lsst.pex::policy::PolicyString
a representation of a string containing Policy parameter data
Definition: PolicyString.h:52

Variable Documentation

lsst.datarel.utils.haveCfht = True

Definition at line 32 of file utils.py.

lsst.datarel.utils.haveLsstSim = True

Definition at line 33 of file utils.py.

Generated on Tue Apr 21 2015 10:05:07 for LSSTApplications by   doxygen 1.8.5