LSSTApplications  10.0-2-g4f67435,11.0.rc2+1,11.0.rc2+12,11.0.rc2+3,11.0.rc2+4,11.0.rc2+5,11.0.rc2+6,11.0.rc2+7,11.0.rc2+8
LSSTDataManagementBasePackage
Functions | Variables
lsst.datarel.ingest Namespace Reference

Functions

def _line_to_args
 
def makeArgumentParser
 
def makeRules
 

Variables

list __all__
 

Function Documentation

def lsst.datarel.ingest._line_to_args (   self,
  line 
)
private

Definition at line 37 of file ingest.py.

37 
38 def _line_to_args(self, line):
39  for arg in shlex.split(line, comments=True, posix=True):
40  if not arg.strip():
41  continue
42  yield arg
43 
def lsst.datarel.ingest.makeArgumentParser (   description,
  inRootsRequired = True,
  addRegistryOption = True 
)

Definition at line 44 of file ingest.py.

44 
45 def makeArgumentParser(description, inRootsRequired=True, addRegistryOption=True):
46  parser = argparse.ArgumentParser(
47  description=description,
48  fromfile_prefix_chars="@",
49  formatter_class=argparse.RawDescriptionHelpFormatter,
50  epilog="Data IDs are expected to be of the form:\n"
51  " --id k1=v11[^v12[...]] [k2=v21[^v22[...]]\n"
52  "\n"
53  "Examples:\n"
54  " 1. --id visit=12345 raft=1,1\n"
55  " 2. --id skyTile=12345\n"
56  " 3. --id visit=12 raft=1,2^2,2 sensor=1,1^1,3\n"
57  "\n"
58  "The first example identifies all sensors in raft 1,1 of visit "
59  "12345. The second identifies all sources/objects in sky-tile "
60  "12345. The cross product is computed for keys with multiple "
61  "values, so the third example is equivalent to:\n"
62  " --id visit=12 raft=1,2 sensor=1,1\n"
63  " --id visit=12 raft=1,2 sensor=1,3\n"
64  " --id visit=12 raft=2,2 sensor=1,1\n"
65  " --id visit=12 raft=2,2 sensor=1,3\n"
66  "\n"
67  "Redundant specification of a data ID will *not* result in "
68  "loads of duplicate data - data IDs are de-duped before ingest "
69  "starts. Note also that the keys allowed in data IDs are "
70  "specific to the type of data the ingestion script deals with. "
71  "For example, one cannot load sensor metadata by sky tile ID, nor "
72  "sources by sensor (CCD) ID.\n"
73  "\n"
74  "Any omitted keys are assumed to take all legal values. So, "
75  "`--id raft=1,2` identifies all sensors of raft 1,2 for all "
76  "available visits.\n"
77  "\n"
78  "Finally, calling an ingestion script multiple times is safe. "
79  "Attempting to load the same data item twice will result "
80  "in an error or (if --strict is specified) or cause previously "
81  "loaded data item to be skipped. Database activity is strictly "
82  "append only.")
83  parser.convert_arg_line_to_args = _line_to_args
84  addDbOptions(parser)
85  parser.add_argument(
86  "-d", "--database", dest="database",
87  help="MySQL database to load CSV files into.")
88  parser.add_argument(
89  "-s", "--strict", dest="strict", action="store_true",
90  help="Error out if previously ingested, incomplete, or otherwise "
91  "suspicious data items are encountered (by default, these are "
92  "skipped). Note that if --database is not supplied, detecting "
93  "previously ingested data is not possible.")
94  parser.add_argument(
95  "-i", "--id", dest="id", nargs="+", default=None, action="append",
96  help="Data ID specifying what to ingest. May appear multiple times.")
97  if addRegistryOption:
98  parser.add_argument(
99  "-R", "--registry", dest="registry", help="Input registry path; "
100  "used for all input roots. If omitted, a file named registry.sqlite3 "
101  "must exist in each input root.")
102  parser.add_argument(
103  "outroot", help="Output directory for CSV files")
104  parser.add_argument(
105  "inroot", nargs="+" if inRootsRequired else "*",
106  help="One or more input root directories")
107  return parser
108 
lsst.datarel.ingest.makeArgumentParser
def makeArgumentParser
Definition: ingest.py:44
def lsst.datarel.ingest.makeRules (   dataIdSpecs,
  camera,
  validKeys 
) 
Return a list of data ID rules from command line --id specifications.
Ensures only the given keys are referenced by the data id specs.

Definition at line 109 of file ingest.py.

109 
110 def makeRules(dataIdSpecs, camera, validKeys):
111  """Return a list of data ID rules from command line --id specifications.
112  Ensures only the given keys are referenced by the data id specs."""
113  if not dataIdSpecs:
114  return None
115  rules = []
116  for id in dataIdSpecs:
117  r = parseDataIdRules(id, camera)
118  for k in r:
119  if k not in validKeys:
120  raise RuntimeError(k + " is not a legal data ID key for this ingest script")
121  rules.append(r)
122  return rules
123 

Variable Documentation

list lsst.datarel.ingest.__all__
Initial value:
1 = ["makeArgumentParser",
2  "makeRules",
3  ]

Definition at line 33 of file ingest.py.

Generated on Wed Sep 16 2015 13:35:47 for LSSTApplications by   doxygen 1.8.5