LSSTApplications  16.0-10-g9d3e444,16.0-11-g09ed895+3,16.0-11-g12e47bd+4,16.0-11-g9bb73b2+10,16.0-12-g5c924a4+10,16.0-15-g7af1f30,16.0-15-gdd5ca33+2,16.0-16-gf0259e2+1,16.0-17-g31abd91+11,16.0-17-g5cf0468+3,16.0-18-g51a54b3+3,16.0-18-ga4d4bcb+5,16.0-18-gcf94535+2,16.0-19-g9d290d5+2,16.0-2-g0febb12+22,16.0-2-g9d5294e+73,16.0-2-ga8830df+7,16.0-21-g3d035912+2,16.0-26-g8e79609,16.0-28-gfc9ea6c+9,16.0-29-ge8801f9+4,16.0-3-ge00e371+38,16.0-4-g18f3627+17,16.0-4-g5f3a788+21,16.0-4-ga3eb747+11,16.0-4-gabf74b7+33,16.0-4-gb13d127+7,16.0-5-g27fb78a+11,16.0-5-g6a53317+38,16.0-5-gb3f8a4b+91,16.0-51-gbbe9c988+3,16.0-6-g9321be7+5,16.0-6-gcbc7b31+47,16.0-6-gf49912c+33,16.0-7-gd2eeba5+56,16.0-75-gbf7a9a820,16.0-8-g21fd5fe+34,16.0-8-g3a9f023+24,16.0-9-gf3bc169+2,16.0-9-gf5c1f43+12,master-gd73dc1d098+5,w.2019.02
LSSTDataManagementBasePackage
Public Member Functions | Static Public Attributes | List of all members
lsst.pipe.base.argumentParser.ArgumentParser Class Reference
Inheritance diagram for lsst.pipe.base.argumentParser.ArgumentParser:
lsst.pipe.base.argumentParser.InputOnlyArgumentParser lsst.pipe.drivers.constructCalibs.CalibArgumentParser lsst.pipe.tasks.ingest.IngestArgumentParser lsst.pipe.tasks.ingestCalibs.IngestCalibsArgumentParser lsst.obs.decam.ingest.DecamIngestArgumentParser

Public Member Functions

def __init__ (self, name, usage="%(prog)s input [options]", kwargs)
 
def add_id_argument (self, name, datasetType, help, level=None, doMakeDataRefList=True, ContainerClass=DataIdContainer)
 
def parse_args (self, config, args=None, log=None, override=None)
 
def handleCamera (self, namespace)
 
def convert_arg_line_to_args (self, arg_line)
 
def addReuseOption (self, choices)
 

Static Public Attributes

bool requireOutput = True
 

Detailed Description

Argument parser for command-line tasks that is based on
`argparse.ArgumentParser`.

Parameters
----------
name : `str`
    Name of top-level task; used to identify camera-specific override
    files.
usage : `str`, optional
    Command-line usage signature.
**kwargs
    Additional keyword arguments for `argparse.ArgumentParser`.

Notes
-----
Users may wish to add additional arguments before calling `parse_args`.

Definition at line 406 of file argumentParser.py.

Constructor & Destructor Documentation

◆ __init__()

def lsst.pipe.base.argumentParser.ArgumentParser.__init__ (   self,
  name,
  usage = "%(prog)s input [options]",
  kwargs 
)

Definition at line 433 of file argumentParser.py.

433  def __init__(self, name, usage="%(prog)s input [options]", **kwargs):
434  self._name = name
435  self._dataIdArgDict = {} # Dict of data identifier specifications, by argument name
436  argparse.ArgumentParser.__init__(self,
437  usage=usage,
438  fromfile_prefix_chars='@',
439  epilog=textwrap.dedent("""Notes:
440  * --config, --configfile, --id, --loglevel and @file may appear multiple times;
441  all values are used, in order left to right
442  * @file reads command-line options from the specified file:
443  * data may be distributed among multiple lines (e.g. one option per line)
444  * data after # is treated as a comment and ignored
445  * blank lines and lines starting with # are ignored
446  * To specify multiple values for an option, do not use = after the option name:
447  * right: --configfile foo bar
448  * wrong: --configfile=foo bar
449  """),
450  formatter_class=argparse.RawDescriptionHelpFormatter,
451  **kwargs)
452  self.add_argument(metavar='input', dest="rawInput",
453  help="path to input data repository, relative to $%s" % (DEFAULT_INPUT_NAME,))
454  self.add_argument("--calib", dest="rawCalib",
455  help="path to input calibration repository, relative to $%s" %
456  (DEFAULT_CALIB_NAME,))
457  self.add_argument("--output", dest="rawOutput",
458  help="path to output data repository (need not exist), relative to $%s" %
459  (DEFAULT_OUTPUT_NAME,))
460  self.add_argument("--rerun", dest="rawRerun", metavar="[INPUT:]OUTPUT",
461  help="rerun name: sets OUTPUT to ROOT/rerun/OUTPUT; "
462  "optionally sets ROOT to ROOT/rerun/INPUT")
463  self.add_argument("-c", "--config", nargs="*", action=ConfigValueAction,
464  help="config override(s), e.g. -c foo=newfoo bar.baz=3", metavar="NAME=VALUE")
465  self.add_argument("-C", "--configfile", dest="configfile", nargs="*", action=ConfigFileAction,
466  help="config override file(s)")
467  self.add_argument("-L", "--loglevel", nargs="*", action=LogLevelAction,
468  help="logging level; supported levels are [trace|debug|info|warn|error|fatal]",
469  metavar="LEVEL|COMPONENT=LEVEL")
470  self.add_argument("--longlog", action="store_true", help="use a more verbose format for the logging")
471  self.add_argument("--debug", action="store_true", help="enable debugging output?")
472  self.add_argument("--doraise", action="store_true",
473  help="raise an exception on error (else log a message and continue)?")
474  self.add_argument("--noExit", action="store_true",
475  help="Do not exit even upon failure (i.e. return a struct to the calling script)")
476  self.add_argument("--profile", help="Dump cProfile statistics to filename")
477  self.add_argument("--show", nargs="+", default=(),
478  help="display the specified information to stdout and quit "
479  "(unless run is specified).")
480  self.add_argument("-j", "--processes", type=int, default=1, help="Number of processes to use")
481  self.add_argument("-t", "--timeout", type=float,
482  help="Timeout for multiprocessing; maximum wall time (sec)")
483  self.add_argument("--clobber-output", action="store_true", dest="clobberOutput", default=False,
484  help=("remove and re-create the output directory if it already exists "
485  "(safe with -j, but not all other forms of parallel execution)"))
486  self.add_argument("--clobber-config", action="store_true", dest="clobberConfig", default=False,
487  help=("backup and then overwrite existing config files instead of checking them "
488  "(safe with -j, but not all other forms of parallel execution)"))
489  self.add_argument("--no-backup-config", action="store_true", dest="noBackupConfig", default=False,
490  help="Don't copy config to file~N backup.")
491  self.add_argument("--clobber-versions", action="store_true", dest="clobberVersions", default=False,
492  help=("backup and then overwrite existing package versions instead of checking"
493  "them (safe with -j, but not all other forms of parallel execution)"))
494  self.add_argument("--no-versions", action="store_true", dest="noVersions", default=False,
495  help="don't check package versions; useful for development")
496  lsstLog.configure_prop("""
497 log4j.rootLogger=INFO, A1
498 log4j.appender.A1=ConsoleAppender
499 log4j.appender.A1.Target=System.out
500 log4j.appender.A1.layout=PatternLayout
501 log4j.appender.A1.layout.ConversionPattern=%c %p: %m%n
502 """)
503 
def __init__(self, minimum, dataRange, Q)

Member Function Documentation

◆ add_id_argument()

def lsst.pipe.base.argumentParser.ArgumentParser.add_id_argument (   self,
  name,
  datasetType,
  help,
  level = None,
  doMakeDataRefList = True,
  ContainerClass = DataIdContainer 
)
Add a data ID argument.


Parameters
----------
name : `str`
    Data ID argument (including leading dashes, if wanted).
datasetType : `str` or `DynamicDatasetType`-type
    Type of dataset. Supply a string for a fixed dataset type.
    For a dynamically determined dataset type, supply
    a `DynamicDatasetType`, such a `DatasetArgument`.
help : `str`
    Help string for the argument.
level : `str`
    The lowest hierarchy level to descend to for this dataset type,
    for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
    Use `""` to use the mapper's default for the dataset type.
    Some container classes may also support `None`, which means
    the level should not be restricted; however the default class,
    `DataIdContainer`, does not support `None`.
doMakeDataRefList : bool, optional
    If `True` (default), construct data references.
ContainerClass : `class`, optional
Class to contain data IDs and data references; the default class
`DataIdContainer` will work for many, but not all, cases.
For example if the dataset type is specified on the command line
then use `DynamicDatasetType`.

Notes
-----
If ``datasetType`` is an instance of `DatasetArgument`,
then add a second argument to specify the dataset type.

The associated data is put into ``namespace.<dataIdArgument.name>``
as an instance of `ContainerClass`; the container includes fields:

- ``idList``: a list of data ID dicts.
- ``refList``: a list of `~lsst.daf.persistence.Butler`
    data references (empty if ``doMakeDataRefList`` is  `False`).

Definition at line 505 of file argumentParser.py.

505  ContainerClass=DataIdContainer):
506  """Add a data ID argument.
507 
508 
509  Parameters
510  ----------
511  name : `str`
512  Data ID argument (including leading dashes, if wanted).
513  datasetType : `str` or `DynamicDatasetType`-type
514  Type of dataset. Supply a string for a fixed dataset type.
515  For a dynamically determined dataset type, supply
516  a `DynamicDatasetType`, such a `DatasetArgument`.
517  help : `str`
518  Help string for the argument.
519  level : `str`
520  The lowest hierarchy level to descend to for this dataset type,
521  for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
522  Use `""` to use the mapper's default for the dataset type.
523  Some container classes may also support `None`, which means
524  the level should not be restricted; however the default class,
525  `DataIdContainer`, does not support `None`.
526  doMakeDataRefList : bool, optional
527  If `True` (default), construct data references.
528  ContainerClass : `class`, optional
529  Class to contain data IDs and data references; the default class
530  `DataIdContainer` will work for many, but not all, cases.
531  For example if the dataset type is specified on the command line
532  then use `DynamicDatasetType`.
533 
534  Notes
535  -----
536  If ``datasetType`` is an instance of `DatasetArgument`,
537  then add a second argument to specify the dataset type.
538 
539  The associated data is put into ``namespace.<dataIdArgument.name>``
540  as an instance of `ContainerClass`; the container includes fields:
541 
542  - ``idList``: a list of data ID dicts.
543  - ``refList``: a list of `~lsst.daf.persistence.Butler`
544  data references (empty if ``doMakeDataRefList`` is `False`).
545  """
546  argName = name.lstrip("-")
547 
548  if argName in self._dataIdArgDict:
549  raise RuntimeError("Data ID argument %s already exists" % (name,))
550  if argName in set(("camera", "config", "butler", "log", "obsPkg")):
551  raise RuntimeError("Data ID argument %s is a reserved name" % (name,))
552 
553  self.add_argument(name, nargs="*", action=IdValueAction, help=help,
554  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
555 
556  dataIdArgument = DataIdArgument(
557  name=argName,
558  datasetType=datasetType,
559  level=level,
560  doMakeDataRefList=doMakeDataRefList,
561  ContainerClass=ContainerClass,
562  )
563 
564  if dataIdArgument.isDynamicDatasetType:
565  datasetType.addArgument(parser=self, idName=argName)
566 
567  self._dataIdArgDict[argName] = dataIdArgument
568 
daf::base::PropertySet * set
Definition: fits.cc:832

◆ addReuseOption()

def lsst.pipe.base.argumentParser.ArgumentParser.addReuseOption (   self,
  choices 
)
Add a "--reuse-outputs-from SUBTASK" option to the argument
parser.

CmdLineTasks that can be restarted at an intermediate step using
outputs from earlier (but still internal) steps should use this
method to allow the user to control whether that happens when
outputs from earlier steps are present.

Parameters
----------
choices : sequence
    A sequence of string names (by convention, top-level subtasks)
    that identify the steps that could be skipped when their
    outputs are already present.  The list is ordered, so when the
    user specifies one step on the command line, all previous steps
    may be skipped as well.  In addition to the choices provided,
    users may pass "all" to indicate that all steps may be thus
    skipped.

When this method is called, the ``namespace`` object returned by
``parse_args`` will contain a ``reuse`` attribute containing
a list of all steps that should be skipped if their outputs
are already present.
If no steps should be skipped, the ``reuse`` will be an empty list.

Definition at line 871 of file argumentParser.py.

871  def addReuseOption(self, choices):
872  """Add a "--reuse-outputs-from SUBTASK" option to the argument
873  parser.
874 
875  CmdLineTasks that can be restarted at an intermediate step using
876  outputs from earlier (but still internal) steps should use this
877  method to allow the user to control whether that happens when
878  outputs from earlier steps are present.
879 
880  Parameters
881  ----------
882  choices : sequence
883  A sequence of string names (by convention, top-level subtasks)
884  that identify the steps that could be skipped when their
885  outputs are already present. The list is ordered, so when the
886  user specifies one step on the command line, all previous steps
887  may be skipped as well. In addition to the choices provided,
888  users may pass "all" to indicate that all steps may be thus
889  skipped.
890 
891  When this method is called, the ``namespace`` object returned by
892  ``parse_args`` will contain a ``reuse`` attribute containing
893  a list of all steps that should be skipped if their outputs
894  are already present.
895  If no steps should be skipped, the ``reuse`` will be an empty list.
896  """
897  choices = list(choices)
898  choices.append("all")
899  self.add_argument("--reuse-outputs-from", dest="reuse", choices=choices,
900  default=[], action=ReuseAction,
901  help=("Skip the given subtask and its predecessors and reuse their outputs "
902  "if those outputs already exist. Use 'all' to specify all subtasks."))
903 
904 
daf::base::PropertyList * list
Definition: fits.cc:833

◆ convert_arg_line_to_args()

def lsst.pipe.base.argumentParser.ArgumentParser.convert_arg_line_to_args (   self,
  arg_line 
)
Allow files of arguments referenced by ``@<path>`` to contain
multiple values on each line.

Parameters
----------
arg_line : `str`
    Line of text read from an argument file.

Definition at line 854 of file argumentParser.py.

854  def convert_arg_line_to_args(self, arg_line):
855  """Allow files of arguments referenced by ``@<path>`` to contain
856  multiple values on each line.
857 
858  Parameters
859  ----------
860  arg_line : `str`
861  Line of text read from an argument file.
862  """
863  arg_line = arg_line.strip()
864  if not arg_line or arg_line.startswith("#"):
865  return
866  for arg in shlex.split(arg_line, comments=True, posix=True):
867  if not arg.strip():
868  continue
869  yield arg
870 

◆ handleCamera()

def lsst.pipe.base.argumentParser.ArgumentParser.handleCamera (   self,
  namespace 
)
Perform camera-specific operations before parsing the command-line.

Parameters
----------
namespace : `argparse.Namespace`
    Namespace (an ) with the following fields:

    - ``camera``: the camera name.
    - ``config``: the config passed to parse_args, with no overrides applied.
    - ``obsPkg``: the ``obs_`` package for this camera.
    - ``log``: a `lsst.log` Log.

Notes
-----
The default implementation does nothing.

Definition at line 835 of file argumentParser.py.

835  def handleCamera(self, namespace):
836  """Perform camera-specific operations before parsing the command-line.
837 
838  Parameters
839  ----------
840  namespace : `argparse.Namespace`
841  Namespace (an ) with the following fields:
842 
843  - ``camera``: the camera name.
844  - ``config``: the config passed to parse_args, with no overrides applied.
845  - ``obsPkg``: the ``obs_`` package for this camera.
846  - ``log``: a `lsst.log` Log.
847 
848  Notes
849  -----
850  The default implementation does nothing.
851  """
852  pass
853 

◆ parse_args()

def lsst.pipe.base.argumentParser.ArgumentParser.parse_args (   self,
  config,
  args = None,
  log = None,
  override = None 
)
Parse arguments for a command-line task.

Parameters
----------
config : `lsst.pex.config.Config`
    Config for the task being run.
args : `list`, optional
    Argument list; if `None` then ``sys.argv[1:]`` is used.
log : `lsst.log.Log`, optional
    `~lsst.log.Log` instance; if `None` use the default log.
override : callable, optional
    A config override function. It must take the root config object
    as its only argument and must modify the config in place.
    This function is called after camera-specific overrides files
    are applied, and before command-line config overrides
    are applied (thus allowing the user the final word).

Returns
-------
namespace : `argparse.Namespace`
    A `~argparse.Namespace` instance containing fields:

    - ``camera``: camera name.
    - ``config``: the supplied config with all overrides applied,
validated and frozen.
    - ``butler``: a `lsst.daf.persistence.Butler` for the data.
    - An entry for each of the data ID arguments registered by
`add_id_argument`, of the type passed to its ``ContainerClass``
keyword (`~lsst.pipe.base.DataIdContainer` by default). It
includes public elements ``idList`` and ``refList``.
    - ``log``: a `lsst.log` Log.
    - An entry for each command-line argument,
with the following exceptions:

      - config is the supplied config, suitably updated.
      - configfile, id and loglevel are all missing.
    - ``obsPkg``: name of the ``obs_`` package for this camera.

Definition at line 569 of file argumentParser.py.

569  def parse_args(self, config, args=None, log=None, override=None):
570  """Parse arguments for a command-line task.
571 
572  Parameters
573  ----------
574  config : `lsst.pex.config.Config`
575  Config for the task being run.
576  args : `list`, optional
577  Argument list; if `None` then ``sys.argv[1:]`` is used.
578  log : `lsst.log.Log`, optional
579  `~lsst.log.Log` instance; if `None` use the default log.
580  override : callable, optional
581  A config override function. It must take the root config object
582  as its only argument and must modify the config in place.
583  This function is called after camera-specific overrides files
584  are applied, and before command-line config overrides
585  are applied (thus allowing the user the final word).
586 
587  Returns
588  -------
589  namespace : `argparse.Namespace`
590  A `~argparse.Namespace` instance containing fields:
591 
592  - ``camera``: camera name.
593  - ``config``: the supplied config with all overrides applied,
594  validated and frozen.
595  - ``butler``: a `lsst.daf.persistence.Butler` for the data.
596  - An entry for each of the data ID arguments registered by
597  `add_id_argument`, of the type passed to its ``ContainerClass``
598  keyword (`~lsst.pipe.base.DataIdContainer` by default). It
599  includes public elements ``idList`` and ``refList``.
600  - ``log``: a `lsst.log` Log.
601  - An entry for each command-line argument,
602  with the following exceptions:
603 
604  - config is the supplied config, suitably updated.
605  - configfile, id and loglevel are all missing.
606  - ``obsPkg``: name of the ``obs_`` package for this camera.
607  """
608  if args is None:
609  args = sys.argv[1:]
610 
611  if len(args) < 1 or args[0].startswith("-") or args[0].startswith("@"):
612  self.print_help()
613  if len(args) == 1 and args[0] in ("-h", "--help"):
614  self.exit()
615  else:
616  self.exit("%s: error: Must specify input as first argument" % self.prog)
617 
618  # Note that --rerun may change namespace.input, but if it does
619  # we verify that the new input has the same mapper class.
620  namespace = argparse.Namespace()
621  namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0])
622  if not os.path.isdir(namespace.input):
623  self.error("Error: input=%r not found" % (namespace.input,))
624 
625  namespace.config = config
626  namespace.log = log if log is not None else lsstLog.Log.getDefaultLogger()
627  mapperClass = dafPersist.Butler.getMapperClass(namespace.input)
628  namespace.camera = mapperClass.getCameraName()
629  namespace.obsPkg = mapperClass.getPackageName()
630 
631  self.handleCamera(namespace)
632 
633  self._applyInitialOverrides(namespace)
634  if override is not None:
635  override(namespace.config)
636 
637  # Add data ID containers to namespace
638  for dataIdArgument in self._dataIdArgDict.values():
639  setattr(namespace, dataIdArgument.name, dataIdArgument.ContainerClass(level=dataIdArgument.level))
640 
641  namespace = argparse.ArgumentParser.parse_args(self, args=args, namespace=namespace)
642  del namespace.configfile
643 
644  self._parseDirectories(namespace)
645 
646  if namespace.clobberOutput:
647  if namespace.output is None:
648  self.error("--clobber-output is only valid with --output or --rerun")
649  elif namespace.output == namespace.input:
650  self.error("--clobber-output is not valid when the output and input repos are the same")
651  if os.path.exists(namespace.output):
652  namespace.log.info("Removing output repo %s for --clobber-output", namespace.output)
653  shutil.rmtree(namespace.output)
654 
655  namespace.log.debug("input=%s", namespace.input)
656  namespace.log.debug("calib=%s", namespace.calib)
657  namespace.log.debug("output=%s", namespace.output)
658 
659  obeyShowArgument(namespace.show, namespace.config, exit=False)
660 
661  # No environment variable or --output or --rerun specified.
662  if self.requireOutput and namespace.output is None and namespace.rerun is None:
663  self.error("no output directory specified.\n"
664  "An output directory must be specified with the --output or --rerun\n"
665  "command-line arguments.\n")
666 
667  butlerArgs = {} # common arguments for butler elements
668  if namespace.calib:
669  butlerArgs = {'mapperArgs': {'calibRoot': namespace.calib}}
670  if namespace.output:
671  outputs = {'root': namespace.output, 'mode': 'rw'}
672  inputs = {'root': namespace.input}
673  inputs.update(butlerArgs)
674  outputs.update(butlerArgs)
675  namespace.butler = dafPersist.Butler(inputs=inputs, outputs=outputs)
676  else:
677  outputs = {'root': namespace.input, 'mode': 'rw'}
678  outputs.update(butlerArgs)
679  namespace.butler = dafPersist.Butler(outputs=outputs)
680 
681  # convert data in each of the identifier lists to proper types
682  # this is done after constructing the butler,
683  # hence after parsing the command line,
684  # because it takes a long time to construct a butler
685  self._processDataIds(namespace)
686  if "data" in namespace.show:
687  for dataIdName in self._dataIdArgDict.keys():
688  for dataRef in getattr(namespace, dataIdName).refList:
689  print("%s dataRef.dataId = %s" % (dataIdName, dataRef.dataId))
690 
691  if namespace.show and "run" not in namespace.show:
692  sys.exit(0)
693 
694  if namespace.debug:
695  try:
696  import debug
697  assert debug # silence pyflakes
698  except ImportError:
699  sys.stderr.write("Warning: no 'debug' module found\n")
700  namespace.debug = False
701 
702  del namespace.loglevel
703 
704  if namespace.longlog:
705  lsstLog.configure_prop("""
706 log4j.rootLogger=INFO, A1
707 log4j.appender.A1=ConsoleAppender
708 log4j.appender.A1.Target=System.out
709 log4j.appender.A1.layout=PatternLayout
710 log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddThh:mm:ss.sss} %c (%X{LABEL})(%F:%L)- %m%n
711 """)
712  del namespace.longlog
713 
714  namespace.config.validate()
715  namespace.config.freeze()
716 
717  return namespace
718 
def obeyShowArgument(showOpts, config=None, exit=False)

Member Data Documentation

◆ requireOutput

bool lsst.pipe.base.argumentParser.ArgumentParser.requireOutput = True
static

Definition at line 430 of file argumentParser.py.


The documentation for this class was generated from the following file: