LSSTApplications  17.0+11,17.0+35,17.0+60,17.0+61,17.0+63,17.0+7,17.0-1-g377950a+35,17.0.1-1-g114240f+2,17.0.1-1-g4d4fbc4+30,17.0.1-1-g55520dc+55,17.0.1-1-g5f4ed7e+59,17.0.1-1-g6dd7d69+22,17.0.1-1-g8de6c91+11,17.0.1-1-gb9095d2+7,17.0.1-1-ge9fec5e+5,17.0.1-1-gf4e0155+63,17.0.1-1-gfc65f5f+56,17.0.1-1-gfc6fb1f+20,17.0.1-10-g87f9f3f+9,17.0.1-12-g112a4bc+3,17.0.1-17-gab9750a3+5,17.0.1-17-gdae4c4a+16,17.0.1-19-g3a24bb2+2,17.0.1-2-g26618f5+35,17.0.1-2-g54f2ebc+9,17.0.1-2-gf403422+1,17.0.1-21-g52a398f+5,17.0.1-26-gd98a1d13,17.0.1-3-g7e86b59+45,17.0.1-3-gb5ca14a,17.0.1-3-gd08d533+46,17.0.1-31-gb0791f330,17.0.1-4-g59d126d+10,17.0.1-5-g3877d06+2,17.0.1-7-g35889ee+7,17.0.1-7-gc7c8782+20,17.0.1-7-gcb7da53+5,17.0.1-9-gc4bbfb2+10,w.2019.24
LSSTDataManagementBasePackage
Public Member Functions | Static Public Attributes | List of all members
lsst.pipe.base.argumentParser.ArgumentParser Class Reference
Inheritance diagram for lsst.pipe.base.argumentParser.ArgumentParser:
lsst.pipe.base.argumentParser.InputOnlyArgumentParser lsst.pipe.drivers.constructCalibs.CalibArgumentParser lsst.pipe.tasks.ingest.IngestArgumentParser lsst.pipe.tasks.ingestCalibs.IngestCalibsArgumentParser lsst.obs.decam.ingest.DecamIngestArgumentParser

Public Member Functions

def __init__ (self, name, usage="%(prog)s input [options]", kwargs)
 
def add_id_argument (self, name, datasetType, help, level=None, doMakeDataRefList=True, ContainerClass=DataIdContainer)
 
def parse_args (self, config, args=None, log=None, override=None)
 
def handleCamera (self, namespace)
 
def convert_arg_line_to_args (self, arg_line)
 
def addReuseOption (self, choices)
 

Static Public Attributes

bool requireOutput = True
 

Detailed Description

Argument parser for command-line tasks that is based on
`argparse.ArgumentParser`.

Parameters
----------
name : `str`
    Name of top-level task; used to identify camera-specific override
    files.
usage : `str`, optional
    Command-line usage signature.
**kwargs
    Additional keyword arguments for `argparse.ArgumentParser`.

Notes
-----
Users may wish to add additional arguments before calling `parse_args`.

Definition at line 407 of file argumentParser.py.

Constructor & Destructor Documentation

◆ __init__()

def lsst.pipe.base.argumentParser.ArgumentParser.__init__ (   self,
  name,
  usage = "%(prog)s input [options]",
  kwargs 
)

Definition at line 434 of file argumentParser.py.

434  def __init__(self, name, usage="%(prog)s input [options]", **kwargs):
435  self._name = name
436  self._dataIdArgDict = {} # Dict of data identifier specifications, by argument name
437  argparse.ArgumentParser.__init__(self,
438  usage=usage,
439  fromfile_prefix_chars='@',
440  epilog=textwrap.dedent("""Notes:
441  * --config, --configfile, --id, --loglevel and @file may appear multiple times;
442  all values are used, in order left to right
443  * @file reads command-line options from the specified file:
444  * data may be distributed among multiple lines (e.g. one option per line)
445  * data after # is treated as a comment and ignored
446  * blank lines and lines starting with # are ignored
447  * To specify multiple values for an option, do not use = after the option name:
448  * right: --configfile foo bar
449  * wrong: --configfile=foo bar
450  """),
451  formatter_class=argparse.RawDescriptionHelpFormatter,
452  **kwargs)
453  self.add_argument(metavar='input', dest="rawInput",
454  help="path to input data repository, relative to $%s" % (DEFAULT_INPUT_NAME,))
455  self.add_argument("--calib", dest="rawCalib",
456  help="path to input calibration repository, relative to $%s" %
457  (DEFAULT_CALIB_NAME,))
458  self.add_argument("--output", dest="rawOutput",
459  help="path to output data repository (need not exist), relative to $%s" %
460  (DEFAULT_OUTPUT_NAME,))
461  self.add_argument("--rerun", dest="rawRerun", metavar="[INPUT:]OUTPUT",
462  help="rerun name: sets OUTPUT to ROOT/rerun/OUTPUT; "
463  "optionally sets ROOT to ROOT/rerun/INPUT")
464  self.add_argument("-c", "--config", nargs="*", action=ConfigValueAction,
465  help="config override(s), e.g. -c foo=newfoo bar.baz=3", metavar="NAME=VALUE")
466  self.add_argument("-C", "--configfile", dest="configfile", nargs="*", action=ConfigFileAction,
467  help="config override file(s)")
468  self.add_argument("-L", "--loglevel", nargs="*", action=LogLevelAction,
469  help="logging level; supported levels are [trace|debug|info|warn|error|fatal]",
470  metavar="LEVEL|COMPONENT=LEVEL")
471  self.add_argument("--longlog", action="store_true", help="use a more verbose format for the logging")
472  self.add_argument("--debug", action="store_true", help="enable debugging output?")
473  self.add_argument("--doraise", action="store_true",
474  help="raise an exception on error (else log a message and continue)?")
475  self.add_argument("--noExit", action="store_true",
476  help="Do not exit even upon failure (i.e. return a struct to the calling script)")
477  self.add_argument("--profile", help="Dump cProfile statistics to filename")
478  self.add_argument("--show", nargs="+", default=(),
479  help="display the specified information to stdout and quit "
480  "(unless run is specified).")
481  self.add_argument("-j", "--processes", type=int, default=1, help="Number of processes to use")
482  self.add_argument("-t", "--timeout", type=float,
483  help="Timeout for multiprocessing; maximum wall time (sec)")
484  self.add_argument("--clobber-output", action="store_true", dest="clobberOutput", default=False,
485  help=("remove and re-create the output directory if it already exists "
486  "(safe with -j, but not all other forms of parallel execution)"))
487  self.add_argument("--clobber-config", action="store_true", dest="clobberConfig", default=False,
488  help=("backup and then overwrite existing config files instead of checking them "
489  "(safe with -j, but not all other forms of parallel execution)"))
490  self.add_argument("--no-backup-config", action="store_true", dest="noBackupConfig", default=False,
491  help="Don't copy config to file~N backup.")
492  self.add_argument("--clobber-versions", action="store_true", dest="clobberVersions", default=False,
493  help=("backup and then overwrite existing package versions instead of checking"
494  "them (safe with -j, but not all other forms of parallel execution)"))
495  self.add_argument("--no-versions", action="store_true", dest="noVersions", default=False,
496  help="don't check package versions; useful for development")
497  lsstLog.configure_prop("""
498 log4j.rootLogger=INFO, A1
499 log4j.appender.A1=ConsoleAppender
500 log4j.appender.A1.Target=System.out
501 log4j.appender.A1.layout=PatternLayout
502 log4j.appender.A1.layout.ConversionPattern=%c %p: %m%n
503 """)
504 
505  # Forward all Python logging to lsst.log
506  lgr = logging.getLogger()
507  lgr.setLevel(logging.INFO) # same as in log4cxx config above
508  lgr.addHandler(lsstLog.LogHandler())
509 
def __init__(self, minimum, dataRange, Q)

Member Function Documentation

◆ add_id_argument()

def lsst.pipe.base.argumentParser.ArgumentParser.add_id_argument (   self,
  name,
  datasetType,
  help,
  level = None,
  doMakeDataRefList = True,
  ContainerClass = DataIdContainer 
)
Add a data ID argument.


Parameters
----------
name : `str`
    Data ID argument (including leading dashes, if wanted).
datasetType : `str` or `DynamicDatasetType`-type
    Type of dataset. Supply a string for a fixed dataset type.
    For a dynamically determined dataset type, supply
    a `DynamicDatasetType`, such a `DatasetArgument`.
help : `str`
    Help string for the argument.
level : `str`
    The lowest hierarchy level to descend to for this dataset type,
    for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
    Use `""` to use the mapper's default for the dataset type.
    Some container classes may also support `None`, which means
    the level should not be restricted; however the default class,
    `DataIdContainer`, does not support `None`.
doMakeDataRefList : bool, optional
    If `True` (default), construct data references.
ContainerClass : `class`, optional
Class to contain data IDs and data references; the default class
`DataIdContainer` will work for many, but not all, cases.
For example if the dataset type is specified on the command line
then use `DynamicDatasetType`.

Notes
-----
If ``datasetType`` is an instance of `DatasetArgument`,
then add a second argument to specify the dataset type.

The associated data is put into ``namespace.<dataIdArgument.name>``
as an instance of `ContainerClass`; the container includes fields:

- ``idList``: a list of data ID dicts.
- ``refList``: a list of `~lsst.daf.persistence.Butler`
    data references (empty if ``doMakeDataRefList`` is  `False`).

Definition at line 511 of file argumentParser.py.

511  ContainerClass=DataIdContainer):
512  """Add a data ID argument.
513 
514 
515  Parameters
516  ----------
517  name : `str`
518  Data ID argument (including leading dashes, if wanted).
519  datasetType : `str` or `DynamicDatasetType`-type
520  Type of dataset. Supply a string for a fixed dataset type.
521  For a dynamically determined dataset type, supply
522  a `DynamicDatasetType`, such a `DatasetArgument`.
523  help : `str`
524  Help string for the argument.
525  level : `str`
526  The lowest hierarchy level to descend to for this dataset type,
527  for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
528  Use `""` to use the mapper's default for the dataset type.
529  Some container classes may also support `None`, which means
530  the level should not be restricted; however the default class,
531  `DataIdContainer`, does not support `None`.
532  doMakeDataRefList : bool, optional
533  If `True` (default), construct data references.
534  ContainerClass : `class`, optional
535  Class to contain data IDs and data references; the default class
536  `DataIdContainer` will work for many, but not all, cases.
537  For example if the dataset type is specified on the command line
538  then use `DynamicDatasetType`.
539 
540  Notes
541  -----
542  If ``datasetType`` is an instance of `DatasetArgument`,
543  then add a second argument to specify the dataset type.
544 
545  The associated data is put into ``namespace.<dataIdArgument.name>``
546  as an instance of `ContainerClass`; the container includes fields:
547 
548  - ``idList``: a list of data ID dicts.
549  - ``refList``: a list of `~lsst.daf.persistence.Butler`
550  data references (empty if ``doMakeDataRefList`` is `False`).
551  """
552  argName = name.lstrip("-")
553 
554  if argName in self._dataIdArgDict:
555  raise RuntimeError("Data ID argument %s already exists" % (name,))
556  if argName in set(("camera", "config", "butler", "log", "obsPkg")):
557  raise RuntimeError("Data ID argument %s is a reserved name" % (name,))
558 
559  self.add_argument(name, nargs="*", action=IdValueAction, help=help,
560  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
561 
562  dataIdArgument = DataIdArgument(
563  name=argName,
564  datasetType=datasetType,
565  level=level,
566  doMakeDataRefList=doMakeDataRefList,
567  ContainerClass=ContainerClass,
568  )
569 
570  if dataIdArgument.isDynamicDatasetType:
571  datasetType.addArgument(parser=self, idName=argName)
572 
573  self._dataIdArgDict[argName] = dataIdArgument
574 
daf::base::PropertySet * set
Definition: fits.cc:884

◆ addReuseOption()

def lsst.pipe.base.argumentParser.ArgumentParser.addReuseOption (   self,
  choices 
)
Add a "--reuse-outputs-from SUBTASK" option to the argument
parser.

CmdLineTasks that can be restarted at an intermediate step using
outputs from earlier (but still internal) steps should use this
method to allow the user to control whether that happens when
outputs from earlier steps are present.

Parameters
----------
choices : sequence
    A sequence of string names (by convention, top-level subtasks)
    that identify the steps that could be skipped when their
    outputs are already present.  The list is ordered, so when the
    user specifies one step on the command line, all previous steps
    may be skipped as well.  In addition to the choices provided,
    users may pass "all" to indicate that all steps may be thus
    skipped.

When this method is called, the ``namespace`` object returned by
``parse_args`` will contain a ``reuse`` attribute containing
a list of all steps that should be skipped if their outputs
are already present.
If no steps should be skipped, the ``reuse`` will be an empty list.

Definition at line 880 of file argumentParser.py.

880  def addReuseOption(self, choices):
881  """Add a "--reuse-outputs-from SUBTASK" option to the argument
882  parser.
883 
884  CmdLineTasks that can be restarted at an intermediate step using
885  outputs from earlier (but still internal) steps should use this
886  method to allow the user to control whether that happens when
887  outputs from earlier steps are present.
888 
889  Parameters
890  ----------
891  choices : sequence
892  A sequence of string names (by convention, top-level subtasks)
893  that identify the steps that could be skipped when their
894  outputs are already present. The list is ordered, so when the
895  user specifies one step on the command line, all previous steps
896  may be skipped as well. In addition to the choices provided,
897  users may pass "all" to indicate that all steps may be thus
898  skipped.
899 
900  When this method is called, the ``namespace`` object returned by
901  ``parse_args`` will contain a ``reuse`` attribute containing
902  a list of all steps that should be skipped if their outputs
903  are already present.
904  If no steps should be skipped, the ``reuse`` will be an empty list.
905  """
906  choices = list(choices)
907  choices.append("all")
908  self.add_argument("--reuse-outputs-from", dest="reuse", choices=choices,
909  default=[], action=ReuseAction,
910  help=("Skip the given subtask and its predecessors and reuse their outputs "
911  "if those outputs already exist. Use 'all' to specify all subtasks."))
912 
913 
daf::base::PropertyList * list
Definition: fits.cc:885

◆ convert_arg_line_to_args()

def lsst.pipe.base.argumentParser.ArgumentParser.convert_arg_line_to_args (   self,
  arg_line 
)
Allow files of arguments referenced by ``@<path>`` to contain
multiple values on each line.

Parameters
----------
arg_line : `str`
    Line of text read from an argument file.

Definition at line 863 of file argumentParser.py.

863  def convert_arg_line_to_args(self, arg_line):
864  """Allow files of arguments referenced by ``@<path>`` to contain
865  multiple values on each line.
866 
867  Parameters
868  ----------
869  arg_line : `str`
870  Line of text read from an argument file.
871  """
872  arg_line = arg_line.strip()
873  if not arg_line or arg_line.startswith("#"):
874  return
875  for arg in shlex.split(arg_line, comments=True, posix=True):
876  if not arg.strip():
877  continue
878  yield arg
879 

◆ handleCamera()

def lsst.pipe.base.argumentParser.ArgumentParser.handleCamera (   self,
  namespace 
)
Perform camera-specific operations before parsing the command-line.

Parameters
----------
namespace : `argparse.Namespace`
    Namespace (an ) with the following fields:

    - ``camera``: the camera name.
    - ``config``: the config passed to parse_args, with no overrides applied.
    - ``obsPkg``: the ``obs_`` package for this camera.
    - ``log``: a `lsst.log` Log.

Notes
-----
The default implementation does nothing.

Definition at line 844 of file argumentParser.py.

844  def handleCamera(self, namespace):
845  """Perform camera-specific operations before parsing the command-line.
846 
847  Parameters
848  ----------
849  namespace : `argparse.Namespace`
850  Namespace (an ) with the following fields:
851 
852  - ``camera``: the camera name.
853  - ``config``: the config passed to parse_args, with no overrides applied.
854  - ``obsPkg``: the ``obs_`` package for this camera.
855  - ``log``: a `lsst.log` Log.
856 
857  Notes
858  -----
859  The default implementation does nothing.
860  """
861  pass
862 

◆ parse_args()

def lsst.pipe.base.argumentParser.ArgumentParser.parse_args (   self,
  config,
  args = None,
  log = None,
  override = None 
)
Parse arguments for a command-line task.

Parameters
----------
config : `lsst.pex.config.Config`
    Config for the task being run.
args : `list`, optional
    Argument list; if `None` then ``sys.argv[1:]`` is used.
log : `lsst.log.Log`, optional
    `~lsst.log.Log` instance; if `None` use the default log.
override : callable, optional
    A config override function. It must take the root config object
    as its only argument and must modify the config in place.
    This function is called after camera-specific overrides files
    are applied, and before command-line config overrides
    are applied (thus allowing the user the final word).

Returns
-------
namespace : `argparse.Namespace`
    A `~argparse.Namespace` instance containing fields:

    - ``camera``: camera name.
    - ``config``: the supplied config with all overrides applied,
validated and frozen.
    - ``butler``: a `lsst.daf.persistence.Butler` for the data.
    - An entry for each of the data ID arguments registered by
`add_id_argument`, of the type passed to its ``ContainerClass``
keyword (`~lsst.pipe.base.DataIdContainer` by default). It
includes public elements ``idList`` and ``refList``.
    - ``log``: a `lsst.log` Log.
    - An entry for each command-line argument,
with the following exceptions:

      - config is the supplied config, suitably updated.
      - configfile, id and loglevel are all missing.
    - ``obsPkg``: name of the ``obs_`` package for this camera.

Definition at line 575 of file argumentParser.py.

575  def parse_args(self, config, args=None, log=None, override=None):
576  """Parse arguments for a command-line task.
577 
578  Parameters
579  ----------
580  config : `lsst.pex.config.Config`
581  Config for the task being run.
582  args : `list`, optional
583  Argument list; if `None` then ``sys.argv[1:]`` is used.
584  log : `lsst.log.Log`, optional
585  `~lsst.log.Log` instance; if `None` use the default log.
586  override : callable, optional
587  A config override function. It must take the root config object
588  as its only argument and must modify the config in place.
589  This function is called after camera-specific overrides files
590  are applied, and before command-line config overrides
591  are applied (thus allowing the user the final word).
592 
593  Returns
594  -------
595  namespace : `argparse.Namespace`
596  A `~argparse.Namespace` instance containing fields:
597 
598  - ``camera``: camera name.
599  - ``config``: the supplied config with all overrides applied,
600  validated and frozen.
601  - ``butler``: a `lsst.daf.persistence.Butler` for the data.
602  - An entry for each of the data ID arguments registered by
603  `add_id_argument`, of the type passed to its ``ContainerClass``
604  keyword (`~lsst.pipe.base.DataIdContainer` by default). It
605  includes public elements ``idList`` and ``refList``.
606  - ``log``: a `lsst.log` Log.
607  - An entry for each command-line argument,
608  with the following exceptions:
609 
610  - config is the supplied config, suitably updated.
611  - configfile, id and loglevel are all missing.
612  - ``obsPkg``: name of the ``obs_`` package for this camera.
613  """
614  if args is None:
615  args = sys.argv[1:]
616 
617  if len(args) < 1 or args[0].startswith("-") or args[0].startswith("@"):
618  self.print_help()
619  if len(args) == 1 and args[0] in ("-h", "--help"):
620  self.exit()
621  else:
622  self.exit("%s: error: Must specify input as first argument" % self.prog)
623 
624  # Note that --rerun may change namespace.input, but if it does
625  # we verify that the new input has the same mapper class.
626  namespace = argparse.Namespace()
627  namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0])
628  if not os.path.isdir(namespace.input):
629  self.error("Error: input=%r not found" % (namespace.input,))
630 
631  namespace.config = config
632  namespace.log = log if log is not None else lsstLog.Log.getDefaultLogger()
633  mapperClass = dafPersist.Butler.getMapperClass(namespace.input)
634  if mapperClass is None:
635  self.error("Error: no mapper specified for input repo %r" % (namespace.input,))
636 
637  namespace.camera = mapperClass.getCameraName()
638  namespace.obsPkg = mapperClass.getPackageName()
639 
640  self.handleCamera(namespace)
641 
642  self._applyInitialOverrides(namespace)
643  if override is not None:
644  override(namespace.config)
645 
646  # Add data ID containers to namespace
647  for dataIdArgument in self._dataIdArgDict.values():
648  setattr(namespace, dataIdArgument.name, dataIdArgument.ContainerClass(level=dataIdArgument.level))
649 
650  namespace = argparse.ArgumentParser.parse_args(self, args=args, namespace=namespace)
651  del namespace.configfile
652 
653  self._parseDirectories(namespace)
654 
655  if namespace.clobberOutput:
656  if namespace.output is None:
657  self.error("--clobber-output is only valid with --output or --rerun")
658  elif namespace.output == namespace.input:
659  self.error("--clobber-output is not valid when the output and input repos are the same")
660  if os.path.exists(namespace.output):
661  namespace.log.info("Removing output repo %s for --clobber-output", namespace.output)
662  shutil.rmtree(namespace.output)
663 
664  namespace.log.debug("input=%s", namespace.input)
665  namespace.log.debug("calib=%s", namespace.calib)
666  namespace.log.debug("output=%s", namespace.output)
667 
668  obeyShowArgument(namespace.show, namespace.config, exit=False)
669 
670  # No environment variable or --output or --rerun specified.
671  if self.requireOutput and namespace.output is None and namespace.rerun is None:
672  self.error("no output directory specified.\n"
673  "An output directory must be specified with the --output or --rerun\n"
674  "command-line arguments.\n")
675 
676  butlerArgs = {} # common arguments for butler elements
677  if namespace.calib:
678  butlerArgs = {'mapperArgs': {'calibRoot': namespace.calib}}
679  if namespace.output:
680  outputs = {'root': namespace.output, 'mode': 'rw'}
681  inputs = {'root': namespace.input}
682  inputs.update(butlerArgs)
683  outputs.update(butlerArgs)
684  namespace.butler = dafPersist.Butler(inputs=inputs, outputs=outputs)
685  else:
686  outputs = {'root': namespace.input, 'mode': 'rw'}
687  outputs.update(butlerArgs)
688  namespace.butler = dafPersist.Butler(outputs=outputs)
689 
690  # convert data in each of the identifier lists to proper types
691  # this is done after constructing the butler,
692  # hence after parsing the command line,
693  # because it takes a long time to construct a butler
694  self._processDataIds(namespace)
695  if "data" in namespace.show:
696  for dataIdName in self._dataIdArgDict.keys():
697  for dataRef in getattr(namespace, dataIdName).refList:
698  print("%s dataRef.dataId = %s" % (dataIdName, dataRef.dataId))
699 
700  if namespace.show and "run" not in namespace.show:
701  sys.exit(0)
702 
703  if namespace.debug:
704  try:
705  import debug
706  assert debug # silence pyflakes
707  except ImportError:
708  sys.stderr.write("Warning: no 'debug' module found\n")
709  namespace.debug = False
710 
711  del namespace.loglevel
712 
713  if namespace.longlog:
714  lsstLog.configure_prop("""
715 log4j.rootLogger=INFO, A1
716 log4j.appender.A1=ConsoleAppender
717 log4j.appender.A1.Target=System.out
718 log4j.appender.A1.layout=PatternLayout
719 log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n
720 """)
721  del namespace.longlog
722 
723  namespace.config.validate()
724  namespace.config.freeze()
725 
726  return namespace
727 
def obeyShowArgument(showOpts, config=None, exit=False)

Member Data Documentation

◆ requireOutput

bool lsst.pipe.base.argumentParser.ArgumentParser.requireOutput = True
static

Definition at line 431 of file argumentParser.py.


The documentation for this class was generated from the following file: