LSSTApplications  17.0,17.0+1,17.0+10,17.0+11,17.0+14,17.0+6,17.0-1-g377950a+5,17.0.1,17.0.1-1-g0d345a5+2,17.0.1-1-g444bd44+2,17.0.1-1-g46e6382+2,17.0.1-1-g4d4fbc4,17.0.1-1-g703d48b+1,17.0.1-1-g9deacb5+2,17.0.1-1-gaef33af,17.0.1-1-gea52513+2,17.0.1-1-gf4e0155+2,17.0.1-1-gfc65f5f+2,17.0.1-1-gfc6fb1f,17.0.1-2-g0ce9737+2,17.0.1-2-g2a2f1b99+2,17.0.1-2-gd73ec07+2,17.0.1-2-gd9aa6e4+1,17.0.1-3-gb71a564+2,17.0.1-3-gc20ba7d+2,17.0.1-4-g41c8d5dc0+1,17.0.1-4-gfa71e81,17.0.1-5-gb7d1e01+1,17.0.1-5-gf0ac6446+2,17.0.1-7-g69836a1+1
LSSTDataManagementBasePackage
Public Member Functions | Static Public Attributes | List of all members
lsst.pipe.base.argumentParser.ArgumentParser Class Reference
Inheritance diagram for lsst.pipe.base.argumentParser.ArgumentParser:
lsst.pipe.base.argumentParser.InputOnlyArgumentParser lsst.pipe.drivers.constructCalibs.CalibArgumentParser lsst.pipe.tasks.ingest.IngestArgumentParser lsst.pipe.tasks.ingestCalibs.IngestCalibsArgumentParser lsst.obs.decam.ingest.DecamIngestArgumentParser

Public Member Functions

def __init__ (self, name, usage="%(prog)s input [options]", kwargs)
 
def add_id_argument (self, name, datasetType, help, level=None, doMakeDataRefList=True, ContainerClass=DataIdContainer)
 
def parse_args (self, config, args=None, log=None, override=None)
 
def handleCamera (self, namespace)
 
def convert_arg_line_to_args (self, arg_line)
 
def addReuseOption (self, choices)
 

Static Public Attributes

bool requireOutput = True
 

Detailed Description

Argument parser for command-line tasks that is based on
`argparse.ArgumentParser`.

Parameters
----------
name : `str`
    Name of top-level task; used to identify camera-specific override
    files.
usage : `str`, optional
    Command-line usage signature.
**kwargs
    Additional keyword arguments for `argparse.ArgumentParser`.

Notes
-----
Users may wish to add additional arguments before calling `parse_args`.

Definition at line 407 of file argumentParser.py.

Constructor & Destructor Documentation

◆ __init__()

def lsst.pipe.base.argumentParser.ArgumentParser.__init__ (   self,
  name,
  usage = "%(prog)s input [options]",
  kwargs 
)

Definition at line 434 of file argumentParser.py.

434  def __init__(self, name, usage="%(prog)s input [options]", **kwargs):
435  self._name = name
436  self._dataIdArgDict = {} # Dict of data identifier specifications, by argument name
437  argparse.ArgumentParser.__init__(self,
438  usage=usage,
439  fromfile_prefix_chars='@',
440  epilog=textwrap.dedent("""Notes:
441  * --config, --configfile, --id, --loglevel and @file may appear multiple times;
442  all values are used, in order left to right
443  * @file reads command-line options from the specified file:
444  * data may be distributed among multiple lines (e.g. one option per line)
445  * data after # is treated as a comment and ignored
446  * blank lines and lines starting with # are ignored
447  * To specify multiple values for an option, do not use = after the option name:
448  * right: --configfile foo bar
449  * wrong: --configfile=foo bar
450  """),
451  formatter_class=argparse.RawDescriptionHelpFormatter,
452  **kwargs)
453  self.add_argument(metavar='input', dest="rawInput",
454  help="path to input data repository, relative to $%s" % (DEFAULT_INPUT_NAME,))
455  self.add_argument("--calib", dest="rawCalib",
456  help="path to input calibration repository, relative to $%s" %
457  (DEFAULT_CALIB_NAME,))
458  self.add_argument("--output", dest="rawOutput",
459  help="path to output data repository (need not exist), relative to $%s" %
460  (DEFAULT_OUTPUT_NAME,))
461  self.add_argument("--rerun", dest="rawRerun", metavar="[INPUT:]OUTPUT",
462  help="rerun name: sets OUTPUT to ROOT/rerun/OUTPUT; "
463  "optionally sets ROOT to ROOT/rerun/INPUT")
464  self.add_argument("-c", "--config", nargs="*", action=ConfigValueAction,
465  help="config override(s), e.g. -c foo=newfoo bar.baz=3", metavar="NAME=VALUE")
466  self.add_argument("-C", "--configfile", dest="configfile", nargs="*", action=ConfigFileAction,
467  help="config override file(s)")
468  self.add_argument("-L", "--loglevel", nargs="*", action=LogLevelAction,
469  help="logging level; supported levels are [trace|debug|info|warn|error|fatal]",
470  metavar="LEVEL|COMPONENT=LEVEL")
471  self.add_argument("--longlog", action="store_true", help="use a more verbose format for the logging")
472  self.add_argument("--debug", action="store_true", help="enable debugging output?")
473  self.add_argument("--doraise", action="store_true",
474  help="raise an exception on error (else log a message and continue)?")
475  self.add_argument("--noExit", action="store_true",
476  help="Do not exit even upon failure (i.e. return a struct to the calling script)")
477  self.add_argument("--profile", help="Dump cProfile statistics to filename")
478  self.add_argument("--show", nargs="+", default=(),
479  help="display the specified information to stdout and quit "
480  "(unless run is specified).")
481  self.add_argument("-j", "--processes", type=int, default=1, help="Number of processes to use")
482  self.add_argument("-t", "--timeout", type=float,
483  help="Timeout for multiprocessing; maximum wall time (sec)")
484  self.add_argument("--clobber-output", action="store_true", dest="clobberOutput", default=False,
485  help=("remove and re-create the output directory if it already exists "
486  "(safe with -j, but not all other forms of parallel execution)"))
487  self.add_argument("--clobber-config", action="store_true", dest="clobberConfig", default=False,
488  help=("backup and then overwrite existing config files instead of checking them "
489  "(safe with -j, but not all other forms of parallel execution)"))
490  self.add_argument("--no-backup-config", action="store_true", dest="noBackupConfig", default=False,
491  help="Don't copy config to file~N backup.")
492  self.add_argument("--clobber-versions", action="store_true", dest="clobberVersions", default=False,
493  help=("backup and then overwrite existing package versions instead of checking"
494  "them (safe with -j, but not all other forms of parallel execution)"))
495  self.add_argument("--no-versions", action="store_true", dest="noVersions", default=False,
496  help="don't check package versions; useful for development")
497  lsstLog.configure_prop("""
498 log4j.rootLogger=INFO, A1
499 log4j.appender.A1=ConsoleAppender
500 log4j.appender.A1.Target=System.out
501 log4j.appender.A1.layout=PatternLayout
502 log4j.appender.A1.layout.ConversionPattern=%c %p: %m%n
503 """)
504 
505  # Forward all Python logging to lsst.log
506  lgr = logging.getLogger()
507  lgr.setLevel(logging.INFO) # same as in log4cxx config above
508  lgr.addHandler(lsstLog.LogHandler())
509 
def __init__(self, minimum, dataRange, Q)

Member Function Documentation

◆ add_id_argument()

def lsst.pipe.base.argumentParser.ArgumentParser.add_id_argument (   self,
  name,
  datasetType,
  help,
  level = None,
  doMakeDataRefList = True,
  ContainerClass = DataIdContainer 
)
Add a data ID argument.


Parameters
----------
name : `str`
    Data ID argument (including leading dashes, if wanted).
datasetType : `str` or `DynamicDatasetType`-type
    Type of dataset. Supply a string for a fixed dataset type.
    For a dynamically determined dataset type, supply
    a `DynamicDatasetType`, such a `DatasetArgument`.
help : `str`
    Help string for the argument.
level : `str`
    The lowest hierarchy level to descend to for this dataset type,
    for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
    Use `""` to use the mapper's default for the dataset type.
    Some container classes may also support `None`, which means
    the level should not be restricted; however the default class,
    `DataIdContainer`, does not support `None`.
doMakeDataRefList : bool, optional
    If `True` (default), construct data references.
ContainerClass : `class`, optional
Class to contain data IDs and data references; the default class
`DataIdContainer` will work for many, but not all, cases.
For example if the dataset type is specified on the command line
then use `DynamicDatasetType`.

Notes
-----
If ``datasetType`` is an instance of `DatasetArgument`,
then add a second argument to specify the dataset type.

The associated data is put into ``namespace.<dataIdArgument.name>``
as an instance of `ContainerClass`; the container includes fields:

- ``idList``: a list of data ID dicts.
- ``refList``: a list of `~lsst.daf.persistence.Butler`
    data references (empty if ``doMakeDataRefList`` is  `False`).

Definition at line 511 of file argumentParser.py.

511  ContainerClass=DataIdContainer):
512  """Add a data ID argument.
513 
514 
515  Parameters
516  ----------
517  name : `str`
518  Data ID argument (including leading dashes, if wanted).
519  datasetType : `str` or `DynamicDatasetType`-type
520  Type of dataset. Supply a string for a fixed dataset type.
521  For a dynamically determined dataset type, supply
522  a `DynamicDatasetType`, such a `DatasetArgument`.
523  help : `str`
524  Help string for the argument.
525  level : `str`
526  The lowest hierarchy level to descend to for this dataset type,
527  for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
528  Use `""` to use the mapper's default for the dataset type.
529  Some container classes may also support `None`, which means
530  the level should not be restricted; however the default class,
531  `DataIdContainer`, does not support `None`.
532  doMakeDataRefList : bool, optional
533  If `True` (default), construct data references.
534  ContainerClass : `class`, optional
535  Class to contain data IDs and data references; the default class
536  `DataIdContainer` will work for many, but not all, cases.
537  For example if the dataset type is specified on the command line
538  then use `DynamicDatasetType`.
539 
540  Notes
541  -----
542  If ``datasetType`` is an instance of `DatasetArgument`,
543  then add a second argument to specify the dataset type.
544 
545  The associated data is put into ``namespace.<dataIdArgument.name>``
546  as an instance of `ContainerClass`; the container includes fields:
547 
548  - ``idList``: a list of data ID dicts.
549  - ``refList``: a list of `~lsst.daf.persistence.Butler`
550  data references (empty if ``doMakeDataRefList`` is `False`).
551  """
552  argName = name.lstrip("-")
553 
554  if argName in self._dataIdArgDict:
555  raise RuntimeError("Data ID argument %s already exists" % (name,))
556  if argName in set(("camera", "config", "butler", "log", "obsPkg")):
557  raise RuntimeError("Data ID argument %s is a reserved name" % (name,))
558 
559  self.add_argument(name, nargs="*", action=IdValueAction, help=help,
560  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
561 
562  dataIdArgument = DataIdArgument(
563  name=argName,
564  datasetType=datasetType,
565  level=level,
566  doMakeDataRefList=doMakeDataRefList,
567  ContainerClass=ContainerClass,
568  )
569 
570  if dataIdArgument.isDynamicDatasetType:
571  datasetType.addArgument(parser=self, idName=argName)
572 
573  self._dataIdArgDict[argName] = dataIdArgument
574 
daf::base::PropertySet * set
Definition: fits.cc:832

◆ addReuseOption()

def lsst.pipe.base.argumentParser.ArgumentParser.addReuseOption (   self,
  choices 
)
Add a "--reuse-outputs-from SUBTASK" option to the argument
parser.

CmdLineTasks that can be restarted at an intermediate step using
outputs from earlier (but still internal) steps should use this
method to allow the user to control whether that happens when
outputs from earlier steps are present.

Parameters
----------
choices : sequence
    A sequence of string names (by convention, top-level subtasks)
    that identify the steps that could be skipped when their
    outputs are already present.  The list is ordered, so when the
    user specifies one step on the command line, all previous steps
    may be skipped as well.  In addition to the choices provided,
    users may pass "all" to indicate that all steps may be thus
    skipped.

When this method is called, the ``namespace`` object returned by
``parse_args`` will contain a ``reuse`` attribute containing
a list of all steps that should be skipped if their outputs
are already present.
If no steps should be skipped, the ``reuse`` will be an empty list.

Definition at line 877 of file argumentParser.py.

877  def addReuseOption(self, choices):
878  """Add a "--reuse-outputs-from SUBTASK" option to the argument
879  parser.
880 
881  CmdLineTasks that can be restarted at an intermediate step using
882  outputs from earlier (but still internal) steps should use this
883  method to allow the user to control whether that happens when
884  outputs from earlier steps are present.
885 
886  Parameters
887  ----------
888  choices : sequence
889  A sequence of string names (by convention, top-level subtasks)
890  that identify the steps that could be skipped when their
891  outputs are already present. The list is ordered, so when the
892  user specifies one step on the command line, all previous steps
893  may be skipped as well. In addition to the choices provided,
894  users may pass "all" to indicate that all steps may be thus
895  skipped.
896 
897  When this method is called, the ``namespace`` object returned by
898  ``parse_args`` will contain a ``reuse`` attribute containing
899  a list of all steps that should be skipped if their outputs
900  are already present.
901  If no steps should be skipped, the ``reuse`` will be an empty list.
902  """
903  choices = list(choices)
904  choices.append("all")
905  self.add_argument("--reuse-outputs-from", dest="reuse", choices=choices,
906  default=[], action=ReuseAction,
907  help=("Skip the given subtask and its predecessors and reuse their outputs "
908  "if those outputs already exist. Use 'all' to specify all subtasks."))
909 
910 
daf::base::PropertyList * list
Definition: fits.cc:833

◆ convert_arg_line_to_args()

def lsst.pipe.base.argumentParser.ArgumentParser.convert_arg_line_to_args (   self,
  arg_line 
)
Allow files of arguments referenced by ``@<path>`` to contain
multiple values on each line.

Parameters
----------
arg_line : `str`
    Line of text read from an argument file.

Definition at line 860 of file argumentParser.py.

860  def convert_arg_line_to_args(self, arg_line):
861  """Allow files of arguments referenced by ``@<path>`` to contain
862  multiple values on each line.
863 
864  Parameters
865  ----------
866  arg_line : `str`
867  Line of text read from an argument file.
868  """
869  arg_line = arg_line.strip()
870  if not arg_line or arg_line.startswith("#"):
871  return
872  for arg in shlex.split(arg_line, comments=True, posix=True):
873  if not arg.strip():
874  continue
875  yield arg
876 

◆ handleCamera()

def lsst.pipe.base.argumentParser.ArgumentParser.handleCamera (   self,
  namespace 
)
Perform camera-specific operations before parsing the command-line.

Parameters
----------
namespace : `argparse.Namespace`
    Namespace (an ) with the following fields:

    - ``camera``: the camera name.
    - ``config``: the config passed to parse_args, with no overrides applied.
    - ``obsPkg``: the ``obs_`` package for this camera.
    - ``log``: a `lsst.log` Log.

Notes
-----
The default implementation does nothing.

Definition at line 841 of file argumentParser.py.

841  def handleCamera(self, namespace):
842  """Perform camera-specific operations before parsing the command-line.
843 
844  Parameters
845  ----------
846  namespace : `argparse.Namespace`
847  Namespace (an ) with the following fields:
848 
849  - ``camera``: the camera name.
850  - ``config``: the config passed to parse_args, with no overrides applied.
851  - ``obsPkg``: the ``obs_`` package for this camera.
852  - ``log``: a `lsst.log` Log.
853 
854  Notes
855  -----
856  The default implementation does nothing.
857  """
858  pass
859 

◆ parse_args()

def lsst.pipe.base.argumentParser.ArgumentParser.parse_args (   self,
  config,
  args = None,
  log = None,
  override = None 
)
Parse arguments for a command-line task.

Parameters
----------
config : `lsst.pex.config.Config`
    Config for the task being run.
args : `list`, optional
    Argument list; if `None` then ``sys.argv[1:]`` is used.
log : `lsst.log.Log`, optional
    `~lsst.log.Log` instance; if `None` use the default log.
override : callable, optional
    A config override function. It must take the root config object
    as its only argument and must modify the config in place.
    This function is called after camera-specific overrides files
    are applied, and before command-line config overrides
    are applied (thus allowing the user the final word).

Returns
-------
namespace : `argparse.Namespace`
    A `~argparse.Namespace` instance containing fields:

    - ``camera``: camera name.
    - ``config``: the supplied config with all overrides applied,
validated and frozen.
    - ``butler``: a `lsst.daf.persistence.Butler` for the data.
    - An entry for each of the data ID arguments registered by
`add_id_argument`, of the type passed to its ``ContainerClass``
keyword (`~lsst.pipe.base.DataIdContainer` by default). It
includes public elements ``idList`` and ``refList``.
    - ``log``: a `lsst.log` Log.
    - An entry for each command-line argument,
with the following exceptions:

      - config is the supplied config, suitably updated.
      - configfile, id and loglevel are all missing.
    - ``obsPkg``: name of the ``obs_`` package for this camera.

Definition at line 575 of file argumentParser.py.

575  def parse_args(self, config, args=None, log=None, override=None):
576  """Parse arguments for a command-line task.
577 
578  Parameters
579  ----------
580  config : `lsst.pex.config.Config`
581  Config for the task being run.
582  args : `list`, optional
583  Argument list; if `None` then ``sys.argv[1:]`` is used.
584  log : `lsst.log.Log`, optional
585  `~lsst.log.Log` instance; if `None` use the default log.
586  override : callable, optional
587  A config override function. It must take the root config object
588  as its only argument and must modify the config in place.
589  This function is called after camera-specific overrides files
590  are applied, and before command-line config overrides
591  are applied (thus allowing the user the final word).
592 
593  Returns
594  -------
595  namespace : `argparse.Namespace`
596  A `~argparse.Namespace` instance containing fields:
597 
598  - ``camera``: camera name.
599  - ``config``: the supplied config with all overrides applied,
600  validated and frozen.
601  - ``butler``: a `lsst.daf.persistence.Butler` for the data.
602  - An entry for each of the data ID arguments registered by
603  `add_id_argument`, of the type passed to its ``ContainerClass``
604  keyword (`~lsst.pipe.base.DataIdContainer` by default). It
605  includes public elements ``idList`` and ``refList``.
606  - ``log``: a `lsst.log` Log.
607  - An entry for each command-line argument,
608  with the following exceptions:
609 
610  - config is the supplied config, suitably updated.
611  - configfile, id and loglevel are all missing.
612  - ``obsPkg``: name of the ``obs_`` package for this camera.
613  """
614  if args is None:
615  args = sys.argv[1:]
616 
617  if len(args) < 1 or args[0].startswith("-") or args[0].startswith("@"):
618  self.print_help()
619  if len(args) == 1 and args[0] in ("-h", "--help"):
620  self.exit()
621  else:
622  self.exit("%s: error: Must specify input as first argument" % self.prog)
623 
624  # Note that --rerun may change namespace.input, but if it does
625  # we verify that the new input has the same mapper class.
626  namespace = argparse.Namespace()
627  namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0])
628  if not os.path.isdir(namespace.input):
629  self.error("Error: input=%r not found" % (namespace.input,))
630 
631  namespace.config = config
632  namespace.log = log if log is not None else lsstLog.Log.getDefaultLogger()
633  mapperClass = dafPersist.Butler.getMapperClass(namespace.input)
634  namespace.camera = mapperClass.getCameraName()
635  namespace.obsPkg = mapperClass.getPackageName()
636 
637  self.handleCamera(namespace)
638 
639  self._applyInitialOverrides(namespace)
640  if override is not None:
641  override(namespace.config)
642 
643  # Add data ID containers to namespace
644  for dataIdArgument in self._dataIdArgDict.values():
645  setattr(namespace, dataIdArgument.name, dataIdArgument.ContainerClass(level=dataIdArgument.level))
646 
647  namespace = argparse.ArgumentParser.parse_args(self, args=args, namespace=namespace)
648  del namespace.configfile
649 
650  self._parseDirectories(namespace)
651 
652  if namespace.clobberOutput:
653  if namespace.output is None:
654  self.error("--clobber-output is only valid with --output or --rerun")
655  elif namespace.output == namespace.input:
656  self.error("--clobber-output is not valid when the output and input repos are the same")
657  if os.path.exists(namespace.output):
658  namespace.log.info("Removing output repo %s for --clobber-output", namespace.output)
659  shutil.rmtree(namespace.output)
660 
661  namespace.log.debug("input=%s", namespace.input)
662  namespace.log.debug("calib=%s", namespace.calib)
663  namespace.log.debug("output=%s", namespace.output)
664 
665  obeyShowArgument(namespace.show, namespace.config, exit=False)
666 
667  # No environment variable or --output or --rerun specified.
668  if self.requireOutput and namespace.output is None and namespace.rerun is None:
669  self.error("no output directory specified.\n"
670  "An output directory must be specified with the --output or --rerun\n"
671  "command-line arguments.\n")
672 
673  butlerArgs = {} # common arguments for butler elements
674  if namespace.calib:
675  butlerArgs = {'mapperArgs': {'calibRoot': namespace.calib}}
676  if namespace.output:
677  outputs = {'root': namespace.output, 'mode': 'rw'}
678  inputs = {'root': namespace.input}
679  inputs.update(butlerArgs)
680  outputs.update(butlerArgs)
681  namespace.butler = dafPersist.Butler(inputs=inputs, outputs=outputs)
682  else:
683  outputs = {'root': namespace.input, 'mode': 'rw'}
684  outputs.update(butlerArgs)
685  namespace.butler = dafPersist.Butler(outputs=outputs)
686 
687  # convert data in each of the identifier lists to proper types
688  # this is done after constructing the butler,
689  # hence after parsing the command line,
690  # because it takes a long time to construct a butler
691  self._processDataIds(namespace)
692  if "data" in namespace.show:
693  for dataIdName in self._dataIdArgDict.keys():
694  for dataRef in getattr(namespace, dataIdName).refList:
695  print("%s dataRef.dataId = %s" % (dataIdName, dataRef.dataId))
696 
697  if namespace.show and "run" not in namespace.show:
698  sys.exit(0)
699 
700  if namespace.debug:
701  try:
702  import debug
703  assert debug # silence pyflakes
704  except ImportError:
705  sys.stderr.write("Warning: no 'debug' module found\n")
706  namespace.debug = False
707 
708  del namespace.loglevel
709 
710  if namespace.longlog:
711  lsstLog.configure_prop("""
712 log4j.rootLogger=INFO, A1
713 log4j.appender.A1=ConsoleAppender
714 log4j.appender.A1.Target=System.out
715 log4j.appender.A1.layout=PatternLayout
716 log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddThh:mm:ss.sss} %c (%X{LABEL})(%F:%L)- %m%n
717 """)
718  del namespace.longlog
719 
720  namespace.config.validate()
721  namespace.config.freeze()
722 
723  return namespace
724 
def obeyShowArgument(showOpts, config=None, exit=False)

Member Data Documentation

◆ requireOutput

bool lsst.pipe.base.argumentParser.ArgumentParser.requireOutput = True
static

Definition at line 431 of file argumentParser.py.


The documentation for this class was generated from the following file: