lsst.pipe.base.argumentParser.ArgumentParser Class Reference

Inheritance diagram for lsst.pipe.base.argumentParser.ArgumentParser:

Public Member Functions
def	__init__ (self, name, usage="%(prog)s input [options]", **kwargs)
def	add_id_argument (self, name, datasetType, help, level=None, doMakeDataRefList=True, ContainerClass=DataIdContainer)
def	parse_args (self, config, args=None, log=None, override=None)
def	handleCamera (self, namespace)
def	convert_arg_line_to_args (self, arg_line)
def	addReuseOption (self, choices)

Static Public Attributes
bool	requireOutput = True

Detailed Description

Argument parser for command-line tasks that is based on
`argparse.ArgumentParser`.

Parameters
----------
name : `str`
    Name of top-level task; used to identify camera-specific override
    files.
usage : `str`, optional
    Command-line usage signature.
**kwargs
    Additional keyword arguments for `argparse.ArgumentParser`.

Notes
-----
Users may wish to add additional arguments before calling `parse_args`.

Definition at line 408 of file argumentParser.py.

Constructor & Destructor Documentation

__init__()

def lsst.pipe.base.argumentParser.ArgumentParser.__init__	(		self,
			name,
			usage = "%(prog)s input [options]",
		**	kwargs
	)

Definition at line 435 of file argumentParser.py.

    def __init__(self, name, usage="%(prog)s input [options]", **kwargs):
        self._name = name
        self._dataIdArgDict = {}  # Dict of data identifier specifications, by argument name
        argparse.ArgumentParser.__init__(self,
                                         usage=usage,
                                         fromfile_prefix_chars='@',
                                         epilog=textwrap.dedent("""Notes:
            * --config, --configfile, --id, --loglevel and @file may appear multiple times;
                all values are used, in order left to right
            * @file reads command-line options from the specified file:
                * data may be distributed among multiple lines (e.g. one option per line)
                * data after # is treated as a comment and ignored
                * blank lines and lines starting with # are ignored
            * To specify multiple values for an option, do not use = after the option name:
                * right: --configfile foo bar
                * wrong: --configfile=foo bar
            """),
                                         formatter_class=argparse.RawDescriptionHelpFormatter,
                                         **kwargs)
        self.add_argument(metavar='input', dest="rawInput",
                          help=f"path to input data repository, relative to ${DEFAULT_INPUT_NAME}")
        self.add_argument("--calib", dest="rawCalib",
                          help=f"path to input calibration repository, relative to ${DEFAULT_CALIB_NAME}")
        self.add_argument("--output", dest="rawOutput",
                          help="path to output data repository (need not exist), "
                               f"relative to ${DEFAULT_OUTPUT_NAME}")
        self.add_argument("--rerun", dest="rawRerun", metavar="[INPUT:]OUTPUT",
                          help="rerun name: sets OUTPUT to ROOT/rerun/OUTPUT; "
                               "optionally sets ROOT to ROOT/rerun/INPUT")
        self.add_argument("-c", "--config", nargs="*", action=ConfigValueAction,
                          help="config override(s), e.g. -c foo=newfoo bar.baz=3", metavar="NAME=VALUE")
        self.add_argument("-C", "--configfile", dest="configfile", nargs="*", action=ConfigFileAction,
                          help="config override file(s)")
        self.add_argument("-L", "--loglevel", nargs="*", action=LogLevelAction,
                          help="logging level; supported levels are [trace|debug|info|warn|error|fatal]",
                          metavar="LEVEL|COMPONENT=LEVEL")
        self.add_argument("--longlog", action="store_true", help="use a more verbose format for the logging")
        self.add_argument("--debug", action="store_true", help="enable debugging output?")
        self.add_argument("--doraise", action="store_true",
                          help="raise an exception on error (else log a message and continue)?")
        self.add_argument("--noExit", action="store_true",
                          help="Do not exit even upon failure (i.e. return a struct to the calling script)")
        self.add_argument("--profile", help="Dump cProfile statistics to filename")
        self.add_argument("--show", nargs="+", default=(),
                          help="display the specified information to stdout and quit "
                               "(unless run is specified); information is "
                               "(config[=PATTERN]|history=PATTERN|tasks|data|run)")
        self.add_argument("-j", "--processes", type=int, default=1, help="Number of processes to use")
        self.add_argument("-t", "--timeout", type=float,
                          help="Timeout for multiprocessing; maximum wall time (sec)")
        self.add_argument("--clobber-output", action="store_true", dest="clobberOutput", default=False,
                          help=("remove and re-create the output directory if it already exists "
                                "(safe with -j, but not all other forms of parallel execution)"))
        self.add_argument("--clobber-config", action="store_true", dest="clobberConfig", default=False,
                          help=("backup and then overwrite existing config files instead of checking them "
                                "(safe with -j, but not all other forms of parallel execution)"))
        self.add_argument("--no-backup-config", action="store_true", dest="noBackupConfig", default=False,
                          help="Don't copy config to file~N backup.")
        self.add_argument("--clobber-versions", action="store_true", dest="clobberVersions", default=False,
                          help=("backup and then overwrite existing package versions instead of checking"
                                "them (safe with -j, but not all other forms of parallel execution)"))
        self.add_argument("--no-versions", action="store_true", dest="noVersions", default=False,
                          help="don't check package versions; useful for development")
        lsstLog.configure_prop("""
log4j.rootLogger=INFO, A1
log4j.appender.A1=ConsoleAppender
log4j.appender.A1.Target=System.out
log4j.appender.A1.layout=PatternLayout
log4j.appender.A1.layout.ConversionPattern=%c %p: %m%n
""")
 
        # Forward all Python logging to lsst.log
        lgr = logging.getLogger()
        lgr.setLevel(logging.INFO)  # same as in log4cxx config above
        lgr.addHandler(lsstLog.LogHandler())
 

Member Function Documentation

add_id_argument()

def lsst.pipe.base.argumentParser.ArgumentParser.add_id_argument	(		self,
			name,
			datasetType,
			help,
			level = None,
			doMakeDataRefList = True,
			ContainerClass = DataIdContainer
	)

Add a data ID argument.


Parameters
----------
name : `str`
    Data ID argument (including leading dashes, if wanted).
datasetType : `str` or `DynamicDatasetType`-type
    Type of dataset. Supply a string for a fixed dataset type.
    For a dynamically determined dataset type, supply
    a `DynamicDatasetType`, such a `DatasetArgument`.
help : `str`
    Help string for the argument.
level : `str`
    The lowest hierarchy level to descend to for this dataset type,
    for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
    Use `""` to use the mapper's default for the dataset type.
    Some container classes may also support `None`, which means
    the level should not be restricted; however the default class,
    `DataIdContainer`, does not support `None`.
doMakeDataRefList : bool, optional
    If `True` (default), construct data references.
ContainerClass : `class`, optional
Class to contain data IDs and data references; the default class
`DataIdContainer` will work for many, but not all, cases.
For example if the dataset type is specified on the command line
then use `DynamicDatasetType`.

Notes
-----
If ``datasetType`` is an instance of `DatasetArgument`,
then add a second argument to specify the dataset type.

The associated data is put into ``namespace.<dataIdArgument.name>``
as an instance of `ContainerClass`; the container includes fields:

- ``idList``: a list of data ID dicts.
- ``refList``: a list of `~lsst.daf.persistence.Butler`
    data references (empty if ``doMakeDataRefList`` is  `False`).

Definition at line 511 of file argumentParser.py.

    def add_id_argument(self, name, datasetType, help, level=None, doMakeDataRefList=True,
                        ContainerClass=DataIdContainer):
        """Add a data ID argument.
 
 
        Parameters
        ----------
        name : `str`
            Data ID argument (including leading dashes, if wanted).
        datasetType : `str` or `DynamicDatasetType`-type
            Type of dataset. Supply a string for a fixed dataset type.
            For a dynamically determined dataset type, supply
            a `DynamicDatasetType`, such a `DatasetArgument`.
        help : `str`
            Help string for the argument.
        level : `str`
            The lowest hierarchy level to descend to for this dataset type,
            for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
            Use `""` to use the mapper's default for the dataset type.
            Some container classes may also support `None`, which means
            the level should not be restricted; however the default class,
            `DataIdContainer`, does not support `None`.
        doMakeDataRefList : bool, optional
            If `True` (default), construct data references.
        ContainerClass : `class`, optional
        Class to contain data IDs and data references; the default class
        `DataIdContainer` will work for many, but not all, cases.
        For example if the dataset type is specified on the command line
        then use `DynamicDatasetType`.
 
        Notes
        -----
        If ``datasetType`` is an instance of `DatasetArgument`,
        then add a second argument to specify the dataset type.
 
        The associated data is put into ``namespace.<dataIdArgument.name>``
        as an instance of `ContainerClass`; the container includes fields:
 
        - ``idList``: a list of data ID dicts.
        - ``refList``: a list of `~lsst.daf.persistence.Butler`
            data references (empty if ``doMakeDataRefList`` is  `False`).
        """
        argName = name.lstrip("-")
 
        if argName in self._dataIdArgDict:
            raise RuntimeError(f"Data ID argument {name} already exists")
        if argName in set(("camera", "config", "butler", "log", "obsPkg")):
            raise RuntimeError(f"Data ID argument {name} is a reserved name")
 
        self.add_argument(name, nargs="*", action=IdValueAction, help=help,
                          metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
 
        dataIdArgument = DataIdArgument(
            name=argName,
            datasetType=datasetType,
            level=level,
            doMakeDataRefList=doMakeDataRefList,
            ContainerClass=ContainerClass,
        )
 
        if dataIdArgument.isDynamicDatasetType:
            datasetType.addArgument(parser=self, idName=argName)
 
        self._dataIdArgDict[argName] = dataIdArgument
 

addReuseOption()

def lsst.pipe.base.argumentParser.ArgumentParser.addReuseOption	(		self,
			choices
	)

Add a "--reuse-outputs-from SUBTASK" option to the argument
parser.

CmdLineTasks that can be restarted at an intermediate step using
outputs from earlier (but still internal) steps should use this
method to allow the user to control whether that happens when
outputs from earlier steps are present.

Parameters
----------
choices : sequence
    A sequence of string names (by convention, top-level subtasks)
    that identify the steps that could be skipped when their
    outputs are already present.  The list is ordered, so when the
    user specifies one step on the command line, all previous steps
    may be skipped as well.  In addition to the choices provided,
    users may pass "all" to indicate that all steps may be thus
    skipped.

When this method is called, the ``namespace`` object returned by
``parse_args`` will contain a ``reuse`` attribute containing
a list of all steps that should be skipped if their outputs
are already present.
If no steps should be skipped, the ``reuse`` will be an empty list.

Definition at line 883 of file argumentParser.py.

    def addReuseOption(self, choices):
        """Add a "--reuse-outputs-from SUBTASK" option to the argument
        parser.
 
        CmdLineTasks that can be restarted at an intermediate step using
        outputs from earlier (but still internal) steps should use this
        method to allow the user to control whether that happens when
        outputs from earlier steps are present.
 
        Parameters
        ----------
        choices : sequence
            A sequence of string names (by convention, top-level subtasks)
            that identify the steps that could be skipped when their
            outputs are already present.  The list is ordered, so when the
            user specifies one step on the command line, all previous steps
            may be skipped as well.  In addition to the choices provided,
            users may pass "all" to indicate that all steps may be thus
            skipped.
 
        When this method is called, the ``namespace`` object returned by
        ``parse_args`` will contain a ``reuse`` attribute containing
        a list of all steps that should be skipped if their outputs
        are already present.
        If no steps should be skipped, the ``reuse`` will be an empty list.
        """
        choices = list(choices)
        choices.append("all")
        self.add_argument("--reuse-outputs-from", dest="reuse", choices=choices,
                          default=[], action=ReuseAction,
                          help=("Skip the given subtask and its predecessors and reuse their outputs "
                                "if those outputs already exist.  Use 'all' to specify all subtasks."))
 
 

convert_arg_line_to_args()

def lsst.pipe.base.argumentParser.ArgumentParser.convert_arg_line_to_args	(		self,
			arg_line
	)

Allow files of arguments referenced by ``@<path>`` to contain
multiple values on each line.

Parameters
----------
arg_line : `str`
    Line of text read from an argument file.

Definition at line 866 of file argumentParser.py.

    def convert_arg_line_to_args(self, arg_line):
        """Allow files of arguments referenced by ``@<path>`` to contain
        multiple values on each line.
 
        Parameters
        ----------
        arg_line : `str`
            Line of text read from an argument file.
        """
        arg_line = arg_line.strip()
        if not arg_line or arg_line.startswith("#"):
            return
        for arg in shlex.split(arg_line, comments=True, posix=True):
            if not arg.strip():
                continue
            yield arg
 

handleCamera()

def lsst.pipe.base.argumentParser.ArgumentParser.handleCamera	(		self,
			namespace
	)

Perform camera-specific operations before parsing the command-line.

Parameters
----------
namespace : `argparse.Namespace`
    Namespace (an ) with the following fields:

    - ``camera``: the camera name.
    - ``config``: the config passed to parse_args, with no overrides
      applied.
    - ``obsPkg``: the ``obs_`` package for this camera.
    - ``log``: a `lsst.log` Log.

Notes
-----
The default implementation does nothing.

Definition at line 846 of file argumentParser.py.

    def handleCamera(self, namespace):
        """Perform camera-specific operations before parsing the command-line.
 
        Parameters
        ----------
        namespace : `argparse.Namespace`
            Namespace (an ) with the following fields:
 
            - ``camera``: the camera name.
            - ``config``: the config passed to parse_args, with no overrides
              applied.
            - ``obsPkg``: the ``obs_`` package for this camera.
            - ``log``: a `lsst.log` Log.
 
        Notes
        -----
        The default implementation does nothing.
        """
        pass
 

parse_args()

def lsst.pipe.base.argumentParser.ArgumentParser.parse_args	(		self,
			config,
			args = None,
			log = None,
			override = None
	)

Parse arguments for a command-line task.

Parameters
----------
config : `lsst.pex.config.Config`
    Config for the task being run.
args : `list`, optional
    Argument list; if `None` then ``sys.argv[1:]`` is used.
log : `lsst.log.Log`, optional
    `~lsst.log.Log` instance; if `None` use the default log.
override : callable, optional
    A config override function. It must take the root config object
    as its only argument and must modify the config in place.
    This function is called after camera-specific overrides files
    are applied, and before command-line config overrides
    are applied (thus allowing the user the final word).

Returns
-------
namespace : `argparse.Namespace`
    A `~argparse.Namespace` instance containing fields:

    - ``camera``: camera name.
    - ``config``: the supplied config with all overrides applied,
validated and frozen.
    - ``butler``: a `lsst.daf.persistence.Butler` for the data.
    - An entry for each of the data ID arguments registered by
`add_id_argument`, of the type passed to its ``ContainerClass``
keyword (`~lsst.pipe.base.DataIdContainer` by default). It
includes public elements ``idList`` and ``refList``.
    - ``log``: a `lsst.log` Log.
    - An entry for each command-line argument,
with the following exceptions:

      - config is the supplied config, suitably updated.
      - configfile, id and loglevel are all missing.
    - ``obsPkg``: name of the ``obs_`` package for this camera.

Definition at line 576 of file argumentParser.py.

    def parse_args(self, config, args=None, log=None, override=None):
        """Parse arguments for a command-line task.
 
        Parameters
        ----------
        config : `lsst.pex.config.Config`
            Config for the task being run.
        args : `list`, optional
            Argument list; if `None` then ``sys.argv[1:]`` is used.
        log : `lsst.log.Log`, optional
            `~lsst.log.Log` instance; if `None` use the default log.
        override : callable, optional
            A config override function. It must take the root config object
            as its only argument and must modify the config in place.
            This function is called after camera-specific overrides files
            are applied, and before command-line config overrides
            are applied (thus allowing the user the final word).
 
        Returns
        -------
        namespace : `argparse.Namespace`
            A `~argparse.Namespace` instance containing fields:
 
            - ``camera``: camera name.
            - ``config``: the supplied config with all overrides applied,
                validated and frozen.
            - ``butler``: a `lsst.daf.persistence.Butler` for the data.
            - An entry for each of the data ID arguments registered by
                `add_id_argument`, of the type passed to its ``ContainerClass``
                keyword (`~lsst.pipe.base.DataIdContainer` by default). It
                includes public elements ``idList`` and ``refList``.
            - ``log``: a `lsst.log` Log.
            - An entry for each command-line argument,
                with the following exceptions:
 
              - config is the supplied config, suitably updated.
              - configfile, id and loglevel are all missing.
            - ``obsPkg``: name of the ``obs_`` package for this camera.
        """
        if args is None:
            args = sys.argv[1:]
 
        if len(args) < 1 or args[0].startswith("-") or args[0].startswith("@"):
            self.print_help()
            if len(args) == 1 and args[0] in ("-h", "--help"):
                self.exit()
            else:
                self.exit(f"{self.prog}: error: Must specify input as first argument")
 
        # Note that --rerun may change namespace.input, but if it does
        # we verify that the new input has the same mapper class.
        namespace = argparse.Namespace()
        namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0])
        if not os.path.isdir(namespace.input):
            self.error(f"Error: input={namespace.input!r} not found")
 
        namespace.config = config
        namespace.log = log if log is not None else lsstLog.Log.getDefaultLogger()
        mapperClass = dafPersist.Butler.getMapperClass(namespace.input)
        if mapperClass is None:
            self.error(f"Error: no mapper specified for input repo {namespace.input!r}")
 
        namespace.camera = mapperClass.getCameraName()
        namespace.obsPkg = mapperClass.getPackageName()
 
        self.handleCamera(namespace)
 
        self._applyInitialOverrides(namespace)
        if override is not None:
            override(namespace.config)
 
        # Add data ID containers to namespace
        for dataIdArgument in self._dataIdArgDict.values():
            setattr(namespace, dataIdArgument.name, dataIdArgument.ContainerClass(level=dataIdArgument.level))
 
        namespace = argparse.ArgumentParser.parse_args(self, args=args, namespace=namespace)
        del namespace.configfile
 
        self._parseDirectories(namespace)
 
        if namespace.clobberOutput:
            if namespace.output is None:
                self.error("--clobber-output is only valid with --output or --rerun")
            elif namespace.output == namespace.input:
                self.error("--clobber-output is not valid when the output and input repos are the same")
            if os.path.exists(namespace.output):
                namespace.log.info("Removing output repo %s for --clobber-output", namespace.output)
                shutil.rmtree(namespace.output)
 
        namespace.log.debug("input=%s", namespace.input)
        namespace.log.debug("calib=%s", namespace.calib)
        namespace.log.debug("output=%s", namespace.output)
 
        obeyShowArgument(namespace.show, namespace.config, exit=False)
 
        # No environment variable or --output or --rerun specified.
        if self.requireOutput and namespace.output is None and namespace.rerun is None:
            self.error("no output directory specified.\n"
                       "An output directory must be specified with the --output or --rerun\n"
                       "command-line arguments.\n")
 
        butlerArgs = {}  # common arguments for butler elements
        if namespace.calib:
            butlerArgs = {'mapperArgs': {'calibRoot': namespace.calib}}
        if namespace.output:
            outputs = {'root': namespace.output, 'mode': 'rw'}
            inputs = {'root': namespace.input}
            inputs.update(butlerArgs)
            outputs.update(butlerArgs)
            namespace.butler = dafPersist.Butler(inputs=inputs, outputs=outputs)
        else:
            outputs = {'root': namespace.input, 'mode': 'rw'}
            outputs.update(butlerArgs)
            namespace.butler = dafPersist.Butler(outputs=outputs)
 
        # convert data in each of the identifier lists to proper types
        # this is done after constructing the butler,
        # hence after parsing the command line,
        # because it takes a long time to construct a butler
        self._processDataIds(namespace)
        if "data" in namespace.show:
            for dataIdName in self._dataIdArgDict.keys():
                for dataRef in getattr(namespace, dataIdName).refList:
                    print(f"{dataIdName} dataRef.dataId = {dataRef.dataId}")
 
        if namespace.show and "run" not in namespace.show:
            sys.exit(0)
 
        if namespace.debug:
            try:
                import debug
                assert debug  # silence pyflakes
            except ImportError:
                print("Warning: no 'debug' module found", file=sys.stderr)
                namespace.debug = False
 
        del namespace.loglevel
 
        if namespace.longlog:
            lsstLog.configure_prop("""
log4j.rootLogger=INFO, A1
log4j.appender.A1=ConsoleAppender
log4j.appender.A1.Target=System.out
log4j.appender.A1.layout=PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n
""")
        del namespace.longlog
 
        namespace.config.validate()
        namespace.config.freeze()
 
        return namespace
 

Member Data Documentation

requireOutput

bool lsst.pipe.base.argumentParser.ArgumentParser.requireOutput = True

static

Definition at line 432 of file argumentParser.py.

---

The documentation for this class was generated from the following file:

• /j/snowflake/release/lsstsw/stack/cb4e2dc/Linux64/pipe_base/20.0.0-25-g3dcad98+544a109665/python/lsst/pipe/base/argumentParser.py