CLI

themis-quickcheck

Primary subcommands:

• scores
• failures
• latency

Benchmark-first filters:

• --slice
• --dimension key=value

themis

The parent CLI currently exposes config-report workflows.

quickcheck

Small CLI for reading projection summaries from a Themis SQLite database.

add_quickcheck_arguments

add_quickcheck_arguments(subparsers: _SubParsersAction[Any]) -> None

Attach quickcheck subcommands to an argparse subparser collection.

PARAMETER	DESCRIPTION
subparsers	The argparse subparser collection that should receive the failures, scores, and latency commands. TYPE: _SubParsersAction[Any]

Source code in themis/cli/quickcheck.py

def add_quickcheck_arguments(subparsers: argparse._SubParsersAction[Any]) -> None:
    """Attach quickcheck subcommands to an argparse subparser collection.

    Args:
        subparsers: The argparse subparser collection that should receive the
            `failures`, `scores`, and `latency` commands.
    """

    failures = subparsers.add_parser("failures")
    failures.add_argument("--db", required=True)
    failures.add_argument("--limit", type=int, default=10)
    failures.add_argument("--transform-hash")
    failures.add_argument("--evaluation-hash")

    scores = subparsers.add_parser("scores")
    scores.add_argument("--db", required=True)
    scores.add_argument("--metric")
    scores.add_argument("--slice")
    scores.add_argument(
        "--dimension",
        action="append",
        default=[],
        help="Exact-match dimension filter in key=value form.",
    )
    scores.add_argument("--evaluation-hash")

    latency = subparsers.add_parser("latency")
    latency.add_argument("--db", required=True)
    latency.add_argument("--transform-hash")
    latency.add_argument("--evaluation-hash")

add_quickcheck_subparser

add_quickcheck_subparser(
    subparsers: _SubParsersAction[Any],
) -> argparse.ArgumentParser

Add the quickcheck command to a parent CLI.

PARAMETER	DESCRIPTION
subparsers	Parent CLI subparser collection that should receive the quickcheck command. TYPE: _SubParsersAction[Any]

RETURNS	DESCRIPTION
ArgumentParser	The configured quickcheck parser that was attached to the parent CLI.

Source code in themis/cli/quickcheck.py

def add_quickcheck_subparser(
    subparsers: argparse._SubParsersAction[Any],
) -> argparse.ArgumentParser:
    """Add the quickcheck command to a parent CLI.

    Args:
        subparsers: Parent CLI subparser collection that should receive the
            `quickcheck` command.

    Returns:
        The configured quickcheck parser that was attached to the parent CLI.
    """

    parser = subparsers.add_parser("quickcheck")
    return configure_quickcheck_parser(parser)

build_parser

build_parser(*, prog: str = 'themis-quickcheck') -> argparse.ArgumentParser

Build the quickcheck CLI parser.

PARAMETER	DESCRIPTION
prog	Program name displayed in usage text. TYPE: str DEFAULT: 'themis-quickcheck'

RETURNS	DESCRIPTION
ArgumentParser	A fully configured parser for the standalone quickcheck CLI.

Source code in themis/cli/quickcheck.py

def build_parser(*, prog: str = "themis-quickcheck") -> argparse.ArgumentParser:
    """Build the quickcheck CLI parser.

    Args:
        prog: Program name displayed in usage text.

    Returns:
        A fully configured parser for the standalone quickcheck CLI.
    """

    parser = argparse.ArgumentParser(prog=prog)
    return configure_quickcheck_parser(parser)

configure_quickcheck_parser

configure_quickcheck_parser(parser: ArgumentParser) -> argparse.ArgumentParser

Configure one parser to serve the quickcheck CLI.

PARAMETER	DESCRIPTION
parser	The parser to configure for quickcheck dispatch. TYPE: ArgumentParser

RETURNS	DESCRIPTION
ArgumentParser	The same parser after subcommands and dispatch metadata are attached.

Source code in themis/cli/quickcheck.py

def configure_quickcheck_parser(
    parser: argparse.ArgumentParser,
) -> argparse.ArgumentParser:
    """Configure one parser to serve the quickcheck CLI.

    Args:
        parser: The parser to configure for quickcheck dispatch.

    Returns:
        The same parser after subcommands and dispatch metadata are attached.
    """

    subparsers = parser.add_subparsers(dest="command", required=True)
    add_quickcheck_arguments(subparsers)
    parser.set_defaults(handler=run_with_args, _parser=parser)
    return parser

main

main(argv: list[str] | None = None) -> int

Run the quickcheck CLI and dispatch to the selected summary command.

PARAMETER	DESCRIPTION
argv	Optional argument vector to parse instead of sys.argv. TYPE: list[str] | None DEFAULT: None

RETURNS	DESCRIPTION
int	A shell-compatible exit status for the selected quickcheck command.

RAISES	DESCRIPTION
SystemExit	Propagated by argparse when invalid CLI input is supplied.

Source code in themis/cli/quickcheck.py

def main(argv: list[str] | None = None) -> int:
    """Run the quickcheck CLI and dispatch to the selected summary command.

    Args:
        argv: Optional argument vector to parse instead of `sys.argv`.

    Returns:
        A shell-compatible exit status for the selected quickcheck command.

    Raises:
        SystemExit: Propagated by argparse when invalid CLI input is supplied.
    """

    parser = build_parser()
    args = parser.parse_args(argv)
    return run_with_args(args)

run_with_args

run_with_args(args: Namespace) -> int

Execute the parsed quickcheck command.

PARAMETER	DESCRIPTION
args	Parsed quickcheck arguments including the selected subcommand and any overlay filters. TYPE: Namespace

RETURNS	DESCRIPTION
int	A shell-compatible exit status for the selected quickcheck query.

Source code in themis/cli/quickcheck.py

def run_with_args(args: argparse.Namespace) -> int:
    """Execute the parsed quickcheck command.

    Args:
        args: Parsed quickcheck arguments including the selected subcommand and
            any overlay filters.

    Returns:
        A shell-compatible exit status for the selected quickcheck query.
    """

    db_path = Path(args.db)
    with _connect(str(db_path)) as conn:
        if args.command == "failures":
            return _run_failures(
                conn,
                limit=args.limit,
                transform_hash=args.transform_hash,
                evaluation_hash=args.evaluation_hash,
            )
        if args.command == "scores":
            return _run_scores(
                conn,
                metric_id=args.metric,
                slice_id=args.slice,
                dimension_filters=args.dimension,
                evaluation_hash=args.evaluation_hash,
            )
        if args.command == "latency":
            return _run_latency(
                conn,
                transform_hash=args.transform_hash,
                evaluation_hash=args.evaluation_hash,
            )
    return 2

main

Parent CLI for Themis commands.

build_parser

build_parser() -> argparse.ArgumentParser

Build the top-level Themis CLI parser.

RETURNS	DESCRIPTION
ArgumentParser	A parser configured with the currently supported Themis subcommands.

Source code in themis/cli/main.py

def build_parser() -> argparse.ArgumentParser:
    """Build the top-level Themis CLI parser.

    Returns:
        A parser configured with the currently supported Themis subcommands.
    """

    parser = argparse.ArgumentParser(prog="themis")
    subparsers = parser.add_subparsers(dest="subcommand", required=True)

    quickcheck_cli.add_quickcheck_subparser(subparsers)
    report_cli.add_report_subparser(subparsers)
    return parser

main

main(argv: list[str] | None = None) -> int

Run the top-level Themis CLI.

PARAMETER	DESCRIPTION
argv	Optional argument vector to parse instead of sys.argv. TYPE: list[str] | None DEFAULT: None

RETURNS	DESCRIPTION
int	A shell-compatible exit status produced by the selected subcommand.

RAISES	DESCRIPTION
SystemExit	Normalized into the returned shell exit status when argparse or a subcommand exits.

Source code in themis/cli/main.py

def main(argv: list[str] | None = None) -> int:
    """Run the top-level Themis CLI.

    Args:
        argv: Optional argument vector to parse instead of `sys.argv`.

    Returns:
        A shell-compatible exit status produced by the selected subcommand.

    Raises:
        SystemExit: Normalized into the returned shell exit status when argparse
            or a subcommand exits.
    """

    parser = build_parser()
    try:
        args = parser.parse_args(argv)
    except SystemExit as exc:
        return _system_exit_code(exc.code)

    handler = getattr(args, "handler", None)
    if handler is None:
        parser.error("Unknown command.")
    try:
        return handler(args)
    except SystemExit as exc:
        return _system_exit_code(exc.code)