Skip to content

First persisted run

What you will build

You will run Themis against SQLite, reopen the stored state, and export a report without regenerating the run.

Prerequisites

  • familiarity with Experiment(...)
  • writable local filesystem

Steps

  1. Create a SQLite-backed experiment.
  2. Execute it with an explicit store.
  3. Inspect execution state and export a Markdown report.
from __future__ import annotations

from pathlib import Path

from themis import (
    Experiment,
    Reporter,
    get_execution_state,
    get_run_snapshot,
    sqlite_store,
)
from themis.core.config import EvaluationConfig, GenerationConfig, StorageConfig
from themis.core.models import Case, Dataset


def run_example(root: Path) -> dict[str, object]:
    """Run against SQLite, inspect the stored state, and export a report."""

    store_path = root / "runs" / "themis.sqlite3"
    store_path.parent.mkdir(parents=True, exist_ok=True)
    store = sqlite_store(store_path)
    experiment = Experiment(
        generation=GenerationConfig(
            generator="builtin/demo_generator",
            candidate_policy={"num_samples": 1},
            reducer="builtin/majority_vote",
        ),
        evaluation=EvaluationConfig(
            metrics=["builtin/exact_match"],
            parsers=["builtin/json_identity"],
        ),
        storage=StorageConfig(store="sqlite", parameters={"path": str(store_path)}),
        datasets=[
            Dataset(
                dataset_id="sample",
                cases=[
                    Case(
                        case_id="case-1",
                        input={"question": "2+2"},
                        expected_output={"answer": "4"},
                    )
                ],
            )
        ],
        seeds=[7],
    )

    result = experiment.run(store=store)
    snapshot = get_run_snapshot(store, result.run_id)
    state = get_execution_state(store, result.run_id)
    report = Reporter(store).export_markdown(result.run_id)
    return {
        "run_id": result.run_id,
        "status": result.status.value,
        "store_path": str(store_path),
        "snapshot_run_id": snapshot.run_id,
        "state_status": state.status.value,
        "report_preview": report.splitlines()[:4],
    }


if __name__ == "__main__":
    print(run_example(Path(".")))

Expected results

Inspect after the run:

  • the SQLite file exists under runs/
  • get_execution_state(...) returns the persisted run state
  • Reporter.export_markdown(...) gives you a portable report without re-running the experiment

Common failure points

  • using memory storage when later inspection is required
  • forgetting that report/export/compare workflows need persisted state

Next steps