Skip to content

Protocols

These are the stable extension boundaries for package users.

DatasetProvider

Bases: Protocol

Query-aware dataset provider used by the benchmark-first public surface.

Source code in themis/contracts/protocols.py 
168
169
170
171
172
173
174
175
176
177
178
@runtime_checkable
class DatasetProvider(Protocol):
    """Query-aware dataset provider used by the benchmark-first public surface."""

    def scan(
        self,
        slice_spec: DatasetSliceSpec,
        query: DatasetQuerySpec,
    ) -> Sequence[DatasetItem]:
        """Return slice items after applying the requested dataset query."""
        ...

scan 

scan(
    slice_spec: DatasetSliceSpec, query: DatasetQuerySpec
) -> Sequence[DatasetItem]

Return slice items after applying the requested dataset query.

Source code in themis/contracts/protocols.py 
172
173
174
175
176
177
178
def scan(
    self,
    slice_spec: DatasetSliceSpec,
    query: DatasetQuerySpec,
) -> Sequence[DatasetItem]:
    """Return slice items after applying the requested dataset query."""
    ...

InferenceEngine

Bases: Protocol

Inference backend responsible for producing one candidate result.

Source code in themis/contracts/protocols.py 
60
61
62
63
64
65
66
67
68
69
70
71
@runtime_checkable
class InferenceEngine(Protocol):
    """Inference backend responsible for producing one candidate result."""

    def infer(
        self,
        trial: TrialSpec,
        context: DatasetContext,
        runtime: RuntimeContext,
    ) -> InferenceResult:
        """Return the raw inference output for one planned trial."""
        ...

infer 

infer(
    trial: TrialSpec, context: DatasetContext, runtime: RuntimeContext
) -> InferenceResult

Return the raw inference output for one planned trial.

Source code in themis/contracts/protocols.py 
64
65
66
67
68
69
70
71
def infer(
    self,
    trial: TrialSpec,
    context: DatasetContext,
    runtime: RuntimeContext,
) -> InferenceResult:
    """Return the raw inference output for one planned trial."""
    ...

Extractor

Bases: Protocol

Structured parser that turns raw model output into extracted data.

Source code in themis/contracts/protocols.py 
74
75
76
77
78
79
80
81
82
83
84
85
@runtime_checkable
class Extractor(Protocol):
    """Structured parser that turns raw model output into extracted data."""

    def extract(
        self,
        trial: TrialSpec,
        candidate: CandidateRecord,
        config: Mapping[str, JSONValueType] | None = None,
    ) -> ExtractionRecord:
        """Parse one candidate into a structured extraction record."""
        ...

extract 

extract(
    trial: TrialSpec,
    candidate: CandidateRecord,
    config: Mapping[str, JSONValueType] | None = None,
) -> ExtractionRecord

Parse one candidate into a structured extraction record.

Source code in themis/contracts/protocols.py 
78
79
80
81
82
83
84
85
def extract(
    self,
    trial: TrialSpec,
    candidate: CandidateRecord,
    config: Mapping[str, JSONValueType] | None = None,
) -> ExtractionRecord:
    """Parse one candidate into a structured extraction record."""
    ...

Metric

Bases: Protocol

Scorer that turns a candidate plus context into one MetricScore.

Source code in themis/contracts/protocols.py 
88
89
90
91
92
93
94
95
96
97
98
99
@runtime_checkable
class Metric(Protocol):
    """Scorer that turns a candidate plus context into one `MetricScore`."""

    def score(
        self,
        trial: TrialSpec,
        candidate: CandidateRecord,
        context: MetricContext,
    ) -> MetricScore:
        """Score one candidate against the provided metric context."""
        ...

score 

score(
    trial: TrialSpec, candidate: CandidateRecord, context: MetricContext
) -> MetricScore

Score one candidate against the provided metric context.

Source code in themis/contracts/protocols.py 
92
93
94
95
96
97
98
99
def score(
    self,
    trial: TrialSpec,
    candidate: CandidateRecord,
    context: MetricContext,
) -> MetricScore:
    """Score one candidate against the provided metric context."""
    ...

JudgeService

Bases: Protocol

Service object used by metrics that need extra judge-model calls.

Source code in themis/contracts/protocols.py 
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@runtime_checkable
class JudgeService(Protocol):
    """Service object used by metrics that need extra judge-model calls."""

    def judge(
        self,
        metric_id: str,
        parent_candidate: CandidateRecord,
        judge_spec: JudgeInferenceSpec,
        prompt: PromptTemplateSpec,
        runtime: MetricContext,
    ) -> InferenceRecord:
        """Run one judge-model call and return the resulting inference record."""
        ...

    def consume_audit_trail(self, candidate_hash: str) -> JudgeAuditTrail | None:
        """Return and clear any judge audit trail recorded for a candidate."""
        ...

consume_audit_trail 

consume_audit_trail(candidate_hash: str) -> JudgeAuditTrail | None

Return and clear any judge audit trail recorded for a candidate.

Source code in themis/contracts/protocols.py 
117
118
119
def consume_audit_trail(self, candidate_hash: str) -> JudgeAuditTrail | None:
    """Return and clear any judge audit trail recorded for a candidate."""
    ...

judge 

judge(
    metric_id: str,
    parent_candidate: CandidateRecord,
    judge_spec: JudgeInferenceSpec,
    prompt: PromptTemplateSpec,
    runtime: MetricContext,
) -> InferenceRecord

Run one judge-model call and return the resulting inference record.

Source code in themis/contracts/protocols.py 
106
107
108
109
110
111
112
113
114
115
def judge(
    self,
    metric_id: str,
    parent_candidate: CandidateRecord,
    judge_spec: JudgeInferenceSpec,
    prompt: PromptTemplateSpec,
    runtime: MetricContext,
) -> InferenceRecord:
    """Run one judge-model call and return the resulting inference record."""
    ...

PipelineHook

Bases: Protocol

Pure transforms around inference, extraction, and evaluation stages.

Source code in themis/contracts/protocols.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
@runtime_checkable
class PipelineHook(Protocol):
    """Pure transforms around inference, extraction, and evaluation stages."""

    def pre_inference(self, trial: TrialSpec, prompt: RenderedPrompt) -> RenderedPrompt:
        """Adjust the rendered prompt before inference runs."""
        ...

    def post_inference(
        self, trial: TrialSpec, result: InferenceResult
    ) -> InferenceResult:
        """Adjust the inference result before extraction begins."""
        ...

    def pre_extraction(
        self, trial: TrialSpec, candidate: CandidateRecord
    ) -> CandidateRecord:
        """Adjust a candidate before extractor execution."""
        ...

    def post_extraction(
        self, trial: TrialSpec, candidate: CandidateRecord
    ) -> CandidateRecord:
        """Adjust a candidate after extraction completes."""
        ...

    def pre_eval(self, trial: TrialSpec, candidate: CandidateRecord) -> CandidateRecord:
        """Adjust a candidate before metric scoring."""
        ...

    def post_eval(
        self, trial: TrialSpec, candidate: CandidateRecord
    ) -> CandidateRecord:
        """Adjust a candidate after metric scoring."""
        ...

post_eval 

post_eval(trial: TrialSpec, candidate: CandidateRecord) -> CandidateRecord

Adjust a candidate after metric scoring.

Source code in themis/contracts/protocols.py 
152
153
154
155
156
def post_eval(
    self, trial: TrialSpec, candidate: CandidateRecord
) -> CandidateRecord:
    """Adjust a candidate after metric scoring."""
    ...

post_extraction 

post_extraction(
    trial: TrialSpec, candidate: CandidateRecord
) -> CandidateRecord

Adjust a candidate after extraction completes.

Source code in themis/contracts/protocols.py 
142
143
144
145
146
def post_extraction(
    self, trial: TrialSpec, candidate: CandidateRecord
) -> CandidateRecord:
    """Adjust a candidate after extraction completes."""
    ...

post_inference 

post_inference(trial: TrialSpec, result: InferenceResult) -> InferenceResult

Adjust the inference result before extraction begins.

Source code in themis/contracts/protocols.py 
130
131
132
133
134
def post_inference(
    self, trial: TrialSpec, result: InferenceResult
) -> InferenceResult:
    """Adjust the inference result before extraction begins."""
    ...

pre_eval 

pre_eval(trial: TrialSpec, candidate: CandidateRecord) -> CandidateRecord

Adjust a candidate before metric scoring.

Source code in themis/contracts/protocols.py 
148
149
150
def pre_eval(self, trial: TrialSpec, candidate: CandidateRecord) -> CandidateRecord:
    """Adjust a candidate before metric scoring."""
    ...

pre_extraction 

pre_extraction(trial: TrialSpec, candidate: CandidateRecord) -> CandidateRecord

Adjust a candidate before extractor execution.

Source code in themis/contracts/protocols.py 
136
137
138
139
140
def pre_extraction(
    self, trial: TrialSpec, candidate: CandidateRecord
) -> CandidateRecord:
    """Adjust a candidate before extractor execution."""
    ...

pre_inference 

pre_inference(trial: TrialSpec, prompt: RenderedPrompt) -> RenderedPrompt

Adjust the rendered prompt before inference runs.

Source code in themis/contracts/protocols.py 
126
127
128
def pre_inference(self, trial: TrialSpec, prompt: RenderedPrompt) -> RenderedPrompt:
    """Adjust the rendered prompt before inference runs."""
    ...