Evaluator SDK

pip install failproofai-sdk

from failproofai_sdk.evaluator import ConditionResult, EvalResult, Evaluator, Metric, Score

app = Evaluator(name="customer-production", version="2026.08.1")


@app.eval(
    "tool_efficiency",
    version="1.0.0",
    labels=["tools", "deterministic"],
    when=lambda session: ConditionResult(session.count("tool_use") > 0, "no_tool_calls"),
)
def tool_efficiency(session):
    calls = session.events_of_type("tool_use")
    distinct = {e.payload.get("tool_name") for e in calls if e.payload.get("tool_name")}
    value = len(distinct) / len(calls)
    return EvalResult(
        score=Score(value, passed=value >= 0.7),
        metrics={"tool_call_count": Metric(len(calls), unit="events")},
        reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls",
    )


@app.eval(
    "answer_relevance",
    version="judge-v1",
    labels=["llm_judge", "relevance"],
    when=lambda session: ConditionResult(
        session.count("human_input") > 0 and session.count("model_response") > 0,
        "no_exchange",
    ),
    timeout_seconds=30,
)
async def answer_relevance(session):
    question = session.events_of_type("human_input")[-1].payload.get("response")
    answer = session.events_of_type("model_response")[-1].payload.get("content")
    value, reasoning = await ask_judge(question, answer)  # your LLM call: a 0-1 score and why
    return EvalResult(score=Score(value, passed=value >= 0.7), reasoning=reasoning)


if __name__ == "__main__":
    app.run_from_env()

FAILPROOFAI_EVALUATOR_URL=https://app.befailproof.ai python evaluator.py