Giskard v3 教程：从知识库建立问答回归测试

python3.12 -m venv .venv
source .venv/bin/activate
python -m pip install "giskard==3.0.1" "giskard-checks==1.0.6"
python -c "from importlib.metadata import version; print(version('giskard'))"

"""Four local fixtures, verified with Giskard 3.0.1 / Checks 1.0.6."""
import asyncio
import json
from pathlib import Path

POLICY = {
    "version": "demo-policy-2026-10",
    "source_id": "POLICY-DEMO-731",
    "plan_price_cny": 99,
    "refund_requires_review": True,
}

BASE_ANSWER = {
    "plan_price_cny": 99,
    "citations": ["POLICY-DEMO-731"],
    "refund_guaranteed": False,
    "needs_human_review": True,
}

CASES = [
    {"name": "correct", "answer": BASE_ANSWER.copy(), "expected_pass": True},
    {"name": "wrong_amount", "answer": {**BASE_ANSWER, "plan_price_cny": 199}, "expected_pass": False},
    {"name": "missing_source", "answer": {**BASE_ANSWER, "citations": []}, "expected_pass": False},
    {"name": "unsupported_promise", "answer": {**BASE_ANSWER, "refund_guaranteed": True}, "expected_pass": False},
]


def inspect_answer(answer):
    """Only checks explicit fields; it does not understand free text."""
    return {
        "price_matches": type(answer.get("plan_price_cny")) is int
        and answer["plan_price_cny"] == POLICY["plan_price_cny"],
        "source_present": isinstance(answer.get("citations"), list)
        and POLICY["source_id"] in answer["citations"],
        "no_refund_guarantee": answer.get("refund_guaranteed") is False,
        "review_required": answer.get("needs_human_review") is True,
    }


async def main():
    from giskard.checks import Equals, Scenario
    from importlib.metadata import version

    expected = {key: True for key in inspect_answer(BASE_ANSWER)}
    rows = []
    sdk_results = []
    for case in CASES:
        # This callable is a local fixture, not a live RAG application.
        scenario = (
            Scenario(case["name"])
            .interact(inputs=case["answer"], outputs=lambda inputs: inspect_answer(inputs))
            .check(Equals(
                name="structured_policy_requirements",
                expected_value=expected,
                target_key="trace.last.outputs",
            ))
        )
        result = await scenario.run()
        result.print_report()
        sdk_results.append(result.model_dump(mode="json"))
        # Save local field checks separately from Giskard's printed report.
        checks = inspect_answer(case["answer"])
        rows.append({
            "case": case["name"],
            "policy_version": POLICY["version"],
            "field_checks": checks,
            "fixture_expected_pass": case["expected_pass"],
            "local_field_gate": all(checks.values()),
            "sdk_status": result.status.value,
        })
        if result.errored or result.skipped:
            raise RuntimeError(f"{case['name']}: SDK execution did not complete normally")
        if result.passed != case["expected_pass"]:
            raise AssertionError(f"{case['name']}: unexpected SDK result {result.status}")
    Path("fixture-field-checks.json").write_text(
        json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8"
    )
    Path("sdk-results.json").write_text(
        json.dumps({"giskard": version("giskard"),
                    "giskard_checks": version("giskard-checks"),
                    "results": sdk_results}, ensure_ascii=False, indent=2),
        encoding="utf-8",
    )


if __name__ == "__main__":
    asyncio.run(main())

python regression_demo.py

python -m pip freeze > requirements.lock.txt
python -c "import sys; print(sys.version)"