Datasets

from mcp_eval import Case, Dataset, ToolWasCalled, ResponseContains

cases = [
  Case(
    name="fetch_example",
    inputs="Fetch https://example.com",
    evaluators=[ToolWasCalled("fetch"), ResponseContains("Example Domain")],
  )
]

dataset = Dataset(name="Fetch Suite", cases=cases)
report = await dataset.evaluate(lambda inputs, agent, session: agent.generate_str(inputs))
report.print(include_input=True, include_output=True)

report = await dataset.evaluate(
  lambda inputs, agent, session: agent.generate_str(inputs),
  max_concurrency=4,
)

name: "Basic Fetch Dataset"
server_name: "fetch"
cases:
  - name: "simple_fetch"
    inputs: "Fetch https://example.com"
    expected_output: "Example Domain"
    evaluators:
      - ToolWasCalled:
          tool_name: "fetch"
      - ResponseContains:
          text: "Example Domain"