mcp-eval Documentation

# Install mcp-eval globally (for CLI)
uv tool install mcpevals

# Add mcp-eval dependency to your project
uv add mcpevals

# Initialize your project (interactive setup)
mcp-eval init

# Add your MCP server to test
mcp-eval server add

# Auto-generate tests with an LLM
mcp-eval generate

# Run decorator/dataset tests
mcp-eval run tests/

# Run pytest tests (use pytest)
uv run pytest -q tests

# Install mcp-eval
pip install mcpevals

# Initialize your project
mcp-eval init

# Add your MCP server
mcp-eval server add

# Run decorator/dataset tests
mcp-eval run tests/

# Run pytest tests (use pytest)
pytest -q tests

from mcp_eval import task, Expect

@task("Verify fetch server works correctly")
async def test_fetch(agent, session):
    # Ask the agent to fetch a webpage
    response = await agent.generate_str("Fetch https://example.com and summarize it")
    
    # Assert the right tool was called
    await session.assert_that(Expect.tools.was_called("fetch"))
    
    # Verify the content is correct
    await session.assert_that(Expect.content.contains("Example Domain"), response=response)
    
    # Check performance
    await session.assert_that(Expect.performance.response_time_under(5000))

import pytest
from mcp_eval import create_agent, Expect

@pytest.mark.asyncio
async def test_fetch_with_pytest():
    agent = await create_agent("claude-3-5-sonnet")
    response = await agent.generate_str("Fetch https://example.com")
    
    assert "Example Domain" in response
    assert agent.tools_called == ["fetch"]