Mcp eval test generator

# Generate 10 pytest-style tests
mcp-eval generate \
  --style pytest \
  --n-examples 10 \
  --provider anthropic \
  --model claude-3-5-sonnet-20241022

# Generate decorator-style tests
mcp-eval generate \
  --style decorators \
  --n-examples 8 \
  --output tests/generated_tests.py

# Generate dataset tests
mcp-eval generate \
  --style dataset \
  --n-examples 15 \
  --refine  # Add additional assertions

# Generate with specific server
mcp-eval generate \
  --server-name my_server \
  --style pytest \
  --n-examples 10 \
  --extra-instructions "Focus on error handling and edge cases"

# Update existing test file
mcp-eval generate \
  --update tests/test_basic.py \
  --style pytest \
  --n-examples 5 \
  --provider anthropic

# Generate from discovered tools
mcp-eval generate \
  --discover-tools \
  --style decorators \
  --n-examples 12

ScenarioSpec(
    name="test_basic_functionality",
    description="Tests basic tool usage",
    prompt="User-facing prompt for the agent",
    expected_output="Optional expected result",
    assertions=[
        ToolWasCalledSpec(tool_name="fetch", min_times=1),
        ResponseContainsSpec(text="success", case_sensitive=False),
        LLMJudgeSpec(rubric="Quality evaluation criteria", min_score=0.8)
    ]
)

# Tool assertions
ToolWasCalledSpec(kind="tool_was_called", tool_name="fetch", min_times=1)
ToolCalledWithSpec(kind="tool_called_with", tool_name="fetch", arguments={"url": "..."})
ToolOutputMatchesSpec(
    kind="tool_output_matches",
    tool_name="fetch",
    expected_output="data",
    match_type="contains"  # exact|contains|regex|partial
)

# Content assertions
ResponseContainsSpec(kind="response_contains", text="expected", case_sensitive=False)
NotContainsSpec(kind="not_contains", text="forbidden", case_sensitive=False)

# Performance assertions  
MaxIterationsSpec(kind="max_iterations", max_iterations=3)
ResponseTimeUnderSpec(kind="response_time_under", ms=5000)

# Judge assertions
LLMJudgeSpec(kind="llm_judge", rubric="Evaluation criteria", min_score=0.8)

# Sequence assertions
ToolSequenceSpec(
    kind="tool_sequence",
    sequence=["validate", "process", "save"],
    allow_other_calls=False
)

"""Generated tests for {{ server_name }} MCP server."""

import pytest
from mcp_eval import Expect
from mcp_eval.session import TestAgent

{% for scenario in scenarios %}
@pytest.mark.asyncio
async def {{ scenario.name|py_ident }}(mcp_agent: TestAgent):
    """{{ scenario.description or scenario.name }}"""
    response = await mcp_agent.generate_str({{ scenario.prompt|py }})
    
    {% for assertion in scenario.assertions %}
    await mcp_agent.session.assert_that(
        {{ render_assertion(assertion) }},
        name="{{ assertion_name(assertion) }}"
    )
    {% endfor %}
{% endfor %}

"""Generated tests for {{ server_name }} MCP server."""

from mcp_eval import task, setup, Expect
from mcp_eval.session import TestAgent, TestSession

@setup
def configure():
    """Setup for generated tests."""
    pass

{% for scenario in scenarios %}
@task({{ scenario.name|py }})
async def {{ scenario.name|py_ident }}(agent: TestAgent, session: TestSession):
    """{{ scenario.description or scenario.name }}"""
    response = await agent.generate_str({{ scenario.prompt|py }})
    
    {% for assertion in scenario.assertions %}
    await session.assert_that(
        {{ render_assertion(assertion) }},
        name={{ assertion_name(assertion)|py }},
        response=response
    )
    {% endfor %}
{% endfor %}

# List available tools
mcp-eval server list --verbose

# Use discovered tools for generation
mcp-eval generate --discover-tools --style pytest

# Generate initial tests
mcp-eval generate --n-examples 10 --output tests/generated.py

# Refine with additional assertions
mcp-eval generate --refine --target-file tests/generated.py

# Add custom scenarios
mcp-eval update --target-file tests/generated.py --n-examples 5

extra_instructions = """
Focus on:
1. Error handling scenarios
2. Performance under load  
3. Edge cases with malformed input
4. Security considerations
5. Multi-tool workflows
"""

# Use in generation
mcp-eval generate \
  --extra-instructions "$extra_instructions" \
  --n-examples 15

# Generate comprehensive test suite
mcp-eval generate \
  --server-name fetch \
  --style pytest \
  --n-examples 12 \
  --extra-instructions "Include tests for various URL types, error handling, and content extraction"

# Generated scenarios will include:
# - Basic URL fetching
# - Invalid URL handling
# - Different content types (HTML, JSON, etc.)
# - Large content handling
# - Timeout scenarios
# - Concurrent fetches

mcp-eval generate \
  --server-name calculator \
  --style decorators \
  --n-examples 10 \
  --extra-instructions "Test all operations, edge cases like division by zero, and operation chaining"

# Generated scenarios:
# - Basic arithmetic (add, subtract, multiply, divide)
# - Division by zero handling
# - Large number operations
# - Decimal precision
# - Operation sequences
# - Invalid input handling

mcp-eval generate \
  --style dataset \
  --n-examples 20 \
  --server-name database \
  --extra-instructions "Create diverse query patterns and data manipulation scenarios"

# Creates Dataset with cases for:
# - SELECT queries
# - INSERT operations
# - UPDATE statements
# - DELETE operations
# - Transaction handling
# - Query errors

@setup
def prepare_test_data():
    """Add test data preparation"""
    create_test_files()
    
@teardown
def cleanup_test_data():
    """Clean up after tests"""
    remove_test_files()

# Add to generated test
metrics = session.get_metrics()
assert metrics.cost_estimate < 0.10, "Cost exceeded budget"
assert len(metrics.tool_calls) <= 5, "Too many tool calls"

@pytest.mark.parametrize("url,expected", [
    ("https://example.com", "Example Domain"),
    ("https://httpbin.org/json", "slideshow"),
])
async def test_parametrized(mcp_agent, url, expected):
    # Enhanced generated test with parameters
    pass