API: Catalog

from mcp_eval.catalog import Expect

# Content assertions
Expect.content.contains("success")
Expect.content.regex(r"\d+ results?")

# Tool assertions
Expect.tools.was_called("fetch")
Expect.tools.success_rate(min_rate=0.95)
Expect.tools.sequence(["validate", "process", "format"])

# Performance assertions
Expect.performance.max_iterations(3)
Expect.performance.response_time_under(5000)

# LLM judge assertions
Expect.judge.llm("Must be professional and accurate")
Expect.judge.multi_criteria([criterion1, criterion2])

# Path efficiency assertions
Expect.path.efficiency(
    expected_tool_sequence=["fetch", "parse"],
    allow_extra_steps=1
)

# Check if content contains a substring
await session.assert_that(
    Expect.content.contains("Example Domain"),
    response=response,
    name="has_expected_text"
)

# Exact match (use regex or precise substring)
await session.assert_that(
    Expect.content.regex(r"^The answer is 42$"),
    response=response
)

# Regular expression matching
await session.assert_that(
    Expect.content.regex(r"Temperature: \d+°[CF]"),
    response=response,
    name="has_temperature"
)

# Case-insensitive matching
await session.assert_that(
    Expect.content.contains("SUCCESS", case_sensitive=False)
)

# Multiple conditions (all must pass)
for expected in ["result", "complete", "successful"]:
    await session.assert_that(
        Expect.content.contains(expected),
        response=response
    )

# Verify a specific tool was called
await session.assert_that(
    Expect.tools.was_called("calculator"),
    name="used_calculator"
)

# Check tool wasn't called (negative assertion)
await session.assert_that(
    Expect.tools.count("dangerous_tool", 0),
    name="safety_check"
)

# Verify exact sequence of tools
await session.assert_that(
    Expect.tools.sequence(["auth", "fetch", "parse"]),
    name="correct_order"
)

# Check tool was called with specific arguments
await session.assert_that(
    Expect.tools.called_with(
        tool_name="fetch",
        expected_args={"url": "https://api.example.com"}
    )
)

# Verify success rate across all tool calls
await session.assert_that(
    Expect.tools.success_rate(min_rate=0.95),
    name="high_reliability"
)

# Check specific tool's output
await session.assert_that(
    Expect.tools.output_matches(
        tool_name="weather_api",
        expected_output="sunny",
        match_type="contains"  # or "exact", "regex", "partial"
    )
)

# Maximum response time in milliseconds
await session.assert_that(
    Expect.performance.response_time_under(5000),
    name="fast_response"
)

# Limit conversation iterations
await session.assert_that(
    Expect.performance.max_iterations(3),
    name="efficient_solution"
)

# Inspect tokens/cost programmatically via session.get_metrics()
metrics = session.get_metrics()

# LLM token usage (if available)
if metrics.llm_metrics:
    print(f"Input tokens: {metrics.llm_metrics.input_tokens}")
    print(f"Output tokens: {metrics.llm_metrics.output_tokens}")
    print(f"Total tokens: {metrics.llm_metrics.total_tokens}")

# Cost estimate (if computed)
print(f"Estimated cost: ${metrics.cost_estimate:.4f}")

# Basic quality check
await session.assert_that(
    Expect.judge.llm(
        rubric="""
        The response should:
        - Be professional and courteous
        - Provide accurate information
        - Be concise (under 100 words)
        """,
        min_score=0.8,
        include_input=True  # Give judge full context
    ),
    response=response,
    name="quality_check"
)

from mcp_eval.evaluators import EvaluationCriterion

# Define weighted criteria
criteria = [
    EvaluationCriterion(
        name="accuracy",
        description="All facts are correct",
        weight=3.0,  # Most important
        min_score=0.9
    ),
    EvaluationCriterion(
        name="completeness",
        description="Addresses all user requirements",
        weight=2.0,
        min_score=0.8
    ),
    EvaluationCriterion(
        name="clarity",
        description="Easy to understand",
        weight=1.0,
        min_score=0.7
    )
]

# Apply multi-criteria judge
await session.assert_that(
    Expect.judge.multi_criteria(
        criteria=criteria,
        aggregate_method="weighted",  # or "min", "average"
        require_all_pass=False,
        use_cot=True  # Chain-of-thought reasoning
    ),
    response=response,
    name="comprehensive_quality"
)

# Use specific model for judging
judge = Expect.judge.llm(
    rubric="Evaluate for technical accuracy",
    min_score=0.85,
    model="claude-3-opus-20240229"
)

await session.assert_that(judge, response=response)

# Define the ideal execution path
await session.assert_that(
    Expect.path.efficiency(
        expected_tool_sequence=["validate", "process", "save"],
        tool_usage_limits={
            "validate": 1,  # Should validate only once
            "process": 1,   # Process only once
            "save": 1       # Save only once
        },
        allow_extra_steps=0,
        penalize_backtracking=True
    ),
    name="golden_path"
)

# Allow some variation while ensuring key waypoints
await session.assert_that(
    # For waypoint/pattern checks, combine Expect.tools.sequence and
    # Expect.tools.count along with Expect.path.efficiency
)

@task("Complete workflow test")
async def test_full_workflow(agent, session):
    """Test the entire user journey with multiple checkpoints."""
    
    response = await agent.generate_str(
        "Fetch weather for NYC and format it nicely"
    )
    
    # Content quality
    await session.assert_that(
        Expect.content.regex(r"\d+°[CF]"),
        response=response,
        name="has_temperature"
    )
    
    # Tool usage
    await session.assert_that(
        Expect.tools.sequence(["weather_api", "format_tool"]),
        name="correct_tools"
    )
    
    # Performance
    await session.assert_that(
        Expect.performance.response_time_under(3000),
        name="fast_enough"
    )
    
    # Quality judgment
    await session.assert_that(
        Expect.judge.llm(
            "Output should be well-formatted and user-friendly",
            min_score=0.8
        ),
        response=response,
        name="quality_output"
    )