Fiddler Evals SDK

# Step 1: Install
pip install "fiddler-evals>=0.8.0"

# Step 2: Initialize connection
from fiddler_evals import init

init(
    url='https://your-org.fiddler.ai',
    token='your-access-token'
)

# Step 3: Create project and application
from fiddler_evals import Project, Application, Dataset

project = Project.get_or_create(name='my_eval_project')
application = Application.get_or_create(
    name='my_llm_app',
    project_id=project.id
)

# Step 4: Create dataset and add test cases
from fiddler_evals.pydantic_models.dataset import NewDatasetItem

dataset = Dataset.create(
    name='experiment_dataset',
    application_id=application.id,
    description='Test cases for LLM experiments'
)

test_cases = [
    NewDatasetItem(
        inputs={"question": "What is the capital of France?"},
        expected_outputs={"answer": "Paris is the capital of France"},
        metadata={"type": "Factual", "category": "Geography"}
    ),
]
dataset = dataset.update_items(items=test_cases).dataset

# Step 5: Run evaluation
from fiddler_evals import evaluate
from fiddler_evals.evaluators import AnswerRelevance, Conciseness, Coherence

MODEL = "openai/gpt-4o"
CREDENTIAL = "your-credential-name"

def my_llm_task(inputs, extras, metadata):
    """Your LLM application logic"""
    question = inputs.get("question", "")

    # Call your LLM here
    answer = f"Mock response to: {question}"
    return {"answer": answer}

# Map evaluator parameters to your task outputs
score_mapping = {
    "user_query": lambda x: x["inputs"]["question"],
    "rag_response": "answer",
    "prompt": lambda x: x["inputs"]["question"],
    "response": "answer",
}

results = evaluate(
    dataset=dataset,
    task=my_llm_task,
    evaluators=[
        AnswerRelevance(model=MODEL, credential=CREDENTIAL),
        Conciseness(model=MODEL, credential=CREDENTIAL),
        Coherence(model=MODEL, credential=CREDENTIAL)
    ],
    name_prefix="my_experiment",
    score_fn_kwargs_mapping=score_mapping
)

# Step 6: Analyze results in Fiddler UI
print(f"✅ Evaluated {len(results.results)} test cases")

from fiddler_evals import init, evaluate, Dataset
from fiddler_evals.evaluators import (
    AnswerRelevance,
    Conciseness,
    FTLResponseFaithfulness
)

MODEL = "openai/gpt-4o"
CREDENTIAL = "your-credential-name"

# Initialize connection
init(url='https://your-org.fiddler.ai', token='your-access-token')

# Get existing dataset
dataset = Dataset.get_by_name(
    name='llm_outputs',
    application_id=application.id
)

# Define your LLM task
def evaluate_llm(inputs, extras, metadata):
    question = inputs['question']
    context = extras.get('context', '')

    # Your LLM call here
    response = my_llm_model.generate(question, context)

    return {
        "answer": response,
        "question": question,
        "context": context
    }

# Run evaluation with multiple evaluators
results = evaluate(
    dataset=dataset,
    task=evaluate_llm,
    evaluators=[
        AnswerRelevance(model=MODEL, credential=CREDENTIAL),
        Conciseness(model=MODEL, credential=CREDENTIAL),
        FTLResponseFaithfulness()  # Centor Models don't require model= parameter
    ],
    name_prefix="llm-eval",
    description="Comprehensive LLM experiment",
    metadata={"model_version": "v2.1", "environment": "production"},
    score_fn_kwargs_mapping={
        "user_query": lambda x: x["inputs"]["question"],
        "rag_response": "answer",
        "response": "answer",
        "context": "context",
    },
    max_workers=4  # Parallel processing
)

# Access results programmatically
for result in results.results:
    item = result.experiment_item
    print(f"\nTest Case: {item.dataset_item_id}")
    print(f"Status: {item.status}")
    print(f"Duration: {item.duration_ms}ms")

    for score in result.scores:
        print(f"  {score.name}: {score.value}")

from fiddler_evals.evaluators.base import Evaluator
from fiddler_evals.pydantic_models.score import Score

class LengthEvaluator(Evaluator):
    """Custom evaluator for response length"""

    def __init__(self, min_length=10, max_length=200):
        super().__init__()
        self.min_length = min_length
        self.max_length = max_length

    def score(self, output: str) -> Score:
        length = len(output.strip())

        if length < self.min_length:
            score_value = 0.0
            reasoning = f"Too short ({length} chars, min {self.min_length})"
        elif length > self.max_length:
            score_value = 0.5
            reasoning = f"Too long ({length} chars, max {self.max_length})"
        else:
            score_value = 1.0
            reasoning = f"Appropriate length ({length} chars)"

        return Score(
            name="length_check",
            evaluator_name=self.name,
            value=score_value,
            reasoning=reasoning
        )

# Use custom evaluator alongside built-in ones
results = evaluate(
    dataset=dataset,
    task=my_llm_task,
    evaluators=[
        AnswerRelevance(model="openai/gpt-4o", credential="your-credential-name"),
        Conciseness(model="openai/gpt-4o", credential="your-credential-name"),
        LengthEvaluator(min_length=15, max_length=100)
    ],
    score_fn_kwargs_mapping={
        "user_query": lambda x: x["inputs"]["question"],
        "rag_response": "answer",
        "response": "answer",
        "output": "answer",
    }
)

from fiddler_evals import items_from_csv, items_from_jsonl, items_from_pandas

# From CSV file
dataset = dataset.update_items(items=items_from_csv(
    file_path='test_cases.csv',
    input_columns=['question'],
    expected_output_columns=['answer'],
    metadata_columns=['category', 'difficulty']
)).dataset

# From JSONL file
dataset = dataset.update_items(items=items_from_jsonl(
    file_path='test_cases.jsonl',
    input_keys=['question'],
    expected_output_keys=['answer'],
    metadata_keys=['category']
)).dataset

# From pandas DataFrame
import pandas as pd

df = pd.DataFrame({
    'question': ['What is AI?', 'Explain ML'],
    'expected_answer': ['AI is...', 'ML is...'],
    'category': ['definition', 'definition']
})

dataset = dataset.update_items(items=items_from_pandas(
    df=df,
    input_columns=['question'],
    expected_output_columns=['expected_answer'],
    metadata_columns=['category']
)).dataset

dataset_v1 = Dataset.get_by_name(
    name='experiment_dataset',
    application_id=application.id,
    version=1  # the version that the first update_items() call saved; use the number you want
)
results = evaluate(
    dataset=dataset_v1,
    task=my_llm_task,
    evaluators=[Conciseness(model=MODEL, credential=CREDENTIAL)],
    name_prefix="past_version",
    score_fn_kwargs_mapping={"response": "answer"}
)

from fiddler_evals import ScoreStatus, ExperimentItemStatus

# Analyze individual results
for i, result in enumerate(results.results):
    item = result.experiment_item
    scores = result.scores

    print(f"\n📝 Test Case {i + 1}:")
    print(f"   Status: {item.status}")
    print(f"   Duration: {item.duration_ms}ms")

    if item.status == ExperimentItemStatus.SUCCESS:
        for score in scores:
            status_emoji = "✅" if score.status == ScoreStatus.SUCCESS else "❌"
            print(f"     {status_emoji} {score.name}: {score.value}")
            if score.reasoning:
                print(f"        {score.reasoning}")

# Calculate summary statistics
from collections import defaultdict

evaluator_scores = defaultdict(list)
for result in results.results:
    for score in result.scores:
        if score.value is not None:
            evaluator_scores[score.name].append(score.value)

print("\n🎯 Summary by Evaluator:")
for evaluator_name, values in evaluator_scores.items():
    avg_score = sum(values) / len(values) if values else 0
    print(f"   {evaluator_name}: {avg_score:.3f} (avg)")

# Process multiple test cases concurrently
results = evaluate(
    dataset=large_dataset,
    task=my_llm_task,
    evaluators=[
        AnswerRelevance(model="openai/gpt-4o", credential="your-credential-name"),
        Conciseness(model="openai/gpt-4o", credential="your-credential-name"),
    ],
    score_fn_kwargs_mapping=score_mapping,
    max_workers=8,  # Use 8 parallel workers
    name_prefix="parallel-eval"
)

# Track experiments with custom metadata
results = evaluate(
    dataset=dataset,
    task=my_llm_task,
    evaluators=[AnswerRelevance(model="openai/gpt-4o", credential="your-credential-name")],
    score_fn_kwargs_mapping=score_mapping,
    name_prefix="model-comparison",
    description="Comparing GPT-4 vs GPT-3.5",
    metadata={
        "model_name": "gpt-4",
        "temperature": 0.7,
        "max_tokens": 1000,
        "evaluation_date": "2024-01-15",
        "environment": "production",
        "version": "v2.1"
    }
)

# Map evaluator parameters to your task output structure
results = evaluate(
    dataset=dataset,
    task=my_llm_task,
    evaluators=[
        AnswerRelevance(model="openai/gpt-4o", credential="your-credential-name"),  # Needs: user_query, rag_response
        Conciseness(model="openai/gpt-4o", credential="your-credential-name"),      # Needs: response
    ],
    score_fn_kwargs_mapping={
        # Map evaluator parameters to task output keys
        "user_query": lambda x: x["inputs"]["question"],  # Lambda for nested values
        "rag_response": "answer",       # Simple key mapping
        "response": "answer",           # Multiple evaluators can use same output
        "context": lambda x: x["outputs"].get("context", "")  # With defaults
    }
)

# Verify installation
pip list | grep fiddler-evals

# Reinstall if needed
pip uninstall fiddler-evals
pip install "fiddler-evals>=0.8.0"

# Check Python version (requires 3.11+)
python --version

from fiddler_evals.evaluators import AnswerRelevance

evaluator = AnswerRelevance(model="openai/gpt-4o", credential="your-credential-name")
score = evaluator.score(
    user_query="What is AI?",
    rag_response="AI is artificial intelligence"
)
print(f"Score: {score.value}, Reasoning: {score.reasoning}")

# Use parallel processing
results = evaluate(
    dataset=dataset,
    task=my_llm_task,
    evaluators=[
        AnswerRelevance(model="openai/gpt-4o", credential="your-credential-name"),
        Conciseness(model="openai/gpt-4o", credential="your-credential-name"),
    ],
    score_fn_kwargs_mapping=score_mapping,
    max_workers=4  # Adjust based on your system
)

# Or process in smaller batches
for i in range(0, len(all_test_cases), 100):
    batch = all_test_cases[i:i+100]
    batch_dataset = Dataset.create(name=f"batch_{i}", application_id=application.id)
    batch_dataset = batch_dataset.update_items(items=batch).dataset
    results = evaluate(dataset=batch_dataset, task=my_llm_task, evaluators=[...], score_fn_kwargs_mapping=score_mapping)