CustomJudgeSpec

from fiddler_evals.evaluators.custom_judge import (
    CustomJudge, CustomJudgeSpec, Message, InputFieldSpec,
    OutputFieldTransform,
)

FAITHFULNESS_SPEC = CustomJudgeSpec(
    prompt_template=[
        Message(role='system', content='Judge faithfulness.'),
        Message(role='user', content=(
            'Question: {{ query }}\n'
            'Documents: {{ docs }}\n'
            'Answer: {{ answer }}'
        )),
    ],
    inputs={
        'query': InputFieldSpec(required=True),
        'docs': InputFieldSpec(required=True),
        'answer': InputFieldSpec(required=True),
    },
    llm_response_fields={
        'is_faithful': {
            'type': 'string',
            'choices': ['faithful', 'not_faithful'],
        },
    },
    output_fields={
        'label': {
            'type': 'string',
            'choices': ['yes', 'no'],
            'transform': OutputFieldTransform(
                source_field='is_faithful',
                value_map={
                    'faithful': 'yes',
                    'not_faithful': 'no',
                },
            ),
        },
    },
)

evaluator = CustomJudge(
    prompt_spec=FAITHFULNESS_SPEC,
    model='openai/gpt-4o',
)