Error in Custom Evaluator: Returning EvaluationResult instead of Dict
getting an issue where I've defined a custom evaluator and am returning an EvaluationResult object but get an error because a dict is expected
"""
Accuracy evaluation using structured AI output.
"""
from typing import Any, Dict
from phoenix.experiments.evaluators import create_evaluator
from phoenix.experiments.types import EvaluationResult
from src.ai.prompt_runner import PromptRunner
@create_evaluator(kind="llm")
async def accuracy(input: Dict[str, Any], output: str, expected: str) -> Dict[str, Any]:
"""
Evaluate accuracy using structured AI output.
Args:
input: Input data containing doc_path
output: Generated summary string from the task
expected: Expected summary string
Returns:
Dict with label, explanation, and score
"""
prompt_runner = PromptRunner()
# Handle case where expected might be a dict or string
reference_answer = (
expected
if isinstance(expected, str)
else expected.get("expected_summary", str(expected))
)
eval_result = await prompt_runner.evaluate_accuracy(
input_text=input["doc_path"],
reference_answer=reference_answer,
answer=output,
)
return EvaluationResult(
label=eval_result.label,
explanation=eval_result.explanation,
score=eval_result.score,
)code where it errors out:
if result:
# Filter out None values for OpenTelemetry attributes
attributes: dict[str, Any] = {}
if (score := result.get("score")) is not None:
attributes["evaluation.score"] = score
if (label := result.get("label")) is not None:
attributes["evaluation.label"] = label
if attributes:
span.set_attributes(attributes)
span.set_attribute(OPENINFERENCE_SPAN_KIND, EVALUATOR)
span.set_status(status)error:
AttributeError: 'EvaluationResult' object has no attribute 'get'before I was returning a dict rather than an EvaluationResult object but ended up with a different issue
