Pranav P.

"""
Accuracy evaluation using structured AI output.
"""

from typing import Any, Dict
from phoenix.experiments.evaluators import create_evaluator
from phoenix.experiments.types import EvaluationResult
from src.ai.prompt_runner import PromptRunner


@create_evaluator(kind="llm")
async def accuracy(input: Dict[str, Any], output: str, expected: str) -> Dict[str, Any]:
    """
    Evaluate accuracy using structured AI output.

    Args:
        input: Input data containing doc_path
        output: Generated summary string from the task
        expected: Expected summary string

    Returns:
        Dict with label, explanation, and score
    """

    prompt_runner = PromptRunner()

    # Handle case where expected might be a dict or string
    reference_answer = (
        expected
        if isinstance(expected, str)
        else expected.get("expected_summary", str(expected))
    )

    eval_result = await prompt_runner.evaluate_accuracy(
        input_text=input["doc_path"],
        reference_answer=reference_answer,
        answer=output,
    )

    return EvaluationResult(
        label=eval_result.label,
        explanation=eval_result.explanation,
        score=eval_result.score,
    )

if result:
                # Filter out None values for OpenTelemetry attributes
                attributes: dict[str, Any] = {}
                if (score := result.get("score")) is not None:
                    attributes["evaluation.score"] = score
                if (label := result.get("label")) is not None:
                    attributes["evaluation.label"] = label
                if attributes:
                    span.set_attributes(attributes)
            span.set_attribute(OPENINFERENCE_SPAN_KIND, EVALUATOR)
            span.set_status(status)

AttributeError: 'EvaluationResult' object has no attribute 'get'

{'RGF0YXNldEV4YW1wbGU6Mw==': Example(
    id="RGF0YXNldEV4YW1wbGU6Mw==",
    [1m[94minput[0m={
        "doc_path": "test_data/409A Valuation Report.pdf"
    },
    [1m[94moutput[0m={
        "expected_summary": "This is a 409A valuation report document that..."
    },
    [1m[94mmetadata[0m={
        "metadata": {
            "topic": "financial"
        }
    },
), 'RGF0YXNldEV4YW1wbGU6NA==': Example(
    id="RGF0YXNldEV4YW1wbGU6NA==",
    [1m[94minput[0m={
        "doc_path": "test_data/409A Valuation Report.pdf"
    },
    [1m[94moutput[0m={
        "expected_summary": "This is a 409A valuation report document that..."
    },
    [1m[94mmetadata[0m={
        "metadata": {
            "topic": "financial"
        }
    },
)}

    df = pd.DataFrame(
        [
            {
                "doc_path": pdf_path,
                "expected_summary": "This is a 409A valuation report document that provides a detailed analysis and valuation of company shares for tax and compliance purposes.",
                "metadata": {"topic": "financial"},
            },
            {
                "doc_path": pdf_path,
                "expected_summary": "This is a 409A valuation report document that provides a detailed analysis and valuation of company shares for tax and compliance purposes.",
                "metadata": {"topic": "financial"},
            },
        ]
    )

    return phoenix_client.upload_dataset(
        dataframe=df,
        dataset_name=dataset_name,
        input_keys=["doc_path"],
        output_keys=["expected_summary"],
        metadata_keys=["metadata"],
    )

cache_key = (example["id"], repetition_number)
TypeError: string indices must be integers, not 'str'

because Example is just
example
'RGF0YXNldEV4YW1wbGU6Mw=='