Building Custom Evaluators with LLM in Code

(
    HallucinationEvaluator,
    RelevanceEvaluator,
    ToxicityEvaluator,
    QAEvaluator,
    SummarizationEvaluator,
) = map(
    lambda args: _create_llm_evaluator_subclass(*args),
    (
        (
            "HallucinationEvaluator",
            EvalCriteria.HALLUCINATION.value,
            'Leverages an LLM to evaluate whether a response (stored under an "output" column) is a hallucination given a query (stored under an "input" column) and one or more retrieved documents (stored under a "reference" column).',  # noqa: E501
        ),
        (
            "RelevanceEvaluator",
            EvalCriteria.RELEVANCE.value,
            'Leverages an LLM to evaluate whether a retrieved document (stored under a "reference" column) is relevant or irrelevant to the corresponding query (stored under the "input" column).',  # noqa: E501
        ),
        (
            "ToxicityEvaluator",
            EvalCriteria.TOXICITY.value,
            'Leverages an LLM to evaluate whether the string stored under the "input" column contains racist, sexist, chauvinistic, biased, or otherwise toxic content.',  # noqa: E501
        ),
        (
            "QAEvaluator",
            EvalCriteria.QA.value,
            'Leverages an LLM to evaluate whether a response (stored under an "output" column) is correct or incorrect given a query (stored under an "input" column) and one or more retrieved documents (stored under a "reference" column).',  # noqa: E501
        ),
        (
            "SummarizationEvaluator",
            EvalCriteria.SUMMARIZATION.value,
            'Leverages an LLM to evaluate whether a summary (stored under an "output" column) provides an accurate synopsis of an input document (stored under a "input" column).',  # noqa: E501
        )
    ),
)