Toggle Menu
Prev Up Next

# Source code for langchain.evaluation.parsing.base

    """Evaluators for parsing strings."""
    import json
    from operator import eq
    from typing import Any, Callable, Optional, Union, cast

    from langchain_core.utils.json import parse_json_markdown

    from langchain.evaluation.schema import StringEvaluator

    class JsonValidityEvaluator(StringEvaluator):
        """Evaluate whether the prediction is valid JSON.

        This evaluator checks if the prediction is a valid JSON string. It does not
            require any input or reference.

        Attributes:
            requires_input (bool): Whether this evaluator requires an input
                string. Always False.
            requires_reference (bool): Whether this evaluator requires a
                reference string. Always False.
            evaluation_name (str): The name of the evaluation metric.
                Always "json".

        Examples:
            >>> evaluator = JsonValidityEvaluator()
            >>> prediction = '{"name": "John", "age": 30, "city": "New York"}'
            >>> evaluator.evaluate(prediction)
            {'score': 1}

            >>> prediction = '{"name": "John", "age": 30, "city": "New York",}'
            >>> evaluator.evaluate(prediction)
            {'score': 0, 'reasoning': 'Expecting property name enclosed in double quotes'}
        """

        def __init__(self, **kwargs: Any) -> None:
            super().__init__()

        @property
        def requires_input(self) -> bool:
            return False

        @property
        def requires_reference(self) -> bool:
            return False

        @property
        def evaluation_name(self) -> str:
            return "json_validity"

        def _evaluate_strings(
            self,
            prediction: str,
            input: Optional[str] = None,
            reference: Optional[str] = None,
            **kwargs: Any,
        ) -> dict:
            """Evaluate the prediction string.

            Args:
                prediction (str): The prediction string to evaluate.
                input (str, optional): Not used in this evaluator. Defaults to None.
                reference (str, optional): Not used in this evaluator. Defaults to None.

            Returns:
                dict: A dictionary containing the evaluation score. The score is 1 if
                the prediction is valid JSON, and 0 otherwise.
                    If the prediction is not valid JSON, the dictionary also contains
                    a "reasoning" field with the error message.

            """
            try:
                parse_json_markdown(prediction, parser=json.loads)
                return {"score": 1}
            except Exception as e:
                return {"score": 0, "reasoning": str(e)}

    class JsonEqualityEvaluator(StringEvaluator):
        """Evaluate whether the prediction is equal to the reference after
            parsing both as JSON.

        This evaluator checks if the prediction, after parsing as JSON, is equal
            to the reference,
        which is also parsed as JSON. It does not require an input string.

        Attributes:
            requires_input (bool): Whether this evaluator requires an
                input string. Always False.
            requires_reference (bool): Whether this evaluator requires
                a reference string. Always True.
            evaluation_name (str): The name of the evaluation metric.
                Always "parsed_equality".

        Examples:
            >>> evaluator = JsonEqualityEvaluator()
            >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 1}')
            {'score': True}
            >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 2}')
            {'score': False}

            >>> evaluator = JsonEqualityEvaluator(operator=lambda x, y: x['a'] == y['a'])
            >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 1}')
            {'score': True}
            >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 2}')
            {'score': False}

        """

        def __init__(self, operator: Optional[Callable] = None, **kwargs: Any) -> None:
            super().__init__()
            self.operator = operator or eq

        @property
        def requires_input(self) -> bool:
            return False

        @property
        def requires_reference(self) -> bool:
            return True

        @property
        def evaluation_name(self) -> str:
            return "json_equality"

        def _parse_json(
            self,
            string: Any,
        ) -> Union[dict, list, None, float, bool, int, str]:
            if isinstance(string, str):
                return parse_json_markdown(string)
            return string

        def _evaluate_strings(
            self,
            prediction: str,
            input: Optional[str] = None,
            reference: Optional[str] = None,
            **kwargs: Any,
        ) -> dict:
            """Evaluate the prediction string.

            Args:
                prediction (str): The prediction string to evaluate.
                input (str, optional): Not used in this evaluator.
                reference (str): The reference string to compare against.

            Returns:
                dict: A dictionary containing the evaluation score.
            """
            parsed = self._parse_json(prediction)
            label = self._parse_json(cast(str, reference))
            if isinstance(label, list):
                if not isinstance(parsed, list):
                    return {"score": 0}
                parsed = sorted(parsed, key=lambda x: str(x))
                label = sorted(label, key=lambda x: str(x))
            return {"score": self.operator(parsed, label)}