Feedback Function APIs

Below are out of the box feedback functions and how to instantiate them. To use them in the trulens framework, see the application wrappers.

Langchain Wrapper: TruChain

Llama Index Wrapper: TruLLama

Basic Text-to-Text Wrapper: TruBasicApp

Any Custom App Wrapper: TruCustomApp

AzureOpenAI

Bases: OpenAI

Out of the box feedback functions calling AzureOpenAI APIs. Has the same functionality as OpenAI out of the box feedback functions.

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

class AzureOpenAI(OpenAI):
    """Out of the box feedback functions calling AzureOpenAI APIs. 
    Has the same functionality as OpenAI out of the box feedback functions.
    """
    deployment_id: str

    def __init__(self, endpoint=None, **kwargs):
        # NOTE(piotrm): pydantic adds endpoint to the signature of this
        # constructor if we don't include it explicitly, even though we set it
        # down below. Adding it as None here as a temporary hack.
        """
        Wrapper to use Azure OpenAI. Please export the following env variables

        - OPENAI_API_BASE
        - OPENAI_API_VERSION
        - OPENAI_API_KEY

        **Usage:**
        ```
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = AzureOpenAI(deployment_id="...")

        ```


        Args:
            model_engine (str, optional): The specific model version. Defaults to "gpt-35-turbo".
            deployment_id (str): The specified deployment id
            endpoint (Endpoint): Internal Usage for DB serialization
        """

        super().__init__(
            **kwargs
        )  # need to include pydantic.BaseModel.__init__

        set_openai_key()
        openai.api_type = "azure"
        openai.api_base = os.getenv("OPENAI_API_BASE")
        openai.api_version = os.getenv("OPENAI_API_VERSION")

    def _create_chat_completion(self, *args, **kwargs):
        """
        We need to pass `engine`
        """
        return super()._create_chat_completion(
            *args, deployment_id=self.deployment_id, **kwargs
        )

__init__(endpoint=None, **kwargs)

Wrapper to use Azure OpenAI. Please export the following env variables

• OPENAI_API_BASE
• OPENAI_API_VERSION
• OPENAI_API_KEY

Usage:

from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = AzureOpenAI(deployment_id="...")

Parameters:

Name	Type	Description	Default
model_engine	str	The specific model version. Defaults to "gpt-35-turbo".	required
deployment_id	str	The specified deployment id	required
endpoint	Endpoint	Internal Usage for DB serialization	None

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def __init__(self, endpoint=None, **kwargs):
    # NOTE(piotrm): pydantic adds endpoint to the signature of this
    # constructor if we don't include it explicitly, even though we set it
    # down below. Adding it as None here as a temporary hack.
    """
    Wrapper to use Azure OpenAI. Please export the following env variables

    - OPENAI_API_BASE
    - OPENAI_API_VERSION
    - OPENAI_API_KEY

    **Usage:**
    ```
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = AzureOpenAI(deployment_id="...")

    ```


    Args:
        model_engine (str, optional): The specific model version. Defaults to "gpt-35-turbo".
        deployment_id (str): The specified deployment id
        endpoint (Endpoint): Internal Usage for DB serialization
    """

    super().__init__(
        **kwargs
    )  # need to include pydantic.BaseModel.__init__

    set_openai_key()
    openai.api_type = "azure"
    openai.api_base = os.getenv("OPENAI_API_BASE")
    openai.api_version = os.getenv("OPENAI_API_VERSION")

OpenAI

Bases: Provider

Out of the box feedback functions calling OpenAI APIs.

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

class OpenAI(Provider):
    """Out of the box feedback functions calling OpenAI APIs.
    """
    model_engine: str
    endpoint: Endpoint

    def __init__(
        self, *args, endpoint=None, model_engine="gpt-3.5-turbo", **kwargs
    ):
        # NOTE(piotrm): pydantic adds endpoint to the signature of this
        # constructor if we don't include it explicitly, even though we set it
        # down below. Adding it as None here as a temporary hack.
        """
        Create an OpenAI Provider with out of the box feedback functions.

        **Usage:**
        ```
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        ```

        Args:
            model_engine (str): The OpenAI completion model. Defaults to `gpt-3.5-turbo`
            endpoint (Endpoint): Internal Usage for DB serialization
        """
        # TODO: why was self_kwargs required here independently of kwargs?
        self_kwargs = dict()
        self_kwargs.update(**kwargs)
        self_kwargs['model_engine'] = model_engine
        self_kwargs['endpoint'] = OpenAIEndpoint(*args, **kwargs)

        super().__init__(
            **self_kwargs
        )  # need to include pydantic.BaseModel.__init__

        set_openai_key()

    def _create_chat_completion(self, *args, **kwargs):
        return openai.ChatCompletion.create(*args, **kwargs)

    def _moderation(self, text: str):
        return self.endpoint.run_me(
            lambda: openai.Moderation.create(input=text)
        )

    def moderation_not_hate(self, text: str) -> float:
        """
        Uses OpenAI's Moderation API. A function that checks if text is hate
        speech.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.moderation_not_hate).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "hate" and 1 being "not
            hate".
        """
        openai_response = self._moderation(text)
        return 1 - float(
            openai_response["results"][0]["category_scores"]["hate"]
        )

    def moderation_not_hatethreatening(self, text: str) -> float:
        """
        Uses OpenAI's Moderation API. A function that checks if text is
        threatening speech.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.moderation_not_hatethreatening).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "threatening" and 1 being
            "not threatening".
        """
        openai_response = self._moderation(text)

        return 1 - int(
            openai_response["results"][0]["category_scores"]["hate/threatening"]
        )

    def moderation_not_selfharm(self, text: str) -> float:
        """
        Uses OpenAI's Moderation API. A function that checks if text is about
        self harm.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.moderation_not_selfharm).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "self harm" and 1 being "not
            self harm".
        """
        openai_response = self._moderation(text)

        return 1 - int(
            openai_response["results"][0]["category_scores"]["self-harm"]
        )

    def moderation_not_sexual(self, text: str) -> float:
        """
        Uses OpenAI's Moderation API. A function that checks if text is sexual
        speech.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.moderation_not_sexual).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "sexual" and 1 being "not
            sexual".
        """
        openai_response = self._moderation(text)

        return 1 - int(
            openai_response["results"][0]["category_scores"]["sexual"]
        )

    def moderation_not_sexualminors(self, text: str) -> float:
        """
        Uses OpenAI's Moderation API. A function that checks if text is about
        sexual minors.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.moderation_not_sexualminors).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "sexual minors" and 1 being
            "not sexual minors".
        """
        openai_response = self._moderation(text)

        return 1 - int(
            openai_response["results"][0]["category_scores"]["sexual/minors"]
        )

    def moderation_not_violence(self, text: str) -> float:
        """
        Uses OpenAI's Moderation API. A function that checks if text is about
        violence.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.moderation_not_violence).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "violence" and 1 being "not
            violence".
        """
        openai_response = self._moderation(text)

        return 1 - int(
            openai_response["results"][0]["category_scores"]["violence"]
        )

    def moderation_not_violencegraphic(self, text: str) -> float:
        """
        Uses OpenAI's Moderation API. A function that checks if text is about
        graphic violence.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.moderation_not_violencegraphic).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "graphic violence" and 1
            being "not graphic violence".
        """
        openai_response = self._moderation(text)

        return 1 - int(
            openai_response["results"][0]["category_scores"]["violence/graphic"]
        )

    def _find_relevant_string(self, full_source, hypothesis):
        return self.endpoint.run_me(
            lambda: self._create_chat_completion(
                model=self.model_engine,
                temperature=0.0,
                messages=[
                    {
                        "role":
                            "system",
                        "content":
                            str.format(
                                prompts.SYSTEM_FIND_SUPPORTING,
                                prompt=full_source,
                            )
                    }, {
                        "role":
                            "user",
                        "content":
                            str.format(
                                prompts.USER_FIND_SUPPORTING,
                                response=hypothesis
                            )
                    }
                ]
            )["choices"][0]["message"]["content"]
        )

    def _summarized_groundedness(self, premise: str, hypothesis: str) -> float:
        """ A groundedness measure best used for summarized premise against simple hypothesis.
        This OpenAI implementation uses information overlap prompts.

        Args:
            premise (str): Summarized source sentences.
            hypothesis (str): Single statement setnece.

        Returns:
            float: Information Overlap
        """
        return re_1_10_rating(
            self.endpoint.run_me(
                lambda: self._create_chat_completion(
                    model=self.model_engine,
                    temperature=0.0,
                    messages=[
                        {
                            "role":
                                "system",
                            "content":
                                str.format(
                                    prompts.LLM_GROUNDEDNESS,
                                    premise=premise,
                                    hypothesis=hypothesis,
                                )
                        }
                    ]
                )["choices"][0]["message"]["content"]
            )
        ) / 10

    def _groundedness_doc_in_out(
        self, premise: str, hypothesis: str, chain_of_thought=True
    ) -> str:
        """An LLM prompt using the entire document for premise and entire statement document for hypothesis

        Args:
            premise (str): A source document
            hypothesis (str): A statement to check

        Returns:
            str: An LLM response using a scorecard template
        """
        if chain_of_thought:
            system_prompt = prompts.LLM_GROUNDEDNESS_FULL_SYSTEM
        else:
            system_prompt = prompts.LLM_GROUNDEDNESS_SYSTEM_NO_COT
        return self.endpoint.run_me(
            lambda: self._create_chat_completion(
                model=self.model_engine,
                temperature=0.0,
                messages=[
                    {
                        "role": "system",
                        "content": system_prompt
                    }, {
                        "role":
                            "user",
                        "content":
                            str.format(
                                prompts.LLM_GROUNDEDNESS_FULL_PROMPT,
                                premise=premise,
                                hypothesis=hypothesis
                            )
                    }
                ]
            )["choices"][0]["message"]["content"]
        )

    def _extract_score_and_reasons_from_response(
        self, system_prompt: str, user_prompt: str = None, normalize=10
    ):
        """Extractor for our LLM prompts. If CoT is used; it will look for "Supporting Evidence" template.
        Otherwise, it will look for the typical 1-10 scoring.

        Args:
            system_prompt (str): A pre-formated system prompt

        Returns:
            The score and reason metadata if available.
        """
        llm_messages = [{"role": "system", "content": system_prompt}]
        if user_prompt is not None:
            llm_messages.append({"role": "user", "content": user_prompt})

        response = self.endpoint.run_me(
            lambda: self._create_chat_completion(
                model=self.model_engine, temperature=0.0, messages=llm_messages
            )["choices"][0]["message"]["content"]
        )
        if "Supporting Evidence" in response:
            score = 0
            for line in response.split('\n'):
                if "Score" in line:
                    score = re_1_10_rating(line) / normalize
            return score, {"reason": response}
        else:
            return re_1_10_rating(response) / normalize

    def qs_relevance(self, question: str, statement: str) -> float:
        """
        Uses OpenAI's Chat Completion App. A function that completes a
        template to check the relevance of the statement to the question.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.qs_relevance).on_input_output() 
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Usage on RAG Contexts:
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.qs_relevance).on_input().on(
            TruLlama.select_source_nodes().node.text # See note below
        ).aggregate(np.mean) 

        ```
        The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)



        Args:
            question (str): A question being asked. 
            statement (str): A statement to the question.

        Returns:
            float: A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".
        """
        system_prompt = str.format(
            prompts.QS_RELEVANCE, question=question, statement=statement
        )
        return self._extract_score_and_reasons_from_response(system_prompt)

    def qs_relevance_with_cot_reasons(
        self, question: str, statement: str
    ) -> float:
        """
        Uses OpenAI's Chat Completion App. A function that completes a
        template to check the relevance of the statement to the question.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.qs_relevance_with_cot_reasons).on_input_output() 
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Usage on RAG Contexts:
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.qs_relevance_with_cot_reasons).on_input().on(
            TruLlama.select_source_nodes().node.text # See note below
        ).aggregate(np.mean) 

        ```
        The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)



        Args:
            question (str): A question being asked. 
            statement (str): A statement to the question.

        Returns:
            float: A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".
        """
        system_prompt = str.format(
            prompts.QS_RELEVANCE, question=question, statement=statement
        )
        system_prompt = system_prompt.replace(
            "RELEVANCE:", prompts.COT_REASONS_TEMPLATE
        )
        return self._extract_score_and_reasons_from_response(system_prompt)

    def relevance(self, prompt: str, response: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the relevance of the response to a prompt.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.relevance).on_input_output()
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Usage on RAG Contexts:
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.relevance).on_input().on(
            TruLlama.select_source_nodes().node.text # See note below
        ).aggregate(np.mean) 

        ```
        The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


        Args:
            prompt (str): A text prompt to an agent. 
            response (str): The agent's response to the prompt.

        Returns:
            float: A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".
        """
        system_prompt = str.format(
            prompts.PR_RELEVANCE, prompt=prompt, response=response
        )
        return self._extract_score_and_reasons_from_response(system_prompt)

    def relevance_with_cot_reasons(self, prompt: str, response: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the relevance of the response to a prompt.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.relevance_with_cot_reasons).on_input_output()
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Usage on RAG Contexts:
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.relevance_with_cot_reasons).on_input().on(
            TruLlama.select_source_nodes().node.text # See note below
        ).aggregate(np.mean) 

        ```
        The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


        Args:
            prompt (str): A text prompt to an agent. 
            response (str): The agent's response to the prompt.

        Returns:
            float: A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".
        """
        system_prompt = str.format(
            prompts.PR_RELEVANCE, prompt=prompt, response=response
        )
        system_prompt = system_prompt.replace(
            "RELEVANCE:", prompts.COT_REASONS_TEMPLATE
        )
        return self._extract_score_and_reasons_from_response(system_prompt)

    def sentiment(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the sentiment of some text.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.sentiment).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "negative sentiment" and 1 being "positive sentiment".
        """
        system_prompt = prompts.SENTIMENT_SYSTEM_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def sentiment_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the sentiment of some text.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.sentiment_with_cot_reasons).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "negative sentiment" and 1 being "positive sentiment".
        """

        system_prompt = prompts.SENTIMENT_SYSTEM_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def model_agreement(self, prompt: str, response: str) -> float:
        """
        Uses OpenAI's Chat GPT Model. A function that gives Chat GPT the same
        prompt and gets a response, encouraging truthfulness. A second template
        is given to Chat GPT with a prompt that the original response is
        correct, and measures whether previous Chat GPT's response is similar.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.model_agreement).on_input_output() 
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            prompt (str): A text prompt to an agent. 
            response (str): The agent's response to the prompt.

        Returns:
            float: A value between 0 and 1. 0 being "not in agreement" and 1 being "in agreement".
        """
        logger.warning(
            "model_agreement has been deprecated. Use GroundTruthAgreement(ground_truth) instead."
        )
        oai_chat_response = self.endpoint.run_me(
            lambda: self._create_chat_completion(
                model=self.model_engine,
                temperature=0.0,
                messages=[
                    {
                        "role": "system",
                        "content": prompts.CORRECT_SYSTEM_PROMPT
                    }, {
                        "role": "user",
                        "content": prompt
                    }
                ]
            )["choices"][0]["message"]["content"]
        )
        agreement_txt = self._get_answer_agreement(
            prompt, response, oai_chat_response, self.model_engine
        )
        return re_1_10_rating(agreement_txt) / 10

    def conciseness(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the conciseness of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.conciseness).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "not concise" and 1 being "concise".
        """

        system_prompt = prompts.LANGCHAIN_CONCISENESS_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def correctness(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the correctness of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.correctness).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "not correct" and 1 being "correct".
        """

        system_prompt = prompts.LANGCHAIN_CORRECTNESS_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def correctness_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the correctness of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.correctness_with_cot_reasons).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "not correct" and 1 being "correct".
        """

        system_prompt = prompts.LANGCHAIN_CORRECTNESS_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def coherence(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the coherence of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.coherence).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "not coherent" and 1 being "coherent".
        """
        system_prompt = prompts.LANGCHAIN_COHERENCE_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def coherence_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the coherence of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.coherence_with_cot_reasons).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "not coherent" and 1 being "coherent".
        """
        system_prompt = prompts.LANGCHAIN_COHERENCE_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def harmfulness(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the harmfulness of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.harmfulness).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "harmful" and 1 being "not harmful".
        """
        system_prompt = prompts.LANGCHAIN_HARMFULNESS_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def harmfulness_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the harmfulness of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.harmfulness_with_cot_reasons).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): The text to evaluate.


        Returns:
            float: A value between 0 and 1. 0 being "harmful" and 1 being "not harmful".
        """

        system_prompt = prompts.LANGCHAIN_HARMFULNESS_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def maliciousness(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the maliciousness of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.maliciousness).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "malicious" and 1 being "not malicious".
        """
        system_prompt = prompts.LANGCHAIN_MALICIOUSNESS_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def maliciousness_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the maliciousness of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.maliciousness_with_cot_reasons).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "malicious" and 1 being "not malicious".
        """
        system_prompt = prompts.LANGCHAIN_MALICIOUSNESS_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def helpfulness(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the helpfulness of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.helpfulness).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "not helpful" and 1 being "helpful".
        """
        system_prompt = prompts.LANGCHAIN_HELPFULNESS_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def helpfulness_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the helpfulness of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.helpfulness_with_cot_reasons).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "not helpful" and 1 being "helpful".
        """

        system_prompt = prompts.LANGCHAIN_HELPFULNESS_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def controversiality(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the controversiality of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.controversiality).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "controversial" and 1 being "not controversial".
        """
        system_prompt = prompts.LANGCHAIN_CONTROVERSIALITY_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def controversiality_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the controversiality of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.controversiality_with_cot_reasons).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "controversial" and 1 being "not controversial".
        """
        system_prompt = prompts.LANGCHAIN_CONTROVERSIALITY_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def misogyny(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the misogyny of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.misogyny).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "misogynist" and 1 being "not misogynist".
        """
        system_prompt = prompts.LANGCHAIN_MISOGYNY_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def misogyny_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the misogyny of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.misogyny_with_cot_reasons).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "misogynist" and 1 being "not misogynist".
        """
        system_prompt = prompts.LANGCHAIN_MISOGYNY_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def criminality(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the criminality of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.criminality).on_output()
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "criminal" and 1 being "not criminal".

        """
        system_prompt = prompts.LANGCHAIN_CRIMINALITY_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def criminality_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the criminality of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.criminality_with_cot_reasons).on_output()
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "criminal" and 1 being "not criminal".
        """

        system_prompt = prompts.LANGCHAIN_CRIMINALITY_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def insensitivity(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the insensitivity of some text. Prompt credit to Langchain Eval.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.insensitivity).on_output()
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "insensitive" and 1 being "not insensitive".
        """
        system_prompt = prompts.LANGCHAIN_INSENSITIVITY_PROMPT
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def insensitivity_with_cot_reasons(self, text: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check the insensitivity of some text. Prompt credit to Langchain Eval.
        Also uses chain of thought methodology and emits the reasons.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.insensitivity_with_cot_reasons).on_output()
        ```
        The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): The text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "insensitive" and 1 being "not insensitive".
        """

        system_prompt = prompts.LANGCHAIN_INSENSITIVITY_PROMPT
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(
            system_prompt, user_prompt=text
        )

    def _get_answer_agreement(
        self, prompt, response, check_response, model_engine="gpt-3.5-turbo"
    ):
        oai_chat_response = self.endpoint.run_me(
            lambda: self._create_chat_completion(
                model=model_engine,
                temperature=0.0,
                messages=[
                    {
                        "role":
                            "system",
                        "content":
                            prompts.AGREEMENT_SYSTEM_PROMPT %
                            (prompt, response)
                    }, {
                        "role": "user",
                        "content": check_response
                    }
                ]
            )["choices"][0]["message"]["content"]
        )
        return oai_chat_response

    def summary_with_cot_reasons(self, source: str, summary: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that tries to distill main points and compares a summary against those main points.
        This feedback function only has a chain of thought implementation as it is extremely important in function assessment. 

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.summary_with_cot_reasons).on_input_output()
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            source (str): Text corresponding to source material. 
            summary (str): Text corresponding to a summary.

        Returns:
            float: A value between 0 and 1. 0 being "main points missed" and 1 being "no main points missed".
        """
        system_prompt = str.format(
            prompts.SUMMARIZATION_PROMPT, source=source, summary=summary
        )
        return self._extract_score_and_reasons_from_response(system_prompt)

    def stereotypes(self, prompt: str, response: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check adding assumed stereotypes in the response when not present in the prompt.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.stereotypes).on_input_output()
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            prompt (str): A text prompt to an agent. 
            response (str): The agent's response to the prompt.

        Returns:
            float: A value between 0 and 1. 0 being "assumed stereotypes" and 1 being "no assumed stereotypes".
        """
        system_prompt = str.format(
            prompts.STEREOTYPES_PROMPT, prompt=prompt, response=response
        )
        return self._extract_score_and_reasons_from_response(system_prompt)

    def stereotypes_with_cot_reasons(self, prompt: str, response: str) -> float:
        """
        Uses OpenAI's Chat Completion Model. A function that completes a
        template to check adding assumed stereotypes in the response when not present in the prompt.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()

        feedback = Feedback(openai_provider.stereotypes_with_cot_reasons).on_input_output()
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            prompt (str): A text prompt to an agent. 
            response (str): The agent's response to the prompt.

        Returns:
            float: A value between 0 and 1. 0 being "assumed stereotypes" and 1 being "no assumed stereotypes".
        """
        system_prompt = str.format(
            prompts.STEREOTYPES_PROMPT, prompt=prompt, response=response
        )
        system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
        return self._extract_score_and_reasons_from_response(system_prompt)

__init__(*args, endpoint=None, model_engine='gpt-3.5-turbo', **kwargs)

Create an OpenAI Provider with out of the box feedback functions.

Usage:

from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

Parameters:

Name	Type	Description	Default
model_engine	str	The OpenAI completion model. Defaults to gpt-3.5-turbo	'gpt-3.5-turbo'
endpoint	Endpoint	Internal Usage for DB serialization	None

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def __init__(
    self, *args, endpoint=None, model_engine="gpt-3.5-turbo", **kwargs
):
    # NOTE(piotrm): pydantic adds endpoint to the signature of this
    # constructor if we don't include it explicitly, even though we set it
    # down below. Adding it as None here as a temporary hack.
    """
    Create an OpenAI Provider with out of the box feedback functions.

    **Usage:**
    ```
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    ```

    Args:
        model_engine (str): The OpenAI completion model. Defaults to `gpt-3.5-turbo`
        endpoint (Endpoint): Internal Usage for DB serialization
    """
    # TODO: why was self_kwargs required here independently of kwargs?
    self_kwargs = dict()
    self_kwargs.update(**kwargs)
    self_kwargs['model_engine'] = model_engine
    self_kwargs['endpoint'] = OpenAIEndpoint(*args, **kwargs)

    super().__init__(
        **self_kwargs
    )  # need to include pydantic.BaseModel.__init__

    set_openai_key()

coherence(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the coherence of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.coherence).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not coherent" and 1 being "coherent".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def coherence(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the coherence of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.coherence).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "not coherent" and 1 being "coherent".
    """
    system_prompt = prompts.LANGCHAIN_COHERENCE_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

coherence_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the coherence of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.coherence_with_cot_reasons).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not coherent" and 1 being "coherent".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def coherence_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the coherence of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.coherence_with_cot_reasons).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "not coherent" and 1 being "coherent".
    """
    system_prompt = prompts.LANGCHAIN_COHERENCE_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

conciseness(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the conciseness of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.conciseness).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not concise" and 1 being "concise".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def conciseness(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the conciseness of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.conciseness).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "not concise" and 1 being "concise".
    """

    system_prompt = prompts.LANGCHAIN_CONCISENESS_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

controversiality(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the controversiality of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.controversiality).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "controversial" and 1 being "not controversial".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def controversiality(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the controversiality of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.controversiality).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "controversial" and 1 being "not controversial".
    """
    system_prompt = prompts.LANGCHAIN_CONTROVERSIALITY_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

controversiality_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the controversiality of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.controversiality_with_cot_reasons).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "controversial" and 1 being "not controversial".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def controversiality_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the controversiality of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.controversiality_with_cot_reasons).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "controversial" and 1 being "not controversial".
    """
    system_prompt = prompts.LANGCHAIN_CONTROVERSIALITY_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

correctness(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the correctness of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.correctness).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not correct" and 1 being "correct".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def correctness(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the correctness of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.correctness).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "not correct" and 1 being "correct".
    """

    system_prompt = prompts.LANGCHAIN_CORRECTNESS_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

correctness_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the correctness of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.correctness_with_cot_reasons).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not correct" and 1 being "correct".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def correctness_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the correctness of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.correctness_with_cot_reasons).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "not correct" and 1 being "correct".
    """

    system_prompt = prompts.LANGCHAIN_CORRECTNESS_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

criminality(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the criminality of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.criminality).on_output()

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "criminal" and 1 being "not criminal".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def criminality(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the criminality of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.criminality).on_output()
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "criminal" and 1 being "not criminal".

    """
    system_prompt = prompts.LANGCHAIN_CRIMINALITY_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

criminality_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the criminality of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.criminality_with_cot_reasons).on_output()

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "criminal" and 1 being "not criminal".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def criminality_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the criminality of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.criminality_with_cot_reasons).on_output()
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "criminal" and 1 being "not criminal".
    """

    system_prompt = prompts.LANGCHAIN_CRIMINALITY_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

harmfulness(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the harmfulness of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.harmfulness).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "harmful" and 1 being "not harmful".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def harmfulness(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the harmfulness of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.harmfulness).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "harmful" and 1 being "not harmful".
    """
    system_prompt = prompts.LANGCHAIN_HARMFULNESS_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

harmfulness_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the harmfulness of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.harmfulness_with_cot_reasons).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "harmful" and 1 being "not harmful".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def harmfulness_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the harmfulness of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.harmfulness_with_cot_reasons).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): The text to evaluate.


    Returns:
        float: A value between 0 and 1. 0 being "harmful" and 1 being "not harmful".
    """

    system_prompt = prompts.LANGCHAIN_HARMFULNESS_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

helpfulness(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the helpfulness of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.helpfulness).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not helpful" and 1 being "helpful".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def helpfulness(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the helpfulness of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.helpfulness).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "not helpful" and 1 being "helpful".
    """
    system_prompt = prompts.LANGCHAIN_HELPFULNESS_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

helpfulness_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the helpfulness of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.helpfulness_with_cot_reasons).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not helpful" and 1 being "helpful".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def helpfulness_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the helpfulness of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.helpfulness_with_cot_reasons).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "not helpful" and 1 being "helpful".
    """

    system_prompt = prompts.LANGCHAIN_HELPFULNESS_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

insensitivity(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the insensitivity of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.insensitivity).on_output()

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "insensitive" and 1 being "not insensitive".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def insensitivity(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the insensitivity of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.insensitivity).on_output()
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "insensitive" and 1 being "not insensitive".
    """
    system_prompt = prompts.LANGCHAIN_INSENSITIVITY_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

insensitivity_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the insensitivity of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.insensitivity_with_cot_reasons).on_output()

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "insensitive" and 1 being "not insensitive".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def insensitivity_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the insensitivity of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.insensitivity_with_cot_reasons).on_output()
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "insensitive" and 1 being "not insensitive".
    """

    system_prompt = prompts.LANGCHAIN_INSENSITIVITY_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

maliciousness(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the maliciousness of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.maliciousness).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "malicious" and 1 being "not malicious".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def maliciousness(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the maliciousness of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.maliciousness).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "malicious" and 1 being "not malicious".
    """
    system_prompt = prompts.LANGCHAIN_MALICIOUSNESS_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

maliciousness_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the maliciousness of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.maliciousness_with_cot_reasons).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "malicious" and 1 being "not malicious".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def maliciousness_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the maliciousness of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.maliciousness_with_cot_reasons).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "malicious" and 1 being "not malicious".
    """
    system_prompt = prompts.LANGCHAIN_MALICIOUSNESS_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

misogyny(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the misogyny of some text. Prompt credit to Langchain Eval.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.misogyny).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "misogynist" and 1 being "not misogynist".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def misogyny(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the misogyny of some text. Prompt credit to Langchain Eval.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.misogyny).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "misogynist" and 1 being "not misogynist".
    """
    system_prompt = prompts.LANGCHAIN_MISOGYNY_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

misogyny_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the misogyny of some text. Prompt credit to Langchain Eval. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.misogyny_with_cot_reasons).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	The text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "misogynist" and 1 being "not misogynist".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def misogyny_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the misogyny of some text. Prompt credit to Langchain Eval.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.misogyny_with_cot_reasons).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): The text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "misogynist" and 1 being "not misogynist".
    """
    system_prompt = prompts.LANGCHAIN_MISOGYNY_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

model_agreement(prompt, response)

Uses OpenAI's Chat GPT Model. A function that gives Chat GPT the same prompt and gets a response, encouraging truthfulness. A second template is given to Chat GPT with a prompt that the original response is correct, and measures whether previous Chat GPT's response is similar.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.model_agreement).on_input_output() 

The on_input_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not in agreement" and 1 being "in agreement".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def model_agreement(self, prompt: str, response: str) -> float:
    """
    Uses OpenAI's Chat GPT Model. A function that gives Chat GPT the same
    prompt and gets a response, encouraging truthfulness. A second template
    is given to Chat GPT with a prompt that the original response is
    correct, and measures whether previous Chat GPT's response is similar.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.model_agreement).on_input_output() 
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        prompt (str): A text prompt to an agent. 
        response (str): The agent's response to the prompt.

    Returns:
        float: A value between 0 and 1. 0 being "not in agreement" and 1 being "in agreement".
    """
    logger.warning(
        "model_agreement has been deprecated. Use GroundTruthAgreement(ground_truth) instead."
    )
    oai_chat_response = self.endpoint.run_me(
        lambda: self._create_chat_completion(
            model=self.model_engine,
            temperature=0.0,
            messages=[
                {
                    "role": "system",
                    "content": prompts.CORRECT_SYSTEM_PROMPT
                }, {
                    "role": "user",
                    "content": prompt
                }
            ]
        )["choices"][0]["message"]["content"]
    )
    agreement_txt = self._get_answer_agreement(
        prompt, response, oai_chat_response, self.model_engine
    )
    return re_1_10_rating(agreement_txt) / 10

moderation_not_hate(text)

Uses OpenAI's Moderation API. A function that checks if text is hate speech.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.moderation_not_hate).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "hate" and 1 being "not
	float	hate".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def moderation_not_hate(self, text: str) -> float:
    """
    Uses OpenAI's Moderation API. A function that checks if text is hate
    speech.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.moderation_not_hate).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "hate" and 1 being "not
        hate".
    """
    openai_response = self._moderation(text)
    return 1 - float(
        openai_response["results"][0]["category_scores"]["hate"]
    )

moderation_not_hatethreatening(text)

Uses OpenAI's Moderation API. A function that checks if text is threatening speech.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.moderation_not_hatethreatening).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "threatening" and 1 being
	float	"not threatening".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def moderation_not_hatethreatening(self, text: str) -> float:
    """
    Uses OpenAI's Moderation API. A function that checks if text is
    threatening speech.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.moderation_not_hatethreatening).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "threatening" and 1 being
        "not threatening".
    """
    openai_response = self._moderation(text)

    return 1 - int(
        openai_response["results"][0]["category_scores"]["hate/threatening"]
    )

moderation_not_selfharm(text)

Uses OpenAI's Moderation API. A function that checks if text is about self harm.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.moderation_not_selfharm).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "self harm" and 1 being "not
	float	self harm".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def moderation_not_selfharm(self, text: str) -> float:
    """
    Uses OpenAI's Moderation API. A function that checks if text is about
    self harm.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.moderation_not_selfharm).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "self harm" and 1 being "not
        self harm".
    """
    openai_response = self._moderation(text)

    return 1 - int(
        openai_response["results"][0]["category_scores"]["self-harm"]
    )

moderation_not_sexual(text)

Uses OpenAI's Moderation API. A function that checks if text is sexual speech.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.moderation_not_sexual).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "sexual" and 1 being "not
	float	sexual".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def moderation_not_sexual(self, text: str) -> float:
    """
    Uses OpenAI's Moderation API. A function that checks if text is sexual
    speech.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.moderation_not_sexual).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "sexual" and 1 being "not
        sexual".
    """
    openai_response = self._moderation(text)

    return 1 - int(
        openai_response["results"][0]["category_scores"]["sexual"]
    )

moderation_not_sexualminors(text)

Uses OpenAI's Moderation API. A function that checks if text is about sexual minors.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.moderation_not_sexualminors).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "sexual minors" and 1 being
	float	"not sexual minors".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def moderation_not_sexualminors(self, text: str) -> float:
    """
    Uses OpenAI's Moderation API. A function that checks if text is about
    sexual minors.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.moderation_not_sexualminors).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "sexual minors" and 1 being
        "not sexual minors".
    """
    openai_response = self._moderation(text)

    return 1 - int(
        openai_response["results"][0]["category_scores"]["sexual/minors"]
    )

moderation_not_violence(text)

Uses OpenAI's Moderation API. A function that checks if text is about violence.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.moderation_not_violence).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "violence" and 1 being "not
	float	violence".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def moderation_not_violence(self, text: str) -> float:
    """
    Uses OpenAI's Moderation API. A function that checks if text is about
    violence.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.moderation_not_violence).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "violence" and 1 being "not
        violence".
    """
    openai_response = self._moderation(text)

    return 1 - int(
        openai_response["results"][0]["category_scores"]["violence"]
    )

moderation_not_violencegraphic(text)

Uses OpenAI's Moderation API. A function that checks if text is about graphic violence.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.moderation_not_violencegraphic).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "graphic violence" and 1
	float	being "not graphic violence".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def moderation_not_violencegraphic(self, text: str) -> float:
    """
    Uses OpenAI's Moderation API. A function that checks if text is about
    graphic violence.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.moderation_not_violencegraphic).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "graphic violence" and 1
        being "not graphic violence".
    """
    openai_response = self._moderation(text)

    return 1 - int(
        openai_response["results"][0]["category_scores"]["violence/graphic"]
    )

qs_relevance(question, statement)

Uses OpenAI's Chat Completion App. A function that completes a template to check the relevance of the statement to the question.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.qs_relevance).on_input_output() 

The on_input_output() selector can be changed. See Feedback Function Guide

Usage on RAG Contexts:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.qs_relevance).on_input().on(
    TruLlama.select_source_nodes().node.text # See note below
).aggregate(np.mean) 

The on(...) selector can be changed. See Feedback Function Guide : Selectors

Parameters:

Name	Type	Description	Default
question	str	A question being asked.	required
statement	str	A statement to the question.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def qs_relevance(self, question: str, statement: str) -> float:
    """
    Uses OpenAI's Chat Completion App. A function that completes a
    template to check the relevance of the statement to the question.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.qs_relevance).on_input_output() 
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Usage on RAG Contexts:
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.qs_relevance).on_input().on(
        TruLlama.select_source_nodes().node.text # See note below
    ).aggregate(np.mean) 

    ```
    The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)



    Args:
        question (str): A question being asked. 
        statement (str): A statement to the question.

    Returns:
        float: A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".
    """
    system_prompt = str.format(
        prompts.QS_RELEVANCE, question=question, statement=statement
    )
    return self._extract_score_and_reasons_from_response(system_prompt)

qs_relevance_with_cot_reasons(question, statement)

Uses OpenAI's Chat Completion App. A function that completes a template to check the relevance of the statement to the question. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.qs_relevance_with_cot_reasons).on_input_output() 

The on_input_output() selector can be changed. See Feedback Function Guide

Usage on RAG Contexts:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.qs_relevance_with_cot_reasons).on_input().on(
    TruLlama.select_source_nodes().node.text # See note below
).aggregate(np.mean) 

The on(...) selector can be changed. See Feedback Function Guide : Selectors

Parameters:

Name	Type	Description	Default
question	str	A question being asked.	required
statement	str	A statement to the question.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def qs_relevance_with_cot_reasons(
    self, question: str, statement: str
) -> float:
    """
    Uses OpenAI's Chat Completion App. A function that completes a
    template to check the relevance of the statement to the question.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.qs_relevance_with_cot_reasons).on_input_output() 
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Usage on RAG Contexts:
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.qs_relevance_with_cot_reasons).on_input().on(
        TruLlama.select_source_nodes().node.text # See note below
    ).aggregate(np.mean) 

    ```
    The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)



    Args:
        question (str): A question being asked. 
        statement (str): A statement to the question.

    Returns:
        float: A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".
    """
    system_prompt = str.format(
        prompts.QS_RELEVANCE, question=question, statement=statement
    )
    system_prompt = system_prompt.replace(
        "RELEVANCE:", prompts.COT_REASONS_TEMPLATE
    )
    return self._extract_score_and_reasons_from_response(system_prompt)

relevance(prompt, response)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the relevance of the response to a prompt.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.relevance).on_input_output()

The on_input_output() selector can be changed. See Feedback Function Guide

Usage on RAG Contexts:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.relevance).on_input().on(
    TruLlama.select_source_nodes().node.text # See note below
).aggregate(np.mean) 

The on(...) selector can be changed. See Feedback Function Guide : Selectors

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def relevance(self, prompt: str, response: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the relevance of the response to a prompt.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.relevance).on_input_output()
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Usage on RAG Contexts:
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.relevance).on_input().on(
        TruLlama.select_source_nodes().node.text # See note below
    ).aggregate(np.mean) 

    ```
    The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


    Args:
        prompt (str): A text prompt to an agent. 
        response (str): The agent's response to the prompt.

    Returns:
        float: A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".
    """
    system_prompt = str.format(
        prompts.PR_RELEVANCE, prompt=prompt, response=response
    )
    return self._extract_score_and_reasons_from_response(system_prompt)

relevance_with_cot_reasons(prompt, response)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the relevance of the response to a prompt. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.relevance_with_cot_reasons).on_input_output()

The on_input_output() selector can be changed. See Feedback Function Guide

Usage on RAG Contexts:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.relevance_with_cot_reasons).on_input().on(
    TruLlama.select_source_nodes().node.text # See note below
).aggregate(np.mean) 

The on(...) selector can be changed. See Feedback Function Guide : Selectors

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def relevance_with_cot_reasons(self, prompt: str, response: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the relevance of the response to a prompt.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.relevance_with_cot_reasons).on_input_output()
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Usage on RAG Contexts:
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.relevance_with_cot_reasons).on_input().on(
        TruLlama.select_source_nodes().node.text # See note below
    ).aggregate(np.mean) 

    ```
    The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


    Args:
        prompt (str): A text prompt to an agent. 
        response (str): The agent's response to the prompt.

    Returns:
        float: A value between 0 and 1. 0 being "not relevant" and 1 being "relevant".
    """
    system_prompt = str.format(
        prompts.PR_RELEVANCE, prompt=prompt, response=response
    )
    system_prompt = system_prompt.replace(
        "RELEVANCE:", prompts.COT_REASONS_TEMPLATE
    )
    return self._extract_score_and_reasons_from_response(system_prompt)

sentiment(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the sentiment of some text.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.sentiment).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "negative sentiment" and 1 being "positive sentiment".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def sentiment(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the sentiment of some text.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.sentiment).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "negative sentiment" and 1 being "positive sentiment".
    """
    system_prompt = prompts.SENTIMENT_SYSTEM_PROMPT
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

sentiment_with_cot_reasons(text)

Uses OpenAI's Chat Completion Model. A function that completes a template to check the sentiment of some text. Also uses chain of thought methodology and emits the reasons.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.sentiment_with_cot_reasons).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "negative sentiment" and 1 being "positive sentiment".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def sentiment_with_cot_reasons(self, text: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check the sentiment of some text.
    Also uses chain of thought methodology and emits the reasons.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.sentiment_with_cot_reasons).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "negative sentiment" and 1 being "positive sentiment".
    """

    system_prompt = prompts.SENTIMENT_SYSTEM_PROMPT
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(
        system_prompt, user_prompt=text
    )

stereotypes(prompt, response)

Uses OpenAI's Chat Completion Model. A function that completes a template to check adding assumed stereotypes in the response when not present in the prompt.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.stereotypes).on_input_output()

The on_input_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "assumed stereotypes" and 1 being "no assumed stereotypes".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def stereotypes(self, prompt: str, response: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check adding assumed stereotypes in the response when not present in the prompt.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.stereotypes).on_input_output()
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        prompt (str): A text prompt to an agent. 
        response (str): The agent's response to the prompt.

    Returns:
        float: A value between 0 and 1. 0 being "assumed stereotypes" and 1 being "no assumed stereotypes".
    """
    system_prompt = str.format(
        prompts.STEREOTYPES_PROMPT, prompt=prompt, response=response
    )
    return self._extract_score_and_reasons_from_response(system_prompt)

stereotypes_with_cot_reasons(prompt, response)

Uses OpenAI's Chat Completion Model. A function that completes a template to check adding assumed stereotypes in the response when not present in the prompt.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.stereotypes_with_cot_reasons).on_input_output()

The on_input_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "assumed stereotypes" and 1 being "no assumed stereotypes".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def stereotypes_with_cot_reasons(self, prompt: str, response: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that completes a
    template to check adding assumed stereotypes in the response when not present in the prompt.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.stereotypes_with_cot_reasons).on_input_output()
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        prompt (str): A text prompt to an agent. 
        response (str): The agent's response to the prompt.

    Returns:
        float: A value between 0 and 1. 0 being "assumed stereotypes" and 1 being "no assumed stereotypes".
    """
    system_prompt = str.format(
        prompts.STEREOTYPES_PROMPT, prompt=prompt, response=response
    )
    system_prompt = system_prompt + prompts.COT_REASONS_TEMPLATE
    return self._extract_score_and_reasons_from_response(system_prompt)

summary_with_cot_reasons(source, summary)

Uses OpenAI's Chat Completion Model. A function that tries to distill main points and compares a summary against those main points. This feedback function only has a chain of thought implementation as it is extremely important in function assessment.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()

feedback = Feedback(openai_provider.summary_with_cot_reasons).on_input_output()

The on_input_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
source	str	Text corresponding to source material.	required
summary	str	Text corresponding to a summary.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "main points missed" and 1 being "no main points missed".

Source code in trulens_eval/trulens_eval/feedback/provider/openai.py

def summary_with_cot_reasons(self, source: str, summary: str) -> float:
    """
    Uses OpenAI's Chat Completion Model. A function that tries to distill main points and compares a summary against those main points.
    This feedback function only has a chain of thought implementation as it is extremely important in function assessment. 

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()

    feedback = Feedback(openai_provider.summary_with_cot_reasons).on_input_output()
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        source (str): Text corresponding to source material. 
        summary (str): Text corresponding to a summary.

    Returns:
        float: A value between 0 and 1. 0 being "main points missed" and 1 being "no main points missed".
    """
    system_prompt = str.format(
        prompts.SUMMARIZATION_PROMPT, source=source, summary=summary
    )
    return self._extract_score_and_reasons_from_response(system_prompt)

Huggingface

Bases: Provider

Out of the box feedback functions calling Huggingface APIs.

Source code in trulens_eval/trulens_eval/feedback/provider/hugs.py

class Huggingface(Provider):
    """
    Out of the box feedback functions calling Huggingface APIs.
    """

    endpoint: Endpoint

    def __init__(self, name: str = None, endpoint=None, **kwargs):
        # NOTE(piotrm): pydantic adds endpoint to the signature of this
        # constructor if we don't include it explicitly, even though we set it
        # down below. Adding it as None here as a temporary hack.
        """
        Create a Huggingface Provider with out of the box feedback functions.

        **Usage:**
        ```python
        from trulens_eval.feedback.provider.hugs import Huggingface
        huggingface_provider = Huggingface()
        ```

        Args:
            endpoint (Endpoint): Internal Usage for DB serialization
        """

        kwargs['name'] = name

        self_kwargs = dict()
        if endpoint is None:
            self_kwargs['endpoint'] = HuggingfaceEndpoint(**kwargs)
        else:
            self_kwargs['endpoint'] = endpoint

        self_kwargs['name'] = name or "huggingface"

        super().__init__(
            **self_kwargs
        )  # need to include pydantic.BaseModel.__init__

    @_tci
    def language_match(self, text1: str, text2: str) -> float:
        """
        Uses Huggingface's papluca/xlm-roberta-base-language-detection model. A
        function that uses language detection on `text1` and `text2` and
        calculates the probit difference on the language detected on text1. The
        function is: `1.0 - (|probit_language_text1(text1) -
        probit_language_text1(text2))`

        **Usage:**
        ```python
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.hugs import Huggingface
        huggingface_provider = Huggingface()

        feedback = Feedback(huggingface_provider.language_match).on_input_output() 
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function
        Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text1 (str): Text to evaluate.
            text2 (str): Comparative text to evaluate.

        Returns:

            float: A value between 0 and 1. 0 being "different languages" and 1
            being "same languages".
        """

        def get_scores(text):
            payload = {"inputs": text}
            hf_response = self.endpoint.post(
                url=HUGS_LANGUAGE_API_URL, payload=payload, timeout=30
            )
            return {r['label']: r['score'] for r in hf_response}

        max_length = 500
        scores1: AsyncResult[Dict] = TP().promise(
            get_scores, text=text1[:max_length]
        )
        scores2: AsyncResult[Dict] = TP().promise(
            get_scores, text=text2[:max_length]
        )

        scores1: Dict = scores1.get()
        scores2: Dict = scores2.get()

        langs = list(scores1.keys())
        prob1 = np.array([scores1[k] for k in langs])
        prob2 = np.array([scores2[k] for k in langs])
        diff = prob1 - prob2

        l1 = 1.0 - (np.linalg.norm(diff, ord=1)) / 2.0

        return l1, dict(text1_scores=scores1, text2_scores=scores2)

    @_tci
    def positive_sentiment(self, text: str) -> float:
        """
        Uses Huggingface's cardiffnlp/twitter-roberta-base-sentiment model. A
        function that uses a sentiment classifier on `text`.

        **Usage:**
        ```python
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.hugs import Huggingface
        huggingface_provider = Huggingface()

        feedback = Feedback(huggingface_provider.positive_sentiment).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function
        Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "negative sentiment" and 1
            being "positive sentiment".
        """

        max_length = 500
        truncated_text = text[:max_length]
        payload = {"inputs": truncated_text}

        hf_response = self.endpoint.post(
            url=HUGS_SENTIMENT_API_URL, payload=payload
        )

        for label in hf_response:
            if label['label'] == 'LABEL_2':
                return label['score']

    @_tci
    def not_toxic(self, text: str) -> float:
        """
        Uses Huggingface's martin-ha/toxic-comment-model model. A function that
        uses a toxic comment classifier on `text`.

        **Usage:**
        ```python
        from trulens_eval import Feedback
        from trulens_eval.feedback.provider.hugs import Huggingface
        huggingface_provider = Huggingface()

        feedback = Feedback(huggingface_provider.not_toxic).on_output() 
        ```
        The `on_output()` selector can be changed. See [Feedback Function
        Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            text (str): Text to evaluate.

        Returns:
            float: A value between 0 and 1. 0 being "toxic" and 1 being "not
            toxic".
        """

        assert len(text) > 0, "Input cannot be blank."

        max_length = 500
        truncated_text = text[:max_length]
        payload = {"inputs": truncated_text}
        hf_response = self.endpoint.post(
            url=HUGS_TOXIC_API_URL, payload=payload
        )

        for label in hf_response:
            if label['label'] == 'toxic':
                return label['score']

    @_tci
    def _summarized_groundedness(self, premise: str, hypothesis: str) -> float:
        """ A groundedness measure best used for summarized premise against simple hypothesis.
        This Huggingface implementation uses NLI.

        Args:
            premise (str): NLI Premise
            hypothesis (str): NLI Hypothesis

        Returns:
            float: NLI Entailment
        """

        if not '.' == premise[len(premise) - 1]:
            premise = premise + '.'
        nli_string = premise + ' ' + hypothesis
        payload = {"inputs": nli_string}
        hf_response = self.endpoint.post(url=HUGS_NLI_API_URL, payload=payload)

        for label in hf_response:
            if label['label'] == 'entailment':
                return label['score']

    @_tci
    def _doc_groundedness(self, premise: str, hypothesis: str) -> float:
        """
        A groundedness measure for full document premise against hypothesis.
        This Huggingface implementation uses DocNLI. The Hypoethsis still only
        works on single small hypothesis.

        Args:
            premise (str): NLI Premise
            hypothesis (str): NLI Hypothesis

        Returns:
            float: NLI Entailment
        """
        nli_string = premise + ' [SEP] ' + hypothesis
        payload = {"inputs": nli_string}
        hf_response = self.endpoint.post(
            url=HUGS_DOCNLI_API_URL, payload=payload
        )

        for label in hf_response:
            if label['label'] == 'entailment':
                return label['score']

__init__(name=None, endpoint=None, **kwargs)

Create a Huggingface Provider with out of the box feedback functions.

Usage:

from trulens_eval.feedback.provider.hugs import Huggingface
huggingface_provider = Huggingface()

Parameters:

Name	Type	Description	Default
endpoint	Endpoint	Internal Usage for DB serialization	None

Source code in trulens_eval/trulens_eval/feedback/provider/hugs.py

def __init__(self, name: str = None, endpoint=None, **kwargs):
    # NOTE(piotrm): pydantic adds endpoint to the signature of this
    # constructor if we don't include it explicitly, even though we set it
    # down below. Adding it as None here as a temporary hack.
    """
    Create a Huggingface Provider with out of the box feedback functions.

    **Usage:**
    ```python
    from trulens_eval.feedback.provider.hugs import Huggingface
    huggingface_provider = Huggingface()
    ```

    Args:
        endpoint (Endpoint): Internal Usage for DB serialization
    """

    kwargs['name'] = name

    self_kwargs = dict()
    if endpoint is None:
        self_kwargs['endpoint'] = HuggingfaceEndpoint(**kwargs)
    else:
        self_kwargs['endpoint'] = endpoint

    self_kwargs['name'] = name or "huggingface"

    super().__init__(
        **self_kwargs
    )  # need to include pydantic.BaseModel.__init__

language_match(text1, text2)

Uses Huggingface's papluca/xlm-roberta-base-language-detection model. A function that uses language detection on text1 and text2 and calculates the probit difference on the language detected on text1. The function is: 1.0 - (|probit_language_text1(text1) - probit_language_text1(text2))

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.hugs import Huggingface
huggingface_provider = Huggingface()

feedback = Feedback(huggingface_provider.language_match).on_input_output() 

The on_input_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text1	str	Text to evaluate.	required
text2	str	Comparative text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "different languages" and 1
	float	being "same languages".

Source code in trulens_eval/trulens_eval/feedback/provider/hugs.py

@_tci
def language_match(self, text1: str, text2: str) -> float:
    """
    Uses Huggingface's papluca/xlm-roberta-base-language-detection model. A
    function that uses language detection on `text1` and `text2` and
    calculates the probit difference on the language detected on text1. The
    function is: `1.0 - (|probit_language_text1(text1) -
    probit_language_text1(text2))`

    **Usage:**
    ```python
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.hugs import Huggingface
    huggingface_provider = Huggingface()

    feedback = Feedback(huggingface_provider.language_match).on_input_output() 
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function
    Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text1 (str): Text to evaluate.
        text2 (str): Comparative text to evaluate.

    Returns:

        float: A value between 0 and 1. 0 being "different languages" and 1
        being "same languages".
    """

    def get_scores(text):
        payload = {"inputs": text}
        hf_response = self.endpoint.post(
            url=HUGS_LANGUAGE_API_URL, payload=payload, timeout=30
        )
        return {r['label']: r['score'] for r in hf_response}

    max_length = 500
    scores1: AsyncResult[Dict] = TP().promise(
        get_scores, text=text1[:max_length]
    )
    scores2: AsyncResult[Dict] = TP().promise(
        get_scores, text=text2[:max_length]
    )

    scores1: Dict = scores1.get()
    scores2: Dict = scores2.get()

    langs = list(scores1.keys())
    prob1 = np.array([scores1[k] for k in langs])
    prob2 = np.array([scores2[k] for k in langs])
    diff = prob1 - prob2

    l1 = 1.0 - (np.linalg.norm(diff, ord=1)) / 2.0

    return l1, dict(text1_scores=scores1, text2_scores=scores2)

not_toxic(text)

Uses Huggingface's martin-ha/toxic-comment-model model. A function that uses a toxic comment classifier on text.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.hugs import Huggingface
huggingface_provider = Huggingface()

feedback = Feedback(huggingface_provider.not_toxic).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "toxic" and 1 being "not
	float	toxic".

Source code in trulens_eval/trulens_eval/feedback/provider/hugs.py

@_tci
def not_toxic(self, text: str) -> float:
    """
    Uses Huggingface's martin-ha/toxic-comment-model model. A function that
    uses a toxic comment classifier on `text`.

    **Usage:**
    ```python
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.hugs import Huggingface
    huggingface_provider = Huggingface()

    feedback = Feedback(huggingface_provider.not_toxic).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function
    Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "toxic" and 1 being "not
        toxic".
    """

    assert len(text) > 0, "Input cannot be blank."

    max_length = 500
    truncated_text = text[:max_length]
    payload = {"inputs": truncated_text}
    hf_response = self.endpoint.post(
        url=HUGS_TOXIC_API_URL, payload=payload
    )

    for label in hf_response:
        if label['label'] == 'toxic':
            return label['score']

positive_sentiment(text)

Uses Huggingface's cardiffnlp/twitter-roberta-base-sentiment model. A function that uses a sentiment classifier on text.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback.provider.hugs import Huggingface
huggingface_provider = Huggingface()

feedback = Feedback(huggingface_provider.positive_sentiment).on_output() 

The on_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
text	str	Text to evaluate.	required

Returns:

Name	Type	Description
float	float	A value between 0 and 1. 0 being "negative sentiment" and 1
	float	being "positive sentiment".

Source code in trulens_eval/trulens_eval/feedback/provider/hugs.py

@_tci
def positive_sentiment(self, text: str) -> float:
    """
    Uses Huggingface's cardiffnlp/twitter-roberta-base-sentiment model. A
    function that uses a sentiment classifier on `text`.

    **Usage:**
    ```python
    from trulens_eval import Feedback
    from trulens_eval.feedback.provider.hugs import Huggingface
    huggingface_provider = Huggingface()

    feedback = Feedback(huggingface_provider.positive_sentiment).on_output() 
    ```
    The `on_output()` selector can be changed. See [Feedback Function
    Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        text (str): Text to evaluate.

    Returns:
        float: A value between 0 and 1. 0 being "negative sentiment" and 1
        being "positive sentiment".
    """

    max_length = 500
    truncated_text = text[:max_length]
    payload = {"inputs": truncated_text}

    hf_response = self.endpoint.post(
        url=HUGS_SENTIMENT_API_URL, payload=payload
    )

    for label in hf_response:
        if label['label'] == 'LABEL_2':
            return label['score']

Groundedness

Bases: SerialModel, WithClassInfo

Measures Groundedness.

Source code in trulens_eval/trulens_eval/feedback/groundedness.py

class Groundedness(SerialModel, WithClassInfo):
    """Measures Groundedness.
    """
    groundedness_provider: Provider
    summarize_provider: Provider

    def __init__(
        self,
        summarize_provider: Provider = None,
        groundedness_provider: Provider = None
    ):
        """Instantiates the groundedness providers. Currently the groundedness functions work well with a summarizer.
        This class will use an OpenAI summarizer to find the relevant strings in a text. The groundedness_provider can 
        either be an llm with OpenAI or NLI with huggingface.

        Usage 1:
        ```
        from trulens_eval.feedback import Groundedness
        from trulens_eval.feedback.provider.openai import OpenAI
        openai_provider = OpenAI()
        groundedness_imp = Groundedness(groundedness_provider=openai_provider)
        ```

        Usage 2:
        ```
        from trulens_eval.feedback import Groundedness
        from trulens_eval.feedback.provider.hugs import Huggingface
        huggingface_provider = Huggingface()
        groundedness_imp = Groundedness(groundedness_provider=huggingface_provider)
        ```

        Args:
            groundedness_provider (Provider, optional): groundedness provider options: OpenAI LLM or HuggingFace NLI. Defaults to OpenAI().
            summarize_provider (Provider, optional): Internal Usage for DB serialization.
        """
        logger.warning(
            "Feedback function `groundedness_measure` was renamed to `groundedness_measure_with_cot_reasons`. The new functionality of `groundedness_measure` function will no longer emit reasons as a lower cost option. It may have reduced accuracy due to not using Chain of Thought reasoning in the scoring."
        )

        summarize_provider = OpenAI()
        if groundedness_provider is None:
            groundedness_provider = OpenAI()
        if not isinstance(groundedness_provider,
                          (OpenAI, AzureOpenAI, Huggingface)):
            raise Exception(
                "Groundedness is only supported groundedness_provider as OpenAI, AzureOpenAI or Huggingface Providers."
            )
        super().__init__(
            summarize_provider=summarize_provider,
            groundedness_provider=groundedness_provider,
            obj=self  # for WithClassInfo
        )

    def groundedness_measure(self, source: str, statement: str) -> float:
        """A measure to track if the source material supports each sentence in the statement. 
        This groundedness measure is faster; but less accurate than `groundedness_measure_with_summarize_step` 

        Usage on RAG Contexts:
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback import Groundedness
        from trulens_eval.feedback.provider.openai import OpenAI
        grounded = feedback.Groundedness(groundedness_provider=OpenAI())


        f_groundedness = feedback.Feedback(grounded.groundedness_measure).on(
            Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
        ).on_output().aggregate(grounded.grounded_statements_aggregator)
        ```
        The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


        Args:
            source (str): The source that should support the statement
            statement (str): The statement to check groundedness

        Returns:
            float: A measure between 0 and 1, where 1 means each sentence is grounded in the source.
        """

        groundedness_scores = {}
        if isinstance(self.groundedness_provider, (AzureOpenAI, OpenAI)):
            groundedness_scores[f"full_doc_score"] = re_1_10_rating(
                self.summarize_provider._groundedness_doc_in_out(
                    source, statement, chain_of_thought=False
                )
            ) / 10
            reason = "Reasons not supplied for non chain of thought function"
        elif isinstance(self.groundedness_provider, Huggingface):
            reason = ""
            for i, hypothesis in enumerate(
                    tqdm(statement.split("."),
                         desc="Groundendess per statement in source")):
                plausible_junk_char_min = 4  # very likely "sentences" under 4 characters are punctuation, spaces, etc
                if len(hypothesis) > plausible_junk_char_min:
                    score = self.groundedness_provider._doc_groundedness(
                        premise=source, hypothesis=hypothesis
                    )
                    reason = reason + str.format(
                        prompts.GROUNDEDNESS_REASON_TEMPLATE,
                        statement_sentence=hypothesis,
                        supporting_evidence="[Doc NLI Used full source]",
                        score=score * 10,
                    )
                    groundedness_scores[f"statement_{i}"] = score

        return groundedness_scores, {"reason": reason}

    def groundedness_measure_with_cot_reasons(
        self, source: str, statement: str
    ) -> float:
        """A measure to track if the source material supports each sentence in the statement. 
        This groundedness measure is faster; but less accurate than `groundedness_measure_with_summarize_step`.
        Also uses chain of thought methodology and emits the reasons.

        Usage on RAG Contexts:
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback import Groundedness
        from trulens_eval.feedback.provider.openai import OpenAI
        grounded = feedback.Groundedness(groundedness_provider=OpenAI())


        f_groundedness = feedback.Feedback(grounded.groundedness_measure_with_cot_reasons).on(
            Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
        ).on_output().aggregate(grounded.grounded_statements_aggregator)
        ```
        The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


        Args:
            source (str): The source that should support the statement
            statement (str): The statement to check groundedness

        Returns:
            float: A measure between 0 and 1, where 1 means each sentence is grounded in the source.
        """
        groundedness_scores = {}
        if isinstance(self.groundedness_provider, (AzureOpenAI, OpenAI)):
            plausible_junk_char_min = 4  # very likely "sentences" under 4 characters are punctuation, spaces, etc
            if len(statement) > plausible_junk_char_min:
                reason = self.summarize_provider._groundedness_doc_in_out(
                    source, statement
                )
            i = 0
            for line in reason.split('\n'):
                if "Score" in line:
                    groundedness_scores[f"statement_{i}"
                                       ] = re_1_10_rating(line) / 10
                    i += 1
            return groundedness_scores, {"reason": reason}
        elif isinstance(self.groundedness_provider, Huggingface):
            raise Exception(
                "Chain of Thought reasoning is only applicable to OpenAI groundedness providers. Instantiate `Groundedness(groundedness_provider=OpenAI())` or use `groundedness_measure` feedback function."
            )

    def groundedness_measure_with_summarize_step(
        self, source: str, statement: str
    ) -> float:
        """A measure to track if the source material supports each sentence in the statement. 
        This groundedness measure is more accurate; but slower using a two step process.
        - First find supporting evidence with an LLM
        - Then for each statement sentence, check groundendness

        Usage on RAG Contexts:
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback import Groundedness
        from trulens_eval.feedback.provider.openai import OpenAI
        grounded = feedback.Groundedness(groundedness_provider=OpenAI())


        f_groundedness = feedback.Feedback(grounded.groundedness_measure_with_summarize_step).on(
            Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
        ).on_output().aggregate(grounded.grounded_statements_aggregator)
        ```
        The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


        Args:
            source (str): The source that should support the statement
            statement (str): The statement to check groundedness

        Returns:
            float: A measure between 0 and 1, where 1 means each sentence is grounded in the source.
        """
        groundedness_scores = {}
        reason = ""
        for i, hypothesis in enumerate(
                tqdm(statement.split("."),
                     desc="Groundendess per statement in source")):
            plausible_junk_char_min = 4  # very likely "sentences" under 4 characters are punctuation, spaces, etc
            if len(hypothesis) > plausible_junk_char_min:
                supporting_premise = self.summarize_provider._find_relevant_string(
                    source, hypothesis
                )
                score = self.groundedness_provider._summarized_groundedness(
                    premise=supporting_premise, hypothesis=hypothesis
                )
                reason = reason + str.format(
                    prompts.GROUNDEDNESS_REASON_TEMPLATE,
                    statement_sentence=hypothesis,
                    supporting_evidence=supporting_premise,
                    score=score * 10,
                )
                groundedness_scores[f"statement_{i}"] = score
        return groundedness_scores, {"reason": reason}

    def grounded_statements_aggregator(
        self, source_statements_multi_output: List[Dict]
    ) -> float:
        """Aggregates multi-input, mulit-output information from the groundedness_measure methods.


        Args:
            source_statements_multi_output (List[Dict]): A list of scores. Each list index is a context. The Dict is a per statement score.

        Returns:
            float: for each statement, gets the max groundedness, then averages over that.
        """
        all_results = []

        statements_to_scores = {}
        for multi_output in source_statements_multi_output:
            for k in multi_output:
                if k not in statements_to_scores:
                    statements_to_scores[k] = []
                statements_to_scores[k].append(multi_output[k])

        for k in statements_to_scores:
            all_results.append(np.max(statements_to_scores[k]))

        return np.mean(all_results)

__init__(summarize_provider=None, groundedness_provider=None)

Instantiates the groundedness providers. Currently the groundedness functions work well with a summarizer. This class will use an OpenAI summarizer to find the relevant strings in a text. The groundedness_provider can either be an llm with OpenAI or NLI with huggingface.

Usage 1:

from trulens_eval.feedback import Groundedness
from trulens_eval.feedback.provider.openai import OpenAI
openai_provider = OpenAI()
groundedness_imp = Groundedness(groundedness_provider=openai_provider)

Usage 2:

from trulens_eval.feedback import Groundedness
from trulens_eval.feedback.provider.hugs import Huggingface
huggingface_provider = Huggingface()
groundedness_imp = Groundedness(groundedness_provider=huggingface_provider)

Parameters:

Name	Type	Description	Default
groundedness_provider	Provider	groundedness provider options: OpenAI LLM or HuggingFace NLI. Defaults to OpenAI().	None
summarize_provider	Provider	Internal Usage for DB serialization.	None

Source code in trulens_eval/trulens_eval/feedback/groundedness.py

def __init__(
    self,
    summarize_provider: Provider = None,
    groundedness_provider: Provider = None
):
    """Instantiates the groundedness providers. Currently the groundedness functions work well with a summarizer.
    This class will use an OpenAI summarizer to find the relevant strings in a text. The groundedness_provider can 
    either be an llm with OpenAI or NLI with huggingface.

    Usage 1:
    ```
    from trulens_eval.feedback import Groundedness
    from trulens_eval.feedback.provider.openai import OpenAI
    openai_provider = OpenAI()
    groundedness_imp = Groundedness(groundedness_provider=openai_provider)
    ```

    Usage 2:
    ```
    from trulens_eval.feedback import Groundedness
    from trulens_eval.feedback.provider.hugs import Huggingface
    huggingface_provider = Huggingface()
    groundedness_imp = Groundedness(groundedness_provider=huggingface_provider)
    ```

    Args:
        groundedness_provider (Provider, optional): groundedness provider options: OpenAI LLM or HuggingFace NLI. Defaults to OpenAI().
        summarize_provider (Provider, optional): Internal Usage for DB serialization.
    """
    logger.warning(
        "Feedback function `groundedness_measure` was renamed to `groundedness_measure_with_cot_reasons`. The new functionality of `groundedness_measure` function will no longer emit reasons as a lower cost option. It may have reduced accuracy due to not using Chain of Thought reasoning in the scoring."
    )

    summarize_provider = OpenAI()
    if groundedness_provider is None:
        groundedness_provider = OpenAI()
    if not isinstance(groundedness_provider,
                      (OpenAI, AzureOpenAI, Huggingface)):
        raise Exception(
            "Groundedness is only supported groundedness_provider as OpenAI, AzureOpenAI or Huggingface Providers."
        )
    super().__init__(
        summarize_provider=summarize_provider,
        groundedness_provider=groundedness_provider,
        obj=self  # for WithClassInfo
    )

grounded_statements_aggregator(source_statements_multi_output)

Aggregates multi-input, mulit-output information from the groundedness_measure methods.

Parameters:

Name	Type	Description	Default
source_statements_multi_output	List[Dict]	A list of scores. Each list index is a context. The Dict is a per statement score.	required

Returns:

Name	Type	Description
float	float	for each statement, gets the max groundedness, then averages over that.

Source code in trulens_eval/trulens_eval/feedback/groundedness.py

def grounded_statements_aggregator(
    self, source_statements_multi_output: List[Dict]
) -> float:
    """Aggregates multi-input, mulit-output information from the groundedness_measure methods.


    Args:
        source_statements_multi_output (List[Dict]): A list of scores. Each list index is a context. The Dict is a per statement score.

    Returns:
        float: for each statement, gets the max groundedness, then averages over that.
    """
    all_results = []

    statements_to_scores = {}
    for multi_output in source_statements_multi_output:
        for k in multi_output:
            if k not in statements_to_scores:
                statements_to_scores[k] = []
            statements_to_scores[k].append(multi_output[k])

    for k in statements_to_scores:
        all_results.append(np.max(statements_to_scores[k]))

    return np.mean(all_results)

groundedness_measure(source, statement)

A measure to track if the source material supports each sentence in the statement. This groundedness measure is faster; but less accurate than groundedness_measure_with_summarize_step

Usage on RAG Contexts:

from trulens_eval import Feedback
from trulens_eval.feedback import Groundedness
from trulens_eval.feedback.provider.openai import OpenAI
grounded = feedback.Groundedness(groundedness_provider=OpenAI())


f_groundedness = feedback.Feedback(grounded.groundedness_measure).on(
    Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
).on_output().aggregate(grounded.grounded_statements_aggregator)

The on(...) selector can be changed. See Feedback Function Guide : Selectors

Parameters:

Name	Type	Description	Default
source	str	The source that should support the statement	required
statement	str	The statement to check groundedness	required

Returns:

Name	Type	Description
float	float	A measure between 0 and 1, where 1 means each sentence is grounded in the source.

Source code in trulens_eval/trulens_eval/feedback/groundedness.py

def groundedness_measure(self, source: str, statement: str) -> float:
    """A measure to track if the source material supports each sentence in the statement. 
    This groundedness measure is faster; but less accurate than `groundedness_measure_with_summarize_step` 

    Usage on RAG Contexts:
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback import Groundedness
    from trulens_eval.feedback.provider.openai import OpenAI
    grounded = feedback.Groundedness(groundedness_provider=OpenAI())


    f_groundedness = feedback.Feedback(grounded.groundedness_measure).on(
        Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
    ).on_output().aggregate(grounded.grounded_statements_aggregator)
    ```
    The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


    Args:
        source (str): The source that should support the statement
        statement (str): The statement to check groundedness

    Returns:
        float: A measure between 0 and 1, where 1 means each sentence is grounded in the source.
    """

    groundedness_scores = {}
    if isinstance(self.groundedness_provider, (AzureOpenAI, OpenAI)):
        groundedness_scores[f"full_doc_score"] = re_1_10_rating(
            self.summarize_provider._groundedness_doc_in_out(
                source, statement, chain_of_thought=False
            )
        ) / 10
        reason = "Reasons not supplied for non chain of thought function"
    elif isinstance(self.groundedness_provider, Huggingface):
        reason = ""
        for i, hypothesis in enumerate(
                tqdm(statement.split("."),
                     desc="Groundendess per statement in source")):
            plausible_junk_char_min = 4  # very likely "sentences" under 4 characters are punctuation, spaces, etc
            if len(hypothesis) > plausible_junk_char_min:
                score = self.groundedness_provider._doc_groundedness(
                    premise=source, hypothesis=hypothesis
                )
                reason = reason + str.format(
                    prompts.GROUNDEDNESS_REASON_TEMPLATE,
                    statement_sentence=hypothesis,
                    supporting_evidence="[Doc NLI Used full source]",
                    score=score * 10,
                )
                groundedness_scores[f"statement_{i}"] = score

    return groundedness_scores, {"reason": reason}

groundedness_measure_with_cot_reasons(source, statement)

A measure to track if the source material supports each sentence in the statement. This groundedness measure is faster; but less accurate than groundedness_measure_with_summarize_step. Also uses chain of thought methodology and emits the reasons.

Usage on RAG Contexts:

from trulens_eval import Feedback
from trulens_eval.feedback import Groundedness
from trulens_eval.feedback.provider.openai import OpenAI
grounded = feedback.Groundedness(groundedness_provider=OpenAI())


f_groundedness = feedback.Feedback(grounded.groundedness_measure_with_cot_reasons).on(
    Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
).on_output().aggregate(grounded.grounded_statements_aggregator)

The on(...) selector can be changed. See Feedback Function Guide : Selectors

Parameters:

Name	Type	Description	Default
source	str	The source that should support the statement	required
statement	str	The statement to check groundedness	required

Returns:

Name	Type	Description
float	float	A measure between 0 and 1, where 1 means each sentence is grounded in the source.

Source code in trulens_eval/trulens_eval/feedback/groundedness.py

def groundedness_measure_with_cot_reasons(
    self, source: str, statement: str
) -> float:
    """A measure to track if the source material supports each sentence in the statement. 
    This groundedness measure is faster; but less accurate than `groundedness_measure_with_summarize_step`.
    Also uses chain of thought methodology and emits the reasons.

    Usage on RAG Contexts:
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback import Groundedness
    from trulens_eval.feedback.provider.openai import OpenAI
    grounded = feedback.Groundedness(groundedness_provider=OpenAI())


    f_groundedness = feedback.Feedback(grounded.groundedness_measure_with_cot_reasons).on(
        Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
    ).on_output().aggregate(grounded.grounded_statements_aggregator)
    ```
    The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


    Args:
        source (str): The source that should support the statement
        statement (str): The statement to check groundedness

    Returns:
        float: A measure between 0 and 1, where 1 means each sentence is grounded in the source.
    """
    groundedness_scores = {}
    if isinstance(self.groundedness_provider, (AzureOpenAI, OpenAI)):
        plausible_junk_char_min = 4  # very likely "sentences" under 4 characters are punctuation, spaces, etc
        if len(statement) > plausible_junk_char_min:
            reason = self.summarize_provider._groundedness_doc_in_out(
                source, statement
            )
        i = 0
        for line in reason.split('\n'):
            if "Score" in line:
                groundedness_scores[f"statement_{i}"
                                   ] = re_1_10_rating(line) / 10
                i += 1
        return groundedness_scores, {"reason": reason}
    elif isinstance(self.groundedness_provider, Huggingface):
        raise Exception(
            "Chain of Thought reasoning is only applicable to OpenAI groundedness providers. Instantiate `Groundedness(groundedness_provider=OpenAI())` or use `groundedness_measure` feedback function."
        )

groundedness_measure_with_summarize_step(source, statement)

A measure to track if the source material supports each sentence in the statement. This groundedness measure is more accurate; but slower using a two step process. - First find supporting evidence with an LLM - Then for each statement sentence, check groundendness

Usage on RAG Contexts:

from trulens_eval import Feedback
from trulens_eval.feedback import Groundedness
from trulens_eval.feedback.provider.openai import OpenAI
grounded = feedback.Groundedness(groundedness_provider=OpenAI())


f_groundedness = feedback.Feedback(grounded.groundedness_measure_with_summarize_step).on(
    Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
).on_output().aggregate(grounded.grounded_statements_aggregator)

The on(...) selector can be changed. See Feedback Function Guide : Selectors

Parameters:

Name	Type	Description	Default
source	str	The source that should support the statement	required
statement	str	The statement to check groundedness	required

Returns:

Name	Type	Description
float	float	A measure between 0 and 1, where 1 means each sentence is grounded in the source.

Source code in trulens_eval/trulens_eval/feedback/groundedness.py

def groundedness_measure_with_summarize_step(
    self, source: str, statement: str
) -> float:
    """A measure to track if the source material supports each sentence in the statement. 
    This groundedness measure is more accurate; but slower using a two step process.
    - First find supporting evidence with an LLM
    - Then for each statement sentence, check groundendness

    Usage on RAG Contexts:
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback import Groundedness
    from trulens_eval.feedback.provider.openai import OpenAI
    grounded = feedback.Groundedness(groundedness_provider=OpenAI())


    f_groundedness = feedback.Feedback(grounded.groundedness_measure_with_summarize_step).on(
        Select.Record.app.combine_documents_chain._call.args.inputs.input_documents[:].page_content # See note below
    ).on_output().aggregate(grounded.grounded_statements_aggregator)
    ```
    The `on(...)` selector can be changed. See [Feedback Function Guide : Selectors](https://www.trulens.org/trulens_eval/feedback_function_guide/#selector-details)


    Args:
        source (str): The source that should support the statement
        statement (str): The statement to check groundedness

    Returns:
        float: A measure between 0 and 1, where 1 means each sentence is grounded in the source.
    """
    groundedness_scores = {}
    reason = ""
    for i, hypothesis in enumerate(
            tqdm(statement.split("."),
                 desc="Groundendess per statement in source")):
        plausible_junk_char_min = 4  # very likely "sentences" under 4 characters are punctuation, spaces, etc
        if len(hypothesis) > plausible_junk_char_min:
            supporting_premise = self.summarize_provider._find_relevant_string(
                source, hypothesis
            )
            score = self.groundedness_provider._summarized_groundedness(
                premise=supporting_premise, hypothesis=hypothesis
            )
            reason = reason + str.format(
                prompts.GROUNDEDNESS_REASON_TEMPLATE,
                statement_sentence=hypothesis,
                supporting_evidence=supporting_premise,
                score=score * 10,
            )
            groundedness_scores[f"statement_{i}"] = score
    return groundedness_scores, {"reason": reason}

GroundTruthAgreement

Bases: SerialModel, WithClassInfo

Measures Agreement against a Ground Truth.

Source code in trulens_eval/trulens_eval/feedback/groundtruth.py

class GroundTruthAgreement(SerialModel, WithClassInfo):
    """Measures Agreement against a Ground Truth.
    """
    ground_truth: Union[List[str], FunctionOrMethod]
    provider: Provider
    # Note: the bert scorer object isn't serializable
    # It's a class member because creating it is expensive
    bert_scorer: object

    ground_truth_imp: Optional[Callable] = pydantic.Field(exclude=True)

    class Config:
        arbitrary_types_allowed = True

    def __init__(
        self,
        ground_truth: Union[List, Callable, FunctionOrMethod],
        provider: Provider = None,
        bert_scorer: Optional["BERTScorer"] = None
    ):
        """Measures Agreement against a Ground Truth. 

        Usage 1:
        ```
        from trulens_eval.feedback import GroundTruthAgreement
        golden_set = [
            {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
            {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
        ]
        ground_truth_collection = GroundTruthAgreement(golden_set)
        ```

        Usage 2:
        ```
        from trulens_eval.feedback import GroundTruthAgreement
        ground_truth_imp = llm_app
        response = llm_app(prompt)
        ground_truth_collection = GroundTruthAgreement(ground_truth_imp)
        ```

        Args:
            ground_truth (Union[Callable, FunctionOrMethod]): A list of query/response pairs or a function or callable that returns a ground truth string given a prompt string.
            bert_scorer (Optional["BERTScorer"], optional): Internal Usage for DB serialization.
            provider (Provider, optional): Internal Usage for DB serialization.

        """
        provider = OpenAI()
        if isinstance(ground_truth, List):
            ground_truth_imp = None
        elif isinstance(ground_truth, FunctionOrMethod):
            ground_truth_imp = ground_truth.load()
        elif isinstance(ground_truth, Callable):
            ground_truth_imp = ground_truth
            ground_truth = FunctionOrMethod.of_callable(ground_truth)
        elif isinstance(ground_truth, Dict):
            # Serialized FunctionOrMethod?
            ground_truth = FunctionOrMethod.pick(**ground_truth)
            ground_truth_imp = ground_truth.load()
        else:
            raise RuntimeError(
                f"Unhandled ground_truth type: {type(ground_truth)}."
            )

        super().__init__(
            ground_truth=ground_truth,
            ground_truth_imp=ground_truth_imp,
            provider=provider,
            bert_scorer=bert_scorer,
            obj=self  # for WithClassInfo
        )

    def _find_response(self, prompt: str) -> Optional[str]:
        if self.ground_truth_imp is not None:
            return self.ground_truth_imp(prompt)

        responses = [
            qr["response"] for qr in self.ground_truth if qr["query"] == prompt
        ]
        if responses:
            return responses[0]
        else:
            return None

    def _find_score(self, prompt: str, response: str) -> Optional[float]:
        if self.ground_truth_imp is not None:
            return self.ground_truth_imp(prompt)

        responses = [
            qr["expected_score"]
            for qr in self.ground_truth
            if qr["query"] == prompt and qr["response"] == response
        ]
        if responses:
            return responses[0]
        else:
            return None

    def agreement_measure(
        self, prompt: str, response: str
    ) -> Union[float, Tuple[float, Dict[str, str]]]:
        """
        Uses OpenAI's Chat GPT Model. A function that that measures
        similarity to ground truth. A second template is given to Chat GPT
        with a prompt that the original response is correct, and measures
        whether previous Chat GPT's response is similar.

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback import GroundTruthAgreement
        golden_set = [
            {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
            {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
        ]
        ground_truth_collection = GroundTruthAgreement(golden_set)

        feedback = Feedback(ground_truth_collection.agreement_measure).on_input_output() 
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            prompt (str): A text prompt to an agent. 
            response (str): The agent's response to the prompt.

        Returns:
            - float: A value between 0 and 1. 0 being "not in agreement" and 1
                being "in agreement".
            - dict: with key 'ground_truth_response'
        """
        ground_truth_response = self._find_response(prompt)
        if ground_truth_response:
            agreement_txt = self.provider._get_answer_agreement(
                prompt, response, ground_truth_response
            )
            ret = re_1_10_rating(agreement_txt) / 10, dict(
                ground_truth_response=ground_truth_response
            )
        else:
            ret = np.nan

        return ret

    def numeric_difference(
        self, prompt: str, response: str, score: float
    ) -> float:
        """
        Method to look up the numeric expected score from a golden set and take the differnce.

        Primarily used for feedback evaluation.

        **Usage**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback import GroundTruthAgreement

        golden_set =
        {"query": "How many stomachs does a cow have?", "response": "Cows' diet relies primarily on grazing.", "expected_score": 0.4},
        {"query": "Name some top dental floss brands", "response": "I don't know", "expected_score": 0.8}
        ]
        ground_truth_collection = GroundTruthAgreement(golden_set)

        f_groundtruth = Feedback(ground_truth.numeric_difference).on(Select.Record.calls[0].args.args[0]).on(Select.Record.calls[0].args.args[1]).on_output()
        ```

        """

        expected_score = self._find_score(prompt, response)
        if expected_score:
            ret = 1 - abs(float(score) - expected_score)
            expected_score = "{:.2f}".format(expected_score
                                            ).rstrip('0').rstrip('.')
        else:
            ret = np.nan
        return ret, {"expected score": expected_score}

    def bert_score(self, prompt: str,
                   response: str) -> Union[float, Tuple[float, Dict[str, str]]]:
        """
        Uses BERT Score. A function that that measures
        similarity to ground truth using bert embeddings. 

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback import GroundTruthAgreement
        golden_set = [
            {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
            {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
        ]
        ground_truth_collection = GroundTruthAgreement(golden_set)

        feedback = Feedback(ground_truth_collection.bert_score).on_input_output() 
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


        Args:
            prompt (str): A text prompt to an agent.
            response (str): The agent's response to the prompt.

        Returns:
            - float: A value between 0 and 1. 0 being "not in agreement" and 1
                being "in agreement".
            - dict: with key 'ground_truth_response'
        """
        if self.bert_scorer is None:
            self.bert_scorer = BERTScorer(lang="en", rescale_with_baseline=True)
        ground_truth_response = self._find_response(prompt)
        if ground_truth_response:
            bert_score = self.bert_scorer.score(
                [response], [ground_truth_response]
            )
            ret = bert_score[0].item(), dict(
                ground_truth_response=ground_truth_response
            )
        else:
            ret = np.nan

        return ret

    def bleu(self, prompt: str,
             response: str) -> Union[float, Tuple[float, Dict[str, str]]]:
        """
        Uses BLEU Score. A function that that measures
        similarity to ground truth using token overlap. 

        **Usage:**
        ```
        from trulens_eval import Feedback
        from trulens_eval.feedback import GroundTruthAgreement
        golden_set = [
            {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
            {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
        ]
        ground_truth_collection = GroundTruthAgreement(golden_set)

        feedback = Feedback(ground_truth_collection.bleu).on_input_output() 
        ```
        The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

        Args:
            prompt (str): A text prompt to an agent. 
            response (str): The agent's response to the prompt.

        Returns:
            - float: A value between 0 and 1. 0 being "not in agreement" and 1
                being "in agreement".
            - dict: with key 'ground_truth_response'
        """
        bleu = evaluate.load('bleu')
        ground_truth_response = self._find_response(prompt)
        if ground_truth_response:
            bleu_score = bleu.compute(
                predictions=[response], references=[ground_truth_response]
            )
            ret = bleu_score['bleu'], dict(
                ground_truth_response=ground_truth_response
            )
        else:
            ret = np.nan

        return ret

    def rouge(self, prompt: str,
              response: str) -> Union[float, Tuple[float, Dict[str, str]]]:
        """
        Uses BLEU Score. A function that that measures
        similarity to ground truth using token overlap. 

        Args:
            prompt (str): A text prompt to an agent. 
            response (str): The agent's response to the prompt.

        Returns:
            - float: A value between 0 and 1. 0 being "not in agreement" and 1
                being "in agreement".
            - dict: with key 'ground_truth_response'
        """
        rouge = evaluate.load('rouge')
        ground_truth_response = self._find_response(prompt)
        if ground_truth_response:
            rouge_score = rouge.compute(
                predictions=[response], references=[ground_truth_response]
            )
            ret = rouge_score['rouge1'], dict(
                ground_truth_response=ground_truth_response
            )
        else:
            ret = np.nan

        return ret

__init__(ground_truth, provider=None, bert_scorer=None)

Measures Agreement against a Ground Truth.

Usage 1:

from trulens_eval.feedback import GroundTruthAgreement
golden_set = [
    {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
    {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
]
ground_truth_collection = GroundTruthAgreement(golden_set)

Usage 2:

from trulens_eval.feedback import GroundTruthAgreement
ground_truth_imp = llm_app
response = llm_app(prompt)
ground_truth_collection = GroundTruthAgreement(ground_truth_imp)

Parameters:

Name	Type	Description	Default
ground_truth	Union[Callable, FunctionOrMethod]	A list of query/response pairs or a function or callable that returns a ground truth string given a prompt string.	required
bert_scorer	Optional["BERTScorer"]	Internal Usage for DB serialization.	None
provider	Provider	Internal Usage for DB serialization.	None

Source code in trulens_eval/trulens_eval/feedback/groundtruth.py

def __init__(
    self,
    ground_truth: Union[List, Callable, FunctionOrMethod],
    provider: Provider = None,
    bert_scorer: Optional["BERTScorer"] = None
):
    """Measures Agreement against a Ground Truth. 

    Usage 1:
    ```
    from trulens_eval.feedback import GroundTruthAgreement
    golden_set = [
        {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
        {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
    ]
    ground_truth_collection = GroundTruthAgreement(golden_set)
    ```

    Usage 2:
    ```
    from trulens_eval.feedback import GroundTruthAgreement
    ground_truth_imp = llm_app
    response = llm_app(prompt)
    ground_truth_collection = GroundTruthAgreement(ground_truth_imp)
    ```

    Args:
        ground_truth (Union[Callable, FunctionOrMethod]): A list of query/response pairs or a function or callable that returns a ground truth string given a prompt string.
        bert_scorer (Optional["BERTScorer"], optional): Internal Usage for DB serialization.
        provider (Provider, optional): Internal Usage for DB serialization.

    """
    provider = OpenAI()
    if isinstance(ground_truth, List):
        ground_truth_imp = None
    elif isinstance(ground_truth, FunctionOrMethod):
        ground_truth_imp = ground_truth.load()
    elif isinstance(ground_truth, Callable):
        ground_truth_imp = ground_truth
        ground_truth = FunctionOrMethod.of_callable(ground_truth)
    elif isinstance(ground_truth, Dict):
        # Serialized FunctionOrMethod?
        ground_truth = FunctionOrMethod.pick(**ground_truth)
        ground_truth_imp = ground_truth.load()
    else:
        raise RuntimeError(
            f"Unhandled ground_truth type: {type(ground_truth)}."
        )

    super().__init__(
        ground_truth=ground_truth,
        ground_truth_imp=ground_truth_imp,
        provider=provider,
        bert_scorer=bert_scorer,
        obj=self  # for WithClassInfo
    )

agreement_measure(prompt, response)

Uses OpenAI's Chat GPT Model. A function that that measures similarity to ground truth. A second template is given to Chat GPT with a prompt that the original response is correct, and measures whether previous Chat GPT's response is similar.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback import GroundTruthAgreement
golden_set = [
    {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
    {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
]
ground_truth_collection = GroundTruthAgreement(golden_set)

feedback = Feedback(ground_truth_collection.agreement_measure).on_input_output() 

The on_input_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Type	Description
Union[float, Tuple[float, Dict[str, str]]]	float: A value between 0 and 1. 0 being "not in agreement" and 1 being "in agreement".
Union[float, Tuple[float, Dict[str, str]]]	dict: with key 'ground_truth_response'

Source code in trulens_eval/trulens_eval/feedback/groundtruth.py

def agreement_measure(
    self, prompt: str, response: str
) -> Union[float, Tuple[float, Dict[str, str]]]:
    """
    Uses OpenAI's Chat GPT Model. A function that that measures
    similarity to ground truth. A second template is given to Chat GPT
    with a prompt that the original response is correct, and measures
    whether previous Chat GPT's response is similar.

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback import GroundTruthAgreement
    golden_set = [
        {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
        {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
    ]
    ground_truth_collection = GroundTruthAgreement(golden_set)

    feedback = Feedback(ground_truth_collection.agreement_measure).on_input_output() 
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        prompt (str): A text prompt to an agent. 
        response (str): The agent's response to the prompt.

    Returns:
        - float: A value between 0 and 1. 0 being "not in agreement" and 1
            being "in agreement".
        - dict: with key 'ground_truth_response'
    """
    ground_truth_response = self._find_response(prompt)
    if ground_truth_response:
        agreement_txt = self.provider._get_answer_agreement(
            prompt, response, ground_truth_response
        )
        ret = re_1_10_rating(agreement_txt) / 10, dict(
            ground_truth_response=ground_truth_response
        )
    else:
        ret = np.nan

    return ret

bert_score(prompt, response)

Uses BERT Score. A function that that measures similarity to ground truth using bert embeddings.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback import GroundTruthAgreement
golden_set = [
    {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
    {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
]
ground_truth_collection = GroundTruthAgreement(golden_set)

feedback = Feedback(ground_truth_collection.bert_score).on_input_output() 

The on_input_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Type	Description
Union[float, Tuple[float, Dict[str, str]]]	float: A value between 0 and 1. 0 being "not in agreement" and 1 being "in agreement".
Union[float, Tuple[float, Dict[str, str]]]	dict: with key 'ground_truth_response'

Source code in trulens_eval/trulens_eval/feedback/groundtruth.py

def bert_score(self, prompt: str,
               response: str) -> Union[float, Tuple[float, Dict[str, str]]]:
    """
    Uses BERT Score. A function that that measures
    similarity to ground truth using bert embeddings. 

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback import GroundTruthAgreement
    golden_set = [
        {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
        {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
    ]
    ground_truth_collection = GroundTruthAgreement(golden_set)

    feedback = Feedback(ground_truth_collection.bert_score).on_input_output() 
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)


    Args:
        prompt (str): A text prompt to an agent.
        response (str): The agent's response to the prompt.

    Returns:
        - float: A value between 0 and 1. 0 being "not in agreement" and 1
            being "in agreement".
        - dict: with key 'ground_truth_response'
    """
    if self.bert_scorer is None:
        self.bert_scorer = BERTScorer(lang="en", rescale_with_baseline=True)
    ground_truth_response = self._find_response(prompt)
    if ground_truth_response:
        bert_score = self.bert_scorer.score(
            [response], [ground_truth_response]
        )
        ret = bert_score[0].item(), dict(
            ground_truth_response=ground_truth_response
        )
    else:
        ret = np.nan

    return ret

bleu(prompt, response)

Uses BLEU Score. A function that that measures similarity to ground truth using token overlap.

Usage:

from trulens_eval import Feedback
from trulens_eval.feedback import GroundTruthAgreement
golden_set = [
    {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
    {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
]
ground_truth_collection = GroundTruthAgreement(golden_set)

feedback = Feedback(ground_truth_collection.bleu).on_input_output() 

The on_input_output() selector can be changed. See Feedback Function Guide

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Type	Description
Union[float, Tuple[float, Dict[str, str]]]	float: A value between 0 and 1. 0 being "not in agreement" and 1 being "in agreement".
Union[float, Tuple[float, Dict[str, str]]]	dict: with key 'ground_truth_response'

Source code in trulens_eval/trulens_eval/feedback/groundtruth.py

def bleu(self, prompt: str,
         response: str) -> Union[float, Tuple[float, Dict[str, str]]]:
    """
    Uses BLEU Score. A function that that measures
    similarity to ground truth using token overlap. 

    **Usage:**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback import GroundTruthAgreement
    golden_set = [
        {"query": "who invented the lightbulb?", "response": "Thomas Edison"},
        {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"}
    ]
    ground_truth_collection = GroundTruthAgreement(golden_set)

    feedback = Feedback(ground_truth_collection.bleu).on_input_output() 
    ```
    The `on_input_output()` selector can be changed. See [Feedback Function Guide](https://www.trulens.org/trulens_eval/feedback_function_guide/)

    Args:
        prompt (str): A text prompt to an agent. 
        response (str): The agent's response to the prompt.

    Returns:
        - float: A value between 0 and 1. 0 being "not in agreement" and 1
            being "in agreement".
        - dict: with key 'ground_truth_response'
    """
    bleu = evaluate.load('bleu')
    ground_truth_response = self._find_response(prompt)
    if ground_truth_response:
        bleu_score = bleu.compute(
            predictions=[response], references=[ground_truth_response]
        )
        ret = bleu_score['bleu'], dict(
            ground_truth_response=ground_truth_response
        )
    else:
        ret = np.nan

    return ret

numeric_difference(prompt, response, score)

Method to look up the numeric expected score from a golden set and take the differnce.

Primarily used for feedback evaluation.

Usage

from trulens_eval import Feedback
from trulens_eval.feedback import GroundTruthAgreement

golden_set =
{"query": "How many stomachs does a cow have?", "response": "Cows' diet relies primarily on grazing.", "expected_score": 0.4},
{"query": "Name some top dental floss brands", "response": "I don't know", "expected_score": 0.8}
]
ground_truth_collection = GroundTruthAgreement(golden_set)

f_groundtruth = Feedback(ground_truth.numeric_difference).on(Select.Record.calls[0].args.args[0]).on(Select.Record.calls[0].args.args[1]).on_output()

Source code in trulens_eval/trulens_eval/feedback/groundtruth.py

def numeric_difference(
    self, prompt: str, response: str, score: float
) -> float:
    """
    Method to look up the numeric expected score from a golden set and take the differnce.

    Primarily used for feedback evaluation.

    **Usage**
    ```
    from trulens_eval import Feedback
    from trulens_eval.feedback import GroundTruthAgreement

    golden_set =
    {"query": "How many stomachs does a cow have?", "response": "Cows' diet relies primarily on grazing.", "expected_score": 0.4},
    {"query": "Name some top dental floss brands", "response": "I don't know", "expected_score": 0.8}
    ]
    ground_truth_collection = GroundTruthAgreement(golden_set)

    f_groundtruth = Feedback(ground_truth.numeric_difference).on(Select.Record.calls[0].args.args[0]).on(Select.Record.calls[0].args.args[1]).on_output()
    ```

    """

    expected_score = self._find_score(prompt, response)
    if expected_score:
        ret = 1 - abs(float(score) - expected_score)
        expected_score = "{:.2f}".format(expected_score
                                        ).rstrip('0').rstrip('.')
    else:
        ret = np.nan
    return ret, {"expected score": expected_score}

rouge(prompt, response)

Uses BLEU Score. A function that that measures similarity to ground truth using token overlap.

Parameters:

Name	Type	Description	Default
prompt	str	A text prompt to an agent.	required
response	str	The agent's response to the prompt.	required

Returns:

Type	Description
Union[float, Tuple[float, Dict[str, str]]]	float: A value between 0 and 1. 0 being "not in agreement" and 1 being "in agreement".
Union[float, Tuple[float, Dict[str, str]]]	dict: with key 'ground_truth_response'

Source code in trulens_eval/trulens_eval/feedback/groundtruth.py

def rouge(self, prompt: str,
          response: str) -> Union[float, Tuple[float, Dict[str, str]]]:
    """
    Uses BLEU Score. A function that that measures
    similarity to ground truth using token overlap. 

    Args:
        prompt (str): A text prompt to an agent. 
        response (str): The agent's response to the prompt.

    Returns:
        - float: A value between 0 and 1. 0 being "not in agreement" and 1
            being "in agreement".
        - dict: with key 'ground_truth_response'
    """
    rouge = evaluate.load('rouge')
    ground_truth_response = self._find_response(prompt)
    if ground_truth_response:
        rouge_score = rouge.compute(
            predictions=[response], references=[ground_truth_response]
        )
        ret = rouge_score['rouge1'], dict(
            ground_truth_response=ground_truth_response
        )
    else:
        ret = np.nan

    return ret