📓 Logging Human Feedback¶

In many situations, it can be useful to log human feedback from your users about your LLM app's performance. Combining human feedback along with automated feedback can help you drill down on subsets of your app that underperform, and uncover new failure modes. This example will walk you through a simple example of recording human feedback with TruLens.

Create app¶

In [ ]:

# Create a simple (fake) app.

from trulens.apps.app import TruApp
from trulens.core.otel.instrument import instrument
from trulens.core.session import TruSession

tru_session = TruSession()
tru_session.reset_database()


class MyApp:
    @instrument()
    def completion(self, prompt: str) -> str:
        return "I don't actually do anything, I'm just a placeholder!"


app = MyApp()
tru_app = TruApp(app, app_name="MyApp", app_version="v1")

# Create a simple (fake) app. from trulens.apps.app import TruApp from trulens.core.otel.instrument import instrument from trulens.core.session import TruSession tru_session = TruSession() tru_session.reset_database() class MyApp: @instrument() def completion(self, prompt: str) -> str: return "I don't actually do anything, I'm just a placeholder!" app = MyApp() tru_app = TruApp(app, app_name="MyApp", app_version="v1")

In [ ]:

# Invoke the app.

with tru_app as recording:
    app.completion("Give me 10 names for a colorful sock company")

# Get the record to add the feedback to.
record = recording.get()

# Invoke the app. with tru_app as recording: app.completion("Give me 10 names for a colorful sock company") # Get the record to add the feedback to. record = recording.get()

Create a mechanism for recording human feedback¶

Be sure to click an emoji in the record to record human_feedback to log.

In [ ]:

from ipywidgets import Button
from ipywidgets import HBox
from ipywidgets import Label
from ipywidgets import VBox
from trulens.core.session import TruSession

thumbs_up_button = Button(description="👍")
thumbs_down_button = Button(description="👎")


def update_feedback(human_feedback: float) -> None:
    # add the human feedback to a particular app and record
    tru_session.add_feedback_result(
        record=record,
        feedback_name="Human Feedback",
        feedback_result=human_feedback,
        higher_is_better=True,
    )
    tru_session.force_flush()


def on_thumbs_up_button_clicked(b):
    update_feedback(human_feedback=1)
    print("👍")


def on_thumbs_down_button_clicked(b):
    update_feedback(human_feedback=0)
    print("👎")


thumbs_up_button.on_click(on_thumbs_up_button_clicked)
thumbs_down_button.on_click(on_thumbs_down_button_clicked)

VBox([
    Label(record.main_input),
    Label(record.main_output),
    HBox([thumbs_up_button, thumbs_down_button]),
])

from ipywidgets import Button from ipywidgets import HBox from ipywidgets import Label from ipywidgets import VBox from trulens.core.session import TruSession thumbs_up_button = Button(description="👍") thumbs_down_button = Button(description="👎") def update_feedback(human_feedback: float) -> None: # add the human feedback to a particular app and record tru_session.add_feedback_result( record=record, feedback_name="Human Feedback", feedback_result=human_feedback, higher_is_better=True, ) tru_session.force_flush() def on_thumbs_up_button_clicked(b): update_feedback(human_feedback=1) print("👍") def on_thumbs_down_button_clicked(b): update_feedback(human_feedback=0) print("👎") thumbs_up_button.on_click(on_thumbs_up_button_clicked) thumbs_down_button.on_click(on_thumbs_down_button_clicked) VBox([ Label(record.main_input), Label(record.main_output), HBox([thumbs_up_button, thumbs_down_button]), ])

In [ ]:

# After clicking on a thumbs up or down button, the leaderboard will update.

tru_session.get_leaderboard(app_ids=[tru_app.app_id])

# After clicking on a thumbs up or down button, the leaderboard will update. tru_session.get_leaderboard(app_ids=[tru_app.app_id])