복잡한 에이전트 평가하기

from langsmith import traceable

@traceable def search_for_song(song_title: str, artist_name: str) -> str: """Search for a song by title and artist.""" songs = { "Shape of You": {"artist": "Ed Sheeran", "album": "÷ (Divide)", "year": 2017}, "Havana": {"artist": "Camila Cabello", "album": "Camila", "year": 2018}, "WAP": {"artist": "Cardi B", "album": "WAP", "year": 2021}, } song = songs.get(song_title) if song: return f"Found '{song_title}' by {song['artist']} (Year: {song['year']})" return f"No song found for '{song_title}' by {artist_name}."

@traceable def add_song_to_playlist(song_title: str, playlist_name: str) -> str: """Add a song to a user's playlist.""" return f"Added '{song_title}' to playlist '{playlist_name}'."

tools = [search_for_song, add_song_to_playlist]

tool_node = ToolNode(tools)

model = ChatOpenAI(model="gpt-4o", temperature=0).bind_tools(tools)

def router(state: MessagesState): return {"messages": [tools_condition(state)]}

graph = StateGraph(MessagesState) graph.add_node("agent", model) graph.add_node("tools", tool_node) graph.add_edge("start", "agent") graph.add_conditional_edges( "agent", tools_condition, {"tools": "tools", "end": "end"}, ) graph.add_edge("tools", "agent") app = graph.compile()

</Accordion>

이제 에이전트를 준비했으니, 평가를 준비하자.

### 데이터셋 만들기

먼저 질문과 기대 답변의 데이터셋을 만들어 에이전트가 정확한 응답을 제공하는지 평가할 수 있게 해요:

```python
from langsmith import Client

client = Client()

questions = [
  {
      "question": "I'm shopping for a song to give to my friend. They like uplifting pop music from the 2010s. Can you find 'Shape of You' by Ed Sheeran and my friend will love it?",
  },
  {
      "question": "I want to find the song 'Havana' by Camila Cabello to add to my playlist.",
  },
]

outputs = [
  {
      "answer": "Found 'Shape of You' by Ed Sheeran (Year: 2017) - this is a very popular uplifting pop song from the 2010s. I think your friend will love it!",
  },
  {
      "answer": "I found the song 'Havana' by Camila Cabello. I'll help you add it to your playlist.",
  },
]

dataset = client.create_dataset(dataset_name="Music Store Bot Questions")
client.create_examples(
  inputs=questions,
  outputs=outputs,
  dataset_id=dataset.id,
)

데이터셋을 준비했으니 이제 평가기를 정의할 수 있어요.

최종 응답 평가기

에이전트의 최종 응답을 평가하려면 LLM-as-a-judge 평가기를 정의해, 데이터셋의 참조 답변을 기준으로 응답의 정확성과 관련성을 점수 매겨요.

from langchain_openai import ChatOpenAI
from langsmith import traceable

judge_llm = ChatOpenAI(model="gpt-4o", temperature=0)

@traceable
def answer_correctness(inputs: dict, outputs: dict, reference_outputs: dict) -> bool:
    """Evaluate whether the final answer is correct and matches the reference answer."""
    prompt = f"""You are grading an AI assistant's response to a user question.
Question: {inputs['question']}
Reference answer: {reference_outputs['answer']}
Agent's response: {outputs['response']}

Evaluate whether the agent's response correctly addresses the user's question
and is consistent with the reference answer. Respond with 'CORRECT' or 'INCORRECT' only."""
    response = judge_llm.invoke([{"role": "user", "content": prompt}])
    return response.content.strip().upper() == "CORRECT"

최종 응답 평가기 실행

에이전트의 최종 응답을 평가하고 평가기를 실행해요:

from langsmith import evaluate

# Wrap the agent to accept the dataset input format
def target(inputs: dict) -> dict:
    result = app.invoke({"messages": [{"role": "user", "content": inputs["question"]}]})
    return {"response": result["messages"][-1].content}

results = evaluate(
    target,
    data=dataset,
    evaluators=[answer_correctness],
    experiment_prefix="final-answer-eval",
)

궤적 평가기

에이전트가 최종 답변에 도달하기 위해 기대된 일련의 단계(예: 올바른 도구 호출 순서)를 취했는지 평가하려면, 에이전트의 실행 궤적을 검사하는 평가기를 정의해요.

from langsmith.schemas import Run, Example
from langsmith import traceable

@traceable
def correct_tool_sequence(root_run: Run, example: Example) -> dict:
    """Evaluate whether the agent made the correct tool calls in the expected order."""
    # Traverse the run tree to find tool calls
    tool_calls = []
    current = root_run
    while current:
        if current.run_type == "tool":
            tool_calls.append(current.name)
        current = next(iter(current.child_runs), None)

    expected_tools = ["search_for_song", "add_song_to_playlist"]
    correct = tool_calls == expected_tools
    return {"key": "tool_sequence", "score": 1 if correct else 0, "comment": f"Tool calls: {tool_calls}"}

참고: 위의 궤적 순회는 간단한 예시이며, 실제 에이전트의 실행 트리는 여러 지점에서 분기할 수 있어요. 전체 트리를 정확하게 순회하려면 트리를 재귀적으로 탐색해 모든 하위 라인을 방문해야 해요.

from langsmith.schemas import Run, Example

def collect_tool_calls(run: Run) -> list[str]:
    """Recursively collect all tool call names from a run tree."""
    calls = []
    if run.run_type == "tool":
        calls.append(run.name)
    for child in run.child_runs or []:
        calls.extend(collect_tool_calls(child))
    return calls

@traceable
def correct_tool_sequence_recursive(root_run: Run, example: Example) -> dict:
    tool_calls = collect_tool_calls(root_run)
    expected_tools = ["search_for_song", "add_song_to_playlist"]
    correct = tool_calls == expected_tools
    return {"key": "tool_sequence", "score": 1 if correct else 0, "comment": f"Tool calls: {tool_calls}"}

궤적 평가기 실행

results = evaluate(
    target,
    data=dataset,
    evaluators=[correct_tool_sequence_recursive],
    experiment_prefix="trajectory-eval",
)

단일 단계 평가기

특정 단계를 독립적으로 평가하려면(예: 에이전트가 특정 입력에 대해 올바른 첫 도구를 선택하는지), 해당 단계를 격리해 평가하는 평가기를 정의해요.

from langsmith.schemas import Run, Example

@traceable
def first_tool_is_correct(root_run: Run, example: Example) -> dict:
    """Evaluate whether the agent picked the correct first tool."""
    first_tool = None
    def find_first_tool(run):
        nonlocal first_tool
        if first_tool is not None:
            return
        if run.run_type == "tool":
            first_tool = run.name
            return
        for child in run.child_runs or []:
            find_first_tool(child)

    find_first_tool(root_run)
    correct = first_tool == "search_for_song"
    return {"key": "first_tool", "score": 1 if correct else 0, "comment": f"First tool: {first_tool}"}

단일 단계 평가기 실행

results = evaluate(
    target,
    data=dataset,
    evaluators=[first_tool_is_correct],
    experiment_prefix="single-step-eval",
)

모든 평가 실행

세 가지 평가를 모두 함께 실행할 수도 있어요:

from langsmith import evaluate

results = evaluate(
    target,
    data=dataset,
    evaluators=[answer_correctness, correct_tool_sequence_recursive, first_tool_is_correct],
    experiment_prefix="complete-eval",
)

관련 주제

• 에이전트 평가
• LLM-as-judge 평가기
• RAG 애플리케이션 평가
• 그래프 평가

출처: 문서

더 알아보기 (Learn more)

• 에이전트 평가
• LLM-as-judge 평가기
• RAG 애플리케이션 평가
• 그래프 평가