AI Agent 评估系统实战：使用 LangSmith + 自定义指标量化 Agent 性能与可靠性

首先配置 LangSmith 追踪器并初始化 Agent：

from langchain.agents import initialize_agent, Tool
from langchain_openai import ChatOpenAI
from langsmith import traceable

# 初始化 LLM
llm = ChatOpenAI(model="gpt-4o", temperature=0)

# 定义自定义工具
def search_web(query: str) -> str:
    """搜索网络获取信息"""
    return f"搜索 '{query}' 的结果..."

# 初始化工具列表
tools = [
    Tool(
        name="WebSearch",
        func=search_web,
        description="用于搜索实时信息"
    )
]

# 初始化 Agent
agent = initialize_agent(
    tools=tools,
    llm=llm,
    agent="zero-shot-react-description"
)

使用 @traceable 装饰器自动记录执行轨迹：

from langsmith import traceable

@traceable(run_type="chain", name="Agent Execution")
def run_agent_with_trace(query: str):
    """运行 Agent 并记录 Trace"""
    response = agent.run(query)
    return response

# 执行测试
result = run_agent_with_trace("帮我写一封产品推广邮件")
print(f"结果：{result}")

运行后，在 LangSmith 仪表盘查看完整的执行 Trace，包括：

• 每一步的工具调用和参数
• LLM 输入输出和 token 消耗
• 执行时间和错误信息

LangSmith 支持 Python 函数形式的自定义评估器：

from langsmith.evaluation import evaluate
from langsmith.schemas import Run, Example

def tool_selection_accuracy(run: Run, example: Example) -> dict:
    """评估工具选择准确率"""
    # 检查是否调用了正确的工具
    tool_calls = run.serialized.get('tool_calls', [])
    expected_tool = example.outputs.get('expected_tool')

    if expected_tool:
        correct = any(call['name'] == expected_tool for call in tool_calls)
        return {'score': 1.0 if correct else 0.0, 'key': 'tool_accuracy'}
    return {'score': None}

def planning_quality(run: Run, example: Example) -> dict:
    """评估规划质量：步骤是否连贯、是否有冗余"""
    steps = run.outputs.get('steps', [])

    # 检查步骤连贯性
    has_redundant = len(steps) > 10  # 简化示例
    has_logical_flow = len(steps) >= 2

    score = 0.5
    if has_logical_flow and not has_redundant:
        score = 1.0
    elif has_redundant:
        score = 0.3

    return {'score': score, 'key': 'planning_quality'}

让 LLM 充当评判者，评估 Agent 输出质量：

from langsmith import Client
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate

client = Client()
judge_llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)

def llm_judge(run: Run, example: Example) -> dict:
    """使用 LLM 评估回答的相关性和准确性"""

    prompt = ChatPromptTemplate.from_messages([
        ("system", "你是 AI Agent 评估专家。请评估以下回答的质量。"),
        ("human", """问题：{question}
期望答案：{expected}
实际回答：{actual}

请从以下维度打分（1-5 分）：
1. 相关性：回答是否切题
2. 准确性：信息是否正确
3. 完整性：是否覆盖关键点

返回 JSON 格式：{{"relevance": 1-5, "accuracy": 1-5, "completeness": 1-5}}""")

    chain = prompt | judge_llm
    response = chain.invoke({
        'question': example.inputs['query'],
        'expected': example.outputs.get('expected_answer', 'N/A'),
        'actual': run.outputs.get('output', '')
    })

    import json
    scores = json.loads(response.content)
    return {
        'score': (scores['relevance'] + scores['accuracy'] + scores['completeness']) / 15,
        'key': 'llm_judge'
    }

使用 LangSmith 的 evaluate 函数批量运行评估：

def run_agent(query: str) -> str:
    """Agent 运行函数"""
    return agent.run(query)

# 准备测试数据集
test_examples = [
    {"inputs": {"query": "帮我写一封产品推广邮件"}, "outputs": {"expected_tool": "WebSearch"}},
    {"inputs": {"query": "查询今天北京的天气"}, "outputs": {"expected_tool": "WebSearch"}},
    {"inputs": {"query": "总结这篇文章的核心观点"}, "outputs": {"expected_tool": None}},
]

# 执行评估
results = evaluate(
    run_agent,
    data=test_examples,
    evaluators=[
        tool_selection_accuracy,
        planning_quality,
        llm_judge
    ],
    experiment_prefix="agent-eval-march-2026"
)

# 查看结果
for result in results:
    print(f"得分：{result.scores}")

在生产环境部署监控：

from langsmith import Client

client = Client()

# 创建监控仪表盘
dashboard = client.create_dashboard(
    name="Production Agent Monitor",
    filters={
        "project_name": "agent-production",
        "time_range": "last_24h"
    }
)

# 配置告警规则
def check_budget_threshold():
    """检查成本是否超出阈值"""
    runs = client.list_runs(
        project_name="agent-production",
        execution_order=1
    )

    total_cost = sum(run.total_cost or 0 for run in runs)

    if total_cost > 100:  # 100 美元阈值
        send_alert(f"⚠️ 成本告警：当前花费 ${total_cost:.2f}")

    return total_cost

基于评估结果持续优化 Agent：

# 从生产 Trace 中收集困难样本
hard_examples = client.list_examples(
    dataset_name="production-failures",
    filters={"score_lt": 0.5}  # 低分样本
)

# 使用困难样本进行回归测试
regression_results = evaluate(
    run_agent,
    data=hard_examples,
    evaluators=[llm_judge],
    experiment_prefix="regression-test"
)

# 对比不同版本的表现
v1_scores = [r.scores['llm_judge'] for r in regression_results.v1]
v2_scores = [r.scores['llm_judge'] for r in regression_results.v2]

print(f"V1 平均分：{sum(v1_scores)/len(v1_scores):.2f}")
print(f"V2 平均分：{sum(v2_scores)/len(v2_scores):.2f}")