"""扩展基准：多步轨迹 + 多轮对话用例。 bench50_v2.py 只测「单发话语 → 首个工具选择」；本脚本补它测不到的： - 多步 ReAct 轨迹（喂回固定工具结果，跑到模型给出最终文字为止） - 多轮接续（翻页、代词回指、澄清后接续） - 回答夹带新需求 - 「改审核人」类消歧（改带"审核"字样的字段 ≠ 审核操作） - 长会话回指（30 轮后引用最早实体） - 跨轮流程接续（弹表单后隔轮补字段，值须逐字保留） - 提议卡后的状态纪律（不重复提议、不声称已执行） 用法： uv run --no-project bench_ext.py --arch old # 现产线：统一 prompt + 6 工具 uv run --no-project bench_ext.py --arch new # skill-ReAct（Phase 2 落地后从仓库资源读 prompt/skill） uv run --no-project bench_ext.py --arch old --only M5 """ import json import sys import time import urllib.request URL = "http://112.82.245.194:41434/v1/chat/completions" MODEL = "qwen3.6-27b-iq3:latest" MAX_STEPS = 8 # ---------------- 与 bench50_v2 相同的生产复刻 prompt / 工具定义 ---------------- import pathlib # noqa: E402 from bench50_v2 import (UNIFIED_PROMPT, ALL6, DOMAIN_MAP, fn, CASES as FIFTY_CASES, # noqa: E402 T_FINDFORMS, T_READFORM, T_LOOKUP, T_ASK) ROOT = pathlib.Path(__file__).resolve().parent.parent SKILLS_DIR = ROOT / "src/main/resources/skills" T_USESKILL = fn("useSkill", "载入一项技能（业务流程的权威步骤说明），返回完整步骤文本。凡要 新建/报价/问价/修改/作废/审核 等" "会改动数据的业务，或不确定流程怎么走，先用它载入对应技能再动手。技能名见 system prompt 的技能清单。", {"skillName": {"type": "string", "description": "技能名（见 system prompt 的技能清单）"}}, ["skillName"]) # rearch3：模型侧工具全部只读/只渲染——collectForm/previewChange 与生产 @Tool 文本同源 T_COLLECT_NEW = fn("collectForm", "在对话里弹出一张 ERP 表单让用户一次性填齐多个字段（而不是逐个追问、更不是自己猜）。用于字段较多的新建/录入场景" "（尤其**新建报价**）。entityKeyword = 单据类型（如 报价 / 客户）；knownFieldsJson = 用户已说的字段(中文名->值)用于预填" "（如 {\"产品名称\":\"纸盒\",\"数量\":\"1000\",\"长(L)\":\"50\"}）。客户/产品/物料会渲染成下拉让用户从真实数据里选。" "用户填完点【保存】后，系统自动校验并提交待办——你**无需**再做任何写操作，也绝不能说已保存。", {"entityKeyword": {"type": "string", "description": "要新建的实体/单据类型，如 报价 / 客户 / 物料"}, "knownFieldsJson": {"type": "string", "description": "可选：用户已说的字段 JSON（中文名->值），用于预填表单"}}, ["entityKeyword"]) T_PREVIEW = fn("previewChange", "为一次写操作生成**预览卡**（只读：本工具绝不写入数据，也不生成任何单据）。卡片展示记录当前内容、" "改动或状态变化高亮，并带对应操作按钮（保存/审核/反审核/作废/取消作废/删除）；只有用户点了按钮，" "系统才会提交待办。action：update=改字段（recordKeyword+fieldChinese+newValue）；" "invalid=作废（业务单据要“删除/取消”一律用它，可复原）；cancelInvalid=复原/取消作废；examine=审核；" "cancelExamine=销审/反审核；delete=物理删除（业务单据别用）。**新增不用本工具**（新增走 collectForm）。" "本工具自行定位主表与记录，无需先 findForms。预览卡出现后你就停下等用户，绝不能说已保存/已完成。", {"action": {"type": "string", "description": "动作：update|invalid|cancelInvalid|examine|cancelExamine|delete"}, "entityKeyword": {"type": "string", "description": "实体/单据类型，如 报价 / 客户 / 销售订单"}, "recordKeyword": {"type": "string", "description": "记录名称或单号关键词"}, "fieldChinese": {"type": "string", "description": "要改的字段中文名（仅 update）"}, "newValue": {"type": "string", "description": "新值（仅 update，逐字照抄用户原话）"}}, ["action", "entityKeyword", "recordKeyword"]) def load_skills(): """{名称: (一句话用途, 正文)} —— 与生产 SkillService 同源同格式（首行=名，次行=用途，余=正文）。""" out = {} for p in sorted(SKILLS_DIR.glob("*.md")): parts = p.read_text(encoding="utf-8").split("\n", 2) if len(parts) >= 3: out[parts[0].strip()] = (parts[1].strip(), parts[2].strip()) return out def build_context(arch): """返回 (system_prompt, tools)。new 架构从仓库资源文件读取（与生产同源），避免复刻漂移。 old = rearch1 统一 prompt + proposeWrite（历史基线）；new = rearch3 skill + 只读工具（previewChange）。""" if arch == "old": return UNIFIED_PROMPT, ALL6 if arch == "new": template = (ROOT / "src/main/resources/prompts/system.txt").read_text(encoding="utf-8") skills = load_skills() index = "".join(f"- {name}：{brief}\n" for name, (brief, _) in skills.items()) sp = template.replace("{DOMAIN_MAP}", DOMAIN_MAP).replace("{SKILL_INDEX}", index) return sp, [T_USESKILL, T_FINDFORMS, T_READFORM, T_LOOKUP, T_COLLECT_NEW, T_PREVIEW, T_ASK] raise SystemExit(f"未知 arch: {arch}") # ---------------- 固定工具结果（fixtures） ---------------- CUSTOMERS = { "必胜客": {"客户名称": "必胜客", "联系电话": "13912345678", "地址": "上海市静安区南京西路1266号", "销售员": "王五"}, "上海创远包装": {"客户名称": "上海创远包装", "联系电话": "021-58991234", "地址": "浦东新区川沙路500号", "销售员": "李四"}, "苏州华为": {"客户名称": "苏州华为", "联系电话": "0512-67771234", "地址": "苏州工业园区", "销售员": "赵六"}, "杭州大华印务": {"客户名称": "杭州大华印务", "联系电话": "0571-87654321", "地址": "杭州市余杭区", "销售员": "钱七"}, } FORMS = { "报价": [{"formName": "报价单", "formId": "FQ01", "moduleId": "M-BJ", "source": "quoquotationmaster"}], "客户": [{"formName": "客户资料", "formId": "FC01", "moduleId": "M-JC", "source": "custmaster"}], "送货": [{"formName": "送货单", "formId": "FD01", "moduleId": "M-XS", "source": "deliverymaster"}], "销售": [{"formName": "销售订单", "formId": "FS01", "moduleId": "M-XS", "source": "salesorder"}], "订单": [{"formName": "销售订单", "formId": "FS01", "moduleId": "M-XS", "source": "salesorder"}], "库存": [{"formName": "库存表", "formId": "FI01", "moduleId": "M-KC", "source": "inventory"}], "物料": [{"formName": "物料资料", "formId": "FM01", "moduleId": "M-JC", "source": "materials"}], "供应商": [{"formName": "供应商资料", "formId": "FG01", "moduleId": "M-JC", "source": "suppliermaster"}], "采购": [{"formName": "采购订单", "formId": "FP01", "moduleId": "M-CG", "source": "purchaseorder"}], "应收": [{"formName": "应收账款", "formId": "FA01", "moduleId": "M-CW", "source": "receivable"}], "生产": [{"formName": "生产任务单", "formId": "FR01", "moduleId": "M-SC", "source": "productiontask"}], } READFORM_PAGES = { ("FC01", 1): "共 6 条记录，第 1 页：必胜客、上海创远包装、苏州华为、杭州大华印务", ("FC01", 2): "共 6 条记录，第 2 页：宁波天海包装、无锡礼盒厂", ("FD01", 1): "共 2 条记录，第 1 页：DH202607005（客户 必胜客，未审核）、DH202607008（客户 苏州华为，已审核）", ("FQ01", 1): "共 2 条记录，第 1 页：BJD202607001（客户 必胜客，产品 披萨盒，数量 5000，未审核）、" "BJD202607082（客户 苏州华为，产品 彩盒，数量 1000，已审核）", ("FI01", 1): "共 5 条记录，第 1 页（按库存量降序）：瓦楞纸 30,000 张、铜版纸 12,000 张、300g白卡纸 8,500 张、" "灰板纸 6,200 张、油墨(青) 410 桶", ("FM01", 1): "共 4 条记录，第 1 页：铜版纸、300g白卡纸、瓦楞纸、灰板纸", ("FS01", 1): "共 3 条记录，第 1 页：SO202607010（客户 必胜客，金额 356,000 元）、SO202607011（客户 上海创远包装，金额 120,000 元）、" "SO202607012（客户 必胜客，金额 89,000 元）", ("FG01", 1): "共 2 条记录，第 1 页：金田纸业（电话 0571-88881234）、杭州新新材料", ("FP01", 1): "共 2 条记录，第 1 页：PO202607003（供应商 金田纸业）、PO202607004（供应商 杭州新新材料）", ("FA01", 1): "共 1 条记录：应收账款合计 1,560,000 元（其中逾期 230,000 元）", ("FR01", 1): "共 4 条记录，第 1 页：MO202607021（披萨盒 印刷中）、MO202607022（彩盒 模切）、" "MO202607023（画册 装订）、MO202607024（纸箱 待排程）", } RECORDS = { "BJD202607001": {"单号": "BJD202607001", "客户": "必胜客", "产品": "披萨盒", "数量": "5000", "审核状态": "未审核", "审核人": "", "业务员": "王五"}, "BJD202607082": {"单号": "BJD202607082", "客户": "苏州华为", "产品": "彩盒", "数量": "1000", "审核状态": "已审核", "审核人": "赵六", "业务员": "李四"}, "DH202607005": {"单号": "DH202607005", "客户": "必胜客", "审核状态": "未审核"}, "DH202607008": {"单号": "DH202607008", "客户": "苏州华为", "审核状态": "已审核"}, "铜版纸": {"物料名称": "铜版纸", "库存": "12000 张"}, "300g白卡纸": {"物料名称": "300g白卡纸", "库存": "8500 张"}, "金田纸业": {"供应商名称": "金田纸业", "联系电话": "0571-88881234"}, } def fixture_result(name, args): args = args or {} if name == "findForms": kw = args.get("keyword", "") for k, forms in FORMS.items(): if k in kw: return json.dumps(forms, ensure_ascii=False) return json.dumps([{"formName": kw + "列表", "formId": "FX99", "moduleId": "M-QT", "source": "misc"}], ensure_ascii=False) if name == "readFormData": page = int(args.get("page") or 1) kw = args.get("keyword") or "" if kw: for rname, rec in {**CUSTOMERS, **RECORDS}.items(): if rname in kw or kw in rname: return f"共 1 条记录：{json.dumps(rec, ensure_ascii=False)}" return f"共 0 条记录（keyword={kw} 无匹配）" key = (args.get("formId", ""), page) if key in READFORM_PAGES: return READFORM_PAGES[key] return f"共 0 条记录（formId={args.get('formId','')} page={page}）" if name == "lookupRecord": rk = args.get("recordKeyword", "") for rname, rec in {**CUSTOMERS, **RECORDS}.items(): if rname in rk or rk in rname: return json.dumps(rec, ensure_ascii=False) return f"未找到「{rk}」对应的记录" if name == "collectForm": return (f"[已弹出「{args.get('entityKeyword','')}」表单，预填={args.get('knownFieldsJson') or '{}'}。" "等待用户在表单里补齐并点提交，本轮到此为止。]") if name == "proposeWrite": return (f"[已生成待确认提议 OP-TEST-1：action={args.get('action','')}，实体={args.get('entityKeyword','')}，" f"记录={args.get('recordKeyword','')}。仅提议，未执行；请用户点【确认】。]") if name == "previewChange": return (f"[已生成预览卡 PV-TEST-1：action={args.get('action','')}，实体={args.get('entityKeyword','')}，" f"记录={args.get('recordKeyword','')}。只读预览，未写入；等待用户点卡上按钮。]") if name == "askUser": return "[问题已发给用户，等待回答，本轮到此为止。]" if name == "useSkill": skills = load_skills() want = args.get("skillName", "") for sname, (_, body) in skills.items(): if sname == want or sname in want or want in sname: return body return "没有叫「" + want + "」的技能。可用技能：" + "、".join(skills) return "OK" # ---------------- 轨迹模拟器 ---------------- def call(body, timeout=200): req = urllib.request.Request( URL, data=json.dumps(body).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer ollama"}) with urllib.request.urlopen(req, timeout=timeout) as r: return json.loads(r.read()) def run_trajectory_once(system_prompt, tools, history, utterance): """history: [(role, text)] 已渲染的往轮。返回 (calls, final_text, steps, err)。""" messages = [{"role": "system", "content": system_prompt}] for role, text in history: messages.append({"role": role, "content": text}) messages.append({"role": "user", "content": utterance}) calls = [] for step in range(MAX_STEPS): body = {"model": MODEL, "stream": False, "temperature": 0.1, "top_p": 0.9, "reasoning_effort": "none", "tools": tools, "messages": messages} try: resp = call(body) except Exception as ex: return calls, None, step, str(ex) msg = resp["choices"][0]["message"] tcs = msg.get("tool_calls") or [] if not tcs: return calls, (msg.get("content") or "").strip(), step + 1, None messages.append({"role": "assistant", "content": msg.get("content") or "", "tool_calls": tcs}) for tc in tcs: name = tc["function"]["name"] try: args = json.loads(tc["function"].get("arguments") or "{}") except Exception: args = {} calls.append((name, args)) messages.append({"role": "tool", "tool_call_id": tc.get("id", "tc"), "content": fixture_result(name, args)}) return calls, None, MAX_STEPS, "step-cap" GUARD_NUDGE = "你上一条回答没有调用任何工具、数字疑似编造。请先用工具查询真实数据，再重新回答这个问题：" def run_trajectory(system_prompt, tools, history, utterance): """带生产反编造护栏的轨迹：零工具却答出数字 → 注入纠正话术重试一次（复刻 AgentChatController）。 判分归一：previewChange（rearch3 只读预览工具）与 proposeWrite 语义同位，统一按 proposeWrite 判。""" calls, text, steps, err = run_trajectory_once(system_prompt, tools, history, utterance) if not (err or calls or not text or not any(c.isdigit() for c in text)): retry_hist = history + [("user", utterance), ("assistant", text)] calls, text, steps2, err = run_trajectory_once(system_prompt, tools, retry_hist, GUARD_NUDGE + utterance) steps += steps2 calls = [("proposeWrite" if n == "previewChange" else n, a) for n, a in calls] return calls, text, steps, err # ---------------- 判分辅助 ---------------- def argstr(args): return json.dumps(args or {}, ensure_ascii=False) def find(calls, name, pred=None): for n, a in calls: if n == name and (pred is None or pred(a)): return a return None def any_read(calls, needle=None): for n, a in calls: if n in ("findForms", "readFormData", "lookupRecord"): if needle is None or needle in argstr(a): return a return None def asked(calls, final_text): return find(calls, "askUser") is not None or (final_text and ("？" in final_text or "?" in final_text)) # ---------------- 用例 ---------------- # 每条: id, cat, history, utter, judge(calls, text) -> ("PASS"|"PARTIAL"|"FAIL", reason) def j_shenheren_update(target_val): def judge(calls, text): p = find(calls, "proposeWrite") if p and p.get("action") == "update": if target_val in argstr(p): return "PASS", "" return "PARTIAL", f"update 但参数缺 {target_val}" if p and p.get("action") in ("examine", "cancelExamine"): return "FAIL", f"误判为审核操作 action={p.get('action')}" if asked(calls, text): return "PARTIAL", "反问（信息已足够）" return "FAIL", "未产生 update 提议" return judge def mk(id_, cat, history, utter, judge, desc): return {"id": id_, "cat": cat, "history": history, "utter": utter, "judge": judge, "desc": desc} def long_history(): pairs = [ ("查一下杭州大华印务的资料", "杭州大华印务：印刷类客户，地址杭州市余杭区，销售员钱七。"), ("必胜客上个月下了几单？", "必胜客上个月共 3 张销售订单。"), ("铜版纸库存还有多少？", "铜版纸当前库存 12,000 张。"), ("上海创远包装的地址？", "浦东新区川沙路500号。"), ("这个月开了几张送货单？", "本月共 18 张送货单。"), ("苏州华为的销售员是谁？", "赵六。"), ("白卡纸多少库存？", "300g 白卡纸库存 8,500 张。"), ("最近有新采购订单吗？", "最近一周有 2 张采购订单。"), ("应收账款总额？", "当前应收账款合计 156 万元。"), ("宁波天海包装是我们客户吗？", "是，客户资料里有宁波天海包装。"), ("今天的生产任务？", "今日 4 个生产任务，均在排程中。"), ("金田纸业的电话？", "0571-88881234。"), ("上季度销售额最高的客户？", "必胜客，合计 89 万元。"), ("画册类产品最近报价几张？", "最近一个月 5 张画册报价单。"), ("无锡礼盒厂有欠款吗？", "无锡礼盒厂当前无逾期欠款。"), ] h = [] for q, a in pairs: h.append(("user", q)) h.append(("assistant", a)) return h def j_m4(calls, text): if any_read(calls, "杭州大华印务"): return "PASS", "" if "钱七" in (text or ""): return "PASS", "从历史正确回忆（未重查）" if asked(calls, text): return "PARTIAL", "反问是哪个客户（历史里可查）" return "FAIL", "既没读 杭州大华印务 也没答对销售员" def j_m5(calls, text): c = find(calls, "collectForm") if c: s = argstr(c) if "大16开" in s and "5000" in s: return "PASS", "" if "16开" in s and "大16开" not in s: return "FAIL", "尺寸值被改写（大16开→16开），违反逐字照抄" return "PARTIAL", f"collectForm 但预填不全: {s[:80]}" p = find(calls, "proposeWrite", lambda a: a.get("action") == "create") if p and "大16开" in argstr(p): return "PASS", "直接 create 且值逐字" return "FAIL", "未回到报价表单流程" def j_m7(calls, text): if find(calls, "proposeWrite"): return "FAIL", "重复生成提议" done_words = [w for w in ("已修改", "已完成", "修改成功") if w in (text or "")] if done_words: return "FAIL", f"声称已执行: {done_words}" if any_read(calls, "必胜客"): return "PASS", "" return "FAIL", "未查询必胜客地址" def j_m3(calls, text): upd = find(calls, "proposeWrite", lambda a: a.get("action") == "update" and "13800138000" in argstr(a) and "必胜客" in argstr(a)) qry = any_read(calls, "上海") if upd and qry: return "PASS", "" if upd: return "PARTIAL", "只完成修改提议，丢了夹带的查询" if qry: return "PARTIAL", "只做了查询，丢了修改" return "FAIL", "两个请求都没完成" CASES = [ # --- 「审核」字样消歧：改带审核字样的字段是 update，不是 examine --- mk("S1", "消歧", [], "把BJD202607001的审核人改成张三", j_shenheren_update("张三"), "改审核人=update"), mk("S2", "消歧", [], "报价单BJD202607082的审核人换成李四", j_shenheren_update("李四"), "换审核人=update"), mk("S3", "消歧", [], "查一下BJD202607001是谁审核的", lambda calls, text: ("PASS", "") if any_read(calls) or (text and not any(ch.isdigit() for ch in text)) else ("FAIL", "既没读也没答"), "查审核人=查询"), mk("S4", "消歧", [], "BJD202607001不用审了，帮我撤回来", lambda calls, text: ("PASS", "") if find(calls, "proposeWrite", lambda a: a.get("action") == "cancelExamine") else (("PARTIAL", "反问") if asked(calls, text) else ("FAIL", "未识别为销审")), "撤回审核=cancelExamine"), # 上下文无候选、记录指代不清 → 反问哪张与直接 cancelInvalid 提议同为正确行为 mk("S5", "消歧", [], "上次作废的那张送货单帮我恢复一下", lambda calls, text: ("PASS", "") if find(calls, "proposeWrite", lambda a: a.get("action") == "cancelInvalid") or asked(calls, text) else ("FAIL", "未识别为复原"), "恢复作废=cancelInvalid或问哪张"), # --- 多轮接续 --- mk("M1", "多轮接续", [("user", "查一下客户列表"), ("assistant", "共 6 个客户，第 1 页：必胜客、上海创远包装、苏州华为、杭州大华印务。（表单：客户资料 formId=FC01 moduleId=M-JC）")], "下一页", lambda calls, text: ("PASS", "") if find(calls, "readFormData", lambda a: int(a.get("page") or 1) == 2) else ("FAIL", "没有翻到第 2 页"), "翻页接续"), mk("M2", "多轮接续", [("user", "查一下必胜客的资料"), ("assistant", "必胜客：地址 上海市静安区南京西路1266号，销售员 王五。")], "他的电话是多少", lambda calls, text: (("PASS", "") if "13912345678" in (text or "") else ("PARTIAL", "读了但答案未含固定电话")) if any_read(calls, "必胜客") else ("FAIL", "未解析代词「他」=必胜客"), "代词回指"), mk("M6", "多轮接续", [("user", "作废那张送货单"), ("assistant", "请问是哪一张送货单？候选：DH202607005（必胜客）、DH202607008（苏州华为）")], "第一张", lambda calls, text: ("PASS", "") if find(calls, "proposeWrite", lambda a: a.get("action") in ("invalid",) and "DH202607005" in argstr(a)) else (("PARTIAL", "再次反问") if asked(calls, text) else ("FAIL", "未把「第一张」解析为 DH202607005")), "澄清后接续"), # --- 夹带新需求 --- mk("M3", "夹带", [("user", "把必胜客的电话改一下"), ("assistant", "请问要改成什么号码？")], "13800138000，另外查下上海有哪些客户", j_m3, "回答+新查询"), # --- 长会话回指 --- mk("M4", "长会话回指", long_history(), "最早问的那个客户，帮我查下它的销售员", j_m4, "30 轮后回指最早实体"), # --- 跨轮表单接续 --- mk("M5", "跨轮表单", [("user", "报价纸盒"), ("assistant", "[已为您弹出「报价」表单，请填写客户、产品、数量、尺寸等字段后提交。]"), ("user", "先查下必胜客的电话"), ("assistant", "必胜客电话：13912345678。")], "报价那个，尺寸大16开，数量5000", j_m5, "隔轮补字段，值逐字"), # --- 提议卡后的状态纪律 --- mk("M7", "提议纪律", [("user", "把必胜客电话改成13800138000"), ("assistant", "[已生成待确认提议 OP123：修改 客户「必胜客」联系电话 → 13800138000]，请点击【确认】执行。")], "顺便查下它的地址", j_m7, "提议后查询：不重提、不谎称已执行"), ] # ---------------- bench50 用例的全轨迹判分（新旧架构同口径对照） ---------------- def judge_fifty(exp, calls, text): """与 bench50_v2.judge 同语义，但看整条轨迹而非首步。useSkill 是自由动作，不计对错。""" biz = [(n, a) for n, a in calls if n != "useSkill"] actions = [(a or {}).get("action", "") for n, a in biz if n == "proposeWrite"] names = [n for n, _ in biz] ask = asked(calls, text) if exp == "查询": if any(n in READ_TOOLS_50 for n in names): return "ok" if not biz and text and not any(c.isdigit() for c in text): return "ok" return "fail" if exp == "新增": if "collectForm" in names or "create" in actions: return "ok" return "clarify" if ask else "fail" if exp == "修改": if "update" in actions or "askUser" in names or "lookupRecord" in names: return "ok" return "clarify" if ask else "fail" if exp == "删除": if any(a in ("invalid", "delete", "cancelInvalid") for a in actions): return "ok" return "clarify" if ask else "fail" if exp == "审核": if any(a in ("examine", "cancelExamine") for a in actions): return "ok" return "clarify" if ask else "fail" if exp == "闲聊": return "ok" if not biz and text else "fail" if exp == "不清楚": if "askUser" in names or (not biz and text): return "ok" return "fail" return "fail" READ_TOOLS_50 = {"findForms", "readFormData", "lookupRecord"} def run_fifty(arch, only=None): system_prompt, tools = build_context(arch) cases = [(u, e) for u, e in FIFTY_CASES if only is None or only in u] print(f"bench50 全轨迹判分 arch={arch} model={MODEL} cases={len(cases)}") strict = 0 lenient = 0 per = {} fails = [] for utt, exp in cases: t0 = time.time() calls, text, steps, err = run_trajectory(system_prompt, tools, [], utt) dt = time.time() - t0 v = "fail" if err else judge_fifty(exp, calls, text) if err: fails.append((utt, exp, f"ERROR {err}")) strict += v == "ok" lenient += v in ("ok", "clarify") s, l, n = per.get(exp, (0, 0, 0)) per[exp] = (s + (v == "ok"), l + (v in ("ok", "clarify")), n + 1) mark = {"ok": "✓", "clarify": "◐", "fail": "✗"}[v] traj = " → ".join(n for n, _ in calls) or "文字" if v == "fail" and not err: fails.append((utt, exp, traj + (" 「" + (text or "")[:60].replace("\n", " ") + "」" if text else ""))) print(f"{mark} [{exp}] {utt} → {traj} ({steps}步 {dt:.0f}s)") n = len(cases) print(f"\n[{arch}] bench50 全轨迹: 严格 {strict}/{n} = {strict/n*100:.0f}% 宽松 {lenient}/{n} = {lenient/n*100:.0f}%") print("按意图分（严格/宽松/总数）:", {k: f"{s}/{l}/{c}" for k, (s, l, c) in per.items()}) for utt, exp, d in fails: print(f" 失败: 「{utt}」[{exp}] → {d}") def main(): arch = "old" only = None argv = sys.argv[1:] if "--arch" in argv: arch = argv[argv.index("--arch") + 1] if "--only" in argv: only = argv[argv.index("--only") + 1] if "--fifty" in argv: run_fifty(arch, only) return system_prompt, tools = build_context(arch) cases = [c for c in CASES if only is None or c["id"] == only] print(f"arch={arch} model={MODEL} cases={len(cases)}") results = [] for c in cases: t0 = time.time() calls, text, steps, err = run_trajectory(system_prompt, tools, c["history"], c["utter"]) dt = time.time() - t0 if err: verdict, reason = "FAIL", f"ERROR {err}" else: verdict, reason = c["judge"](calls, text) results.append((c, verdict, reason)) mark = {"PASS": "✓", "PARTIAL": "◐", "FAIL": "✗"}[verdict] traj = " → ".join(n for n, _ in calls) or "文字" print(f'{mark} [{c["cat"]}] {c["id"]} {c["desc"]}: {traj}' f'{" | " + reason if reason else ""} ({steps}步 {dt:.0f}s)') if verdict != "PASS" and text: print(f' 最终答复: {text[:100]}') n = len(results) strict = sum(v == "PASS" for _, v, _ in results) lenient = sum(v in ("PASS", "PARTIAL") for _, v, _ in results) by_cat = {} for c, v, _ in results: s, l, t = by_cat.get(c["cat"], (0, 0, 0)) by_cat[c["cat"]] = (s + (v == "PASS"), l + (v != "FAIL"), t + 1) print(f"\n[{arch}] 严格 {strict}/{n} 宽松 {lenient}/{n}") for cat, (s, l, t) in by_cat.items(): print(f" {cat}: 严格 {s}/{t} 宽松 {l}/{t}") if __name__ == "__main__": main()