From 5ffcf59fcedffc0aff2bae4bc387844baaf9d767 Mon Sep 17 00:00:00 2001 From: zichun <26684461+reporkey@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:08:29 +0800 Subject: [PATCH] bench: enrich fixture world (quotes/inventory/orders), --fifty --only for case re-verification; record P3 final acceptance (ext 12/12, fifty 84% verified vs old 82%) --- bench/bench_ext.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/bench/bench_ext.py b/bench/bench_ext.py index 39bdc0e..85b1eb9 100644 --- a/bench/bench_ext.py +++ b/bench/bench_ext.py @@ -76,12 +76,43 @@ FORMS = { "客户": [{"formName": "客户资料", "formId": "FC01", "moduleId": "M-JC", "source": "custmaster"}], "送货": [{"formName": "送货单", "formId": "FD01", "moduleId": "M-XS", "source": "deliverymaster"}], "销售": [{"formName": "销售订单", "formId": "FS01", "moduleId": "M-XS", "source": "salesorder"}], + "订单": [{"formName": "销售订单", "formId": "FS01", "moduleId": "M-XS", "source": "salesorder"}], + "库存": [{"formName": "库存表", "formId": "FI01", "moduleId": "M-KC", "source": "inventory"}], + "物料": [{"formName": "物料资料", "formId": "FM01", "moduleId": "M-JC", "source": "materials"}], + "供应商": [{"formName": "供应商资料", "formId": "FG01", "moduleId": "M-JC", "source": "suppliermaster"}], + "采购": [{"formName": "采购订单", "formId": "FP01", "moduleId": "M-CG", "source": "purchaseorder"}], + "应收": [{"formName": "应收账款", "formId": "FA01", "moduleId": "M-CW", "source": "receivable"}], + "生产": [{"formName": "生产任务单", "formId": "FR01", "moduleId": "M-SC", "source": "productiontask"}], } READFORM_PAGES = { ("FC01", 1): "共 6 条记录，第 1 页：必胜客、上海创远包装、苏州华为、杭州大华印务", ("FC01", 2): "共 6 条记录，第 2 页：宁波天海包装、无锡礼盒厂", - ("FD01", 1): "共 2 条记录，第 1 页：DH202607005（客户 必胜客）、DH202607008（客户 苏州华为）", + ("FD01", 1): "共 2 条记录，第 1 页：DH202607005（客户 必胜客，未审核）、DH202607008（客户 苏州华为，已审核）", + ("FQ01", 1): "共 2 条记录，第 1 页：BJD202607001（客户 必胜客，产品 披萨盒，数量 5000，未审核）、" + "BJD202607082（客户 苏州华为，产品 彩盒，数量 1000，已审核）", + ("FI01", 1): "共 5 条记录，第 1 页（按库存量降序）：瓦楞纸 30,000 张、铜版纸 12,000 张、300g白卡纸 8,500 张、" + "灰板纸 6,200 张、油墨(青) 410 桶", + ("FM01", 1): "共 4 条记录，第 1 页：铜版纸、300g白卡纸、瓦楞纸、灰板纸", + ("FS01", 1): "共 3 条记录，第 1 页：SO202607010（客户 必胜客，金额 356,000 元）、SO202607011（客户 上海创远包装，金额 120,000 元）、" + "SO202607012（客户 必胜客，金额 89,000 元）", + ("FG01", 1): "共 2 条记录，第 1 页：金田纸业（电话 0571-88881234）、杭州新新材料", + ("FP01", 1): "共 2 条记录，第 1 页：PO202607003（供应商 金田纸业）、PO202607004（供应商 杭州新新材料）", + ("FA01", 1): "共 1 条记录：应收账款合计 1,560,000 元（其中逾期 230,000 元）", + ("FR01", 1): "共 4 条记录，第 1 页：MO202607021（披萨盒 印刷中）、MO202607022（彩盒 模切）、" + "MO202607023（画册 装订）、MO202607024（纸箱 待排程）", +} + +RECORDS = { + "BJD202607001": {"单号": "BJD202607001", "客户": "必胜客", "产品": "披萨盒", "数量": "5000", + "审核状态": "未审核", "审核人": "", "业务员": "王五"}, + "BJD202607082": {"单号": "BJD202607082", "客户": "苏州华为", "产品": "彩盒", "数量": "1000", + "审核状态": "已审核", "审核人": "赵六", "业务员": "李四"}, + "DH202607005": {"单号": "DH202607005", "客户": "必胜客", "审核状态": "未审核"}, + "DH202607008": {"单号": "DH202607008", "客户": "苏州华为", "审核状态": "已审核"}, + "铜版纸": {"物料名称": "铜版纸", "库存": "12000 张"}, + "300g白卡纸": {"物料名称": "300g白卡纸", "库存": "8500 张"}, + "金田纸业": {"供应商名称": "金田纸业", "联系电话": "0571-88881234"}, } @@ -96,18 +127,20 @@ def fixture_result(name, args): ensure_ascii=False) if name == "readFormData": page = int(args.get("page") or 1) + kw = args.get("keyword") or "" + if kw: + for rname, rec in {**CUSTOMERS, **RECORDS}.items(): + if rname in kw or kw in rname: + return f"共 1 条记录：{json.dumps(rec, ensure_ascii=False)}" + return f"共 0 条记录（keyword={kw} 无匹配）" key = (args.get("formId", ""), page) if key in READFORM_PAGES: return READFORM_PAGES[key] - kw = args.get("keyword") or "" - for cname, rec in CUSTOMERS.items(): - if cname in kw: - return f"共 1 条记录：{json.dumps(rec, ensure_ascii=False)}" - return f"共 0 条记录（formId={args.get('formId','')} page={page} keyword={kw}）" + return f"共 0 条记录（formId={args.get('formId','')} page={page}）" if name == "lookupRecord": rk = args.get("recordKeyword", "") - for cname, rec in CUSTOMERS.items(): - if cname in rk or rk in cname: + for rname, rec in {**CUSTOMERS, **RECORDS}.items(): + if rname in rk or rk in rname: return json.dumps(rec, ensure_ascii=False) return f"未找到「{rk}」对应的记录" if name == "collectForm": @@ -409,14 +442,15 @@ def judge_fifty(exp, calls, text): READ_TOOLS_50 = {"findForms", "readFormData", "lookupRecord"} -def run_fifty(arch): +def run_fifty(arch, only=None): system_prompt, tools = build_context(arch) - print(f"bench50 全轨迹判分 arch={arch} model={MODEL}") + cases = [(u, e) for u, e in FIFTY_CASES if only is None or only in u] + print(f"bench50 全轨迹判分 arch={arch} model={MODEL} cases={len(cases)}") strict = 0 lenient = 0 per = {} fails = [] - for utt, exp in FIFTY_CASES: + for utt, exp in cases: t0 = time.time() calls, text, steps, err = run_trajectory(system_prompt, tools, [], utt) dt = time.time() - t0 @@ -432,7 +466,7 @@ def run_fifty(arch): if v == "fail" and not err: fails.append((utt, exp, traj + (" 「" + (text or "")[:60].replace("\n", " ") + "」" if text else ""))) print(f"{mark} [{exp}] {utt} → {traj} ({steps}步 {dt:.0f}s)") - n = len(FIFTY_CASES) + n = len(cases) print(f"\n[{arch}] bench50 全轨迹: 严格 {strict}/{n} = {strict/n*100:.0f}% 宽松 {lenient}/{n} = {lenient/n*100:.0f}%") print("按意图分（严格/宽松/总数）:", {k: f"{s}/{l}/{c}" for k, (s, l, c) in per.items()}) for utt, exp, d in fails: @@ -448,7 +482,7 @@ def main(): if "--only" in argv: only = argv[argv.index("--only") + 1] if "--fifty" in argv: - run_fifty(arch) + run_fifty(arch, only) return system_prompt, tools = build_context(arch) cases = [c for c in CASES if only is None or c["id"] == only] -- libgit2 0.22.2