朱子纯 / xlyNewAi

Commit 5ffcf59fcedffc0aff2bae4bc387844baaf9d767

Authored by zichun
1 parent c4ac1971

bench: enrich fixture world (quotes/inventory/orders), --fifty --only for case r… 

…e-verification; record P3 final acceptance (ext 12/12, fifty 84% verified vs old 82%)
Inline Side-by-side
Showing 1 changed file with 47 additions and 13 deletions
bench/bench_ext.py
  View file @5ffcf59
... ... @@ -76,12 +76,43 @@ FORMS = {
76 76 "客户": [{"formName": "客户资料", "formId": "FC01", "moduleId": "M-JC", "source": "custmaster"}],
77 77 "送货": [{"formName": "送货单", "formId": "FD01", "moduleId": "M-XS", "source": "deliverymaster"}],
78 78 "销售": [{"formName": "销售订单", "formId": "FS01", "moduleId": "M-XS", "source": "salesorder"}],
  79 + "订单": [{"formName": "销售订单", "formId": "FS01", "moduleId": "M-XS", "source": "salesorder"}],
  80 + "库存": [{"formName": "库存表", "formId": "FI01", "moduleId": "M-KC", "source": "inventory"}],
  81 + "物料": [{"formName": "物料资料", "formId": "FM01", "moduleId": "M-JC", "source": "materials"}],
  82 + "供应商": [{"formName": "供应商资料", "formId": "FG01", "moduleId": "M-JC", "source": "suppliermaster"}],
  83 + "采购": [{"formName": "采购订单", "formId": "FP01", "moduleId": "M-CG", "source": "purchaseorder"}],
  84 + "应收": [{"formName": "应收账款", "formId": "FA01", "moduleId": "M-CW", "source": "receivable"}],
  85 + "生产": [{"formName": "生产任务单", "formId": "FR01", "moduleId": "M-SC", "source": "productiontask"}],
79 86 }
80 87  
81 88 READFORM_PAGES = {
82 89 ("FC01", 1): "共 6 条记录，第 1 页：必胜客、上海创远包装、苏州华为、杭州大华印务",
83 90 ("FC01", 2): "共 6 条记录，第 2 页：宁波天海包装、无锡礼盒厂",
84   - ("FD01", 1): "共 2 条记录，第 1 页：DH202607005（客户 必胜客）、DH202607008（客户 苏州华为）",
  91 + ("FD01", 1): "共 2 条记录，第 1 页：DH202607005（客户 必胜客，未审核）、DH202607008（客户 苏州华为，已审核）",
  92 + ("FQ01", 1): "共 2 条记录，第 1 页：BJD202607001（客户 必胜客，产品 披萨盒，数量 5000，未审核）、"
  93 + "BJD202607082（客户 苏州华为，产品 彩盒，数量 1000，已审核）",
  94 + ("FI01", 1): "共 5 条记录，第 1 页（按库存量降序）：瓦楞纸 30,000 张、铜版纸 12,000 张、300g白卡纸 8,500 张、"
  95 + "灰板纸 6,200 张、油墨(青) 410 桶",
  96 + ("FM01", 1): "共 4 条记录，第 1 页：铜版纸、300g白卡纸、瓦楞纸、灰板纸",
  97 + ("FS01", 1): "共 3 条记录，第 1 页：SO202607010（客户 必胜客，金额 356,000 元）、SO202607011（客户 上海创远包装，金额 120,000 元）、"
  98 + "SO202607012（客户 必胜客，金额 89,000 元）",
  99 + ("FG01", 1): "共 2 条记录，第 1 页：金田纸业（电话 0571-88881234）、杭州新新材料",
  100 + ("FP01", 1): "共 2 条记录，第 1 页：PO202607003（供应商 金田纸业）、PO202607004（供应商 杭州新新材料）",
  101 + ("FA01", 1): "共 1 条记录：应收账款合计 1,560,000 元（其中逾期 230,000 元）",
  102 + ("FR01", 1): "共 4 条记录，第 1 页：MO202607021（披萨盒 印刷中）、MO202607022（彩盒 模切）、"
  103 + "MO202607023（画册 装订）、MO202607024（纸箱 待排程）",
  104 +}
  105 +
  106 +RECORDS = {
  107 + "BJD202607001": {"单号": "BJD202607001", "客户": "必胜客", "产品": "披萨盒", "数量": "5000",
  108 + "审核状态": "未审核", "审核人": "", "业务员": "王五"},
  109 + "BJD202607082": {"单号": "BJD202607082", "客户": "苏州华为", "产品": "彩盒", "数量": "1000",
  110 + "审核状态": "已审核", "审核人": "赵六", "业务员": "李四"},
  111 + "DH202607005": {"单号": "DH202607005", "客户": "必胜客", "审核状态": "未审核"},
  112 + "DH202607008": {"单号": "DH202607008", "客户": "苏州华为", "审核状态": "已审核"},
  113 + "铜版纸": {"物料名称": "铜版纸", "库存": "12000 张"},
  114 + "300g白卡纸": {"物料名称": "300g白卡纸", "库存": "8500 张"},
  115 + "金田纸业": {"供应商名称": "金田纸业", "联系电话": "0571-88881234"},
85 116 }
86 117  
87 118  
... ... @@ -96,18 +127,20 @@ def fixture_result(name, args):
96 127 ensure_ascii=False)
97 128 if name == "readFormData":
98 129 page = int(args.get("page") or 1)
  130 + kw = args.get("keyword") or ""
  131 + if kw:
  132 + for rname, rec in {**CUSTOMERS, **RECORDS}.items():
  133 + if rname in kw or kw in rname:
  134 + return f"共 1 条记录：{json.dumps(rec, ensure_ascii=False)}"
  135 + return f"共 0 条记录（keyword={kw} 无匹配）"
99 136 key = (args.get("formId", ""), page)
100 137 if key in READFORM_PAGES:
101 138 return READFORM_PAGES[key]
102   - kw = args.get("keyword") or ""
103   - for cname, rec in CUSTOMERS.items():
104   - if cname in kw:
105   - return f"共 1 条记录：{json.dumps(rec, ensure_ascii=False)}"
106   - return f"共 0 条记录（formId={args.get('formId','')} page={page} keyword={kw}）"
  139 + return f"共 0 条记录（formId={args.get('formId','')} page={page}）"
107 140 if name == "lookupRecord":
108 141 rk = args.get("recordKeyword", "")
109   - for cname, rec in CUSTOMERS.items():
110   - if cname in rk or rk in cname:
  142 + for rname, rec in {**CUSTOMERS, **RECORDS}.items():
  143 + if rname in rk or rk in rname:
111 144 return json.dumps(rec, ensure_ascii=False)
112 145 return f"未找到「{rk}」对应的记录"
113 146 if name == "collectForm":
... ... @@ -409,14 +442,15 @@ def judge_fifty(exp, calls, text):
409 442 READ_TOOLS_50 = {"findForms", "readFormData", "lookupRecord"}
410 443  
411 444  
412   -def run_fifty(arch):
  445 +def run_fifty(arch, only=None):
413 446 system_prompt, tools = build_context(arch)
414   - print(f"bench50 全轨迹判分 arch={arch} model={MODEL}")
  447 + cases = [(u, e) for u, e in FIFTY_CASES if only is None or only in u]
  448 + print(f"bench50 全轨迹判分 arch={arch} model={MODEL} cases={len(cases)}")
415 449 strict = 0
416 450 lenient = 0
417 451 per = {}
418 452 fails = []
419   - for utt, exp in FIFTY_CASES:
  453 + for utt, exp in cases:
420 454 t0 = time.time()
421 455 calls, text, steps, err = run_trajectory(system_prompt, tools, [], utt)
422 456 dt = time.time() - t0
... ... @@ -432,7 +466,7 @@ def run_fifty(arch):
432 466 if v == "fail" and not err:
433 467 fails.append((utt, exp, traj + (" 「" + (text or "")[:60].replace("\n", " ") + "」" if text else "")))
434 468 print(f"{mark} [{exp}] {utt} → {traj} ({steps}步 {dt:.0f}s)")
435   - n = len(FIFTY_CASES)
  469 + n = len(cases)
436 470 print(f"\n[{arch}] bench50 全轨迹: 严格 {strict}/{n} = {strict/n*100:.0f}% 宽松 {lenient}/{n} = {lenient/n*100:.0f}%")
437 471 print("按意图分（严格/宽松/总数）:", {k: f"{s}/{l}/{c}" for k, (s, l, c) in per.items()})
438 472 for utt, exp, d in fails:
... ... @@ -448,7 +482,7 @@ def main():
448 482 if "--only" in argv:
449 483 only = argv[argv.index("--only") + 1]
450 484 if "--fifty" in argv:
451   - run_fifty(arch)
  485 + run_fifty(arch, only)
452 486 return
453 487 system_prompt, tools = build_context(arch)
454 488 cases = [c for c in CASES if only is None or c["id"] == only]
... ...