bench: enrich fixture world (quotes/inventory/orders), --fifty --only for case r…

…e-verification; record P3 final acceptance (ext 12/12, fifty 84% verified vs old 82%)

bench: enrich fixture world (quotes/inventory/orders), --fifty --only for case r…
…e-verification; record P3 final acceptance (ext 12/12, fifty 84% verified vs old 82%)
zichun
1 parent c4ac1971
Showing 1 changed file with 47 additions and 13 deletions
bench/bench_ext.py
@@ -76,12 +76,43 @@ FORMS = {
     "客户": [{"formName": "客户资料", "formId": "FC01", "moduleId": "M-JC", "source": "custmaster"}],
     "送货": [{"formName": "送货单", "formId": "FD01", "moduleId": "M-XS", "source": "deliverymaster"}],
     "销售": [{"formName": "销售订单", "formId": "FS01", "moduleId": "M-XS", "source": "salesorder"}],
+    "订单": [{"formName": "销售订单", "formId": "FS01", "moduleId": "M-XS", "source": "salesorder"}],
+    "库存": [{"formName": "库存表", "formId": "FI01", "moduleId": "M-KC", "source": "inventory"}],
+    "物料": [{"formName": "物料资料", "formId": "FM01", "moduleId": "M-JC", "source": "materials"}],
+    "供应商": [{"formName": "供应商资料", "formId": "FG01", "moduleId": "M-JC", "source": "suppliermaster"}],
+    "采购": [{"formName": "采购订单", "formId": "FP01", "moduleId": "M-CG", "source": "purchaseorder"}],
+    "应收": [{"formName": "应收账款", "formId": "FA01", "moduleId": "M-CW", "source": "receivable"}],
+    "生产": [{"formName": "生产任务单", "formId": "FR01", "moduleId": "M-SC", "source": "productiontask"}],
 }
 READFORM_PAGES = {
     ("FC01", 1): "共 6 条记录，第 1 页：必胜客、上海创远包装、苏州华为、杭州大华印务",
     ("FC01", 2): "共 6 条记录，第 2 页：宁波天海包装、无锡礼盒厂",
-    ("FD01", 1): "共 2 条记录，第 1 页：DH202607005（客户 必胜客）、DH202607008（客户 苏州华为）",
+    ("FD01", 1): "共 2 条记录，第 1 页：DH202607005（客户 必胜客，未审核）、DH202607008（客户 苏州华为，已审核）",
+    ("FQ01", 1): "共 2 条记录，第 1 页：BJD202607001（客户 必胜客，产品 披萨盒，数量 5000，未审核）、"
+                 "BJD202607082（客户 苏州华为，产品 彩盒，数量 1000，已审核）",
+    ("FI01", 1): "共 5 条记录，第 1 页（按库存量降序）：瓦楞纸 30,000 张、铜版纸 12,000 张、300g白卡纸 8,500 张、"
+                 "灰板纸 6,200 张、油墨(青) 410 桶",
+    ("FM01", 1): "共 4 条记录，第 1 页：铜版纸、300g白卡纸、瓦楞纸、灰板纸",
+    ("FS01", 1): "共 3 条记录，第 1 页：SO202607010（客户 必胜客，金额 356,000 元）、SO202607011（客户 上海创远包装，金额 120,000 元）、"
+                 "SO202607012（客户 必胜客，金额 89,000 元）",
+    ("FG01", 1): "共 2 条记录，第 1 页：金田纸业（电话 0571-88881234）、杭州新新材料",
+    ("FP01", 1): "共 2 条记录，第 1 页：PO202607003（供应商 金田纸业）、PO202607004（供应商 杭州新新材料）",
+    ("FA01", 1): "共 1 条记录：应收账款合计 1,560,000 元（其中逾期 230,000 元）",
+    ("FR01", 1): "共 4 条记录，第 1 页：MO202607021（披萨盒 印刷中）、MO202607022（彩盒 模切）、"
+                 "MO202607023（画册 装订）、MO202607024（纸箱 待排程）",
+}
+
+RECORDS = {
+    "BJD202607001": {"单号": "BJD202607001", "客户": "必胜客", "产品": "披萨盒", "数量": "5000",
+                     "审核状态": "未审核", "审核人": "", "业务员": "王五"},
+    "BJD202607082": {"单号": "BJD202607082", "客户": "苏州华为", "产品": "彩盒", "数量": "1000",
+                     "审核状态": "已审核", "审核人": "赵六", "业务员": "李四"},
+    "DH202607005": {"单号": "DH202607005", "客户": "必胜客", "审核状态": "未审核"},
+    "DH202607008": {"单号": "DH202607008", "客户": "苏州华为", "审核状态": "已审核"},
+    "铜版纸": {"物料名称": "铜版纸", "库存": "12000 张"},
+    "300g白卡纸": {"物料名称": "300g白卡纸", "库存": "8500 张"},
+    "金田纸业": {"供应商名称": "金田纸业", "联系电话": "0571-88881234"},
 }
@@ -96,18 +127,20 @@ def fixture_result(name, args):
                           ensure_ascii=False)
     if name == "readFormData":
         page = int(args.get("page") or 1)
+        kw = args.get("keyword") or ""
+        if kw:
+            for rname, rec in {**CUSTOMERS, **RECORDS}.items():
+                if rname in kw or kw in rname:
+                    return f"共 1 条记录：{json.dumps(rec, ensure_ascii=False)}"
+            return f"共 0 条记录（keyword={kw} 无匹配）"
         key = (args.get("formId", ""), page)
         if key in READFORM_PAGES:
             return READFORM_PAGES[key]
-        kw = args.get("keyword") or ""
-        for cname, rec in CUSTOMERS.items():
-            if cname in kw:
-                return f"共 1 条记录：{json.dumps(rec, ensure_ascii=False)}"
-        return f"共 0 条记录（formId={args.get('formId','')} page={page} keyword={kw}）"
+        return f"共 0 条记录（formId={args.get('formId','')} page={page}）"
     if name == "lookupRecord":
         rk = args.get("recordKeyword", "")
-        for cname, rec in CUSTOMERS.items():
-            if cname in rk or rk in cname:
+        for rname, rec in {**CUSTOMERS, **RECORDS}.items():
+            if rname in rk or rk in rname:
                 return json.dumps(rec, ensure_ascii=False)
         return f"未找到「{rk}」对应的记录"
     if name == "collectForm":
@@ -409,14 +442,15 @@ def judge_fifty(exp, calls, text):
 READ_TOOLS_50 = {"findForms", "readFormData", "lookupRecord"}
-def run_fifty(arch):
+def run_fifty(arch, only=None):
     system_prompt, tools = build_context(arch)
-    print(f"bench50 全轨迹判分  arch={arch}  model={MODEL}")
+    cases = [(u, e) for u, e in FIFTY_CASES if only is None or only in u]
+    print(f"bench50 全轨迹判分  arch={arch}  model={MODEL}  cases={len(cases)}")
     strict = 0
     lenient = 0
     per = {}
     fails = []
-    for utt, exp in FIFTY_CASES:
+    for utt, exp in cases:
         t0 = time.time()
         calls, text, steps, err = run_trajectory(system_prompt, tools, [], utt)
         dt = time.time() - t0
@@ -432,7 +466,7 @@ def run_fifty(arch):
         if v == "fail" and not err:
             fails.append((utt, exp, traj + (" 「" + (text or "")[:60].replace("\n", " ") + "」" if text else "")))
         print(f"{mark} [{exp}] {utt}  →  {traj}  ({steps}步 {dt:.0f}s)")
-    n = len(FIFTY_CASES)
+    n = len(cases)
     print(f"\n[{arch}] bench50 全轨迹: 严格 {strict}/{n} = {strict/n*100:.0f}%   宽松 {lenient}/{n} = {lenient/n*100:.0f}%")
     print("按意图分（严格/宽松/总数）:", {k: f"{s}/{l}/{c}" for k, (s, l, c) in per.items()})
     for utt, exp, d in fails:
@@ -448,7 +482,7 @@ def main():
     if "--only" in argv:
         only = argv[argv.index("--only") + 1]
     if "--fifty" in argv:
-        run_fifty(arch)
+        run_fifty(arch, only)
         return
     system_prompt, tools = build_context(arch)
     cases = [c for c in CASES if only is None or c["id"] == only]