添加向量库

qianbao
1 parent da8e40f1
Showing 7 changed files with 418 additions and 26 deletions
src/main/java/com/xly/milvus/service/AiGlobalAgentQuestionSqlEmitterService.java
src/main/java/com/xly/milvus/service/impl/AiGlobalAgentQuestionSqlEmitterServiceImpl.java
src/main/java/com/xly/milvus/service/impl/MilvusServiceImpl.java
src/main/java/com/xly/milvus/service/impl/VectorizationServiceImpl.java
src/main/java/com/xly/service/XlyErpService.java
src/main/java/com/xly/thread/AiUserAgentQuestionThread.java
src/main/java/com/xly/util/SqlWhereUtil.java
@@ -17,4 +17,5 @@ public interface AiGlobalAgentQuestionSqlEmitterService {
  
     Map<String, Object> queryAiGlobalAgentQuestionSqlEmitter(String searchText, String collectionName);
  
+    void removeAndCreateCollection(String collectionName);
 }
 \ No newline at end of file
@@ -68,10 +68,13 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
         if(ObjectUtil.isEmpty(sSqlContent)){
             sSqlContent = StrUtil.EMPTY;
         }
-
+        //调用数据库插入数据库]
+        if(ObjectUtil.isEmpty(data.get("userInput"))){
+            Object userInput = ObjectUtil.isNotEmpty(data.get("sUserInput"))?data.get("sUserInput").toString():data.get("userInput");
+            data.put("userInput",userInput);
+        }
         // 2. 转换为Milvus格式
         JsonObject row = convertToMilvusRow(data, vector,sQuestion,sSqlContent,cachType,sKey);
-
         //创建集合
 //        createCollection(collectionName);
         createCollectionIfNotExists(collectionName);
@@ -85,11 +88,6 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
         InsertResp insertResp = milvusClient.insert(insertReq);
         //是否初始化。初始化不需要插入到数据库
         if(!isInit){
-            //调用数据库插入数据库]
-            if(ObjectUtil.isEmpty(data.get("userInput"))){
-                Object userInput = ObjectUtil.isNotEmpty(data.get("sUserInput"))?data.get("sUserInput").toString():data.get("userInput");
-                data.put("userInput",userInput);
-            }
             Map<String, Object> searMap = dynamicExeDbService.getDoProMap(sProName, data);
             dynamicExeDbService.getCallPro(searMap, sProName);
         }
@@ -104,9 +102,10 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
         // 2. 设置范围搜索参数
         Map<String, Object> searchParams = new HashMap<>();
         searchParams.put("nprobe", 10);
+        searchParams.put("radius", 0.94f);  // 只返回相似度 >= 0.94 的结果
         // 对于 IP 度量，相似度范围在 [minScore, maxScore]
-        searchParams.put("radius", 0.98);      // 最小相似度
-        searchParams.put("range_filter", 1); // 最大相似度
+//        searchParams.put("radius", -1);      // 最小相似度
+//        searchParams.put("range_filter", 1); // 最大相似度
         // 1. 确保集合已加载
         ensureCollectionLoaded(collectionName);
  
@@ -136,7 +135,7 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
                 .annsField("vector")  // 向量字段名
                 .topK(10)  // 返回最相似的10条
                 .metricType(IndexParam.MetricType.IP)  // 内积相似度
-                .outputFields(Arrays.asList("sQuestion", "sSqlContent", "data_id","cachType", "create_time","metadata"))
+                .outputFields(Arrays.asList("sQuestion", "sSqlContent","sUserInput", "data_id","cachType", "create_time","metadata"))
                 .searchParams(searchParams)
                 .build();
         // 5. 执行搜索
@@ -146,6 +145,7 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
         if(ObjectUtil.isEmpty(searchResults)){
             return result;
         }
+        //默认相似度最高的在第一条
         List<SearchResp.SearchResult> firstResultList = searchResults.get(0);
         if(ObjectUtil.isEmpty(firstResultList)){
             return result;
@@ -259,6 +259,7 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
         row.addProperty("sKey", sKey);
         row.addProperty("data_id", data.get("sId").toString());
         row.addProperty("sQuestion", sQuestion);
+        row.addProperty("sUserInput", data.get("userInput").toString());
         row.addProperty("sSqlContent", sSqlContent);
         row.addProperty("cachType", cachType);
         // 创建时间字段 - 必须提供！
@@ -295,12 +296,6 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
      * 创建集合（定义字段结构）
      */
     private void createCollection(String collectionName) {
-        //删除现有集合
-//        DropCollectionReq dropCollectionReq = DropCollectionReq.builder()
-//                .collectionName(collectionName)
-//                .build();
-//        milvusClient.dropCollection(dropCollectionReq);
-
         // 定义字段列表
         List<CreateCollectionReq.FieldSchema> fieldSchemas = Arrays.asList(
                 // 1. 主键字段
@@ -325,6 +320,13 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
                         .name("sQuestion")
                         .dataType(DataType.VarChar)
                         .maxLength(5000)
+                        .description("用户问题(上下文)")
+                        .build(),
+
+                CreateCollectionReq.FieldSchema.builder()
+                        .name("sUserInput")
+                        .dataType(DataType.VarChar)
+                        .maxLength(5000)
                         .description("用户问题")
                         .build(),
  
@@ -396,6 +398,19 @@ public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgent
         createIndexesStepByStep(collectionName);
     }
  
+
+    /**
+     * 创建集合（定义字段结构）
+     */
+    public void removeAndCreateCollection(String collectionName) {
+        //删除现有集合
+        DropCollectionReq dropCollectionReq = DropCollectionReq.builder()
+                .collectionName(collectionName)
+                .build();
+        milvusClient.dropCollection(dropCollectionReq);
+        createCollection(collectionName);
+    }
+
     /*
     * 分步创建索引，便于监控每个索引的状态
     */
@@ -170,8 +170,6 @@ public class MilvusServiceImpl implements MilvusService {
                     sb.append(sUserInput).append("已经存在").append("\n");
                 }
                 //更新数据是否是全局问题，等待朱总确定是否需要
-
-
             }
         });
         if(ObjectUtil.isEmpty(sb)){
@@ -187,15 +185,18 @@ public class MilvusServiceImpl implements MilvusService {
     public TTSResponseDTO initGlobalAgentQuestion(Map<String, Object> reqMap) {
         //全局问题初始化
         List<Map<String,Object>> data = getGlobalAgentQuestion();
+        String collectionName = "ai_global_agent_question_sql";
+        //删除所有向量
+        aiGlobalAgentQuestionSqlEmitterService.removeAndCreateCollection(collectionName);
         data.forEach(one->{
             //查询向量库是否存在
             String sSceneId = one.get("sSceneId").toString();
             String sMethodId = one.get("sMethodId").toString();
             String sUserInput = one.get("sUserInput").toString();
             String sSqlContent = one.get("sSqlContent").toString();
-            String searchText = sSceneId+"_"+sMethodId+"_"+sUserInput;
+            String searchText = String.format("场景：%s 方法：%s 客户问题：%s", sSceneId, sMethodId, sUserInput);
             if(ObjectUtil.isNotEmpty(sUserInput)){
-                aiGlobalAgentQuestionSqlEmitterService.addAiGlobalAgentQuestionSqlEmitter(searchText,one,sUserInput,sSqlContent,"MYSQL","ai_global_agent_question_sql",true);
+                aiGlobalAgentQuestionSqlEmitterService.addAiGlobalAgentQuestionSqlEmitter(searchText,one,sUserInput,sSqlContent,"MYSQL",collectionName,true);
             }
         });
         return TTSResponseDTO.builder()
@@ -36,9 +36,12 @@ public class VectorizationServiceImpl implements VectorizationService {
             dev.langchain4j.data.embedding.Embedding embedding = embeddingModel.embed(text).content();
             float[] vectorArray = embedding.vector();
  
+            // L2归一化：将向量长度归一化为1
+            float[] normalizedArray = normalizeVector(vectorArray);
+
             // 转换为List<Float>
             List<Float> vector = new ArrayList<>();
-            for (float v : vectorArray) {
+            for (float v : normalizedArray) {
                 vector.add(v);
             }
  
@@ -49,6 +52,28 @@ public class VectorizationServiceImpl implements VectorizationService {
         }
     }
  
+    /**
+     * L2归一化：使向量的模长为1
+     */
+    private float[] normalizeVector(float[] vector) {
+        // 计算L2范数
+        double norm = 0.0;
+        for (float v : vector) {
+            norm += v * v;
+        }
+        norm = Math.sqrt(norm);
+
+        // 归一化
+        if (norm > 0) {
+            float[] normalized = new float[vector.length];
+            for (int i = 0; i < vector.length; i++) {
+                normalized[i] = (float)(vector[i] / norm);
+            }
+            return normalized;
+        }
+        return vector;
+    }
+
     @Override
     public List<List<Float>> batchTextToVector(List<String> texts) {
         if (texts == null || texts.isEmpty()) {
@@ -800,7 +800,7 @@ public class XlyErpService {
      **/
     private Map<String,Object> getDynamicTableCach(UserSceneSession session,String input){
         try{
-            String searchText = session.getCurrentScene().getSId()+"_"+session.getCurrentTool().getSId()+"_"+input;
+            String searchText = String.format("场景：%s 方法：%s 客户问题：%s", session.getCurrentScene().getSId(), session.getCurrentTool().getSId(), input);
             //根据问题查询向量库
             Map<String,Object> serMap = aiGlobalAgentQuestionSqlEmitterService.queryAiGlobalAgentQuestionSqlEmitter(searchText, "ai_global_agent_question_sql");
             return serMap;
@@ -10,7 +10,7 @@ import com.xly.milvus.service.AiGlobalAgentQuestionSqlEmitterService;
 import com.xly.service.DynamicExeDbService;
 import com.xly.service.RedisService;
 import com.xly.util.SqlValidateUtil;
-import com.xly.util.SqlWhereHelper;
+import com.xly.util.SqlWhereUtil;
 import dev.langchain4j.data.message.ChatMessage;
 import dev.langchain4j.data.message.ChatMessageType;
 import java.util.HashMap;
@@ -55,11 +55,12 @@ public class AiUserAgentQuestionThread implements Runnable {
             String sReidKey = SqlValidateUtil.getsKey( sSceneId, sMethodId, sQuestionGroupNo);
             redisService.set(sReidKey,sSqlContent);
         }
-        String sKey = sSceneId+"_"+sMethodId +"_"+sQuestion;
+        String searchText = String.format("场景：%s 方法：%s 客户问题：%s", sSceneId, sMethodId, sQuestion);
+                // sSceneId+"_"+sMethodId +"_"+sQuestion;
 //                SqlValidateUtil.getsKey( sSceneId, sMethodId, SqlValidateUtil.getsQuestion(session.getSUserQuestionList()));
         //存入向量库 存在SQL语句并且没有where  并且执行成功
-        if(!SqlWhereHelper.hasWhereButNoCompareOperators(sSqlContent) && ObjectUtil.isNotEmpty(sSqlContent) && bSucess){
-            aiGlobalAgentQuestionSqlEmitterService.addAiGlobalAgentQuestionSqlEmitter(sKey,data,sQuestion,sSqlContent,cachType,"ai_global_agent_question_sql",false);
+        if(!SqlWhereUtil.hasValidConditionAfterClean(sSqlContent) && ObjectUtil.isNotEmpty(sSqlContent) && bSucess){
+            aiGlobalAgentQuestionSqlEmitterService.addAiGlobalAgentQuestionSqlEmitter(searchText,data,sQuestion,sSqlContent,cachType,"ai_global_agent_question_sql",false);
         }
         //调用数据库插入数据库
         Map<String, Object> searMap = dynamicExeDbService.getDoProMap(sProName, data);
+package com.xly.util;
+
+import cn.hutool.core.util.StrUtil;
+import lombok.extern.slf4j.Slf4j;
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.expression.*;
+import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
+import net.sf.jsqlparser.expression.operators.conditional.OrExpression;
+import net.sf.jsqlparser.expression.operators.relational.*;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.Statement;
+import net.sf.jsqlparser.statement.delete.Delete;
+import net.sf.jsqlparser.statement.select.PlainSelect;
+import net.sf.jsqlparser.statement.select.Select;
+import net.sf.jsqlparser.statement.update.Update;
+import net.sf.jsqlparser.expression.LongValue;
+
+@Slf4j
+public class SqlWhereUtil {
+
+    /**
+     * 去除为空的条件 和 bCheck=1 的条件
+     * 然后判断是否还有有效的 WHERE 条件
+     *
+     * @param sql 原始SQL
+     * @return true: 去除后还有有效条件; false: 去除后没有条件了
+     */
+    public static boolean hasValidConditionAfterClean(String sql) {
+        if (StrUtil.isBlank(sql)) {
+            return false;
+        }
+
+        try {
+            Statement statement = CCJSqlParserUtil.parse(sql);
+            Expression whereExpression = null;
+
+            if (statement instanceof Select) {
+                Select select = (Select) statement;
+                PlainSelect plainSelect = select.getPlainSelect();
+                if (plainSelect != null) {
+                    whereExpression = plainSelect.getWhere();
+                }
+            } else if (statement instanceof Update) {
+                Update update = (Update) statement;
+                whereExpression = update.getWhere();
+            } else if (statement instanceof Delete) {
+                Delete delete = (Delete) statement;
+                whereExpression = delete.getWhere();
+            }
+
+            if (whereExpression == null) {
+                return false;
+            }
+
+            // 清理条件
+            Expression cleanedExpression = cleanExpression(whereExpression);
+            return cleanedExpression != null;
+
+        } catch (JSQLParserException e) {
+            log.warn("SQL 解析失败，使用简单判断: {}", sql, e);
+            return hasValidConditionAfterCleanSimple(sql);
+        }
+    }
+
+    /**
+     * 递归清理表达式
+     * 返回 null 表示整个表达式被清空
+     */
+    private static Expression cleanExpression(Expression expression) {
+        if (expression == null) {
+            return null;
+        }
+
+        // 处理 AND 表达式
+        if (expression instanceof AndExpression) {
+            AndExpression and = (AndExpression) expression;
+            Expression left = cleanExpression(and.getLeftExpression());
+            Expression right = cleanExpression(and.getRightExpression());
+
+            if (left == null && right == null) {
+                return null;
+            } else if (left == null) {
+                return right;
+            } else if (right == null) {
+                return left;
+            } else {
+                return new AndExpression(left, right);
+            }
+        }
+
+        // 处理 OR 表达式
+        if (expression instanceof OrExpression) {
+            OrExpression or = (OrExpression) expression;
+            Expression left = cleanExpression(or.getLeftExpression());
+            Expression right = cleanExpression(or.getRightExpression());
+
+            if (left == null && right == null) {
+                return null;
+            } else if (left == null) {
+                return right;
+            } else if (right == null) {
+                return left;
+            } else {
+                return new OrExpression(left, right);
+            }
+        }
+
+        // 处理括号表达式
+        if (expression instanceof Parenthesis) {
+            Parenthesis parenthesis = (Parenthesis) expression;
+            Expression content = cleanExpression(parenthesis.getExpression());
+            if (content == null) {
+                return null;
+            }
+            return new Parenthesis(content);
+        }
+
+        // 判断是否为需要去除的条件
+        if (shouldRemoveCondition(expression)) {
+            return null;
+        }
+
+        return expression;
+    }
+
+    /**
+     * 判断条件是否应该被去除
+     */
+    private static boolean shouldRemoveCondition(Expression expression) {
+        // 1. IS NULL / IS NOT NULL
+        if (expression instanceof IsNullExpression) {
+            return true;
+        }
+
+        // 2. 等值比较: = '' , != '' , <> ''
+        if (expression instanceof EqualsTo) {
+            EqualsTo equals = (EqualsTo) expression;
+            if (isEmptyString(equals.getRightExpression()) || isEmptyString(equals.getLeftExpression())) {
+                return true;
+            }
+            // bCheck = 1
+            if (isCheckOneCondition(equals)) {
+                return true;
+            }
+        }
+
+        // !=
+        if (expression instanceof NotEqualsTo) {
+            NotEqualsTo notEquals = (NotEqualsTo) expression;
+            if (isEmptyString(notEquals.getRightExpression()) || isEmptyString(notEquals.getLeftExpression())) {
+                return true;
+            }
+        }
+
+        // 3. 函数判断为空: IFNULL(col, '') = '' , TRIM(col) = '' , NVL(col, '') = '' 等
+        if (expression instanceof EqualsTo) {
+            EqualsTo equals = (EqualsTo) expression;
+            if (isFunctionReturnEmptyCheck(equals)) {
+                return true;
+            }
+        }
+
+        // 4. 其他比较操作符中可能包含空值判断
+        if (expression instanceof ComparisonOperator) {
+            ComparisonOperator comp = (ComparisonOperator) expression;
+            if (isEmptyString(comp.getLeftExpression()) || isEmptyString(comp.getRightExpression())) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * 判断表达式是否为空字符串常量 ''
+     */
+    private static boolean isEmptyString(Expression expression) {
+        if (expression == null) {
+            return false;
+        }
+        if (expression instanceof StringValue) {
+            StringValue stringValue = (StringValue) expression;
+            return "".equals(stringValue.getValue());
+        }
+        return false;
+    }
+
+    /**
+     * 判断是否为 bCheck = 1 或 1 = bCheck
+     */
+    private static boolean isCheckOneCondition(EqualsTo equals) {
+        Expression left = equals.getLeftExpression();
+        Expression right = equals.getRightExpression();
+
+        // bCheck = 1
+        if (isColumnNameCheck(left) && isOneValue(right)) {
+            return true;
+        }
+        // 1 = bCheck
+        if (isOneValue(left) && isColumnNameCheck(right)) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * 判断是否为 bCheck 列（不区分大小写）
+     */
+    private static boolean isColumnNameCheck(Expression expression) {
+        if (expression instanceof Column) {
+            Column column = (Column) expression;
+            String columnName = column.getColumnName();
+            return columnName != null && columnName.equalsIgnoreCase("bCheck");
+        }
+        return false;
+    }
+
+    /**
+     * 判断是否为数字 1
+     */
+    private static boolean isOneValue(Expression expression) {
+        if (expression instanceof LongValue) {
+            LongValue longValue = (LongValue) expression;
+            return longValue.getValue() == 1;
+        }
+        return false;
+    }
+
+    /**
+     * 判断是否为函数返回空值判断，如：IFNULL(col, '') = ''
+     */
+    private static boolean isFunctionReturnEmptyCheck(EqualsTo equals) {
+        Expression left = equals.getLeftExpression();
+        Expression right = equals.getRightExpression();
+
+        // 情况1: 函数 = ''
+        if (isNullReturnFunction(left) && isEmptyString(right)) {
+            return true;
+        }
+        // 情况2: '' = 函数
+        if (isEmptyString(left) && isNullReturnFunction(right)) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * 判断是否为可能返回 NULL 的函数（在空值判断场景下）
+     * 如: IFNULL(col, '') , NVL(col, '') , TRIM(col) , COALESCE(col, '')
+     */
+    private static boolean isNullReturnFunction(Expression expression) {
+        if (!(expression instanceof Function)) {
+            return false;
+        }
+
+        Function function = (Function) expression;
+        String funcName = function.getName();
+
+        if (funcName == null) {
+            return false;
+        }
+
+        String upperName = funcName.toUpperCase();
+
+        // 常见的空值处理函数
+        switch (upperName) {
+            case "IFNULL":
+            case "NVL":
+            case "NVL2":
+            case "COALESCE":
+            case "NULLIF":
+                return true;
+            case "TRIM":
+            case "LTRIM":
+            case "RTRIM":
+                // TRIM 函数去除空格后判断为空
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    // ===================== 简单判断方法（降级方案） =====================
+
+    /**
+     * 简单判断：去除条件后是否还有有效条件
+     */
+    private static boolean hasValidConditionAfterCleanSimple(String sql) {
+        if (StrUtil.isBlank(sql)) {
+            return false;
+        }
+
+        String upper = sql.toUpperCase();
+
+        // 先找到 WHERE 位置
+        int whereIdx = upper.indexOf("WHERE");
+        if (whereIdx == -1) {
+            return false;
+        }
+
+        // 提取 WHERE 后面的条件
+        String whereClause = sql.substring(whereIdx + 5);
+
+        // 去除 GROUP BY、ORDER BY、LIMIT 之后的内容
+        int groupIdx = upper.indexOf("GROUP BY", whereIdx);
+        int orderIdx = upper.indexOf("ORDER BY", whereIdx);
+        int limitIdx = upper.indexOf("LIMIT", whereIdx);
+        int endIdx = sql.length();
+        if (groupIdx != -1) endIdx = Math.min(endIdx, groupIdx);
+        if (orderIdx != -1) endIdx = Math.min(endIdx, orderIdx);
+        if (limitIdx != -1) endIdx = Math.min(endIdx, limitIdx);
+
+        String condition = sql.substring(whereIdx + 5, endIdx);
+
+        // 去除需要过滤的条件
+        String cleaned = condition;
+
+        // 去除 IS NULL / IS NOT NULL
+        cleaned = cleaned.replaceAll("(?i)\\bIS\\s+(NOT\\s+)?NULL\\b", "");
+
+        // 去除 = '' / != '' / <> ''
+        cleaned = cleaned.replaceAll("(?i)\\s*(=|!=|<>)\\s*''\\s*", "");
+        cleaned = cleaned.replaceAll("(?i)\\s*''\\s*(=|!=|<>)\\s*", "");
+
+        // 去除 bCheck = 1
+        cleaned = cleaned.replaceAll("(?i)\\bbCheck\\s*=\\s*1\\b", "");
+
+        // 去除函数空值判断: IFNULL(..., '') = ''
+        cleaned = cleaned.replaceAll("(?i)\\b(IFNULL|NVL|COALESCE|TRIM)\\s*\\([^)]+\\)\\s*=\\s*''", "");
+        cleaned = cleaned.replaceAll("(?i)\\b''\\s*=\\s*(IFNULL|NVL|COALESCE|TRIM)\\s*\\([^)]+\\)", "");
+
+        // 去除多余的 AND/OR
+        cleaned = cleaned.replaceAll("(?i)\\s+(AND|OR)\\s+", " ");
+        cleaned = cleaned.trim();
+
+        // 判断是否还有有效条件（非空且不是纯粹的 true/false）
+        if (StrUtil.isBlank(cleaned)) {
+            return false;
+        }
+
+        // 如果只剩下 1=1 或 true 等恒真条件，也算无效
+        if (cleaned.matches("(?i)^\\s*(1\\s*=\\s*1|true)\\s*$")) {
+            return false;
+        }
+
+        return true;
+    }
+}
 \ No newline at end of file