diff --git a/pom.xml b/pom.xml index 7eb61bf..52b4ae8 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ 2.17.2 1.17.2 - 4.4.0 + 2.6.15 @@ -49,6 +49,30 @@ spring-boot-starter-web + + + io.milvus + milvus-sdk-java + ${milvus.version} + + + + dev.langchain4j + langchain4j-embeddings-all-minilm-l6-v2 + ${langchain4j.version} + + + + + + + + + + org.springframework.boot + spring-boot-starter-actuator + + org.springframework.boot spring-boot-starter-validation @@ -63,6 +87,12 @@ spring-boot-starter-aop + + com.google.code.gson + gson + 2.10.1 + + org.springframework.cloud @@ -270,69 +300,12 @@ 1.3.2 - - - - - - - - - - - - - - - - - - - jakarta.persistence jakarta.persistence-api 3.1.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -341,23 +314,13 @@ ${langchain4j.version} - - - - - - dev.langchain4j langchain4j ${langchain4j.version} - - - - - + dev.langchain4j langchain4j-ollama @@ -371,13 +334,6 @@ 1.17.0 - - - - - - - @@ -392,12 +348,6 @@ 2.9.1 - - - - - - diff --git a/src/main/java/com/xly/agent/ErpAiAgent.java b/src/main/java/com/xly/agent/ErpAiAgent.java index 727c684..e120d7d 100644 --- a/src/main/java/com/xly/agent/ErpAiAgent.java +++ b/src/main/java/com/xly/agent/ErpAiAgent.java @@ -27,10 +27,13 @@ public interface ErpAiAgent { * 入参：用户问题、执行的SQL、表结构、JSON格式结果 */ @SystemMessage(""" - 你是专业的业务数据分析师，严格遵循以下**通用规则**解释查询结果，适用于所有业务场景： + 你是专业的业务数据分析师，严格遵循以下**通用规则**解释查询结果，适用于所有业务场景： 1. 解释风格：贴合业务场景，无任何SQL专业术语，用口语化、简洁的商业语言说明，避免技术词汇； 2. 数据准确：严格按照JSON执行结果解释，不夸大、不遗漏、不编造数据，数值与结果完全一致； - 3. 输出格式：仅返回解释内容，不要列出ID,无多余标题、换行、符号，结果为空时直接返回“未查询到相关数据”； + 3. 输出格式：仅返回解释内容，不要列出ID,无多余标题、换行、符号,结果为空时直接返回“未查询到相关数据” + 3.1. 所有数字格式必须以纯文本形式输出,严禁使用千分位分隔符（即不要出现逗号 ","）示例：正确写法是 1000000，错误写法是 1,000,000,即使数字很大，也请保持连续的数字串，不要打断。 + 3.2 所有日期请转换为 YYYY-MM-DD 格式（例如：2026-03-15）,严禁包含时间部分（如小时、分钟、秒）（例如：2026-03-15 00:00:00），也不要包含时区信息。” + 3.3. 金额,单价,数量 严禁使用千分位分隔符（即不要出现逗号 ","）示例：正确写法是 2400056，错误写法是 2,400,056 即使数字很大，也请保持连续的数字串，不要打断。 4. 长度控制：单条解释不超过150字，条理清晰，重点突出核心数据/趋势； 5. 禁止重复：不重复用户问题、不重复执行的SQL语句，仅针对结果做业务解读。 """) @@ -48,4 +51,132 @@ public interface ErpAiAgent { @V("sql") String sql, @V("tableStruct") String tableStruct, @V("result") String result); + + /** + * 动态表结构：自然语言解释SQL执行结果 + * 入参：用户问题、执行的SQL、表结构、JSON格式结果 + */ + @SystemMessage(""" + 你是专业的业务数据分析师，请分析以下查询结果： + 【用户问题】 + {{userInput}} + 【数据字段说明】 + {{sMilvusFiledDescription}} + 【查询结果数据（JSON格式）】 + {{result}} + 【分析要求】 + 1. 解释风格：贴合业务场景，无任何SQL专业术语，用口语化、简洁的商业语言说明，避免技术词汇； + 2. 数据准确：严格按照JSON执行结果解释，不夸大、不遗漏、不编造数据，数值与结果完全一致； + 3. 输出格式： + 3.1. 如果用户要求"表格形式展示"，先输出简短文字说明，然后输出Markdown格式的表格 + 3.2. 如果用户未要求表格，仅返回解释内容，不要列出ID,无多余标题、换行、符号 + 3.3. 结果为空时直接返回"未查询到相关数据" + 3.4. 所有数字格式必须以纯文本形式输出,严禁使用千分位分隔符（即不要出现逗号 ","） + 3.5. 所有日期请转换为 YYYY-MM-DD 格式，严禁包含时间部分 + 4. 长度控制：单条解释不超过150字，条理清晰，重点突出核心数据/趋势； + 5. 禁止重复：不重复用户问题、不重复执行的SQL语句，仅针对结果做业务解读。 + """) + @UserMessage(""" + 【用户查询】 + {{userInput}} + 【字段说明】 + {{sMilvusFiledDescription}} + 【查询结果】 + 用户原始查询：{{userInput}} + 执行查询向量库后结果（JSON格式）：{{result}} + 请根据上述信息+通用规则，对查询结果做业务解释： + """) + String explainMilvusResult(@MemoryId String userId, + @V("userInput") String userInput, + @V("sMilvusFiledDescription") String sMilvusFiledDescription, + @V("result") String result); + + /** + * AI路由判断接口 + * true: 走聚合查询（MySQL） + * false: 走向量检索（Milvus） + */ +// @SystemMessage(""" +// 你是一个智能查询路由专家，请根据用户需求判断应该使用哪种查询方式。 +// +// 判断标准： +// 1. 返回 true（聚合查询/MySQL）的场景： +// - 需要计算统计指标：总数、总和、平均值、最大/最小值、占比 +// - 需要数据汇总：分组统计、排行榜、TopN +// - 包含关键词：统计、求和、汇总、排名、平均、数量、总额、最高、最低、占比、分组、分析、趋势 +// - 示例：统计本月销售总额、查询销量前10的商品、各品类占比分析 +// +// 2. 返回 false（向量检索/Milvus）的场景： +// - 查询明细数据：XXX的销售订单明细、XXX的客户信息、具体内容详情 +// - 查找相似内容：根据语义查找相关文档、推荐相似商品 +// - 模糊匹配：不确定具体关键词，需要语义理解 +// - 内容检索：查找包含特定概念的文档 +// - 包含关键词：明细、详情、查询明细、查找、搜索、匹配、推荐、相似、相关、类似 +// - 示例：李留记的销售订单明细、查询关于人工智能的文档、找相似的图片 +// +// 重要规则： +// - 只返回 true 或 false，不要返回其他内容 +// - 不要解释，不要添加额外文字 +// - 如果用户要求"表格形式展示"，返回 false（明细查询） +// - 如果用户指定具体人名、具体对象，返回 false（明细查询） +// """) +// @UserMessage("用户需求：{{userInput}}") + @SystemMessage(""" + 你是一个智能查询路由专家。请根据【用户需求】，只返回 true 或 false + - 如果用户需求包含以下关键词：统计、求和、汇总、排名、TopN、平均、数量、总额、最高、最低、占比、分组，则返回true + - 如果用户需求属于模糊匹配、普通语义检索，查询明细，(例如:查询报价单明细,查询客户信息)，则返回false + - 查询明细数据：XXX的销售订单明细、XXX的客户信息、具体内容详情，则返回false + - 模糊匹配：不确定具体关键词，需要语义理解，则返回false + """) + @UserMessage(""" + 【用户需求】 + {{userInput}} + """) + Boolean routeQuery(@MemoryId String userId, @V("userInput") String userInput); + + /** + * 生成 Milvus 过滤条件 + */ + @SystemMessage(""" + MILVUS 标量过滤条件生成规则（严格遵守）： + 1. 语法规范： + - 允许的操作符：==, !=, like + - 逻辑组合：&& (AND), || (OR) + - 所有字段都是字符串类型，值必须使用单引号包裹 + - 字符串中的单引号需要转义：'O''Reilly' + 2. 可用字段（只能使用这些字段）： + - {{sMilvusFiled}} + 字段说明： + - {{sMilvusFiledDescription}} + 3. 重要规则： + - 只使用上述可用字段，不要创建新字段 + - 如果用户提到了文档类型（如"报价单"、"订单"等），但可用字段中没有类型字段，则忽略该条件 + - 只提取有明确值的字段条件 + 4. 生成规则： + - 如果没有提取到任何具体条件，返回空字符串 + - 从用户输入中提取明确的字段条件 + - 识别模式：字段名 + 操作符 + 值 + - 示例： + * "单据号 INV001" → sBillNo == 'INV001' + * "客户编号 C001" → sCustomerNo == 'C001' + * "销售人员张三" → sSalesManName == '张三' + * "产品包含手机" → sProductStyle like '%手机%' + 5. 输出格式： + - 仅返回纯过滤条件，无任何解释、换行、备注 + - 单条件：sBillNo == 'INV001' + - 多条件：(sBillNo == 'INV001' && sCustomerNo == 'C001') + - 无条件：直接返回空字符串 + """) + @UserMessage(""" + 【用户查询】 + - {{userInput}} + 【可用字段】 + - {{sMilvusFiled}} + 【字段说明】 + - {{sMilvusFiledDescription}} + """) + String getMilvusFilter(@MemoryId String userId, + @V("userInput") String userInput, + @V("sMilvusFiled") String sMilvusFiled, + @V("sMilvusFiledDescription") String sMilvusFiledDescription); } diff --git a/src/main/java/com/xly/config/OperableChatMemoryProvider.java b/src/main/java/com/xly/config/OperableChatMemoryProvider.java index 24f2952..c7aa9bd 100644 --- a/src/main/java/com/xly/config/OperableChatMemoryProvider.java +++ b/src/main/java/com/xly/config/OperableChatMemoryProvider.java @@ -121,6 +121,30 @@ public class OperableChatMemoryProvider implements ChatMemoryProvider { // 步骤4: 完全重新设置消息列表 return rebuildMemoryWithMessages(memoryId, currentMessages); } + + public List deleteUserLasterMessageBySize(Object memoryId,Integer size) { + if (Objects.isNull(memoryId) || size==0) { + return getCurrentChatMessages(memoryId); + } + // 步骤1: 获取当前所有消息 + ChatMemory currentMemory = this.get(memoryId); + List currentMessages = new ArrayList<>(currentMemory.messages()); + // 从后往前查找内容匹配的最后一条消息 + int indexToDelete = currentMessages.size(); + // 如果找到匹配的消息 + if (indexToDelete >= 0) { + List filteredMessages = new ArrayList<>(currentMessages); + for(int i=0;i1){ + filteredMessages.remove(indexToDelete); + } + } + return rebuildMemoryWithMessages(memoryId, filteredMessages); + } + // 步骤4: 完全重新设置消息列表 + return rebuildMemoryWithMessages(memoryId, currentMessages); + } /** * 批量删除多条消息 * @param memoryId 会话ID diff --git a/src/main/java/com/xly/entity/AiResponseAccumulator.java b/src/main/java/com/xly/entity/AiResponseAccumulator.java new file mode 100644 index 0000000..4dde7dd --- /dev/null +++ b/src/main/java/com/xly/entity/AiResponseAccumulator.java @@ -0,0 +1,89 @@ +package com.xly.entity; + +import com.xly.entity.AiResponseDTO; +import cn.hutool.core.util.StrUtil; +import lombok.extern.slf4j.Slf4j; + +/** + * AI响应累积器 + */ +@Slf4j +public class AiResponseAccumulator { + + private final String requestId; + private final StringBuilder aiTextBuilder = new StringBuilder(); + private final StringBuilder systemTextBuilder = new StringBuilder(); + private String sSceneName; + private String sMethodName; + private String sReturnType; + private int totalChunks = 0; + private int processedChunks = 0; + private final long startTime; + + public AiResponseAccumulator(String requestId) { + this.requestId = requestId; + this.startTime = System.currentTimeMillis(); + } + + /** + * 累积单个AI响应 + */ + public void accumulate(AiResponseDTO response) { + processedChunks++; + + // 更新总块数 + if (response.getTotalChunks() != null && response.getTotalChunks() > 0) { + this.totalChunks = response.getTotalChunks(); + } + + // 累积AI文本片段 + if (StrUtil.isNotBlank(response.getTextFragment())) { + aiTextBuilder.append(response.getTextFragment()); + } + + // 累积系统文本片段 + if (StrUtil.isNotBlank(response.getSystemTextFragment())) { + systemTextBuilder.append(response.getSystemTextFragment()); + } + + // 更新元数据（取最后一次非空值） + if (StrUtil.isNotBlank(response.getSSceneName())) { + this.sSceneName = response.getSSceneName(); + } + if (StrUtil.isNotBlank(response.getSMethodName())) { + this.sMethodName = response.getSMethodName(); + } + if (StrUtil.isNotBlank(response.getSReturnType())) { + this.sReturnType = response.getSReturnType(); + } + + log.debug("累积进度: requestId={}, 已处理={}/{}块, AI文本长度={}", + requestId, processedChunks, totalChunks, aiTextBuilder.length()); + } + + /** + * 获取完整的响应 + */ + public AiResponseDTO getCompleteResponse() { + AiResponseDTO response = new AiResponseDTO(); + response.setRequestId(requestId); + response.setAiText(aiTextBuilder.toString()); + response.setSystemText(systemTextBuilder.toString()); + response.setFullAiText(aiTextBuilder.toString()); + response.setFullSystemText(systemTextBuilder.toString()); + response.setSSceneName(sSceneName); + response.setSMethodName(sMethodName); + response.setSReturnType(sReturnType != null ? sReturnType : "MARKDOWN"); + response.setTotalChunks(totalChunks); + response.setChunkIndex(processedChunks - 1); + response.setIsLastChunk(true); + response.setElapsedTime(System.currentTimeMillis() - startTime); + + log.info("ERP累积完成: requestId={}, 总块数={}, AI文本长度={}, 系统文本长度={}, 耗时={}ms", + requestId, totalChunks, + aiTextBuilder.length(), systemTextBuilder.length(), + response.getElapsedTime()); + + return response; + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/entity/AiResponseDTO.java b/src/main/java/com/xly/entity/AiResponseDTO.java index edf20d2..bc5fa66 100644 --- a/src/main/java/com/xly/entity/AiResponseDTO.java +++ b/src/main/java/com/xly/entity/AiResponseDTO.java @@ -9,9 +9,12 @@ import lombok.Data; import lombok.NoArgsConstructor; import java.io.Serializable; +import java.util.List; +import java.util.Map; /** * TTS响应数据传输对象 + * 增强版：支持流式处理 */ @Data @Builder @@ -20,14 +23,230 @@ import java.io.Serializable; public class AiResponseDTO implements Serializable { private static final long serialVersionUID = 1L; - // AI文字部分 + + // ============ 原有字段 ============ + + /** + * AI文字部分 + */ private String aiText; - //系统拼接返回的文字部分 + + /** + * 系统拼接返回的文字部分 + */ private String systemText; - //业务场景名称 + + /** + * 业务场景名称 + */ private String sSceneName; - //业务方法名称 + + /** + * 业务方法名称 + */ private String sMethodName; + + /** + * 返回类型，默认MARKDOWN + */ private String sReturnType = ReturnTypeCode.MAKEDOWN.getCode(); + // ============ 新增字段：流式处理支持 ============ + + /** + * 请求ID，用于追踪整个流式请求 + */ + private String requestId; + + /** + * 响应码 + */ + private Integer code; + + /** + * 响应消息 + */ + private String message; + + /** + * 处理状态：PROCESSING, COMPLETED, FAILED + */ + private String status; + + /** + * 当前处理的块编号（从0开始） + */ + private Integer chunkIndex; + + /** + * 总块数 + */ + private Integer totalChunks; + + /** + * 是否是最后一块 + */ + private Boolean isLastChunk; + + /** + * 文本片段（用于流式传输） + * 当aiText过大时，可以分段传输 + */ + private String textFragment; + + /** + * 系统文本片段（用于流式传输） + */ + private String systemTextFragment; + + /** + * 累积的完整AI文本（仅在最后一块时返回） + */ + private String fullAiText; + + /** + * 累积的完整系统文本（仅在最后一块时返回） + */ + private String fullSystemText; + + /** + * 处理进度（0-100） + */ + private Integer progress; + + /** + * 时间戳 + */ + private Long timestamp; + + /** + * 处理耗时（毫秒） + */ + private Long elapsedTime; + + /** + * 扩展字段，用于存储额外的元数据 + */ + private Map metadata; + + /** + * 错误信息（当status=FAILED时） + */ + private String errorMessage; + + /** + * 错误码（当status=FAILED时） + */ + private String errorCode; + + // ============ 便捷方法 ============ + + /** + * 判断是否处理成功 + */ + public boolean isSuccess() { + return code != null && code == 200; + } + + /** + * 判断是否处理中 + */ + public boolean isProcessing() { + return "PROCESSING".equals(status); + } + + /** + * 判断是否已完成 + */ + public boolean isCompleted() { + return "COMPLETED".equals(status); + } + + /** + * 判断是否失败 + */ + public boolean isFailed() { + return "FAILED".equals(status); + } + + /** + * 获取完整的文本（AI文本 + 系统文本） + */ + public String getFullText() { + StringBuilder sb = new StringBuilder(); + if (StrUtil.isNotBlank(aiText)) { + sb.append(aiText); + } + if (StrUtil.isNotBlank(systemText)) { + sb.append(systemText); + } + return sb.toString(); + } + + /** + * 创建处理中的响应 + */ + public static AiResponseDTO processing(String requestId, String textFragment, + Integer chunkIndex, Integer totalChunks) { + return AiResponseDTO.builder() + .requestId(requestId) + .code(200) + .message("Processing") + .status("PROCESSING") + .textFragment(textFragment) + .chunkIndex(chunkIndex) + .totalChunks(totalChunks) + .timestamp(System.currentTimeMillis()) + .progress(calculateProgress(chunkIndex, totalChunks)) + .build(); + } + + /** + * 创建完成响应 + */ + public static AiResponseDTO completed(String requestId, String fullAiText, + String fullSystemText, String sSceneName, + String sMethodName, String sReturnType) { + return AiResponseDTO.builder() + .requestId(requestId) + .code(200) + .message("Completed") + .status("COMPLETED") + .fullAiText(fullAiText) + .fullSystemText(fullSystemText) + .aiText(fullAiText) + .systemText(fullSystemText) + .sSceneName(sSceneName) + .sMethodName(sMethodName) + .sReturnType(sReturnType) + .progress(100) + .timestamp(System.currentTimeMillis()) + .isLastChunk(true) + .build(); + } + + /** + * 创建失败响应 + */ + public static AiResponseDTO failed(String requestId, String errorMessage, String errorCode) { + return AiResponseDTO.builder() + .requestId(requestId) + .code(500) + .message("Failed") + .status("FAILED") + .errorMessage(errorMessage) + .errorCode(errorCode) + .timestamp(System.currentTimeMillis()) + .build(); + } + + /** + * 计算进度 + */ + private static Integer calculateProgress(Integer chunkIndex, Integer totalChunks) { + if (chunkIndex == null || totalChunks == null || totalChunks == 0) { + return 0; + } + return (int) ((chunkIndex + 1) * 100.0 / totalChunks); + } } \ No newline at end of file diff --git a/src/main/java/com/xly/entity/ToolMeta.java b/src/main/java/com/xly/entity/ToolMeta.java index c64fe5e..42e14cb 100644 --- a/src/main/java/com/xly/entity/ToolMeta.java +++ b/src/main/java/com/xly/entity/ToolMeta.java @@ -47,4 +47,7 @@ public class ToolMeta { private List paramRuleListCheck;//需要校验 private List paramRuleListAll;//所有的 private LocalDateTime tMakeDate; + private String sVectorfiled; + private String sVectorjson; + } diff --git a/src/main/java/com/xly/milvus/bean/CustomSearchResultsWrapper.java b/src/main/java/com/xly/milvus/bean/CustomSearchResultsWrapper.java new file mode 100644 index 0000000..ce73bee --- /dev/null +++ b/src/main/java/com/xly/milvus/bean/CustomSearchResultsWrapper.java @@ -0,0 +1,22 @@ +package com.xly.milvus.bean; + +import io.milvus.grpc.SearchResultData; +import io.milvus.response.SearchResultsWrapper; +import java.util.List; + +/** + * 自定义SearchResultsWrapper，用于访问protected方法 + */ +public class CustomSearchResultsWrapper extends SearchResultsWrapper { + + public CustomSearchResultsWrapper(SearchResultData results) { + super(results); + } + + /** + * 公开访问getOutputFields方法 + */ + public List getOutputFieldsPublic() { + return super.getOutputFields(); + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/bean/SimilaritySearchRequest.java b/src/main/java/com/xly/milvus/bean/SimilaritySearchRequest.java new file mode 100644 index 0000000..54c3719 --- /dev/null +++ b/src/main/java/com/xly/milvus/bean/SimilaritySearchRequest.java @@ -0,0 +1,30 @@ +package com.xly.milvus.bean; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; + +/** + * 相似度查询请求实体 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SimilaritySearchRequest { + private List queryVector; // 查询向量 + private String queryText; // 查询文本（如果有文本转向量服务） + private Integer topK = 10; // 返回数量 + private Double minScore; // 最小相似度得分 + private Double maxScore; // 最大相似度得分 + private String metricType = "IP"; // 距离类型: IP(内积), L2(欧氏距离), COSINE(余弦) + private List outputFields; // 输出字段 + private Map filter; // 过滤条件 + private String partitionName; // 分区名称 + private Boolean withScore = true; // 是否返回得分 + private Boolean withDistance = false; // 是否返回距离 +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/bean/SimilaritySearchResult.java b/src/main/java/com/xly/milvus/bean/SimilaritySearchResult.java new file mode 100644 index 0000000..1bd32e5 --- /dev/null +++ b/src/main/java/com/xly/milvus/bean/SimilaritySearchResult.java @@ -0,0 +1,63 @@ +package com.xly.milvus.bean; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +/** + * 相似度查询结果实体 - 增强版 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SimilaritySearchResult { + private Long id; // 整数ID + private String strId; // 字符串ID + private Float score; // 相似度得分 + private Double normalizedScore; // 归一化后的得分（0-1之间） + private Map fields; // 字段数据 + private String collectionName; // 集合名称 + + /** + * 获取ID（优先返回字符串ID，如果没有则返回整数ID） + */ + public String getId() { + if (strId != null && !strId.isEmpty()) { + return strId; + } + return id != null ? String.valueOf(id) : null; + } + + /** + * 获取字段值 + */ + public Object getField(String fieldName) { + if (fields != null) { + return fields.get(fieldName); + } + return null; + } + + /** + * 获取字符串字段值 + */ + public String getStringField(String fieldName) { + Object value = getField(fieldName); + return value != null ? value.toString() : null; + } + + /** + * 获取整数字段值 + */ + public Integer getIntField(String fieldName) { + Object value = getField(fieldName); + if (value instanceof Number) { + return ((Number) value).intValue(); + } + return null; + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/config/EmbeddingConfig.java b/src/main/java/com/xly/milvus/config/EmbeddingConfig.java new file mode 100644 index 0000000..fd5c348 --- /dev/null +++ b/src/main/java/com/xly/milvus/config/EmbeddingConfig.java @@ -0,0 +1,20 @@ +package com.xly.milvus.config; + +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * LangChain4j 嵌入模型配置类 + */ +@Configuration +public class EmbeddingConfig { + + @Bean + public EmbeddingModel embeddingModel() { + // 使用 All-MiniLM-L6-v2 嵌入模型 + // 这是一个轻量级的模型，维度为384，适合在本地运行 + return new AllMiniLmL6V2EmbeddingModel(); + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/config/MilvusConfig.java b/src/main/java/com/xly/milvus/config/MilvusConfig.java new file mode 100644 index 0000000..a38e673 --- /dev/null +++ b/src/main/java/com/xly/milvus/config/MilvusConfig.java @@ -0,0 +1,162 @@ +package com.xly.milvus.config; + +import io.milvus.v2.client.ConnectConfig; +import io.milvus.v2.client.MilvusClientV2; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.retry.annotation.EnableRetry; +import org.springframework.retry.support.RetryTemplate; + +@Slf4j +@Configuration +@EnableRetry +public class MilvusConfig { + + @Value("${milvus.host:localhost}") + private String host; + + @Value("${milvus.port:19530}") + private Integer port; + + @Value("${milvus.database:default}") + private String database; + + @Value("${milvus.username:}") + private String username; + + @Value("${milvus.password:}") + private String password; + + @Value("${milvus.connect-timeout:10000}") + private Long connectTimeout; + + @Value("${milvus.rpc-deadline:10000}") + private Long rpcDeadline; + + @Value("${milvus.keep-alive-time:300000}") + private Long keepAliveTime; + + @Value("${milvus.keep-alive-timeout:5000}") + private Long keepAliveTimeout; + + @Value("${milvus.secure:false}") + private Boolean secure; + + @Value("${milvus.validate-on-startup:true}") + private boolean validateOnStartup; + + @Value("${milvus.fail-on-connect-error:false}") + private boolean failOnConnectError; + + @Value("${milvus.enable-precheck:true}") + private boolean enablePrecheck; + + @Bean(destroyMethod = "close") + public MilvusClientV2 milvusClient() { + try { + String uri = String.format("http://%s:%d", host, port); + // 构建连接配置 + ConnectConfig.ConnectConfigBuilder configBuilder = ConnectConfig.builder() + .uri(uri) + .dbName("default") + .connectTimeoutMs(connectTimeout) + .rpcDeadlineMs(rpcDeadline) + .keepAliveTimeMs(keepAliveTime) + .keepAliveTimeoutMs(keepAliveTimeout) + .keepAliveWithoutCalls(true) + .secure(secure) + .enablePrecheck(enablePrecheck); + + // 添加认证信息（如果有） + if (username != null && !username.isEmpty()) { + configBuilder.username(username); + if (password != null) { + configBuilder.password(password); + } + } + + ConnectConfig connectConfig = configBuilder.build(); + + // 创建客户端 + MilvusClientV2 client = new MilvusClientV2(connectConfig); + // 检查xlyerp数据库是否存在，如果不存在则创建 + var databases = client.listDatabases(); + if (!databases.getDatabaseNames().contains(database)) { + log.info("数据库 xlyerp 不存在，正在创建..."); + client.createDatabase( + io.milvus.v2.service.database.request.CreateDatabaseReq.builder() + .databaseName(database) + .build() + ); + log.info("数据库 xlyerp 创建成功"); + } + + // 切换到xlyerp数据库 + client.useDatabase (database); + log.info("切换到数据库: {}",database); + + // 启动验证 + if (validateOnStartup) { + validateConnection(client); + } + return client; + } catch (Exception e) { + log.error("创建Milvus客户端时发生异常: {}", e.getMessage(), e); + throw new RuntimeException("无法创建Milvus客户端", e); + } + + } + + /** + * 验证Milvus连接 - 基于源码中可用的方法 + */ + private void validateConnection(MilvusClientV2 client) { + try { + // 方法1: 检查服务器版本（源码中存在getServerVersion()） + String serverVersion = client.getServerVersion(); + + // 方法2: 列出数据库（源码中存在listDatabases()） + var databases = client.listDatabases(); + + // 方法3: 检查健康状态（源码中存在checkHealth()） + var health = client.checkHealth(); + + System.out.println("✅ Milvus连接成功"); + System.out.println(" - 服务器版本: " + serverVersion); + System.out.println(" - 数据库: " + databases); + System.out.println(" - 健康状态: " + health); + + // 验证指定数据库是否存在 + if (database != null && !database.isEmpty()) { + boolean dbExists = databases.getDatabaseNames().contains(database); + if (!dbExists && !"default".equals(database)) { + System.err.println("⚠️ 警告: 指定数据库 '" + database + "' 不存在"); + } + } + + } catch (Exception e) { + String errorMsg = String.format("❌ Milvus连接失败: %s:%d - %s",host, port, e.getMessage()); + + if (failOnConnectError) { + throw new IllegalStateException(errorMsg, e); + } else { + System.err.println(errorMsg); + System.err.println(" ⚠️ 应用将继续启动，但Milvus功能可能不可用"); + } + } + } + + /** + * 配置重试模板（可选） + */ + @Bean + public RetryTemplate milvusRetryTemplate() { + return RetryTemplate.builder() + .maxAttempts(3) + .exponentialBackoff(1000, 2, 10000) + .retryOn(Exception.class) + .build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/config/MilvusHealthIndicator.java b/src/main/java/com/xly/milvus/config/MilvusHealthIndicator.java new file mode 100644 index 0000000..5f5c819 --- /dev/null +++ b/src/main/java/com/xly/milvus/config/MilvusHealthIndicator.java @@ -0,0 +1,68 @@ +package com.xly.milvus.config; + +import io.milvus.v2.client.MilvusClientV2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +@Component +public class MilvusHealthIndicator implements HealthIndicator { + + @Autowired(required = false) + private MilvusClientV2 milvusClient; + + @Override + public Health health() { + if (milvusClient == null) { + return Health.down() + .withDetail("error", "Milvus客户端未初始化") + .build(); + } + + try { + // 检查客户端是否就绪（源码中存在clientIsReady()） + boolean isReady = milvusClient.clientIsReady(); + + if (!isReady) { + return Health.down() + .withDetail("error", "客户端未就绪") + .build(); + } + + long startTime = System.currentTimeMillis(); + + // 执行健康检查 - 使用源码中存在的方法 + String serverVersion = milvusClient.getServerVersion(); + var databases = milvusClient.listDatabases(); + var healthCheck = milvusClient.checkHealth(); + + long responseTime = System.currentTimeMillis() - startTime; + + Map details = new HashMap<>(); + details.put("serverVersion", serverVersion); + details.put("database", milvusClient.currentUsedDatabase()); + details.put("databases", databases.getDatabaseNames()); + details.put("healthStatus", healthCheck.getIsHealthy()); + details.put("responseTime", responseTime + "ms"); + details.put("clientReady", isReady); + + if (healthCheck.getQuotaStates() != null) { + details.put("quotaStates", healthCheck.getQuotaStates()); + } + + return Health.up() + .withDetails(details) + .build(); + + } catch (Exception e) { + return Health.down() + .withDetail("error", e.getMessage()) + .withDetail("clientReady", milvusClient.clientIsReady()) + .build(); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/config/MilvusRetryConfig.java b/src/main/java/com/xly/milvus/config/MilvusRetryConfig.java new file mode 100644 index 0000000..1e3bb54 --- /dev/null +++ b/src/main/java/com/xly/milvus/config/MilvusRetryConfig.java @@ -0,0 +1,57 @@ +package com.xly.milvus.config; + +import io.milvus.v2.client.RetryConfig; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MilvusRetryConfig { + + @Value("${milvus.retry.max-retry-times:75}") + private int maxRetryTimes; + + @Value("${milvus.retry.initial-backoff-ms:10}") + private long initialBackOffMs; + + @Value("${milvus.retry.max-backoff-ms:3000}") + private long maxBackOffMs; + + @Value("${milvus.retry.backoff-multiplier:3}") + private int backOffMultiplier; + + @Value("${milvus.retry.retry-on-rate-limit:true}") + private boolean retryOnRateLimit; + + @Value("${milvus.retry.max-retry-timeout-ms:0}") + private long maxRetryTimeoutMs; + + @Value("${milvus.retry.enabled:true}") + private boolean retryEnabled; + + /** + * 方法名改为 createRetryConfig，避免与类名冲突 + */ + @Bean + public RetryConfig createRetryConfig() { + if (!retryEnabled) { + return RetryConfig.builder() + .maxRetryTimes(1) + .initialBackOffMs(0) + .maxBackOffMs(0) + .backOffMultiplier(1) + .retryOnRateLimit(false) + .maxRetryTimeoutMs(0) + .build(); + } + + return RetryConfig.builder() + .maxRetryTimes(maxRetryTimes) + .initialBackOffMs(initialBackOffMs) + .maxBackOffMs(maxBackOffMs) + .backOffMultiplier(backOffMultiplier) + .retryOnRateLimit(retryOnRateLimit) + .maxRetryTimeoutMs(maxRetryTimeoutMs) + .build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/config/MilvusStartupValidator.java b/src/main/java/com/xly/milvus/config/MilvusStartupValidator.java new file mode 100644 index 0000000..e3e1f3a --- /dev/null +++ b/src/main/java/com/xly/milvus/config/MilvusStartupValidator.java @@ -0,0 +1,79 @@ +package com.xly.milvus.config; + +import io.milvus.v2.client.MilvusClientV2; +import io.milvus.v2.service.collection.request.HasCollectionReq; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Slf4j +@Component +@Order(1) +public class MilvusStartupValidator implements ApplicationRunner { + + @Autowired + private MilvusClientV2 milvusClient; + + @Value("${milvus.validation.collections:}") + private List validationCollections; + + @Value("${milvus.validation.timeout:5000}") + private long validationTimeout; + + @Override + public void run(ApplicationArguments args) throws Exception { + log.info("开始Milvus启动验证..."); + + try { + // 1. 验证客户端就绪状态 + boolean isReady = milvusClient.clientIsReady(); + log.info("客户端就绪状态: {}", isReady); + + // 2. 获取服务器版本 + String serverVersion = milvusClient.getServerVersion(); + log.info("Milvus服务器版本: {}", serverVersion); + + // 3. 列出所有数据库 + var databases = milvusClient.listDatabases(); + log.info("可用数据库: {}", databases.getDatabaseNames()); + + // 4. 检查当前使用的数据库 + String currentDb = milvusClient.currentUsedDatabase(); + log.info("当前数据库: {}", currentDb); + + // 5. 健康检查 + var healthCheck = milvusClient.checkHealth(); + log.info("健康状态: {}", healthCheck.getIsHealthy()); + if (healthCheck.getQuotaStates() != null && !healthCheck.getQuotaStates().isEmpty()) { + log.info("配额状态: {}", healthCheck.getQuotaStates()); + } + + // 6. 验证指定的集合（可选） + if (validationCollections != null && !validationCollections.isEmpty()) { + for (String collectionName : validationCollections) { + boolean exists = milvusClient.hasCollection( + HasCollectionReq.builder() + .collectionName(collectionName) + .build() + ); + log.info("集合 '{}' 存在: {}", collectionName, exists); + } + } + + log.info("✅ Milvus启动验证完成"); + + } catch (Exception e) { + log.error("❌ Milvus启动验证失败: {}", e.getMessage(), e); + // 可以根据配置决定是否抛出异常 + // throw new RuntimeException("Milvus验证失败", e); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/config/SearchResult.java b/src/main/java/com/xly/milvus/config/SearchResult.java new file mode 100644 index 0000000..55da993 --- /dev/null +++ b/src/main/java/com/xly/milvus/config/SearchResult.java @@ -0,0 +1,22 @@ +package com.xly.milvus.config; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +/** + * 搜索结果实体 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SearchResult { + private Long id; + private Float score; + private Map fields; + private String collectionName; +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/service/AiGlobalAgentQuestionSqlEmitterService.java b/src/main/java/com/xly/milvus/service/AiGlobalAgentQuestionSqlEmitterService.java new file mode 100644 index 0000000..c98c0d7 --- /dev/null +++ b/src/main/java/com/xly/milvus/service/AiGlobalAgentQuestionSqlEmitterService.java @@ -0,0 +1,20 @@ +package com.xly.milvus.service; + + +import java.util.Map; + +public interface AiGlobalAgentQuestionSqlEmitterService { + + /*** + * @Author 钱豹 + * @Date 15:27 2026/3/19 + * @Param [data, sQuestion, sSqlContent, collectionName] + * @return void + * @Description 插入向量库 + **/ + void addAiGlobalAgentQuestionSqlEmitter(String sKey,Map data, String sQuestion, String sSqlContent, String collectionName); + + + Map queryAiGlobalAgentQuestionSqlEmitter(String searchText, String collectionName); + +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/service/EmbeddingService.java b/src/main/java/com/xly/milvus/service/EmbeddingService.java new file mode 100644 index 0000000..93985f7 --- /dev/null +++ b/src/main/java/com/xly/milvus/service/EmbeddingService.java @@ -0,0 +1,17 @@ +package com.xly.milvus.service; + +import java.util.List; + +public interface EmbeddingService { + + /** + * 生成单个文本的向量 + */ + public List generateEmbedding(String text); + + /** + * 批量生成向量（高效版） + * 利用 LangChain4j 内置的并行化能力，显著提升性能 + */ + public List> generateEmbeddings(List texts); +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/service/MilvusService.java b/src/main/java/com/xly/milvus/service/MilvusService.java new file mode 100644 index 0000000..b0c800c --- /dev/null +++ b/src/main/java/com/xly/milvus/service/MilvusService.java @@ -0,0 +1,58 @@ +package com.xly.milvus.service; + +import com.xly.tts.bean.TTSResponseDTO; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 向量化服务接口 + */ +public interface MilvusService { + + /*** + * @Author 钱豹 + * @Date 22:17 2026/3/24 + * @Param + * @return + * @Description 初始化数据 + **/ + + TTSResponseDTO initDataToMilvus(Map reqMap); + + + /** + * 创建集合（如果不存在） + */ + void createCollectionIfNotExists(String collectionName, String sVectorfiled, String sVectorjson, Boolean bRset); + + + /*** + * @Author 钱豹 + * @Date 21:39 2026/3/24 + * @Param [collectionName, sVectorfiled, sVectorjson, data] + * @return long + * @Description 批量插入数据 + **/ + long addDataToCollection(String collectionName, String sVectorfiled, String sVectorjson, List> data); + + /*** + * @Author 钱豹 + * @Date 10:39 2026/3/25 + * @Param + * @return + * @Description 向量库查询 + **/ + List> getDataToCollection(String collectionName, String milvusFilter,String searchText,Integer size,List fields); + + /*** + * @Author 钱豹 + * @Date 10:56 2026/3/25 + * @Param [sVectorfiled] + * @return java.util.Map + * @Description 获取配置 + **/ + Map getMilvusFiled(String sVectorfiled); +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/service/VectorizationService.java b/src/main/java/com/xly/milvus/service/VectorizationService.java new file mode 100644 index 0000000..ca47d70 --- /dev/null +++ b/src/main/java/com/xly/milvus/service/VectorizationService.java @@ -0,0 +1,23 @@ +package com.xly.milvus.service; + +import java.util.List; + +/** + * 向量化服务接口 + */ +public interface VectorizationService { + + /** + * 将文本向量化 + * @param text 文本内容 + * @return 向量数组 + */ + List textToVector(String text); + + /** + * 批量向量化 + * @param texts 文本列表 + * @return 向量列表 + */ + List> batchTextToVector(List texts); +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/service/impl/AiGlobalAgentQuestionSqlEmitterServiceImpl.java b/src/main/java/com/xly/milvus/service/impl/AiGlobalAgentQuestionSqlEmitterServiceImpl.java new file mode 100644 index 0000000..bacaa16 --- /dev/null +++ b/src/main/java/com/xly/milvus/service/impl/AiGlobalAgentQuestionSqlEmitterServiceImpl.java @@ -0,0 +1,716 @@ +package com.xly.milvus.service.impl; + +import cn.hutool.core.collection.ConcurrentHashSet; +import cn.hutool.core.util.ObjectUtil; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.xly.milvus.service.AiGlobalAgentQuestionSqlEmitterService; +import com.xly.milvus.service.VectorizationService; +import com.xly.milvus.util.MapToJsonConverter; +import io.milvus.v2.client.MilvusClientV2; +import io.milvus.v2.common.ConsistencyLevel; +import io.milvus.v2.common.DataType; +import io.milvus.v2.common.IndexBuildState; +import io.milvus.v2.common.IndexParam; +import io.milvus.v2.service.collection.request.*; +import io.milvus.v2.service.index.request.CreateIndexReq; +import io.milvus.v2.service.index.request.DescribeIndexReq; +import io.milvus.v2.service.index.request.DropIndexReq; +import io.milvus.v2.service.index.response.DescribeIndexResp; +import io.milvus.v2.service.vector.request.InsertReq; +import io.milvus.v2.service.vector.request.SearchReq; +import io.milvus.v2.service.vector.request.data.FloatVec; +import io.milvus.v2.service.vector.response.InsertResp; +import io.milvus.v2.service.vector.response.SearchResp; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.*; + +@Slf4j +@Service("aiGlobalAgentQuestionSqlEmitterService") +@RequiredArgsConstructor +public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgentQuestionSqlEmitterService { + + private final MilvusClientV2 milvusClient; + private final VectorizationService vectorizationService; + + // 或者从配置文件读取 + @Value("${milvus.vector.dimension:384}") + private int VECTOR_DIM; + + // 缓存已加载的集合 + private final Set loadedCollections = new ConcurrentHashSet<>(); + + /*** + * @Author 钱豹 + * @Date 13:06 2026/3/19 + * @Param [] + * @return void + * @Description 插入数据 + **/ + @Override + public void addAiGlobalAgentQuestionSqlEmitter(String sKey,Map data,String sQuestion,String sSqlContent,String collectionName) { + // 向量化 + List vector = vectorizationService.textToVector(sKey); + + if (vector == null || vector.isEmpty()) { + throw new RuntimeException("向量化失败"); + } + + // 2. 转换为Milvus格式 + JsonObject row = convertToMilvusRow(data, vector,sQuestion,sSqlContent,sKey); + + //创建集合 +// createCollection(collectionName); + createCollectionIfNotExists(collectionName); + + // 3. 插入到Milvus + InsertReq insertReq = InsertReq.builder() + .collectionName(collectionName) + .data(List.of(row)) + .build(); + + InsertResp insertResp = milvusClient.insert(insertReq); + System.out.println("成功插入 " + insertResp.getInsertCnt() + " 条数据"); + System.out.println(" - 数据预览:"); + } + + @Override + public Map queryAiGlobalAgentQuestionSqlEmitter(String searchText, String collectionName) { + Map result = new HashMap<>(); + log.info("开始相似度查询: collection={}, searchText={}", collectionName, searchText); + // 2. 设置范围搜索参数 + Map searchParams = new HashMap<>(); + searchParams.put("nprobe", 10); + // 对于 IP 度量，相似度范围在 [minScore, maxScore] + searchParams.put("radius", 0.9); // 最小相似度 + searchParams.put("range_filter", 1); // 最大相似度 + // 1. 确保集合已加载 + ensureCollectionLoaded(collectionName); + + // 1. 向量化搜索文本 + List vectorList = vectorizationService.textToVector(searchText); + if (vectorList == null || vectorList.isEmpty()) { + throw new RuntimeException("向量化失败"); + } + // 2. 转换为 float[] + float[] floatArray = new float[vectorList.size()]; + for (int i = 0; i < vectorList.size(); i++) { + floatArray[i] = vectorList.get(i); + } + // 查询最近插入的数据（按时间倒序） +// QueryReq queryReq = QueryReq.builder() +// .collectionName(collectionName) +// .outputFields(Arrays.asList("sQuestion", "sSqlContent", "data_id", "create_time","metadata")) +// .limit(100) +// .build(); +// QueryResp queryResp = milvusClient.query(queryReq); + + // 3. 创建 Milvus FloatVec 对象 + FloatVec floatVec = new FloatVec(floatArray); + // 4. 构建搜索请求 + SearchReq searchReq = SearchReq.builder() + .collectionName(collectionName) + .data(Collections.singletonList(floatVec)) + .annsField("vector") // 向量字段名 + .topK(10) // 返回最相似的10条 + .metricType(IndexParam.MetricType.IP) // 内积相似度 + .outputFields(Arrays.asList("sQuestion", "sSqlContent", "data_id", "create_time","metadata")) + .searchParams(searchParams) + .build(); + // 5. 执行搜索 + SearchResp searchResp = milvusClient.search(searchReq); + // 6. 处理结果 + List> searchResults = searchResp.getSearchResults(); + if(ObjectUtil.isEmpty(searchResults)){ + return result; + } + List firstResultList = searchResults.get(0); + if(ObjectUtil.isEmpty(firstResultList)){ + return result; + } + firstResultList.sort((a, b) -> Float.compare(b.getScore(), a.getScore())); + SearchResp.SearchResult item = firstResultList.get(0); + Map itemMap = new HashMap<>(); + itemMap.put("score", item.getScore()); + itemMap.put("id", item.getId()); + itemMap.putAll(item.getEntity()); + return itemMap; + } + + /** + * 确保集合已加载 + */ + private void ensureCollectionLoaded(String collectionName) { + try { + // 如果已经加载过，直接返回 + if (loadedCollections.contains(collectionName)) { + return; + } + log.info("检查集合加载状态: {}", collectionName); + // 检查集合是否存在 + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() + .collectionName(collectionName) + .build(); + + boolean exists = milvusClient.hasCollection(hasCollectionReq); + + if (!exists) { + log.error("集合不存在: {}", collectionName); + throw new RuntimeException("集合不存在: " + collectionName); + } + + // 获取加载状态 + GetLoadStateReq getLoadStateReq = GetLoadStateReq.builder() + .collectionName(collectionName) + .build(); + + boolean isLoaded = milvusClient.getLoadState(getLoadStateReq); + + if (!isLoaded) { + log.info("加载集合到内存: {}", collectionName); + + // 加载集合 + LoadCollectionReq loadCollectionReq = LoadCollectionReq.builder() + .collectionName(collectionName) + .build(); + + milvusClient.loadCollection(loadCollectionReq); + + // 等待加载完成 + waitForCollectionLoaded(collectionName); + + loadedCollections.add(collectionName); + log.info("集合加载完成: {}", collectionName); + } else { + loadedCollections.add(collectionName); + log.info("集合已加载: {}", collectionName); + } + + } catch (Exception e) { + log.error("确保集合加载失败", e); + throw new RuntimeException("集合加载失败: " + collectionName, e); + } + } + + /** + * 等待集合加载完成 + */ + private void waitForCollectionLoaded(String collectionName) { + int maxRetries = 30; + int retryInterval = 1000; // 1秒 + + for (int i = 0; i < maxRetries; i++) { + try { + GetLoadStateReq getLoadStateReq = GetLoadStateReq.builder() + .collectionName(collectionName) + .build(); + + boolean isLoaded = milvusClient.getLoadState(getLoadStateReq); + + if (isLoaded) { + log.info("集合加载状态确认: {}", collectionName); + return; + } + + Thread.sleep(retryInterval); + + } catch (Exception e) { + log.warn("检查加载状态失败，重试 {}/{}", i + 1, maxRetries); + } + } + + throw new RuntimeException("集合加载超时: " + collectionName); + } + + + /** + * 从实体对象构建Milvus插入数据 + */ + public JsonObject convertToMilvusRow(Map data, List vector,String sQuestion,String sSqlContent,String sKey) { + JsonObject row = new JsonObject(); + + // 添加向量 + JsonArray vectorArray = new JsonArray(); + vector.forEach(vectorArray::add); + row.add("vector", vectorArray); + // 添加文本字段 + row.addProperty("sKey", sKey); + row.addProperty("data_id", data.get("sId").toString()); + row.addProperty("sQuestion", sQuestion); + row.addProperty("sSqlContent", sSqlContent); + // 创建时间字段 - 必须提供！ + row.addProperty("create_time", System.currentTimeMillis()); + // 创建时间字段 - 必须提供！ +// row.add("create_time", JsonValue.from(System.currentTimeMillis())); + // 添加业务字段到metadata + JsonObject metadata = MapToJsonConverter.convert(data); + row.add("metadata", metadata); + return row; + } + + /** + * 创建集合（如果不存在） + */ + public void createCollectionIfNotExists(String collectionName) { + try { + // 检查集合是否存在 + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() + .collectionName(collectionName) + .build(); + boolean exists = milvusClient.hasCollection(hasCollectionReq); + if (!exists) { + createCollection(collectionName); + log.info("集合 {} 创建成功", collectionName); + } + } catch (Exception e) { + log.error("检查/创建集合失败: {}", collectionName, e); + throw new RuntimeException("初始化Milvus集合失败", e); + } + } + + /** + * 创建集合（定义字段结构） + */ + private void createCollection(String collectionName) { + //删除现有集合 +// DropCollectionReq dropCollectionReq = DropCollectionReq.builder() +// .collectionName(collectionName) +// .build(); +// milvusClient.dropCollection(dropCollectionReq); + + // 定义字段列表 + List fieldSchemas = Arrays.asList( + // 1. 主键字段 + CreateCollectionReq.FieldSchema.builder() + .name("id") + .dataType(DataType.Int64) + .isPrimaryKey(true) + .autoID(true) // 使用自动ID + .description("主键ID") + .build(), + + // 2. 向量字段 + CreateCollectionReq.FieldSchema.builder() + .name("vector") + .dataType(DataType.FloatVector) + .dimension(VECTOR_DIM) + .description("向量字段，用于相似性搜索") + .build(), + + // 3. 问题字段 + CreateCollectionReq.FieldSchema.builder() + .name("sQuestion") + .dataType(DataType.VarChar) + .maxLength(1000) + .description("用户问题") + .build(), + + // 4. SQL内容字段 + CreateCollectionReq.FieldSchema.builder() + .name("sSqlContent") + .dataType(DataType.VarChar) + .maxLength(5000) // SQL可能较长 + .description("SQL语句") + .build(), + + // 5. 数据ID字段 + CreateCollectionReq.FieldSchema.builder() + .name("data_id") + .dataType(DataType.VarChar) + .maxLength(100) + .description("原始数据ID") + .build(), + + // 6. 创建时间字段 + CreateCollectionReq.FieldSchema.builder() + .name("create_time") + .dataType(DataType.Int64) + .description("创建时间戳") + .build(), + + // 7. 元数据字段（使用JSON类型存储额外数据） + CreateCollectionReq.FieldSchema.builder() + .name("metadata") + .dataType(DataType.JSON) + .description("额外元数据") + .build(), + CreateCollectionReq.FieldSchema.builder() + .name("sKey") + .dataType(DataType.VarChar) + .maxLength(100) + .description("存入的vector转换前数据") + .build() + ); + + // 创建集合schema + CreateCollectionReq.CollectionSchema schema = + CreateCollectionReq.CollectionSchema.builder() + .fieldSchemaList(fieldSchemas) + .enableDynamicField(true) + .build(); + + // 创建集合请求 + CreateCollectionReq createCollectionReq = CreateCollectionReq.builder() + .collectionName(collectionName) + .collectionSchema(schema) + .consistencyLevel(ConsistencyLevel.BOUNDED) + .build(); + + // 执行创建集合 + milvusClient.createCollection(createCollectionReq); + + //创建索引 + createIndexesStepByStep(collectionName); + } + + /* + * 分步创建索引，便于监控每个索引的状态 + */ + private void createIndexesStepByStep(String collectionName) { + log.info("开始为集合创建索引: {}", collectionName); + createAllIndexes(collectionName); +// // 1. 创建向量索引 +// createVectorIndex(collectionName); +// +// // 2. 创建标量索引 +// createScalarIndexes(collectionName); + } + + /** + * 创建向量索引 + * IVF_FLAT 向量相似度搜索 常用的向量索引，平衡性能和召回率 + * STL_SORT 标量字段排序 适用于数字、时间等需要排序的字段 + * INVERTED 文本字段过滤 倒排索引，适用于文本字段的精确匹配 + * TRIE 字符串前缀匹配 适用于前缀查询 + * BITMAP 枚举值过滤 适用于低基数字段 + */ + private void createVectorIndex(String collectionName) { + log.info("创建向量索引: {}", collectionName); + + Map extraParams = new HashMap<>(); + extraParams.put("nlist", 128); + + IndexParam vectorIndex = IndexParam.builder() + .fieldName("vector") + .indexName("idx_vector") + .indexType(IndexParam.IndexType.IVF_FLAT) + .metricType(IndexParam.MetricType.IP) + .extraParams(extraParams) + .build(); + + CreateIndexReq createIndexReq = CreateIndexReq.builder() + .collectionName(collectionName) + .indexParams(Collections.singletonList(vectorIndex)) + .sync(true) + .timeout(60000L) + .build(); + + milvusClient.createIndex(createIndexReq); + log.info("向量索引创建完成"); + } + + /** + * 创建标量索引 + */ + private void createScalarIndexes(String collectionName) { + log.info("创建标量索引: {}", collectionName); + + // 为 create_time 字段创建索引 + IndexParam timeIndex = IndexParam.builder() + .fieldName("create_time") + .indexName("idx_create_time") + .indexType(IndexParam.IndexType.STL_SORT) // 排序索引 + .build(); + + CreateIndexReq timeIndexReq = CreateIndexReq.builder() + .collectionName(collectionName) + .indexParams(Collections.singletonList(timeIndex)) + .sync(true) + .timeout(30000L) + .build(); + + milvusClient.createIndex(timeIndexReq); + log.info("create_time 索引创建完成"); + + // 为 question 字段创建倒排索引（支持文本过滤） + IndexParam questionIndex = IndexParam.builder() + .fieldName("sQuestion") + .indexName("idx_question") + .indexType(IndexParam.IndexType.TRIE) // 倒排索引 + .build(); + + CreateIndexReq questionIndexReq = CreateIndexReq.builder() + .collectionName(collectionName) + .indexParams(Collections.singletonList(questionIndex)) + .sync(true) + .timeout(30000L) + .build(); + + milvusClient.createIndex(questionIndexReq); + log.info("question 索引创建完成"); + + // 为 data_id 字段创建索引 + IndexParam idIndex = IndexParam.builder() + .fieldName("data_id") + .indexName("idx_data_id") + .indexType(IndexParam.IndexType.TRIE) + .build(); + + CreateIndexReq idIndexReq = CreateIndexReq.builder() + .collectionName(collectionName) + .indexParams(Collections.singletonList(idIndex)) + .sync(true) + .timeout(30000L) + .build(); + + milvusClient.createIndex(idIndexReq); + log.info("data_id 索引创建完成"); + } + + /** + * 重建索引（解决索引未就绪问题） + */ + public boolean rebuildIndex(String collectionName) { + log.info("========== 开始重建索引 =========="); + log.info("集合名称: {}", collectionName); + + try { + // 1. 先检查集合是否存在 + HasCollectionReq hasReq = HasCollectionReq.builder() + .collectionName(collectionName) + .build(); + + if (!milvusClient.hasCollection(hasReq)) { + log.error("集合不存在: {}", collectionName); + return false; + } + + // 2. 先释放集合（如果已加载） + try { + ReleaseCollectionReq releaseReq = ReleaseCollectionReq.builder() + .collectionName(collectionName) + .build(); + milvusClient.releaseCollection(releaseReq); + log.info("集合已释放: {}", collectionName); + Thread.sleep(2000); // 等待释放完成 + } catch (Exception e) { + log.warn("释放集合失败（可能未加载）: {}", e.getMessage()); + } + + // 3. 删除原有索引 + try { + DropIndexReq dropIndexReq = DropIndexReq.builder() + .collectionName(collectionName) + .fieldName("vector") // 指定向量字段 + .build(); + + milvusClient.dropIndex(dropIndexReq); + log.info("原有索引已删除"); + Thread.sleep(2000); // 等待删除完成 + } catch (Exception e) { + log.warn("删除索引失败（可能不存在）: {}", e.getMessage()); + } + + // 4. 创建新索引 + createVectorIndex(collectionName); + + // 5. 等待索引就绪 + boolean indexReady = waitForIndexReady(collectionName, "vector", 60); + if (!indexReady) { + log.error("索引未就绪"); + return false; + } + + // 6. 重新加载集合 + LoadCollectionReq loadReq = LoadCollectionReq.builder() + .collectionName(collectionName) + .build(); + + milvusClient.loadCollection(loadReq); + log.info("集合重新加载成功"); + + // 7. 验证加载状态 + boolean loaded = waitForLoad(collectionName, 30); + if (!loaded) { + log.error("集合加载失败"); + return false; + } + + log.info("✅ 索引重建完成，集合已就绪: {}", collectionName); + return true; + + } catch (Exception e) { + log.error("重建索引失败", e); + return false; + } + } + + /** + * 等待索引就绪 + */ + private boolean waitForIndexReady(String collectionName, String fieldName, int timeoutSeconds) { + log.info("等待索引就绪: {}.{}，超时: {}秒", collectionName, fieldName, timeoutSeconds); + + for (int i = 0; i < timeoutSeconds; i++) { + try { + DescribeIndexReq describeIndexReq = DescribeIndexReq.builder() + .collectionName(collectionName) + .build(); + + DescribeIndexResp describeIndexResp = milvusClient.describeIndex(describeIndexReq); + + List indexDescs = describeIndexResp.getIndexDescriptions(); + + for (DescribeIndexResp.IndexDesc desc : indexDescs) { + if (fieldName.equals(desc.getFieldName())) { + IndexBuildState state = desc.getIndexState(); + + log.info("索引状态: {}, 进度: {}/{}", + state, desc.getIndexedRows(), desc.getTotalRows()); + + if (state == IndexBuildState.Finished) { + log.info("✅ 索引就绪"); + return true; + } else if (state == IndexBuildState.Failed) { + log.error("❌ 索引构建失败: {}", desc.getIndexFailedReason()); + return false; + } + break; + } + } + + Thread.sleep(1000); + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("等待被中断"); + return false; + } catch (Exception e) { + log.warn("检查索引状态失败: {}/{}", i + 1, timeoutSeconds); + } + } + + log.error("等待索引就绪超时"); + return false; + } + + /** + * 等待集合加载完成 + */ + private boolean waitForLoad(String collectionName, int timeoutSeconds) { + log.info("等待集合加载: {}，超时: {}秒", collectionName, timeoutSeconds); + + for (int i = 0; i < timeoutSeconds; i++) { + try { + GetLoadStateReq loadStateReq = GetLoadStateReq.builder() + .collectionName(collectionName) + .build(); + + boolean isLoaded = milvusClient.getLoadState(loadStateReq); + + if (isLoaded) { + log.info("✅ 集合加载完成"); + return true; + } + + Thread.sleep(1000); + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (Exception e) { + log.warn("检查加载状态失败: {}/{}", i + 1, timeoutSeconds); + } + } + + log.error("集合加载超时"); + return false; + } + + + /** + * 批量创建所有索引（向量索引 + 多个标量索引） + */ + private void createAllIndexes(String collectionName) { + log.info("开始为集合批量创建索引: {}", collectionName); + + // 1. 准备所有索引参数 + List allIndexParams = new ArrayList<>(); + + // 1.1 向量索引 + Map vectorExtraParams = new HashMap<>(); + vectorExtraParams.put("nlist", 256); // 聚类中心数：sqrt(384) * 13 ≈ 256 + vectorExtraParams.put("nprobe", 32); // 搜索时检查的聚类数 + + IndexParam vectorIndex = IndexParam.builder() + .fieldName("vector") + .indexName("idx_vector_rebuild") + .indexType(IndexParam.IndexType.IVF_FLAT) + .metricType(IndexParam.MetricType.IP) + .extraParams(vectorExtraParams) + .build(); + allIndexParams.add(vectorIndex); + + // 1.2 create_time 字段索引（用于时间范围查询） + IndexParam timeIndex = IndexParam.builder() + .fieldName("create_time") + .indexName("idx_create_time") + .indexType(IndexParam.IndexType.STL_SORT) // 排序索引 + .build(); + allIndexParams.add(timeIndex); + + // 1.3 question 字段倒排索引（用于文本过滤） + IndexParam questionIndex = IndexParam.builder() + .fieldName("sQuestion") + .indexName("idx_question") + .indexType(IndexParam.IndexType.INVERTED) // 倒排索引 + .build(); + allIndexParams.add(questionIndex); + + // 1.4 data_id 字段索引（用于精确匹配） + IndexParam idIndex = IndexParam.builder() + .fieldName("data_id") + .indexName("idx_data_id") + .indexType(IndexParam.IndexType.INVERTED) + .build(); + allIndexParams.add(idIndex); + + IndexParam sKey = IndexParam.builder() + .fieldName("sKey") + .indexName("s_key") + .indexType(IndexParam.IndexType.INVERTED) + .build(); + allIndexParams.add(sKey); + + // 1.5 sql_content 字段索引（如果需要） +// IndexParam sqlIndex = IndexParam.builder() +// .fieldName("sql_content") +// .indexName("idx_sql_content") +// .indexType(IndexParam.IndexType.INVERTED) +// .build(); +// allIndexParams.add(sqlIndex); + + // 2. 批量创建索引 + try { + CreateIndexReq createIndexReq = CreateIndexReq.builder() + .collectionName(collectionName) + .indexParams(allIndexParams) // 一次性传入所有索引 + .sync(true) // 同步等待 + .timeout(120000L) // 总超时时间120秒 + .build(); + + milvusClient.createIndex(createIndexReq); + log.info("所有索引批量创建成功: {}", collectionName); + + } catch (Exception e) { + log.error("批量创建索引失败: {}", e.getMessage()); + createScalarIndexes(collectionName); + } + } + + +} diff --git a/src/main/java/com/xly/milvus/service/impl/EmbeddingServiceImpl.java b/src/main/java/com/xly/milvus/service/impl/EmbeddingServiceImpl.java new file mode 100644 index 0000000..e865ebd --- /dev/null +++ b/src/main/java/com/xly/milvus/service/impl/EmbeddingServiceImpl.java @@ -0,0 +1,84 @@ +package com.xly.milvus.service.impl; + +import com.xly.milvus.service.EmbeddingService; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +@Service("embeddingService") +@RequiredArgsConstructor +public class EmbeddingServiceImpl implements EmbeddingService { + + private final EmbeddingModel embeddingModel; + + /** + * 生成单个文本的向量 + */ + public List generateEmbedding(String text) { + if (text == null || text.trim().isEmpty()) { + log.warn("Input text is empty"); + return null; + } + + try { + // 0.35.0 API: embed 方法返回 Response + Embedding embedding = embeddingModel.embed(text).content(); + + // 将 float[] 转换为 List 供 Milvus 使用 + float[] vectorArray = embedding.vector(); + List vectorList = new ArrayList<>(vectorArray.length); + for (float v : vectorArray) { + vectorList.add(v); + } + return vectorList; + } catch (Exception e) { + log.error("Error generating embedding for text: {}", text, e); + return null; + } + } + + /** + * 批量生成向量（高效版） + * 利用 LangChain4j 内置的并行化能力，显著提升性能 + */ + public List> generateEmbeddings(List texts) { + if (texts == null || texts.isEmpty()) { + return Collections.emptyList(); + } + + try { + // 将文本转换为 TextSegment 列表 + List segments = texts.stream() + .map(TextSegment::from) + .collect(Collectors.toList()); + + // 批量嵌入，内部自动并行化 + List embeddings = embeddingModel.embedAll(segments).content(); + + // 转换格式 + return embeddings.stream() + .map(embedding -> { + float[] array = embedding.vector(); + List list = new ArrayList<>(array.length); + for (float v : array) { + list.add(v); + } + return list; + }) + .collect(Collectors.toList()); + + } catch (Exception e) { + log.error("Error generating embeddings in batch", e); + return Collections.emptyList(); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/service/impl/MilvusServiceImpl.java b/src/main/java/com/xly/milvus/service/impl/MilvusServiceImpl.java new file mode 100644 index 0000000..3204d0e --- /dev/null +++ b/src/main/java/com/xly/milvus/service/impl/MilvusServiceImpl.java @@ -0,0 +1,788 @@ +package com.xly.milvus.service.impl; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.collection.ConcurrentHashSet; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.thread.ThreadUtil; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.xly.milvus.service.MilvusService; +import com.xly.milvus.service.VectorizationService; +import com.xly.milvus.util.MapToJsonConverter; +import com.xly.service.DynamicExeDbService; +import com.xly.tts.bean.TTSResponseDTO; +import io.milvus.response.SearchResultsWrapper; +import io.milvus.v2.client.MilvusClientV2; +import io.milvus.v2.common.ConsistencyLevel; +import io.milvus.v2.common.DataType; +import io.milvus.v2.common.IndexParam; +import io.milvus.v2.service.collection.request.*; +import io.milvus.v2.service.collection.response.DescribeCollectionResp; +import io.milvus.v2.service.vector.request.DeleteReq; +import io.milvus.v2.service.vector.request.InsertReq; +import io.milvus.v2.service.vector.request.SearchReq; +import io.milvus.v2.service.vector.request.data.FloatVec; +import io.milvus.v2.service.vector.response.InsertResp; +import io.milvus.v2.service.vector.response.QueryResp; +import io.milvus.v2.service.vector.response.SearchResp; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * 向量化服务实现 - 使用LangChain4j的All-MiniLM-L6-v2模型 + */ +@Slf4j +@Service(value = "milvusService") +@RequiredArgsConstructor +public class MilvusServiceImpl implements MilvusService { + + private final MilvusClientV2 milvusClient; + private final VectorizationService vectorizationService; + private final DynamicExeDbService dynamicExeDbService; + + // 或者从配置文件读取 + @Value("${milvus.vector.dimension:384}") + private int VECTOR_DIM; + + // 缓存已经初始化过的 Milvus 集合（线程安全） + private final Set loadedCollections = new ConcurrentHashSet<>(); + + + + /*** + * @Author 钱豹 + * @Date 22:18 2026/3/24 + * @Param [reqMap] + * @return void + * @Description 初始化结构以及数据 + **/ + @Override + public TTSResponseDTO initDataToMilvus(Map reqMap) { + if(ObjectUtil.isEmpty(reqMap.get("sInputTabelName"))){ + return TTSResponseDTO.builder() + .code(-1) + .message("输入表名") + .build(); + } + if(ObjectUtil.isEmpty(reqMap.get("sVectorfiled"))){ + return TTSResponseDTO.builder() + .code(-1) + .message("向量库标量字段") + .build(); + } + if(ObjectUtil.isEmpty(reqMap.get("sVectorjson"))){ + return TTSResponseDTO.builder() + .code(-1) + .message("向量化内容JSON") + .build(); + } + + String sInputTabelName = reqMap.get("sInputTabelName").toString(); + String sVectorfiled = reqMap.get("sVectorfiled").toString(); + String sVectorjson = reqMap.get("sVectorjson").toString(); + + String tUpdateDate = DateUtil.now(); + String tUpdateDateUp = getUpdateDateUp(sInputTabelName); + //获取需要同步地数据 + List> data = getAddData(sInputTabelName,tUpdateDate, tUpdateDateUp); + //创建集合 + createCollectionIfNotExists(sInputTabelName, sVectorfiled, sVectorjson,true); + if(ObjectUtil.isNotEmpty(data)){ + //插入数据 + long num= addDataToCollection(sInputTabelName, sVectorfiled, sVectorjson,data); + } + addAiMilvusVectorRecord(sInputTabelName,tUpdateDate, tUpdateDateUp); + return TTSResponseDTO.builder() + .code(200) + .message("success") + .build(); + } + + public String getUpdateDateUp(String sInputTabelName) { + Map serDataMap = new HashMap<>(); + String sSql ="SELECT DATE_FORMAT(tUpdateDate,'%Y-%m-%d %H:%i:%s') AS tUpdateDate FROM ai_milvus_vector_record WHERE sInputTabelName = #{sInputTabelName}"; + serDataMap.put("sInputTabelName",sInputTabelName); + List> data = this.dynamicExeDbService.findSql(serDataMap,sSql); + if(ObjectUtil.isEmpty(data)){ + return "2000-03-24"; + } + return data.get(0).get("tUpdateDate").toString(); + } + + /*** + * @Author 钱豹 + * @Date 22:24 2026/3/24 + * @Param + * @return + * @Description 获取需要同步地数据 + **/ + public List> getAddData(String sInputTabelName,String tUpdateDate,String tUpdateDateUp) { + //获取需要同步地数据 + Map serDataMap = new HashMap<>(); + serDataMap.put("tUpdateDate",tUpdateDate); + serDataMap.put("tUpdateDateUp",tUpdateDateUp); + String sSql = String.format("SELECT * FROM %s WHERE tUpdateDate >= #{tUpdateDateUp} AND tUpdateDate < #{tUpdateDate}",sInputTabelName); + return this.dynamicExeDbService.findSql(serDataMap,sSql); + } + + /*** + * @Author 钱豹 + * @Date 22:32 2026/3/24 + * @Param [sInputTabelName, tUpdateDate] + * @return java.util.List> + * @Description 获取更新地数据 + **/ + public void addAiMilvusVectorRecord(String sInputTabelName,String tUpdateDate,String tUpdateDateUp) { + //获取需要同步地数据 + Map dMap = new HashMap<>(); + dMap.put("sInputTabelName",sInputTabelName); + String sSql = "DELETE FROM ai_milvus_vector_record WHERE sInputTabelName = #{sInputTabelName}"; + dynamicExeDbService.delSql(dMap,sSql); + dMap.put("tUpdateDate",tUpdateDate); + dMap.put("tUpdateDateUp",tUpdateDateUp); + sSql = String.format("INSERT INTO ai_milvus_vector_record(sId,sInputTabelName,tUpdateDate,tUpdateDateUp)VALUES(newId(),#{sInputTabelName},#{tUpdateDate},#{tUpdateDateUp})"); + dynamicExeDbService.addSql(dMap,sSql); + } + + + /*** + * @Author 钱豹 + * @Date 16:31 2026/3/24 + * @Param [collectionName] + * @return void + * @Description 创建集合 + **/ + @Override + public void createCollectionIfNotExists(String collectionName,String sVectorfiled,String sVectorjson,Boolean bRset) { + createCollection(collectionName,sVectorfiled,bRset); + log.info("集合 {} 创建成功", collectionName); + // 集合缓存：只加载一次 + if (!loadedCollections.contains(collectionName)) { + loadedCollections.add(collectionName); + } + + } + + /*** + * @Author 钱豹 + * @Date 20:59 2026/3/24 + * @Param [collectionName, sVectorfiled, sVectorjson, bRset] + * @return void + * @Description 新增数据集合 + **/ + @Override + public long addDataToCollection(String collectionName, String sVectorfiled, String sVectorjson,List> data){ + + // 1. 参数校验（防止空参数导致崩溃） + if (ObjectUtil.isEmpty(collectionName) || CollUtil.isEmpty(data)) { + throw new IllegalArgumentException("参数异常：集合名/slaveId/数据不能为空"); + } + + // 1. 转换为Milvus格式 + List rows = convertToMilvusRow(data, sVectorfiled, sVectorjson); + if (CollUtil.isEmpty(rows)) { + return 0l; // 无数据直接返回 + } + + // 3.先删除再插入 + // 1. 构建删除请求 + // 过滤条件：匹配唯一键 + // 核心：从 data 中提取所有 sSlaveId，批量删除 + List slaveIdList = data.stream() + .map(map -> map.get("sSlaveId")) // 取每条数据的slaveId + .filter(Objects::nonNull) + .map(String::valueOf) + .distinct() // 去重 + .toList(); + + if (slaveIdList.isEmpty()) { + throw new RuntimeException("未获取到slaveId，无法删除旧数据"); + } + + // 拼接 Milvus 删除条件：sSlaveId in ['111','222','333'] + String filter = String.format("sSlaveId in [%s]", + slaveIdList.stream() + .map(id -> "'" + id + "'") + .collect(Collectors.joining(",")) + ); + + // 批量删除 + DeleteReq deleteReq = DeleteReq.builder() + .collectionName(collectionName) + .filter(filter) + .build(); + milvusClient.delete(deleteReq); + + // 短暂等待 Milvus 数据同步 + ThreadUtil.sleep(100); + + // 4. 插入到Milvus（批量） + InsertReq insertReq = InsertReq.builder() + .collectionName(collectionName) + .data(rows) + .build(); + InsertResp insertResp = milvusClient.insert(insertReq); + return insertResp.getInsertCnt(); + } + + /*** + * @Author 钱豹 + * @Date 13:29 2026/3/25 + * @Param [sVectorfiled] + * @return java.util.Map + * @Description 返回组装动态内容 + **/ + @Override + public Map getMilvusFiled(String sVectorfiled){ + String[] sVectorfiledArray = sVectorfiled.split(","); + List sFileds = new ArrayList<>(); + List sFiledDescriptions = new ArrayList<>(); + List> titleList = new LinkedList<>(); + for(String sVectorfiledOne : sVectorfiledArray){ + Map title = new HashMap<>(); + + String[] sVectorfiledOneArray = sVectorfiledOne.split(":"); + String sDescriptions = sVectorfiledOneArray[0]; + String sName = sVectorfiledOneArray[1]; + sFileds.add(sName); + // 处理描述中可能包含的换行，保持缩进一致 +// String formattedDesc = sDescriptions.replace("\n", "\n "); +// sFiledDescriptions.add(String.format(" - %s: %s", sName, formattedDesc)); + String formattedDesc =String.format("%s: %s", sName, sDescriptions); + sFiledDescriptions.add(formattedDesc); + title.put("sName",sName); + title.put("sTitle",sDescriptions); + titleList.add(title); + } + Map rMap = new HashMap<>(); + rMap.put("sMilvusFiled", String.join(",", sFileds)); + rMap.put("sMilvusFiledDescription", String.join(",", sFiledDescriptions)); + rMap.put("sFileds", sFileds); + rMap.put("title", titleList); + return rMap; + } + + @Override + public List> getDataToCollection(String collectionName, String milvusFilter,String searchText,Integer size,List fields){ + log.info("开始相似度查询: collection={}, searchText={}", collectionName, searchText); + // 2. 设置范围搜索参数 + Map searchParams = new HashMap<>(); + searchParams.put("nprobe", 10); + // 对于 IP 度量，相似度范围在 [minScore, maxScore] + searchParams.put("radius", 0.9); // 最小相似度 + searchParams.put("range_filter", 1); // 最大相似度 + // 1. 确保集合已加载 +// ensureCollectionLoaded(collectionName); + // 1. 向量化搜索文本 + List vectorList = vectorizationService.textToVector(searchText); + if (vectorList == null || vectorList.isEmpty()) { + throw new RuntimeException("向量化失败"); + } + // 2. 转换为 float[] + float[] floatArray = new float[vectorList.size()]; + for (int i = 0; i < vectorList.size(); i++) { + floatArray[i] = vectorList.get(i); + } + + // 3. 创建 Milvus FloatVec 对象 + FloatVec floatVec = new FloatVec(floatArray); + log.info("查询向量库条件{}",milvusFilter); + milvusFilter = isValidMilvusFilter(milvusFilter)?milvusFilter : null; + log.info("实际查询向量库条件{}",milvusFilter); + // 4. 构建搜索请求 + SearchReq searchReq = SearchReq.builder() + .collectionName(collectionName) + .data(Collections.singletonList(floatVec)) + .annsField("vector") // 向量字段名 + .topK(size) // 返回最相似的10条 + .metricType(IndexParam.MetricType.IP) // 内积相似度 + .outputFields(fields) +// .searchParams(searchParams) + .filter(milvusFilter) + .build(); + // 5. 执行搜索 + SearchResp searchResp = milvusClient.search(searchReq); + + // 6. 处理结果 + return processMilvusResults(searchResp); + } + + + /** + * 判断 Milvus 过滤条件是否有效 + * @param milvusFilter 过滤条件字符串 + * @return true: 有效条件, false: 无效条件 + */ + public boolean isValidMilvusFilter(String milvusFilter) { + // 1. 空值判断 + if (milvusFilter == null || milvusFilter.trim().isEmpty()) { + return false; + } + + String filter = milvusFilter.trim(); + + // 2. 基本格式检查：不能是纯布尔值 + if ("true".equalsIgnoreCase(filter) || "false".equalsIgnoreCase(filter)) { + return false; + } + + // 3. 检查是否包含有效的操作符 + boolean hasValidOperator = filter.matches(".*[=!<>]=?.*") || filter.contains(" like "); + if (!hasValidOperator) { + return false; + } + + // 4. 检查字符串值是否使用单引号包裹 + // 匹配模式：字段名 操作符 '值' + Pattern pattern = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*\\s*(==|!=|>=|<=|>|<|like)\\s*('[^']*'|\\d+)"); + Matcher matcher = pattern.matcher(filter); + + // 5. 对于复合条件，递归检查 + if (filter.contains("&&") || filter.contains("||")) { + // 分割复合条件（简单处理，生产环境需要更完善的解析） + String[] conditions = filter.split("&&|\\|\\|"); + for (String condition : conditions) { + condition = condition.trim().replaceAll("^[()]+|[()]+$", ""); // 去除括号 + if (!isValidSimpleCondition(condition)) { + return false; + } + } + return true; + } + + // 6. 检查简单条件 + return isValidSimpleCondition(filter); + } + + /** + * 验证简单条件（不包含 && 和 ||） + */ + private boolean isValidSimpleCondition(String condition) { + if (condition == null || condition.trim().isEmpty()) { + return false; + } + condition = condition.trim(); + // 匹配简单条件的正则 + // 格式：字段名 操作符 值 + // 字段名：字母开头，包含字母数字下划线 + // 操作符：==, !=, >=, <=, >, <, like + // 值：单引号字符串 或 数字 + String regex = "^\\s*([a-zA-Z_][a-zA-Z0-9_]*)\\s*" + // 字段名 + "(==|!=|>=|<=|>|<|like)\\s*" + // 操作符 + "('([^'\\\\]|\\\\.)*'|\\d+(\\.\\d+)?)\\s*$"; // 值 + + if (!condition.matches(regex)) { + return false; + } + // 额外检查：like 操作符的值必须包含 % + if (condition.contains(" like ")) { + String value = condition.split("like")[1].trim(); + if (!value.contains("%")) { + return false; // like 必须使用 % 通配符 + } + } + return true; + } + + + /** + * 处理 Milvus 查询结果（完整版） + */ + /** + * 处理 Milvus 查询结果（完整版） + */ + private List> processMilvusResults(SearchResp response) { + List> results = new ArrayList<>(); + if (response == null) { + log.warn("Milvus 响应为空"); + return results; + } + List> searchResults = response.getSearchResults(); + if (searchResults == null || searchResults.isEmpty()) { + log.warn("Milvus 搜索结果为空"); + return results; + } + // 遍历每个查询的结果集（通常只有一个查询） + for (List resultList : searchResults) { + // 遍历每个搜索结果 + for (SearchResp.SearchResult result : resultList) { + Map item = new HashMap<>(); + // 获取相似度分数 + Float score = result.getScore(); + if (score != null) { + item.put("score", score); + } + // 获取实体字段数据 + Map entity = result.getEntity(); + // 将所有字段添加到结果中 + item.putAll(entity); + results.add(item); + } + } + log.info("处理完成，共 {} 条搜索结果", results.size()); + return results; + } + /** + * 从实体对象构建Milvus插入数据 + */ + public List convertToMilvusRow(List> data, String sVectorfiled,String sVectorjson) { + List rows = new ArrayList<>(); + if (CollUtil.isEmpty(data)) { + return rows; + } + // 批量遍历，逐个转换 + for (Map map : data) { + JsonObject jsonObject = convertToMilvusRowOne(map, sVectorfiled, sVectorjson); + rows.add(jsonObject); + } + return rows; + } + + /*** + * @Author 钱豹 + * @Date 21:31 2026/3/24 + * @Param [data, sVectorfiled, sVectorjson] + * @return com.google.gson.JsonObject + * @Description 单个转换 + **/ + public JsonObject convertToMilvusRowOne(Map data, String sVectorfiled,String sVectorjson) { + + // ====================== 修复 1：使用真实的向量化文本 ====================== + // 从 sVectorjson 或 data 中获取要向量化的字段值 + StringBuffer vectorText = new StringBuffer(); + getVectorText(data, vectorText, sVectorjson); + // 向量化 + List vector = vectorizationService.textToVector(vectorText.toString()); + if (vector == null || vector.isEmpty()) { + throw new RuntimeException("向量化失败，文本内容：" + vectorText); + } + JsonObject row = new JsonObject(); + // 添加向量 + JsonArray vectorArray = new JsonArray(); + vector.forEach(vectorArray::add); + row.add("vector", vectorArray); + + // ====================== 修复 2：sSlaveId 空值安全 ====================== + Object slaveIdObj = data.get("sSlaveId"); + if (slaveIdObj == null) { + throw new RuntimeException("数据中缺少 sSlaveId 字段，无法插入Milvus"); + } + row.addProperty("sSlaveId", slaveIdObj.toString()); + + // 创建时间 + row.addProperty("create_time", System.currentTimeMillis()); + + // 业务元数据 + JsonObject metadata = MapToJsonConverter.convert(data); + row.add("metadata", metadata); + + // 动态字段 + String[] sVectorfiledArray = sVectorfiled.split(","); + for (String vectorFieldOne : sVectorfiledArray) { + String[] fieldArr = vectorFieldOne.split(":"); + if (fieldArr.length < 2) { + continue; + } + String fieldName = fieldArr[1]; + Object value = ObjectUtil.isEmpty(data.get(fieldName)) ? getDefaultData(fieldName) : data.get(fieldName); + // 通用类型安全添加（你之前的优化方法） + addToJsonObject(row, fieldName, value); + } + + return row; + } + + private void getVectorText(Map data, StringBuffer vectorText,String sVectorjson){ + // 动态字段 + String[] sVectorjsonArray = sVectorjson.split(";"); + for (String sVectorjsonOne : sVectorjsonArray) { + String sText; + String[] fieldArr = sVectorjsonOne.split(":"); + if (fieldArr.length < 2) { + continue; + } + String fieldName = fieldArr[1]; + Object value = ObjectUtil.isEmpty(data.get(fieldName)) ? getDefaultData(fieldName) : data.get(fieldName); + if (ObjectUtil.isEmpty(value)) { + sText = StrUtil.EMPTY; + }else{ + sText = value.toString(); + } + vectorText.append(" ").append(fieldArr[0]).append(sText); + } + } + + + /*******************************************************内部方法********************************************************************************/ + + /** + * 安全将 Object 加入 Gson JsonObject，自动识别类型 + */ + private void addToJsonObject(JsonObject row, String fieldName, Object value) { + if (value == null) { + row.addProperty(fieldName, ""); + return; + } + // 基本类型直接添加 + if (value instanceof String) { + row.addProperty(fieldName, (String) value); + } else if (value instanceof Number) { + row.addProperty(fieldName, (Number) value); + } else if (value instanceof Boolean) { + row.addProperty(fieldName, (Boolean) value); + } + // List / 数组类型 + else if (value instanceof List) { + JsonArray jsonArray = new JsonArray(); + for (Object item : (List) value) { + addJsonElement(jsonArray, item); + } + row.add(fieldName, jsonArray); + } + // 其他对象转字符串 + else { + row.addProperty(fieldName, value.toString()); + } + } + /** + * 递归处理 List 元素（支持无限层嵌套 List） + */ + private void addJsonElement(JsonArray jsonArray, Object item) { + if (item == null) { + jsonArray.add((String) null); + return; + } + if (item instanceof String) { + jsonArray.add((String) item); + } else if (item instanceof Number) { + jsonArray.add((Number) item); + } else if (item instanceof Boolean) { + jsonArray.add((Boolean) item); + } else if (item instanceof List) { + // 递归处理嵌套列表 + JsonArray nestedArray = new JsonArray(); + for (Object nestedItem : (List) item) { + addJsonElement(nestedArray, nestedItem); + } + jsonArray.add(nestedArray); + } else { + jsonArray.add(item.toString()); + } + } + + /** + * 创建集合（定义字段结构） + */ + private void createCollection(String collectionName,String sVectorfiled,Boolean bRset) { + // 6. 重新加载集合 + LoadCollectionReq loadReq = LoadCollectionReq.builder() + .collectionName(collectionName) + .build(); + + //是否删除集合 重新创建 + if (bRset){ + // 1. 删除旧集合 + milvusClient.dropCollection(DropCollectionReq.builder() + .collectionName(collectionName) + .build()); + + } + // 检查集合是否存在 + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() + .collectionName(collectionName) + .build(); + boolean exists = milvusClient.hasCollection(hasCollectionReq); + if (exists) { + return; + } + // 定义字段列表 + List fieldSchemas = new ArrayList<>(); + // 1. 准备所有索引参数 + List allIndexParams = new ArrayList<>(); + //定义字段列表 + // 1. 主键字段 + fieldSchemas.add( CreateCollectionReq.FieldSchema.builder() + .name("id") + .dataType(DataType.Int64) + .isPrimaryKey(true) + .autoID(true) // 使用自动ID + .description("主键ID") + .build()); + // 3. 主键字段 + fieldSchemas.add(CreateCollectionReq.FieldSchema.builder() + .name("sSlaveId") + .dataType(DataType.VarChar) +// .isPrimaryKey(true) //索引创建 + .maxLength(100) + .description("原始数据主键ID") + .build()); + // 2. 向量字段 + fieldSchemas.add(CreateCollectionReq.FieldSchema.builder() + .name("vector") + .dataType(DataType.FloatVector) + .dimension(VECTOR_DIM) + .description("向量字段，用于相似性搜索") + .build()); + + // 4. 创建时间字段 + fieldSchemas.add( CreateCollectionReq.FieldSchema.builder() + .name("create_time") + .dataType(DataType.Int64) + .description("创建时间戳") + .build()); + // 5. 元数据字段（使用JSON类型存储额外数据） + fieldSchemas.add(CreateCollectionReq.FieldSchema.builder() + .name("metadata") + .dataType(DataType.JSON) + .description("额外元数据") + .build()); + //动态字段创建 + String[] sVectorfiledArray = sVectorfiled.split(","); + for(String sVectorfiledOne : sVectorfiledArray){ + String[] sVectorfiledOneArray = sVectorfiledOne.split(":"); + String sName = sVectorfiledOneArray[1]; + String sDescription = sVectorfiledOneArray[0]; + DataType dataType = processField(sVectorfiledOneArray[1]); + fieldSchemas.add(CreateCollectionReq.FieldSchema.builder() + .name(sName) + .dataType(dataType) + .description(sDescription) + .isPrimaryKey(false) // 如果不是主键 + .isNullable(true) // 允许为空 +// .defaultValue("") // 如果有默认值 + // SQL可能较长 + .maxLength(1000) + .build()); + } + //创建索引 + createAllIndexes(sVectorfiled,allIndexParams); + + // 创建集合schema + CreateCollectionReq.CollectionSchema schema = + CreateCollectionReq.CollectionSchema.builder() + .fieldSchemaList(fieldSchemas) + .enableDynamicField(true) + .build(); + // 创建集合请求 + CreateCollectionReq createCollectionReq = CreateCollectionReq.builder() + .collectionName(collectionName) + .collectionSchema(schema) + .indexParams(allIndexParams)//索引集合 + .consistencyLevel(ConsistencyLevel.BOUNDED) + .build(); + + // 执行创建集合 + milvusClient.createCollection(createCollectionReq); + milvusClient.loadCollection(loadReq); + log.info("集合重新加载成功"); + } + /** + * 批量创建所有索引（向量索引 + 多个标量索引） + */ + private void createAllIndexes(String sVectorfiled,List allIndexParams) { + + // 1.1 向量索引 + Map vectorExtraParams = new HashMap<>(8); + vectorExtraParams.put("nlist", 256); // 聚类中心数：sqrt(384) * 13 ≈ 256 + vectorExtraParams.put("nprobe", 32); // 搜索时检查的聚类数 + + IndexParam vectorIndex = IndexParam.builder() + .fieldName("vector") + .indexName("idx_vector_rebuild") + .indexType(IndexParam.IndexType.IVF_FLAT) + .metricType(IndexParam.MetricType.IP) + .extraParams(vectorExtraParams) + .build(); + allIndexParams.add(vectorIndex); + + // 1.2 create_time 字段索引（用于时间范围查询） + IndexParam timeIndex = IndexParam.builder() + .fieldName("create_time") + .indexName("idx_create_time") + .indexType(IndexParam.IndexType.STL_SORT) // 排序索引 + .build(); + allIndexParams.add(timeIndex); + + // 1.4 data_id 字段索引（用于精确匹配） + IndexParam idIndex = IndexParam.builder() + .fieldName("sSlaveId") + .indexName("idx_data_id") + .indexType(IndexParam.IndexType.TRIE) + .build(); + allIndexParams.add(idIndex); + //动态字段创建 + String[] sVectorfiledArray = sVectorfiled.split(","); + for(String sVectorfiledOne : sVectorfiledArray){ + String[] sVectorfiledOneArray = sVectorfiledOne.split(":"); + String sName = sVectorfiledOneArray[1]; + IndexParam.IndexType indexType =indexField(sVectorfiledOneArray[1]); + allIndexParams.add(IndexParam.builder() + .fieldName(sName) + .indexName(sName) + .indexType(indexType) + .build()); + } + + } + + /*** + * @Author 钱豹 + * @Date 21:10 2026/3/24 + * @Param + * @return + * @Description //TODO + **/ + public Object getDefaultData(String sKey) { + if(sKey.startsWith("d") || sKey.startsWith("i")){ + return BigDecimal.ZERO; + }else if(sKey.startsWith("b")){ + return false; + }else{ + return StrUtil.EMPTY; + } + } + /*** + * @Author 钱豹 + * @Date 20:44 2026/3/24 + * @Param [sKey] + * @return io.milvus.v2.common.DataType + * @Description 字段类型 + **/ + public DataType processField(String sKey) { + if(sKey.startsWith("d")){ + return DataType.Double; + }else if(sKey.startsWith("i")){ + return DataType.Int64; + }else if(sKey.startsWith("b")){ + return DataType.Bool; + }else{ + return DataType.VarChar; + } + } + + /*** + * @Author 钱豹 + * @Date 20:44 2026/3/24 + * @Param [sKey] + * @return io.milvus.v2.common.DataType + * @Description 索引类型 + **/ + public IndexParam.IndexType indexField(String sKey) { + return IndexParam.IndexType.TRIE; + } + + + + + +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/service/impl/VectorizationServiceImpl.java b/src/main/java/com/xly/milvus/service/impl/VectorizationServiceImpl.java new file mode 100644 index 0000000..5037b76 --- /dev/null +++ b/src/main/java/com/xly/milvus/service/impl/VectorizationServiceImpl.java @@ -0,0 +1,62 @@ +package com.xly.milvus.service.impl; + +import com.xly.milvus.service.VectorizationService; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 向量化服务实现 - 使用LangChain4j的All-MiniLM-L6-v2模型 + */ +@Slf4j +@Service +public class VectorizationServiceImpl implements VectorizationService { + + private final EmbeddingModel embeddingModel; + + public VectorizationServiceImpl() { + // 初始化嵌入模型 + this.embeddingModel = new AllMiniLmL6V2EmbeddingModel(); + log.info("向量化模型初始化成功"); + } + + @Override + public List textToVector(String text) { + if (text == null || text.trim().isEmpty()) { + return new ArrayList<>(); + } + + try { + // 使用LangChain4j生成向量 + dev.langchain4j.data.embedding.Embedding embedding = embeddingModel.embed(text).content(); + float[] vectorArray = embedding.vector(); + + // 转换为List + List vector = new ArrayList<>(); + for (float v : vectorArray) { + vector.add(v); + } + + return vector; + } catch (Exception e) { + log.error("文本向量化失败: {}", e.getMessage(), e); + throw new RuntimeException("文本向量化失败", e); + } + } + + @Override + public List> batchTextToVector(List texts) { + if (texts == null || texts.isEmpty()) { + return new ArrayList<>(); + } + + return texts.stream() + .map(this::textToVector) + .collect(Collectors.toList()); + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/test/MilvusCompleteTest.java b/src/main/java/com/xly/milvus/test/MilvusCompleteTest.java new file mode 100644 index 0000000..4e4b632 --- /dev/null +++ b/src/main/java/com/xly/milvus/test/MilvusCompleteTest.java @@ -0,0 +1,536 @@ +package com.xly.milvus.test; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.milvus.v2.client.ConnectConfig; +import io.milvus.v2.client.MilvusClientV2; +import io.milvus.v2.common.DataType; +import io.milvus.v2.common.IndexParam; +import io.milvus.v2.common.ConsistencyLevel; +import io.milvus.v2.service.collection.request.*; +import io.milvus.v2.service.collection.response.GetCollectionStatsResp; +import io.milvus.v2.service.index.request.CreateIndexReq; +import io.milvus.v2.service.vector.request.InsertReq; +import io.milvus.v2.service.vector.request.SearchReq; +import io.milvus.v2.service.vector.request.QueryReq; +import io.milvus.v2.service.vector.request.data.FloatVec; +import io.milvus.v2.service.vector.response.InsertResp; +import io.milvus.v2.service.vector.response.SearchResp; +import io.milvus.v2.service.vector.response.QueryResp; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Milvus Java SDK 2.6.15 完整测试代码 + * + * 功能： + * 1. 连接Milvus服务器 + * 2. 创建集合（包含多种字段类型） + * 3. 创建向量索引 + * 4. 插入测试数据 + * 5. 查询数据（标量过滤） + * 6. 向量相似性搜索 + * 7. 获取集合统计信息 + * 8. 清理资源 + * + * @author xly + * @version 1.0 + */ +public class MilvusCompleteTest { + + // ==================== 配置参数 ==================== + private static final String MILVUS_HOST = "121.43.128.225"; // 您的Milvus服务器IP + private static final int MILVUS_PORT = 19530; // Milvus服务端口 + private static final String COLLECTION_NAME = "complete_test"; // 集合名称 + private static final int VECTOR_DIM = 128; // 向量维度 + private static final int INSERT_COUNT = 15; // 插入数据条数 + + private MilvusClientV2 client; + + public static void main(String[] args) { + MilvusCompleteTest test = new MilvusCompleteTest(); + try { + // 执行测试流程 + test.run(); + } catch (Exception e) { + System.err.println("❌ 测试过程中发生错误: " + e.getMessage()); + e.printStackTrace(); + } + } + + /** + * 执行完整的测试流程 + */ + private void run() { + printHeader("Milvus Java SDK 2.6.15 完整测试"); + + // 1. 连接Milvus + connect(); + + try { + // 2. 清理旧集合 + cleanup(); + + // 3. 创建集合 + createCollection(); + + // 4. 创建索引 + createIndex(); + + // 5. 插入数据 + insertData(); + + // 6. 查询数据 + queryData(); + + // 7. 向量搜索 + searchData(); + + // 8. 获取集合统计 + getCollectionStats(); + + printSuccess("所有测试完成！"); + + } catch (Exception e) { + System.err.println("❌ 测试失败: " + e.getMessage()); + throw e; + } finally { + // 9. 清理资源 + releaseAndClose(); + } + } + + /** + * 连接Milvus服务器 + */ + /** + * 连接Milvus服务器 + */ + private void connect() { + printSection("连接Milvus服务器"); + + try { + ConnectConfig connectConfig = ConnectConfig.builder() + .uri("http://" + MILVUS_HOST + ":" + MILVUS_PORT) + .connectTimeoutMs(TimeUnit.SECONDS.toMillis(30)) // 连接超时30秒 + .keepAliveTimeMs(TimeUnit.MINUTES.toMillis(5)) // 保持连接5分钟 + .keepAliveTimeoutMs(TimeUnit.SECONDS.toMillis(5)) // keep-alive超时5秒 + .keepAliveWithoutCalls(true) // 无调用时保持连接 + .rpcDeadlineMs(TimeUnit.SECONDS.toMillis(10)) // RPC超时10秒 + .enablePrecheck(true) // 启用预检查 + .build(); + + client = new MilvusClientV2(connectConfig); + + // 验证连接 + String serverVersion = client.getServerVersion(); + System.out.println("✅ 连接成功！"); + System.out.println(" - 服务器地址: " + MILVUS_HOST + ":" + MILVUS_PORT); + System.out.println(" - 服务器版本: " + serverVersion); + + } catch (Exception e) { + System.err.println("❌ 连接失败: " + e.getMessage()); + throw new RuntimeException("无法连接到Milvus服务器", e); + } + } + + /** + * 清理已存在的集合 + */ + private void cleanup() { + printSection("清理环境"); + + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() + .collectionName(COLLECTION_NAME) + .build(); + + boolean exists = client.hasCollection(hasCollectionReq); + if (exists) { + DropCollectionReq dropCollectionReq = DropCollectionReq.builder() + .collectionName(COLLECTION_NAME) + .build(); + client.dropCollection(dropCollectionReq); + System.out.println("✅ 已删除旧集合: " + COLLECTION_NAME); + } else { + System.out.println("⏭️ 无需清理，集合不存在"); + } + } + + /** + * 创建集合 + */ + private void createCollection() { + printSection("创建集合"); + + // 定义字段列表 + List fieldSchemas = Arrays.asList( + // 1. 主键字段 + CreateCollectionReq.FieldSchema.builder() + .name("id") + .dataType(DataType.Int64) + .isPrimaryKey(true) + .autoID(false) + .description("主键ID") + .build(), + + // 2. 向量字段 + CreateCollectionReq.FieldSchema.builder() + .name("vector") + .dataType(DataType.FloatVector) + .dimension(VECTOR_DIM) + .description("向量字段，用于相似性搜索") + .build(), + + // 3. 标题字段 + CreateCollectionReq.FieldSchema.builder() + .name("title") + .dataType(DataType.VarChar) + .maxLength(200) + .description("标题") + .build(), + + // 4. 内容字段 + CreateCollectionReq.FieldSchema.builder() + .name("content") + .dataType(DataType.VarChar) + .maxLength(1000) + .description("详细内容") + .build(), + + // 5. 分类字段 + CreateCollectionReq.FieldSchema.builder() + .name("category") + .dataType(DataType.VarChar) + .maxLength(50) + .description("分类标签") + .build(), + + // 6. 得分字段 + CreateCollectionReq.FieldSchema.builder() + .name("score") + .dataType(DataType.Float) + .description("评分") + .build(), + + // 7. 时间戳字段 + CreateCollectionReq.FieldSchema.builder() + .name("create_time") + .dataType(DataType.Int64) + .description("创建时间戳") + .build(), + + // 8. 标签数组字段 + CreateCollectionReq.FieldSchema.builder() + .name("tags") + .dataType(DataType.Array) + .elementType(DataType.VarChar) + .maxCapacity(10) + .description("标签数组") + .build() + ); + + // 创建集合schema + CreateCollectionReq.CollectionSchema schema = + CreateCollectionReq.CollectionSchema.builder() + .fieldSchemaList(fieldSchemas) + .enableDynamicField(true) // 启用动态字段 + .build(); + + // 创建集合请求 + CreateCollectionReq createCollectionReq = CreateCollectionReq.builder() + .collectionName(COLLECTION_NAME) + .collectionSchema(schema) + .consistencyLevel(ConsistencyLevel.BOUNDED) // 有界一致性 + .build(); + + // 执行创建 + client.createCollection(createCollectionReq); + System.out.println("✅ 集合创建成功: " + COLLECTION_NAME); + System.out.println(" - 向量维度: " + VECTOR_DIM); + System.out.println(" - 字段数量: " + fieldSchemas.size()); + } + + /** + * 创建索引 + */ + /** + * 创建索引 + */ + private void createIndex() { + printSection("创建索引"); + + // 创建索引参数 + Map extraParams = new HashMap<>(); + extraParams.put("nlist", 1024); + + IndexParam indexParam = IndexParam.builder() + .fieldName("vector") + .indexType(IndexParam.IndexType.IVF_FLAT) + .metricType(IndexParam.MetricType.L2) + .extraParams(extraParams) + .build(); + + // 创建索引请求 + CreateIndexReq createIndexReq = CreateIndexReq.builder() + .collectionName(COLLECTION_NAME) + .indexParams(Collections.singletonList(indexParam)) + .build(); + + // 执行创建 + client.createIndex(createIndexReq); + System.out.println("✅ 索引创建成功"); + System.out.println(" - 索引类型: IVF_FLAT"); + System.out.println(" - 度量方式: L2 (欧氏距离)"); + + // 修正：使用 LoadCollectionReq 加载集合到内存 + LoadCollectionReq loadCollectionReq = LoadCollectionReq.builder() + .collectionName(COLLECTION_NAME) + .build(); + client.loadCollection(loadCollectionReq); + System.out.println("✅ 集合已加载到内存"); + } + + /** + * 插入测试数据 + */ + private void insertData() { + printSection("插入测试数据"); + + List rows = new ArrayList<>(); + Random random = new Random(42); // 固定种子，保证可重复性 + + // 测试数据 + String[] categories = {"科技", "体育", "娱乐", "教育", "财经"}; + String[] titles = { + "人工智能发展现状与未来趋势", "机器学习入门实战教程", + "深度学习在图像识别中的应用", "NBA季后赛精彩回顾", + "世界杯预选赛最新战况", "奥运会筹备工作进展", + "热门电影推荐排行榜", "音乐榜单TOP10", + "综艺节目收视率分析", "在线教育行业发展报告", + "编程语言流行趋势", "高效学习方法分享", + "股市投资策略分析", "区块链技术应用前景", + "数字货币市场动态" + }; + + long currentTime = System.currentTimeMillis() / 1000; + + // 生成测试数据 + for (int i = 1; i <= INSERT_COUNT; i++) { + JsonObject row = new JsonObject(); + + // ID + row.addProperty("id", (long) i); + + // 向量 + JsonArray vectorArray = new JsonArray(); + for (int j = 0; j < VECTOR_DIM; j++) { + vectorArray.add(random.nextFloat()); + } + row.add("vector", vectorArray); + + // 标题 + int index = (i - 1) % titles.length; + row.addProperty("title", titles[index]); + + // 内容 + row.addProperty("content", "这是第" + i + "条测试数据的详细内容。用于测试Milvus的插入和查询功能。"); + + // 分类 + row.addProperty("category", categories[random.nextInt(categories.length)]); + + // 得分 + row.addProperty("score", random.nextFloat() * 100); + + // 创建时间 + row.addProperty("create_time", currentTime - random.nextInt(86400) * i); + + // 标签数组 + JsonArray tagsArray = new JsonArray(); + tagsArray.add("tag_" + random.nextInt(5)); + tagsArray.add("tag_" + random.nextInt(5)); + row.add("tags", tagsArray); + + rows.add(row); + } + + // 批量插入 + InsertReq insertReq = InsertReq.builder() + .collectionName(COLLECTION_NAME) + .data(rows) + .build(); + + InsertResp insertResp = client.insert(insertReq); + System.out.println("✅ 成功插入 " + insertResp.getInsertCnt() + " 条数据"); + System.out.println(" - 数据预览:"); + + // 显示前3条数据的部分信息 + for (int i = 0; i < Math.min(3, rows.size()); i++) { + JsonObject row = rows.get(i); + System.out.printf(" [%d] ID: %d, 标题: %s, 分类: %s, 得分: %.2f%n", + i + 1, + row.get("id").getAsLong(), + row.get("title").getAsString(), + row.get("category").getAsString(), + row.get("score").getAsFloat()); + } + } + + /** + * 查询数据（标量过滤） + */ + private void queryData() { + printSection("标量查询"); + + // 查询得分大于50的数据 + QueryReq queryReq = QueryReq.builder() + .collectionName(COLLECTION_NAME) + .filter("score > 50") // 过滤条件 + .outputFields(Arrays.asList("id", "title", "category", "score")) + .limit(10) + .build(); + + QueryResp queryResp = client.query(queryReq); + + List results = queryResp.getQueryResults(); + System.out.println("查询条件: score > 50"); + System.out.println("查询结果: " + results.size() + " 条数据"); + + if (!results.isEmpty()) { + System.out.println("结果列表:"); + for (QueryResp.QueryResult result : results) { + Map entity = result.getEntity(); + System.out.printf(" ID: %d | 标题: %s | 分类: %s | 得分: %.2f%n", + (Long) entity.get("id"), + entity.get("title"), + entity.get("category"), + (Float) entity.get("score")); + } + } + } + + /** + * 向量相似性搜索 + */ + private void searchData() { + printSection("向量相似性搜索"); + + // 创建查询向量 + List vectorList = new ArrayList<>(); + Random random = new Random(); + for (int j = 0; j < VECTOR_DIM; j++) { + vectorList.add(random.nextFloat()); + } + FloatVec queryVector = new FloatVec(vectorList); + + // 构建搜索参数 + Map searchParams = new HashMap<>(); + searchParams.put("nprobe", 10); + + // 构建搜索请求 + SearchReq searchReq = SearchReq.builder() + .collectionName(COLLECTION_NAME) + .data(Collections.singletonList(queryVector)) + .annsField("vector") + .limit(5) + .outputFields(Arrays.asList("title", "category", "score", "tags")) + .metricType(IndexParam.MetricType.L2) + .searchParams(searchParams) + .build(); + + // 执行搜索 + SearchResp searchResp = client.search(searchReq); + + // 打印结果 + System.out.println("最相似的5条结果:"); + List> results = searchResp.getSearchResults(); + + if (!results.isEmpty() && !results.get(0).isEmpty()) { + for (int i = 0; i < results.get(0).size(); i++) { + SearchResp.SearchResult result = results.get(0).get(i); + Map entity = result.getEntity(); + + System.out.printf(" %d. ID: %d | 距离: %.4f%n", + i + 1, + (Long) result.getId(), + result.getScore()); + System.out.printf(" 标题: %s | 分类: %s | 得分: %.2f%n", + entity.get("title"), + entity.get("category"), + (Float) entity.get("score")); + System.out.printf(" 标签: %s%n", + entity.get("tags")); + } + } else { + System.out.println(" 没有找到结果"); + } + } + + /** + * 获取集合统计信息 + */ + private void getCollectionStats() { + printSection("集合统计信息"); + + GetCollectionStatsReq statsReq = GetCollectionStatsReq.builder() + .collectionName(COLLECTION_NAME) + .build(); + + GetCollectionStatsResp statsResp = client.getCollectionStats(statsReq); + + System.out.println("集合名称: " + COLLECTION_NAME); + System.out.println("实体数量: " + statsResp.getNumOfEntities()); + } + + /** + * 释放集合并关闭连接 + */ + private void releaseAndClose() { + printSection("清理资源"); + + if (client != null) { + try { + // 释放集合 + ReleaseCollectionReq releaseCollectionReq = ReleaseCollectionReq.builder() + .collectionName(COLLECTION_NAME) + .build(); + client.releaseCollection(releaseCollectionReq); + System.out.println("✅ 集合已释放"); + + // 关闭连接 + client.close(); + System.out.println("✅ 连接已关闭"); + + } catch (Exception e) { + System.err.println("⚠️ 清理资源时出错: " + e.getMessage()); + } + } + } + + /** + * 打印标题 + */ + private void printHeader(String title) { + System.out.println("\n" + "=".repeat(60)); + System.out.println(" " + title); + System.out.println("=".repeat(60)); + } + + /** + * 打印章节标题 + */ + private void printSection(String section) { + System.out.println("\n" + "-".repeat(40)); + System.out.println(" " + section); + System.out.println("-".repeat(40)); + } + + /** + * 打印成功信息 + */ + private void printSuccess(String message) { + System.out.println("\n" + "=".repeat(60)); + System.out.println(" ✅ " + message); + System.out.println("=".repeat(60)); + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/util/MapToJsonConverter.java b/src/main/java/com/xly/milvus/util/MapToJsonConverter.java new file mode 100644 index 0000000..61f1924 --- /dev/null +++ b/src/main/java/com/xly/milvus/util/MapToJsonConverter.java @@ -0,0 +1,117 @@ +package com.xly.milvus.util; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.util.List; +import java.util.Map; + +/** + * Map 转 JsonObject 工具类 + */ +public class MapToJsonConverter { + + /** + * 将 Map 转换为 JsonObject + */ + public static JsonObject convert(Map map) { + JsonObject jsonObject = new JsonObject(); + + if (map == null || map.isEmpty()) { + return jsonObject; + } + + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + + addToJson(jsonObject, key, value); + } + + return jsonObject; + } + + /** + * 递归添加值到JsonObject + */ + @SuppressWarnings("unchecked") + private static void addToJson(JsonObject jsonObject, String key, Object value) { + if (value == null) { + jsonObject.add(key, null); + } else if (value instanceof String) { + jsonObject.addProperty(key, (String) value); + } else if (value instanceof Number) { + jsonObject.addProperty(key, (Number) value); + } else if (value instanceof Boolean) { + jsonObject.addProperty(key, (Boolean) value); + } else if (value instanceof Character) { + jsonObject.addProperty(key, (Character) value); + } else if (value instanceof Map) { + jsonObject.add(key, convert((Map) value)); + } else if (value instanceof List) { + jsonObject.add(key, convertList((List) value)); + } else if (value instanceof Object[]) { + jsonObject.add(key, convertArray((Object[]) value)); + } else { + // 其他类型转换为字符串 + jsonObject.addProperty(key, value.toString()); + } + } + + /** + * 转换List为JsonArray + */ + private static JsonArray convertList(List list) { + JsonArray jsonArray = new JsonArray(); + + for (Object item : list) { + if (item == null) { + jsonArray.add((JsonObject) null); + } else if (item instanceof String) { + jsonArray.add((String) item); + } else if (item instanceof Number) { + jsonArray.add((Number) item); + } else if (item instanceof Boolean) { + jsonArray.add((Boolean) item); + } else if (item instanceof Character) { + jsonArray.add((Character) item); + } else if (item instanceof Map) { + jsonArray.add(convert((Map) item)); + } else if (item instanceof List) { + jsonArray.add(convertList((List) item)); + } else { + jsonArray.add(item.toString()); + } + } + + return jsonArray; + } + + /** + * 转换数组为JsonArray + */ + private static JsonArray convertArray(Object[] array) { + JsonArray jsonArray = new JsonArray(); + + for (Object item : array) { + if (item == null) { + jsonArray.add((JsonObject) null); + } else if (item instanceof String) { + jsonArray.add((String) item); + } else if (item instanceof Number) { + jsonArray.add((Number) item); + } else if (item instanceof Boolean) { + jsonArray.add((Boolean) item); + } else if (item instanceof Character) { + jsonArray.add((Character) item); + } else if (item instanceof Map) { + jsonArray.add(convert((Map) item)); + } else if (item instanceof List) { + jsonArray.add(convertList((List) item)); + } else { + jsonArray.add(item.toString()); + } + } + + return jsonArray; + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/util/MilvusSchemaBuilder.java b/src/main/java/com/xly/milvus/util/MilvusSchemaBuilder.java new file mode 100644 index 0000000..51ca4e2 --- /dev/null +++ b/src/main/java/com/xly/milvus/util/MilvusSchemaBuilder.java @@ -0,0 +1,214 @@ +package com.xly.milvus.util; + +import io.milvus.v2.common.DataType; +import io.milvus.v2.service.collection.request.CreateCollectionReq; +import java.util.HashMap; +import java.util.Map; + +/** + * Milvus字段Schema构建工具类 + * 基于源码的 FieldSchema.FieldSchemaBuilder + */ +public class MilvusSchemaBuilder { + + /** + * 创建主键ID字段（Int64类型，自增） + */ + public static CreateCollectionReq.FieldSchema createAutoIdField(String fieldName) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.Int64) + .isPrimaryKey(true) + .autoID(true) + .description("自增主键ID") + .build(); + } + + /** + * 创建主键ID字段（String类型，不自增） + */ + public static CreateCollectionReq.FieldSchema createStringIdField(String fieldName, int maxLength) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.VarChar) + .maxLength(maxLength) + .isPrimaryKey(true) + .autoID(false) + .description("业务主键ID") + .build(); + } + + /** + * 创建向量字段 + */ + public static CreateCollectionReq.FieldSchema createVectorField(String fieldName, int dimension) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.FloatVector) + .dimension(dimension) + .description("向量字段，维度: " + dimension) + .build(); + } + + /** + * 创建文本字段（VarChar类型） + */ + public static CreateCollectionReq.FieldSchema createTextField(String fieldName, int maxLength) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.VarChar) + .maxLength(maxLength) + .description("文本内容") + .build(); + } + + /** + * 创建JSON元数据字段 + */ + public static CreateCollectionReq.FieldSchema createJsonField(String fieldName) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.JSON) + .description("JSON元数据") + .build(); + } + + /** + * 创建整型字段 + */ + public static CreateCollectionReq.FieldSchema createIntField(String fieldName, String description) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.Int64) + .description(description) + .build(); + } + + /** + * 创建整型字段（可空） + */ + public static CreateCollectionReq.FieldSchema createIntField(String fieldName, + String description, + boolean isNullable) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.Int64) + .description(description) + .isNullable(isNullable) + .build(); + } + + /** + * 创建整型字段（带默认值） + */ + public static CreateCollectionReq.FieldSchema createIntField(String fieldName, + String description, + long defaultValue) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.Int64) + .description(description) + .defaultValue(defaultValue) + .build(); + } + + /** + * 创建浮点型字段 + */ + public static CreateCollectionReq.FieldSchema createFloatField(String fieldName, String description) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.Float) + .description(description) + .build(); + } + + /** + * 创建布尔型字段 + */ + public static CreateCollectionReq.FieldSchema createBoolField(String fieldName, String description) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.Bool) + .description(description) + .build(); + } + + /** + * 创建布尔型字段（带默认值） + */ + public static CreateCollectionReq.FieldSchema createBoolField(String fieldName, + String description, + boolean defaultValue) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.Bool) + .description(description) + .defaultValue(defaultValue) + .build(); + } + + /** + * 创建数组字段 + */ + public static CreateCollectionReq.FieldSchema createArrayField(String fieldName, + DataType elementType, + int maxCapacity, + String description) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.Array) + .elementType(elementType) + .maxCapacity(maxCapacity) + .description(description) + .build(); + } + + /** + * 创建分区键字段 + */ + public static CreateCollectionReq.FieldSchema createPartitionKeyField(String fieldName, + DataType dataType, + String description) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(dataType) + .isPartitionKey(true) + .description(description) + .build(); + } + + /** + * 创建聚类键字段 + */ + public static CreateCollectionReq.FieldSchema createClusteringKeyField(String fieldName, + DataType dataType, + String description) { + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(dataType) + .isClusteringKey(true) + .description(description) + .build(); + } + + /** + * 创建带分析器的文本字段（用于全文搜索） + */ + public static CreateCollectionReq.FieldSchema createAnalyzedTextField(String fieldName, + int maxLength, + String analyzerType) { + Map analyzerParams = new HashMap<>(); + analyzerParams.put("type", analyzerType); // "english", "chinese", "standard" 等 + + return CreateCollectionReq.FieldSchema.builder() + .name(fieldName) + .dataType(DataType.VarChar) + .maxLength(maxLength) + .enableAnalyzer(true) + .analyzerParams(analyzerParams) + .enableMatch(true) + .description("支持全文搜索的文本字段") + .build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/milvus/web/MilvusController.java b/src/main/java/com/xly/milvus/web/MilvusController.java new file mode 100644 index 0000000..80ff82e --- /dev/null +++ b/src/main/java/com/xly/milvus/web/MilvusController.java @@ -0,0 +1,49 @@ +package com.xly.milvus.web; + +import com.xly.milvus.service.MilvusService; +import com.xly.runner.AppStartupRunner; +import com.xly.service.DynamicExeDbService; +import com.xly.service.UserSceneSessionService; +import com.xly.tool.DynamicToolProvider; +import com.xly.tts.bean.*; +import com.xly.tts.service.LocalAudioCache; +import com.xly.tts.service.PythonTtsProxyService; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +@Slf4j +@RestController +@RequestMapping("/api/milvus") +@RequiredArgsConstructor +public class MilvusController { + + private final MilvusService milvusService; + + /*** + * @Author 钱豹 + * @Date 14:32 2026/2/10 + * @Param [request] + * @return org.springframework.http.ResponseEntity + * @Description 初始化AI所有变量 热启动 + **/ + @PostMapping("/init") + public ResponseEntity init(@RequestBody Map reqMap) { + TTSResponseDTO responseDTO = milvusService.initDataToMilvus(reqMap); + return ResponseEntity.ok(responseDTO); + } + + +} \ No newline at end of file diff --git a/src/main/java/com/xly/service/XlyErpService.java b/src/main/java/com/xly/service/XlyErpService.java index 30cb1b3..4fd724a 100644 --- a/src/main/java/com/xly/service/XlyErpService.java +++ b/src/main/java/com/xly/service/XlyErpService.java @@ -7,6 +7,7 @@ import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; import com.xly.agent.ChatiAgent; import com.xly.agent.DynamicTableNl2SqlAiAgent; import com.xly.agent.ErpAiAgent; @@ -17,15 +18,14 @@ import com.xly.constant.ReturnTypeCode; import com.xly.entity.*; import com.xly.exception.sqlexception.SqlGenerateException; import com.xly.exception.sqlexception.SqlValidateException; +import com.xly.milvus.service.AiGlobalAgentQuestionSqlEmitterService; +import com.xly.milvus.service.MilvusService; import com.xly.runner.AppStartupRunner; import com.xly.thread.AiSqlErrorHistoryThread; import com.xly.thread.AiUserAgentQuestionThread; import com.xly.thread.MultiThreadPoolServer; import com.xly.tool.DynamicToolProvider; -import com.xly.util.EnhancedErrorGuidance; -import com.xly.util.InputPreprocessor; -import com.xly.util.SqlValidateUtil; -import com.xly.util.ValiDataUtil; +import com.xly.util.*; import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; @@ -33,15 +33,22 @@ import dev.langchain4j.data.message.ChatMessageType; import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.ollama.OllamaChatModel; import dev.langchain4j.service.AiServices; +import dev.langchain4j.service.MemoryId; +import dev.langchain4j.service.V; +import io.milvus.v2.common.DataType; +import io.milvus.v2.service.collection.request.CreateCollectionReq; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.time.DateFormatUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.IdGenerator; +import reactor.core.publisher.Flux; import java.time.Duration; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.IntStream; @Service @RequiredArgsConstructor @@ -56,6 +63,8 @@ public class XlyErpService { private final OperableChatMemoryProvider operableChatMemoryProvider; private final DynamicExeDbService dynamicExeDbService; private final RedisService redisService; + private final AiGlobalAgentQuestionSqlEmitterService aiGlobalAgentQuestionSqlEmitterService; + private final MilvusService milvusService; //执行动态语句 执行异常的情况下 最多执行次数 private final Integer maxRetries = 5; @@ -67,6 +76,51 @@ public class XlyErpService { @Value("${langchain4j.ollama.sql-model-name}") private String sqlModelName; + + + /** + * 新的流式方法 - 返回 Flux + * 每个AiResponseDTO包含一个文本片段 + */ + /** + * 模拟的erpUserInputStream实现 + */ + public Flux erpUserInputStream(String userInput, String sUserId, + String sUserName, String sBrandsId, + String sSubsidiaryId, String sUserType, + String authorization) { + String requestId = UUID.randomUUID().toString(); + + // 按句子分割 + String[] sentences = userInput.split("(?<=[。！？.!?])"); + int totalChunks = sentences.length; + + return Flux.range(0, totalChunks) + .delayElements(Duration.ofMillis(200)) + .map(i -> { + String sentence = sentences[i].trim(); + if (sentence.isEmpty()) return null; + + return AiResponseDTO.builder() + .requestId(requestId) + .code(200) + .message("ERP_CHUNK") + .status("PROCESSING") + .textFragment(sentence) + .chunkIndex(i) + .totalChunks(totalChunks) + .isLastChunk(i == totalChunks - 1) + .progress((i + 1) * 100 / totalChunks) + .timestamp(System.currentTimeMillis()) + .sSceneName("客服咨询") + .sMethodName("chat") + .sReturnType("MARKDOWN") + .build(); + }) + .filter(Objects::nonNull); + } + + /*** * @Author 钱豹 * @Date 19:18 2026/1/27 @@ -116,8 +170,15 @@ public class XlyErpService { if (aiAgent == null){ return getChatiAgent (input,session); } - //用户输入添加方法 - String sResponMessage = aiAgent.chat(userId, input); + String sResponMessage = StrUtil.EMPTY; + //用户输入添加方法(如果没有方法，动态SQL方法不需要) + if(!(ObjectUtil.isNotEmpty(session.getCurrentTool()) + && ObjectUtil.isNotEmpty(session.getCurrentTool().getSInputTabelName()) + && ObjectUtil.isNotEmpty(session.getCurrentTool().getSStructureMemo())) + ){ + sResponMessage = aiAgent.chat(userId, input); + } + if(ObjectUtil.isNotEmpty(session.getCurrentTool()) && !ObjectUtil.isNotEmpty(session.getCurrentTool().getSInputTabelName()) ){ @@ -132,7 +193,14 @@ public class XlyErpService { && ObjectUtil.isNotEmpty(session.getCurrentTool().getSInputTabelName()) && ObjectUtil.isNotEmpty(session.getCurrentTool().getSStructureMemo())) ){ - sResponMessage = getDynamicTableSql(session, input, userId, userInput,0,StrUtil.EMPTY,StrUtil.EMPTY,"0",StrUtil.EMPTY, aiAgent); + //查询是否走向量库 还是数据库查询 + Boolean isAggregation = aiAgent.routeQuery(session.getUserId(), input); + if(!isAggregation){ + //获取常量库内容 + sResponMessage = getMilvus(session, input, aiAgent); + }else { + sResponMessage = getDynamicTableSql(session, input, userId, userInput,0,StrUtil.EMPTY,StrUtil.EMPTY,"0",StrUtil.EMPTY, aiAgent); + } return AiResponseDTO.builder().sSceneName(sceneName).sMethodName(methodName).aiText(sResponMessage).sReturnType(ReturnTypeCode.HTML.getCode()).build(); } else if (ObjectUtil.isNotEmpty(session.getCurrentTool())) { //2.处理工具参数采集结束后业务逻辑处理 @@ -198,6 +266,75 @@ public class XlyErpService { return AiResponseDTO.builder().sSceneName(sceneName).sMethodName(StrUtil.EMPTY).aiText(StrUtil.EMPTY).systemText("清除记忆成功！").sReturnType(ReturnTypeCode.HTML.getCode()).build(); } + /*** + * @Author 钱豹 + * @Date 10:16 2026/3/25 + * @Param [session, input, userId, userInput, attempt, errorSql, errorMessage, iErroCount, historySqlList, aiAgent] + * @return java.lang.String + * @Description 查询向量库 + **/ + private String getMilvus(UserSceneSession session,String userInput,ErpAiAgent aiAgent){ + String resultExplain = "信息模糊,请提供更具体的问题或指令"; + try{ + String sVectorfiled = session.getCurrentTool().getSVectorfiled(); + String sInputTabelName = session.getCurrentTool().getSInputTabelName(); + Map rMap = milvusService.getMilvusFiled(sVectorfiled); + String sMilvusFiled = rMap.get("sMilvusFiled").toString(); + String sMilvusFiledDescription = rMap.get("sMilvusFiledDescription").toString(); + List fields = (List) rMap.get("sFileds"); +// List> title = (List>) rMap.get("title"); + String milvusFilter = aiAgent.getMilvusFilter(session.getUserId(),userInput, sMilvusFiled, sMilvusFiledDescription); + List> data = milvusService.getDataToCollection(sInputTabelName, milvusFilter,userInput,100,fields); + //采用表格形式显示 + resultExplain = aiAgent.explainMilvusResult(session.getUserId(),userInput,sMilvusFiledDescription,JSONObject.toJSONString(data)); + //buildMarkdownTableWithStream(data, title); + return resultExplain; + }catch (Exception e){ + e.printStackTrace(); + } + return resultExplain; + } + /*** + * @Author 钱豹 + * @Date 13:19 2026/3/25 + * @Param [data, title] + * @return java.lang.String + * @Description 数据转成MarkdownTable + **/ + public String buildMarkdownTableWithStream(List> data, List> title) { + if (data == null || data.isEmpty()) { + return "暂无数据"; + } + + // 动态构建表头 + StringBuilder headerBuilder = new StringBuilder("|"); + StringBuilder separatorBuilder = new StringBuilder("|"); + + for (Map column : title) { + String displayName = column.get("sTitle"); // 中文显示名称 + headerBuilder.append(" ").append(displayName).append(" |"); + separatorBuilder.append("------|"); + } + String header = headerBuilder.toString() + "\n" + separatorBuilder.toString() + "\n"; + // 构建数据行 + String rows = IntStream.range(0, data.size()) + .mapToObj(i -> { + Map item = data.get(i); + StringBuilder rowBuilder = new StringBuilder("|"); + + for (Map column : title) { + String fieldName = column.get("sName"); + Object value = item.getOrDefault(fieldName, ""); + rowBuilder.append(" ").append(value).append(" |"); + } + + return rowBuilder.toString(); + }) + .collect(Collectors.joining("\n")); + + return header + rows; + } + /*** * @Author 钱豹 @@ -283,48 +420,54 @@ public class XlyErpService { **/ private String getDynamicTableSqlExec(UserSceneSession session,String input,String userId,String userInput,String errorSql,String errorMessage,String iErroCount,String historySqlList,ErpAiAgent aiAgent){ // 1. 构建自然语言转SQLAgent， - List> sqlResult = new ArrayList<>(); + List> sqlResult; String cleanSql = StrUtil.EMPTY; String rawSql; - String tableStruct; - String sError_mes = StrUtil.EMPTY; - Boolean doSqlErro = false; - List chatMessage = operableChatMemoryProvider.getCurrentChatMessages(session.getUserId()); + String tableStruct = session.getCurrentTool().getSStructureMemo(); + String sError_mes; + Boolean doAddSql = false; + List chatMessage = new ArrayList<>(); try{ - DynamicTableNl2SqlAiAgent aiDynamicTableNl2SqlAiAgent = createDynamicTableNl2SqlAiAgent(userId, input, session); - String tableNames = session.getCurrentTool().getSInputTabelName(); - // "订单表:viw_salsalesorder,客户信息表:elecustomer,结算方式表:sispayment,产品表(无单价,无金额,无数量):viw_product_sort,销售人员表:viw_sissalesman_depart"; - tableStruct = session.getCurrentTool().getSStructureMemo(); - String sDataNow = DateUtil.format(new Date(), DatePattern.CHINESE_DATE_TIME_FORMAT); - - if(ObjectUtil.isEmpty(errorSql) && ObjectUtil.isEmpty(errorMessage)){ - rawSql = aiDynamicTableNl2SqlAiAgent.generateMysqlSql(userId,tableNames,tableStruct,sDataNow,userInput); - }else{ - rawSql = aiDynamicTableNl2SqlAiAgent.regenerateSqlWithError(userId, tableNames,tableStruct,sDataNow,userInput,errorSql,errorMessage,iErroCount,historySqlList); - } - log.info("rawSql："+rawSql); - if (rawSql == null || rawSql.trim().isEmpty()) { - throw new SqlValidateException("SQL EMPTY"); + //获取缓存动态SQL +// cleanSql = getDynamicTableNl2Sql(session,input); + //如果之前已查询直接返回 + if(ObjectUtil.isEmpty(cleanSql)){ + DynamicTableNl2SqlAiAgent aiDynamicTableNl2SqlAiAgent = createDynamicTableNl2SqlAiAgent(userId, input, session); + chatMessage = operableChatMemoryProvider.getCurrentChatMessages(session.getUserId()); + String tableNames = session.getCurrentTool().getSInputTabelName(); + // "订单表:viw_salsalesorder,客户信息表:elecustomer,结算方式表:sispayment,产品表(无单价,无金额,无数量):viw_product_sort,销售人员表:viw_sissalesman_depart"; + String sDataNow = DateUtil.now(); + //DateFormatUtils.format(new Date(), "yyyy年MM月dd日HH时mm分ss秒"); +// String sDataNow = DateUtil.format(new Date(), DatePattern.CHINESE_DATE_TIME_FORMAT); + + if(ObjectUtil.isEmpty(errorSql) && ObjectUtil.isEmpty(errorMessage)){ + rawSql = aiDynamicTableNl2SqlAiAgent.generateMysqlSql(userId,tableNames,tableStruct,sDataNow,userInput); + }else{ + rawSql = aiDynamicTableNl2SqlAiAgent.regenerateSqlWithError(userId, tableNames,tableStruct,sDataNow,userInput,errorSql,errorMessage,iErroCount,historySqlList); + } + log.info("rawSql："+rawSql); + if (rawSql == null || rawSql.trim().isEmpty()) { + throw new SqlValidateException("SQL EMPTY"); + } + // 2. 清理SQL多余符号 + 生产级强校验（核心安全保障，不可省略） + cleanSql = SqlValidateUtil.cleanSqlSymbol(rawSql); + SqlValidateUtil.validateMysqlSql(cleanSql); + doAddSql = true; } - // 2. 清理SQL多余符号 + 生产级强校验（核心安全保障，不可省略） - cleanSql = SqlValidateUtil.cleanSqlSymbol(rawSql); - // String[] cleanSqlA = rawSql.split(";"); - // if(cleanSqlA.length>1){ - // cleanSql = cleanSqlA[cleanSqlA.length-1]; - // } - SqlValidateUtil.validateMysqlSql(cleanSql); - // 4. 执行SQL获取结构化结果 - // Map params = new HashMap<>(); +// List chatMessage2 = operableChatMemoryProvider.getCurrentChatMessages(session.getUserId()); try{ sqlResult = dynamicExeDbService.findSql(new HashMap<>(),cleanSql); }catch (Exception e){ throw new SqlGenerateException(e.getMessage()+" OLDSQL "+cleanSql); } }catch (SqlValidateException e){ + //删除记录 +// operableChatMemoryProvider.deleteUserLasterMessageBySize(userId,3); sError_mes = e.getMessage(); doAiSqlErrorHistoryThread(session, StrUtil.EMPTY, cleanSql, sError_mes,input); throw e; }catch (SqlGenerateException e){ +// operableChatMemoryProvider.deleteUserLasterMessageBySize(userId,3); sError_mes = e.getMessage(); doAiSqlErrorHistoryThread(session, StrUtil.EMPTY, cleanSql, sError_mes,input); throw e; @@ -337,9 +480,11 @@ public class XlyErpService { if(Integer.valueOf(iErroCount)>0){ doAiSqlErrorHistoryThread(session, cleanSql, StrUtil.EMPTY, StrUtil.EMPTY,input); } - - //执行操作记录表 - doAiUserAgentQuestion(session,input,cleanSql,chatMessage); + //插入常用操作 + if(doAddSql){ + //执行操作记录表 + doAiUserAgentQuestion(session,input,cleanSql,chatMessage); + } String sText = aiAgent.explainSqlResult( userId, userInput, @@ -350,6 +495,28 @@ public class XlyErpService { return sText; } + /*** + * @Author 钱豹 + * @Date 17:04 2026/3/19 + * @Param [session] + * @return java.lang.String + * @Description 获取动态SQL（历史中查询） + **/ + private String getDynamicTableNl2Sql(UserSceneSession session,String input){ +// String sReidKey = SqlValidateUtil.getsKey( session.getCurrentScene().getSId(), session.getCurrentTool().getSId(), input); +// Object sSql = redisService.get(sReidKey); +// if(ObjectUtil.isNotEmpty(sSql)){ +// return sSql.toString(); +// } + String searchText = session.getCurrentScene().getSId()+"_"+session.getCurrentTool().getSId()+input; + //SqlValidateUtil.getsKey( session.getCurrentScene().getSId(), session.getCurrentTool().getSId(), SqlValidateUtil.getsQuestion(session.getSUserQuestionList())); + //根据问题查询向量库 + Map serMap = aiGlobalAgentQuestionSqlEmitterService.queryAiGlobalAgentQuestionSqlEmitter(searchText, "ai_global_agent_question_sql"); + if(ObjectUtil.isNotEmpty(serMap)){ + return serMap.get("sSqlContent").toString(); + } + return null; + } /*** * @Author 钱豹 @@ -485,7 +652,11 @@ public class XlyErpService { .build(); UserSceneSessionService.ERP_AGENT_CACHE.put(userId, aiAgent); // 初始化AiService 以防止热加载太慢 找不到相应的方法 - aiAgent.chat(userId, "initAiService"); + try{ + aiAgent.chat(userId, "initAiService"); + }catch (Exception e){ + e.printStackTrace(); + } log.info("用户{}Agent构建完成，已选场景：{},场景ID{}", userId, session.isSceneSelected() ? session.getCurrentScene().getSSceneName() : "未选（全场景匹配）", dynamicToolProvider.sSceneIdMap.get(userId)); } return aiAgent; diff --git a/src/main/java/com/xly/thread/AiUserAgentQuestionThread.java b/src/main/java/com/xly/thread/AiUserAgentQuestionThread.java index 9071995..1827efa 100644 --- a/src/main/java/com/xly/thread/AiUserAgentQuestionThread.java +++ b/src/main/java/com/xly/thread/AiUserAgentQuestionThread.java @@ -1,17 +1,18 @@ package com.xly.thread; +import cn.hutool.core.lang.generator.UUIDGenerator; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; -import com.xly.config.OperableChatMemoryProvider; import com.xly.config.SpringContextHolder; import com.xly.entity.UserSceneSession; +import com.xly.milvus.service.AiGlobalAgentQuestionSqlEmitterService; import com.xly.service.DynamicExeDbService; +import com.xly.service.RedisService; +import com.xly.util.MD5Util; +import com.xly.util.SqlValidateUtil; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.ChatMessageType; -import jnr.ffi.annotations.In; - -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,17 +37,30 @@ public class AiUserAgentQuestionThread implements Runnable { String sSceneId = session.getCurrentScene().getSId(); String sMethodId = session.getCurrentTool().getSId(); DynamicExeDbService dynamicExeDbService = SpringContextHolder.getBean(DynamicExeDbService.class); + RedisService redisService = SpringContextHolder.getBean(RedisService.class); + AiGlobalAgentQuestionSqlEmitterService aiGlobalAgentQuestionSqlEmitterService = SpringContextHolder.getBean(AiGlobalAgentQuestionSqlEmitterService.class); String sQuestionGroupNo = session.getSUserQuestionList().get(0); Integer bRedis = (session.getSUserQuestionList().size()==1)?1:0; Map data = getMap(sSceneId, sMethodId,bRedis,sQuestionGroupNo); - data.put("sQuestion",getsQuestion(session.getSUserQuestionList())); + data.put("sQuestion",SqlValidateUtil.getsQuestion(session.getSUserQuestionList())); + data.put("sId",new UUIDGenerator().next()); + //插入Redis缓存 + if(bRedis==1 && ObjectUtil.isNotEmpty(sSqlContent)){ + String sReidKey = SqlValidateUtil.getsKey( sSceneId, sMethodId, sQuestionGroupNo); + redisService.set(sReidKey,sSqlContent); + } + String sKey = sSceneId+"_"+sMethodId +"_"+sQuestion; +// SqlValidateUtil.getsKey( sSceneId, sMethodId, SqlValidateUtil.getsQuestion(session.getSUserQuestionList())); + //存入向量库 + aiGlobalAgentQuestionSqlEmitterService.addAiGlobalAgentQuestionSqlEmitter(sKey,data,sQuestion,sSqlContent,"ai_global_agent_question_sql"); + //调用数据库插入数据库 Map searMap = dynamicExeDbService.getDoProMap(sProName, data); dynamicExeDbService.getCallPro(searMap, sProName); } - private String getsQuestion(List sUserQuestionList){ - return String.join(",", sUserQuestionList); - } + + + //获取组ID private String getQuestionGroupNo(){ diff --git a/src/main/java/com/xly/token/RedisTokenManager.java b/src/main/java/com/xly/token/RedisTokenManager.java index 5bfddfc..c8b1196 100644 --- a/src/main/java/com/xly/token/RedisTokenManager.java +++ b/src/main/java/com/xly/token/RedisTokenManager.java @@ -1,4 +1,4 @@ -//package com.xly.token; +package com.xly.token;//package com.xly.token; // //import cn.hutool.core.util.ObjectUtil; //import cn.hutool.core.util.StrUtil; diff --git a/src/main/java/com/xly/tts/bean/TTSResponseDTO.java b/src/main/java/com/xly/tts/bean/TTSResponseDTO.java index 1fc8f65..cf9200c 100644 --- a/src/main/java/com/xly/tts/bean/TTSResponseDTO.java +++ b/src/main/java/com/xly/tts/bean/TTSResponseDTO.java @@ -6,6 +6,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import reactor.core.publisher.Flux; import java.io.Serializable; @@ -70,6 +71,8 @@ public class TTSResponseDTO implements Serializable { private String sReturnType = ReturnTypeCode.MAKEDOWN.getCode(); + private Boolean ErpComplete; + /** * 创建失败响应 */ diff --git a/src/main/java/com/xly/tts/service/PythonTtsProxyService.java b/src/main/java/com/xly/tts/service/PythonTtsProxyService.java index 187c78b..f30d1f3 100644 --- a/src/main/java/com/xly/tts/service/PythonTtsProxyService.java +++ b/src/main/java/com/xly/tts/service/PythonTtsProxyService.java @@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import com.xly.constant.BusinessCode; import com.xly.constant.ReturnTypeCode; +import com.xly.entity.AiResponseAccumulator; import com.xly.entity.AiResponseDTO; import com.xly.service.UserSceneSessionService; import com.xly.service.XlyErpService; @@ -16,15 +17,15 @@ import org.springframework.core.io.InputStreamResource; import org.springframework.http.*; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; +import reactor.core.publisher.Flux; import javax.annotation.PostConstruct; import java.io.*; +import java.time.Duration; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.net.URL; -import java.net.HttpURLConnection; import java.io.InputStream; @Slf4j @@ -81,6 +82,111 @@ public class PythonTtsProxyService { return synthesizeStreamAi(request, voiceText); } + /** + * 流式ERP + 流式TTS合成 + * 先流式输出ERP文本，完成后自动开始TTS合成 + * 使用现有TTSResponseDTO字段 + */ + public Flux synthesizeStreamAiStream(TTSRequestDTO request) { + String userInput = request.getText(); + String sUserId = request.getUserid(); + String sUserName = request.getUsername(); + String sBrandsId = request.getBrandsid(); + String sSubsidiaryId = request.getSubsidiaryid(); + String sUserType = request.getUsertype(); + String authorization = request.getAuthorization(); + + String requestId = UUID.randomUUID().toString(); + log.info("开始流式处理: requestId={}, userId={}", requestId, sUserId); + + // 创建累积器（用于累积完整的AiResponseDTO） + AiResponseAccumulator accumulator = new AiResponseAccumulator(requestId); + + // 1. 处理ERP流，将AiResponseDTO转换为TTSResponseDTO + Flux erpStream = xlyErpService.erpUserInputStream( + userInput, sUserId, sUserName, sBrandsId, + sSubsidiaryId, sUserType, authorization + ) + .doOnNext(aiResponse -> { + // 设置请求ID + aiResponse.setRequestId(requestId); + // 后台累积完整文本（为后续TTS做准备） + accumulator.accumulate(aiResponse); + log.debug("收到ERP片段: requestId={}, chunk={}/{}", + requestId, + aiResponse.getChunkIndex(), + aiResponse.getTotalChunks()); + }) + .map(aiResponse -> { + // 将AiResponseDTO转换为TTSResponseDTO + // 使用processedText字段传递AI文本片段 + // 使用systemText字段传递系统文本片段 + return TTSResponseDTO.builder() + .code(200) + .message("ERP_CHUNK") // message字段标记为ERP文本块 + .requestId(requestId) + .processedText(aiResponse.getTextFragment()) // 用processedText传递AI文本片段 + .systemText(aiResponse.getSystemTextFragment()) // 用systemText传递系统文本片段 + .sSceneName(aiResponse.getSSceneName()) + .sMethodName(aiResponse.getSMethodName()) + .sReturnType(aiResponse.getSReturnType()) + .timestamp(System.currentTimeMillis()) + .build(); + }); + + // 2. ERP完成后，发送完成标记，然后开始TTS合成 + return erpStream + .concatWith(Flux.defer(() -> { + // 获取完整的累积结果 + AiResponseDTO completeResponse = accumulator.getCompleteResponse(); + + // 验证ERP结果 + if (StrUtil.isBlank(completeResponse.getAiText())) { + log.warn("ERP返回空文本: requestId={}", requestId); + return Flux.error(new RuntimeException("ERP返回空文本")); + } + + log.info("ERP流式处理完成，开始TTS合成: requestId={}, aiText长度={}", + requestId, completeResponse.getAiText().length()); + + // 3. 发送ERP完成消息（使用完整文本） + TTSResponseDTO erpCompleteDto = TTSResponseDTO.builder() + .code(200) + .message("ERP_COMPLETE") // message标记完成 + .requestId(requestId) + .processedText(completeResponse.getAiText()) // 完整AI文本 + .systemText(completeResponse.getSystemText()) // 完整系统文本 + .sSceneName(completeResponse.getSSceneName()) + .sMethodName(completeResponse.getSMethodName()) + .sReturnType(completeResponse.getSReturnType()) + .timestamp(System.currentTimeMillis()) + .build(); + + // 4. 调用TTS合成（返回TTSResponseDTO流） + Flux ttsStream = synthesizeStreamAiNew(request, completeResponse) + .doOnNext(ttsResponse -> { + ttsResponse.setRequestId(requestId); + ttsResponse.setMessage("TTS_SEGMENT"); // message标记为TTS音频段 + }); + + // 先发送ERP完成消息，再发送TTS流 + return Flux.concat(Flux.just(erpCompleteDto), ttsStream); + })) + // 超时控制 + .timeout(Duration.ofSeconds(120)) + // 错误处理 + .onErrorResume(e -> { + log.error("流式处理失败: requestId={}, error={}", requestId, e.getMessage()); + return Flux.just(TTSResponseDTO.error(requestId, 500, e.getMessage())); + }) + // 日志记录 + .doOnComplete(() -> log.info("流式处理完成: requestId={}", requestId)) + .doOnCancel(() -> log.warn("流式处理取消: requestId={}", requestId)); + } + + + + public ResponseEntity cleanMemory(TTSRequestDTO request) { String sUserId = request.getUserid(); String sUserName = request.getUsername(); @@ -123,7 +229,7 @@ public class PythonTtsProxyService { } /** - * 【保持原有返回类型】不动！内部流式请求Python + * 内部流式请求Python */ public ResponseEntity synthesizeStreamAi(TTSRequestDTO request, AiResponseDTO aiResponseDTO) { String aiText = aiResponseDTO.getAiText(); @@ -132,12 +238,10 @@ public class PythonTtsProxyService { systemText = StrUtil.EMPTY; } String voiceTextNew = AdvancedSymbolRemover.removePunctuationHtml(aiText); - // ============================ // 【绝对唯一】不会重复、不会覆盖 // ============================ String cacheKey = request.getUserid() + "_" + System.nanoTime(); - TTSResponseDTO dto = TTSResponseDTO.builder() .code(200) .message("success") @@ -159,7 +263,6 @@ public class PythonTtsProxyService { if (!voiceless || ObjectUtil.isEmpty(voiceTextNew)) { return ResponseEntity.ok(dto); } - // 平均分割文字 List textParts = splitTextSmart(voiceTextNew, 30); dto.setAudioSize(textParts.size()); @@ -199,14 +302,91 @@ public class PythonTtsProxyService { } } }, executorService); - return ResponseEntity.ok(dto); } + + /** + * 内部流式请求Python + */ + public Flux synthesizeStreamAiNew(TTSRequestDTO request, AiResponseDTO aiResponseDTO) { + String aiText = aiResponseDTO.getAiText(); + String systemText = aiResponseDTO.getSystemText(); + if (ObjectUtil.isEmpty(systemText)) { + systemText = StrUtil.EMPTY; + } + String voiceTextNew = AdvancedSymbolRemover.removePunctuationHtml(aiText); + // ============================ + // 【绝对唯一】不会重复、不会覆盖 + // ============================ + String cacheKey = request.getUserid() + "_" + System.nanoTime(); + TTSResponseDTO dto = TTSResponseDTO.builder() + .code(200) + .message("success") + .cacheKey(cacheKey) // 前端靠这个取自己的分段 + .originalText(request.getText()) + .processedText(aiText) + .audioText(voiceTextNew) + .systemText(systemText) + .voice(request.getVoice()) + .sSceneName(aiResponseDTO.getSSceneName()) + .sMethodName(aiResponseDTO.getSMethodName()) + .sReturnType(aiResponseDTO.getSReturnType()) + .sCommonts(BusinessCode.COMMONTS.getMessage()) + .timestamp(System.currentTimeMillis()) + .textLength((aiText + systemText).length()) + .build(); + + boolean voiceless = Boolean.TRUE.equals(request.getVoiceless()); + if (!voiceless || ObjectUtil.isEmpty(voiceTextNew)) { + return Flux.just(dto); + } + // 平均分割文字 + List textParts = splitTextSmart(voiceTextNew, 30); + dto.setAudioSize(textParts.size()); + // 异步分段合成 + CompletableFuture.runAsync(() -> { + for (int i = 0; i < textParts.size(); i++) { + String part = textParts.get(i); + if (ObjectUtil.isEmpty(part)) continue; + + try { + Map params = new HashMap<>(); + params.put("text", part); + params.put("voice", request.getVoice()); + params.put("rate", request.getRate() != null ? request.getRate() : "+10%"); + params.put("volume", request.getVolume() != null ? request.getVolume() : "+0%"); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.setAccept(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); + HttpEntity> entity = new HttpEntity<>(params, headers); + + ResponseEntity response = restTemplate.exchange( + pythonServiceUrl + "/stream-synthesize", + HttpMethod.POST, entity, byte[].class + ); + + if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { + String base64 = Base64.getEncoder().encodeToString(response.getBody()); + + // ============================ + // 【关键】带序号存储！前端靠序号知道顺序！ + // ============================ + LocalAudioCache.addPiece(cacheKey, i, part, base64); + } + } catch (Exception e) { + log.warn("分段合成失败: {}", e.getMessage()); + } + } + }, executorService); + return Flux.just(dto); + } + + // ============================================== + // 智能分段：优先按 。！？； ， 空格 断开 + // 不会把一句话生硬切断，更自然 // ============================================== -// 智能分段：优先按 。！？； ， 空格 断开 -// 不会把一句话生硬切断，更自然 -// ============================================== private List splitTextSmart(String text, int maxLength) { List parts = new ArrayList<>(); if (text == null || text.isEmpty()) return parts; diff --git a/src/main/java/com/xly/util/MD5Util.java b/src/main/java/com/xly/util/MD5Util.java new file mode 100644 index 0000000..a500076 --- /dev/null +++ b/src/main/java/com/xly/util/MD5Util.java @@ -0,0 +1,106 @@ +package com.xly.util; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * MD5加密工具类 + */ +public class MD5Util { + + /** + * 将字符串进行MD5加密 + * @param input 输入字符串 + * @return MD5加密后的32位小写字符串 + */ + public static String encrypt(String input) { + if (input == null || input.isEmpty()) { + return null; + } + + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] messageDigest = md.digest(input.getBytes()); + + // 将字节数组转换为十六进制字符串 + StringBuilder hexString = new StringBuilder(); + for (byte b : messageDigest) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("MD5加密失败", e); + } + } + + /** + * 将字符串进行MD5加密（大写） + * @param input 输入字符串 + * @return MD5加密后的32位大写字符串 + */ + public static String encryptToUpperCase(String input) { + String result = encrypt(input); + return result != null ? result.toUpperCase() : null; + } + + /** + * 验证字符串与MD5值是否匹配 + * @param input 输入字符串 + * @param md5 MD5值 + * @return 是否匹配 + */ + public static boolean verify(String input, String md5) { + String encrypted = encrypt(input); + return encrypted != null && encrypted.equalsIgnoreCase(md5); + } + + /** + * 获取文件的MD5值 + * @param bytes 文件字节数组 + * @return 文件MD5值 + */ + public static String getFileMD5(byte[] bytes) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] messageDigest = md.digest(bytes); + + StringBuilder hexString = new StringBuilder(); + for (byte b : messageDigest) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("文件MD5计算失败", e); + } + } + + /** + * MD5加盐加密 + * @param input 输入字符串 + * @param salt 盐值 + * @return 加盐后的MD5值 + */ + public static String encryptWithSalt(String input, String salt) { + return encrypt(input + salt); + } + + /** + * 双重MD5加密 + * @param input 输入字符串 + * @return 双重MD5加密结果 + */ + public static String doubleEncrypt(String input) { + String first = encrypt(input); + return encrypt(first); + } +} \ No newline at end of file diff --git a/src/main/java/com/xly/util/SqlValidateUtil.java b/src/main/java/com/xly/util/SqlValidateUtil.java index 0d7fd99..b276a4e 100644 --- a/src/main/java/com/xly/util/SqlValidateUtil.java +++ b/src/main/java/com/xly/util/SqlValidateUtil.java @@ -103,4 +103,13 @@ public class SqlValidateUtil { .replaceAll("\\n|\\r", " ") .trim(); } + + public static String getsKey(String sSceneId,String sMethodId,String sQuestion){ +// sSceneId+sMethodId+sQuestion + return sSceneId+"_"+sMethodId+"_"+ MD5Util.encrypt(sQuestion); + } + + public static String getsQuestion(List sUserQuestionList){ + return String.join(",", sUserQuestionList); + } } diff --git a/src/main/java/com/xly/web/TTSStreamController.java b/src/main/java/com/xly/web/TTSStreamController.java index ff6fc28..c72a9d1 100644 --- a/src/main/java/com/xly/web/TTSStreamController.java +++ b/src/main/java/com/xly/web/TTSStreamController.java @@ -7,6 +7,7 @@ import com.xly.tool.DynamicToolProvider; import com.xly.tts.bean.*; import com.xly.tts.service.LocalAudioCache; import com.xly.tts.service.PythonTtsProxyService; +import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.InputStreamResource; @@ -14,6 +15,8 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -82,6 +85,24 @@ public class TTSStreamController { return pythonTtsProxyService.synthesizeStreamAi(request); } + /** + * 流式合成语音（代理到Python服务） + */ + @PostMapping(value = "/stream/queryFlux", + consumes = {MediaType.APPLICATION_JSON_VALUE}, + produces = {MediaType.APPLICATION_OCTET_STREAM_VALUE, + MediaType.APPLICATION_JSON_VALUE}) + public Flux streamFlux(@Valid @RequestBody Mono requestMono) { + return requestMono.flatMapMany(request -> { + log.info("处理请求: requestId={}, text长度={}", request.getUserid(), request.getText().length()); + return pythonTtsProxyService.synthesizeStreamAiStream(request); + }) + .doOnSubscribe(sub -> log.debug("流式订阅开始")) + .doOnCancel(() -> log.debug("流式请求被取消")) + .doOnComplete(() -> log.debug("流式响应完成")) + .doOnError(e -> log.error("流式处理错误", e)); + } + @GetMapping("/audio/piece") public ResponseEntity> getPiece( @RequestParam String cacheKey, @@ -210,81 +231,6 @@ public class TTSStreamController { } /** - * SSE流式输出（Server-Sent Events） - */ - @GetMapping(value = "/sse-stream", produces = "text/event-stream") - public ResponseEntity sseStream( - @RequestParam String text, - @RequestParam(defaultValue = "zh-CN-XiaoxiaoNeural") String voice) { - - log.info("收到SSE流式请求: voice={}", voice); - - TTSRequestDTO request = new TTSRequestDTO(); - request.setText(text); - request.setVoice(voice); - - StreamingResponseBody responseBody = outputStream -> { - try { - outputStream.write(("event: audio-start\ndata: \n\n").getBytes()); - outputStream.flush(); - - // 调用Python服务获取音频 - ResponseEntity response = pythonTtsProxyService.synthesizeStream(request); - - if (response.getBody() != null) { - InputStream inputStream = response.getBody().getInputStream(); - byte[] buffer = new byte[1024]; - int bytesRead; - - int totalBytes = 0; - while ((bytesRead = inputStream.read(buffer)) != -1) { - totalBytes += bytesRead; - - // 发送进度事件 - String progressEvent = String.format( - "event: progress\ndata: {\"bytes\":%d}\n\n", totalBytes); - outputStream.write(progressEvent.getBytes()); - outputStream.flush(); - - // 发送音频数据（base64编码） - String base64Data = java.util.Base64.getEncoder().encodeToString( - java.util.Arrays.copyOfRange(buffer, 0, bytesRead)); - String audioEvent = String.format( - "event: audio-data\ndata: {\"chunk\":\"%s\"}\n\n", base64Data); - outputStream.write(audioEvent.getBytes()); - outputStream.flush(); - } - - // 发送完成事件 - String completeEvent = String.format( - "event: audio-complete\ndata: {\"total_bytes\":%d}\n\n", totalBytes); - outputStream.write(completeEvent.getBytes()); - outputStream.flush(); - } else { - outputStream.write(("event: error\ndata: {\"message\":\"合成失败\"}\n\n").getBytes()); - outputStream.flush(); - } - - } catch (Exception e) { -// log.error("SSE流式输出异常: {}", e.getMessage(), e); - try { - outputStream.write(("event: error\ndata: {\"message\":\"" + - e.getMessage().replace("\"", "\\\"") + "\"}\n\n").getBytes()); - outputStream.flush(); - } catch (Exception ex) { - // 忽略关闭错误 - } - } - }; - - return ResponseEntity.ok() - .header("Content-Type", "text/event-stream") - .header("Cache-Control", "no-cache") - .header("X-Accel-Buffering", "no") // 禁用Nginx缓冲 - .body(responseBody); - } - - /** * 测试接口 */ @GetMapping("/test") diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 27ee7ab..b237890 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -7,6 +7,8 @@ logging: com.xly: debug com.xlyflow: debug org.springframework: warn + ai.djl: DEBUG + dev.langchain4j: DEBUG server: port: 8099 @@ -52,35 +54,99 @@ spring: # REDIS (RedisProperties) data: redis: - host: localhost - port: 6379 # Redis 端口 - database: 0 # 使用的数据库索引（默认为0） - password: # 密码（如果没有设置则为空） - timeout: 3000ms # 连接超时时间 + host: 127.0.0.1 + password: xlyXLY2015 + port: 16379 + database: 0 # index + timeout: 30000ms # 连接超时时长（毫秒） + block-when-exhausted: true #redis配置结束 lettuce: pool: max-active: 8 # 连接池最大连接数 max-idle: 8 # 连接池最大空闲连接 - min-idle: 0 # 连接池最小空闲连接 + min-idle: 0 # 连接池最小空闲连接' + +milvus: + host: 112.82.245.194 + port: 19530 + database: xlymilvus + username: + password: + collection: + question-sql: question_sql_vectors + sync: + batch-size: 100 + vector: + dimension: 384 # All-MiniLM-L6-v2 模型的维度 + text: + max-length: 65535 + schema: + type: base # base, detailed, partitioned, searchable + # 重试配置 + retry: + enabled: true # 是否启用重试 + max-retry-times: 75 # 最大重试次数（SDK默认值） + initial-backoff-ms: 10 # 初始退避时间(ms) + max-backoff-ms: 3000 # 最大退避时间(ms) + backoff-multiplier: 3 # 退避乘数 + retry-on-rate-limit: true # 遇到限流是否重试 + max-retry-timeout-ms: 0 # 最大重试超时时间(0表示不限制) + + # 操作级别的重试配置（可选，覆盖默认值） + operations: + query: + max-retry-times: 3 + initial-backoff-ms: 100 + max-backoff-ms: 2000 + insert: + max-retry-times: 2 + initial-backoff-ms: 50 + max-backoff-ms: 1000 + search: + max-retry-times: 3 + initial-backoff-ms: 200 + max-backoff-ms: 3000 + max-retry-timeout-ms: 10000 + +# Actuator配置 +management: + endpoints: + web: + exposure: + include: health,info,metrics + endpoint: + health: + show-details: always + health: + milvus: + enabled: true # application.yml 或 application.properties langchain4j: ollama: # 聊天模型配置（用于一般对话） - base-url: http://121.43.128.225:11434 + base-url: http://112.82.245.194:11434 chat-model-name: qwen2.5:7b-instruct +# chat-model-name: qwen3.5:9b # SQL/代码模型配置（专门用于代码和SQL生成） - sql-model-name: qwen2.5-coder:32b + sql-model-name: qwen2.5-coder:7b +# sql-model-name: qwen2.5-coder:32b +# sql-model-name: mdq100/qwen3.5-coder:35b # 或者如果两个模型在同一服务器，可以使用同一个URL +# ollama: +# # 聊天模型配置（用于一般对话） +# base-url: http://112.82.245.194:11434 +# chat-model-name: qwen3.5:9b +# # SQL/代码模型配置（专门用于代码和SQL生成） +# sql-model-name: mdq100/qwen3.5-coder:35b + mybatis: mapper-locations: classpath:mapper/*.xml type-aliases-package: com.xly.entity configuration: map-underscore-to-camel-case: true log-impl: org.apache.ibatis.logging.stdout.StdOutImpl - - # 情感预设缓存 cache: enabled: true diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml index f020377..07613be 100644 --- a/src/main/resources/logback-spring.xml +++ b/src/main/resources/logback-spring.xml @@ -118,7 +118,8 @@ - + + diff --git a/src/main/resources/mapper/DynamicExeDbMapper.xml b/src/main/resources/mapper/DynamicExeDbMapper.xml index fba3162..e133d36 100644 --- a/src/main/resources/mapper/DynamicExeDbMapper.xml +++ b/src/main/resources/mapper/DynamicExeDbMapper.xml @@ -37,7 +37,7 @@ - + diff --git a/src/main/resources/templates/chat.html b/src/main/resources/templates/chat.html index 1f464d1..329b1af 100644 --- a/src/main/resources/templates/chat.html +++ b/src/main/resources/templates/chat.html @@ -466,7 +466,7 @@ let brandsid= "1111111111"; let subsidiaryid= "1111111111"; let usertype= "sysadmin"; - let authorization="CE444885A9BCFDDE1FD793F8A0931301E9D7DE6CEDD9DE4B83ECE2219C7829A8F3419238942A93E9AD666629E18D159AF7FE144A6407DE745BA0AEC8B235FC1D4CAE6F9AC893752209A98011A981375391D4466816B7D3D1AF306E28B989121C538155B7ADAEE71E899235DC1122F426"; + let authorization="1EDB99C9BF070115F7A57AC43D8CB09F0B8C49F979DAB63A2AEA84B372B2B42BF3419238942A93E9AD666629E18D159AF7FE144A6407DE745BA0AEC8B235FC1D35611629BD9166D2BBFC3B7AF31FDF60A31A297DF9BF51740C90173D4CC922B3538155B7ADAEE71E899235DC1122F426"; let hrefLock = window.location.origin+"/xlyAi"; const CONFIG = {