Commit fce865d46b450ce838d63cd34dbe2871bd6f967c
Merge remote-tracking branch 'origin/master'
Showing
39 changed files
with
4368 additions
and
234 deletions
pom.xml
| @@ -39,7 +39,7 @@ | @@ -39,7 +39,7 @@ | ||
| 39 | <jackson.version>2.17.2</jackson.version> | 39 | <jackson.version>2.17.2</jackson.version> |
| 40 | <json-schema.version>1.17.2</json-schema.version> | 40 | <json-schema.version>1.17.2</json-schema.version> |
| 41 | <!-- 向量数据 --> | 41 | <!-- 向量数据 --> |
| 42 | - <weaviate.version>4.4.0</weaviate.version> | 42 | + <milvus.version>2.6.15</milvus.version> |
| 43 | </properties> | 43 | </properties> |
| 44 | 44 | ||
| 45 | <dependencies> | 45 | <dependencies> |
| @@ -49,6 +49,30 @@ | @@ -49,6 +49,30 @@ | ||
| 49 | <artifactId>spring-boot-starter-web</artifactId> | 49 | <artifactId>spring-boot-starter-web</artifactId> |
| 50 | </dependency> | 50 | </dependency> |
| 51 | 51 | ||
| 52 | + <!-- Milvus Java SDK 核心依赖 --> | ||
| 53 | + <dependency> | ||
| 54 | + <groupId>io.milvus</groupId> | ||
| 55 | + <artifactId>milvus-sdk-java</artifactId> | ||
| 56 | + <version>${milvus.version}</version> | ||
| 57 | + </dependency> | ||
| 58 | + | ||
| 59 | + <dependency> | ||
| 60 | + <groupId>dev.langchain4j</groupId> | ||
| 61 | + <artifactId>langchain4j-embeddings-all-minilm-l6-v2</artifactId> | ||
| 62 | + <version>${langchain4j.version}</version> | ||
| 63 | + </dependency> | ||
| 64 | + | ||
| 65 | +<!-- <dependency>--> | ||
| 66 | +<!-- <groupId>com.baomidou</groupId>--> | ||
| 67 | +<!-- <artifactId>mybatis-plus-boot-starter</artifactId>--> | ||
| 68 | +<!-- <version>3.5.3.1</version>--> | ||
| 69 | +<!-- </dependency>--> | ||
| 70 | + | ||
| 71 | + <dependency> | ||
| 72 | + <groupId>org.springframework.boot</groupId> | ||
| 73 | + <artifactId>spring-boot-starter-actuator</artifactId> | ||
| 74 | + </dependency> | ||
| 75 | + | ||
| 52 | <dependency> | 76 | <dependency> |
| 53 | <groupId>org.springframework.boot</groupId> | 77 | <groupId>org.springframework.boot</groupId> |
| 54 | <artifactId>spring-boot-starter-validation</artifactId> | 78 | <artifactId>spring-boot-starter-validation</artifactId> |
| @@ -63,6 +87,12 @@ | @@ -63,6 +87,12 @@ | ||
| 63 | <artifactId>spring-boot-starter-aop</artifactId> | 87 | <artifactId>spring-boot-starter-aop</artifactId> |
| 64 | </dependency> | 88 | </dependency> |
| 65 | 89 | ||
| 90 | + <dependency> | ||
| 91 | + <groupId>com.google.code.gson</groupId> | ||
| 92 | + <artifactId>gson</artifactId> | ||
| 93 | + <version>2.10.1</version> | ||
| 94 | + </dependency> | ||
| 95 | + | ||
| 66 | <!-- Spring Cloud Context 依赖 --> | 96 | <!-- Spring Cloud Context 依赖 --> |
| 67 | <dependency> | 97 | <dependency> |
| 68 | <groupId>org.springframework.cloud</groupId> | 98 | <groupId>org.springframework.cloud</groupId> |
| @@ -270,69 +300,12 @@ | @@ -270,69 +300,12 @@ | ||
| 270 | <version>1.3.2</version> | 300 | <version>1.3.2</version> |
| 271 | </dependency> | 301 | </dependency> |
| 272 | 302 | ||
| 273 | - <!-- H2 Database --> | ||
| 274 | - <!-- 最新稳定版本 --> | ||
| 275 | - <!-- <dependency>--> | ||
| 276 | - <!-- <groupId>com.h2database</groupId>--> | ||
| 277 | - <!-- <artifactId>h2</artifactId>--> | ||
| 278 | - <!-- <version>2.2.224</version>--> | ||
| 279 | - <!-- </dependency>--> | ||
| 280 | - | ||
| 281 | - <!-- 数据验证 --> | ||
| 282 | - <!-- <dependency>--> | ||
| 283 | - <!-- <groupId>org.hibernate.orm</groupId>--> | ||
| 284 | - <!-- <artifactId>hibernate-core</artifactId>--> | ||
| 285 | - <!-- <version>6.4.4.Final</version>--> | ||
| 286 | - <!-- </dependency>--> | ||
| 287 | - <!-- <dependency>--> | ||
| 288 | - <!-- <groupId>org.hibernate.validator</groupId>--> | ||
| 289 | - <!-- <artifactId>hibernate-validator</artifactId>--> | ||
| 290 | - <!-- <version>8.0.1.Final</version>--> | ||
| 291 | - <!-- </dependency>--> | ||
| 292 | <dependency> | 303 | <dependency> |
| 293 | <groupId>jakarta.persistence</groupId> | 304 | <groupId>jakarta.persistence</groupId> |
| 294 | <artifactId>jakarta.persistence-api</artifactId> | 305 | <artifactId>jakarta.persistence-api</artifactId> |
| 295 | <version>3.1.0</version> | 306 | <version>3.1.0</version> |
| 296 | </dependency> | 307 | </dependency> |
| 297 | 308 | ||
| 298 | - | ||
| 299 | - <!-- <dependency>--> | ||
| 300 | - <!-- <groupId>dev.langchain4j</groupId>--> | ||
| 301 | - <!-- <artifactId>langchain4j-document-parser-apache-pdfbox</artifactId>--> | ||
| 302 | - <!-- <version>${langchain4jpdf.version}</version>--> | ||
| 303 | - <!-- </dependency>--> | ||
| 304 | - | ||
| 305 | - | ||
| 306 | - <!-- 工具调用 --> | ||
| 307 | - <!-- <dependency>--> | ||
| 308 | - <!-- <groupId>dev.langchain4j</groupId>--> | ||
| 309 | - <!-- <artifactId>langchain4j</artifactId>--> | ||
| 310 | - <!-- <version>${langchain4-10.version}</version>--> | ||
| 311 | - <!-- </dependency>--> | ||
| 312 | - | ||
| 313 | - <!-- ========== 官方 SDK（可选） ========== --> | ||
| 314 | - <!-- OpenAI 官方 SDK --> | ||
| 315 | - <!-- <dependency>--> | ||
| 316 | - <!-- <groupId>com.theokanning.openai-gpt3-java</groupId>--> | ||
| 317 | - <!-- <artifactId>service</artifactId>--> | ||
| 318 | - <!-- <version>${openai-java.version}</version>--> | ||
| 319 | - <!-- </dependency>--> | ||
| 320 | - | ||
| 321 | - <!-- <!– Anthropic 官方 SDK –>--> | ||
| 322 | - <!-- <dependency>--> | ||
| 323 | - <!-- <groupId>com.anthropics</groupId>--> | ||
| 324 | - <!-- <artifactId>anthropic-sdk-java</artifactId>--> | ||
| 325 | - <!-- <version>${anthropic-sdk.version}</version>--> | ||
| 326 | - <!-- </dependency>--> | ||
| 327 | - | ||
| 328 | - <!-- 嵌入向量 --> | ||
| 329 | - <!-- <dependency>--> | ||
| 330 | - <!-- <groupId>dev.langchain4j</groupId>--> | ||
| 331 | - <!-- <artifactId>langchain4j-embeddings</artifactId>--> | ||
| 332 | - <!-- <version>${langchain4jbeta.version}</version>--> | ||
| 333 | - <!-- </dependency>--> | ||
| 334 | - | ||
| 335 | - | ||
| 336 | <!-- ========== LangChain4j 1.10.0 ========== --> | 309 | <!-- ========== LangChain4j 1.10.0 ========== --> |
| 337 | <!-- 核心库 --> | 310 | <!-- 核心库 --> |
| 338 | <dependency> | 311 | <dependency> |
| @@ -341,23 +314,13 @@ | @@ -341,23 +314,13 @@ | ||
| 341 | <version>${langchain4j.version}</version> | 314 | <version>${langchain4j.version}</version> |
| 342 | </dependency> | 315 | </dependency> |
| 343 | 316 | ||
| 344 | - <!-- OpenAI 集成（支持 Function Calling） --> | ||
| 345 | -<!-- <dependency>--> | ||
| 346 | -<!-- <groupId>dev.langchain4j</groupId>--> | ||
| 347 | -<!-- <artifactId>langchain4j-open-ai</artifactId>--> | ||
| 348 | -<!-- <version>${langchain4j.version}</version>--> | ||
| 349 | -<!-- </dependency>--> | ||
| 350 | 317 | ||
| 351 | <dependency> | 318 | <dependency> |
| 352 | <groupId>dev.langchain4j</groupId> | 319 | <groupId>dev.langchain4j</groupId> |
| 353 | <artifactId>langchain4j</artifactId> | 320 | <artifactId>langchain4j</artifactId> |
| 354 | <version>${langchain4j.version}</version> | 321 | <version>${langchain4j.version}</version> |
| 355 | </dependency> | 322 | </dependency> |
| 356 | - <!-- <dependency>--> | ||
| 357 | - <!-- <groupId>dev.langchain4j</groupId>--> | ||
| 358 | - <!-- <artifactId>langchain4j-embeddings</artifactId>--> | ||
| 359 | - <!-- <version>${langchain4j.version}</version>--> | ||
| 360 | - <!-- </dependency>--> | 323 | + |
| 361 | <dependency> | 324 | <dependency> |
| 362 | <groupId>dev.langchain4j</groupId> | 325 | <groupId>dev.langchain4j</groupId> |
| 363 | <artifactId>langchain4j-ollama</artifactId> | 326 | <artifactId>langchain4j-ollama</artifactId> |
| @@ -371,13 +334,6 @@ | @@ -371,13 +334,6 @@ | ||
| 371 | <version>1.17.0</version> | 334 | <version>1.17.0</version> |
| 372 | </dependency> | 335 | </dependency> |
| 373 | 336 | ||
| 374 | - <!-- 文档加载器 --> | ||
| 375 | - <!-- Tika 文档解析器（支持多种格式） --> | ||
| 376 | - <!-- <dependency>--> | ||
| 377 | - <!-- <groupId>dev.langchain4j</groupId>--> | ||
| 378 | - <!-- <artifactId>langchain4j-document-parser-tika</artifactId>--> | ||
| 379 | - <!-- <version>${langchain4j.version}</version>--> | ||
| 380 | - <!-- </dependency>--> | ||
| 381 | 337 | ||
| 382 | <!-- 或者使用 Apache Tika 直接 --> | 338 | <!-- 或者使用 Apache Tika 直接 --> |
| 383 | <dependency> | 339 | <dependency> |
| @@ -392,12 +348,6 @@ | @@ -392,12 +348,6 @@ | ||
| 392 | <version>2.9.1</version> | 348 | <version>2.9.1</version> |
| 393 | </dependency> | 349 | </dependency> |
| 394 | 350 | ||
| 395 | - <!-- 嵌入向量 --> | ||
| 396 | - <!-- <dependency>--> | ||
| 397 | - <!-- <groupId>dev.langchain4j</groupId>--> | ||
| 398 | - <!-- <artifactId>langchain4j-embeddings</artifactId>--> | ||
| 399 | - <!-- <version>${langchain4jbeta.version}</version>--> | ||
| 400 | - <!-- </dependency>--> | ||
| 401 | 351 | ||
| 402 | <!-- Spring Retry --> | 352 | <!-- Spring Retry --> |
| 403 | <dependency> | 353 | <dependency> |
src/main/java/com/xly/agent/ErpAiAgent.java
| @@ -27,10 +27,13 @@ public interface ErpAiAgent { | @@ -27,10 +27,13 @@ public interface ErpAiAgent { | ||
| 27 | * 入参：用户问题、执行的SQL、表结构、JSON格式结果 | 27 | * 入参：用户问题、执行的SQL、表结构、JSON格式结果 |
| 28 | */ | 28 | */ |
| 29 | @SystemMessage(""" | 29 | @SystemMessage(""" |
| 30 | - 你是专业的业务数据分析师，严格遵循以下**通用规则**解释查询结果，适用于所有业务场景： | 30 | + 你是专业的业务数据分析师，严格遵循以下**通用规则**解释查询结果，适用于所有业务场景： |
| 31 | 1. 解释风格：贴合业务场景，无任何SQL专业术语，用口语化、简洁的商业语言说明，避免技术词汇； | 31 | 1. 解释风格：贴合业务场景，无任何SQL专业术语，用口语化、简洁的商业语言说明，避免技术词汇； |
| 32 | 2. 数据准确：严格按照JSON执行结果解释，不夸大、不遗漏、不编造数据，数值与结果完全一致； | 32 | 2. 数据准确：严格按照JSON执行结果解释，不夸大、不遗漏、不编造数据，数值与结果完全一致； |
| 33 | - 3. 输出格式：仅返回解释内容，不要列出ID,无多余标题、换行、符号，结果为空时直接返回“未查询到相关数据”； | 33 | + 3. 输出格式：仅返回解释内容，不要列出ID,无多余标题、换行、符号,结果为空时直接返回“未查询到相关数据” |
| 34 | + 3.1. 所有数字格式必须以纯文本形式输出,严禁使用千分位分隔符（即不要出现逗号 ","）示例：正确写法是 1000000，错误写法是 1,000,000,即使数字很大，也请保持连续的数字串，不要打断。 | ||
| 35 | + 3.2 所有日期请转换为 YYYY-MM-DD 格式（例如：2026-03-15）,严禁包含时间部分（如小时、分钟、秒）（例如：2026-03-15 00:00:00），也不要包含时区信息。” | ||
| 36 | + 3.3. 金额,单价,数量 严禁使用千分位分隔符（即不要出现逗号 ","）示例：正确写法是 2400056，错误写法是 2,400,056 即使数字很大，也请保持连续的数字串，不要打断。 | ||
| 34 | 4. 长度控制：单条解释不超过150字，条理清晰，重点突出核心数据/趋势； | 37 | 4. 长度控制：单条解释不超过150字，条理清晰，重点突出核心数据/趋势； |
| 35 | 5. 禁止重复：不重复用户问题、不重复执行的SQL语句，仅针对结果做业务解读。 | 38 | 5. 禁止重复：不重复用户问题、不重复执行的SQL语句，仅针对结果做业务解读。 |
| 36 | """) | 39 | """) |
| @@ -48,4 +51,132 @@ public interface ErpAiAgent { | @@ -48,4 +51,132 @@ public interface ErpAiAgent { | ||
| 48 | @V("sql") String sql, | 51 | @V("sql") String sql, |
| 49 | @V("tableStruct") String tableStruct, | 52 | @V("tableStruct") String tableStruct, |
| 50 | @V("result") String result); | 53 | @V("result") String result); |
| 54 | + | ||
| 55 | + /** | ||
| 56 | + * 动态表结构：自然语言解释SQL执行结果 | ||
| 57 | + * 入参：用户问题、执行的SQL、表结构、JSON格式结果 | ||
| 58 | + */ | ||
| 59 | + @SystemMessage(""" | ||
| 60 | + 你是专业的业务数据分析师，请分析以下查询结果： | ||
| 61 | + 【用户问题】 | ||
| 62 | + {{userInput}} | ||
| 63 | + 【数据字段说明】 | ||
| 64 | + {{sMilvusFiledDescription}} | ||
| 65 | + 【查询结果数据（JSON格式）】 | ||
| 66 | + {{result}} | ||
| 67 | + 【分析要求】 | ||
| 68 | + 1. 解释风格：贴合业务场景，无任何SQL专业术语，用口语化、简洁的商业语言说明，避免技术词汇； | ||
| 69 | + 2. 数据准确：严格按照JSON执行结果解释，不夸大、不遗漏、不编造数据，数值与结果完全一致； | ||
| 70 | + 3. 输出格式： | ||
| 71 | + 3.1. 如果用户要求"表格形式展示"，先输出简短文字说明，然后输出Markdown格式的表格 | ||
| 72 | + 3.2. 如果用户未要求表格，仅返回解释内容，不要列出ID,无多余标题、换行、符号 | ||
| 73 | + 3.3. 结果为空时直接返回"未查询到相关数据" | ||
| 74 | + 3.4. 所有数字格式必须以纯文本形式输出,严禁使用千分位分隔符（即不要出现逗号 ","） | ||
| 75 | + 3.5. 所有日期请转换为 YYYY-MM-DD 格式，严禁包含时间部分 | ||
| 76 | + 4. 长度控制：单条解释不超过150字，条理清晰，重点突出核心数据/趋势； | ||
| 77 | + 5. 禁止重复：不重复用户问题、不重复执行的SQL语句，仅针对结果做业务解读。 | ||
| 78 | + """) | ||
| 79 | + @UserMessage(""" | ||
| 80 | + 【用户查询】 | ||
| 81 | + {{userInput}} | ||
| 82 | + 【字段说明】 | ||
| 83 | + {{sMilvusFiledDescription}} | ||
| 84 | + 【查询结果】 | ||
| 85 | + 用户原始查询：{{userInput}} | ||
| 86 | + 执行查询向量库后结果（JSON格式）：{{result}} | ||
| 87 | + 请根据上述信息+通用规则，对查询结果做业务解释： | ||
| 88 | + """) | ||
| 89 | + String explainMilvusResult(@MemoryId String userId, | ||
| 90 | + @V("userInput") String userInput, | ||
| 91 | + @V("sMilvusFiledDescription") String sMilvusFiledDescription, | ||
| 92 | + @V("result") String result); | ||
| 93 | + | ||
| 94 | + /** | ||
| 95 | + * AI路由判断接口 | ||
| 96 | + * true: 走聚合查询（MySQL） | ||
| 97 | + * false: 走向量检索（Milvus） | ||
| 98 | + */ | ||
| 99 | +// @SystemMessage(""" | ||
| 100 | +// 你是一个智能查询路由专家，请根据用户需求判断应该使用哪种查询方式。 | ||
| 101 | +// | ||
| 102 | +// 判断标准： | ||
| 103 | +// 1. 返回 true（聚合查询/MySQL）的场景： | ||
| 104 | +// - 需要计算统计指标：总数、总和、平均值、最大/最小值、占比 | ||
| 105 | +// - 需要数据汇总：分组统计、排行榜、TopN | ||
| 106 | +// - 包含关键词：统计、求和、汇总、排名、平均、数量、总额、最高、最低、占比、分组、分析、趋势 | ||
| 107 | +// - 示例：统计本月销售总额、查询销量前10的商品、各品类占比分析 | ||
| 108 | +// | ||
| 109 | +// 2. 返回 false（向量检索/Milvus）的场景： | ||
| 110 | +// - 查询明细数据：XXX的销售订单明细、XXX的客户信息、具体内容详情 | ||
| 111 | +// - 查找相似内容：根据语义查找相关文档、推荐相似商品 | ||
| 112 | +// - 模糊匹配：不确定具体关键词，需要语义理解 | ||
| 113 | +// - 内容检索：查找包含特定概念的文档 | ||
| 114 | +// - 包含关键词：明细、详情、查询明细、查找、搜索、匹配、推荐、相似、相关、类似 | ||
| 115 | +// - 示例：李留记的销售订单明细、查询关于人工智能的文档、找相似的图片 | ||
| 116 | +// | ||
| 117 | +// 重要规则： | ||
| 118 | +// - 只返回 true 或 false，不要返回其他内容 | ||
| 119 | +// - 不要解释，不要添加额外文字 | ||
| 120 | +// - 如果用户要求"表格形式展示"，返回 false（明细查询） | ||
| 121 | +// - 如果用户指定具体人名、具体对象，返回 false（明细查询） | ||
| 122 | +// """) | ||
| 123 | +// @UserMessage("用户需求：{{userInput}}") | ||
| 124 | + @SystemMessage(""" | ||
| 125 | + 你是一个智能查询路由专家。请根据【用户需求】，只返回 true 或 false | ||
| 126 | + - 如果用户需求包含以下关键词：统计、求和、汇总、排名、TopN、平均、数量、总额、最高、最低、占比、分组，则返回true | ||
| 127 | + - 如果用户需求属于模糊匹配、普通语义检索，查询明细，(例如:查询报价单明细,查询客户信息)，则返回false | ||
| 128 | + - 查询明细数据：XXX的销售订单明细、XXX的客户信息、具体内容详情，则返回false | ||
| 129 | + - 模糊匹配：不确定具体关键词，需要语义理解，则返回false | ||
| 130 | + """) | ||
| 131 | + @UserMessage(""" | ||
| 132 | + 【用户需求】 | ||
| 133 | + {{userInput}} | ||
| 134 | + """) | ||
| 135 | + Boolean routeQuery(@MemoryId String userId, @V("userInput") String userInput); | ||
| 136 | + | ||
| 137 | + /** | ||
| 138 | + * 生成 Milvus 过滤条件 | ||
| 139 | + */ | ||
| 140 | + @SystemMessage(""" | ||
| 141 | + MILVUS 标量过滤条件生成规则（严格遵守）： | ||
| 142 | + 1. 语法规范： | ||
| 143 | + - 允许的操作符：==, !=, like | ||
| 144 | + - 逻辑组合：&& (AND), || (OR) | ||
| 145 | + - 所有字段都是字符串类型，值必须使用单引号包裹 | ||
| 146 | + - 字符串中的单引号需要转义：'O''Reilly' | ||
| 147 | + 2. 可用字段（只能使用这些字段）： | ||
| 148 | + - {{sMilvusFiled}} | ||
| 149 | + 字段说明： | ||
| 150 | + - {{sMilvusFiledDescription}} | ||
| 151 | + 3. 重要规则： | ||
| 152 | + - 只使用上述可用字段，不要创建新字段 | ||
| 153 | + - 如果用户提到了文档类型（如"报价单"、"订单"等），但可用字段中没有类型字段，则忽略该条件 | ||
| 154 | + - 只提取有明确值的字段条件 | ||
| 155 | + 4. 生成规则： | ||
| 156 | + - 如果没有提取到任何具体条件，返回空字符串 | ||
| 157 | + - 从用户输入中提取明确的字段条件 | ||
| 158 | + - 识别模式：字段名 + 操作符 + 值 | ||
| 159 | + - 示例： | ||
| 160 | + * "单据号 INV001" → sBillNo == 'INV001' | ||
| 161 | + * "客户编号 C001" → sCustomerNo == 'C001' | ||
| 162 | + * "销售人员张三" → sSalesManName == '张三' | ||
| 163 | + * "产品包含手机" → sProductStyle like '%手机%' | ||
| 164 | + 5. 输出格式： | ||
| 165 | + - 仅返回纯过滤条件，无任何解释、换行、备注 | ||
| 166 | + - 单条件：sBillNo == 'INV001' | ||
| 167 | + - 多条件：(sBillNo == 'INV001' && sCustomerNo == 'C001') | ||
| 168 | + - 无条件：直接返回空字符串 | ||
| 169 | + """) | ||
| 170 | + @UserMessage(""" | ||
| 171 | + 【用户查询】 | ||
| 172 | + - {{userInput}} | ||
| 173 | + 【可用字段】 | ||
| 174 | + - {{sMilvusFiled}} | ||
| 175 | + 【字段说明】 | ||
| 176 | + - {{sMilvusFiledDescription}} | ||
| 177 | + """) | ||
| 178 | + String getMilvusFilter(@MemoryId String userId, | ||
| 179 | + @V("userInput") String userInput, | ||
| 180 | + @V("sMilvusFiled") String sMilvusFiled, | ||
| 181 | + @V("sMilvusFiledDescription") String sMilvusFiledDescription); | ||
| 51 | } | 182 | } |
src/main/java/com/xly/config/OperableChatMemoryProvider.java
| @@ -121,6 +121,30 @@ public class OperableChatMemoryProvider implements ChatMemoryProvider { | @@ -121,6 +121,30 @@ public class OperableChatMemoryProvider implements ChatMemoryProvider { | ||
| 121 | // 步骤4: 完全重新设置消息列表 | 121 | // 步骤4: 完全重新设置消息列表 |
| 122 | return rebuildMemoryWithMessages(memoryId, currentMessages); | 122 | return rebuildMemoryWithMessages(memoryId, currentMessages); |
| 123 | } | 123 | } |
| 124 | + | ||
| 125 | + public List<ChatMessage> deleteUserLasterMessageBySize(Object memoryId,Integer size) { | ||
| 126 | + if (Objects.isNull(memoryId) || size==0) { | ||
| 127 | + return getCurrentChatMessages(memoryId); | ||
| 128 | + } | ||
| 129 | + // 步骤1: 获取当前所有消息 | ||
| 130 | + ChatMemory currentMemory = this.get(memoryId); | ||
| 131 | + List<ChatMessage> currentMessages = new ArrayList<>(currentMemory.messages()); | ||
| 132 | + // 从后往前查找内容匹配的最后一条消息 | ||
| 133 | + int indexToDelete = currentMessages.size(); | ||
| 134 | + // 如果找到匹配的消息 | ||
| 135 | + if (indexToDelete >= 0) { | ||
| 136 | + List<ChatMessage> filteredMessages = new ArrayList<>(currentMessages); | ||
| 137 | + for(int i=0;i<size;i++){ | ||
| 138 | + indexToDelete = indexToDelete -1; | ||
| 139 | + if(indexToDelete>1){ | ||
| 140 | + filteredMessages.remove(indexToDelete); | ||
| 141 | + } | ||
| 142 | + } | ||
| 143 | + return rebuildMemoryWithMessages(memoryId, filteredMessages); | ||
| 144 | + } | ||
| 145 | + // 步骤4: 完全重新设置消息列表 | ||
| 146 | + return rebuildMemoryWithMessages(memoryId, currentMessages); | ||
| 147 | + } | ||
| 124 | /** | 148 | /** |
| 125 | * 批量删除多条消息 | 149 | * 批量删除多条消息 |
| 126 | * @param memoryId 会话ID | 150 | * @param memoryId 会话ID |
src/main/java/com/xly/entity/AiResponseAccumulator.java
0 → 100644
| 1 | +package com.xly.entity; | ||
| 2 | + | ||
| 3 | +import com.xly.entity.AiResponseDTO; | ||
| 4 | +import cn.hutool.core.util.StrUtil; | ||
| 5 | +import lombok.extern.slf4j.Slf4j; | ||
| 6 | + | ||
| 7 | +/** | ||
| 8 | + * AI响应累积器 | ||
| 9 | + */ | ||
| 10 | +@Slf4j | ||
| 11 | +public class AiResponseAccumulator { | ||
| 12 | + | ||
| 13 | + private final String requestId; | ||
| 14 | + private final StringBuilder aiTextBuilder = new StringBuilder(); | ||
| 15 | + private final StringBuilder systemTextBuilder = new StringBuilder(); | ||
| 16 | + private String sSceneName; | ||
| 17 | + private String sMethodName; | ||
| 18 | + private String sReturnType; | ||
| 19 | + private int totalChunks = 0; | ||
| 20 | + private int processedChunks = 0; | ||
| 21 | + private final long startTime; | ||
| 22 | + | ||
| 23 | + public AiResponseAccumulator(String requestId) { | ||
| 24 | + this.requestId = requestId; | ||
| 25 | + this.startTime = System.currentTimeMillis(); | ||
| 26 | + } | ||
| 27 | + | ||
| 28 | + /** | ||
| 29 | + * 累积单个AI响应 | ||
| 30 | + */ | ||
| 31 | + public void accumulate(AiResponseDTO response) { | ||
| 32 | + processedChunks++; | ||
| 33 | + | ||
| 34 | + // 更新总块数 | ||
| 35 | + if (response.getTotalChunks() != null && response.getTotalChunks() > 0) { | ||
| 36 | + this.totalChunks = response.getTotalChunks(); | ||
| 37 | + } | ||
| 38 | + | ||
| 39 | + // 累积AI文本片段 | ||
| 40 | + if (StrUtil.isNotBlank(response.getTextFragment())) { | ||
| 41 | + aiTextBuilder.append(response.getTextFragment()); | ||
| 42 | + } | ||
| 43 | + | ||
| 44 | + // 累积系统文本片段 | ||
| 45 | + if (StrUtil.isNotBlank(response.getSystemTextFragment())) { | ||
| 46 | + systemTextBuilder.append(response.getSystemTextFragment()); | ||
| 47 | + } | ||
| 48 | + | ||
| 49 | + // 更新元数据（取最后一次非空值） | ||
| 50 | + if (StrUtil.isNotBlank(response.getSSceneName())) { | ||
| 51 | + this.sSceneName = response.getSSceneName(); | ||
| 52 | + } | ||
| 53 | + if (StrUtil.isNotBlank(response.getSMethodName())) { | ||
| 54 | + this.sMethodName = response.getSMethodName(); | ||
| 55 | + } | ||
| 56 | + if (StrUtil.isNotBlank(response.getSReturnType())) { | ||
| 57 | + this.sReturnType = response.getSReturnType(); | ||
| 58 | + } | ||
| 59 | + | ||
| 60 | + log.debug("累积进度: requestId={}, 已处理={}/{}块, AI文本长度={}", | ||
| 61 | + requestId, processedChunks, totalChunks, aiTextBuilder.length()); | ||
| 62 | + } | ||
| 63 | + | ||
| 64 | + /** | ||
| 65 | + * 获取完整的响应 | ||
| 66 | + */ | ||
| 67 | + public AiResponseDTO getCompleteResponse() { | ||
| 68 | + AiResponseDTO response = new AiResponseDTO(); | ||
| 69 | + response.setRequestId(requestId); | ||
| 70 | + response.setAiText(aiTextBuilder.toString()); | ||
| 71 | + response.setSystemText(systemTextBuilder.toString()); | ||
| 72 | + response.setFullAiText(aiTextBuilder.toString()); | ||
| 73 | + response.setFullSystemText(systemTextBuilder.toString()); | ||
| 74 | + response.setSSceneName(sSceneName); | ||
| 75 | + response.setSMethodName(sMethodName); | ||
| 76 | + response.setSReturnType(sReturnType != null ? sReturnType : "MARKDOWN"); | ||
| 77 | + response.setTotalChunks(totalChunks); | ||
| 78 | + response.setChunkIndex(processedChunks - 1); | ||
| 79 | + response.setIsLastChunk(true); | ||
| 80 | + response.setElapsedTime(System.currentTimeMillis() - startTime); | ||
| 81 | + | ||
| 82 | + log.info("ERP累积完成: requestId={}, 总块数={}, AI文本长度={}, 系统文本长度={}, 耗时={}ms", | ||
| 83 | + requestId, totalChunks, | ||
| 84 | + aiTextBuilder.length(), systemTextBuilder.length(), | ||
| 85 | + response.getElapsedTime()); | ||
| 86 | + | ||
| 87 | + return response; | ||
| 88 | + } | ||
| 89 | +} | ||
| 0 | \ No newline at end of file | 90 | \ No newline at end of file |
src/main/java/com/xly/entity/AiResponseDTO.java
| @@ -9,9 +9,12 @@ import lombok.Data; | @@ -9,9 +9,12 @@ import lombok.Data; | ||
| 9 | import lombok.NoArgsConstructor; | 9 | import lombok.NoArgsConstructor; |
| 10 | 10 | ||
| 11 | import java.io.Serializable; | 11 | import java.io.Serializable; |
| 12 | +import java.util.List; | ||
| 13 | +import java.util.Map; | ||
| 12 | 14 | ||
| 13 | /** | 15 | /** |
| 14 | * TTS响应数据传输对象 | 16 | * TTS响应数据传输对象 |
| 17 | + * 增强版：支持流式处理 | ||
| 15 | */ | 18 | */ |
| 16 | @Data | 19 | @Data |
| 17 | @Builder | 20 | @Builder |
| @@ -20,14 +23,230 @@ import java.io.Serializable; | @@ -20,14 +23,230 @@ import java.io.Serializable; | ||
| 20 | public class AiResponseDTO implements Serializable { | 23 | public class AiResponseDTO implements Serializable { |
| 21 | 24 | ||
| 22 | private static final long serialVersionUID = 1L; | 25 | private static final long serialVersionUID = 1L; |
| 23 | - // AI文字部分 | 26 | + |
| 27 | + // ============ 原有字段 ============ | ||
| 28 | + | ||
| 29 | + /** | ||
| 30 | + * AI文字部分 | ||
| 31 | + */ | ||
| 24 | private String aiText; | 32 | private String aiText; |
| 25 | - //系统拼接返回的文字部分 | 33 | + |
| 34 | + /** | ||
| 35 | + * 系统拼接返回的文字部分 | ||
| 36 | + */ | ||
| 26 | private String systemText; | 37 | private String systemText; |
| 27 | - //业务场景名称 | 38 | + |
| 39 | + /** | ||
| 40 | + * 业务场景名称 | ||
| 41 | + */ | ||
| 28 | private String sSceneName; | 42 | private String sSceneName; |
| 29 | - //业务方法名称 | 43 | + |
| 44 | + /** | ||
| 45 | + * 业务方法名称 | ||
| 46 | + */ | ||
| 30 | private String sMethodName; | 47 | private String sMethodName; |
| 48 | + | ||
| 49 | + /** | ||
| 50 | + * 返回类型，默认MARKDOWN | ||
| 51 | + */ | ||
| 31 | private String sReturnType = ReturnTypeCode.MAKEDOWN.getCode(); | 52 | private String sReturnType = ReturnTypeCode.MAKEDOWN.getCode(); |
| 32 | 53 | ||
| 54 | + // ============ 新增字段：流式处理支持 ============ | ||
| 55 | + | ||
| 56 | + /** | ||
| 57 | + * 请求ID，用于追踪整个流式请求 | ||
| 58 | + */ | ||
| 59 | + private String requestId; | ||
| 60 | + | ||
| 61 | + /** | ||
| 62 | + * 响应码 | ||
| 63 | + */ | ||
| 64 | + private Integer code; | ||
| 65 | + | ||
| 66 | + /** | ||
| 67 | + * 响应消息 | ||
| 68 | + */ | ||
| 69 | + private String message; | ||
| 70 | + | ||
| 71 | + /** | ||
| 72 | + * 处理状态：PROCESSING, COMPLETED, FAILED | ||
| 73 | + */ | ||
| 74 | + private String status; | ||
| 75 | + | ||
| 76 | + /** | ||
| 77 | + * 当前处理的块编号（从0开始） | ||
| 78 | + */ | ||
| 79 | + private Integer chunkIndex; | ||
| 80 | + | ||
| 81 | + /** | ||
| 82 | + * 总块数 | ||
| 83 | + */ | ||
| 84 | + private Integer totalChunks; | ||
| 85 | + | ||
| 86 | + /** | ||
| 87 | + * 是否是最后一块 | ||
| 88 | + */ | ||
| 89 | + private Boolean isLastChunk; | ||
| 90 | + | ||
| 91 | + /** | ||
| 92 | + * 文本片段（用于流式传输） | ||
| 93 | + * 当aiText过大时，可以分段传输 | ||
| 94 | + */ | ||
| 95 | + private String textFragment; | ||
| 96 | + | ||
| 97 | + /** | ||
| 98 | + * 系统文本片段（用于流式传输） | ||
| 99 | + */ | ||
| 100 | + private String systemTextFragment; | ||
| 101 | + | ||
| 102 | + /** | ||
| 103 | + * 累积的完整AI文本（仅在最后一块时返回） | ||
| 104 | + */ | ||
| 105 | + private String fullAiText; | ||
| 106 | + | ||
| 107 | + /** | ||
| 108 | + * 累积的完整系统文本（仅在最后一块时返回） | ||
| 109 | + */ | ||
| 110 | + private String fullSystemText; | ||
| 111 | + | ||
| 112 | + /** | ||
| 113 | + * 处理进度（0-100） | ||
| 114 | + */ | ||
| 115 | + private Integer progress; | ||
| 116 | + | ||
| 117 | + /** | ||
| 118 | + * 时间戳 | ||
| 119 | + */ | ||
| 120 | + private Long timestamp; | ||
| 121 | + | ||
| 122 | + /** | ||
| 123 | + * 处理耗时（毫秒） | ||
| 124 | + */ | ||
| 125 | + private Long elapsedTime; | ||
| 126 | + | ||
| 127 | + /** | ||
| 128 | + * 扩展字段，用于存储额外的元数据 | ||
| 129 | + */ | ||
| 130 | + private Map<String, Object> metadata; | ||
| 131 | + | ||
| 132 | + /** | ||
| 133 | + * 错误信息（当status=FAILED时） | ||
| 134 | + */ | ||
| 135 | + private String errorMessage; | ||
| 136 | + | ||
| 137 | + /** | ||
| 138 | + * 错误码（当status=FAILED时） | ||
| 139 | + */ | ||
| 140 | + private String errorCode; | ||
| 141 | + | ||
| 142 | + // ============ 便捷方法 ============ | ||
| 143 | + | ||
| 144 | + /** | ||
| 145 | + * 判断是否处理成功 | ||
| 146 | + */ | ||
| 147 | + public boolean isSuccess() { | ||
| 148 | + return code != null && code == 200; | ||
| 149 | + } | ||
| 150 | + | ||
| 151 | + /** | ||
| 152 | + * 判断是否处理中 | ||
| 153 | + */ | ||
| 154 | + public boolean isProcessing() { | ||
| 155 | + return "PROCESSING".equals(status); | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + /** | ||
| 159 | + * 判断是否已完成 | ||
| 160 | + */ | ||
| 161 | + public boolean isCompleted() { | ||
| 162 | + return "COMPLETED".equals(status); | ||
| 163 | + } | ||
| 164 | + | ||
| 165 | + /** | ||
| 166 | + * 判断是否失败 | ||
| 167 | + */ | ||
| 168 | + public boolean isFailed() { | ||
| 169 | + return "FAILED".equals(status); | ||
| 170 | + } | ||
| 171 | + | ||
| 172 | + /** | ||
| 173 | + * 获取完整的文本（AI文本 + 系统文本） | ||
| 174 | + */ | ||
| 175 | + public String getFullText() { | ||
| 176 | + StringBuilder sb = new StringBuilder(); | ||
| 177 | + if (StrUtil.isNotBlank(aiText)) { | ||
| 178 | + sb.append(aiText); | ||
| 179 | + } | ||
| 180 | + if (StrUtil.isNotBlank(systemText)) { | ||
| 181 | + sb.append(systemText); | ||
| 182 | + } | ||
| 183 | + return sb.toString(); | ||
| 184 | + } | ||
| 185 | + | ||
| 186 | + /** | ||
| 187 | + * 创建处理中的响应 | ||
| 188 | + */ | ||
| 189 | + public static AiResponseDTO processing(String requestId, String textFragment, | ||
| 190 | + Integer chunkIndex, Integer totalChunks) { | ||
| 191 | + return AiResponseDTO.builder() | ||
| 192 | + .requestId(requestId) | ||
| 193 | + .code(200) | ||
| 194 | + .message("Processing") | ||
| 195 | + .status("PROCESSING") | ||
| 196 | + .textFragment(textFragment) | ||
| 197 | + .chunkIndex(chunkIndex) | ||
| 198 | + .totalChunks(totalChunks) | ||
| 199 | + .timestamp(System.currentTimeMillis()) | ||
| 200 | + .progress(calculateProgress(chunkIndex, totalChunks)) | ||
| 201 | + .build(); | ||
| 202 | + } | ||
| 203 | + | ||
| 204 | + /** | ||
| 205 | + * 创建完成响应 | ||
| 206 | + */ | ||
| 207 | + public static AiResponseDTO completed(String requestId, String fullAiText, | ||
| 208 | + String fullSystemText, String sSceneName, | ||
| 209 | + String sMethodName, String sReturnType) { | ||
| 210 | + return AiResponseDTO.builder() | ||
| 211 | + .requestId(requestId) | ||
| 212 | + .code(200) | ||
| 213 | + .message("Completed") | ||
| 214 | + .status("COMPLETED") | ||
| 215 | + .fullAiText(fullAiText) | ||
| 216 | + .fullSystemText(fullSystemText) | ||
| 217 | + .aiText(fullAiText) | ||
| 218 | + .systemText(fullSystemText) | ||
| 219 | + .sSceneName(sSceneName) | ||
| 220 | + .sMethodName(sMethodName) | ||
| 221 | + .sReturnType(sReturnType) | ||
| 222 | + .progress(100) | ||
| 223 | + .timestamp(System.currentTimeMillis()) | ||
| 224 | + .isLastChunk(true) | ||
| 225 | + .build(); | ||
| 226 | + } | ||
| 227 | + | ||
| 228 | + /** | ||
| 229 | + * 创建失败响应 | ||
| 230 | + */ | ||
| 231 | + public static AiResponseDTO failed(String requestId, String errorMessage, String errorCode) { | ||
| 232 | + return AiResponseDTO.builder() | ||
| 233 | + .requestId(requestId) | ||
| 234 | + .code(500) | ||
| 235 | + .message("Failed") | ||
| 236 | + .status("FAILED") | ||
| 237 | + .errorMessage(errorMessage) | ||
| 238 | + .errorCode(errorCode) | ||
| 239 | + .timestamp(System.currentTimeMillis()) | ||
| 240 | + .build(); | ||
| 241 | + } | ||
| 242 | + | ||
| 243 | + /** | ||
| 244 | + * 计算进度 | ||
| 245 | + */ | ||
| 246 | + private static Integer calculateProgress(Integer chunkIndex, Integer totalChunks) { | ||
| 247 | + if (chunkIndex == null || totalChunks == null || totalChunks == 0) { | ||
| 248 | + return 0; | ||
| 249 | + } | ||
| 250 | + return (int) ((chunkIndex + 1) * 100.0 / totalChunks); | ||
| 251 | + } | ||
| 33 | } | 252 | } |
| 34 | \ No newline at end of file | 253 | \ No newline at end of file |
src/main/java/com/xly/entity/ToolMeta.java
| @@ -47,4 +47,7 @@ public class ToolMeta { | @@ -47,4 +47,7 @@ public class ToolMeta { | ||
| 47 | private List<ParamRule> paramRuleListCheck;//需要校验 | 47 | private List<ParamRule> paramRuleListCheck;//需要校验 |
| 48 | private List<ParamRule> paramRuleListAll;//所有的 | 48 | private List<ParamRule> paramRuleListAll;//所有的 |
| 49 | private LocalDateTime tMakeDate; | 49 | private LocalDateTime tMakeDate; |
| 50 | + private String sVectorfiled; | ||
| 51 | + private String sVectorjson; | ||
| 52 | + | ||
| 50 | } | 53 | } |
src/main/java/com/xly/milvus/bean/CustomSearchResultsWrapper.java
0 → 100644
| 1 | +package com.xly.milvus.bean; | ||
| 2 | + | ||
| 3 | +import io.milvus.grpc.SearchResultData; | ||
| 4 | +import io.milvus.response.SearchResultsWrapper; | ||
| 5 | +import java.util.List; | ||
| 6 | + | ||
| 7 | +/** | ||
| 8 | + * 自定义SearchResultsWrapper，用于访问protected方法 | ||
| 9 | + */ | ||
| 10 | +public class CustomSearchResultsWrapper extends SearchResultsWrapper { | ||
| 11 | + | ||
| 12 | + public CustomSearchResultsWrapper(SearchResultData results) { | ||
| 13 | + super(results); | ||
| 14 | + } | ||
| 15 | + | ||
| 16 | + /** | ||
| 17 | + * 公开访问getOutputFields方法 | ||
| 18 | + */ | ||
| 19 | + public List<String> getOutputFieldsPublic() { | ||
| 20 | + return super.getOutputFields(); | ||
| 21 | + } | ||
| 22 | +} | ||
| 0 | \ No newline at end of file | 23 | \ No newline at end of file |
src/main/java/com/xly/milvus/bean/SimilaritySearchRequest.java
0 → 100644
| 1 | +package com.xly.milvus.bean; | ||
| 2 | + | ||
| 3 | +import lombok.AllArgsConstructor; | ||
| 4 | +import lombok.Builder; | ||
| 5 | +import lombok.Data; | ||
| 6 | +import lombok.NoArgsConstructor; | ||
| 7 | + | ||
| 8 | +import java.util.List; | ||
| 9 | +import java.util.Map; | ||
| 10 | + | ||
| 11 | +/** | ||
| 12 | + * 相似度查询请求实体 | ||
| 13 | + */ | ||
| 14 | +@Data | ||
| 15 | +@Builder | ||
| 16 | +@NoArgsConstructor | ||
| 17 | +@AllArgsConstructor | ||
| 18 | +public class SimilaritySearchRequest { | ||
| 19 | + private List<Float> queryVector; // 查询向量 | ||
| 20 | + private String queryText; // 查询文本（如果有文本转向量服务） | ||
| 21 | + private Integer topK = 10; // 返回数量 | ||
| 22 | + private Double minScore; // 最小相似度得分 | ||
| 23 | + private Double maxScore; // 最大相似度得分 | ||
| 24 | + private String metricType = "IP"; // 距离类型: IP(内积), L2(欧氏距离), COSINE(余弦) | ||
| 25 | + private List<String> outputFields; // 输出字段 | ||
| 26 | + private Map<String, Object> filter; // 过滤条件 | ||
| 27 | + private String partitionName; // 分区名称 | ||
| 28 | + private Boolean withScore = true; // 是否返回得分 | ||
| 29 | + private Boolean withDistance = false; // 是否返回距离 | ||
| 30 | +} | ||
| 0 | \ No newline at end of file | 31 | \ No newline at end of file |
src/main/java/com/xly/milvus/bean/SimilaritySearchResult.java
0 → 100644
| 1 | +package com.xly.milvus.bean; | ||
| 2 | + | ||
| 3 | +import lombok.AllArgsConstructor; | ||
| 4 | +import lombok.Builder; | ||
| 5 | +import lombok.Data; | ||
| 6 | +import lombok.NoArgsConstructor; | ||
| 7 | + | ||
| 8 | +import java.util.Map; | ||
| 9 | + | ||
| 10 | +/** | ||
| 11 | + * 相似度查询结果实体 - 增强版 | ||
| 12 | + */ | ||
| 13 | +@Data | ||
| 14 | +@Builder | ||
| 15 | +@NoArgsConstructor | ||
| 16 | +@AllArgsConstructor | ||
| 17 | +public class SimilaritySearchResult { | ||
| 18 | + private Long id; // 整数ID | ||
| 19 | + private String strId; // 字符串ID | ||
| 20 | + private Float score; // 相似度得分 | ||
| 21 | + private Double normalizedScore; // 归一化后的得分（0-1之间） | ||
| 22 | + private Map<String, Object> fields; // 字段数据 | ||
| 23 | + private String collectionName; // 集合名称 | ||
| 24 | + | ||
| 25 | + /** | ||
| 26 | + * 获取ID（优先返回字符串ID，如果没有则返回整数ID） | ||
| 27 | + */ | ||
| 28 | + public String getId() { | ||
| 29 | + if (strId != null && !strId.isEmpty()) { | ||
| 30 | + return strId; | ||
| 31 | + } | ||
| 32 | + return id != null ? String.valueOf(id) : null; | ||
| 33 | + } | ||
| 34 | + | ||
| 35 | + /** | ||
| 36 | + * 获取字段值 | ||
| 37 | + */ | ||
| 38 | + public Object getField(String fieldName) { | ||
| 39 | + if (fields != null) { | ||
| 40 | + return fields.get(fieldName); | ||
| 41 | + } | ||
| 42 | + return null; | ||
| 43 | + } | ||
| 44 | + | ||
| 45 | + /** | ||
| 46 | + * 获取字符串字段值 | ||
| 47 | + */ | ||
| 48 | + public String getStringField(String fieldName) { | ||
| 49 | + Object value = getField(fieldName); | ||
| 50 | + return value != null ? value.toString() : null; | ||
| 51 | + } | ||
| 52 | + | ||
| 53 | + /** | ||
| 54 | + * 获取整数字段值 | ||
| 55 | + */ | ||
| 56 | + public Integer getIntField(String fieldName) { | ||
| 57 | + Object value = getField(fieldName); | ||
| 58 | + if (value instanceof Number) { | ||
| 59 | + return ((Number) value).intValue(); | ||
| 60 | + } | ||
| 61 | + return null; | ||
| 62 | + } | ||
| 63 | +} | ||
| 0 | \ No newline at end of file | 64 | \ No newline at end of file |
src/main/java/com/xly/milvus/config/EmbeddingConfig.java
0 → 100644
| 1 | +package com.xly.milvus.config; | ||
| 2 | + | ||
| 3 | +import dev.langchain4j.model.embedding.EmbeddingModel; | ||
| 4 | +import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; | ||
| 5 | +import org.springframework.context.annotation.Bean; | ||
| 6 | +import org.springframework.context.annotation.Configuration; | ||
| 7 | + | ||
| 8 | +/** | ||
| 9 | + * LangChain4j 嵌入模型配置类 | ||
| 10 | + */ | ||
| 11 | +@Configuration | ||
| 12 | +public class EmbeddingConfig { | ||
| 13 | + | ||
| 14 | + @Bean | ||
| 15 | + public EmbeddingModel embeddingModel() { | ||
| 16 | + // 使用 All-MiniLM-L6-v2 嵌入模型 | ||
| 17 | + // 这是一个轻量级的模型，维度为384，适合在本地运行 | ||
| 18 | + return new AllMiniLmL6V2EmbeddingModel(); | ||
| 19 | + } | ||
| 20 | +} | ||
| 0 | \ No newline at end of file | 21 | \ No newline at end of file |
src/main/java/com/xly/milvus/config/MilvusConfig.java
0 → 100644
| 1 | +package com.xly.milvus.config; | ||
| 2 | + | ||
| 3 | +import io.milvus.v2.client.ConnectConfig; | ||
| 4 | +import io.milvus.v2.client.MilvusClientV2; | ||
| 5 | +import lombok.extern.slf4j.Slf4j; | ||
| 6 | +import org.springframework.beans.factory.annotation.Value; | ||
| 7 | +import org.springframework.context.annotation.Bean; | ||
| 8 | +import org.springframework.context.annotation.Configuration; | ||
| 9 | +import org.springframework.retry.annotation.EnableRetry; | ||
| 10 | +import org.springframework.retry.support.RetryTemplate; | ||
| 11 | + | ||
| 12 | +@Slf4j | ||
| 13 | +@Configuration | ||
| 14 | +@EnableRetry | ||
| 15 | +public class MilvusConfig { | ||
| 16 | + | ||
| 17 | + @Value("${milvus.host:localhost}") | ||
| 18 | + private String host; | ||
| 19 | + | ||
| 20 | + @Value("${milvus.port:19530}") | ||
| 21 | + private Integer port; | ||
| 22 | + | ||
| 23 | + @Value("${milvus.database:default}") | ||
| 24 | + private String database; | ||
| 25 | + | ||
| 26 | + @Value("${milvus.username:}") | ||
| 27 | + private String username; | ||
| 28 | + | ||
| 29 | + @Value("${milvus.password:}") | ||
| 30 | + private String password; | ||
| 31 | + | ||
| 32 | + @Value("${milvus.connect-timeout:10000}") | ||
| 33 | + private Long connectTimeout; | ||
| 34 | + | ||
| 35 | + @Value("${milvus.rpc-deadline:10000}") | ||
| 36 | + private Long rpcDeadline; | ||
| 37 | + | ||
| 38 | + @Value("${milvus.keep-alive-time:300000}") | ||
| 39 | + private Long keepAliveTime; | ||
| 40 | + | ||
| 41 | + @Value("${milvus.keep-alive-timeout:5000}") | ||
| 42 | + private Long keepAliveTimeout; | ||
| 43 | + | ||
| 44 | + @Value("${milvus.secure:false}") | ||
| 45 | + private Boolean secure; | ||
| 46 | + | ||
| 47 | + @Value("${milvus.validate-on-startup:true}") | ||
| 48 | + private boolean validateOnStartup; | ||
| 49 | + | ||
| 50 | + @Value("${milvus.fail-on-connect-error:false}") | ||
| 51 | + private boolean failOnConnectError; | ||
| 52 | + | ||
| 53 | + @Value("${milvus.enable-precheck:true}") | ||
| 54 | + private boolean enablePrecheck; | ||
| 55 | + | ||
| 56 | + @Bean(destroyMethod = "close") | ||
| 57 | + public MilvusClientV2 milvusClient() { | ||
| 58 | + try { | ||
| 59 | + String uri = String.format("http://%s:%d", host, port); | ||
| 60 | + // 构建连接配置 | ||
| 61 | + ConnectConfig.ConnectConfigBuilder configBuilder = ConnectConfig.builder() | ||
| 62 | + .uri(uri) | ||
| 63 | + .dbName("default") | ||
| 64 | + .connectTimeoutMs(connectTimeout) | ||
| 65 | + .rpcDeadlineMs(rpcDeadline) | ||
| 66 | + .keepAliveTimeMs(keepAliveTime) | ||
| 67 | + .keepAliveTimeoutMs(keepAliveTimeout) | ||
| 68 | + .keepAliveWithoutCalls(true) | ||
| 69 | + .secure(secure) | ||
| 70 | + .enablePrecheck(enablePrecheck); | ||
| 71 | + | ||
| 72 | + // 添加认证信息（如果有） | ||
| 73 | + if (username != null && !username.isEmpty()) { | ||
| 74 | + configBuilder.username(username); | ||
| 75 | + if (password != null) { | ||
| 76 | + configBuilder.password(password); | ||
| 77 | + } | ||
| 78 | + } | ||
| 79 | + | ||
| 80 | + ConnectConfig connectConfig = configBuilder.build(); | ||
| 81 | + | ||
| 82 | + // 创建客户端 | ||
| 83 | + MilvusClientV2 client = new MilvusClientV2(connectConfig); | ||
| 84 | + // 检查xlyerp数据库是否存在，如果不存在则创建 | ||
| 85 | + var databases = client.listDatabases(); | ||
| 86 | + if (!databases.getDatabaseNames().contains(database)) { | ||
| 87 | + log.info("数据库 xlyerp 不存在，正在创建..."); | ||
| 88 | + client.createDatabase( | ||
| 89 | + io.milvus.v2.service.database.request.CreateDatabaseReq.builder() | ||
| 90 | + .databaseName(database) | ||
| 91 | + .build() | ||
| 92 | + ); | ||
| 93 | + log.info("数据库 xlyerp 创建成功"); | ||
| 94 | + } | ||
| 95 | + | ||
| 96 | + // 切换到xlyerp数据库 | ||
| 97 | + client.useDatabase (database); | ||
| 98 | + log.info("切换到数据库: {}",database); | ||
| 99 | + | ||
| 100 | + // 启动验证 | ||
| 101 | + if (validateOnStartup) { | ||
| 102 | + validateConnection(client); | ||
| 103 | + } | ||
| 104 | + return client; | ||
| 105 | + } catch (Exception e) { | ||
| 106 | + log.error("创建Milvus客户端时发生异常: {}", e.getMessage(), e); | ||
| 107 | + throw new RuntimeException("无法创建Milvus客户端", e); | ||
| 108 | + } | ||
| 109 | + | ||
| 110 | + } | ||
| 111 | + | ||
| 112 | + /** | ||
| 113 | + * 验证Milvus连接 - 基于源码中可用的方法 | ||
| 114 | + */ | ||
| 115 | + private void validateConnection(MilvusClientV2 client) { | ||
| 116 | + try { | ||
| 117 | + // 方法1: 检查服务器版本（源码中存在getServerVersion()） | ||
| 118 | + String serverVersion = client.getServerVersion(); | ||
| 119 | + | ||
| 120 | + // 方法2: 列出数据库（源码中存在listDatabases()） | ||
| 121 | + var databases = client.listDatabases(); | ||
| 122 | + | ||
| 123 | + // 方法3: 检查健康状态（源码中存在checkHealth()） | ||
| 124 | + var health = client.checkHealth(); | ||
| 125 | + | ||
| 126 | + System.out.println("✅ Milvus连接成功"); | ||
| 127 | + System.out.println(" - 服务器版本: " + serverVersion); | ||
| 128 | + System.out.println(" - 数据库: " + databases); | ||
| 129 | + System.out.println(" - 健康状态: " + health); | ||
| 130 | + | ||
| 131 | + // 验证指定数据库是否存在 | ||
| 132 | + if (database != null && !database.isEmpty()) { | ||
| 133 | + boolean dbExists = databases.getDatabaseNames().contains(database); | ||
| 134 | + if (!dbExists && !"default".equals(database)) { | ||
| 135 | + System.err.println("⚠️ 警告: 指定数据库 '" + database + "' 不存在"); | ||
| 136 | + } | ||
| 137 | + } | ||
| 138 | + | ||
| 139 | + } catch (Exception e) { | ||
| 140 | + String errorMsg = String.format("❌ Milvus连接失败: %s:%d - %s",host, port, e.getMessage()); | ||
| 141 | + | ||
| 142 | + if (failOnConnectError) { | ||
| 143 | + throw new IllegalStateException(errorMsg, e); | ||
| 144 | + } else { | ||
| 145 | + System.err.println(errorMsg); | ||
| 146 | + System.err.println(" ⚠️ 应用将继续启动，但Milvus功能可能不可用"); | ||
| 147 | + } | ||
| 148 | + } | ||
| 149 | + } | ||
| 150 | + | ||
| 151 | + /** | ||
| 152 | + * 配置重试模板（可选） | ||
| 153 | + */ | ||
| 154 | + @Bean | ||
| 155 | + public RetryTemplate milvusRetryTemplate() { | ||
| 156 | + return RetryTemplate.builder() | ||
| 157 | + .maxAttempts(3) | ||
| 158 | + .exponentialBackoff(1000, 2, 10000) | ||
| 159 | + .retryOn(Exception.class) | ||
| 160 | + .build(); | ||
| 161 | + } | ||
| 162 | +} | ||
| 0 | \ No newline at end of file | 163 | \ No newline at end of file |
src/main/java/com/xly/milvus/config/MilvusHealthIndicator.java
0 → 100644
| 1 | +package com.xly.milvus.config; | ||
| 2 | + | ||
| 3 | +import io.milvus.v2.client.MilvusClientV2; | ||
| 4 | +import org.springframework.beans.factory.annotation.Autowired; | ||
| 5 | +import org.springframework.boot.actuate.health.Health; | ||
| 6 | +import org.springframework.boot.actuate.health.HealthIndicator; | ||
| 7 | +import org.springframework.stereotype.Component; | ||
| 8 | + | ||
| 9 | +import java.util.HashMap; | ||
| 10 | +import java.util.Map; | ||
| 11 | + | ||
| 12 | +@Component | ||
| 13 | +public class MilvusHealthIndicator implements HealthIndicator { | ||
| 14 | + | ||
| 15 | + @Autowired(required = false) | ||
| 16 | + private MilvusClientV2 milvusClient; | ||
| 17 | + | ||
| 18 | + @Override | ||
| 19 | + public Health health() { | ||
| 20 | + if (milvusClient == null) { | ||
| 21 | + return Health.down() | ||
| 22 | + .withDetail("error", "Milvus客户端未初始化") | ||
| 23 | + .build(); | ||
| 24 | + } | ||
| 25 | + | ||
| 26 | + try { | ||
| 27 | + // 检查客户端是否就绪（源码中存在clientIsReady()） | ||
| 28 | + boolean isReady = milvusClient.clientIsReady(); | ||
| 29 | + | ||
| 30 | + if (!isReady) { | ||
| 31 | + return Health.down() | ||
| 32 | + .withDetail("error", "客户端未就绪") | ||
| 33 | + .build(); | ||
| 34 | + } | ||
| 35 | + | ||
| 36 | + long startTime = System.currentTimeMillis(); | ||
| 37 | + | ||
| 38 | + // 执行健康检查 - 使用源码中存在的方法 | ||
| 39 | + String serverVersion = milvusClient.getServerVersion(); | ||
| 40 | + var databases = milvusClient.listDatabases(); | ||
| 41 | + var healthCheck = milvusClient.checkHealth(); | ||
| 42 | + | ||
| 43 | + long responseTime = System.currentTimeMillis() - startTime; | ||
| 44 | + | ||
| 45 | + Map<String, Object> details = new HashMap<>(); | ||
| 46 | + details.put("serverVersion", serverVersion); | ||
| 47 | + details.put("database", milvusClient.currentUsedDatabase()); | ||
| 48 | + details.put("databases", databases.getDatabaseNames()); | ||
| 49 | + details.put("healthStatus", healthCheck.getIsHealthy()); | ||
| 50 | + details.put("responseTime", responseTime + "ms"); | ||
| 51 | + details.put("clientReady", isReady); | ||
| 52 | + | ||
| 53 | + if (healthCheck.getQuotaStates() != null) { | ||
| 54 | + details.put("quotaStates", healthCheck.getQuotaStates()); | ||
| 55 | + } | ||
| 56 | + | ||
| 57 | + return Health.up() | ||
| 58 | + .withDetails(details) | ||
| 59 | + .build(); | ||
| 60 | + | ||
| 61 | + } catch (Exception e) { | ||
| 62 | + return Health.down() | ||
| 63 | + .withDetail("error", e.getMessage()) | ||
| 64 | + .withDetail("clientReady", milvusClient.clientIsReady()) | ||
| 65 | + .build(); | ||
| 66 | + } | ||
| 67 | + } | ||
| 68 | +} | ||
| 0 | \ No newline at end of file | 69 | \ No newline at end of file |
src/main/java/com/xly/milvus/config/MilvusRetryConfig.java
0 → 100644
| 1 | +package com.xly.milvus.config; | ||
| 2 | + | ||
| 3 | +import io.milvus.v2.client.RetryConfig; | ||
| 4 | +import org.springframework.beans.factory.annotation.Value; | ||
| 5 | +import org.springframework.context.annotation.Bean; | ||
| 6 | +import org.springframework.context.annotation.Configuration; | ||
| 7 | + | ||
| 8 | +@Configuration | ||
| 9 | +public class MilvusRetryConfig { | ||
| 10 | + | ||
| 11 | + @Value("${milvus.retry.max-retry-times:75}") | ||
| 12 | + private int maxRetryTimes; | ||
| 13 | + | ||
| 14 | + @Value("${milvus.retry.initial-backoff-ms:10}") | ||
| 15 | + private long initialBackOffMs; | ||
| 16 | + | ||
| 17 | + @Value("${milvus.retry.max-backoff-ms:3000}") | ||
| 18 | + private long maxBackOffMs; | ||
| 19 | + | ||
| 20 | + @Value("${milvus.retry.backoff-multiplier:3}") | ||
| 21 | + private int backOffMultiplier; | ||
| 22 | + | ||
| 23 | + @Value("${milvus.retry.retry-on-rate-limit:true}") | ||
| 24 | + private boolean retryOnRateLimit; | ||
| 25 | + | ||
| 26 | + @Value("${milvus.retry.max-retry-timeout-ms:0}") | ||
| 27 | + private long maxRetryTimeoutMs; | ||
| 28 | + | ||
| 29 | + @Value("${milvus.retry.enabled:true}") | ||
| 30 | + private boolean retryEnabled; | ||
| 31 | + | ||
| 32 | + /** | ||
| 33 | + * 方法名改为 createRetryConfig，避免与类名冲突 | ||
| 34 | + */ | ||
| 35 | + @Bean | ||
| 36 | + public RetryConfig createRetryConfig() { | ||
| 37 | + if (!retryEnabled) { | ||
| 38 | + return RetryConfig.builder() | ||
| 39 | + .maxRetryTimes(1) | ||
| 40 | + .initialBackOffMs(0) | ||
| 41 | + .maxBackOffMs(0) | ||
| 42 | + .backOffMultiplier(1) | ||
| 43 | + .retryOnRateLimit(false) | ||
| 44 | + .maxRetryTimeoutMs(0) | ||
| 45 | + .build(); | ||
| 46 | + } | ||
| 47 | + | ||
| 48 | + return RetryConfig.builder() | ||
| 49 | + .maxRetryTimes(maxRetryTimes) | ||
| 50 | + .initialBackOffMs(initialBackOffMs) | ||
| 51 | + .maxBackOffMs(maxBackOffMs) | ||
| 52 | + .backOffMultiplier(backOffMultiplier) | ||
| 53 | + .retryOnRateLimit(retryOnRateLimit) | ||
| 54 | + .maxRetryTimeoutMs(maxRetryTimeoutMs) | ||
| 55 | + .build(); | ||
| 56 | + } | ||
| 57 | +} | ||
| 0 | \ No newline at end of file | 58 | \ No newline at end of file |
src/main/java/com/xly/milvus/config/MilvusStartupValidator.java
0 → 100644
| 1 | +package com.xly.milvus.config; | ||
| 2 | + | ||
| 3 | +import io.milvus.v2.client.MilvusClientV2; | ||
| 4 | +import io.milvus.v2.service.collection.request.HasCollectionReq; | ||
| 5 | +import lombok.extern.slf4j.Slf4j; | ||
| 6 | +import org.slf4j.Logger; | ||
| 7 | +import org.slf4j.LoggerFactory; | ||
| 8 | +import org.springframework.beans.factory.annotation.Autowired; | ||
| 9 | +import org.springframework.beans.factory.annotation.Value; | ||
| 10 | +import org.springframework.boot.ApplicationArguments; | ||
| 11 | +import org.springframework.boot.ApplicationRunner; | ||
| 12 | +import org.springframework.core.annotation.Order; | ||
| 13 | +import org.springframework.stereotype.Component; | ||
| 14 | + | ||
| 15 | +import java.util.List; | ||
| 16 | + | ||
| 17 | +@Slf4j | ||
| 18 | +@Component | ||
| 19 | +@Order(1) | ||
| 20 | +public class MilvusStartupValidator implements ApplicationRunner { | ||
| 21 | + | ||
| 22 | + @Autowired | ||
| 23 | + private MilvusClientV2 milvusClient; | ||
| 24 | + | ||
| 25 | + @Value("${milvus.validation.collections:}") | ||
| 26 | + private List<String> validationCollections; | ||
| 27 | + | ||
| 28 | + @Value("${milvus.validation.timeout:5000}") | ||
| 29 | + private long validationTimeout; | ||
| 30 | + | ||
| 31 | + @Override | ||
| 32 | + public void run(ApplicationArguments args) throws Exception { | ||
| 33 | + log.info("开始Milvus启动验证..."); | ||
| 34 | + | ||
| 35 | + try { | ||
| 36 | + // 1. 验证客户端就绪状态 | ||
| 37 | + boolean isReady = milvusClient.clientIsReady(); | ||
| 38 | + log.info("客户端就绪状态: {}", isReady); | ||
| 39 | + | ||
| 40 | + // 2. 获取服务器版本 | ||
| 41 | + String serverVersion = milvusClient.getServerVersion(); | ||
| 42 | + log.info("Milvus服务器版本: {}", serverVersion); | ||
| 43 | + | ||
| 44 | + // 3. 列出所有数据库 | ||
| 45 | + var databases = milvusClient.listDatabases(); | ||
| 46 | + log.info("可用数据库: {}", databases.getDatabaseNames()); | ||
| 47 | + | ||
| 48 | + // 4. 检查当前使用的数据库 | ||
| 49 | + String currentDb = milvusClient.currentUsedDatabase(); | ||
| 50 | + log.info("当前数据库: {}", currentDb); | ||
| 51 | + | ||
| 52 | + // 5. 健康检查 | ||
| 53 | + var healthCheck = milvusClient.checkHealth(); | ||
| 54 | + log.info("健康状态: {}", healthCheck.getIsHealthy()); | ||
| 55 | + if (healthCheck.getQuotaStates() != null && !healthCheck.getQuotaStates().isEmpty()) { | ||
| 56 | + log.info("配额状态: {}", healthCheck.getQuotaStates()); | ||
| 57 | + } | ||
| 58 | + | ||
| 59 | + // 6. 验证指定的集合（可选） | ||
| 60 | + if (validationCollections != null && !validationCollections.isEmpty()) { | ||
| 61 | + for (String collectionName : validationCollections) { | ||
| 62 | + boolean exists = milvusClient.hasCollection( | ||
| 63 | + HasCollectionReq.builder() | ||
| 64 | + .collectionName(collectionName) | ||
| 65 | + .build() | ||
| 66 | + ); | ||
| 67 | + log.info("集合 '{}' 存在: {}", collectionName, exists); | ||
| 68 | + } | ||
| 69 | + } | ||
| 70 | + | ||
| 71 | + log.info("Milvus启动验证完成"); | ||
| 72 | + | ||
| 73 | + } catch (Exception e) { | ||
| 74 | + log.error("Milvus启动验证失败: {}", e.getMessage(), e); | ||
| 75 | + // 可以根据配置决定是否抛出异常 | ||
| 76 | + // throw new RuntimeException("Milvus验证失败", e); | ||
| 77 | + } | ||
| 78 | + } | ||
| 79 | +} | ||
| 0 | \ No newline at end of file | 80 | \ No newline at end of file |
src/main/java/com/xly/milvus/config/SearchResult.java
0 → 100644
| 1 | +package com.xly.milvus.config; | ||
| 2 | + | ||
| 3 | +import lombok.AllArgsConstructor; | ||
| 4 | +import lombok.Builder; | ||
| 5 | +import lombok.Data; | ||
| 6 | +import lombok.NoArgsConstructor; | ||
| 7 | + | ||
| 8 | +import java.util.Map; | ||
| 9 | + | ||
| 10 | +/** | ||
| 11 | + * 搜索结果实体 | ||
| 12 | + */ | ||
| 13 | +@Data | ||
| 14 | +@Builder | ||
| 15 | +@NoArgsConstructor | ||
| 16 | +@AllArgsConstructor | ||
| 17 | +public class SearchResult { | ||
| 18 | + private Long id; | ||
| 19 | + private Float score; | ||
| 20 | + private Map<String, Object> fields; | ||
| 21 | + private String collectionName; | ||
| 22 | +} | ||
| 0 | \ No newline at end of file | 23 | \ No newline at end of file |
src/main/java/com/xly/milvus/service/AiGlobalAgentQuestionSqlEmitterService.java
0 → 100644
| 1 | +package com.xly.milvus.service; | ||
| 2 | + | ||
| 3 | + | ||
| 4 | +import java.util.Map; | ||
| 5 | + | ||
| 6 | +public interface AiGlobalAgentQuestionSqlEmitterService { | ||
| 7 | + | ||
| 8 | + /*** | ||
| 9 | + * @Author 钱豹 | ||
| 10 | + * @Date 15:27 2026/3/19 | ||
| 11 | + * @Param [data, sQuestion, sSqlContent, collectionName] | ||
| 12 | + * @return void | ||
| 13 | + * @Description 插入向量库 | ||
| 14 | + **/ | ||
| 15 | + void addAiGlobalAgentQuestionSqlEmitter(String sKey,Map<String,Object> data, String sQuestion, String sSqlContent, String collectionName); | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + Map<String, Object> queryAiGlobalAgentQuestionSqlEmitter(String searchText, String collectionName); | ||
| 19 | + | ||
| 20 | +} | ||
| 0 | \ No newline at end of file | 21 | \ No newline at end of file |
src/main/java/com/xly/milvus/service/EmbeddingService.java
0 → 100644
| 1 | +package com.xly.milvus.service; | ||
| 2 | + | ||
| 3 | +import java.util.List; | ||
| 4 | + | ||
| 5 | +public interface EmbeddingService { | ||
| 6 | + | ||
| 7 | + /** | ||
| 8 | + * 生成单个文本的向量 | ||
| 9 | + */ | ||
| 10 | + public List<Float> generateEmbedding(String text); | ||
| 11 | + | ||
| 12 | + /** | ||
| 13 | + * 批量生成向量（高效版） | ||
| 14 | + * 利用 LangChain4j 内置的并行化能力，显著提升性能 | ||
| 15 | + */ | ||
| 16 | + public List<List<Float>> generateEmbeddings(List<String> texts); | ||
| 17 | +} | ||
| 0 | \ No newline at end of file | 18 | \ No newline at end of file |
src/main/java/com/xly/milvus/service/MilvusService.java
0 → 100644
| 1 | +package com.xly.milvus.service; | ||
| 2 | + | ||
| 3 | +import com.xly.tts.bean.TTSResponseDTO; | ||
| 4 | + | ||
| 5 | +import java.util.ArrayList; | ||
| 6 | +import java.util.HashMap; | ||
| 7 | +import java.util.List; | ||
| 8 | +import java.util.Map; | ||
| 9 | + | ||
| 10 | +/** | ||
| 11 | + * 向量化服务接口 | ||
| 12 | + */ | ||
| 13 | +public interface MilvusService { | ||
| 14 | + | ||
| 15 | + /*** | ||
| 16 | + * @Author 钱豹 | ||
| 17 | + * @Date 22:17 2026/3/24 | ||
| 18 | + * @Param | ||
| 19 | + * @return | ||
| 20 | + * @Description 初始化数据 | ||
| 21 | + **/ | ||
| 22 | + | ||
| 23 | + TTSResponseDTO initDataToMilvus(Map<String, Object> reqMap); | ||
| 24 | + | ||
| 25 | + | ||
| 26 | + /** | ||
| 27 | + * 创建集合（如果不存在） | ||
| 28 | + */ | ||
| 29 | + void createCollectionIfNotExists(String collectionName, String sVectorfiled, String sVectorjson, Boolean bRset); | ||
| 30 | + | ||
| 31 | + | ||
| 32 | + /*** | ||
| 33 | + * @Author 钱豹 | ||
| 34 | + * @Date 21:39 2026/3/24 | ||
| 35 | + * @Param [collectionName, sVectorfiled, sVectorjson, data] | ||
| 36 | + * @return long | ||
| 37 | + * @Description 批量插入数据 | ||
| 38 | + **/ | ||
| 39 | + long addDataToCollection(String collectionName, String sVectorfiled, String sVectorjson, List<Map<String, Object>> data); | ||
| 40 | + | ||
| 41 | + /*** | ||
| 42 | + * @Author 钱豹 | ||
| 43 | + * @Date 10:39 2026/3/25 | ||
| 44 | + * @Param | ||
| 45 | + * @return | ||
| 46 | + * @Description 向量库查询 | ||
| 47 | + **/ | ||
| 48 | + List<Map<String, Object>> getDataToCollection(String collectionName, String milvusFilter,String searchText,Integer size,List<String> fields); | ||
| 49 | + | ||
| 50 | + /*** | ||
| 51 | + * @Author 钱豹 | ||
| 52 | + * @Date 10:56 2026/3/25 | ||
| 53 | + * @Param [sVectorfiled] | ||
| 54 | + * @return java.util.Map<java.lang.String,java.lang.Object> | ||
| 55 | + * @Description 获取配置 | ||
| 56 | + **/ | ||
| 57 | + Map<String,Object> getMilvusFiled(String sVectorfiled); | ||
| 58 | +} | ||
| 0 | \ No newline at end of file | 59 | \ No newline at end of file |
src/main/java/com/xly/milvus/service/VectorizationService.java
0 → 100644
| 1 | +package com.xly.milvus.service; | ||
| 2 | + | ||
| 3 | +import java.util.List; | ||
| 4 | + | ||
| 5 | +/** | ||
| 6 | + * 向量化服务接口 | ||
| 7 | + */ | ||
| 8 | +public interface VectorizationService { | ||
| 9 | + | ||
| 10 | + /** | ||
| 11 | + * 将文本向量化 | ||
| 12 | + * @param text 文本内容 | ||
| 13 | + * @return 向量数组 | ||
| 14 | + */ | ||
| 15 | + List<Float> textToVector(String text); | ||
| 16 | + | ||
| 17 | + /** | ||
| 18 | + * 批量向量化 | ||
| 19 | + * @param texts 文本列表 | ||
| 20 | + * @return 向量列表 | ||
| 21 | + */ | ||
| 22 | + List<List<Float>> batchTextToVector(List<String> texts); | ||
| 23 | +} | ||
| 0 | \ No newline at end of file | 24 | \ No newline at end of file |
src/main/java/com/xly/milvus/service/impl/AiGlobalAgentQuestionSqlEmitterServiceImpl.java
0 → 100644
| 1 | +package com.xly.milvus.service.impl; | ||
| 2 | + | ||
| 3 | +import cn.hutool.core.collection.ConcurrentHashSet; | ||
| 4 | +import cn.hutool.core.util.ObjectUtil; | ||
| 5 | +import com.google.gson.JsonArray; | ||
| 6 | +import com.google.gson.JsonObject; | ||
| 7 | +import com.xly.milvus.service.AiGlobalAgentQuestionSqlEmitterService; | ||
| 8 | +import com.xly.milvus.service.VectorizationService; | ||
| 9 | +import com.xly.milvus.util.MapToJsonConverter; | ||
| 10 | +import io.milvus.v2.client.MilvusClientV2; | ||
| 11 | +import io.milvus.v2.common.ConsistencyLevel; | ||
| 12 | +import io.milvus.v2.common.DataType; | ||
| 13 | +import io.milvus.v2.common.IndexBuildState; | ||
| 14 | +import io.milvus.v2.common.IndexParam; | ||
| 15 | +import io.milvus.v2.service.collection.request.*; | ||
| 16 | +import io.milvus.v2.service.index.request.CreateIndexReq; | ||
| 17 | +import io.milvus.v2.service.index.request.DescribeIndexReq; | ||
| 18 | +import io.milvus.v2.service.index.request.DropIndexReq; | ||
| 19 | +import io.milvus.v2.service.index.response.DescribeIndexResp; | ||
| 20 | +import io.milvus.v2.service.vector.request.InsertReq; | ||
| 21 | +import io.milvus.v2.service.vector.request.SearchReq; | ||
| 22 | +import io.milvus.v2.service.vector.request.data.FloatVec; | ||
| 23 | +import io.milvus.v2.service.vector.response.InsertResp; | ||
| 24 | +import io.milvus.v2.service.vector.response.SearchResp; | ||
| 25 | +import lombok.RequiredArgsConstructor; | ||
| 26 | +import lombok.extern.slf4j.Slf4j; | ||
| 27 | +import org.springframework.beans.factory.annotation.Value; | ||
| 28 | +import org.springframework.stereotype.Service; | ||
| 29 | + | ||
| 30 | +import java.util.*; | ||
| 31 | + | ||
| 32 | +@Slf4j | ||
| 33 | +@Service("aiGlobalAgentQuestionSqlEmitterService") | ||
| 34 | +@RequiredArgsConstructor | ||
| 35 | +public class AiGlobalAgentQuestionSqlEmitterServiceImpl implements AiGlobalAgentQuestionSqlEmitterService { | ||
| 36 | + | ||
| 37 | + private final MilvusClientV2 milvusClient; | ||
| 38 | + private final VectorizationService vectorizationService; | ||
| 39 | + | ||
| 40 | + // 或者从配置文件读取 | ||
| 41 | + @Value("${milvus.vector.dimension:384}") | ||
| 42 | + private int VECTOR_DIM; | ||
| 43 | + | ||
| 44 | + // 缓存已加载的集合 | ||
| 45 | + private final Set<String> loadedCollections = new ConcurrentHashSet<>(); | ||
| 46 | + | ||
| 47 | + /*** | ||
| 48 | + * @Author 钱豹 | ||
| 49 | + * @Date 13:06 2026/3/19 | ||
| 50 | + * @Param [] | ||
| 51 | + * @return void | ||
| 52 | + * @Description 插入数据 | ||
| 53 | + **/ | ||
| 54 | + @Override | ||
| 55 | + public void addAiGlobalAgentQuestionSqlEmitter(String sKey,Map<String,Object> data,String sQuestion,String sSqlContent,String collectionName) { | ||
| 56 | + // 向量化 | ||
| 57 | + List<Float> vector = vectorizationService.textToVector(sKey); | ||
| 58 | + | ||
| 59 | + if (vector == null || vector.isEmpty()) { | ||
| 60 | + throw new RuntimeException("向量化失败"); | ||
| 61 | + } | ||
| 62 | + | ||
| 63 | + // 2. 转换为Milvus格式 | ||
| 64 | + JsonObject row = convertToMilvusRow(data, vector,sQuestion,sSqlContent,sKey); | ||
| 65 | + | ||
| 66 | + //创建集合 | ||
| 67 | +// createCollection(collectionName); | ||
| 68 | + createCollectionIfNotExists(collectionName); | ||
| 69 | + | ||
| 70 | + // 3. 插入到Milvus | ||
| 71 | + InsertReq insertReq = InsertReq.builder() | ||
| 72 | + .collectionName(collectionName) | ||
| 73 | + .data(List.of(row)) | ||
| 74 | + .build(); | ||
| 75 | + | ||
| 76 | + InsertResp insertResp = milvusClient.insert(insertReq); | ||
| 77 | + System.out.println("成功插入 " + insertResp.getInsertCnt() + " 条数据"); | ||
| 78 | + System.out.println(" - 数据预览:"); | ||
| 79 | + } | ||
| 80 | + | ||
| 81 | + @Override | ||
| 82 | + public Map<String, Object> queryAiGlobalAgentQuestionSqlEmitter(String searchText, String collectionName) { | ||
| 83 | + Map<String, Object> result = new HashMap<>(); | ||
| 84 | + log.info("开始相似度查询: collection={}, searchText={}", collectionName, searchText); | ||
| 85 | + // 2. 设置范围搜索参数 | ||
| 86 | + Map<String, Object> searchParams = new HashMap<>(); | ||
| 87 | + searchParams.put("nprobe", 10); | ||
| 88 | + // 对于 IP 度量，相似度范围在 [minScore, maxScore] | ||
| 89 | + searchParams.put("radius", 0.9); // 最小相似度 | ||
| 90 | + searchParams.put("range_filter", 1); // 最大相似度 | ||
| 91 | + // 1. 确保集合已加载 | ||
| 92 | + ensureCollectionLoaded(collectionName); | ||
| 93 | + | ||
| 94 | + // 1. 向量化搜索文本 | ||
| 95 | + List<Float> vectorList = vectorizationService.textToVector(searchText); | ||
| 96 | + if (vectorList == null || vectorList.isEmpty()) { | ||
| 97 | + throw new RuntimeException("向量化失败"); | ||
| 98 | + } | ||
| 99 | + // 2. 转换为 float[] | ||
| 100 | + float[] floatArray = new float[vectorList.size()]; | ||
| 101 | + for (int i = 0; i < vectorList.size(); i++) { | ||
| 102 | + floatArray[i] = vectorList.get(i); | ||
| 103 | + } | ||
| 104 | + // 查询最近插入的数据（按时间倒序） | ||
| 105 | +// QueryReq queryReq = QueryReq.builder() | ||
| 106 | +// .collectionName(collectionName) | ||
| 107 | +// .outputFields(Arrays.asList("sQuestion", "sSqlContent", "data_id", "create_time","metadata")) | ||
| 108 | +// .limit(100) | ||
| 109 | +// .build(); | ||
| 110 | +// QueryResp queryResp = milvusClient.query(queryReq); | ||
| 111 | + | ||
| 112 | + // 3. 创建 Milvus FloatVec 对象 | ||
| 113 | + FloatVec floatVec = new FloatVec(floatArray); | ||
| 114 | + // 4. 构建搜索请求 | ||
| 115 | + SearchReq searchReq = SearchReq.builder() | ||
| 116 | + .collectionName(collectionName) | ||
| 117 | + .data(Collections.singletonList(floatVec)) | ||
| 118 | + .annsField("vector") // 向量字段名 | ||
| 119 | + .topK(10) // 返回最相似的10条 | ||
| 120 | + .metricType(IndexParam.MetricType.IP) // 内积相似度 | ||
| 121 | + .outputFields(Arrays.asList("sQuestion", "sSqlContent", "data_id", "create_time","metadata")) | ||
| 122 | + .searchParams(searchParams) | ||
| 123 | + .build(); | ||
| 124 | + // 5. 执行搜索 | ||
| 125 | + SearchResp searchResp = milvusClient.search(searchReq); | ||
| 126 | + // 6. 处理结果 | ||
| 127 | + List<List<SearchResp.SearchResult>> searchResults = searchResp.getSearchResults(); | ||
| 128 | + if(ObjectUtil.isEmpty(searchResults)){ | ||
| 129 | + return result; | ||
| 130 | + } | ||
| 131 | + List<SearchResp.SearchResult> firstResultList = searchResults.get(0); | ||
| 132 | + if(ObjectUtil.isEmpty(firstResultList)){ | ||
| 133 | + return result; | ||
| 134 | + } | ||
| 135 | + firstResultList.sort((a, b) -> Float.compare(b.getScore(), a.getScore())); | ||
| 136 | + SearchResp.SearchResult item = firstResultList.get(0); | ||
| 137 | + Map<String, Object> itemMap = new HashMap<>(); | ||
| 138 | + itemMap.put("score", item.getScore()); | ||
| 139 | + itemMap.put("id", item.getId()); | ||
| 140 | + itemMap.putAll(item.getEntity()); | ||
| 141 | + return itemMap; | ||
| 142 | + } | ||
| 143 | + | ||
| 144 | + /** | ||
| 145 | + * 确保集合已加载 | ||
| 146 | + */ | ||
| 147 | + private void ensureCollectionLoaded(String collectionName) { | ||
| 148 | + try { | ||
| 149 | + // 如果已经加载过，直接返回 | ||
| 150 | + if (loadedCollections.contains(collectionName)) { | ||
| 151 | + return; | ||
| 152 | + } | ||
| 153 | + log.info("检查集合加载状态: {}", collectionName); | ||
| 154 | + // 检查集合是否存在 | ||
| 155 | + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() | ||
| 156 | + .collectionName(collectionName) | ||
| 157 | + .build(); | ||
| 158 | + | ||
| 159 | + boolean exists = milvusClient.hasCollection(hasCollectionReq); | ||
| 160 | + | ||
| 161 | + if (!exists) { | ||
| 162 | + log.error("集合不存在: {}", collectionName); | ||
| 163 | + throw new RuntimeException("集合不存在: " + collectionName); | ||
| 164 | + } | ||
| 165 | + | ||
| 166 | + // 获取加载状态 | ||
| 167 | + GetLoadStateReq getLoadStateReq = GetLoadStateReq.builder() | ||
| 168 | + .collectionName(collectionName) | ||
| 169 | + .build(); | ||
| 170 | + | ||
| 171 | + boolean isLoaded = milvusClient.getLoadState(getLoadStateReq); | ||
| 172 | + | ||
| 173 | + if (!isLoaded) { | ||
| 174 | + log.info("加载集合到内存: {}", collectionName); | ||
| 175 | + | ||
| 176 | + // 加载集合 | ||
| 177 | + LoadCollectionReq loadCollectionReq = LoadCollectionReq.builder() | ||
| 178 | + .collectionName(collectionName) | ||
| 179 | + .build(); | ||
| 180 | + | ||
| 181 | + milvusClient.loadCollection(loadCollectionReq); | ||
| 182 | + | ||
| 183 | + // 等待加载完成 | ||
| 184 | + waitForCollectionLoaded(collectionName); | ||
| 185 | + | ||
| 186 | + loadedCollections.add(collectionName); | ||
| 187 | + log.info("集合加载完成: {}", collectionName); | ||
| 188 | + } else { | ||
| 189 | + loadedCollections.add(collectionName); | ||
| 190 | + log.info("集合已加载: {}", collectionName); | ||
| 191 | + } | ||
| 192 | + | ||
| 193 | + } catch (Exception e) { | ||
| 194 | + log.error("确保集合加载失败", e); | ||
| 195 | + throw new RuntimeException("集合加载失败: " + collectionName, e); | ||
| 196 | + } | ||
| 197 | + } | ||
| 198 | + | ||
| 199 | + /** | ||
| 200 | + * 等待集合加载完成 | ||
| 201 | + */ | ||
| 202 | + private void waitForCollectionLoaded(String collectionName) { | ||
| 203 | + int maxRetries = 30; | ||
| 204 | + int retryInterval = 1000; // 1秒 | ||
| 205 | + | ||
| 206 | + for (int i = 0; i < maxRetries; i++) { | ||
| 207 | + try { | ||
| 208 | + GetLoadStateReq getLoadStateReq = GetLoadStateReq.builder() | ||
| 209 | + .collectionName(collectionName) | ||
| 210 | + .build(); | ||
| 211 | + | ||
| 212 | + boolean isLoaded = milvusClient.getLoadState(getLoadStateReq); | ||
| 213 | + | ||
| 214 | + if (isLoaded) { | ||
| 215 | + log.info("集合加载状态确认: {}", collectionName); | ||
| 216 | + return; | ||
| 217 | + } | ||
| 218 | + | ||
| 219 | + Thread.sleep(retryInterval); | ||
| 220 | + | ||
| 221 | + } catch (Exception e) { | ||
| 222 | + log.warn("检查加载状态失败，重试 {}/{}", i + 1, maxRetries); | ||
| 223 | + } | ||
| 224 | + } | ||
| 225 | + | ||
| 226 | + throw new RuntimeException("集合加载超时: " + collectionName); | ||
| 227 | + } | ||
| 228 | + | ||
| 229 | + | ||
| 230 | + /** | ||
| 231 | + * 从实体对象构建Milvus插入数据 | ||
| 232 | + */ | ||
| 233 | + public JsonObject convertToMilvusRow(Map<String,Object> data, List<Float> vector,String sQuestion,String sSqlContent,String sKey) { | ||
| 234 | + JsonObject row = new JsonObject(); | ||
| 235 | + | ||
| 236 | + // 添加向量 | ||
| 237 | + JsonArray vectorArray = new JsonArray(); | ||
| 238 | + vector.forEach(vectorArray::add); | ||
| 239 | + row.add("vector", vectorArray); | ||
| 240 | + // 添加文本字段 | ||
| 241 | + row.addProperty("sKey", sKey); | ||
| 242 | + row.addProperty("data_id", data.get("sId").toString()); | ||
| 243 | + row.addProperty("sQuestion", sQuestion); | ||
| 244 | + row.addProperty("sSqlContent", sSqlContent); | ||
| 245 | + // 创建时间字段 - 必须提供！ | ||
| 246 | + row.addProperty("create_time", System.currentTimeMillis()); | ||
| 247 | + // 创建时间字段 - 必须提供！ | ||
| 248 | +// row.add("create_time", JsonValue.from(System.currentTimeMillis())); | ||
| 249 | + // 添加业务字段到metadata | ||
| 250 | + JsonObject metadata = MapToJsonConverter.convert(data); | ||
| 251 | + row.add("metadata", metadata); | ||
| 252 | + return row; | ||
| 253 | + } | ||
| 254 | + | ||
| 255 | + /** | ||
| 256 | + * 创建集合（如果不存在） | ||
| 257 | + */ | ||
| 258 | + public void createCollectionIfNotExists(String collectionName) { | ||
| 259 | + try { | ||
| 260 | + // 检查集合是否存在 | ||
| 261 | + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() | ||
| 262 | + .collectionName(collectionName) | ||
| 263 | + .build(); | ||
| 264 | + boolean exists = milvusClient.hasCollection(hasCollectionReq); | ||
| 265 | + if (!exists) { | ||
| 266 | + createCollection(collectionName); | ||
| 267 | + log.info("集合 {} 创建成功", collectionName); | ||
| 268 | + } | ||
| 269 | + } catch (Exception e) { | ||
| 270 | + log.error("检查/创建集合失败: {}", collectionName, e); | ||
| 271 | + throw new RuntimeException("初始化Milvus集合失败", e); | ||
| 272 | + } | ||
| 273 | + } | ||
| 274 | + | ||
| 275 | + /** | ||
| 276 | + * 创建集合（定义字段结构） | ||
| 277 | + */ | ||
| 278 | + private void createCollection(String collectionName) { | ||
| 279 | + //删除现有集合 | ||
| 280 | +// DropCollectionReq dropCollectionReq = DropCollectionReq.builder() | ||
| 281 | +// .collectionName(collectionName) | ||
| 282 | +// .build(); | ||
| 283 | +// milvusClient.dropCollection(dropCollectionReq); | ||
| 284 | + | ||
| 285 | + // 定义字段列表 | ||
| 286 | + List<CreateCollectionReq.FieldSchema> fieldSchemas = Arrays.asList( | ||
| 287 | + // 1. 主键字段 | ||
| 288 | + CreateCollectionReq.FieldSchema.builder() | ||
| 289 | + .name("id") | ||
| 290 | + .dataType(DataType.Int64) | ||
| 291 | + .isPrimaryKey(true) | ||
| 292 | + .autoID(true) // 使用自动ID | ||
| 293 | + .description("主键ID") | ||
| 294 | + .build(), | ||
| 295 | + | ||
| 296 | + // 2. 向量字段 | ||
| 297 | + CreateCollectionReq.FieldSchema.builder() | ||
| 298 | + .name("vector") | ||
| 299 | + .dataType(DataType.FloatVector) | ||
| 300 | + .dimension(VECTOR_DIM) | ||
| 301 | + .description("向量字段，用于相似性搜索") | ||
| 302 | + .build(), | ||
| 303 | + | ||
| 304 | + // 3. 问题字段 | ||
| 305 | + CreateCollectionReq.FieldSchema.builder() | ||
| 306 | + .name("sQuestion") | ||
| 307 | + .dataType(DataType.VarChar) | ||
| 308 | + .maxLength(1000) | ||
| 309 | + .description("用户问题") | ||
| 310 | + .build(), | ||
| 311 | + | ||
| 312 | + // 4. SQL内容字段 | ||
| 313 | + CreateCollectionReq.FieldSchema.builder() | ||
| 314 | + .name("sSqlContent") | ||
| 315 | + .dataType(DataType.VarChar) | ||
| 316 | + .maxLength(5000) // SQL可能较长 | ||
| 317 | + .description("SQL语句") | ||
| 318 | + .build(), | ||
| 319 | + | ||
| 320 | + // 5. 数据ID字段 | ||
| 321 | + CreateCollectionReq.FieldSchema.builder() | ||
| 322 | + .name("data_id") | ||
| 323 | + .dataType(DataType.VarChar) | ||
| 324 | + .maxLength(100) | ||
| 325 | + .description("原始数据ID") | ||
| 326 | + .build(), | ||
| 327 | + | ||
| 328 | + // 6. 创建时间字段 | ||
| 329 | + CreateCollectionReq.FieldSchema.builder() | ||
| 330 | + .name("create_time") | ||
| 331 | + .dataType(DataType.Int64) | ||
| 332 | + .description("创建时间戳") | ||
| 333 | + .build(), | ||
| 334 | + | ||
| 335 | + // 7. 元数据字段（使用JSON类型存储额外数据） | ||
| 336 | + CreateCollectionReq.FieldSchema.builder() | ||
| 337 | + .name("metadata") | ||
| 338 | + .dataType(DataType.JSON) | ||
| 339 | + .description("额外元数据") | ||
| 340 | + .build(), | ||
| 341 | + CreateCollectionReq.FieldSchema.builder() | ||
| 342 | + .name("sKey") | ||
| 343 | + .dataType(DataType.VarChar) | ||
| 344 | + .maxLength(100) | ||
| 345 | + .description("存入的vector转换前数据") | ||
| 346 | + .build() | ||
| 347 | + ); | ||
| 348 | + | ||
| 349 | + // 创建集合schema | ||
| 350 | + CreateCollectionReq.CollectionSchema schema = | ||
| 351 | + CreateCollectionReq.CollectionSchema.builder() | ||
| 352 | + .fieldSchemaList(fieldSchemas) | ||
| 353 | + .enableDynamicField(true) | ||
| 354 | + .build(); | ||
| 355 | + | ||
| 356 | + // 创建集合请求 | ||
| 357 | + CreateCollectionReq createCollectionReq = CreateCollectionReq.builder() | ||
| 358 | + .collectionName(collectionName) | ||
| 359 | + .collectionSchema(schema) | ||
| 360 | + .consistencyLevel(ConsistencyLevel.BOUNDED) | ||
| 361 | + .build(); | ||
| 362 | + | ||
| 363 | + // 执行创建集合 | ||
| 364 | + milvusClient.createCollection(createCollectionReq); | ||
| 365 | + | ||
| 366 | + //创建索引 | ||
| 367 | + createIndexesStepByStep(collectionName); | ||
| 368 | + } | ||
| 369 | + | ||
| 370 | + /* | ||
| 371 | + * 分步创建索引，便于监控每个索引的状态 | ||
| 372 | + */ | ||
| 373 | + private void createIndexesStepByStep(String collectionName) { | ||
| 374 | + log.info("开始为集合创建索引: {}", collectionName); | ||
| 375 | + createAllIndexes(collectionName); | ||
| 376 | +// // 1. 创建向量索引 | ||
| 377 | +// createVectorIndex(collectionName); | ||
| 378 | +// | ||
| 379 | +// // 2. 创建标量索引 | ||
| 380 | +// createScalarIndexes(collectionName); | ||
| 381 | + } | ||
| 382 | + | ||
| 383 | + /** | ||
| 384 | + * 创建向量索引 | ||
| 385 | + * IVF_FLAT 向量相似度搜索 常用的向量索引，平衡性能和召回率 | ||
| 386 | + * STL_SORT 标量字段排序 适用于数字、时间等需要排序的字段 | ||
| 387 | + * INVERTED 文本字段过滤 倒排索引，适用于文本字段的精确匹配 | ||
| 388 | + * TRIE 字符串前缀匹配 适用于前缀查询 | ||
| 389 | + * BITMAP 枚举值过滤 适用于低基数字段 | ||
| 390 | + */ | ||
| 391 | + private void createVectorIndex(String collectionName) { | ||
| 392 | + log.info("创建向量索引: {}", collectionName); | ||
| 393 | + | ||
| 394 | + Map<String, Object> extraParams = new HashMap<>(); | ||
| 395 | + extraParams.put("nlist", 128); | ||
| 396 | + | ||
| 397 | + IndexParam vectorIndex = IndexParam.builder() | ||
| 398 | + .fieldName("vector") | ||
| 399 | + .indexName("idx_vector") | ||
| 400 | + .indexType(IndexParam.IndexType.IVF_FLAT) | ||
| 401 | + .metricType(IndexParam.MetricType.IP) | ||
| 402 | + .extraParams(extraParams) | ||
| 403 | + .build(); | ||
| 404 | + | ||
| 405 | + CreateIndexReq createIndexReq = CreateIndexReq.builder() | ||
| 406 | + .collectionName(collectionName) | ||
| 407 | + .indexParams(Collections.singletonList(vectorIndex)) | ||
| 408 | + .sync(true) | ||
| 409 | + .timeout(60000L) | ||
| 410 | + .build(); | ||
| 411 | + | ||
| 412 | + milvusClient.createIndex(createIndexReq); | ||
| 413 | + log.info("向量索引创建完成"); | ||
| 414 | + } | ||
| 415 | + | ||
| 416 | + /** | ||
| 417 | + * 创建标量索引 | ||
| 418 | + */ | ||
| 419 | + private void createScalarIndexes(String collectionName) { | ||
| 420 | + log.info("创建标量索引: {}", collectionName); | ||
| 421 | + | ||
| 422 | + // 为 create_time 字段创建索引 | ||
| 423 | + IndexParam timeIndex = IndexParam.builder() | ||
| 424 | + .fieldName("create_time") | ||
| 425 | + .indexName("idx_create_time") | ||
| 426 | + .indexType(IndexParam.IndexType.STL_SORT) // 排序索引 | ||
| 427 | + .build(); | ||
| 428 | + | ||
| 429 | + CreateIndexReq timeIndexReq = CreateIndexReq.builder() | ||
| 430 | + .collectionName(collectionName) | ||
| 431 | + .indexParams(Collections.singletonList(timeIndex)) | ||
| 432 | + .sync(true) | ||
| 433 | + .timeout(30000L) | ||
| 434 | + .build(); | ||
| 435 | + | ||
| 436 | + milvusClient.createIndex(timeIndexReq); | ||
| 437 | + log.info("create_time 索引创建完成"); | ||
| 438 | + | ||
| 439 | + // 为 question 字段创建倒排索引（支持文本过滤） | ||
| 440 | + IndexParam questionIndex = IndexParam.builder() | ||
| 441 | + .fieldName("sQuestion") | ||
| 442 | + .indexName("idx_question") | ||
| 443 | + .indexType(IndexParam.IndexType.TRIE) // 倒排索引 | ||
| 444 | + .build(); | ||
| 445 | + | ||
| 446 | + CreateIndexReq questionIndexReq = CreateIndexReq.builder() | ||
| 447 | + .collectionName(collectionName) | ||
| 448 | + .indexParams(Collections.singletonList(questionIndex)) | ||
| 449 | + .sync(true) | ||
| 450 | + .timeout(30000L) | ||
| 451 | + .build(); | ||
| 452 | + | ||
| 453 | + milvusClient.createIndex(questionIndexReq); | ||
| 454 | + log.info("question 索引创建完成"); | ||
| 455 | + | ||
| 456 | + // 为 data_id 字段创建索引 | ||
| 457 | + IndexParam idIndex = IndexParam.builder() | ||
| 458 | + .fieldName("data_id") | ||
| 459 | + .indexName("idx_data_id") | ||
| 460 | + .indexType(IndexParam.IndexType.TRIE) | ||
| 461 | + .build(); | ||
| 462 | + | ||
| 463 | + CreateIndexReq idIndexReq = CreateIndexReq.builder() | ||
| 464 | + .collectionName(collectionName) | ||
| 465 | + .indexParams(Collections.singletonList(idIndex)) | ||
| 466 | + .sync(true) | ||
| 467 | + .timeout(30000L) | ||
| 468 | + .build(); | ||
| 469 | + | ||
| 470 | + milvusClient.createIndex(idIndexReq); | ||
| 471 | + log.info("data_id 索引创建完成"); | ||
| 472 | + } | ||
| 473 | + | ||
| 474 | + /** | ||
| 475 | + * 重建索引（解决索引未就绪问题） | ||
| 476 | + */ | ||
| 477 | + public boolean rebuildIndex(String collectionName) { | ||
| 478 | + log.info("========== 开始重建索引 =========="); | ||
| 479 | + log.info("集合名称: {}", collectionName); | ||
| 480 | + | ||
| 481 | + try { | ||
| 482 | + // 1. 先检查集合是否存在 | ||
| 483 | + HasCollectionReq hasReq = HasCollectionReq.builder() | ||
| 484 | + .collectionName(collectionName) | ||
| 485 | + .build(); | ||
| 486 | + | ||
| 487 | + if (!milvusClient.hasCollection(hasReq)) { | ||
| 488 | + log.error("集合不存在: {}", collectionName); | ||
| 489 | + return false; | ||
| 490 | + } | ||
| 491 | + | ||
| 492 | + // 2. 先释放集合（如果已加载） | ||
| 493 | + try { | ||
| 494 | + ReleaseCollectionReq releaseReq = ReleaseCollectionReq.builder() | ||
| 495 | + .collectionName(collectionName) | ||
| 496 | + .build(); | ||
| 497 | + milvusClient.releaseCollection(releaseReq); | ||
| 498 | + log.info("集合已释放: {}", collectionName); | ||
| 499 | + Thread.sleep(2000); // 等待释放完成 | ||
| 500 | + } catch (Exception e) { | ||
| 501 | + log.warn("释放集合失败（可能未加载）: {}", e.getMessage()); | ||
| 502 | + } | ||
| 503 | + | ||
| 504 | + // 3. 删除原有索引 | ||
| 505 | + try { | ||
| 506 | + DropIndexReq dropIndexReq = DropIndexReq.builder() | ||
| 507 | + .collectionName(collectionName) | ||
| 508 | + .fieldName("vector") // 指定向量字段 | ||
| 509 | + .build(); | ||
| 510 | + | ||
| 511 | + milvusClient.dropIndex(dropIndexReq); | ||
| 512 | + log.info("原有索引已删除"); | ||
| 513 | + Thread.sleep(2000); // 等待删除完成 | ||
| 514 | + } catch (Exception e) { | ||
| 515 | + log.warn("删除索引失败（可能不存在）: {}", e.getMessage()); | ||
| 516 | + } | ||
| 517 | + | ||
| 518 | + // 4. 创建新索引 | ||
| 519 | + createVectorIndex(collectionName); | ||
| 520 | + | ||
| 521 | + // 5. 等待索引就绪 | ||
| 522 | + boolean indexReady = waitForIndexReady(collectionName, "vector", 60); | ||
| 523 | + if (!indexReady) { | ||
| 524 | + log.error("索引未就绪"); | ||
| 525 | + return false; | ||
| 526 | + } | ||
| 527 | + | ||
| 528 | + // 6. 重新加载集合 | ||
| 529 | + LoadCollectionReq loadReq = LoadCollectionReq.builder() | ||
| 530 | + .collectionName(collectionName) | ||
| 531 | + .build(); | ||
| 532 | + | ||
| 533 | + milvusClient.loadCollection(loadReq); | ||
| 534 | + log.info("集合重新加载成功"); | ||
| 535 | + | ||
| 536 | + // 7. 验证加载状态 | ||
| 537 | + boolean loaded = waitForLoad(collectionName, 30); | ||
| 538 | + if (!loaded) { | ||
| 539 | + log.error("集合加载失败"); | ||
| 540 | + return false; | ||
| 541 | + } | ||
| 542 | + | ||
| 543 | + log.info("✅ 索引重建完成，集合已就绪: {}", collectionName); | ||
| 544 | + return true; | ||
| 545 | + | ||
| 546 | + } catch (Exception e) { | ||
| 547 | + log.error("重建索引失败", e); | ||
| 548 | + return false; | ||
| 549 | + } | ||
| 550 | + } | ||
| 551 | + | ||
| 552 | + /** | ||
| 553 | + * 等待索引就绪 | ||
| 554 | + */ | ||
| 555 | + private boolean waitForIndexReady(String collectionName, String fieldName, int timeoutSeconds) { | ||
| 556 | + log.info("等待索引就绪: {}.{}，超时: {}秒", collectionName, fieldName, timeoutSeconds); | ||
| 557 | + | ||
| 558 | + for (int i = 0; i < timeoutSeconds; i++) { | ||
| 559 | + try { | ||
| 560 | + DescribeIndexReq describeIndexReq = DescribeIndexReq.builder() | ||
| 561 | + .collectionName(collectionName) | ||
| 562 | + .build(); | ||
| 563 | + | ||
| 564 | + DescribeIndexResp describeIndexResp = milvusClient.describeIndex(describeIndexReq); | ||
| 565 | + | ||
| 566 | + List<DescribeIndexResp.IndexDesc> indexDescs = describeIndexResp.getIndexDescriptions(); | ||
| 567 | + | ||
| 568 | + for (DescribeIndexResp.IndexDesc desc : indexDescs) { | ||
| 569 | + if (fieldName.equals(desc.getFieldName())) { | ||
| 570 | + IndexBuildState state = desc.getIndexState(); | ||
| 571 | + | ||
| 572 | + log.info("索引状态: {}, 进度: {}/{}", | ||
| 573 | + state, desc.getIndexedRows(), desc.getTotalRows()); | ||
| 574 | + | ||
| 575 | + if (state == IndexBuildState.Finished) { | ||
| 576 | + log.info("✅ 索引就绪"); | ||
| 577 | + return true; | ||
| 578 | + } else if (state == IndexBuildState.Failed) { | ||
| 579 | + log.error("❌ 索引构建失败: {}", desc.getIndexFailedReason()); | ||
| 580 | + return false; | ||
| 581 | + } | ||
| 582 | + break; | ||
| 583 | + } | ||
| 584 | + } | ||
| 585 | + | ||
| 586 | + Thread.sleep(1000); | ||
| 587 | + | ||
| 588 | + } catch (InterruptedException e) { | ||
| 589 | + Thread.currentThread().interrupt(); | ||
| 590 | + log.warn("等待被中断"); | ||
| 591 | + return false; | ||
| 592 | + } catch (Exception e) { | ||
| 593 | + log.warn("检查索引状态失败: {}/{}", i + 1, timeoutSeconds); | ||
| 594 | + } | ||
| 595 | + } | ||
| 596 | + | ||
| 597 | + log.error("等待索引就绪超时"); | ||
| 598 | + return false; | ||
| 599 | + } | ||
| 600 | + | ||
| 601 | + /** | ||
| 602 | + * 等待集合加载完成 | ||
| 603 | + */ | ||
| 604 | + private boolean waitForLoad(String collectionName, int timeoutSeconds) { | ||
| 605 | + log.info("等待集合加载: {}，超时: {}秒", collectionName, timeoutSeconds); | ||
| 606 | + | ||
| 607 | + for (int i = 0; i < timeoutSeconds; i++) { | ||
| 608 | + try { | ||
| 609 | + GetLoadStateReq loadStateReq = GetLoadStateReq.builder() | ||
| 610 | + .collectionName(collectionName) | ||
| 611 | + .build(); | ||
| 612 | + | ||
| 613 | + boolean isLoaded = milvusClient.getLoadState(loadStateReq); | ||
| 614 | + | ||
| 615 | + if (isLoaded) { | ||
| 616 | + log.info("✅ 集合加载完成"); | ||
| 617 | + return true; | ||
| 618 | + } | ||
| 619 | + | ||
| 620 | + Thread.sleep(1000); | ||
| 621 | + | ||
| 622 | + } catch (InterruptedException e) { | ||
| 623 | + Thread.currentThread().interrupt(); | ||
| 624 | + return false; | ||
| 625 | + } catch (Exception e) { | ||
| 626 | + log.warn("检查加载状态失败: {}/{}", i + 1, timeoutSeconds); | ||
| 627 | + } | ||
| 628 | + } | ||
| 629 | + | ||
| 630 | + log.error("集合加载超时"); | ||
| 631 | + return false; | ||
| 632 | + } | ||
| 633 | + | ||
| 634 | + | ||
| 635 | + /** | ||
| 636 | + * 批量创建所有索引（向量索引 + 多个标量索引） | ||
| 637 | + */ | ||
| 638 | + private void createAllIndexes(String collectionName) { | ||
| 639 | + log.info("开始为集合批量创建索引: {}", collectionName); | ||
| 640 | + | ||
| 641 | + // 1. 准备所有索引参数 | ||
| 642 | + List<IndexParam> allIndexParams = new ArrayList<>(); | ||
| 643 | + | ||
| 644 | + // 1.1 向量索引 | ||
| 645 | + Map<String, Object> vectorExtraParams = new HashMap<>(); | ||
| 646 | + vectorExtraParams.put("nlist", 256); // 聚类中心数：sqrt(384) * 13 ≈ 256 | ||
| 647 | + vectorExtraParams.put("nprobe", 32); // 搜索时检查的聚类数 | ||
| 648 | + | ||
| 649 | + IndexParam vectorIndex = IndexParam.builder() | ||
| 650 | + .fieldName("vector") | ||
| 651 | + .indexName("idx_vector_rebuild") | ||
| 652 | + .indexType(IndexParam.IndexType.IVF_FLAT) | ||
| 653 | + .metricType(IndexParam.MetricType.IP) | ||
| 654 | + .extraParams(vectorExtraParams) | ||
| 655 | + .build(); | ||
| 656 | + allIndexParams.add(vectorIndex); | ||
| 657 | + | ||
| 658 | + // 1.2 create_time 字段索引（用于时间范围查询） | ||
| 659 | + IndexParam timeIndex = IndexParam.builder() | ||
| 660 | + .fieldName("create_time") | ||
| 661 | + .indexName("idx_create_time") | ||
| 662 | + .indexType(IndexParam.IndexType.STL_SORT) // 排序索引 | ||
| 663 | + .build(); | ||
| 664 | + allIndexParams.add(timeIndex); | ||
| 665 | + | ||
| 666 | + // 1.3 question 字段倒排索引（用于文本过滤） | ||
| 667 | + IndexParam questionIndex = IndexParam.builder() | ||
| 668 | + .fieldName("sQuestion") | ||
| 669 | + .indexName("idx_question") | ||
| 670 | + .indexType(IndexParam.IndexType.INVERTED) // 倒排索引 | ||
| 671 | + .build(); | ||
| 672 | + allIndexParams.add(questionIndex); | ||
| 673 | + | ||
| 674 | + // 1.4 data_id 字段索引（用于精确匹配） | ||
| 675 | + IndexParam idIndex = IndexParam.builder() | ||
| 676 | + .fieldName("data_id") | ||
| 677 | + .indexName("idx_data_id") | ||
| 678 | + .indexType(IndexParam.IndexType.INVERTED) | ||
| 679 | + .build(); | ||
| 680 | + allIndexParams.add(idIndex); | ||
| 681 | + | ||
| 682 | + IndexParam sKey = IndexParam.builder() | ||
| 683 | + .fieldName("sKey") | ||
| 684 | + .indexName("s_key") | ||
| 685 | + .indexType(IndexParam.IndexType.INVERTED) | ||
| 686 | + .build(); | ||
| 687 | + allIndexParams.add(sKey); | ||
| 688 | + | ||
| 689 | + // 1.5 sql_content 字段索引（如果需要） | ||
| 690 | +// IndexParam sqlIndex = IndexParam.builder() | ||
| 691 | +// .fieldName("sql_content") | ||
| 692 | +// .indexName("idx_sql_content") | ||
| 693 | +// .indexType(IndexParam.IndexType.INVERTED) | ||
| 694 | +// .build(); | ||
| 695 | +// allIndexParams.add(sqlIndex); | ||
| 696 | + | ||
| 697 | + // 2. 批量创建索引 | ||
| 698 | + try { | ||
| 699 | + CreateIndexReq createIndexReq = CreateIndexReq.builder() | ||
| 700 | + .collectionName(collectionName) | ||
| 701 | + .indexParams(allIndexParams) // 一次性传入所有索引 | ||
| 702 | + .sync(true) // 同步等待 | ||
| 703 | + .timeout(120000L) // 总超时时间120秒 | ||
| 704 | + .build(); | ||
| 705 | + | ||
| 706 | + milvusClient.createIndex(createIndexReq); | ||
| 707 | + log.info("所有索引批量创建成功: {}", collectionName); | ||
| 708 | + | ||
| 709 | + } catch (Exception e) { | ||
| 710 | + log.error("批量创建索引失败: {}", e.getMessage()); | ||
| 711 | + createScalarIndexes(collectionName); | ||
| 712 | + } | ||
| 713 | + } | ||
| 714 | + | ||
| 715 | + | ||
| 716 | +} |
src/main/java/com/xly/milvus/service/impl/EmbeddingServiceImpl.java
0 → 100644
| 1 | +package com.xly.milvus.service.impl; | ||
| 2 | + | ||
| 3 | +import com.xly.milvus.service.EmbeddingService; | ||
| 4 | +import dev.langchain4j.data.embedding.Embedding; | ||
| 5 | +import dev.langchain4j.data.segment.TextSegment; | ||
| 6 | +import dev.langchain4j.model.embedding.EmbeddingModel; | ||
| 7 | +import lombok.RequiredArgsConstructor; | ||
| 8 | +import lombok.extern.slf4j.Slf4j; | ||
| 9 | +import org.springframework.stereotype.Service; | ||
| 10 | + | ||
| 11 | +import java.util.ArrayList; | ||
| 12 | +import java.util.Collections; | ||
| 13 | +import java.util.List; | ||
| 14 | +import java.util.stream.Collectors; | ||
| 15 | + | ||
| 16 | +@Slf4j | ||
| 17 | +@Service("embeddingService") | ||
| 18 | +@RequiredArgsConstructor | ||
| 19 | +public class EmbeddingServiceImpl implements EmbeddingService { | ||
| 20 | + | ||
| 21 | + private final EmbeddingModel embeddingModel; | ||
| 22 | + | ||
| 23 | + /** | ||
| 24 | + * 生成单个文本的向量 | ||
| 25 | + */ | ||
| 26 | + public List<Float> generateEmbedding(String text) { | ||
| 27 | + if (text == null || text.trim().isEmpty()) { | ||
| 28 | + log.warn("Input text is empty"); | ||
| 29 | + return null; | ||
| 30 | + } | ||
| 31 | + | ||
| 32 | + try { | ||
| 33 | + // 0.35.0 API: embed 方法返回 Response<Embedding> | ||
| 34 | + Embedding embedding = embeddingModel.embed(text).content(); | ||
| 35 | + | ||
| 36 | + // 将 float[] 转换为 List<Float> 供 Milvus 使用 | ||
| 37 | + float[] vectorArray = embedding.vector(); | ||
| 38 | + List<Float> vectorList = new ArrayList<>(vectorArray.length); | ||
| 39 | + for (float v : vectorArray) { | ||
| 40 | + vectorList.add(v); | ||
| 41 | + } | ||
| 42 | + return vectorList; | ||
| 43 | + } catch (Exception e) { | ||
| 44 | + log.error("Error generating embedding for text: {}", text, e); | ||
| 45 | + return null; | ||
| 46 | + } | ||
| 47 | + } | ||
| 48 | + | ||
| 49 | + /** | ||
| 50 | + * 批量生成向量（高效版） | ||
| 51 | + * 利用 LangChain4j 内置的并行化能力，显著提升性能 | ||
| 52 | + */ | ||
| 53 | + public List<List<Float>> generateEmbeddings(List<String> texts) { | ||
| 54 | + if (texts == null || texts.isEmpty()) { | ||
| 55 | + return Collections.emptyList(); | ||
| 56 | + } | ||
| 57 | + | ||
| 58 | + try { | ||
| 59 | + // 将文本转换为 TextSegment 列表 | ||
| 60 | + List<TextSegment> segments = texts.stream() | ||
| 61 | + .map(TextSegment::from) | ||
| 62 | + .collect(Collectors.toList()); | ||
| 63 | + | ||
| 64 | + // 批量嵌入，内部自动并行化 | ||
| 65 | + List<Embedding> embeddings = embeddingModel.embedAll(segments).content(); | ||
| 66 | + | ||
| 67 | + // 转换格式 | ||
| 68 | + return embeddings.stream() | ||
| 69 | + .map(embedding -> { | ||
| 70 | + float[] array = embedding.vector(); | ||
| 71 | + List<Float> list = new ArrayList<>(array.length); | ||
| 72 | + for (float v : array) { | ||
| 73 | + list.add(v); | ||
| 74 | + } | ||
| 75 | + return list; | ||
| 76 | + }) | ||
| 77 | + .collect(Collectors.toList()); | ||
| 78 | + | ||
| 79 | + } catch (Exception e) { | ||
| 80 | + log.error("Error generating embeddings in batch", e); | ||
| 81 | + return Collections.emptyList(); | ||
| 82 | + } | ||
| 83 | + } | ||
| 84 | +} | ||
| 0 | \ No newline at end of file | 85 | \ No newline at end of file |
src/main/java/com/xly/milvus/service/impl/MilvusServiceImpl.java
0 → 100644
| 1 | +package com.xly.milvus.service.impl; | ||
| 2 | + | ||
| 3 | +import cn.hutool.core.collection.CollUtil; | ||
| 4 | +import cn.hutool.core.collection.ConcurrentHashSet; | ||
| 5 | +import cn.hutool.core.date.DateUtil; | ||
| 6 | +import cn.hutool.core.thread.ThreadUtil; | ||
| 7 | +import cn.hutool.core.util.ObjectUtil; | ||
| 8 | +import cn.hutool.core.util.StrUtil; | ||
| 9 | +import com.google.gson.JsonArray; | ||
| 10 | +import com.google.gson.JsonObject; | ||
| 11 | +import com.xly.milvus.service.MilvusService; | ||
| 12 | +import com.xly.milvus.service.VectorizationService; | ||
| 13 | +import com.xly.milvus.util.MapToJsonConverter; | ||
| 14 | +import com.xly.service.DynamicExeDbService; | ||
| 15 | +import com.xly.tts.bean.TTSResponseDTO; | ||
| 16 | +import io.milvus.response.SearchResultsWrapper; | ||
| 17 | +import io.milvus.v2.client.MilvusClientV2; | ||
| 18 | +import io.milvus.v2.common.ConsistencyLevel; | ||
| 19 | +import io.milvus.v2.common.DataType; | ||
| 20 | +import io.milvus.v2.common.IndexParam; | ||
| 21 | +import io.milvus.v2.service.collection.request.*; | ||
| 22 | +import io.milvus.v2.service.collection.response.DescribeCollectionResp; | ||
| 23 | +import io.milvus.v2.service.vector.request.DeleteReq; | ||
| 24 | +import io.milvus.v2.service.vector.request.InsertReq; | ||
| 25 | +import io.milvus.v2.service.vector.request.SearchReq; | ||
| 26 | +import io.milvus.v2.service.vector.request.data.FloatVec; | ||
| 27 | +import io.milvus.v2.service.vector.response.InsertResp; | ||
| 28 | +import io.milvus.v2.service.vector.response.QueryResp; | ||
| 29 | +import io.milvus.v2.service.vector.response.SearchResp; | ||
| 30 | +import lombok.RequiredArgsConstructor; | ||
| 31 | +import lombok.extern.slf4j.Slf4j; | ||
| 32 | +import org.springframework.beans.factory.annotation.Value; | ||
| 33 | +import org.springframework.stereotype.Service; | ||
| 34 | + | ||
| 35 | +import java.math.BigDecimal; | ||
| 36 | +import java.util.*; | ||
| 37 | +import java.util.regex.Matcher; | ||
| 38 | +import java.util.regex.Pattern; | ||
| 39 | +import java.util.stream.Collectors; | ||
| 40 | + | ||
| 41 | +/** | ||
| 42 | + * 向量化服务实现 - 使用LangChain4j的All-MiniLM-L6-v2模型 | ||
| 43 | + */ | ||
| 44 | +@Slf4j | ||
| 45 | +@Service(value = "milvusService") | ||
| 46 | +@RequiredArgsConstructor | ||
| 47 | +public class MilvusServiceImpl implements MilvusService { | ||
| 48 | + | ||
| 49 | + private final MilvusClientV2 milvusClient; | ||
| 50 | + private final VectorizationService vectorizationService; | ||
| 51 | + private final DynamicExeDbService dynamicExeDbService; | ||
| 52 | + | ||
| 53 | + // 或者从配置文件读取 | ||
| 54 | + @Value("${milvus.vector.dimension:384}") | ||
| 55 | + private int VECTOR_DIM; | ||
| 56 | + | ||
| 57 | + // 缓存已经初始化过的 Milvus 集合（线程安全） | ||
| 58 | + private final Set<String> loadedCollections = new ConcurrentHashSet<>(); | ||
| 59 | + | ||
| 60 | + | ||
| 61 | + | ||
| 62 | + /*** | ||
| 63 | + * @Author 钱豹 | ||
| 64 | + * @Date 22:18 2026/3/24 | ||
| 65 | + * @Param [reqMap] | ||
| 66 | + * @return void | ||
| 67 | + * @Description 初始化结构以及数据 | ||
| 68 | + **/ | ||
| 69 | + @Override | ||
| 70 | + public TTSResponseDTO initDataToMilvus(Map<String, Object> reqMap) { | ||
| 71 | + if(ObjectUtil.isEmpty(reqMap.get("sInputTabelName"))){ | ||
| 72 | + return TTSResponseDTO.builder() | ||
| 73 | + .code(-1) | ||
| 74 | + .message("输入表名") | ||
| 75 | + .build(); | ||
| 76 | + } | ||
| 77 | + if(ObjectUtil.isEmpty(reqMap.get("sVectorfiled"))){ | ||
| 78 | + return TTSResponseDTO.builder() | ||
| 79 | + .code(-1) | ||
| 80 | + .message("向量库标量字段") | ||
| 81 | + .build(); | ||
| 82 | + } | ||
| 83 | + if(ObjectUtil.isEmpty(reqMap.get("sVectorjson"))){ | ||
| 84 | + return TTSResponseDTO.builder() | ||
| 85 | + .code(-1) | ||
| 86 | + .message("向量化内容JSON") | ||
| 87 | + .build(); | ||
| 88 | + } | ||
| 89 | + | ||
| 90 | + String sInputTabelName = reqMap.get("sInputTabelName").toString(); | ||
| 91 | + String sVectorfiled = reqMap.get("sVectorfiled").toString(); | ||
| 92 | + String sVectorjson = reqMap.get("sVectorjson").toString(); | ||
| 93 | + | ||
| 94 | + String tUpdateDate = DateUtil.now(); | ||
| 95 | + String tUpdateDateUp = getUpdateDateUp(sInputTabelName); | ||
| 96 | + //获取需要同步地数据 | ||
| 97 | + List<Map<String,Object>> data = getAddData(sInputTabelName,tUpdateDate, tUpdateDateUp); | ||
| 98 | + //创建集合 | ||
| 99 | + createCollectionIfNotExists(sInputTabelName, sVectorfiled, sVectorjson,true); | ||
| 100 | + if(ObjectUtil.isNotEmpty(data)){ | ||
| 101 | + //插入数据 | ||
| 102 | + long num= addDataToCollection(sInputTabelName, sVectorfiled, sVectorjson,data); | ||
| 103 | + } | ||
| 104 | + addAiMilvusVectorRecord(sInputTabelName,tUpdateDate, tUpdateDateUp); | ||
| 105 | + return TTSResponseDTO.builder() | ||
| 106 | + .code(200) | ||
| 107 | + .message("success") | ||
| 108 | + .build(); | ||
| 109 | + } | ||
| 110 | + | ||
| 111 | + public String getUpdateDateUp(String sInputTabelName) { | ||
| 112 | + Map<String,Object> serDataMap = new HashMap<>(); | ||
| 113 | + String sSql ="SELECT DATE_FORMAT(tUpdateDate,'%Y-%m-%d %H:%i:%s') AS tUpdateDate FROM ai_milvus_vector_record WHERE sInputTabelName = #{sInputTabelName}"; | ||
| 114 | + serDataMap.put("sInputTabelName",sInputTabelName); | ||
| 115 | + List<Map<String,Object>> data = this.dynamicExeDbService.findSql(serDataMap,sSql); | ||
| 116 | + if(ObjectUtil.isEmpty(data)){ | ||
| 117 | + return "2000-03-24"; | ||
| 118 | + } | ||
| 119 | + return data.get(0).get("tUpdateDate").toString(); | ||
| 120 | + } | ||
| 121 | + | ||
| 122 | + /*** | ||
| 123 | + * @Author 钱豹 | ||
| 124 | + * @Date 22:24 2026/3/24 | ||
| 125 | + * @Param | ||
| 126 | + * @return | ||
| 127 | + * @Description 获取需要同步地数据 | ||
| 128 | + **/ | ||
| 129 | + public List<Map<String,Object>> getAddData(String sInputTabelName,String tUpdateDate,String tUpdateDateUp) { | ||
| 130 | + //获取需要同步地数据 | ||
| 131 | + Map<String,Object> serDataMap = new HashMap<>(); | ||
| 132 | + serDataMap.put("tUpdateDate",tUpdateDate); | ||
| 133 | + serDataMap.put("tUpdateDateUp",tUpdateDateUp); | ||
| 134 | + String sSql = String.format("SELECT * FROM %s WHERE tUpdateDate >= #{tUpdateDateUp} AND tUpdateDate < #{tUpdateDate}",sInputTabelName); | ||
| 135 | + return this.dynamicExeDbService.findSql(serDataMap,sSql); | ||
| 136 | + } | ||
| 137 | + | ||
| 138 | + /*** | ||
| 139 | + * @Author 钱豹 | ||
| 140 | + * @Date 22:32 2026/3/24 | ||
| 141 | + * @Param [sInputTabelName, tUpdateDate] | ||
| 142 | + * @return java.util.List<java.util.Map<java.lang.String,java.lang.Object>> | ||
| 143 | + * @Description 获取更新地数据 | ||
| 144 | + **/ | ||
| 145 | + public void addAiMilvusVectorRecord(String sInputTabelName,String tUpdateDate,String tUpdateDateUp) { | ||
| 146 | + //获取需要同步地数据 | ||
| 147 | + delAiMilvusVectorRecord(sInputTabelName); | ||
| 148 | + Map<String,Object> dMap = new HashMap<>(); | ||
| 149 | + dMap.put("sInputTabelName",sInputTabelName); | ||
| 150 | + dMap.put("tUpdateDate",tUpdateDate); | ||
| 151 | + dMap.put("tUpdateDateUp",tUpdateDateUp); | ||
| 152 | + String sSql = String.format("INSERT INTO ai_milvus_vector_record(sId,sInputTabelName,tUpdateDate,tUpdateDateUp)VALUES(newId(),#{sInputTabelName},#{tUpdateDate},#{tUpdateDateUp})"); | ||
| 153 | + dynamicExeDbService.addSql(dMap,sSql); | ||
| 154 | + } | ||
| 155 | + | ||
| 156 | + /*** | ||
| 157 | + * @Author 钱豹 | ||
| 158 | + * @Date 22:32 2026/3/24 | ||
| 159 | + * @Param [sInputTabelName, tUpdateDate] | ||
| 160 | + * @return java.util.List<java.util.Map<java.lang.String,java.lang.Object>> | ||
| 161 | + * @Description 获取更新地数据 | ||
| 162 | + **/ | ||
| 163 | + public void delAiMilvusVectorRecord(String sInputTabelName) { | ||
| 164 | + //获取需要同步地数据 | ||
| 165 | + Map<String,Object> dMap = new HashMap<>(); | ||
| 166 | + dMap.put("sInputTabelName",sInputTabelName); | ||
| 167 | + String sSql = "DELETE FROM ai_milvus_vector_record WHERE sInputTabelName = #{sInputTabelName}"; | ||
| 168 | + dynamicExeDbService.delSql(dMap,sSql); | ||
| 169 | + } | ||
| 170 | + | ||
| 171 | + | ||
| 172 | + /*** | ||
| 173 | + * @Author 钱豹 | ||
| 174 | + * @Date 16:31 2026/3/24 | ||
| 175 | + * @Param [collectionName] | ||
| 176 | + * @return void | ||
| 177 | + * @Description 创建集合 | ||
| 178 | + **/ | ||
| 179 | + @Override | ||
| 180 | + public void createCollectionIfNotExists(String collectionName,String sVectorfiled,String sVectorjson,Boolean bRset) { | ||
| 181 | + createCollection(collectionName,sVectorfiled,bRset); | ||
| 182 | + log.info("集合 {} 创建成功", collectionName); | ||
| 183 | + // 集合缓存：只加载一次 | ||
| 184 | + if (!loadedCollections.contains(collectionName)) { | ||
| 185 | + loadedCollections.add(collectionName); | ||
| 186 | + } | ||
| 187 | + | ||
| 188 | + } | ||
| 189 | + | ||
| 190 | + /*** | ||
| 191 | + * @Author 钱豹 | ||
| 192 | + * @Date 20:59 2026/3/24 | ||
| 193 | + * @Param [collectionName, sVectorfiled, sVectorjson, bRset] | ||
| 194 | + * @return void | ||
| 195 | + * @Description 新增数据集合 | ||
| 196 | + **/ | ||
| 197 | + @Override | ||
| 198 | + public long addDataToCollection(String collectionName, String sVectorfiled, String sVectorjson,List<Map<String,Object>> data){ | ||
| 199 | + | ||
| 200 | + // 1. 参数校验（防止空参数导致崩溃） | ||
| 201 | + if (ObjectUtil.isEmpty(collectionName) || CollUtil.isEmpty(data)) { | ||
| 202 | + throw new IllegalArgumentException("参数异常：集合名/slaveId/数据不能为空"); | ||
| 203 | + } | ||
| 204 | + | ||
| 205 | + // 1. 转换为Milvus格式 | ||
| 206 | + List<JsonObject> rows = convertToMilvusRow(data, sVectorfiled, sVectorjson); | ||
| 207 | + if (CollUtil.isEmpty(rows)) { | ||
| 208 | + return 0l; // 无数据直接返回 | ||
| 209 | + } | ||
| 210 | + | ||
| 211 | + // 3.先删除再插入 | ||
| 212 | + // 1. 构建删除请求 | ||
| 213 | + // 过滤条件：匹配唯一键 | ||
| 214 | + // 核心：从 data 中提取所有 sSlaveId，批量删除 | ||
| 215 | + List<String> slaveIdList = data.stream() | ||
| 216 | + .map(map -> map.get("sSlaveId")) // 取每条数据的slaveId | ||
| 217 | + .filter(Objects::nonNull) | ||
| 218 | + .map(String::valueOf) | ||
| 219 | + .distinct() // 去重 | ||
| 220 | + .toList(); | ||
| 221 | + | ||
| 222 | + if (slaveIdList.isEmpty()) { | ||
| 223 | + throw new RuntimeException("未获取到slaveId，无法删除旧数据"); | ||
| 224 | + } | ||
| 225 | + | ||
| 226 | + // 拼接 Milvus 删除条件：sSlaveId in ['111','222','333'] | ||
| 227 | + String filter = String.format("sSlaveId in [%s]", | ||
| 228 | + slaveIdList.stream() | ||
| 229 | + .map(id -> "'" + id + "'") | ||
| 230 | + .collect(Collectors.joining(",")) | ||
| 231 | + ); | ||
| 232 | + | ||
| 233 | + // 批量删除 | ||
| 234 | + DeleteReq deleteReq = DeleteReq.builder() | ||
| 235 | + .collectionName(collectionName) | ||
| 236 | + .filter(filter) | ||
| 237 | + .build(); | ||
| 238 | + milvusClient.delete(deleteReq); | ||
| 239 | + | ||
| 240 | + // 短暂等待 Milvus 数据同步 | ||
| 241 | + ThreadUtil.sleep(100); | ||
| 242 | + | ||
| 243 | + // 4. 插入到Milvus（批量） | ||
| 244 | + InsertReq insertReq = InsertReq.builder() | ||
| 245 | + .collectionName(collectionName) | ||
| 246 | + .data(rows) | ||
| 247 | + .build(); | ||
| 248 | + InsertResp insertResp = milvusClient.insert(insertReq); | ||
| 249 | + return insertResp.getInsertCnt(); | ||
| 250 | + } | ||
| 251 | + | ||
| 252 | + /*** | ||
| 253 | + * @Author 钱豹 | ||
| 254 | + * @Date 13:29 2026/3/25 | ||
| 255 | + * @Param [sVectorfiled] | ||
| 256 | + * @return java.util.Map<java.lang.String,java.lang.Object> | ||
| 257 | + * @Description 返回组装动态内容 | ||
| 258 | + **/ | ||
| 259 | + @Override | ||
| 260 | + public Map<String,Object> getMilvusFiled(String sVectorfiled){ | ||
| 261 | + String[] sVectorfiledArray = sVectorfiled.split(","); | ||
| 262 | + List<String> sFileds = new ArrayList<>(); | ||
| 263 | + List<String> sFiledDescriptions = new ArrayList<>(); | ||
| 264 | + List<Map<String,String>> titleList = new LinkedList<>(); | ||
| 265 | + for(String sVectorfiledOne : sVectorfiledArray){ | ||
| 266 | + Map<String,String> title = new HashMap<>(); | ||
| 267 | + | ||
| 268 | + String[] sVectorfiledOneArray = sVectorfiledOne.split(":"); | ||
| 269 | + String sDescriptions = sVectorfiledOneArray[0]; | ||
| 270 | + String sName = sVectorfiledOneArray[1]; | ||
| 271 | + sFileds.add(sName); | ||
| 272 | + // 处理描述中可能包含的换行，保持缩进一致 | ||
| 273 | +// String formattedDesc = sDescriptions.replace("\n", "\n "); | ||
| 274 | +// sFiledDescriptions.add(String.format(" - %s: %s", sName, formattedDesc)); | ||
| 275 | + String formattedDesc =String.format("%s: %s", sName, sDescriptions); | ||
| 276 | + sFiledDescriptions.add(formattedDesc); | ||
| 277 | + title.put("sName",sName); | ||
| 278 | + title.put("sTitle",sDescriptions); | ||
| 279 | + titleList.add(title); | ||
| 280 | + } | ||
| 281 | + Map<String,Object> rMap = new HashMap<>(); | ||
| 282 | + rMap.put("sMilvusFiled", String.join(",", sFileds)); | ||
| 283 | + rMap.put("sMilvusFiledDescription", String.join(",", sFiledDescriptions)); | ||
| 284 | + rMap.put("sFileds", sFileds); | ||
| 285 | + rMap.put("title", titleList); | ||
| 286 | + return rMap; | ||
| 287 | + } | ||
| 288 | + | ||
| 289 | + @Override | ||
| 290 | + public List<Map<String, Object>> getDataToCollection(String collectionName, String milvusFilter,String searchText,Integer size,List<String> fields){ | ||
| 291 | + log.info("开始相似度查询: collection={}, searchText={}", collectionName, searchText); | ||
| 292 | + // 2. 设置范围搜索参数 | ||
| 293 | + Map<String, Object> searchParams = new HashMap<>(); | ||
| 294 | + searchParams.put("nprobe", 10); | ||
| 295 | + // 对于 IP 度量，相似度范围在 [minScore, maxScore] | ||
| 296 | + searchParams.put("radius", 0.9); // 最小相似度 | ||
| 297 | + searchParams.put("range_filter", 1); // 最大相似度 | ||
| 298 | + // 1. 确保集合已加载 | ||
| 299 | +// ensureCollectionLoaded(collectionName); | ||
| 300 | + // 1. 向量化搜索文本 | ||
| 301 | + List<Float> vectorList = vectorizationService.textToVector(searchText); | ||
| 302 | + if (vectorList == null || vectorList.isEmpty()) { | ||
| 303 | + throw new RuntimeException("向量化失败"); | ||
| 304 | + } | ||
| 305 | + // 2. 转换为 float[] | ||
| 306 | + float[] floatArray = new float[vectorList.size()]; | ||
| 307 | + for (int i = 0; i < vectorList.size(); i++) { | ||
| 308 | + floatArray[i] = vectorList.get(i); | ||
| 309 | + } | ||
| 310 | + | ||
| 311 | + // 3. 创建 Milvus FloatVec 对象 | ||
| 312 | + FloatVec floatVec = new FloatVec(floatArray); | ||
| 313 | + log.info("查询向量库条件{}",milvusFilter); | ||
| 314 | + milvusFilter = isValidMilvusFilter(milvusFilter)?milvusFilter : null; | ||
| 315 | + log.info("实际查询向量库条件{}",milvusFilter); | ||
| 316 | + // 4. 构建搜索请求 | ||
| 317 | + SearchReq searchReq = SearchReq.builder() | ||
| 318 | + .collectionName(collectionName) | ||
| 319 | + .data(Collections.singletonList(floatVec)) | ||
| 320 | + .annsField("vector") // 向量字段名 | ||
| 321 | + .topK(size) // 返回最相似的10条 | ||
| 322 | + .metricType(IndexParam.MetricType.IP) // 内积相似度 | ||
| 323 | + .outputFields(fields) | ||
| 324 | +// .searchParams(searchParams) | ||
| 325 | + .filter(milvusFilter) | ||
| 326 | + .build(); | ||
| 327 | + // 5. 执行搜索 | ||
| 328 | + SearchResp searchResp = milvusClient.search(searchReq); | ||
| 329 | + | ||
| 330 | + // 6. 处理结果 | ||
| 331 | + return processMilvusResults(searchResp); | ||
| 332 | + } | ||
| 333 | + | ||
| 334 | + | ||
| 335 | + /** | ||
| 336 | + * 判断 Milvus 过滤条件是否有效 | ||
| 337 | + * @param milvusFilter 过滤条件字符串 | ||
| 338 | + * @return true: 有效条件, false: 无效条件 | ||
| 339 | + */ | ||
| 340 | + public boolean isValidMilvusFilter(String milvusFilter) { | ||
| 341 | + // 1. 空值判断 | ||
| 342 | + if (milvusFilter == null || milvusFilter.trim().isEmpty()) { | ||
| 343 | + return false; | ||
| 344 | + } | ||
| 345 | + | ||
| 346 | + String filter = milvusFilter.trim(); | ||
| 347 | + | ||
| 348 | + // 2. 基本格式检查：不能是纯布尔值 | ||
| 349 | + if ("true".equalsIgnoreCase(filter) || "false".equalsIgnoreCase(filter)) { | ||
| 350 | + return false; | ||
| 351 | + } | ||
| 352 | + | ||
| 353 | + // 3. 检查是否包含有效的操作符 | ||
| 354 | + boolean hasValidOperator = filter.matches(".*[=!<>]=?.*") || filter.contains(" like "); | ||
| 355 | + if (!hasValidOperator) { | ||
| 356 | + return false; | ||
| 357 | + } | ||
| 358 | + | ||
| 359 | + // 4. 检查字符串值是否使用单引号包裹 | ||
| 360 | + // 匹配模式：字段名 操作符 '值' | ||
| 361 | + Pattern pattern = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*\\s*(==|!=|>=|<=|>|<|like)\\s*('[^']*'|\\d+)"); | ||
| 362 | + Matcher matcher = pattern.matcher(filter); | ||
| 363 | + | ||
| 364 | + // 5. 对于复合条件，递归检查 | ||
| 365 | + if (filter.contains("&&") || filter.contains("||")) { | ||
| 366 | + // 分割复合条件（简单处理，生产环境需要更完善的解析） | ||
| 367 | + String[] conditions = filter.split("&&|\\|\\|"); | ||
| 368 | + for (String condition : conditions) { | ||
| 369 | + condition = condition.trim().replaceAll("^[()]+|[()]+$", ""); // 去除括号 | ||
| 370 | + if (!isValidSimpleCondition(condition)) { | ||
| 371 | + return false; | ||
| 372 | + } | ||
| 373 | + } | ||
| 374 | + return true; | ||
| 375 | + } | ||
| 376 | + | ||
| 377 | + // 6. 检查简单条件 | ||
| 378 | + return isValidSimpleCondition(filter); | ||
| 379 | + } | ||
| 380 | + | ||
| 381 | + /** | ||
| 382 | + * 验证简单条件（不包含 && 和 ||） | ||
| 383 | + */ | ||
| 384 | + private boolean isValidSimpleCondition(String condition) { | ||
| 385 | + if (condition == null || condition.trim().isEmpty()) { | ||
| 386 | + return false; | ||
| 387 | + } | ||
| 388 | + condition = condition.trim(); | ||
| 389 | + // 匹配简单条件的正则 | ||
| 390 | + // 格式：字段名 操作符 值 | ||
| 391 | + // 字段名：字母开头，包含字母数字下划线 | ||
| 392 | + // 操作符：==, !=, >=, <=, >, <, like | ||
| 393 | + // 值：单引号字符串 或 数字 | ||
| 394 | + String regex = "^\\s*([a-zA-Z_][a-zA-Z0-9_]*)\\s*" + // 字段名 | ||
| 395 | + "(==|!=|>=|<=|>|<|like)\\s*" + // 操作符 | ||
| 396 | + "('([^'\\\\]|\\\\.)*'|\\d+(\\.\\d+)?)\\s*$"; // 值 | ||
| 397 | + | ||
| 398 | + if (!condition.matches(regex)) { | ||
| 399 | + return false; | ||
| 400 | + } | ||
| 401 | + // 额外检查：like 操作符的值必须包含 % | ||
| 402 | + if (condition.contains(" like ")) { | ||
| 403 | + String value = condition.split("like")[1].trim(); | ||
| 404 | + if (!value.contains("%")) { | ||
| 405 | + return false; // like 必须使用 % 通配符 | ||
| 406 | + } | ||
| 407 | + } | ||
| 408 | + return true; | ||
| 409 | + } | ||
| 410 | + | ||
| 411 | + | ||
| 412 | + /** | ||
| 413 | + * 处理 Milvus 查询结果（完整版） | ||
| 414 | + */ | ||
| 415 | + /** | ||
| 416 | + * 处理 Milvus 查询结果（完整版） | ||
| 417 | + */ | ||
| 418 | + private List<Map<String, Object>> processMilvusResults(SearchResp response) { | ||
| 419 | + List<Map<String, Object>> results = new ArrayList<>(); | ||
| 420 | + if (response == null) { | ||
| 421 | + log.warn("Milvus 响应为空"); | ||
| 422 | + return results; | ||
| 423 | + } | ||
| 424 | + List<List<SearchResp.SearchResult>> searchResults = response.getSearchResults(); | ||
| 425 | + if (searchResults == null || searchResults.isEmpty()) { | ||
| 426 | + log.warn("Milvus 搜索结果为空"); | ||
| 427 | + return results; | ||
| 428 | + } | ||
| 429 | + // 遍历每个查询的结果集（通常只有一个查询） | ||
| 430 | + for (List<SearchResp.SearchResult> resultList : searchResults) { | ||
| 431 | + // 遍历每个搜索结果 | ||
| 432 | + for (SearchResp.SearchResult result : resultList) { | ||
| 433 | + Map<String, Object> item = new HashMap<>(); | ||
| 434 | + // 获取相似度分数 | ||
| 435 | + Float score = result.getScore(); | ||
| 436 | + if (score != null) { | ||
| 437 | + item.put("score", score); | ||
| 438 | + } | ||
| 439 | + // 获取实体字段数据 | ||
| 440 | + Map<String, Object> entity = result.getEntity(); | ||
| 441 | + // 将所有字段添加到结果中 | ||
| 442 | + item.putAll(entity); | ||
| 443 | + results.add(item); | ||
| 444 | + } | ||
| 445 | + } | ||
| 446 | + log.info("处理完成，共 {} 条搜索结果", results.size()); | ||
| 447 | + return results; | ||
| 448 | + } | ||
| 449 | + /** | ||
| 450 | + * 从实体对象构建Milvus插入数据 | ||
| 451 | + */ | ||
| 452 | + public List<JsonObject> convertToMilvusRow(List<Map<String,Object>> data, String sVectorfiled,String sVectorjson) { | ||
| 453 | + List<JsonObject> rows = new ArrayList<>(); | ||
| 454 | + if (CollUtil.isEmpty(data)) { | ||
| 455 | + return rows; | ||
| 456 | + } | ||
| 457 | + // 批量遍历，逐个转换 | ||
| 458 | + for (Map<String, Object> map : data) { | ||
| 459 | + JsonObject jsonObject = convertToMilvusRowOne(map, sVectorfiled, sVectorjson); | ||
| 460 | + rows.add(jsonObject); | ||
| 461 | + } | ||
| 462 | + return rows; | ||
| 463 | + } | ||
| 464 | + | ||
| 465 | + /*** | ||
| 466 | + * @Author 钱豹 | ||
| 467 | + * @Date 21:31 2026/3/24 | ||
| 468 | + * @Param [data, sVectorfiled, sVectorjson] | ||
| 469 | + * @return com.google.gson.JsonObject | ||
| 470 | + * @Description 单个转换 | ||
| 471 | + **/ | ||
| 472 | + public JsonObject convertToMilvusRowOne(Map<String,Object> data, String sVectorfiled,String sVectorjson) { | ||
| 473 | + | ||
| 474 | + // ====================== 修复 1：使用真实的向量化文本 ====================== | ||
| 475 | + // 从 sVectorjson 或 data 中获取要向量化的字段值 | ||
| 476 | + StringBuffer vectorText = new StringBuffer(); | ||
| 477 | + getVectorText(data, vectorText, sVectorjson); | ||
| 478 | + // 向量化 | ||
| 479 | + List<Float> vector = vectorizationService.textToVector(vectorText.toString()); | ||
| 480 | + if (vector == null || vector.isEmpty()) { | ||
| 481 | + throw new RuntimeException("向量化失败，文本内容：" + vectorText); | ||
| 482 | + } | ||
| 483 | + JsonObject row = new JsonObject(); | ||
| 484 | + // 添加向量 | ||
| 485 | + JsonArray vectorArray = new JsonArray(); | ||
| 486 | + vector.forEach(vectorArray::add); | ||
| 487 | + row.add("vector", vectorArray); | ||
| 488 | + | ||
| 489 | + // ====================== 修复 2：sSlaveId 空值安全 ====================== | ||
| 490 | + Object slaveIdObj = data.get("sSlaveId"); | ||
| 491 | + if (slaveIdObj == null) { | ||
| 492 | + throw new RuntimeException("数据中缺少 sSlaveId 字段，无法插入Milvus"); | ||
| 493 | + } | ||
| 494 | + row.addProperty("sSlaveId", slaveIdObj.toString()); | ||
| 495 | + | ||
| 496 | + // 创建时间 | ||
| 497 | + row.addProperty("create_time", System.currentTimeMillis()); | ||
| 498 | + | ||
| 499 | + // 业务元数据 | ||
| 500 | + JsonObject metadata = MapToJsonConverter.convert(data); | ||
| 501 | + row.add("metadata", metadata); | ||
| 502 | + | ||
| 503 | + // 动态字段 | ||
| 504 | + String[] sVectorfiledArray = sVectorfiled.split(","); | ||
| 505 | + for (String vectorFieldOne : sVectorfiledArray) { | ||
| 506 | + String[] fieldArr = vectorFieldOne.split(":"); | ||
| 507 | + if (fieldArr.length < 2) { | ||
| 508 | + continue; | ||
| 509 | + } | ||
| 510 | + String fieldName = fieldArr[1]; | ||
| 511 | + Object value = ObjectUtil.isEmpty(data.get(fieldName)) ? getDefaultData(fieldName) : data.get(fieldName); | ||
| 512 | + // 通用类型安全添加（你之前的优化方法） | ||
| 513 | + addToJsonObject(row, fieldName, value); | ||
| 514 | + } | ||
| 515 | + | ||
| 516 | + return row; | ||
| 517 | + } | ||
| 518 | + | ||
| 519 | + private void getVectorText(Map<String,Object> data, StringBuffer vectorText,String sVectorjson){ | ||
| 520 | + // 动态字段 | ||
| 521 | + String[] sVectorjsonArray = sVectorjson.split(";"); | ||
| 522 | + for (String sVectorjsonOne : sVectorjsonArray) { | ||
| 523 | + String sText; | ||
| 524 | + String[] fieldArr = sVectorjsonOne.split(":"); | ||
| 525 | + if (fieldArr.length < 2) { | ||
| 526 | + continue; | ||
| 527 | + } | ||
| 528 | + String fieldName = fieldArr[1]; | ||
| 529 | + Object value = ObjectUtil.isEmpty(data.get(fieldName)) ? getDefaultData(fieldName) : data.get(fieldName); | ||
| 530 | + if (ObjectUtil.isEmpty(value)) { | ||
| 531 | + sText = StrUtil.EMPTY; | ||
| 532 | + }else{ | ||
| 533 | + sText = value.toString(); | ||
| 534 | + } | ||
| 535 | + vectorText.append(" ").append(fieldArr[0]).append(sText); | ||
| 536 | + } | ||
| 537 | + } | ||
| 538 | + | ||
| 539 | + | ||
| 540 | + /*******************************************************内部方法********************************************************************************/ | ||
| 541 | + | ||
| 542 | + /** | ||
| 543 | + * 安全将 Object 加入 Gson JsonObject，自动识别类型 | ||
| 544 | + */ | ||
| 545 | + private void addToJsonObject(JsonObject row, String fieldName, Object value) { | ||
| 546 | + if (value == null) { | ||
| 547 | + row.addProperty(fieldName, ""); | ||
| 548 | + return; | ||
| 549 | + } | ||
| 550 | + // 基本类型直接添加 | ||
| 551 | + if (value instanceof String) { | ||
| 552 | + row.addProperty(fieldName, (String) value); | ||
| 553 | + } else if (value instanceof Number) { | ||
| 554 | + row.addProperty(fieldName, (Number) value); | ||
| 555 | + } else if (value instanceof Boolean) { | ||
| 556 | + row.addProperty(fieldName, (Boolean) value); | ||
| 557 | + } | ||
| 558 | + // List / 数组类型 | ||
| 559 | + else if (value instanceof List<?>) { | ||
| 560 | + JsonArray jsonArray = new JsonArray(); | ||
| 561 | + for (Object item : (List<?>) value) { | ||
| 562 | + addJsonElement(jsonArray, item); | ||
| 563 | + } | ||
| 564 | + row.add(fieldName, jsonArray); | ||
| 565 | + } | ||
| 566 | + // 其他对象转字符串 | ||
| 567 | + else { | ||
| 568 | + row.addProperty(fieldName, value.toString()); | ||
| 569 | + } | ||
| 570 | + } | ||
| 571 | + /** | ||
| 572 | + * 递归处理 List 元素（支持无限层嵌套 List） | ||
| 573 | + */ | ||
| 574 | + private void addJsonElement(JsonArray jsonArray, Object item) { | ||
| 575 | + if (item == null) { | ||
| 576 | + jsonArray.add((String) null); | ||
| 577 | + return; | ||
| 578 | + } | ||
| 579 | + if (item instanceof String) { | ||
| 580 | + jsonArray.add((String) item); | ||
| 581 | + } else if (item instanceof Number) { | ||
| 582 | + jsonArray.add((Number) item); | ||
| 583 | + } else if (item instanceof Boolean) { | ||
| 584 | + jsonArray.add((Boolean) item); | ||
| 585 | + } else if (item instanceof List<?>) { | ||
| 586 | + // 递归处理嵌套列表 | ||
| 587 | + JsonArray nestedArray = new JsonArray(); | ||
| 588 | + for (Object nestedItem : (List<?>) item) { | ||
| 589 | + addJsonElement(nestedArray, nestedItem); | ||
| 590 | + } | ||
| 591 | + jsonArray.add(nestedArray); | ||
| 592 | + } else { | ||
| 593 | + jsonArray.add(item.toString()); | ||
| 594 | + } | ||
| 595 | + } | ||
| 596 | + | ||
| 597 | + /** | ||
| 598 | + * 创建集合（定义字段结构） | ||
| 599 | + */ | ||
| 600 | + private void createCollection(String collectionName,String sVectorfiled,Boolean bRset) { | ||
| 601 | + // 6. 重新加载集合 | ||
| 602 | + LoadCollectionReq loadReq = LoadCollectionReq.builder() | ||
| 603 | + .collectionName(collectionName) | ||
| 604 | + .build(); | ||
| 605 | + | ||
| 606 | + //是否删除集合 重新创建 | ||
| 607 | + if (bRset){ | ||
| 608 | + // 1. 删除旧集合 | ||
| 609 | + milvusClient.dropCollection(DropCollectionReq.builder() | ||
| 610 | + .collectionName(collectionName) | ||
| 611 | + .build()); | ||
| 612 | + //删除对应的记录表 | ||
| 613 | + delAiMilvusVectorRecord(collectionName); | ||
| 614 | + } | ||
| 615 | + // 检查集合是否存在 | ||
| 616 | + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() | ||
| 617 | + .collectionName(collectionName) | ||
| 618 | + .build(); | ||
| 619 | + boolean exists = milvusClient.hasCollection(hasCollectionReq); | ||
| 620 | + if (exists) { | ||
| 621 | + return; | ||
| 622 | + } | ||
| 623 | + // 定义字段列表 | ||
| 624 | + List<CreateCollectionReq.FieldSchema> fieldSchemas = new ArrayList<>(); | ||
| 625 | + // 1. 准备所有索引参数 | ||
| 626 | + List<IndexParam> allIndexParams = new ArrayList<>(); | ||
| 627 | + //定义字段列表 | ||
| 628 | + // 1. 主键字段 | ||
| 629 | + fieldSchemas.add( CreateCollectionReq.FieldSchema.builder() | ||
| 630 | + .name("id") | ||
| 631 | + .dataType(DataType.Int64) | ||
| 632 | + .isPrimaryKey(true) | ||
| 633 | + .autoID(true) // 使用自动ID | ||
| 634 | + .description("主键ID") | ||
| 635 | + .build()); | ||
| 636 | + // 3. 主键字段 | ||
| 637 | + fieldSchemas.add(CreateCollectionReq.FieldSchema.builder() | ||
| 638 | + .name("sSlaveId") | ||
| 639 | + .dataType(DataType.VarChar) | ||
| 640 | +// .isPrimaryKey(true) //索引创建 | ||
| 641 | + .maxLength(100) | ||
| 642 | + .description("原始数据主键ID") | ||
| 643 | + .build()); | ||
| 644 | + // 2. 向量字段 | ||
| 645 | + fieldSchemas.add(CreateCollectionReq.FieldSchema.builder() | ||
| 646 | + .name("vector") | ||
| 647 | + .dataType(DataType.FloatVector) | ||
| 648 | + .dimension(VECTOR_DIM) | ||
| 649 | + .description("向量字段，用于相似性搜索") | ||
| 650 | + .build()); | ||
| 651 | + | ||
| 652 | + // 4. 创建时间字段 | ||
| 653 | + fieldSchemas.add( CreateCollectionReq.FieldSchema.builder() | ||
| 654 | + .name("create_time") | ||
| 655 | + .dataType(DataType.Int64) | ||
| 656 | + .description("创建时间戳") | ||
| 657 | + .build()); | ||
| 658 | + // 5. 元数据字段（使用JSON类型存储额外数据） | ||
| 659 | + fieldSchemas.add(CreateCollectionReq.FieldSchema.builder() | ||
| 660 | + .name("metadata") | ||
| 661 | + .dataType(DataType.JSON) | ||
| 662 | + .description("额外元数据") | ||
| 663 | + .build()); | ||
| 664 | + //动态字段创建 | ||
| 665 | + String[] sVectorfiledArray = sVectorfiled.split(","); | ||
| 666 | + for(String sVectorfiledOne : sVectorfiledArray){ | ||
| 667 | + String[] sVectorfiledOneArray = sVectorfiledOne.split(":"); | ||
| 668 | + String sName = sVectorfiledOneArray[1]; | ||
| 669 | + String sDescription = sVectorfiledOneArray[0]; | ||
| 670 | + DataType dataType = processField(sVectorfiledOneArray[1]); | ||
| 671 | + fieldSchemas.add(CreateCollectionReq.FieldSchema.builder() | ||
| 672 | + .name(sName) | ||
| 673 | + .dataType(dataType) | ||
| 674 | + .description(sDescription) | ||
| 675 | + .isPrimaryKey(false) // 如果不是主键 | ||
| 676 | + .isNullable(true) // 允许为空 | ||
| 677 | +// .defaultValue("") // 如果有默认值 | ||
| 678 | + // SQL可能较长 | ||
| 679 | + .maxLength(1000) | ||
| 680 | + .build()); | ||
| 681 | + } | ||
| 682 | + //创建索引 | ||
| 683 | + createAllIndexes(sVectorfiled,allIndexParams); | ||
| 684 | + | ||
| 685 | + // 创建集合schema | ||
| 686 | + CreateCollectionReq.CollectionSchema schema = | ||
| 687 | + CreateCollectionReq.CollectionSchema.builder() | ||
| 688 | + .fieldSchemaList(fieldSchemas) | ||
| 689 | + .enableDynamicField(true) | ||
| 690 | + .build(); | ||
| 691 | + // 创建集合请求 | ||
| 692 | + CreateCollectionReq createCollectionReq = CreateCollectionReq.builder() | ||
| 693 | + .collectionName(collectionName) | ||
| 694 | + .collectionSchema(schema) | ||
| 695 | + .indexParams(allIndexParams)//索引集合 | ||
| 696 | + .consistencyLevel(ConsistencyLevel.BOUNDED) | ||
| 697 | + .build(); | ||
| 698 | + | ||
| 699 | + // 执行创建集合 | ||
| 700 | + milvusClient.createCollection(createCollectionReq); | ||
| 701 | + milvusClient.loadCollection(loadReq); | ||
| 702 | + log.info("集合重新加载成功"); | ||
| 703 | + } | ||
| 704 | + /** | ||
| 705 | + * 批量创建所有索引（向量索引 + 多个标量索引） | ||
| 706 | + */ | ||
| 707 | + private void createAllIndexes(String sVectorfiled,List<IndexParam> allIndexParams) { | ||
| 708 | + | ||
| 709 | + // 1.1 向量索引 | ||
| 710 | + Map<String, Object> vectorExtraParams = new HashMap<>(8); | ||
| 711 | + vectorExtraParams.put("nlist", 256); // 聚类中心数：sqrt(384) * 13 ≈ 256 | ||
| 712 | + vectorExtraParams.put("nprobe", 32); // 搜索时检查的聚类数 | ||
| 713 | + | ||
| 714 | + IndexParam vectorIndex = IndexParam.builder() | ||
| 715 | + .fieldName("vector") | ||
| 716 | + .indexName("idx_vector_rebuild") | ||
| 717 | + .indexType(IndexParam.IndexType.IVF_FLAT) | ||
| 718 | + .metricType(IndexParam.MetricType.IP) | ||
| 719 | + .extraParams(vectorExtraParams) | ||
| 720 | + .build(); | ||
| 721 | + allIndexParams.add(vectorIndex); | ||
| 722 | + | ||
| 723 | + // 1.2 create_time 字段索引（用于时间范围查询） | ||
| 724 | + IndexParam timeIndex = IndexParam.builder() | ||
| 725 | + .fieldName("create_time") | ||
| 726 | + .indexName("idx_create_time") | ||
| 727 | + .indexType(IndexParam.IndexType.STL_SORT) // 排序索引 | ||
| 728 | + .build(); | ||
| 729 | + allIndexParams.add(timeIndex); | ||
| 730 | + | ||
| 731 | + // 1.4 data_id 字段索引（用于精确匹配） | ||
| 732 | + IndexParam idIndex = IndexParam.builder() | ||
| 733 | + .fieldName("sSlaveId") | ||
| 734 | + .indexName("idx_data_id") | ||
| 735 | + .indexType(IndexParam.IndexType.TRIE) | ||
| 736 | + .build(); | ||
| 737 | + allIndexParams.add(idIndex); | ||
| 738 | + //动态字段创建 | ||
| 739 | + String[] sVectorfiledArray = sVectorfiled.split(","); | ||
| 740 | + for(String sVectorfiledOne : sVectorfiledArray){ | ||
| 741 | + String[] sVectorfiledOneArray = sVectorfiledOne.split(":"); | ||
| 742 | + String sName = sVectorfiledOneArray[1]; | ||
| 743 | + IndexParam.IndexType indexType =indexField(sVectorfiledOneArray[1]); | ||
| 744 | + allIndexParams.add(IndexParam.builder() | ||
| 745 | + .fieldName(sName) | ||
| 746 | + .indexName(sName) | ||
| 747 | + .indexType(indexType) | ||
| 748 | + .build()); | ||
| 749 | + } | ||
| 750 | + | ||
| 751 | + } | ||
| 752 | + | ||
| 753 | + /*** | ||
| 754 | + * @Author 钱豹 | ||
| 755 | + * @Date 21:10 2026/3/24 | ||
| 756 | + * @Param | ||
| 757 | + * @return | ||
| 758 | + * @Description //TODO | ||
| 759 | + **/ | ||
| 760 | + public Object getDefaultData(String sKey) { | ||
| 761 | + if(sKey.startsWith("d") || sKey.startsWith("i")){ | ||
| 762 | + return BigDecimal.ZERO; | ||
| 763 | + }else if(sKey.startsWith("b")){ | ||
| 764 | + return false; | ||
| 765 | + }else{ | ||
| 766 | + return StrUtil.EMPTY; | ||
| 767 | + } | ||
| 768 | + } | ||
| 769 | + /*** | ||
| 770 | + * @Author 钱豹 | ||
| 771 | + * @Date 20:44 2026/3/24 | ||
| 772 | + * @Param [sKey] | ||
| 773 | + * @return io.milvus.v2.common.DataType | ||
| 774 | + * @Description 字段类型 | ||
| 775 | + **/ | ||
| 776 | + public DataType processField(String sKey) { | ||
| 777 | + if(sKey.startsWith("d")){ | ||
| 778 | + return DataType.Double; | ||
| 779 | + }else if(sKey.startsWith("i")){ | ||
| 780 | + return DataType.Int64; | ||
| 781 | + }else if(sKey.startsWith("b")){ | ||
| 782 | + return DataType.Bool; | ||
| 783 | + }else{ | ||
| 784 | + return DataType.VarChar; | ||
| 785 | + } | ||
| 786 | + } | ||
| 787 | + | ||
| 788 | + /*** | ||
| 789 | + * @Author 钱豹 | ||
| 790 | + * @Date 20:44 2026/3/24 | ||
| 791 | + * @Param [sKey] | ||
| 792 | + * @return io.milvus.v2.common.DataType | ||
| 793 | + * @Description 索引类型 | ||
| 794 | + **/ | ||
| 795 | + public IndexParam.IndexType indexField(String sKey) { | ||
| 796 | + return IndexParam.IndexType.TRIE; | ||
| 797 | + } | ||
| 798 | + | ||
| 799 | + | ||
| 800 | + | ||
| 801 | + | ||
| 802 | + | ||
| 803 | +} | ||
| 0 | \ No newline at end of file | 804 | \ No newline at end of file |
src/main/java/com/xly/milvus/service/impl/VectorizationServiceImpl.java
0 → 100644
| 1 | +package com.xly.milvus.service.impl; | ||
| 2 | + | ||
| 3 | +import com.xly.milvus.service.VectorizationService; | ||
| 4 | +import dev.langchain4j.model.embedding.EmbeddingModel; | ||
| 5 | +import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; | ||
| 6 | +import lombok.extern.slf4j.Slf4j; | ||
| 7 | +import org.springframework.stereotype.Service; | ||
| 8 | + | ||
| 9 | +import java.util.ArrayList; | ||
| 10 | +import java.util.List; | ||
| 11 | +import java.util.stream.Collectors; | ||
| 12 | + | ||
| 13 | +/** | ||
| 14 | + * 向量化服务实现 - 使用LangChain4j的All-MiniLM-L6-v2模型 | ||
| 15 | + */ | ||
| 16 | +@Slf4j | ||
| 17 | +@Service | ||
| 18 | +public class VectorizationServiceImpl implements VectorizationService { | ||
| 19 | + | ||
| 20 | + private final EmbeddingModel embeddingModel; | ||
| 21 | + | ||
| 22 | + public VectorizationServiceImpl() { | ||
| 23 | + // 初始化嵌入模型 | ||
| 24 | + this.embeddingModel = new AllMiniLmL6V2EmbeddingModel(); | ||
| 25 | + log.info("向量化模型初始化成功"); | ||
| 26 | + } | ||
| 27 | + | ||
| 28 | + @Override | ||
| 29 | + public List<Float> textToVector(String text) { | ||
| 30 | + if (text == null || text.trim().isEmpty()) { | ||
| 31 | + return new ArrayList<>(); | ||
| 32 | + } | ||
| 33 | + | ||
| 34 | + try { | ||
| 35 | + // 使用LangChain4j生成向量 | ||
| 36 | + dev.langchain4j.data.embedding.Embedding embedding = embeddingModel.embed(text).content(); | ||
| 37 | + float[] vectorArray = embedding.vector(); | ||
| 38 | + | ||
| 39 | + // 转换为List<Float> | ||
| 40 | + List<Float> vector = new ArrayList<>(); | ||
| 41 | + for (float v : vectorArray) { | ||
| 42 | + vector.add(v); | ||
| 43 | + } | ||
| 44 | + | ||
| 45 | + return vector; | ||
| 46 | + } catch (Exception e) { | ||
| 47 | + log.error("文本向量化失败: {}", e.getMessage(), e); | ||
| 48 | + throw new RuntimeException("文本向量化失败", e); | ||
| 49 | + } | ||
| 50 | + } | ||
| 51 | + | ||
| 52 | + @Override | ||
| 53 | + public List<List<Float>> batchTextToVector(List<String> texts) { | ||
| 54 | + if (texts == null || texts.isEmpty()) { | ||
| 55 | + return new ArrayList<>(); | ||
| 56 | + } | ||
| 57 | + | ||
| 58 | + return texts.stream() | ||
| 59 | + .map(this::textToVector) | ||
| 60 | + .collect(Collectors.toList()); | ||
| 61 | + } | ||
| 62 | +} | ||
| 0 | \ No newline at end of file | 63 | \ No newline at end of file |
src/main/java/com/xly/milvus/test/MilvusCompleteTest.java
0 → 100644
| 1 | +package com.xly.milvus.test; | ||
| 2 | + | ||
| 3 | +import com.google.gson.JsonArray; | ||
| 4 | +import com.google.gson.JsonObject; | ||
| 5 | +import io.milvus.v2.client.ConnectConfig; | ||
| 6 | +import io.milvus.v2.client.MilvusClientV2; | ||
| 7 | +import io.milvus.v2.common.DataType; | ||
| 8 | +import io.milvus.v2.common.IndexParam; | ||
| 9 | +import io.milvus.v2.common.ConsistencyLevel; | ||
| 10 | +import io.milvus.v2.service.collection.request.*; | ||
| 11 | +import io.milvus.v2.service.collection.response.GetCollectionStatsResp; | ||
| 12 | +import io.milvus.v2.service.index.request.CreateIndexReq; | ||
| 13 | +import io.milvus.v2.service.vector.request.InsertReq; | ||
| 14 | +import io.milvus.v2.service.vector.request.SearchReq; | ||
| 15 | +import io.milvus.v2.service.vector.request.QueryReq; | ||
| 16 | +import io.milvus.v2.service.vector.request.data.FloatVec; | ||
| 17 | +import io.milvus.v2.service.vector.response.InsertResp; | ||
| 18 | +import io.milvus.v2.service.vector.response.SearchResp; | ||
| 19 | +import io.milvus.v2.service.vector.response.QueryResp; | ||
| 20 | + | ||
| 21 | +import java.util.*; | ||
| 22 | +import java.util.concurrent.TimeUnit; | ||
| 23 | + | ||
| 24 | +/** | ||
| 25 | + * Milvus Java SDK 2.6.15 完整测试代码 | ||
| 26 | + * | ||
| 27 | + * 功能： | ||
| 28 | + * 1. 连接Milvus服务器 | ||
| 29 | + * 2. 创建集合（包含多种字段类型） | ||
| 30 | + * 3. 创建向量索引 | ||
| 31 | + * 4. 插入测试数据 | ||
| 32 | + * 5. 查询数据（标量过滤） | ||
| 33 | + * 6. 向量相似性搜索 | ||
| 34 | + * 7. 获取集合统计信息 | ||
| 35 | + * 8. 清理资源 | ||
| 36 | + * | ||
| 37 | + * @author xly | ||
| 38 | + * @version 1.0 | ||
| 39 | + */ | ||
| 40 | +public class MilvusCompleteTest { | ||
| 41 | + | ||
| 42 | + // ==================== 配置参数 ==================== | ||
| 43 | + private static final String MILVUS_HOST = "121.43.128.225"; // 您的Milvus服务器IP | ||
| 44 | + private static final int MILVUS_PORT = 19530; // Milvus服务端口 | ||
| 45 | + private static final String COLLECTION_NAME = "complete_test"; // 集合名称 | ||
| 46 | + private static final int VECTOR_DIM = 128; // 向量维度 | ||
| 47 | + private static final int INSERT_COUNT = 15; // 插入数据条数 | ||
| 48 | + | ||
| 49 | + private MilvusClientV2 client; | ||
| 50 | + | ||
| 51 | + public static void main(String[] args) { | ||
| 52 | + MilvusCompleteTest test = new MilvusCompleteTest(); | ||
| 53 | + try { | ||
| 54 | + // 执行测试流程 | ||
| 55 | + test.run(); | ||
| 56 | + } catch (Exception e) { | ||
| 57 | + System.err.println("❌ 测试过程中发生错误: " + e.getMessage()); | ||
| 58 | + e.printStackTrace(); | ||
| 59 | + } | ||
| 60 | + } | ||
| 61 | + | ||
| 62 | + /** | ||
| 63 | + * 执行完整的测试流程 | ||
| 64 | + */ | ||
| 65 | + private void run() { | ||
| 66 | + printHeader("Milvus Java SDK 2.6.15 完整测试"); | ||
| 67 | + | ||
| 68 | + // 1. 连接Milvus | ||
| 69 | + connect(); | ||
| 70 | + | ||
| 71 | + try { | ||
| 72 | + // 2. 清理旧集合 | ||
| 73 | + cleanup(); | ||
| 74 | + | ||
| 75 | + // 3. 创建集合 | ||
| 76 | + createCollection(); | ||
| 77 | + | ||
| 78 | + // 4. 创建索引 | ||
| 79 | + createIndex(); | ||
| 80 | + | ||
| 81 | + // 5. 插入数据 | ||
| 82 | + insertData(); | ||
| 83 | + | ||
| 84 | + // 6. 查询数据 | ||
| 85 | + queryData(); | ||
| 86 | + | ||
| 87 | + // 7. 向量搜索 | ||
| 88 | + searchData(); | ||
| 89 | + | ||
| 90 | + // 8. 获取集合统计 | ||
| 91 | + getCollectionStats(); | ||
| 92 | + | ||
| 93 | + printSuccess("所有测试完成！"); | ||
| 94 | + | ||
| 95 | + } catch (Exception e) { | ||
| 96 | + System.err.println("❌ 测试失败: " + e.getMessage()); | ||
| 97 | + throw e; | ||
| 98 | + } finally { | ||
| 99 | + // 9. 清理资源 | ||
| 100 | + releaseAndClose(); | ||
| 101 | + } | ||
| 102 | + } | ||
| 103 | + | ||
| 104 | + /** | ||
| 105 | + * 连接Milvus服务器 | ||
| 106 | + */ | ||
| 107 | + /** | ||
| 108 | + * 连接Milvus服务器 | ||
| 109 | + */ | ||
| 110 | + private void connect() { | ||
| 111 | + printSection("连接Milvus服务器"); | ||
| 112 | + | ||
| 113 | + try { | ||
| 114 | + ConnectConfig connectConfig = ConnectConfig.builder() | ||
| 115 | + .uri("http://" + MILVUS_HOST + ":" + MILVUS_PORT) | ||
| 116 | + .connectTimeoutMs(TimeUnit.SECONDS.toMillis(30)) // 连接超时30秒 | ||
| 117 | + .keepAliveTimeMs(TimeUnit.MINUTES.toMillis(5)) // 保持连接5分钟 | ||
| 118 | + .keepAliveTimeoutMs(TimeUnit.SECONDS.toMillis(5)) // keep-alive超时5秒 | ||
| 119 | + .keepAliveWithoutCalls(true) // 无调用时保持连接 | ||
| 120 | + .rpcDeadlineMs(TimeUnit.SECONDS.toMillis(10)) // RPC超时10秒 | ||
| 121 | + .enablePrecheck(true) // 启用预检查 | ||
| 122 | + .build(); | ||
| 123 | + | ||
| 124 | + client = new MilvusClientV2(connectConfig); | ||
| 125 | + | ||
| 126 | + // 验证连接 | ||
| 127 | + String serverVersion = client.getServerVersion(); | ||
| 128 | + System.out.println("✅ 连接成功！"); | ||
| 129 | + System.out.println(" - 服务器地址: " + MILVUS_HOST + ":" + MILVUS_PORT); | ||
| 130 | + System.out.println(" - 服务器版本: " + serverVersion); | ||
| 131 | + | ||
| 132 | + } catch (Exception e) { | ||
| 133 | + System.err.println("❌ 连接失败: " + e.getMessage()); | ||
| 134 | + throw new RuntimeException("无法连接到Milvus服务器", e); | ||
| 135 | + } | ||
| 136 | + } | ||
| 137 | + | ||
| 138 | + /** | ||
| 139 | + * 清理已存在的集合 | ||
| 140 | + */ | ||
| 141 | + private void cleanup() { | ||
| 142 | + printSection("清理环境"); | ||
| 143 | + | ||
| 144 | + HasCollectionReq hasCollectionReq = HasCollectionReq.builder() | ||
| 145 | + .collectionName(COLLECTION_NAME) | ||
| 146 | + .build(); | ||
| 147 | + | ||
| 148 | + boolean exists = client.hasCollection(hasCollectionReq); | ||
| 149 | + if (exists) { | ||
| 150 | + DropCollectionReq dropCollectionReq = DropCollectionReq.builder() | ||
| 151 | + .collectionName(COLLECTION_NAME) | ||
| 152 | + .build(); | ||
| 153 | + client.dropCollection(dropCollectionReq); | ||
| 154 | + System.out.println("✅ 已删除旧集合: " + COLLECTION_NAME); | ||
| 155 | + } else { | ||
| 156 | + System.out.println("⏭️ 无需清理，集合不存在"); | ||
| 157 | + } | ||
| 158 | + } | ||
| 159 | + | ||
| 160 | + /** | ||
| 161 | + * 创建集合 | ||
| 162 | + */ | ||
| 163 | + private void createCollection() { | ||
| 164 | + printSection("创建集合"); | ||
| 165 | + | ||
| 166 | + // 定义字段列表 | ||
| 167 | + List<CreateCollectionReq.FieldSchema> fieldSchemas = Arrays.asList( | ||
| 168 | + // 1. 主键字段 | ||
| 169 | + CreateCollectionReq.FieldSchema.builder() | ||
| 170 | + .name("id") | ||
| 171 | + .dataType(DataType.Int64) | ||
| 172 | + .isPrimaryKey(true) | ||
| 173 | + .autoID(false) | ||
| 174 | + .description("主键ID") | ||
| 175 | + .build(), | ||
| 176 | + | ||
| 177 | + // 2. 向量字段 | ||
| 178 | + CreateCollectionReq.FieldSchema.builder() | ||
| 179 | + .name("vector") | ||
| 180 | + .dataType(DataType.FloatVector) | ||
| 181 | + .dimension(VECTOR_DIM) | ||
| 182 | + .description("向量字段，用于相似性搜索") | ||
| 183 | + .build(), | ||
| 184 | + | ||
| 185 | + // 3. 标题字段 | ||
| 186 | + CreateCollectionReq.FieldSchema.builder() | ||
| 187 | + .name("title") | ||
| 188 | + .dataType(DataType.VarChar) | ||
| 189 | + .maxLength(200) | ||
| 190 | + .description("标题") | ||
| 191 | + .build(), | ||
| 192 | + | ||
| 193 | + // 4. 内容字段 | ||
| 194 | + CreateCollectionReq.FieldSchema.builder() | ||
| 195 | + .name("content") | ||
| 196 | + .dataType(DataType.VarChar) | ||
| 197 | + .maxLength(1000) | ||
| 198 | + .description("详细内容") | ||
| 199 | + .build(), | ||
| 200 | + | ||
| 201 | + // 5. 分类字段 | ||
| 202 | + CreateCollectionReq.FieldSchema.builder() | ||
| 203 | + .name("category") | ||
| 204 | + .dataType(DataType.VarChar) | ||
| 205 | + .maxLength(50) | ||
| 206 | + .description("分类标签") | ||
| 207 | + .build(), | ||
| 208 | + | ||
| 209 | + // 6. 得分字段 | ||
| 210 | + CreateCollectionReq.FieldSchema.builder() | ||
| 211 | + .name("score") | ||
| 212 | + .dataType(DataType.Float) | ||
| 213 | + .description("评分") | ||
| 214 | + .build(), | ||
| 215 | + | ||
| 216 | + // 7. 时间戳字段 | ||
| 217 | + CreateCollectionReq.FieldSchema.builder() | ||
| 218 | + .name("create_time") | ||
| 219 | + .dataType(DataType.Int64) | ||
| 220 | + .description("创建时间戳") | ||
| 221 | + .build(), | ||
| 222 | + | ||
| 223 | + // 8. 标签数组字段 | ||
| 224 | + CreateCollectionReq.FieldSchema.builder() | ||
| 225 | + .name("tags") | ||
| 226 | + .dataType(DataType.Array) | ||
| 227 | + .elementType(DataType.VarChar) | ||
| 228 | + .maxCapacity(10) | ||
| 229 | + .description("标签数组") | ||
| 230 | + .build() | ||
| 231 | + ); | ||
| 232 | + | ||
| 233 | + // 创建集合schema | ||
| 234 | + CreateCollectionReq.CollectionSchema schema = | ||
| 235 | + CreateCollectionReq.CollectionSchema.builder() | ||
| 236 | + .fieldSchemaList(fieldSchemas) | ||
| 237 | + .enableDynamicField(true) // 启用动态字段 | ||
| 238 | + .build(); | ||
| 239 | + | ||
| 240 | + // 创建集合请求 | ||
| 241 | + CreateCollectionReq createCollectionReq = CreateCollectionReq.builder() | ||
| 242 | + .collectionName(COLLECTION_NAME) | ||
| 243 | + .collectionSchema(schema) | ||
| 244 | + .consistencyLevel(ConsistencyLevel.BOUNDED) // 有界一致性 | ||
| 245 | + .build(); | ||
| 246 | + | ||
| 247 | + // 执行创建 | ||
| 248 | + client.createCollection(createCollectionReq); | ||
| 249 | + System.out.println("✅ 集合创建成功: " + COLLECTION_NAME); | ||
| 250 | + System.out.println(" - 向量维度: " + VECTOR_DIM); | ||
| 251 | + System.out.println(" - 字段数量: " + fieldSchemas.size()); | ||
| 252 | + } | ||
| 253 | + | ||
| 254 | + /** | ||
| 255 | + * 创建索引 | ||
| 256 | + */ | ||
| 257 | + /** | ||
| 258 | + * 创建索引 | ||
| 259 | + */ | ||
| 260 | + private void createIndex() { | ||
| 261 | + printSection("创建索引"); | ||
| 262 | + | ||
| 263 | + // 创建索引参数 | ||
| 264 | + Map<String, Object> extraParams = new HashMap<>(); | ||
| 265 | + extraParams.put("nlist", 1024); | ||
| 266 | + | ||
| 267 | + IndexParam indexParam = IndexParam.builder() | ||
| 268 | + .fieldName("vector") | ||
| 269 | + .indexType(IndexParam.IndexType.IVF_FLAT) | ||
| 270 | + .metricType(IndexParam.MetricType.L2) | ||
| 271 | + .extraParams(extraParams) | ||
| 272 | + .build(); | ||
| 273 | + | ||
| 274 | + // 创建索引请求 | ||
| 275 | + CreateIndexReq createIndexReq = CreateIndexReq.builder() | ||
| 276 | + .collectionName(COLLECTION_NAME) | ||
| 277 | + .indexParams(Collections.singletonList(indexParam)) | ||
| 278 | + .build(); | ||
| 279 | + | ||
| 280 | + // 执行创建 | ||
| 281 | + client.createIndex(createIndexReq); | ||
| 282 | + System.out.println("✅ 索引创建成功"); | ||
| 283 | + System.out.println(" - 索引类型: IVF_FLAT"); | ||
| 284 | + System.out.println(" - 度量方式: L2 (欧氏距离)"); | ||
| 285 | + | ||
| 286 | + // 修正：使用 LoadCollectionReq 加载集合到内存 | ||
| 287 | + LoadCollectionReq loadCollectionReq = LoadCollectionReq.builder() | ||
| 288 | + .collectionName(COLLECTION_NAME) | ||
| 289 | + .build(); | ||
| 290 | + client.loadCollection(loadCollectionReq); | ||
| 291 | + System.out.println("✅ 集合已加载到内存"); | ||
| 292 | + } | ||
| 293 | + | ||
| 294 | + /** | ||
| 295 | + * 插入测试数据 | ||
| 296 | + */ | ||
| 297 | + private void insertData() { | ||
| 298 | + printSection("插入测试数据"); | ||
| 299 | + | ||
| 300 | + List<JsonObject> rows = new ArrayList<>(); | ||
| 301 | + Random random = new Random(42); // 固定种子，保证可重复性 | ||
| 302 | + | ||
| 303 | + // 测试数据 | ||
| 304 | + String[] categories = {"科技", "体育", "娱乐", "教育", "财经"}; | ||
| 305 | + String[] titles = { | ||
| 306 | + "人工智能发展现状与未来趋势", "机器学习入门实战教程", | ||
| 307 | + "深度学习在图像识别中的应用", "NBA季后赛精彩回顾", | ||
| 308 | + "世界杯预选赛最新战况", "奥运会筹备工作进展", | ||
| 309 | + "热门电影推荐排行榜", "音乐榜单TOP10", | ||
| 310 | + "综艺节目收视率分析", "在线教育行业发展报告", | ||
| 311 | + "编程语言流行趋势", "高效学习方法分享", | ||
| 312 | + "股市投资策略分析", "区块链技术应用前景", | ||
| 313 | + "数字货币市场动态" | ||
| 314 | + }; | ||
| 315 | + | ||
| 316 | + long currentTime = System.currentTimeMillis() / 1000; | ||
| 317 | + | ||
| 318 | + // 生成测试数据 | ||
| 319 | + for (int i = 1; i <= INSERT_COUNT; i++) { | ||
| 320 | + JsonObject row = new JsonObject(); | ||
| 321 | + | ||
| 322 | + // ID | ||
| 323 | + row.addProperty("id", (long) i); | ||
| 324 | + | ||
| 325 | + // 向量 | ||
| 326 | + JsonArray vectorArray = new JsonArray(); | ||
| 327 | + for (int j = 0; j < VECTOR_DIM; j++) { | ||
| 328 | + vectorArray.add(random.nextFloat()); | ||
| 329 | + } | ||
| 330 | + row.add("vector", vectorArray); | ||
| 331 | + | ||
| 332 | + // 标题 | ||
| 333 | + int index = (i - 1) % titles.length; | ||
| 334 | + row.addProperty("title", titles[index]); | ||
| 335 | + | ||
| 336 | + // 内容 | ||
| 337 | + row.addProperty("content", "这是第" + i + "条测试数据的详细内容。用于测试Milvus的插入和查询功能。"); | ||
| 338 | + | ||
| 339 | + // 分类 | ||
| 340 | + row.addProperty("category", categories[random.nextInt(categories.length)]); | ||
| 341 | + | ||
| 342 | + // 得分 | ||
| 343 | + row.addProperty("score", random.nextFloat() * 100); | ||
| 344 | + | ||
| 345 | + // 创建时间 | ||
| 346 | + row.addProperty("create_time", currentTime - random.nextInt(86400) * i); | ||
| 347 | + | ||
| 348 | + // 标签数组 | ||
| 349 | + JsonArray tagsArray = new JsonArray(); | ||
| 350 | + tagsArray.add("tag_" + random.nextInt(5)); | ||
| 351 | + tagsArray.add("tag_" + random.nextInt(5)); | ||
| 352 | + row.add("tags", tagsArray); | ||
| 353 | + | ||
| 354 | + rows.add(row); | ||
| 355 | + } | ||
| 356 | + | ||
| 357 | + // 批量插入 | ||
| 358 | + InsertReq insertReq = InsertReq.builder() | ||
| 359 | + .collectionName(COLLECTION_NAME) | ||
| 360 | + .data(rows) | ||
| 361 | + .build(); | ||
| 362 | + | ||
| 363 | + InsertResp insertResp = client.insert(insertReq); | ||
| 364 | + System.out.println("✅ 成功插入 " + insertResp.getInsertCnt() + " 条数据"); | ||
| 365 | + System.out.println(" - 数据预览:"); | ||
| 366 | + | ||
| 367 | + // 显示前3条数据的部分信息 | ||
| 368 | + for (int i = 0; i < Math.min(3, rows.size()); i++) { | ||
| 369 | + JsonObject row = rows.get(i); | ||
| 370 | + System.out.printf(" [%d] ID: %d, 标题: %s, 分类: %s, 得分: %.2f%n", | ||
| 371 | + i + 1, | ||
| 372 | + row.get("id").getAsLong(), | ||
| 373 | + row.get("title").getAsString(), | ||
| 374 | + row.get("category").getAsString(), | ||
| 375 | + row.get("score").getAsFloat()); | ||
| 376 | + } | ||
| 377 | + } | ||
| 378 | + | ||
| 379 | + /** | ||
| 380 | + * 查询数据（标量过滤） | ||
| 381 | + */ | ||
| 382 | + private void queryData() { | ||
| 383 | + printSection("标量查询"); | ||
| 384 | + | ||
| 385 | + // 查询得分大于50的数据 | ||
| 386 | + QueryReq queryReq = QueryReq.builder() | ||
| 387 | + .collectionName(COLLECTION_NAME) | ||
| 388 | + .filter("score > 50") // 过滤条件 | ||
| 389 | + .outputFields(Arrays.asList("id", "title", "category", "score")) | ||
| 390 | + .limit(10) | ||
| 391 | + .build(); | ||
| 392 | + | ||
| 393 | + QueryResp queryResp = client.query(queryReq); | ||
| 394 | + | ||
| 395 | + List<QueryResp.QueryResult> results = queryResp.getQueryResults(); | ||
| 396 | + System.out.println("查询条件: score > 50"); | ||
| 397 | + System.out.println("查询结果: " + results.size() + " 条数据"); | ||
| 398 | + | ||
| 399 | + if (!results.isEmpty()) { | ||
| 400 | + System.out.println("结果列表:"); | ||
| 401 | + for (QueryResp.QueryResult result : results) { | ||
| 402 | + Map<String, Object> entity = result.getEntity(); | ||
| 403 | + System.out.printf(" ID: %d | 标题: %s | 分类: %s | 得分: %.2f%n", | ||
| 404 | + (Long) entity.get("id"), | ||
| 405 | + entity.get("title"), | ||
| 406 | + entity.get("category"), | ||
| 407 | + (Float) entity.get("score")); | ||
| 408 | + } | ||
| 409 | + } | ||
| 410 | + } | ||
| 411 | + | ||
| 412 | + /** | ||
| 413 | + * 向量相似性搜索 | ||
| 414 | + */ | ||
| 415 | + private void searchData() { | ||
| 416 | + printSection("向量相似性搜索"); | ||
| 417 | + | ||
| 418 | + // 创建查询向量 | ||
| 419 | + List<Float> vectorList = new ArrayList<>(); | ||
| 420 | + Random random = new Random(); | ||
| 421 | + for (int j = 0; j < VECTOR_DIM; j++) { | ||
| 422 | + vectorList.add(random.nextFloat()); | ||
| 423 | + } | ||
| 424 | + FloatVec queryVector = new FloatVec(vectorList); | ||
| 425 | + | ||
| 426 | + // 构建搜索参数 | ||
| 427 | + Map<String, Object> searchParams = new HashMap<>(); | ||
| 428 | + searchParams.put("nprobe", 10); | ||
| 429 | + | ||
| 430 | + // 构建搜索请求 | ||
| 431 | + SearchReq searchReq = SearchReq.builder() | ||
| 432 | + .collectionName(COLLECTION_NAME) | ||
| 433 | + .data(Collections.singletonList(queryVector)) | ||
| 434 | + .annsField("vector") | ||
| 435 | + .limit(5) | ||
| 436 | + .outputFields(Arrays.asList("title", "category", "score", "tags")) | ||
| 437 | + .metricType(IndexParam.MetricType.L2) | ||
| 438 | + .searchParams(searchParams) | ||
| 439 | + .build(); | ||
| 440 | + | ||
| 441 | + // 执行搜索 | ||
| 442 | + SearchResp searchResp = client.search(searchReq); | ||
| 443 | + | ||
| 444 | + // 打印结果 | ||
| 445 | + System.out.println("最相似的5条结果:"); | ||
| 446 | + List<List<SearchResp.SearchResult>> results = searchResp.getSearchResults(); | ||
| 447 | + | ||
| 448 | + if (!results.isEmpty() && !results.get(0).isEmpty()) { | ||
| 449 | + for (int i = 0; i < results.get(0).size(); i++) { | ||
| 450 | + SearchResp.SearchResult result = results.get(0).get(i); | ||
| 451 | + Map<String, Object> entity = result.getEntity(); | ||
| 452 | + | ||
| 453 | + System.out.printf(" %d. ID: %d | 距离: %.4f%n", | ||
| 454 | + i + 1, | ||
| 455 | + (Long) result.getId(), | ||
| 456 | + result.getScore()); | ||
| 457 | + System.out.printf(" 标题: %s | 分类: %s | 得分: %.2f%n", | ||
| 458 | + entity.get("title"), | ||
| 459 | + entity.get("category"), | ||
| 460 | + (Float) entity.get("score")); | ||
| 461 | + System.out.printf(" 标签: %s%n", | ||
| 462 | + entity.get("tags")); | ||
| 463 | + } | ||
| 464 | + } else { | ||
| 465 | + System.out.println(" 没有找到结果"); | ||
| 466 | + } | ||
| 467 | + } | ||
| 468 | + | ||
| 469 | + /** | ||
| 470 | + * 获取集合统计信息 | ||
| 471 | + */ | ||
| 472 | + private void getCollectionStats() { | ||
| 473 | + printSection("集合统计信息"); | ||
| 474 | + | ||
| 475 | + GetCollectionStatsReq statsReq = GetCollectionStatsReq.builder() | ||
| 476 | + .collectionName(COLLECTION_NAME) | ||
| 477 | + .build(); | ||
| 478 | + | ||
| 479 | + GetCollectionStatsResp statsResp = client.getCollectionStats(statsReq); | ||
| 480 | + | ||
| 481 | + System.out.println("集合名称: " + COLLECTION_NAME); | ||
| 482 | + System.out.println("实体数量: " + statsResp.getNumOfEntities()); | ||
| 483 | + } | ||
| 484 | + | ||
| 485 | + /** | ||
| 486 | + * 释放集合并关闭连接 | ||
| 487 | + */ | ||
| 488 | + private void releaseAndClose() { | ||
| 489 | + printSection("清理资源"); | ||
| 490 | + | ||
| 491 | + if (client != null) { | ||
| 492 | + try { | ||
| 493 | + // 释放集合 | ||
| 494 | + ReleaseCollectionReq releaseCollectionReq = ReleaseCollectionReq.builder() | ||
| 495 | + .collectionName(COLLECTION_NAME) | ||
| 496 | + .build(); | ||
| 497 | + client.releaseCollection(releaseCollectionReq); | ||
| 498 | + System.out.println("✅ 集合已释放"); | ||
| 499 | + | ||
| 500 | + // 关闭连接 | ||
| 501 | + client.close(); | ||
| 502 | + System.out.println("✅ 连接已关闭"); | ||
| 503 | + | ||
| 504 | + } catch (Exception e) { | ||
| 505 | + System.err.println("⚠️ 清理资源时出错: " + e.getMessage()); | ||
| 506 | + } | ||
| 507 | + } | ||
| 508 | + } | ||
| 509 | + | ||
| 510 | + /** | ||
| 511 | + * 打印标题 | ||
| 512 | + */ | ||
| 513 | + private void printHeader(String title) { | ||
| 514 | + System.out.println("\n" + "=".repeat(60)); | ||
| 515 | + System.out.println(" " + title); | ||
| 516 | + System.out.println("=".repeat(60)); | ||
| 517 | + } | ||
| 518 | + | ||
| 519 | + /** | ||
| 520 | + * 打印章节标题 | ||
| 521 | + */ | ||
| 522 | + private void printSection(String section) { | ||
| 523 | + System.out.println("\n" + "-".repeat(40)); | ||
| 524 | + System.out.println(" " + section); | ||
| 525 | + System.out.println("-".repeat(40)); | ||
| 526 | + } | ||
| 527 | + | ||
| 528 | + /** | ||
| 529 | + * 打印成功信息 | ||
| 530 | + */ | ||
| 531 | + private void printSuccess(String message) { | ||
| 532 | + System.out.println("\n" + "=".repeat(60)); | ||
| 533 | + System.out.println(" ✅ " + message); | ||
| 534 | + System.out.println("=".repeat(60)); | ||
| 535 | + } | ||
| 536 | +} | ||
| 0 | \ No newline at end of file | 537 | \ No newline at end of file |
src/main/java/com/xly/milvus/util/MapToJsonConverter.java
0 → 100644
| 1 | +package com.xly.milvus.util; | ||
| 2 | + | ||
| 3 | +import com.google.gson.JsonArray; | ||
| 4 | +import com.google.gson.JsonObject; | ||
| 5 | +import java.util.List; | ||
| 6 | +import java.util.Map; | ||
| 7 | + | ||
| 8 | +/** | ||
| 9 | + * Map 转 JsonObject 工具类 | ||
| 10 | + */ | ||
| 11 | +public class MapToJsonConverter { | ||
| 12 | + | ||
| 13 | + /** | ||
| 14 | + * 将 Map<String, Object> 转换为 JsonObject | ||
| 15 | + */ | ||
| 16 | + public static JsonObject convert(Map<String, Object> map) { | ||
| 17 | + JsonObject jsonObject = new JsonObject(); | ||
| 18 | + | ||
| 19 | + if (map == null || map.isEmpty()) { | ||
| 20 | + return jsonObject; | ||
| 21 | + } | ||
| 22 | + | ||
| 23 | + for (Map.Entry<String, Object> entry : map.entrySet()) { | ||
| 24 | + String key = entry.getKey(); | ||
| 25 | + Object value = entry.getValue(); | ||
| 26 | + | ||
| 27 | + addToJson(jsonObject, key, value); | ||
| 28 | + } | ||
| 29 | + | ||
| 30 | + return jsonObject; | ||
| 31 | + } | ||
| 32 | + | ||
| 33 | + /** | ||
| 34 | + * 递归添加值到JsonObject | ||
| 35 | + */ | ||
| 36 | + @SuppressWarnings("unchecked") | ||
| 37 | + private static void addToJson(JsonObject jsonObject, String key, Object value) { | ||
| 38 | + if (value == null) { | ||
| 39 | + jsonObject.add(key, null); | ||
| 40 | + } else if (value instanceof String) { | ||
| 41 | + jsonObject.addProperty(key, (String) value); | ||
| 42 | + } else if (value instanceof Number) { | ||
| 43 | + jsonObject.addProperty(key, (Number) value); | ||
| 44 | + } else if (value instanceof Boolean) { | ||
| 45 | + jsonObject.addProperty(key, (Boolean) value); | ||
| 46 | + } else if (value instanceof Character) { | ||
| 47 | + jsonObject.addProperty(key, (Character) value); | ||
| 48 | + } else if (value instanceof Map) { | ||
| 49 | + jsonObject.add(key, convert((Map<String, Object>) value)); | ||
| 50 | + } else if (value instanceof List) { | ||
| 51 | + jsonObject.add(key, convertList((List<?>) value)); | ||
| 52 | + } else if (value instanceof Object[]) { | ||
| 53 | + jsonObject.add(key, convertArray((Object[]) value)); | ||
| 54 | + } else { | ||
| 55 | + // 其他类型转换为字符串 | ||
| 56 | + jsonObject.addProperty(key, value.toString()); | ||
| 57 | + } | ||
| 58 | + } | ||
| 59 | + | ||
| 60 | + /** | ||
| 61 | + * 转换List为JsonArray | ||
| 62 | + */ | ||
| 63 | + private static JsonArray convertList(List<?> list) { | ||
| 64 | + JsonArray jsonArray = new JsonArray(); | ||
| 65 | + | ||
| 66 | + for (Object item : list) { | ||
| 67 | + if (item == null) { | ||
| 68 | + jsonArray.add((JsonObject) null); | ||
| 69 | + } else if (item instanceof String) { | ||
| 70 | + jsonArray.add((String) item); | ||
| 71 | + } else if (item instanceof Number) { | ||
| 72 | + jsonArray.add((Number) item); | ||
| 73 | + } else if (item instanceof Boolean) { | ||
| 74 | + jsonArray.add((Boolean) item); | ||
| 75 | + } else if (item instanceof Character) { | ||
| 76 | + jsonArray.add((Character) item); | ||
| 77 | + } else if (item instanceof Map) { | ||
| 78 | + jsonArray.add(convert((Map<String, Object>) item)); | ||
| 79 | + } else if (item instanceof List) { | ||
| 80 | + jsonArray.add(convertList((List<?>) item)); | ||
| 81 | + } else { | ||
| 82 | + jsonArray.add(item.toString()); | ||
| 83 | + } | ||
| 84 | + } | ||
| 85 | + | ||
| 86 | + return jsonArray; | ||
| 87 | + } | ||
| 88 | + | ||
| 89 | + /** | ||
| 90 | + * 转换数组为JsonArray | ||
| 91 | + */ | ||
| 92 | + private static JsonArray convertArray(Object[] array) { | ||
| 93 | + JsonArray jsonArray = new JsonArray(); | ||
| 94 | + | ||
| 95 | + for (Object item : array) { | ||
| 96 | + if (item == null) { | ||
| 97 | + jsonArray.add((JsonObject) null); | ||
| 98 | + } else if (item instanceof String) { | ||
| 99 | + jsonArray.add((String) item); | ||
| 100 | + } else if (item instanceof Number) { | ||
| 101 | + jsonArray.add((Number) item); | ||
| 102 | + } else if (item instanceof Boolean) { | ||
| 103 | + jsonArray.add((Boolean) item); | ||
| 104 | + } else if (item instanceof Character) { | ||
| 105 | + jsonArray.add((Character) item); | ||
| 106 | + } else if (item instanceof Map) { | ||
| 107 | + jsonArray.add(convert((Map<String, Object>) item)); | ||
| 108 | + } else if (item instanceof List) { | ||
| 109 | + jsonArray.add(convertList((List<?>) item)); | ||
| 110 | + } else { | ||
| 111 | + jsonArray.add(item.toString()); | ||
| 112 | + } | ||
| 113 | + } | ||
| 114 | + | ||
| 115 | + return jsonArray; | ||
| 116 | + } | ||
| 117 | +} | ||
| 0 | \ No newline at end of file | 118 | \ No newline at end of file |
src/main/java/com/xly/milvus/util/MilvusSchemaBuilder.java
0 → 100644
| 1 | +package com.xly.milvus.util; | ||
| 2 | + | ||
| 3 | +import io.milvus.v2.common.DataType; | ||
| 4 | +import io.milvus.v2.service.collection.request.CreateCollectionReq; | ||
| 5 | +import java.util.HashMap; | ||
| 6 | +import java.util.Map; | ||
| 7 | + | ||
| 8 | +/** | ||
| 9 | + * Milvus字段Schema构建工具类 | ||
| 10 | + * 基于源码的 FieldSchema.FieldSchemaBuilder | ||
| 11 | + */ | ||
| 12 | +public class MilvusSchemaBuilder { | ||
| 13 | + | ||
| 14 | + /** | ||
| 15 | + * 创建主键ID字段（Int64类型，自增） | ||
| 16 | + */ | ||
| 17 | + public static CreateCollectionReq.FieldSchema createAutoIdField(String fieldName) { | ||
| 18 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 19 | + .name(fieldName) | ||
| 20 | + .dataType(DataType.Int64) | ||
| 21 | + .isPrimaryKey(true) | ||
| 22 | + .autoID(true) | ||
| 23 | + .description("自增主键ID") | ||
| 24 | + .build(); | ||
| 25 | + } | ||
| 26 | + | ||
| 27 | + /** | ||
| 28 | + * 创建主键ID字段（String类型，不自增） | ||
| 29 | + */ | ||
| 30 | + public static CreateCollectionReq.FieldSchema createStringIdField(String fieldName, int maxLength) { | ||
| 31 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 32 | + .name(fieldName) | ||
| 33 | + .dataType(DataType.VarChar) | ||
| 34 | + .maxLength(maxLength) | ||
| 35 | + .isPrimaryKey(true) | ||
| 36 | + .autoID(false) | ||
| 37 | + .description("业务主键ID") | ||
| 38 | + .build(); | ||
| 39 | + } | ||
| 40 | + | ||
| 41 | + /** | ||
| 42 | + * 创建向量字段 | ||
| 43 | + */ | ||
| 44 | + public static CreateCollectionReq.FieldSchema createVectorField(String fieldName, int dimension) { | ||
| 45 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 46 | + .name(fieldName) | ||
| 47 | + .dataType(DataType.FloatVector) | ||
| 48 | + .dimension(dimension) | ||
| 49 | + .description("向量字段，维度: " + dimension) | ||
| 50 | + .build(); | ||
| 51 | + } | ||
| 52 | + | ||
| 53 | + /** | ||
| 54 | + * 创建文本字段（VarChar类型） | ||
| 55 | + */ | ||
| 56 | + public static CreateCollectionReq.FieldSchema createTextField(String fieldName, int maxLength) { | ||
| 57 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 58 | + .name(fieldName) | ||
| 59 | + .dataType(DataType.VarChar) | ||
| 60 | + .maxLength(maxLength) | ||
| 61 | + .description("文本内容") | ||
| 62 | + .build(); | ||
| 63 | + } | ||
| 64 | + | ||
| 65 | + /** | ||
| 66 | + * 创建JSON元数据字段 | ||
| 67 | + */ | ||
| 68 | + public static CreateCollectionReq.FieldSchema createJsonField(String fieldName) { | ||
| 69 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 70 | + .name(fieldName) | ||
| 71 | + .dataType(DataType.JSON) | ||
| 72 | + .description("JSON元数据") | ||
| 73 | + .build(); | ||
| 74 | + } | ||
| 75 | + | ||
| 76 | + /** | ||
| 77 | + * 创建整型字段 | ||
| 78 | + */ | ||
| 79 | + public static CreateCollectionReq.FieldSchema createIntField(String fieldName, String description) { | ||
| 80 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 81 | + .name(fieldName) | ||
| 82 | + .dataType(DataType.Int64) | ||
| 83 | + .description(description) | ||
| 84 | + .build(); | ||
| 85 | + } | ||
| 86 | + | ||
| 87 | + /** | ||
| 88 | + * 创建整型字段（可空） | ||
| 89 | + */ | ||
| 90 | + public static CreateCollectionReq.FieldSchema createIntField(String fieldName, | ||
| 91 | + String description, | ||
| 92 | + boolean isNullable) { | ||
| 93 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 94 | + .name(fieldName) | ||
| 95 | + .dataType(DataType.Int64) | ||
| 96 | + .description(description) | ||
| 97 | + .isNullable(isNullable) | ||
| 98 | + .build(); | ||
| 99 | + } | ||
| 100 | + | ||
| 101 | + /** | ||
| 102 | + * 创建整型字段（带默认值） | ||
| 103 | + */ | ||
| 104 | + public static CreateCollectionReq.FieldSchema createIntField(String fieldName, | ||
| 105 | + String description, | ||
| 106 | + long defaultValue) { | ||
| 107 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 108 | + .name(fieldName) | ||
| 109 | + .dataType(DataType.Int64) | ||
| 110 | + .description(description) | ||
| 111 | + .defaultValue(defaultValue) | ||
| 112 | + .build(); | ||
| 113 | + } | ||
| 114 | + | ||
| 115 | + /** | ||
| 116 | + * 创建浮点型字段 | ||
| 117 | + */ | ||
| 118 | + public static CreateCollectionReq.FieldSchema createFloatField(String fieldName, String description) { | ||
| 119 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 120 | + .name(fieldName) | ||
| 121 | + .dataType(DataType.Float) | ||
| 122 | + .description(description) | ||
| 123 | + .build(); | ||
| 124 | + } | ||
| 125 | + | ||
| 126 | + /** | ||
| 127 | + * 创建布尔型字段 | ||
| 128 | + */ | ||
| 129 | + public static CreateCollectionReq.FieldSchema createBoolField(String fieldName, String description) { | ||
| 130 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 131 | + .name(fieldName) | ||
| 132 | + .dataType(DataType.Bool) | ||
| 133 | + .description(description) | ||
| 134 | + .build(); | ||
| 135 | + } | ||
| 136 | + | ||
| 137 | + /** | ||
| 138 | + * 创建布尔型字段（带默认值） | ||
| 139 | + */ | ||
| 140 | + public static CreateCollectionReq.FieldSchema createBoolField(String fieldName, | ||
| 141 | + String description, | ||
| 142 | + boolean defaultValue) { | ||
| 143 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 144 | + .name(fieldName) | ||
| 145 | + .dataType(DataType.Bool) | ||
| 146 | + .description(description) | ||
| 147 | + .defaultValue(defaultValue) | ||
| 148 | + .build(); | ||
| 149 | + } | ||
| 150 | + | ||
| 151 | + /** | ||
| 152 | + * 创建数组字段 | ||
| 153 | + */ | ||
| 154 | + public static CreateCollectionReq.FieldSchema createArrayField(String fieldName, | ||
| 155 | + DataType elementType, | ||
| 156 | + int maxCapacity, | ||
| 157 | + String description) { | ||
| 158 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 159 | + .name(fieldName) | ||
| 160 | + .dataType(DataType.Array) | ||
| 161 | + .elementType(elementType) | ||
| 162 | + .maxCapacity(maxCapacity) | ||
| 163 | + .description(description) | ||
| 164 | + .build(); | ||
| 165 | + } | ||
| 166 | + | ||
| 167 | + /** | ||
| 168 | + * 创建分区键字段 | ||
| 169 | + */ | ||
| 170 | + public static CreateCollectionReq.FieldSchema createPartitionKeyField(String fieldName, | ||
| 171 | + DataType dataType, | ||
| 172 | + String description) { | ||
| 173 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 174 | + .name(fieldName) | ||
| 175 | + .dataType(dataType) | ||
| 176 | + .isPartitionKey(true) | ||
| 177 | + .description(description) | ||
| 178 | + .build(); | ||
| 179 | + } | ||
| 180 | + | ||
| 181 | + /** | ||
| 182 | + * 创建聚类键字段 | ||
| 183 | + */ | ||
| 184 | + public static CreateCollectionReq.FieldSchema createClusteringKeyField(String fieldName, | ||
| 185 | + DataType dataType, | ||
| 186 | + String description) { | ||
| 187 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 188 | + .name(fieldName) | ||
| 189 | + .dataType(dataType) | ||
| 190 | + .isClusteringKey(true) | ||
| 191 | + .description(description) | ||
| 192 | + .build(); | ||
| 193 | + } | ||
| 194 | + | ||
| 195 | + /** | ||
| 196 | + * 创建带分析器的文本字段（用于全文搜索） | ||
| 197 | + */ | ||
| 198 | + public static CreateCollectionReq.FieldSchema createAnalyzedTextField(String fieldName, | ||
| 199 | + int maxLength, | ||
| 200 | + String analyzerType) { | ||
| 201 | + Map<String, Object> analyzerParams = new HashMap<>(); | ||
| 202 | + analyzerParams.put("type", analyzerType); // "english", "chinese", "standard" 等 | ||
| 203 | + | ||
| 204 | + return CreateCollectionReq.FieldSchema.builder() | ||
| 205 | + .name(fieldName) | ||
| 206 | + .dataType(DataType.VarChar) | ||
| 207 | + .maxLength(maxLength) | ||
| 208 | + .enableAnalyzer(true) | ||
| 209 | + .analyzerParams(analyzerParams) | ||
| 210 | + .enableMatch(true) | ||
| 211 | + .description("支持全文搜索的文本字段") | ||
| 212 | + .build(); | ||
| 213 | + } | ||
| 214 | +} | ||
| 0 | \ No newline at end of file | 215 | \ No newline at end of file |
src/main/java/com/xly/milvus/web/MilvusController.java
0 → 100644
| 1 | +package com.xly.milvus.web; | ||
| 2 | + | ||
| 3 | +import com.xly.milvus.service.MilvusService; | ||
| 4 | +import com.xly.runner.AppStartupRunner; | ||
| 5 | +import com.xly.service.DynamicExeDbService; | ||
| 6 | +import com.xly.service.UserSceneSessionService; | ||
| 7 | +import com.xly.tool.DynamicToolProvider; | ||
| 8 | +import com.xly.tts.bean.*; | ||
| 9 | +import com.xly.tts.service.LocalAudioCache; | ||
| 10 | +import com.xly.tts.service.PythonTtsProxyService; | ||
| 11 | +import jakarta.validation.Valid; | ||
| 12 | +import lombok.RequiredArgsConstructor; | ||
| 13 | +import lombok.extern.slf4j.Slf4j; | ||
| 14 | +import org.springframework.core.io.InputStreamResource; | ||
| 15 | +import org.springframework.http.MediaType; | ||
| 16 | +import org.springframework.http.ResponseEntity; | ||
| 17 | +import org.springframework.web.bind.annotation.*; | ||
| 18 | +import reactor.core.publisher.Flux; | ||
| 19 | +import reactor.core.publisher.Mono; | ||
| 20 | + | ||
| 21 | +import javax.annotation.PostConstruct; | ||
| 22 | +import javax.annotation.PreDestroy; | ||
| 23 | +import java.util.List; | ||
| 24 | +import java.util.Map; | ||
| 25 | +import java.util.concurrent.CompletableFuture; | ||
| 26 | + | ||
| 27 | +@Slf4j | ||
| 28 | +@RestController | ||
| 29 | +@RequestMapping("/api/milvus") | ||
| 30 | +@RequiredArgsConstructor | ||
| 31 | +public class MilvusController { | ||
| 32 | + | ||
| 33 | + private final MilvusService milvusService; | ||
| 34 | + | ||
| 35 | + /*** | ||
| 36 | + * @Author 钱豹 | ||
| 37 | + * @Date 14:32 2026/2/10 | ||
| 38 | + * @Param [request] | ||
| 39 | + * @return org.springframework.http.ResponseEntity<com.xly.tts.bean.TTSResponseDTO> | ||
| 40 | + * @Description 初始化AI所有变量 热启动 | ||
| 41 | + **/ | ||
| 42 | + @PostMapping("/init") | ||
| 43 | + public ResponseEntity<TTSResponseDTO> init(@RequestBody Map<String,Object> reqMap) { | ||
| 44 | + TTSResponseDTO responseDTO = milvusService.initDataToMilvus(reqMap); | ||
| 45 | + return ResponseEntity.ok(responseDTO); | ||
| 46 | + } | ||
| 47 | + | ||
| 48 | + | ||
| 49 | +} | ||
| 0 | \ No newline at end of file | 50 | \ No newline at end of file |
src/main/java/com/xly/service/XlyErpService.java
| @@ -7,6 +7,7 @@ import cn.hutool.core.util.IdUtil; | @@ -7,6 +7,7 @@ import cn.hutool.core.util.IdUtil; | ||
| 7 | import cn.hutool.core.util.ObjectUtil; | 7 | import cn.hutool.core.util.ObjectUtil; |
| 8 | import cn.hutool.core.util.StrUtil; | 8 | import cn.hutool.core.util.StrUtil; |
| 9 | import com.alibaba.fastjson2.JSON; | 9 | import com.alibaba.fastjson2.JSON; |
| 10 | +import com.alibaba.fastjson2.JSONObject; | ||
| 10 | import com.xly.agent.ChatiAgent; | 11 | import com.xly.agent.ChatiAgent; |
| 11 | import com.xly.agent.DynamicTableNl2SqlAiAgent; | 12 | import com.xly.agent.DynamicTableNl2SqlAiAgent; |
| 12 | import com.xly.agent.ErpAiAgent; | 13 | import com.xly.agent.ErpAiAgent; |
| @@ -17,15 +18,14 @@ import com.xly.constant.ReturnTypeCode; | @@ -17,15 +18,14 @@ import com.xly.constant.ReturnTypeCode; | ||
| 17 | import com.xly.entity.*; | 18 | import com.xly.entity.*; |
| 18 | import com.xly.exception.sqlexception.SqlGenerateException; | 19 | import com.xly.exception.sqlexception.SqlGenerateException; |
| 19 | import com.xly.exception.sqlexception.SqlValidateException; | 20 | import com.xly.exception.sqlexception.SqlValidateException; |
| 21 | +import com.xly.milvus.service.AiGlobalAgentQuestionSqlEmitterService; | ||
| 22 | +import com.xly.milvus.service.MilvusService; | ||
| 20 | import com.xly.runner.AppStartupRunner; | 23 | import com.xly.runner.AppStartupRunner; |
| 21 | import com.xly.thread.AiSqlErrorHistoryThread; | 24 | import com.xly.thread.AiSqlErrorHistoryThread; |
| 22 | import com.xly.thread.AiUserAgentQuestionThread; | 25 | import com.xly.thread.AiUserAgentQuestionThread; |
| 23 | import com.xly.thread.MultiThreadPoolServer; | 26 | import com.xly.thread.MultiThreadPoolServer; |
| 24 | import com.xly.tool.DynamicToolProvider; | 27 | import com.xly.tool.DynamicToolProvider; |
| 25 | -import com.xly.util.EnhancedErrorGuidance; | ||
| 26 | -import com.xly.util.InputPreprocessor; | ||
| 27 | -import com.xly.util.SqlValidateUtil; | ||
| 28 | -import com.xly.util.ValiDataUtil; | 28 | +import com.xly.util.*; |
| 29 | import dev.langchain4j.agent.tool.ToolExecutionRequest; | 29 | import dev.langchain4j.agent.tool.ToolExecutionRequest; |
| 30 | import dev.langchain4j.data.message.AiMessage; | 30 | import dev.langchain4j.data.message.AiMessage; |
| 31 | import dev.langchain4j.data.message.ChatMessage; | 31 | import dev.langchain4j.data.message.ChatMessage; |
| @@ -33,15 +33,22 @@ import dev.langchain4j.data.message.ChatMessageType; | @@ -33,15 +33,22 @@ import dev.langchain4j.data.message.ChatMessageType; | ||
| 33 | import dev.langchain4j.model.chat.ChatLanguageModel; | 33 | import dev.langchain4j.model.chat.ChatLanguageModel; |
| 34 | import dev.langchain4j.model.ollama.OllamaChatModel; | 34 | import dev.langchain4j.model.ollama.OllamaChatModel; |
| 35 | import dev.langchain4j.service.AiServices; | 35 | import dev.langchain4j.service.AiServices; |
| 36 | +import dev.langchain4j.service.MemoryId; | ||
| 37 | +import dev.langchain4j.service.V; | ||
| 38 | +import io.milvus.v2.common.DataType; | ||
| 39 | +import io.milvus.v2.service.collection.request.CreateCollectionReq; | ||
| 36 | import lombok.RequiredArgsConstructor; | 40 | import lombok.RequiredArgsConstructor; |
| 37 | import lombok.extern.slf4j.Slf4j; | 41 | import lombok.extern.slf4j.Slf4j; |
| 42 | +import org.apache.commons.lang3.time.DateFormatUtils; | ||
| 38 | import org.springframework.beans.factory.annotation.Value; | 43 | import org.springframework.beans.factory.annotation.Value; |
| 39 | import org.springframework.stereotype.Service; | 44 | import org.springframework.stereotype.Service; |
| 40 | import org.springframework.util.IdGenerator; | 45 | import org.springframework.util.IdGenerator; |
| 46 | +import reactor.core.publisher.Flux; | ||
| 41 | 47 | ||
| 42 | import java.time.Duration; | 48 | import java.time.Duration; |
| 43 | import java.util.*; | 49 | import java.util.*; |
| 44 | import java.util.stream.Collectors; | 50 | import java.util.stream.Collectors; |
| 51 | +import java.util.stream.IntStream; | ||
| 45 | 52 | ||
| 46 | @Service | 53 | @Service |
| 47 | @RequiredArgsConstructor | 54 | @RequiredArgsConstructor |
| @@ -56,6 +63,8 @@ public class XlyErpService { | @@ -56,6 +63,8 @@ public class XlyErpService { | ||
| 56 | private final OperableChatMemoryProvider operableChatMemoryProvider; | 63 | private final OperableChatMemoryProvider operableChatMemoryProvider; |
| 57 | private final DynamicExeDbService dynamicExeDbService; | 64 | private final DynamicExeDbService dynamicExeDbService; |
| 58 | private final RedisService redisService; | 65 | private final RedisService redisService; |
| 66 | + private final AiGlobalAgentQuestionSqlEmitterService aiGlobalAgentQuestionSqlEmitterService; | ||
| 67 | + private final MilvusService milvusService; | ||
| 59 | 68 | ||
| 60 | //执行动态语句 执行异常的情况下 最多执行次数 | 69 | //执行动态语句 执行异常的情况下 最多执行次数 |
| 61 | private final Integer maxRetries = 5; | 70 | private final Integer maxRetries = 5; |
| @@ -67,6 +76,51 @@ public class XlyErpService { | @@ -67,6 +76,51 @@ public class XlyErpService { | ||
| 67 | 76 | ||
| 68 | @Value("${langchain4j.ollama.sql-model-name}") | 77 | @Value("${langchain4j.ollama.sql-model-name}") |
| 69 | private String sqlModelName; | 78 | private String sqlModelName; |
| 79 | + | ||
| 80 | + | ||
| 81 | + /** | ||
| 82 | + * 新的流式方法 - 返回 Flux<AiResponseDTO> | ||
| 83 | + * 每个AiResponseDTO包含一个文本片段 | ||
| 84 | + */ | ||
| 85 | + /** | ||
| 86 | + * 模拟的erpUserInputStream实现 | ||
| 87 | + */ | ||
| 88 | + public Flux<AiResponseDTO> erpUserInputStream(String userInput, String sUserId, | ||
| 89 | + String sUserName, String sBrandsId, | ||
| 90 | + String sSubsidiaryId, String sUserType, | ||
| 91 | + String authorization) { | ||
| 92 | + String requestId = UUID.randomUUID().toString(); | ||
| 93 | + | ||
| 94 | + // 按句子分割 | ||
| 95 | + String[] sentences = userInput.split("(?<=[。！？.!?])"); | ||
| 96 | + int totalChunks = sentences.length; | ||
| 97 | + | ||
| 98 | + return Flux.range(0, totalChunks) | ||
| 99 | + .delayElements(Duration.ofMillis(200)) | ||
| 100 | + .map(i -> { | ||
| 101 | + String sentence = sentences[i].trim(); | ||
| 102 | + if (sentence.isEmpty()) return null; | ||
| 103 | + | ||
| 104 | + return AiResponseDTO.builder() | ||
| 105 | + .requestId(requestId) | ||
| 106 | + .code(200) | ||
| 107 | + .message("ERP_CHUNK") | ||
| 108 | + .status("PROCESSING") | ||
| 109 | + .textFragment(sentence) | ||
| 110 | + .chunkIndex(i) | ||
| 111 | + .totalChunks(totalChunks) | ||
| 112 | + .isLastChunk(i == totalChunks - 1) | ||
| 113 | + .progress((i + 1) * 100 / totalChunks) | ||
| 114 | + .timestamp(System.currentTimeMillis()) | ||
| 115 | + .sSceneName("客服咨询") | ||
| 116 | + .sMethodName("chat") | ||
| 117 | + .sReturnType("MARKDOWN") | ||
| 118 | + .build(); | ||
| 119 | + }) | ||
| 120 | + .filter(Objects::nonNull); | ||
| 121 | + } | ||
| 122 | + | ||
| 123 | + | ||
| 70 | /*** | 124 | /*** |
| 71 | * @Author 钱豹 | 125 | * @Author 钱豹 |
| 72 | * @Date 19:18 2026/1/27 | 126 | * @Date 19:18 2026/1/27 |
| @@ -116,8 +170,15 @@ public class XlyErpService { | @@ -116,8 +170,15 @@ public class XlyErpService { | ||
| 116 | if (aiAgent == null){ | 170 | if (aiAgent == null){ |
| 117 | return getChatiAgent (input,session); | 171 | return getChatiAgent (input,session); |
| 118 | } | 172 | } |
| 119 | - //用户输入添加方法 | ||
| 120 | - String sResponMessage = aiAgent.chat(userId, input); | 173 | + String sResponMessage = StrUtil.EMPTY; |
| 174 | + //用户输入添加方法(如果没有方法，动态SQL方法不需要) | ||
| 175 | + if(!(ObjectUtil.isNotEmpty(session.getCurrentTool()) | ||
| 176 | + && ObjectUtil.isNotEmpty(session.getCurrentTool().getSInputTabelName()) | ||
| 177 | + && ObjectUtil.isNotEmpty(session.getCurrentTool().getSStructureMemo())) | ||
| 178 | + ){ | ||
| 179 | + sResponMessage = aiAgent.chat(userId, input); | ||
| 180 | + } | ||
| 181 | + | ||
| 121 | if(ObjectUtil.isNotEmpty(session.getCurrentTool()) | 182 | if(ObjectUtil.isNotEmpty(session.getCurrentTool()) |
| 122 | && !ObjectUtil.isNotEmpty(session.getCurrentTool().getSInputTabelName()) | 183 | && !ObjectUtil.isNotEmpty(session.getCurrentTool().getSInputTabelName()) |
| 123 | ){ | 184 | ){ |
| @@ -132,7 +193,14 @@ public class XlyErpService { | @@ -132,7 +193,14 @@ public class XlyErpService { | ||
| 132 | && ObjectUtil.isNotEmpty(session.getCurrentTool().getSInputTabelName()) | 193 | && ObjectUtil.isNotEmpty(session.getCurrentTool().getSInputTabelName()) |
| 133 | && ObjectUtil.isNotEmpty(session.getCurrentTool().getSStructureMemo())) | 194 | && ObjectUtil.isNotEmpty(session.getCurrentTool().getSStructureMemo())) |
| 134 | ){ | 195 | ){ |
| 135 | - sResponMessage = getDynamicTableSql(session, input, userId, userInput,0,StrUtil.EMPTY,StrUtil.EMPTY,"0",StrUtil.EMPTY, aiAgent); | 196 | + //查询是否走向量库 还是数据库查询 |
| 197 | + Boolean isAggregation = aiAgent.routeQuery(session.getUserId(), input); | ||
| 198 | + if(!isAggregation){ | ||
| 199 | + //获取常量库内容 | ||
| 200 | + sResponMessage = getMilvus(session, input, aiAgent); | ||
| 201 | + }else { | ||
| 202 | + sResponMessage = getDynamicTableSql(session, input, userId, userInput,0,StrUtil.EMPTY,StrUtil.EMPTY,"0",StrUtil.EMPTY, aiAgent); | ||
| 203 | + } | ||
| 136 | return AiResponseDTO.builder().sSceneName(sceneName).sMethodName(methodName).aiText(sResponMessage).sReturnType(ReturnTypeCode.HTML.getCode()).build(); | 204 | return AiResponseDTO.builder().sSceneName(sceneName).sMethodName(methodName).aiText(sResponMessage).sReturnType(ReturnTypeCode.HTML.getCode()).build(); |
| 137 | } else if (ObjectUtil.isNotEmpty(session.getCurrentTool())) { | 205 | } else if (ObjectUtil.isNotEmpty(session.getCurrentTool())) { |
| 138 | //2.处理工具参数采集结束后业务逻辑处理 | 206 | //2.处理工具参数采集结束后业务逻辑处理 |
| @@ -198,6 +266,75 @@ public class XlyErpService { | @@ -198,6 +266,75 @@ public class XlyErpService { | ||
| 198 | return AiResponseDTO.builder().sSceneName(sceneName).sMethodName(StrUtil.EMPTY).aiText(StrUtil.EMPTY).systemText("清除记忆成功！").sReturnType(ReturnTypeCode.HTML.getCode()).build(); | 266 | return AiResponseDTO.builder().sSceneName(sceneName).sMethodName(StrUtil.EMPTY).aiText(StrUtil.EMPTY).systemText("清除记忆成功！").sReturnType(ReturnTypeCode.HTML.getCode()).build(); |
| 199 | } | 267 | } |
| 200 | 268 | ||
| 269 | + /*** | ||
| 270 | + * @Author 钱豹 | ||
| 271 | + * @Date 10:16 2026/3/25 | ||
| 272 | + * @Param [session, input, userId, userInput, attempt, errorSql, errorMessage, iErroCount, historySqlList, aiAgent] | ||
| 273 | + * @return java.lang.String | ||
| 274 | + * @Description 查询向量库 | ||
| 275 | + **/ | ||
| 276 | + private String getMilvus(UserSceneSession session,String userInput,ErpAiAgent aiAgent){ | ||
| 277 | + String resultExplain = "信息模糊,请提供更具体的问题或指令"; | ||
| 278 | + try{ | ||
| 279 | + String sVectorfiled = session.getCurrentTool().getSVectorfiled(); | ||
| 280 | + String sInputTabelName = session.getCurrentTool().getSInputTabelName(); | ||
| 281 | + Map<String,Object> rMap = milvusService.getMilvusFiled(sVectorfiled); | ||
| 282 | + String sMilvusFiled = rMap.get("sMilvusFiled").toString(); | ||
| 283 | + String sMilvusFiledDescription = rMap.get("sMilvusFiledDescription").toString(); | ||
| 284 | + List<String> fields = (List<String>) rMap.get("sFileds"); | ||
| 285 | +// List<Map<String, String>> title = (List<Map<String, String>>) rMap.get("title"); | ||
| 286 | + String milvusFilter = aiAgent.getMilvusFilter(session.getUserId(),userInput, sMilvusFiled, sMilvusFiledDescription); | ||
| 287 | + List<Map<String,Object>> data = milvusService.getDataToCollection(sInputTabelName, milvusFilter,userInput,100,fields); | ||
| 288 | + //采用表格形式显示 | ||
| 289 | + resultExplain = aiAgent.explainMilvusResult(session.getUserId(),userInput,sMilvusFiledDescription,JSONObject.toJSONString(data)); | ||
| 290 | + //buildMarkdownTableWithStream(data, title); | ||
| 291 | + return resultExplain; | ||
| 292 | + }catch (Exception e){ | ||
| 293 | + e.printStackTrace(); | ||
| 294 | + } | ||
| 295 | + return resultExplain; | ||
| 296 | + } | ||
| 297 | + /*** | ||
| 298 | + * @Author 钱豹 | ||
| 299 | + * @Date 13:19 2026/3/25 | ||
| 300 | + * @Param [data, title] | ||
| 301 | + * @return java.lang.String | ||
| 302 | + * @Description 数据转成MarkdownTable | ||
| 303 | + **/ | ||
| 304 | + public String buildMarkdownTableWithStream(List<Map<String, Object>> data, List<Map<String, String>> title) { | ||
| 305 | + if (data == null || data.isEmpty()) { | ||
| 306 | + return "暂无数据"; | ||
| 307 | + } | ||
| 308 | + | ||
| 309 | + // 动态构建表头 | ||
| 310 | + StringBuilder headerBuilder = new StringBuilder("|"); | ||
| 311 | + StringBuilder separatorBuilder = new StringBuilder("|"); | ||
| 312 | + | ||
| 313 | + for (Map<String, String> column : title) { | ||
| 314 | + String displayName = column.get("sTitle"); // 中文显示名称 | ||
| 315 | + headerBuilder.append(" ").append(displayName).append(" |"); | ||
| 316 | + separatorBuilder.append("------|"); | ||
| 317 | + } | ||
| 318 | + String header = headerBuilder.toString() + "\n" + separatorBuilder.toString() + "\n"; | ||
| 319 | + // 构建数据行 | ||
| 320 | + String rows = IntStream.range(0, data.size()) | ||
| 321 | + .mapToObj(i -> { | ||
| 322 | + Map<String, Object> item = data.get(i); | ||
| 323 | + StringBuilder rowBuilder = new StringBuilder("|"); | ||
| 324 | + | ||
| 325 | + for (Map<String, String> column : title) { | ||
| 326 | + String fieldName = column.get("sName"); | ||
| 327 | + Object value = item.getOrDefault(fieldName, ""); | ||
| 328 | + rowBuilder.append(" ").append(value).append(" |"); | ||
| 329 | + } | ||
| 330 | + | ||
| 331 | + return rowBuilder.toString(); | ||
| 332 | + }) | ||
| 333 | + .collect(Collectors.joining("\n")); | ||
| 334 | + | ||
| 335 | + return header + rows; | ||
| 336 | + } | ||
| 337 | + | ||
| 201 | 338 | ||
| 202 | /*** | 339 | /*** |
| 203 | * @Author 钱豹 | 340 | * @Author 钱豹 |
| @@ -283,48 +420,54 @@ public class XlyErpService { | @@ -283,48 +420,54 @@ public class XlyErpService { | ||
| 283 | **/ | 420 | **/ |
| 284 | private String getDynamicTableSqlExec(UserSceneSession session,String input,String userId,String userInput,String errorSql,String errorMessage,String iErroCount,String historySqlList,ErpAiAgent aiAgent){ | 421 | private String getDynamicTableSqlExec(UserSceneSession session,String input,String userId,String userInput,String errorSql,String errorMessage,String iErroCount,String historySqlList,ErpAiAgent aiAgent){ |
| 285 | // 1. 构建自然语言转SQLAgent， | 422 | // 1. 构建自然语言转SQLAgent， |
| 286 | - List<Map<String, Object>> sqlResult = new ArrayList<>(); | 423 | + List<Map<String, Object>> sqlResult; |
| 287 | String cleanSql = StrUtil.EMPTY; | 424 | String cleanSql = StrUtil.EMPTY; |
| 288 | String rawSql; | 425 | String rawSql; |
| 289 | - String tableStruct; | ||
| 290 | - String sError_mes = StrUtil.EMPTY; | ||
| 291 | - Boolean doSqlErro = false; | ||
| 292 | - List<ChatMessage> chatMessage = operableChatMemoryProvider.getCurrentChatMessages(session.getUserId()); | 426 | + String tableStruct = session.getCurrentTool().getSStructureMemo(); |
| 427 | + String sError_mes; | ||
| 428 | + Boolean doAddSql = false; | ||
| 429 | + List<ChatMessage> chatMessage = new ArrayList<>(); | ||
| 293 | try{ | 430 | try{ |
| 294 | - DynamicTableNl2SqlAiAgent aiDynamicTableNl2SqlAiAgent = createDynamicTableNl2SqlAiAgent(userId, input, session); | ||
| 295 | - String tableNames = session.getCurrentTool().getSInputTabelName(); | ||
| 296 | - // "订单表:viw_salsalesorder,客户信息表:elecustomer,结算方式表:sispayment,产品表(无单价,无金额,无数量):viw_product_sort,销售人员表:viw_sissalesman_depart"; | ||
| 297 | - tableStruct = session.getCurrentTool().getSStructureMemo(); | ||
| 298 | - String sDataNow = DateUtil.format(new Date(), DatePattern.CHINESE_DATE_TIME_FORMAT); | ||
| 299 | - | ||
| 300 | - if(ObjectUtil.isEmpty(errorSql) && ObjectUtil.isEmpty(errorMessage)){ | ||
| 301 | - rawSql = aiDynamicTableNl2SqlAiAgent.generateMysqlSql(userId,tableNames,tableStruct,sDataNow,userInput); | ||
| 302 | - }else{ | ||
| 303 | - rawSql = aiDynamicTableNl2SqlAiAgent.regenerateSqlWithError(userId, tableNames,tableStruct,sDataNow,userInput,errorSql,errorMessage,iErroCount,historySqlList); | ||
| 304 | - } | ||
| 305 | - log.info("rawSql："+rawSql); | ||
| 306 | - if (rawSql == null || rawSql.trim().isEmpty()) { | ||
| 307 | - throw new SqlValidateException("SQL EMPTY"); | 431 | + //获取缓存动态SQL |
| 432 | +// cleanSql = getDynamicTableNl2Sql(session,input); | ||
| 433 | + //如果之前已查询直接返回 | ||
| 434 | + if(ObjectUtil.isEmpty(cleanSql)){ | ||
| 435 | + DynamicTableNl2SqlAiAgent aiDynamicTableNl2SqlAiAgent = createDynamicTableNl2SqlAiAgent(userId, input, session); | ||
| 436 | + chatMessage = operableChatMemoryProvider.getCurrentChatMessages(session.getUserId()); | ||
| 437 | + String tableNames = session.getCurrentTool().getSInputTabelName(); | ||
| 438 | + // "订单表:viw_salsalesorder,客户信息表:elecustomer,结算方式表:sispayment,产品表(无单价,无金额,无数量):viw_product_sort,销售人员表:viw_sissalesman_depart"; | ||
| 439 | + String sDataNow = DateUtil.now(); | ||
| 440 | + //DateFormatUtils.format(new Date(), "yyyy年MM月dd日HH时mm分ss秒"); | ||
| 441 | +// String sDataNow = DateUtil.format(new Date(), DatePattern.CHINESE_DATE_TIME_FORMAT); | ||
| 442 | + | ||
| 443 | + if(ObjectUtil.isEmpty(errorSql) && ObjectUtil.isEmpty(errorMessage)){ | ||
| 444 | + rawSql = aiDynamicTableNl2SqlAiAgent.generateMysqlSql(userId,tableNames,tableStruct,sDataNow,userInput); | ||
| 445 | + }else{ | ||
| 446 | + rawSql = aiDynamicTableNl2SqlAiAgent.regenerateSqlWithError(userId, tableNames,tableStruct,sDataNow,userInput,errorSql,errorMessage,iErroCount,historySqlList); | ||
| 447 | + } | ||
| 448 | + log.info("rawSql："+rawSql); | ||
| 449 | + if (rawSql == null || rawSql.trim().isEmpty()) { | ||
| 450 | + throw new SqlValidateException("SQL EMPTY"); | ||
| 451 | + } | ||
| 452 | + // 2. 清理SQL多余符号 + 生产级强校验（核心安全保障，不可省略） | ||
| 453 | + cleanSql = SqlValidateUtil.cleanSqlSymbol(rawSql); | ||
| 454 | + SqlValidateUtil.validateMysqlSql(cleanSql); | ||
| 455 | + doAddSql = true; | ||
| 308 | } | 456 | } |
| 309 | - // 2. 清理SQL多余符号 + 生产级强校验（核心安全保障，不可省略） | ||
| 310 | - cleanSql = SqlValidateUtil.cleanSqlSymbol(rawSql); | ||
| 311 | - // String[] cleanSqlA = rawSql.split(";"); | ||
| 312 | - // if(cleanSqlA.length>1){ | ||
| 313 | - // cleanSql = cleanSqlA[cleanSqlA.length-1]; | ||
| 314 | - // } | ||
| 315 | - SqlValidateUtil.validateMysqlSql(cleanSql); | ||
| 316 | - // 4. 执行SQL获取结构化结果 | ||
| 317 | - // Map<String,Object> params = new HashMap<>(); | 457 | +// List<ChatMessage> chatMessage2 = operableChatMemoryProvider.getCurrentChatMessages(session.getUserId()); |
| 318 | try{ | 458 | try{ |
| 319 | sqlResult = dynamicExeDbService.findSql(new HashMap<>(),cleanSql); | 459 | sqlResult = dynamicExeDbService.findSql(new HashMap<>(),cleanSql); |
| 320 | }catch (Exception e){ | 460 | }catch (Exception e){ |
| 321 | throw new SqlGenerateException(e.getMessage()+" OLDSQL "+cleanSql); | 461 | throw new SqlGenerateException(e.getMessage()+" OLDSQL "+cleanSql); |
| 322 | } | 462 | } |
| 323 | }catch (SqlValidateException e){ | 463 | }catch (SqlValidateException e){ |
| 464 | + //删除记录 | ||
| 465 | +// operableChatMemoryProvider.deleteUserLasterMessageBySize(userId,3); | ||
| 324 | sError_mes = e.getMessage(); | 466 | sError_mes = e.getMessage(); |
| 325 | doAiSqlErrorHistoryThread(session, StrUtil.EMPTY, cleanSql, sError_mes,input); | 467 | doAiSqlErrorHistoryThread(session, StrUtil.EMPTY, cleanSql, sError_mes,input); |
| 326 | throw e; | 468 | throw e; |
| 327 | }catch (SqlGenerateException e){ | 469 | }catch (SqlGenerateException e){ |
| 470 | +// operableChatMemoryProvider.deleteUserLasterMessageBySize(userId,3); | ||
| 328 | sError_mes = e.getMessage(); | 471 | sError_mes = e.getMessage(); |
| 329 | doAiSqlErrorHistoryThread(session, StrUtil.EMPTY, cleanSql, sError_mes,input); | 472 | doAiSqlErrorHistoryThread(session, StrUtil.EMPTY, cleanSql, sError_mes,input); |
| 330 | throw e; | 473 | throw e; |
| @@ -337,9 +480,11 @@ public class XlyErpService { | @@ -337,9 +480,11 @@ public class XlyErpService { | ||
| 337 | if(Integer.valueOf(iErroCount)>0){ | 480 | if(Integer.valueOf(iErroCount)>0){ |
| 338 | doAiSqlErrorHistoryThread(session, cleanSql, StrUtil.EMPTY, StrUtil.EMPTY,input); | 481 | doAiSqlErrorHistoryThread(session, cleanSql, StrUtil.EMPTY, StrUtil.EMPTY,input); |
| 339 | } | 482 | } |
| 340 | - | ||
| 341 | - //执行操作记录表 | ||
| 342 | - doAiUserAgentQuestion(session,input,cleanSql,chatMessage); | 483 | + //插入常用操作 |
| 484 | + if(doAddSql){ | ||
| 485 | + //执行操作记录表 | ||
| 486 | + doAiUserAgentQuestion(session,input,cleanSql,chatMessage); | ||
| 487 | + } | ||
| 343 | String sText = aiAgent.explainSqlResult( | 488 | String sText = aiAgent.explainSqlResult( |
| 344 | userId, | 489 | userId, |
| 345 | userInput, | 490 | userInput, |
| @@ -350,6 +495,28 @@ public class XlyErpService { | @@ -350,6 +495,28 @@ public class XlyErpService { | ||
| 350 | return sText; | 495 | return sText; |
| 351 | } | 496 | } |
| 352 | 497 | ||
| 498 | + /*** | ||
| 499 | + * @Author 钱豹 | ||
| 500 | + * @Date 17:04 2026/3/19 | ||
| 501 | + * @Param [session] | ||
| 502 | + * @return java.lang.String | ||
| 503 | + * @Description 获取动态SQL（历史中查询） | ||
| 504 | + **/ | ||
| 505 | + private String getDynamicTableNl2Sql(UserSceneSession session,String input){ | ||
| 506 | +// String sReidKey = SqlValidateUtil.getsKey( session.getCurrentScene().getSId(), session.getCurrentTool().getSId(), input); | ||
| 507 | +// Object sSql = redisService.get(sReidKey); | ||
| 508 | +// if(ObjectUtil.isNotEmpty(sSql)){ | ||
| 509 | +// return sSql.toString(); | ||
| 510 | +// } | ||
| 511 | + String searchText = session.getCurrentScene().getSId()+"_"+session.getCurrentTool().getSId()+input; | ||
| 512 | + //SqlValidateUtil.getsKey( session.getCurrentScene().getSId(), session.getCurrentTool().getSId(), SqlValidateUtil.getsQuestion(session.getSUserQuestionList())); | ||
| 513 | + //根据问题查询向量库 | ||
| 514 | + Map<String,Object> serMap = aiGlobalAgentQuestionSqlEmitterService.queryAiGlobalAgentQuestionSqlEmitter(searchText, "ai_global_agent_question_sql"); | ||
| 515 | + if(ObjectUtil.isNotEmpty(serMap)){ | ||
| 516 | + return serMap.get("sSqlContent").toString(); | ||
| 517 | + } | ||
| 518 | + return null; | ||
| 519 | + } | ||
| 353 | 520 | ||
| 354 | /*** | 521 | /*** |
| 355 | * @Author 钱豹 | 522 | * @Author 钱豹 |
| @@ -485,7 +652,11 @@ public class XlyErpService { | @@ -485,7 +652,11 @@ public class XlyErpService { | ||
| 485 | .build(); | 652 | .build(); |
| 486 | UserSceneSessionService.ERP_AGENT_CACHE.put(userId, aiAgent); | 653 | UserSceneSessionService.ERP_AGENT_CACHE.put(userId, aiAgent); |
| 487 | // 初始化AiService 以防止热加载太慢 找不到相应的方法 | 654 | // 初始化AiService 以防止热加载太慢 找不到相应的方法 |
| 488 | - aiAgent.chat(userId, "initAiService"); | 655 | + try{ |
| 656 | + aiAgent.chat(userId, "initAiService"); | ||
| 657 | + }catch (Exception e){ | ||
| 658 | + e.printStackTrace(); | ||
| 659 | + } | ||
| 489 | log.info("用户{}Agent构建完成，已选场景：{},场景ID{}", userId, session.isSceneSelected() ? session.getCurrentScene().getSSceneName() : "未选（全场景匹配）", dynamicToolProvider.sSceneIdMap.get(userId)); | 660 | log.info("用户{}Agent构建完成，已选场景：{},场景ID{}", userId, session.isSceneSelected() ? session.getCurrentScene().getSSceneName() : "未选（全场景匹配）", dynamicToolProvider.sSceneIdMap.get(userId)); |
| 490 | } | 661 | } |
| 491 | return aiAgent; | 662 | return aiAgent; |
src/main/java/com/xly/thread/AiUserAgentQuestionThread.java
| 1 | package com.xly.thread; | 1 | package com.xly.thread; |
| 2 | 2 | ||
| 3 | 3 | ||
| 4 | +import cn.hutool.core.lang.generator.UUIDGenerator; | ||
| 4 | import cn.hutool.core.util.ObjectUtil; | 5 | import cn.hutool.core.util.ObjectUtil; |
| 5 | import cn.hutool.core.util.StrUtil; | 6 | import cn.hutool.core.util.StrUtil; |
| 6 | -import com.xly.config.OperableChatMemoryProvider; | ||
| 7 | import com.xly.config.SpringContextHolder; | 7 | import com.xly.config.SpringContextHolder; |
| 8 | import com.xly.entity.UserSceneSession; | 8 | import com.xly.entity.UserSceneSession; |
| 9 | +import com.xly.milvus.service.AiGlobalAgentQuestionSqlEmitterService; | ||
| 9 | import com.xly.service.DynamicExeDbService; | 10 | import com.xly.service.DynamicExeDbService; |
| 11 | +import com.xly.service.RedisService; | ||
| 12 | +import com.xly.util.MD5Util; | ||
| 13 | +import com.xly.util.SqlValidateUtil; | ||
| 10 | import dev.langchain4j.data.message.ChatMessage; | 14 | import dev.langchain4j.data.message.ChatMessage; |
| 11 | import dev.langchain4j.data.message.ChatMessageType; | 15 | import dev.langchain4j.data.message.ChatMessageType; |
| 12 | -import jnr.ffi.annotations.In; | ||
| 13 | - | ||
| 14 | -import java.util.Arrays; | ||
| 15 | import java.util.HashMap; | 16 | import java.util.HashMap; |
| 16 | import java.util.List; | 17 | import java.util.List; |
| 17 | import java.util.Map; | 18 | import java.util.Map; |
| @@ -36,17 +37,30 @@ public class AiUserAgentQuestionThread implements Runnable { | @@ -36,17 +37,30 @@ public class AiUserAgentQuestionThread implements Runnable { | ||
| 36 | String sSceneId = session.getCurrentScene().getSId(); | 37 | String sSceneId = session.getCurrentScene().getSId(); |
| 37 | String sMethodId = session.getCurrentTool().getSId(); | 38 | String sMethodId = session.getCurrentTool().getSId(); |
| 38 | DynamicExeDbService dynamicExeDbService = SpringContextHolder.getBean(DynamicExeDbService.class); | 39 | DynamicExeDbService dynamicExeDbService = SpringContextHolder.getBean(DynamicExeDbService.class); |
| 40 | + RedisService redisService = SpringContextHolder.getBean(RedisService.class); | ||
| 41 | + AiGlobalAgentQuestionSqlEmitterService aiGlobalAgentQuestionSqlEmitterService = SpringContextHolder.getBean(AiGlobalAgentQuestionSqlEmitterService.class); | ||
| 39 | String sQuestionGroupNo = session.getSUserQuestionList().get(0); | 42 | String sQuestionGroupNo = session.getSUserQuestionList().get(0); |
| 40 | Integer bRedis = (session.getSUserQuestionList().size()==1)?1:0; | 43 | Integer bRedis = (session.getSUserQuestionList().size()==1)?1:0; |
| 41 | Map<String, Object> data = getMap(sSceneId, sMethodId,bRedis,sQuestionGroupNo); | 44 | Map<String, Object> data = getMap(sSceneId, sMethodId,bRedis,sQuestionGroupNo); |
| 42 | - data.put("sQuestion",getsQuestion(session.getSUserQuestionList())); | 45 | + data.put("sQuestion",SqlValidateUtil.getsQuestion(session.getSUserQuestionList())); |
| 46 | + data.put("sId",new UUIDGenerator().next()); | ||
| 47 | + //插入Redis缓存 | ||
| 48 | + if(bRedis==1 && ObjectUtil.isNotEmpty(sSqlContent)){ | ||
| 49 | + String sReidKey = SqlValidateUtil.getsKey( sSceneId, sMethodId, sQuestionGroupNo); | ||
| 50 | + redisService.set(sReidKey,sSqlContent); | ||
| 51 | + } | ||
| 52 | + String sKey = sSceneId+"_"+sMethodId +"_"+sQuestion; | ||
| 53 | +// SqlValidateUtil.getsKey( sSceneId, sMethodId, SqlValidateUtil.getsQuestion(session.getSUserQuestionList())); | ||
| 54 | + //存入向量库 | ||
| 55 | + aiGlobalAgentQuestionSqlEmitterService.addAiGlobalAgentQuestionSqlEmitter(sKey,data,sQuestion,sSqlContent,"ai_global_agent_question_sql"); | ||
| 56 | + //调用数据库插入数据库 | ||
| 43 | Map<String, Object> searMap = dynamicExeDbService.getDoProMap(sProName, data); | 57 | Map<String, Object> searMap = dynamicExeDbService.getDoProMap(sProName, data); |
| 44 | dynamicExeDbService.getCallPro(searMap, sProName); | 58 | dynamicExeDbService.getCallPro(searMap, sProName); |
| 45 | } | 59 | } |
| 46 | 60 | ||
| 47 | - private String getsQuestion(List<String> sUserQuestionList){ | ||
| 48 | - return String.join(",", sUserQuestionList); | ||
| 49 | - } | 61 | + |
| 62 | + | ||
| 63 | + | ||
| 50 | 64 | ||
| 51 | //获取组ID | 65 | //获取组ID |
| 52 | private String getQuestionGroupNo(){ | 66 | private String getQuestionGroupNo(){ |
src/main/java/com/xly/token/RedisTokenManager.java
src/main/java/com/xly/tts/bean/TTSResponseDTO.java
| @@ -6,6 +6,7 @@ import lombok.AllArgsConstructor; | @@ -6,6 +6,7 @@ import lombok.AllArgsConstructor; | ||
| 6 | import lombok.Builder; | 6 | import lombok.Builder; |
| 7 | import lombok.Data; | 7 | import lombok.Data; |
| 8 | import lombok.NoArgsConstructor; | 8 | import lombok.NoArgsConstructor; |
| 9 | +import reactor.core.publisher.Flux; | ||
| 9 | 10 | ||
| 10 | import java.io.Serializable; | 11 | import java.io.Serializable; |
| 11 | 12 | ||
| @@ -70,6 +71,8 @@ public class TTSResponseDTO implements Serializable { | @@ -70,6 +71,8 @@ public class TTSResponseDTO implements Serializable { | ||
| 70 | 71 | ||
| 71 | private String sReturnType = ReturnTypeCode.MAKEDOWN.getCode(); | 72 | private String sReturnType = ReturnTypeCode.MAKEDOWN.getCode(); |
| 72 | 73 | ||
| 74 | + private Boolean ErpComplete; | ||
| 75 | + | ||
| 73 | /** | 76 | /** |
| 74 | * 创建失败响应 | 77 | * 创建失败响应 |
| 75 | */ | 78 | */ |
src/main/java/com/xly/tts/service/PythonTtsProxyService.java
| @@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil; | @@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil; | ||
| 4 | import cn.hutool.core.util.StrUtil; | 4 | import cn.hutool.core.util.StrUtil; |
| 5 | import com.xly.constant.BusinessCode; | 5 | import com.xly.constant.BusinessCode; |
| 6 | import com.xly.constant.ReturnTypeCode; | 6 | import com.xly.constant.ReturnTypeCode; |
| 7 | +import com.xly.entity.AiResponseAccumulator; | ||
| 7 | import com.xly.entity.AiResponseDTO; | 8 | import com.xly.entity.AiResponseDTO; |
| 8 | import com.xly.service.UserSceneSessionService; | 9 | import com.xly.service.UserSceneSessionService; |
| 9 | import com.xly.service.XlyErpService; | 10 | import com.xly.service.XlyErpService; |
| @@ -16,15 +17,15 @@ import org.springframework.core.io.InputStreamResource; | @@ -16,15 +17,15 @@ import org.springframework.core.io.InputStreamResource; | ||
| 16 | import org.springframework.http.*; | 17 | import org.springframework.http.*; |
| 17 | import org.springframework.stereotype.Service; | 18 | import org.springframework.stereotype.Service; |
| 18 | import org.springframework.web.client.RestTemplate; | 19 | import org.springframework.web.client.RestTemplate; |
| 20 | +import reactor.core.publisher.Flux; | ||
| 19 | 21 | ||
| 20 | import javax.annotation.PostConstruct; | 22 | import javax.annotation.PostConstruct; |
| 21 | import java.io.*; | 23 | import java.io.*; |
| 24 | +import java.time.Duration; | ||
| 22 | import java.util.*; | 25 | import java.util.*; |
| 23 | import java.util.concurrent.CompletableFuture; | 26 | import java.util.concurrent.CompletableFuture; |
| 24 | import java.util.concurrent.ExecutorService; | 27 | import java.util.concurrent.ExecutorService; |
| 25 | import java.util.concurrent.Executors; | 28 | import java.util.concurrent.Executors; |
| 26 | -import java.net.URL; | ||
| 27 | -import java.net.HttpURLConnection; | ||
| 28 | import java.io.InputStream; | 29 | import java.io.InputStream; |
| 29 | 30 | ||
| 30 | @Slf4j | 31 | @Slf4j |
| @@ -81,6 +82,111 @@ public class PythonTtsProxyService { | @@ -81,6 +82,111 @@ public class PythonTtsProxyService { | ||
| 81 | return synthesizeStreamAi(request, voiceText); | 82 | return synthesizeStreamAi(request, voiceText); |
| 82 | } | 83 | } |
| 83 | 84 | ||
| 85 | + /** | ||
| 86 | + * 流式ERP + 流式TTS合成 | ||
| 87 | + * 先流式输出ERP文本，完成后自动开始TTS合成 | ||
| 88 | + * 使用现有TTSResponseDTO字段 | ||
| 89 | + */ | ||
| 90 | + public Flux<TTSResponseDTO> synthesizeStreamAiStream(TTSRequestDTO request) { | ||
| 91 | + String userInput = request.getText(); | ||
| 92 | + String sUserId = request.getUserid(); | ||
| 93 | + String sUserName = request.getUsername(); | ||
| 94 | + String sBrandsId = request.getBrandsid(); | ||
| 95 | + String sSubsidiaryId = request.getSubsidiaryid(); | ||
| 96 | + String sUserType = request.getUsertype(); | ||
| 97 | + String authorization = request.getAuthorization(); | ||
| 98 | + | ||
| 99 | + String requestId = UUID.randomUUID().toString(); | ||
| 100 | + log.info("开始流式处理: requestId={}, userId={}", requestId, sUserId); | ||
| 101 | + | ||
| 102 | + // 创建累积器（用于累积完整的AiResponseDTO） | ||
| 103 | + AiResponseAccumulator accumulator = new AiResponseAccumulator(requestId); | ||
| 104 | + | ||
| 105 | + // 1. 处理ERP流，将AiResponseDTO转换为TTSResponseDTO | ||
| 106 | + Flux<TTSResponseDTO> erpStream = xlyErpService.erpUserInputStream( | ||
| 107 | + userInput, sUserId, sUserName, sBrandsId, | ||
| 108 | + sSubsidiaryId, sUserType, authorization | ||
| 109 | + ) | ||
| 110 | + .doOnNext(aiResponse -> { | ||
| 111 | + // 设置请求ID | ||
| 112 | + aiResponse.setRequestId(requestId); | ||
| 113 | + // 后台累积完整文本（为后续TTS做准备） | ||
| 114 | + accumulator.accumulate(aiResponse); | ||
| 115 | + log.debug("收到ERP片段: requestId={}, chunk={}/{}", | ||
| 116 | + requestId, | ||
| 117 | + aiResponse.getChunkIndex(), | ||
| 118 | + aiResponse.getTotalChunks()); | ||
| 119 | + }) | ||
| 120 | + .map(aiResponse -> { | ||
| 121 | + // 将AiResponseDTO转换为TTSResponseDTO | ||
| 122 | + // 使用processedText字段传递AI文本片段 | ||
| 123 | + // 使用systemText字段传递系统文本片段 | ||
| 124 | + return TTSResponseDTO.builder() | ||
| 125 | + .code(200) | ||
| 126 | + .message("ERP_CHUNK") // message字段标记为ERP文本块 | ||
| 127 | + .requestId(requestId) | ||
| 128 | + .processedText(aiResponse.getTextFragment()) // 用processedText传递AI文本片段 | ||
| 129 | + .systemText(aiResponse.getSystemTextFragment()) // 用systemText传递系统文本片段 | ||
| 130 | + .sSceneName(aiResponse.getSSceneName()) | ||
| 131 | + .sMethodName(aiResponse.getSMethodName()) | ||
| 132 | + .sReturnType(aiResponse.getSReturnType()) | ||
| 133 | + .timestamp(System.currentTimeMillis()) | ||
| 134 | + .build(); | ||
| 135 | + }); | ||
| 136 | + | ||
| 137 | + // 2. ERP完成后，发送完成标记，然后开始TTS合成 | ||
| 138 | + return erpStream | ||
| 139 | + .concatWith(Flux.defer(() -> { | ||
| 140 | + // 获取完整的累积结果 | ||
| 141 | + AiResponseDTO completeResponse = accumulator.getCompleteResponse(); | ||
| 142 | + | ||
| 143 | + // 验证ERP结果 | ||
| 144 | + if (StrUtil.isBlank(completeResponse.getAiText())) { | ||
| 145 | + log.warn("ERP返回空文本: requestId={}", requestId); | ||
| 146 | + return Flux.error(new RuntimeException("ERP返回空文本")); | ||
| 147 | + } | ||
| 148 | + | ||
| 149 | + log.info("ERP流式处理完成，开始TTS合成: requestId={}, aiText长度={}", | ||
| 150 | + requestId, completeResponse.getAiText().length()); | ||
| 151 | + | ||
| 152 | + // 3. 发送ERP完成消息（使用完整文本） | ||
| 153 | + TTSResponseDTO erpCompleteDto = TTSResponseDTO.builder() | ||
| 154 | + .code(200) | ||
| 155 | + .message("ERP_COMPLETE") // message标记完成 | ||
| 156 | + .requestId(requestId) | ||
| 157 | + .processedText(completeResponse.getAiText()) // 完整AI文本 | ||
| 158 | + .systemText(completeResponse.getSystemText()) // 完整系统文本 | ||
| 159 | + .sSceneName(completeResponse.getSSceneName()) | ||
| 160 | + .sMethodName(completeResponse.getSMethodName()) | ||
| 161 | + .sReturnType(completeResponse.getSReturnType()) | ||
| 162 | + .timestamp(System.currentTimeMillis()) | ||
| 163 | + .build(); | ||
| 164 | + | ||
| 165 | + // 4. 调用TTS合成（返回TTSResponseDTO流） | ||
| 166 | + Flux<TTSResponseDTO> ttsStream = synthesizeStreamAiNew(request, completeResponse) | ||
| 167 | + .doOnNext(ttsResponse -> { | ||
| 168 | + ttsResponse.setRequestId(requestId); | ||
| 169 | + ttsResponse.setMessage("TTS_SEGMENT"); // message标记为TTS音频段 | ||
| 170 | + }); | ||
| 171 | + | ||
| 172 | + // 先发送ERP完成消息，再发送TTS流 | ||
| 173 | + return Flux.concat(Flux.just(erpCompleteDto), ttsStream); | ||
| 174 | + })) | ||
| 175 | + // 超时控制 | ||
| 176 | + .timeout(Duration.ofSeconds(120)) | ||
| 177 | + // 错误处理 | ||
| 178 | + .onErrorResume(e -> { | ||
| 179 | + log.error("流式处理失败: requestId={}, error={}", requestId, e.getMessage()); | ||
| 180 | + return Flux.just(TTSResponseDTO.error(requestId, 500, e.getMessage())); | ||
| 181 | + }) | ||
| 182 | + // 日志记录 | ||
| 183 | + .doOnComplete(() -> log.info("流式处理完成: requestId={}", requestId)) | ||
| 184 | + .doOnCancel(() -> log.warn("流式处理取消: requestId={}", requestId)); | ||
| 185 | + } | ||
| 186 | + | ||
| 187 | + | ||
| 188 | + | ||
| 189 | + | ||
| 84 | public ResponseEntity<TTSResponseDTO> cleanMemory(TTSRequestDTO request) { | 190 | public ResponseEntity<TTSResponseDTO> cleanMemory(TTSRequestDTO request) { |
| 85 | String sUserId = request.getUserid(); | 191 | String sUserId = request.getUserid(); |
| 86 | String sUserName = request.getUsername(); | 192 | String sUserName = request.getUsername(); |
| @@ -123,7 +229,7 @@ public class PythonTtsProxyService { | @@ -123,7 +229,7 @@ public class PythonTtsProxyService { | ||
| 123 | } | 229 | } |
| 124 | 230 | ||
| 125 | /** | 231 | /** |
| 126 | - * 【保持原有返回类型】不动！内部流式请求Python | 232 | + * 内部流式请求Python |
| 127 | */ | 233 | */ |
| 128 | public ResponseEntity<TTSResponseDTO> synthesizeStreamAi(TTSRequestDTO request, AiResponseDTO aiResponseDTO) { | 234 | public ResponseEntity<TTSResponseDTO> synthesizeStreamAi(TTSRequestDTO request, AiResponseDTO aiResponseDTO) { |
| 129 | String aiText = aiResponseDTO.getAiText(); | 235 | String aiText = aiResponseDTO.getAiText(); |
| @@ -132,12 +238,10 @@ public class PythonTtsProxyService { | @@ -132,12 +238,10 @@ public class PythonTtsProxyService { | ||
| 132 | systemText = StrUtil.EMPTY; | 238 | systemText = StrUtil.EMPTY; |
| 133 | } | 239 | } |
| 134 | String voiceTextNew = AdvancedSymbolRemover.removePunctuationHtml(aiText); | 240 | String voiceTextNew = AdvancedSymbolRemover.removePunctuationHtml(aiText); |
| 135 | - | ||
| 136 | // ============================ | 241 | // ============================ |
| 137 | // 【绝对唯一】不会重复、不会覆盖 | 242 | // 【绝对唯一】不会重复、不会覆盖 |
| 138 | // ============================ | 243 | // ============================ |
| 139 | String cacheKey = request.getUserid() + "_" + System.nanoTime(); | 244 | String cacheKey = request.getUserid() + "_" + System.nanoTime(); |
| 140 | - | ||
| 141 | TTSResponseDTO dto = TTSResponseDTO.builder() | 245 | TTSResponseDTO dto = TTSResponseDTO.builder() |
| 142 | .code(200) | 246 | .code(200) |
| 143 | .message("success") | 247 | .message("success") |
| @@ -159,7 +263,6 @@ public class PythonTtsProxyService { | @@ -159,7 +263,6 @@ public class PythonTtsProxyService { | ||
| 159 | if (!voiceless || ObjectUtil.isEmpty(voiceTextNew)) { | 263 | if (!voiceless || ObjectUtil.isEmpty(voiceTextNew)) { |
| 160 | return ResponseEntity.ok(dto); | 264 | return ResponseEntity.ok(dto); |
| 161 | } | 265 | } |
| 162 | - | ||
| 163 | // 平均分割文字 | 266 | // 平均分割文字 |
| 164 | List<String> textParts = splitTextSmart(voiceTextNew, 30); | 267 | List<String> textParts = splitTextSmart(voiceTextNew, 30); |
| 165 | dto.setAudioSize(textParts.size()); | 268 | dto.setAudioSize(textParts.size()); |
| @@ -199,14 +302,91 @@ public class PythonTtsProxyService { | @@ -199,14 +302,91 @@ public class PythonTtsProxyService { | ||
| 199 | } | 302 | } |
| 200 | } | 303 | } |
| 201 | }, executorService); | 304 | }, executorService); |
| 202 | - | ||
| 203 | return ResponseEntity.ok(dto); | 305 | return ResponseEntity.ok(dto); |
| 204 | } | 306 | } |
| 205 | 307 | ||
| 308 | + | ||
| 309 | + /** | ||
| 310 | + * 内部流式请求Python | ||
| 311 | + */ | ||
| 312 | + public Flux<TTSResponseDTO> synthesizeStreamAiNew(TTSRequestDTO request, AiResponseDTO aiResponseDTO) { | ||
| 313 | + String aiText = aiResponseDTO.getAiText(); | ||
| 314 | + String systemText = aiResponseDTO.getSystemText(); | ||
| 315 | + if (ObjectUtil.isEmpty(systemText)) { | ||
| 316 | + systemText = StrUtil.EMPTY; | ||
| 317 | + } | ||
| 318 | + String voiceTextNew = AdvancedSymbolRemover.removePunctuationHtml(aiText); | ||
| 319 | + // ============================ | ||
| 320 | + // 【绝对唯一】不会重复、不会覆盖 | ||
| 321 | + // ============================ | ||
| 322 | + String cacheKey = request.getUserid() + "_" + System.nanoTime(); | ||
| 323 | + TTSResponseDTO dto = TTSResponseDTO.builder() | ||
| 324 | + .code(200) | ||
| 325 | + .message("success") | ||
| 326 | + .cacheKey(cacheKey) // 前端靠这个取自己的分段 | ||
| 327 | + .originalText(request.getText()) | ||
| 328 | + .processedText(aiText) | ||
| 329 | + .audioText(voiceTextNew) | ||
| 330 | + .systemText(systemText) | ||
| 331 | + .voice(request.getVoice()) | ||
| 332 | + .sSceneName(aiResponseDTO.getSSceneName()) | ||
| 333 | + .sMethodName(aiResponseDTO.getSMethodName()) | ||
| 334 | + .sReturnType(aiResponseDTO.getSReturnType()) | ||
| 335 | + .sCommonts(BusinessCode.COMMONTS.getMessage()) | ||
| 336 | + .timestamp(System.currentTimeMillis()) | ||
| 337 | + .textLength((aiText + systemText).length()) | ||
| 338 | + .build(); | ||
| 339 | + | ||
| 340 | + boolean voiceless = Boolean.TRUE.equals(request.getVoiceless()); | ||
| 341 | + if (!voiceless || ObjectUtil.isEmpty(voiceTextNew)) { | ||
| 342 | + return Flux.just(dto); | ||
| 343 | + } | ||
| 344 | + // 平均分割文字 | ||
| 345 | + List<String> textParts = splitTextSmart(voiceTextNew, 30); | ||
| 346 | + dto.setAudioSize(textParts.size()); | ||
| 347 | + // 异步分段合成 | ||
| 348 | + CompletableFuture.runAsync(() -> { | ||
| 349 | + for (int i = 0; i < textParts.size(); i++) { | ||
| 350 | + String part = textParts.get(i); | ||
| 351 | + if (ObjectUtil.isEmpty(part)) continue; | ||
| 352 | + | ||
| 353 | + try { | ||
| 354 | + Map<String, Object> params = new HashMap<>(); | ||
| 355 | + params.put("text", part); | ||
| 356 | + params.put("voice", request.getVoice()); | ||
| 357 | + params.put("rate", request.getRate() != null ? request.getRate() : "+10%"); | ||
| 358 | + params.put("volume", request.getVolume() != null ? request.getVolume() : "+0%"); | ||
| 359 | + | ||
| 360 | + HttpHeaders headers = new HttpHeaders(); | ||
| 361 | + headers.setContentType(MediaType.APPLICATION_JSON); | ||
| 362 | + headers.setAccept(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); | ||
| 363 | + HttpEntity<Map<String, Object>> entity = new HttpEntity<>(params, headers); | ||
| 364 | + | ||
| 365 | + ResponseEntity<byte[]> response = restTemplate.exchange( | ||
| 366 | + pythonServiceUrl + "/stream-synthesize", | ||
| 367 | + HttpMethod.POST, entity, byte[].class | ||
| 368 | + ); | ||
| 369 | + | ||
| 370 | + if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { | ||
| 371 | + String base64 = Base64.getEncoder().encodeToString(response.getBody()); | ||
| 372 | + | ||
| 373 | + // ============================ | ||
| 374 | + // 【关键】带序号存储！前端靠序号知道顺序！ | ||
| 375 | + // ============================ | ||
| 376 | + LocalAudioCache.addPiece(cacheKey, i, part, base64); | ||
| 377 | + } | ||
| 378 | + } catch (Exception e) { | ||
| 379 | + log.warn("分段合成失败: {}", e.getMessage()); | ||
| 380 | + } | ||
| 381 | + } | ||
| 382 | + }, executorService); | ||
| 383 | + return Flux.just(dto); | ||
| 384 | + } | ||
| 385 | + | ||
| 386 | + // ============================================== | ||
| 387 | + // 智能分段：优先按 。！？； ， 空格 断开 | ||
| 388 | + // 不会把一句话生硬切断，更自然 | ||
| 206 | // ============================================== | 389 | // ============================================== |
| 207 | -// 智能分段：优先按 。！？； ， 空格 断开 | ||
| 208 | -// 不会把一句话生硬切断，更自然 | ||
| 209 | -// ============================================== | ||
| 210 | private List<String> splitTextSmart(String text, int maxLength) { | 390 | private List<String> splitTextSmart(String text, int maxLength) { |
| 211 | List<String> parts = new ArrayList<>(); | 391 | List<String> parts = new ArrayList<>(); |
| 212 | if (text == null || text.isEmpty()) return parts; | 392 | if (text == null || text.isEmpty()) return parts; |
src/main/java/com/xly/util/MD5Util.java
0 → 100644
| 1 | +package com.xly.util; | ||
| 2 | + | ||
| 3 | +import java.security.MessageDigest; | ||
| 4 | +import java.security.NoSuchAlgorithmException; | ||
| 5 | + | ||
| 6 | +/** | ||
| 7 | + * MD5加密工具类 | ||
| 8 | + */ | ||
| 9 | +public class MD5Util { | ||
| 10 | + | ||
| 11 | + /** | ||
| 12 | + * 将字符串进行MD5加密 | ||
| 13 | + * @param input 输入字符串 | ||
| 14 | + * @return MD5加密后的32位小写字符串 | ||
| 15 | + */ | ||
| 16 | + public static String encrypt(String input) { | ||
| 17 | + if (input == null || input.isEmpty()) { | ||
| 18 | + return null; | ||
| 19 | + } | ||
| 20 | + | ||
| 21 | + try { | ||
| 22 | + MessageDigest md = MessageDigest.getInstance("MD5"); | ||
| 23 | + byte[] messageDigest = md.digest(input.getBytes()); | ||
| 24 | + | ||
| 25 | + // 将字节数组转换为十六进制字符串 | ||
| 26 | + StringBuilder hexString = new StringBuilder(); | ||
| 27 | + for (byte b : messageDigest) { | ||
| 28 | + String hex = Integer.toHexString(0xff & b); | ||
| 29 | + if (hex.length() == 1) { | ||
| 30 | + hexString.append('0'); | ||
| 31 | + } | ||
| 32 | + hexString.append(hex); | ||
| 33 | + } | ||
| 34 | + return hexString.toString(); | ||
| 35 | + | ||
| 36 | + } catch (NoSuchAlgorithmException e) { | ||
| 37 | + throw new RuntimeException("MD5加密失败", e); | ||
| 38 | + } | ||
| 39 | + } | ||
| 40 | + | ||
| 41 | + /** | ||
| 42 | + * 将字符串进行MD5加密（大写） | ||
| 43 | + * @param input 输入字符串 | ||
| 44 | + * @return MD5加密后的32位大写字符串 | ||
| 45 | + */ | ||
| 46 | + public static String encryptToUpperCase(String input) { | ||
| 47 | + String result = encrypt(input); | ||
| 48 | + return result != null ? result.toUpperCase() : null; | ||
| 49 | + } | ||
| 50 | + | ||
| 51 | + /** | ||
| 52 | + * 验证字符串与MD5值是否匹配 | ||
| 53 | + * @param input 输入字符串 | ||
| 54 | + * @param md5 MD5值 | ||
| 55 | + * @return 是否匹配 | ||
| 56 | + */ | ||
| 57 | + public static boolean verify(String input, String md5) { | ||
| 58 | + String encrypted = encrypt(input); | ||
| 59 | + return encrypted != null && encrypted.equalsIgnoreCase(md5); | ||
| 60 | + } | ||
| 61 | + | ||
| 62 | + /** | ||
| 63 | + * 获取文件的MD5值 | ||
| 64 | + * @param bytes 文件字节数组 | ||
| 65 | + * @return 文件MD5值 | ||
| 66 | + */ | ||
| 67 | + public static String getFileMD5(byte[] bytes) { | ||
| 68 | + try { | ||
| 69 | + MessageDigest md = MessageDigest.getInstance("MD5"); | ||
| 70 | + byte[] messageDigest = md.digest(bytes); | ||
| 71 | + | ||
| 72 | + StringBuilder hexString = new StringBuilder(); | ||
| 73 | + for (byte b : messageDigest) { | ||
| 74 | + String hex = Integer.toHexString(0xff & b); | ||
| 75 | + if (hex.length() == 1) { | ||
| 76 | + hexString.append('0'); | ||
| 77 | + } | ||
| 78 | + hexString.append(hex); | ||
| 79 | + } | ||
| 80 | + return hexString.toString(); | ||
| 81 | + | ||
| 82 | + } catch (NoSuchAlgorithmException e) { | ||
| 83 | + throw new RuntimeException("文件MD5计算失败", e); | ||
| 84 | + } | ||
| 85 | + } | ||
| 86 | + | ||
| 87 | + /** | ||
| 88 | + * MD5加盐加密 | ||
| 89 | + * @param input 输入字符串 | ||
| 90 | + * @param salt 盐值 | ||
| 91 | + * @return 加盐后的MD5值 | ||
| 92 | + */ | ||
| 93 | + public static String encryptWithSalt(String input, String salt) { | ||
| 94 | + return encrypt(input + salt); | ||
| 95 | + } | ||
| 96 | + | ||
| 97 | + /** | ||
| 98 | + * 双重MD5加密 | ||
| 99 | + * @param input 输入字符串 | ||
| 100 | + * @return 双重MD5加密结果 | ||
| 101 | + */ | ||
| 102 | + public static String doubleEncrypt(String input) { | ||
| 103 | + String first = encrypt(input); | ||
| 104 | + return encrypt(first); | ||
| 105 | + } | ||
| 106 | +} | ||
| 0 | \ No newline at end of file | 107 | \ No newline at end of file |
src/main/java/com/xly/util/SqlValidateUtil.java
| @@ -103,4 +103,13 @@ public class SqlValidateUtil { | @@ -103,4 +103,13 @@ public class SqlValidateUtil { | ||
| 103 | .replaceAll("\\n|\\r", " ") | 103 | .replaceAll("\\n|\\r", " ") |
| 104 | .trim(); | 104 | .trim(); |
| 105 | } | 105 | } |
| 106 | + | ||
| 107 | + public static String getsKey(String sSceneId,String sMethodId,String sQuestion){ | ||
| 108 | +// sSceneId+sMethodId+sQuestion | ||
| 109 | + return sSceneId+"_"+sMethodId+"_"+ MD5Util.encrypt(sQuestion); | ||
| 110 | + } | ||
| 111 | + | ||
| 112 | + public static String getsQuestion(List<String> sUserQuestionList){ | ||
| 113 | + return String.join(",", sUserQuestionList); | ||
| 114 | + } | ||
| 106 | } | 115 | } |
src/main/java/com/xly/web/TTSStreamController.java
| @@ -7,6 +7,7 @@ import com.xly.tool.DynamicToolProvider; | @@ -7,6 +7,7 @@ import com.xly.tool.DynamicToolProvider; | ||
| 7 | import com.xly.tts.bean.*; | 7 | import com.xly.tts.bean.*; |
| 8 | import com.xly.tts.service.LocalAudioCache; | 8 | import com.xly.tts.service.LocalAudioCache; |
| 9 | import com.xly.tts.service.PythonTtsProxyService; | 9 | import com.xly.tts.service.PythonTtsProxyService; |
| 10 | +import jakarta.validation.Valid; | ||
| 10 | import lombok.RequiredArgsConstructor; | 11 | import lombok.RequiredArgsConstructor; |
| 11 | import lombok.extern.slf4j.Slf4j; | 12 | import lombok.extern.slf4j.Slf4j; |
| 12 | import org.springframework.core.io.InputStreamResource; | 13 | import org.springframework.core.io.InputStreamResource; |
| @@ -14,6 +15,8 @@ import org.springframework.http.MediaType; | @@ -14,6 +15,8 @@ import org.springframework.http.MediaType; | ||
| 14 | import org.springframework.http.ResponseEntity; | 15 | import org.springframework.http.ResponseEntity; |
| 15 | import org.springframework.web.bind.annotation.*; | 16 | import org.springframework.web.bind.annotation.*; |
| 16 | import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; | 17 | import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; |
| 18 | +import reactor.core.publisher.Flux; | ||
| 19 | +import reactor.core.publisher.Mono; | ||
| 17 | 20 | ||
| 18 | import javax.annotation.PostConstruct; | 21 | import javax.annotation.PostConstruct; |
| 19 | import javax.annotation.PreDestroy; | 22 | import javax.annotation.PreDestroy; |
| @@ -82,6 +85,24 @@ public class TTSStreamController { | @@ -82,6 +85,24 @@ public class TTSStreamController { | ||
| 82 | return pythonTtsProxyService.synthesizeStreamAi(request); | 85 | return pythonTtsProxyService.synthesizeStreamAi(request); |
| 83 | } | 86 | } |
| 84 | 87 | ||
| 88 | + /** | ||
| 89 | + * 流式合成语音（代理到Python服务） | ||
| 90 | + */ | ||
| 91 | + @PostMapping(value = "/stream/queryFlux", | ||
| 92 | + consumes = {MediaType.APPLICATION_JSON_VALUE}, | ||
| 93 | + produces = {MediaType.APPLICATION_OCTET_STREAM_VALUE, | ||
| 94 | + MediaType.APPLICATION_JSON_VALUE}) | ||
| 95 | + public Flux<TTSResponseDTO> streamFlux(@Valid @RequestBody Mono<TTSRequestDTO> requestMono) { | ||
| 96 | + return requestMono.flatMapMany(request -> { | ||
| 97 | + log.info("处理请求: requestId={}, text长度={}", request.getUserid(), request.getText().length()); | ||
| 98 | + return pythonTtsProxyService.synthesizeStreamAiStream(request); | ||
| 99 | + }) | ||
| 100 | + .doOnSubscribe(sub -> log.debug("流式订阅开始")) | ||
| 101 | + .doOnCancel(() -> log.debug("流式请求被取消")) | ||
| 102 | + .doOnComplete(() -> log.debug("流式响应完成")) | ||
| 103 | + .doOnError(e -> log.error("流式处理错误", e)); | ||
| 104 | + } | ||
| 105 | + | ||
| 85 | @GetMapping("/audio/piece") | 106 | @GetMapping("/audio/piece") |
| 86 | public ResponseEntity<Map<String, String>> getPiece( | 107 | public ResponseEntity<Map<String, String>> getPiece( |
| 87 | @RequestParam String cacheKey, | 108 | @RequestParam String cacheKey, |
| @@ -210,81 +231,6 @@ public class TTSStreamController { | @@ -210,81 +231,6 @@ public class TTSStreamController { | ||
| 210 | } | 231 | } |
| 211 | 232 | ||
| 212 | /** | 233 | /** |
| 213 | - * SSE流式输出（Server-Sent Events） | ||
| 214 | - */ | ||
| 215 | - @GetMapping(value = "/sse-stream", produces = "text/event-stream") | ||
| 216 | - public ResponseEntity<StreamingResponseBody> sseStream( | ||
| 217 | - @RequestParam String text, | ||
| 218 | - @RequestParam(defaultValue = "zh-CN-XiaoxiaoNeural") String voice) { | ||
| 219 | - | ||
| 220 | - log.info("收到SSE流式请求: voice={}", voice); | ||
| 221 | - | ||
| 222 | - TTSRequestDTO request = new TTSRequestDTO(); | ||
| 223 | - request.setText(text); | ||
| 224 | - request.setVoice(voice); | ||
| 225 | - | ||
| 226 | - StreamingResponseBody responseBody = outputStream -> { | ||
| 227 | - try { | ||
| 228 | - outputStream.write(("event: audio-start\ndata: \n\n").getBytes()); | ||
| 229 | - outputStream.flush(); | ||
| 230 | - | ||
| 231 | - // 调用Python服务获取音频 | ||
| 232 | - ResponseEntity<InputStreamResource> response = pythonTtsProxyService.synthesizeStream(request); | ||
| 233 | - | ||
| 234 | - if (response.getBody() != null) { | ||
| 235 | - InputStream inputStream = response.getBody().getInputStream(); | ||
| 236 | - byte[] buffer = new byte[1024]; | ||
| 237 | - int bytesRead; | ||
| 238 | - | ||
| 239 | - int totalBytes = 0; | ||
| 240 | - while ((bytesRead = inputStream.read(buffer)) != -1) { | ||
| 241 | - totalBytes += bytesRead; | ||
| 242 | - | ||
| 243 | - // 发送进度事件 | ||
| 244 | - String progressEvent = String.format( | ||
| 245 | - "event: progress\ndata: {\"bytes\":%d}\n\n", totalBytes); | ||
| 246 | - outputStream.write(progressEvent.getBytes()); | ||
| 247 | - outputStream.flush(); | ||
| 248 | - | ||
| 249 | - // 发送音频数据（base64编码） | ||
| 250 | - String base64Data = java.util.Base64.getEncoder().encodeToString( | ||
| 251 | - java.util.Arrays.copyOfRange(buffer, 0, bytesRead)); | ||
| 252 | - String audioEvent = String.format( | ||
| 253 | - "event: audio-data\ndata: {\"chunk\":\"%s\"}\n\n", base64Data); | ||
| 254 | - outputStream.write(audioEvent.getBytes()); | ||
| 255 | - outputStream.flush(); | ||
| 256 | - } | ||
| 257 | - | ||
| 258 | - // 发送完成事件 | ||
| 259 | - String completeEvent = String.format( | ||
| 260 | - "event: audio-complete\ndata: {\"total_bytes\":%d}\n\n", totalBytes); | ||
| 261 | - outputStream.write(completeEvent.getBytes()); | ||
| 262 | - outputStream.flush(); | ||
| 263 | - } else { | ||
| 264 | - outputStream.write(("event: error\ndata: {\"message\":\"合成失败\"}\n\n").getBytes()); | ||
| 265 | - outputStream.flush(); | ||
| 266 | - } | ||
| 267 | - | ||
| 268 | - } catch (Exception e) { | ||
| 269 | -// log.error("SSE流式输出异常: {}", e.getMessage(), e); | ||
| 270 | - try { | ||
| 271 | - outputStream.write(("event: error\ndata: {\"message\":\"" + | ||
| 272 | - e.getMessage().replace("\"", "\\\"") + "\"}\n\n").getBytes()); | ||
| 273 | - outputStream.flush(); | ||
| 274 | - } catch (Exception ex) { | ||
| 275 | - // 忽略关闭错误 | ||
| 276 | - } | ||
| 277 | - } | ||
| 278 | - }; | ||
| 279 | - | ||
| 280 | - return ResponseEntity.ok() | ||
| 281 | - .header("Content-Type", "text/event-stream") | ||
| 282 | - .header("Cache-Control", "no-cache") | ||
| 283 | - .header("X-Accel-Buffering", "no") // 禁用Nginx缓冲 | ||
| 284 | - .body(responseBody); | ||
| 285 | - } | ||
| 286 | - | ||
| 287 | - /** | ||
| 288 | * 测试接口 | 234 | * 测试接口 |
| 289 | */ | 235 | */ |
| 290 | @GetMapping("/test") | 236 | @GetMapping("/test") |
src/main/resources/application.yml
| @@ -7,6 +7,8 @@ logging: | @@ -7,6 +7,8 @@ logging: | ||
| 7 | com.xly: debug | 7 | com.xly: debug |
| 8 | com.xlyflow: debug | 8 | com.xlyflow: debug |
| 9 | org.springframework: warn | 9 | org.springframework: warn |
| 10 | + ai.djl: DEBUG | ||
| 11 | + dev.langchain4j: DEBUG | ||
| 10 | 12 | ||
| 11 | server: | 13 | server: |
| 12 | port: 8099 | 14 | port: 8099 |
| @@ -52,35 +54,99 @@ spring: | @@ -52,35 +54,99 @@ spring: | ||
| 52 | # REDIS (RedisProperties) | 54 | # REDIS (RedisProperties) |
| 53 | data: | 55 | data: |
| 54 | redis: | 56 | redis: |
| 55 | - host: localhost | ||
| 56 | - port: 6379 # Redis 端口 | ||
| 57 | - database: 0 # 使用的数据库索引（默认为0） | ||
| 58 | - password: # 密码（如果没有设置则为空） | ||
| 59 | - timeout: 3000ms # 连接超时时间 | 57 | + host: 127.0.0.1 |
| 58 | + password: xlyXLY2015 | ||
| 59 | + port: 16379 | ||
| 60 | + database: 0 # index | ||
| 61 | + timeout: 30000ms # 连接超时时长（毫秒） | ||
| 62 | + block-when-exhausted: true #redis配置结束 | ||
| 60 | lettuce: | 63 | lettuce: |
| 61 | pool: | 64 | pool: |
| 62 | max-active: 8 # 连接池最大连接数 | 65 | max-active: 8 # 连接池最大连接数 |
| 63 | max-idle: 8 # 连接池最大空闲连接 | 66 | max-idle: 8 # 连接池最大空闲连接 |
| 64 | - min-idle: 0 # 连接池最小空闲连接 | 67 | + min-idle: 0 # 连接池最小空闲连接' |
| 68 | + | ||
| 69 | +milvus: | ||
| 70 | + host: 112.82.245.194 | ||
| 71 | + port: 19530 | ||
| 72 | + database: xlymilvus | ||
| 73 | + username: | ||
| 74 | + password: | ||
| 75 | + collection: | ||
| 76 | + question-sql: question_sql_vectors | ||
| 77 | + sync: | ||
| 78 | + batch-size: 100 | ||
| 79 | + vector: | ||
| 80 | + dimension: 384 # All-MiniLM-L6-v2 模型的维度 | ||
| 81 | + text: | ||
| 82 | + max-length: 65535 | ||
| 83 | + schema: | ||
| 84 | + type: base # base, detailed, partitioned, searchable | ||
| 85 | + # 重试配置 | ||
| 86 | + retry: | ||
| 87 | + enabled: true # 是否启用重试 | ||
| 88 | + max-retry-times: 75 # 最大重试次数（SDK默认值） | ||
| 89 | + initial-backoff-ms: 10 # 初始退避时间(ms) | ||
| 90 | + max-backoff-ms: 3000 # 最大退避时间(ms) | ||
| 91 | + backoff-multiplier: 3 # 退避乘数 | ||
| 92 | + retry-on-rate-limit: true # 遇到限流是否重试 | ||
| 93 | + max-retry-timeout-ms: 0 # 最大重试超时时间(0表示不限制) | ||
| 94 | + | ||
| 95 | + # 操作级别的重试配置（可选，覆盖默认值） | ||
| 96 | + operations: | ||
| 97 | + query: | ||
| 98 | + max-retry-times: 3 | ||
| 99 | + initial-backoff-ms: 100 | ||
| 100 | + max-backoff-ms: 2000 | ||
| 101 | + insert: | ||
| 102 | + max-retry-times: 2 | ||
| 103 | + initial-backoff-ms: 50 | ||
| 104 | + max-backoff-ms: 1000 | ||
| 105 | + search: | ||
| 106 | + max-retry-times: 3 | ||
| 107 | + initial-backoff-ms: 200 | ||
| 108 | + max-backoff-ms: 3000 | ||
| 109 | + max-retry-timeout-ms: 10000 | ||
| 110 | + | ||
| 111 | +# Actuator配置 | ||
| 112 | +management: | ||
| 113 | + endpoints: | ||
| 114 | + web: | ||
| 115 | + exposure: | ||
| 116 | + include: health,info,metrics | ||
| 117 | + endpoint: | ||
| 118 | + health: | ||
| 119 | + show-details: always | ||
| 120 | + health: | ||
| 121 | + milvus: | ||
| 122 | + enabled: true | ||
| 65 | 123 | ||
| 66 | # application.yml 或 application.properties | 124 | # application.yml 或 application.properties |
| 67 | langchain4j: | 125 | langchain4j: |
| 68 | ollama: | 126 | ollama: |
| 69 | # 聊天模型配置（用于一般对话） | 127 | # 聊天模型配置（用于一般对话） |
| 70 | - base-url: http://121.43.128.225:11434 | 128 | + base-url: http://112.82.245.194:11434 |
| 71 | chat-model-name: qwen2.5:7b-instruct | 129 | chat-model-name: qwen2.5:7b-instruct |
| 130 | +# chat-model-name: qwen3.5:9b | ||
| 72 | # SQL/代码模型配置（专门用于代码和SQL生成） | 131 | # SQL/代码模型配置（专门用于代码和SQL生成） |
| 73 | - sql-model-name: qwen2.5-coder:32b | 132 | + sql-model-name: qwen2.5-coder:7b |
| 133 | +# sql-model-name: qwen2.5-coder:32b | ||
| 134 | +# sql-model-name: mdq100/qwen3.5-coder:35b | ||
| 74 | # 或者如果两个模型在同一服务器，可以使用同一个URL | 135 | # 或者如果两个模型在同一服务器，可以使用同一个URL |
| 75 | 136 | ||
| 137 | +# ollama: | ||
| 138 | +# # 聊天模型配置（用于一般对话） | ||
| 139 | +# base-url: http://112.82.245.194:11434 | ||
| 140 | +# chat-model-name: qwen3.5:9b | ||
| 141 | +# # SQL/代码模型配置（专门用于代码和SQL生成） | ||
| 142 | +# sql-model-name: mdq100/qwen3.5-coder:35b | ||
| 143 | + | ||
| 76 | mybatis: | 144 | mybatis: |
| 77 | mapper-locations: classpath:mapper/*.xml | 145 | mapper-locations: classpath:mapper/*.xml |
| 78 | type-aliases-package: com.xly.entity | 146 | type-aliases-package: com.xly.entity |
| 79 | configuration: | 147 | configuration: |
| 80 | map-underscore-to-camel-case: true | 148 | map-underscore-to-camel-case: true |
| 81 | log-impl: org.apache.ibatis.logging.stdout.StdOutImpl | 149 | log-impl: org.apache.ibatis.logging.stdout.StdOutImpl |
| 82 | - | ||
| 83 | - | ||
| 84 | # 情感预设缓存 | 150 | # 情感预设缓存 |
| 85 | cache: | 151 | cache: |
| 86 | enabled: true | 152 | enabled: true |
src/main/resources/logback-spring.xml
| @@ -118,7 +118,8 @@ | @@ -118,7 +118,8 @@ | ||
| 118 | <logger name="java.sql.Connection" level="DEBUG"/> | 118 | <logger name="java.sql.Connection" level="DEBUG"/> |
| 119 | <logger name="java.sql.Statement" level="DEBUG"/> | 119 | <logger name="java.sql.Statement" level="DEBUG"/> |
| 120 | <logger name="java.sql.PreparedStatement" level="DEBUG"/> | 120 | <logger name="java.sql.PreparedStatement" level="DEBUG"/> |
| 121 | - | 121 | + <logger name="ai.djl" level="DEBUG"/> |
| 122 | + <logger name="dev.langchain4j" level="DEBUG"/> | ||
| 122 | <logger name="com.xly" additivity="false"> | 123 | <logger name="com.xly" additivity="false"> |
| 123 | <appender-ref ref="CONSOLE"/> | 124 | <appender-ref ref="CONSOLE"/> |
| 124 | <appender-ref ref="DEBUG" /> | 125 | <appender-ref ref="DEBUG" /> |
src/main/resources/mapper/DynamicExeDbMapper.xml
src/main/resources/templates/chat.html
| @@ -466,7 +466,7 @@ | @@ -466,7 +466,7 @@ | ||
| 466 | let brandsid= "1111111111"; | 466 | let brandsid= "1111111111"; |
| 467 | let subsidiaryid= "1111111111"; | 467 | let subsidiaryid= "1111111111"; |
| 468 | let usertype= "sysadmin"; | 468 | let usertype= "sysadmin"; |
| 469 | - let authorization="CE444885A9BCFDDE1FD793F8A0931301E9D7DE6CEDD9DE4B83ECE2219C7829A8F3419238942A93E9AD666629E18D159AF7FE144A6407DE745BA0AEC8B235FC1D4CAE6F9AC893752209A98011A981375391D4466816B7D3D1AF306E28B989121C538155B7ADAEE71E899235DC1122F426"; | 469 | + let authorization="1EDB99C9BF070115F7A57AC43D8CB09F0B8C49F979DAB63A2AEA84B372B2B42BF3419238942A93E9AD666629E18D159AF7FE144A6407DE745BA0AEC8B235FC1D35611629BD9166D2BBFC3B7AF31FDF60A31A297DF9BF51740C90173D4CC922B3538155B7ADAEE71E899235DC1122F426"; |
| 470 | let hrefLock = window.location.origin+"/xlyAi"; | 470 | let hrefLock = window.location.origin+"/xlyAi"; |
| 471 | 471 | ||
| 472 | const CONFIG = { | 472 | const CONFIG = { |