From 94fd6ace14d380b4302cda93637fc564404b075f Mon Sep 17 00:00:00 2001 From: haungxing <1203316822@qq.com> Date: Wed, 4 Sep 2024 17:17:39 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=90=91=E9=87=8F=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E6=93=8D=E4=BD=9Capi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../database/controller/WeaviateController.java | 38 +++++++++++++++++ .../bigmodel/database/dto/WeaviateSaveDTO.java | 33 +++++++++++++++ .../bigmodel/database/service/WeaviateService.java | 48 +++++++++++----------- .../src/main/resources/template/template.yml | 11 ++++- 4 files changed, 105 insertions(+), 25 deletions(-) create mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java create mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/dto/WeaviateSaveDTO.java diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java new file mode 100644 index 0000000..7683887 --- /dev/null +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java @@ -0,0 +1,38 @@ +package com.hnac.hzims.bigmodel.database.controller; + +import com.hnac.hzims.bigmodel.database.dto.WeaviateSaveDTO; +import com.hnac.hzims.bigmodel.database.service.WeaviateService; +import io.weaviate.client.v1.data.model.WeaviateObject; +import lombok.AllArgsConstructor; +import org.springblade.core.tool.api.R; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +/** + * @Author: huangxing + * @Date: 2024/09/04 14:16 + */ +@RestController +@AllArgsConstructor +@RequestMapping("/weaviate") +public class WeaviateController { + + private final WeaviateService weaviateService; + + @PostMapping("/saveBatch") + public R saveBatch(@RequestBody WeaviateSaveDTO req) { + weaviateService.saveBatch(req.getEntities(), req.getClassName(), req.getAttrsMap()); + return R.success("操作成功！"); + } + + @GetMapping("/list") + public R> list(@RequestParam(value = "id",required = false) String id, @RequestParam("className") String className) { + return R.data(weaviateService.list(id,className)); + } + + @DeleteMapping("/removeById") + public R removeById(@RequestParam(value = "id",required = false) String id, @RequestParam("className") String className) { + return R.status(weaviateService.delete(id,className)); + } +} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/dto/WeaviateSaveDTO.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/dto/WeaviateSaveDTO.java new file mode 100644 index 0000000..715298e --- /dev/null +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/dto/WeaviateSaveDTO.java @@ -0,0 +1,33 @@ +package com.hnac.hzims.bigmodel.database.dto; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +/** + * @Author: huangxing + * @Date: 2024/09/04 14:36 + */ +@Data +@EqualsAndHashCode +public class WeaviateSaveDTO implements Serializable { + + /** + * 向量数据库表名 + */ + private String className; + + /** + * 向量数据库属性名 + */ + private Map attrsMap; + + /** + * 向量数据库存入对象列表 + */ + private List entities; + +} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java index cbc926b..5b0f00e 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java @@ -8,6 +8,7 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.hnac.hzims.bigmodel.configuration.BigModelInvokeApi; import com.hnac.hzinfo.exception.HzServiceException; +import io.weaviate.client.WeaviateClient; import io.weaviate.client.base.Result; import io.weaviate.client.v1.data.api.ObjectCreator; import io.weaviate.client.v1.data.api.ObjectDeleter; @@ -38,10 +39,7 @@ import java.util.stream.IntStream; @Slf4j public class WeaviateService { - private final ObjectCreator objectCreator; - private final ObjectUpdater objectUpdater; - private final ObjectDeleter objectDeleter; - private final ObjectsGetter objectsGetter; + private final WeaviateClient weaviateClient; private final BigModelInvokeApi invokeApi; @Value("${gglm.vectorUrl}") @@ -55,7 +53,7 @@ public class WeaviateService { * @return 保存操作结果 */ public Boolean save(Object entity, String className, List attrs) { - ObjectCreator creator = objectCreator.withClassName(className); + ObjectCreator creator = weaviateClient.data().creator().withClassName(className); if(Func.isNotEmpty(attrs)) { JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity)); List vectors = attrs.stream().map(attr -> jsonObject.getString(attr)).collect(Collectors.toList()); @@ -74,7 +72,7 @@ public class WeaviateService { * @return 保存操作结果 */ public Boolean saveBatch(List entities,String className, Map attrsMap) { - ObjectCreator creator = objectCreator.withClassName(className); + ObjectCreator creator = weaviateClient.data().creator().withClassName(className); List vectorStrs = Lists.newArrayList(); List attrs = Lists.newArrayList(); if(Func.isNotEmpty(attrsMap)) { @@ -91,7 +89,12 @@ public class WeaviateService { Float[] vectors = this.compute(vectorStrs); List> vector = this.splitVector(entities.size(), attrsMap, vectors); for(int i = 0; i < entities.size(); i++) { - creator.withProperties(BeanUtil.toMap(entities.get(i))).withVectors(vector.get(i)).run(); + // log.info("vector:{}",JSON.toJSONString(vector.get(i))); + JSONObject object = JSONObject.parseObject(JSON.toJSONString(entities.get(i))); + Map properties = new HashMap<>(); + object.forEach((k,v) -> properties.put(k,v)); + log.info("properties:{}",JSON.toJSONString(properties)); + creator.withProperties(properties).withVectors(vector.get(i)).run(); } } else { entities.forEach(entity -> creator.withProperties(BeanUtil.toMap(entity)).run()); @@ -105,18 +108,15 @@ public class WeaviateService { * @param className 表名 * @return 删除结果 */ - public Boolean deleteByClassName(String className) { - Result result = objectDeleter.withClassName(className).run(); - return !result.hasErrors(); - } - - /** - * 删除向量数据库（ID） - * @param id 向量数据库ID - * @return 删除结果 - */ - public Boolean deleteById(String id) { - Result result = objectDeleter.withID(id).run(); + public Boolean delete(String id,String className) { + ObjectDeleter deleter = weaviateClient.data().deleter(); + if(Func.isNotEmpty(id)) { + deleter.withID(id); + } + if(Func.isNotEmpty(className)) { + deleter.withClassName(className); + } + Result result = deleter.run(); return !result.hasErrors(); } @@ -126,7 +126,7 @@ public class WeaviateService { * @return 更新结果 */ public Boolean updateById(String id, Object entity, String className, Map attrMap) { - ObjectUpdater updater = objectUpdater.withClassName(className).withID(id).withProperties(BeanUtil.toMap(entity)); + ObjectUpdater updater = weaviateClient.data().updater().withClassName(className).withID(id).withProperties(BeanUtil.toMap(entity)); // 计算向量 Map vector = new HashMap<>(); if(Func.isNotEmpty(attrMap)) { @@ -143,7 +143,8 @@ public class WeaviateService { return !result.hasErrors(); } - public List list(String id,String className) { + public List list(String id,String className) { + ObjectsGetter objectsGetter = weaviateClient.data().objectsGetter(); if(Func.isNotEmpty(id)) { objectsGetter.withID(id); } @@ -154,7 +155,7 @@ public class WeaviateService { if(result.hasErrors()) { throw new HzServiceException("查询失败！"); } - return result.getResult().stream().map(WeaviateObject::getProperties).collect(Collectors.toList()); + return result.getResult(); } /** @@ -176,10 +177,11 @@ public class WeaviateService { List splitIndex = this.getSplitIndex(vectors.size(), attrsMap.size()); AtomicInteger i = new AtomicInteger(); attrsMap.forEach((k,v) -> { - List vector = vectors.subList(splitIndex.get(i.get()), splitIndex.get(i.get() + (vectors.size() / attrsMap.size()))); + List vector = vectors.subList(splitIndex.get(i.get()), splitIndex.get(i.get()) + (vectors.size() / attrsMap.size())); vectorMap.put(k, vector.toArray(new Float[vector.size()])); i.getAndIncrement(); }); + result.add(vectorMap); }); return result; } diff --git a/hzims-service/hzims-big-model/src/main/resources/template/template.yml b/hzims-service/hzims-big-model/src/main/resources/template/template.yml index ea6a966..8674b5f 100644 --- a/hzims-service/hzims-big-model/src/main/resources/template/template.yml +++ b/hzims-service/hzims-big-model/src/main/resources/template/template.yml @@ -63,7 +63,7 @@ gglm: smartReportGeneratePower: "/custom/smart_report_generate_power" assistantAnalyseAsk: "/qa/assistant_analyse_ask" updateKnowledge: "/kn/update_knowledge" - compute: "compute" + compute: "/compute" swagger: base-packages: com.hnac.hzims.bigmodel @@ -84,4 +84,11 @@ xxl: bigmodel: zhipuai: url: https://open.bigmodel.cn/api/paas/v4/chat/completions - apiSecret: dfd23052747674818c7ac6f9922beff1.n2o5JEdfnrLbFU53 \ No newline at end of file + apiSecret: dfd23052747674818c7ac6f9922beff1.n2o5JEdfnrLbFU53 + +weaviate: + datasource: + schema: http + host: 192.168.60.16 + port: 9992 + apiKey: 123