From ea653a473f54926ef16c03affef632add024e59a Mon Sep 17 00:00:00 2001 From: haungxing <1203316822@qq.com> Date: Mon, 26 Aug 2024 14:58:49 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=B8=B4=E6=97=B6=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bigmodel/database/entity/WeaviateEntity.java | 23 +++ .../hzinfo/inspect/obj/entity/ObjectEntity.java | 2 +- .../bigmodel/configuration/BigModelInvokeApi.java | 5 + .../bigmodel/configuration/WeaviateConfig.java | 20 --- .../bigmodel/configuration/WeaviateConfigure.java | 56 +++++++ .../bigmodel/configuration/WeaviateProperties.java | 32 ++++ .../bigmodel/database/service/WeaviateService.java | 183 +++++++++++++++++++++ .../hzims/bigmodel/database/util/WeaviateUtil.java | 11 ++ .../bigmodel/zhipuai/config/ZhipuAiConfig.java | 35 ---- .../src/main/resources/template/template.yml | 2 + 10 files changed, 313 insertions(+), 56 deletions(-) create mode 100644 hzims-service-api/big-model-api/src/main/java/com/hnac/hzims/bigmodel/database/entity/WeaviateEntity.java delete mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfig.java create mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfigure.java create mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateProperties.java create mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java create mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/util/WeaviateUtil.java diff --git a/hzims-service-api/big-model-api/src/main/java/com/hnac/hzims/bigmodel/database/entity/WeaviateEntity.java b/hzims-service-api/big-model-api/src/main/java/com/hnac/hzims/bigmodel/database/entity/WeaviateEntity.java new file mode 100644 index 0000000..6c2b3ba --- /dev/null +++ b/hzims-service-api/big-model-api/src/main/java/com/hnac/hzims/bigmodel/database/entity/WeaviateEntity.java @@ -0,0 +1,23 @@ +package com.hnac.hzims.bigmodel.database.entity; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.io.Serializable; + +/** + * @Author: huangxing + * @Date: 2024/08/22 19:35 + */ +@Data +@EqualsAndHashCode +public class WeaviateEntity implements Serializable { + + @JSONField(name = "item_id") + private String itemId; + + @JSONField(name = "item_name") + private String itemName; + +} diff --git a/hzims-service-api/inspect-api/src/main/java/com/hnac/hzinfo/inspect/obj/entity/ObjectEntity.java b/hzims-service-api/inspect-api/src/main/java/com/hnac/hzinfo/inspect/obj/entity/ObjectEntity.java index 4a7efdb..7ef2305 100644 --- a/hzims-service-api/inspect-api/src/main/java/com/hnac/hzinfo/inspect/obj/entity/ObjectEntity.java +++ b/hzims-service-api/inspect-api/src/main/java/com/hnac/hzinfo/inspect/obj/entity/ObjectEntity.java @@ -49,7 +49,7 @@ public class ObjectEntity extends TenantEntity { * 地理位置 */ @ApiModelProperty(value = "地理位置,长度必须为1到128") - @Size(min=1,max = 128,message ="地理位置长度必须为1到128") + // @Size(min=1,max = 128,message ="地理位置长度必须为1到128") private String address; /** * 经度(东经) diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/BigModelInvokeApi.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/BigModelInvokeApi.java index c11b4b6..b6b842a 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/BigModelInvokeApi.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/BigModelInvokeApi.java @@ -62,4 +62,9 @@ public class BigModelInvokeApi { */ private String updateKnowledge; + /** + * 计算向量 + */ + private String compute; + } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfig.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfig.java deleted file mode 100644 index 022663a..0000000 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfig.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.hnac.hzims.bigmodel.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.reactive.function.client.WebClient; -import reactor.core.publisher.Mono; - -/** - * @Author: huangxing - * @Date: 2024/08/21 15:22 - */ -@Configuration -public class WeaviateConfig { - - @Bean - public WebClient weaviateClient() { - return WebClient.create("http://192.168.60.16:9992"); - } - -} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfigure.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfigure.java new file mode 100644 index 0000000..6bd2f14 --- /dev/null +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfigure.java @@ -0,0 +1,56 @@ +package com.hnac.hzims.bigmodel.configuration; + +import io.weaviate.client.Config; +import io.weaviate.client.WeaviateAuthClient; +import io.weaviate.client.WeaviateClient; +import io.weaviate.client.v1.auth.exception.AuthException; +import io.weaviate.client.v1.data.api.ObjectCreator; +import io.weaviate.client.v1.data.api.ObjectDeleter; +import io.weaviate.client.v1.data.api.ObjectUpdater; +import io.weaviate.client.v1.data.api.ObjectsGetter; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @Author: huangxing + * @Date: 2024/08/22 18:38 + */ +@Configuration +public class WeaviateConfigure { + + private final WeaviateProperties weaviateProperties; + + public WeaviateConfigure(WeaviateProperties weaviateProperties) { + this.weaviateProperties = weaviateProperties; + } + + @Bean + public WeaviateClient weaviateClient() throws AuthException { + Config config = new Config(this.weaviateProperties.getSchema(), this.weaviateProperties.getHost() + ":" + this.weaviateProperties.getPort()); + return WeaviateAuthClient.apiKey(config,this.weaviateProperties.getApiKey()); + } + + @Bean + public ObjectsGetter objectsGetter() throws AuthException { + WeaviateClient weaviateClient = weaviateClient(); + return weaviateClient.data().objectsGetter(); + } + + @Bean + public ObjectCreator objectCreator() throws AuthException { + WeaviateClient weaviateClient = weaviateClient(); + return weaviateClient.data().creator(); + } + + @Bean + public ObjectDeleter deleter() throws AuthException { + WeaviateClient weaviateClient = weaviateClient(); + return weaviateClient.data().deleter(); + } + + @Bean + public ObjectUpdater updater() throws AuthException { + WeaviateClient weaviateClient = weaviateClient(); + return weaviateClient.data().updater(); + } +} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateProperties.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateProperties.java new file mode 100644 index 0000000..468bae9 --- /dev/null +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateProperties.java @@ -0,0 +1,32 @@ +package com.hnac.hzims.bigmodel.configuration; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * @Author: huangxing + * @Date: 2024/08/21 15:22 + */ +@Data +@Component +@ConfigurationProperties(prefix = "weaviate.datasource") +public class WeaviateProperties { + + private String schema; + + private String host; + + private String port; + + /** + * 登录认证KEY + */ + private String apiKey; + + /** + * 数据库表名前缀 + */ + private String classNamePrefix; + +} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java new file mode 100644 index 0000000..931a6a8 --- /dev/null +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java @@ -0,0 +1,183 @@ +package com.hnac.hzims.bigmodel.database.service; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import cn.hutool.json.JSONUtil; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Lists; +import com.google.protobuf.ServiceException; +import com.hnac.hzims.bigmodel.configuration.BigModelInvokeApi; +import com.hnac.hzinfo.exception.HzServiceException; +import io.weaviate.client.base.Result; +import io.weaviate.client.v1.data.api.ObjectCreator; +import io.weaviate.client.v1.data.api.ObjectDeleter; +import io.weaviate.client.v1.data.api.ObjectUpdater; +import io.weaviate.client.v1.data.model.WeaviateObject; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springblade.core.tool.utils.BeanUtil; +import org.springblade.core.tool.utils.Func; +import org.springblade.core.tool.utils.StringUtil; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; + +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * @Author: huangxing + * @Date: 2024/08/22 19:17 + */ +@RequiredArgsConstructor +@Service +@Slf4j +public class WeaviateService { + + private final ObjectCreator objectCreator; + private final ObjectUpdater objectUpdater; + private final ObjectDeleter objectDeleter; + private final BigModelInvokeApi invokeApi; + @Value("${gglm.vectorUrl}") + private final String vectorUrl; + + /** + * 对象保存向量数据库 + * @param entity 保存对象 + * @param className 保存表名 + * @param attrs 待计算的列信息 + * @return 保存操作结果 + */ + public Boolean save(Object entity, String className, List attrs) { + ObjectCreator creator = objectCreator.withClassName(className); + if(Func.isNotEmpty(attrs)) { + JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity)); + List vectors = attrs.stream().map(attr -> jsonObject.getString(attr)).collect(Collectors.toList()); + Float[] compute = this.compute(vectors); + creator.withVector(compute); + } + Result result = creator.withProperties(BeanUtil.toMap(entity)).run(); + return !result.hasErrors(); + } + + /** + * 对象批量保存向量数据库 + * @param entities 保存对象列表 + * @param className 保存表名 + * @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔 + * @return 保存操作结果 + */ + public Boolean saveBatch(List entities,String className, Map attrsMap) { + ObjectCreator creator = objectCreator.withClassName(className); + List vectorStrs = Lists.newArrayList(); + List attrs = Lists.newArrayList(); + if(Func.isNotEmpty(attrsMap)) { + // 格式化数据 + attrsMap.forEach((k,v) -> attrs.add(v)); + // 解析待计算的向量字段 + entities.forEach(entity -> { + List vectorStr = attrs.stream().map(fields -> this.getFieldValue(fields, entity)).filter(Func::isNotEmpty).collect(Collectors.toList()); + vectorStrs.addAll(vectorStr); + }); + } + if(Func.isNotEmpty(vectorStrs)) { + // 若解析出来的向量存在值 + Float[] vectors = this.compute(vectorStrs); + + } else { + entities.forEach(entity -> creator.withProperties(BeanUtil.toMap(entity)).run()); + return true; + } + return false; + } + + /** + * 拆解计算出来的向量Float[] + * @param entitySize 对象列表size + * @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔 + * @param vectorTotal 计算出的向量总量 + * @return 拆解结果 + */ + private List> splitVector(Integer entitySize,Map attrsMap,Float[] vectorTotal) { + List> result = Lists.newArrayList(); + + // 获取待切割的下标 + List indexes = this.getSplitIndex(vectorTotal.length, entitySize); + indexes.forEach(index -> { + + }); + return null; + } + + /** + * 获取将list等量分隔成若干份的列表下标 + * @param size 总数 + * @param splitNum 分隔数量 + * @return 下标集合 + */ + private List getSplitIndex(int size,int splitNum) { + if(size % splitNum != 0) { + throw new HzServiceException("向量计算失败，无法根据同步对象进行等量分隔！"); + } + return IntStream.iterate(0, index -> index + 1) + .limit(splitNum) + .mapToObj(index -> index * (size / splitNum)) + .collect(Collectors.toList()); + } + + + private String getFieldValue(String fields,Object object) { + Class clazz = object.getClass(); + return Func.toStrList(",", fields).stream().map(field -> { + try { + Field declaredField = clazz.getDeclaredField(field); + declaredField.setAccessible(true); + return declaredField.get(object).toString(); + } catch (NoSuchFieldException | IllegalAccessException e) { + return null; + } + }).collect(Collectors.joining(" ")); + } + + /** + * 计算向量值 + * @param vectors 待计算的向量 + * @return 向量值Float[] + */ + private Float[] compute(List vectors) { + // 向量计算 + String url = vectorUrl + invokeApi.getCompute(); + String jsonData = JSONUtil.toJsonStr(vectors); + HttpResponse response = HttpRequest.post(url) + .header("Content-Type", "application/json; charset=utf-8") + .body(jsonData) + .execute(); + byte[] bytes = response.bodyBytes(); + if (bytes.length % 4 != 0) { + throw new HzServiceException("向量计算失败！响应数据长度不是4的倍数"); + } + List chunks = new ArrayList<>(); + int range = bytes.length / 4; + IntStream.range(0, range) + .forEach(index -> { + byte[] chunk = new byte[4]; + int page = index * 4; + chunk[0] = bytes[page]; + chunk[1] = bytes[page + 1]; + chunk[2] = bytes[page + 2]; + chunk[3] = bytes[page + 3]; + chunks.add(chunk); + }); + List floats = chunks.stream().map(b -> { + ByteBuffer buffer = ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN); + return buffer.getFloat(); + }).collect(Collectors.toList()); + return floats.toArray(new Float[floats.size()]); + } +} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/util/WeaviateUtil.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/util/WeaviateUtil.java new file mode 100644 index 0000000..1c056ce --- /dev/null +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/util/WeaviateUtil.java @@ -0,0 +1,11 @@ +package com.hnac.hzims.bigmodel.database.util; + +/** + * @Author: huangxing + * @Date: 2024/08/23 10:09 + */ +public class WeaviateUtil { + + + +} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/config/ZhipuAiConfig.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/config/ZhipuAiConfig.java index b28a3a5..1c5a8b2 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/config/ZhipuAiConfig.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/config/ZhipuAiConfig.java @@ -1,22 +1,11 @@ package com.hnac.hzims.bigmodel.zhipuai.config; -import com.alibaba.fastjson.JSON; import com.zhipu.oapi.ClientV4; -import io.weaviate.client.Config; -import io.weaviate.client.WeaviateAuthClient; -import io.weaviate.client.WeaviateClient; -import io.weaviate.client.base.Result; -import io.weaviate.client.v1.auth.exception.AuthException; -import io.weaviate.client.v1.data.model.WeaviateObject; -import io.weaviate.client.v1.misc.model.Meta; import lombok.AllArgsConstructor; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.util.HashMap; -import java.util.List; - /** * @Author: ypj * @Date: 2024/8/9 17:01 @@ -32,28 +21,4 @@ public class ZhipuAiConfig { public ClientV4 clientV4() { return new ClientV4.Builder(properties.getApiSecret()).build(); } - -// public static void main(String[] args) throws AuthException { -// Config config = new Config("http", "192.168.60.16:9992"); -// WeaviateClient client = WeaviateAuthClient.apiKey(config,"123"); -// // 新增 HZIMS_TEST -//// Result craeteTableTest = client.data().creator().withClassName("HZIMS_TEST").run(); -//// System.out.println(JSON.toJSONString(craeteTableTest)); -//// Result> searchResult = client.data().objectsGetter().withClassName("HZIMS_TEST").run(); -//// System.out.println(JSON.toJSONString(searchResult)); -// // 新增数据 -//// HashMap hashMap = new HashMap() {{ -//// put("id_", "13412423423412"); -//// put("name", "test"); -//// }}; -//// Result hzimsTest = client.data().creator().withClassName("HZIMS_TEST").withProperties(hashMap).withVector(new Float[]{}).run(); -//// System.out.println(JSON.toJSONString(hzimsTest)); -//// Result> searchResult = client.data().objectsGetter().withClassName("HZIMS_TEST").run(); -//// System.out.println(JSON.toJSONString(searchResult)); -// // 删除数据 -// Result hzimsTest = client.data().deleter().withClassName("HZIMS_TEST").withID("14c6847d-b758-475f-968d-64b0e193e43e").run(); -// System.out.println(JSON.toJSONString(hzimsTest)); -// Result> searchResult = client.data().objectsGetter().withClassName("HZIMS_TEST").run(); -// System.out.println(JSON.toJSONString(searchResult)); -// } } diff --git a/hzims-service/hzims-big-model/src/main/resources/template/template.yml b/hzims-service/hzims-big-model/src/main/resources/template/template.yml index 77287f6..ab25e27 100644 --- a/hzims-service/hzims-big-model/src/main/resources/template/template.yml +++ b/hzims-service/hzims-big-model/src/main/resources/template/template.yml @@ -49,6 +49,7 @@ spring: gglm: url: "http://${gglm.host}:${gglm.port}" + vectorUrl: "http://${vector.host}:${vector.port}" api: assistantAsk: "/qa/assistant_ask" assistantStatus: "/qa/assistant_status" @@ -62,6 +63,7 @@ gglm: smartReportGeneratePower: "/custom/smart_report_generate_power" assistantAnalyseAsk: "/qa/assistant_analyse_ask" updateKnowledge: "/kn/update_knowledge" + compute: "compute" swagger: base-packages: com.hnac.hzims.bigmodel