fix: 临时提交

hzims-service-api/big-model-api/src/main/java/com/hnac/hzims/bigmodel/database/entity/WeaviateEntity.java

@@ -0,0 +1,23 @@
+package com.hnac.hzims.bigmodel.database.entity;
+
+import com.alibaba.fastjson.annotation.JSONField;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.io.Serializable;
+
+/**
+ * @Author: huangxing
+ * @Date: 2024/08/22 19:35
+ */
+@Data
+@EqualsAndHashCode
+public class WeaviateEntity implements Serializable {
+
+    @JSONField(name = "item_id")
+    private String itemId;
+
+    @JSONField(name = "item_name")
+    private String itemName;
+
+}

hzims-service-api/inspect-api/src/main/java/com/hnac/hzinfo/inspect/obj/entity/ObjectEntity.java

@@ -49,7 +49,7 @@ public class ObjectEntity extends TenantEntity {
 	 * 地理位置
 	 */
 	@ApiModelProperty(value = "地理位置,长度必须为1到128")
-	@Size(min=1,max = 128,message ="地理位置长度必须为1到128")
+	// @Size(min=1,max = 128,message ="地理位置长度必须为1到128")
 	private String address;
 	/**
 	 * 经度(东经)

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/BigModelInvokeApi.java

@@ -62,4 +62,9 @@ public class BigModelInvokeApi {
      */
     private String updateKnowledge;
 
+    /**
+     * 计算向量
+     */
+    private String compute;
+
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfig.java

@@ -1,20 +0,0 @@
-package com.hnac.hzims.bigmodel.configuration;
-
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.web.reactive.function.client.WebClient;
-import reactor.core.publisher.Mono;
-
-/**
- * @Author: huangxing
- * @Date: 2024/08/21 15:22
- */
-@Configuration
-public class WeaviateConfig {
-
-    @Bean
-    public WebClient weaviateClient() {
-        return WebClient.create("http://192.168.60.16:9992");
-    }
-
-}

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateConfigure.java

@@ -0,0 +1,56 @@
+package com.hnac.hzims.bigmodel.configuration;
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.v1.auth.exception.AuthException;
+import io.weaviate.client.v1.data.api.ObjectCreator;
+import io.weaviate.client.v1.data.api.ObjectDeleter;
+import io.weaviate.client.v1.data.api.ObjectUpdater;
+import io.weaviate.client.v1.data.api.ObjectsGetter;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @Author: huangxing
+ * @Date: 2024/08/22 18:38
+ */
+@Configuration
+public class WeaviateConfigure {
+
+    private final WeaviateProperties weaviateProperties;
+
+    public WeaviateConfigure(WeaviateProperties weaviateProperties) {
+        this.weaviateProperties = weaviateProperties;
+    }
+
+    @Bean
+    public WeaviateClient weaviateClient() throws AuthException {
+        Config config = new Config(this.weaviateProperties.getSchema(), this.weaviateProperties.getHost() + ":" + this.weaviateProperties.getPort());
+        return WeaviateAuthClient.apiKey(config,this.weaviateProperties.getApiKey());
+    }
+
+    @Bean
+    public ObjectsGetter objectsGetter() throws AuthException {
+        WeaviateClient weaviateClient = weaviateClient();
+        return weaviateClient.data().objectsGetter();
+    }
+
+    @Bean
+    public ObjectCreator objectCreator() throws AuthException {
+        WeaviateClient weaviateClient = weaviateClient();
+        return weaviateClient.data().creator();
+    }
+
+    @Bean
+    public ObjectDeleter deleter() throws AuthException {
+        WeaviateClient weaviateClient = weaviateClient();
+        return weaviateClient.data().deleter();
+    }
+
+    @Bean
+    public ObjectUpdater updater() throws AuthException {
+        WeaviateClient weaviateClient = weaviateClient();
+        return weaviateClient.data().updater();
+    }
+}

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/configuration/WeaviateProperties.java

@@ -0,0 +1,32 @@
+package com.hnac.hzims.bigmodel.configuration;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * @Author: huangxing
+ * @Date: 2024/08/21 15:22
+ */
+@Data
+@Component
+@ConfigurationProperties(prefix = "weaviate.datasource")
+public class WeaviateProperties {
+
+    private String schema;
+
+    private String host;
+
+    private String port;
+
+    /**
+     * 登录认证KEY
+     */
+    private String apiKey;
+
+    /**
+     * 数据库表名前缀
+     */
+    private String classNamePrefix;
+
+}

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java

@@ -0,0 +1,183 @@
+package com.hnac.hzims.bigmodel.database.service;
+
+import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpResponse;
+import cn.hutool.json.JSONUtil;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Lists;
+import com.google.protobuf.ServiceException;
+import com.hnac.hzims.bigmodel.configuration.BigModelInvokeApi;
+import com.hnac.hzinfo.exception.HzServiceException;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.data.api.ObjectCreator;
+import io.weaviate.client.v1.data.api.ObjectDeleter;
+import io.weaviate.client.v1.data.api.ObjectUpdater;
+import io.weaviate.client.v1.data.model.WeaviateObject;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springblade.core.tool.utils.BeanUtil;
+import org.springblade.core.tool.utils.Func;
+import org.springblade.core.tool.utils.StringUtil;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.util.Assert;
+
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.*;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * @Author: huangxing
+ * @Date: 2024/08/22 19:17
+ */
+@RequiredArgsConstructor
+@Service
+@Slf4j
+public class WeaviateService {
+
+    private final ObjectCreator objectCreator;
+    private final ObjectUpdater objectUpdater;
+    private final ObjectDeleter objectDeleter;
+    private final BigModelInvokeApi invokeApi;
+    @Value("${gglm.vectorUrl}")
+    private final String vectorUrl;
+
+    /**
+     * 对象保存向量数据库
+     * @param entity 保存对象
+     * @param className 保存表名
+     * @param attrs 待计算的列信息
+     * @return 保存操作结果
+     */
+    public Boolean save(Object entity, String className, List<String> attrs) {
+        ObjectCreator creator = objectCreator.withClassName(className);
+        if(Func.isNotEmpty(attrs)) {
+            JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity));
+            List<String> vectors = attrs.stream().map(attr -> jsonObject.getString(attr)).collect(Collectors.toList());
+            Float[] compute = this.compute(vectors);
+            creator.withVector(compute);
+        }
+        Result<WeaviateObject> result = creator.withProperties(BeanUtil.toMap(entity)).run();
+        return !result.hasErrors();
+    }
+
+    /**
+     * 对象批量保存向量数据库
+     * @param entities 保存对象列表
+     * @param className 保存表名
+     * @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔
+     * @return 保存操作结果
+     */
+    public Boolean saveBatch(List entities,String className, Map<String,String> attrsMap) {
+        ObjectCreator creator = objectCreator.withClassName(className);
+        List<String> vectorStrs = Lists.newArrayList();
+        List<String> attrs = Lists.newArrayList();
+        if(Func.isNotEmpty(attrsMap)) {
+            // 格式化数据
+            attrsMap.forEach((k,v) -> attrs.add(v));
+            // 解析待计算的向量字段
+            entities.forEach(entity -> {
+                List<String> vectorStr = attrs.stream().map(fields -> this.getFieldValue(fields, entity)).filter(Func::isNotEmpty).collect(Collectors.toList());
+                vectorStrs.addAll(vectorStr);
+            });
+        }
+        if(Func.isNotEmpty(vectorStrs)) {
+            // 若解析出来的向量存在值
+            Float[] vectors = this.compute(vectorStrs);
+
+        } else {
+            entities.forEach(entity -> creator.withProperties(BeanUtil.toMap(entity)).run());
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * 拆解计算出来的向量Float[]
+     * @param entitySize 对象列表size
+     * @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔
+     * @param vectorTotal 计算出的向量总量
+     * @return 拆解结果
+     */
+    private List<Map<String,Float[]>> splitVector(Integer entitySize,Map<String,String> attrsMap,Float[] vectorTotal) {
+        List<Map<String,Float[]>> result = Lists.newArrayList();
+        
+        // 获取待切割的下标
+        List<Integer> indexes = this.getSplitIndex(vectorTotal.length, entitySize);
+        indexes.forEach(index -> {
+
+        });
+        return null;
+    }
+
+    /**
+     * 获取将list等量分隔成若干份的列表下标
+     * @param size 总数
+     * @param splitNum 分隔数量
+     * @return 下标集合
+     */
+    private List<Integer> getSplitIndex(int size,int splitNum) {
+        if(size % splitNum != 0) {
+            throw new HzServiceException("向量计算失败，无法根据同步对象进行等量分隔！");
+        }
+        return IntStream.iterate(0, index -> index + 1)
+                .limit(splitNum)
+                .mapToObj(index -> index * (size / splitNum))
+                .collect(Collectors.toList());
+    }
+
+
+    private String getFieldValue(String fields,Object object) {
+        Class clazz = object.getClass();
+        return Func.toStrList(",", fields).stream().map(field -> {
+            try {
+                Field declaredField = clazz.getDeclaredField(field);
+                declaredField.setAccessible(true);
+                return declaredField.get(object).toString();
+            } catch (NoSuchFieldException | IllegalAccessException e) {
+                return null;
+            }
+        }).collect(Collectors.joining(" "));
+    }
+
+    /**
+     * 计算向量值
+     * @param vectors 待计算的向量
+     * @return 向量值Float[]
+     */
+    private Float[] compute(List<String> vectors) {
+        // 向量计算
+        String url = vectorUrl + invokeApi.getCompute();
+        String jsonData = JSONUtil.toJsonStr(vectors);
+        HttpResponse response = HttpRequest.post(url)
+                .header("Content-Type", "application/json; charset=utf-8")
+                .body(jsonData)
+                .execute();
+        byte[] bytes = response.bodyBytes();
+        if (bytes.length % 4 != 0) {
+            throw new HzServiceException("向量计算失败！响应数据长度不是4的倍数");
+        }
+        List<byte[]> chunks = new ArrayList<>();
+        int range = bytes.length / 4;
+        IntStream.range(0, range)
+                .forEach(index -> {
+                    byte[] chunk = new byte[4];
+                    int page = index * 4;
+                    chunk[0] = bytes[page];
+                    chunk[1] = bytes[page + 1];
+                    chunk[2] = bytes[page + 2];
+                    chunk[3] = bytes[page + 3];
+                    chunks.add(chunk);
+                });
+        List<Float> floats = chunks.stream().map(b -> {
+            ByteBuffer buffer = ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN);
+            return buffer.getFloat();
+        }).collect(Collectors.toList());
+        return floats.toArray(new Float[floats.size()]);
+    }
+}

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/util/WeaviateUtil.java

@@ -0,0 +1,11 @@
+package com.hnac.hzims.bigmodel.database.util;
+
+/**
+ * @Author: huangxing
+ * @Date: 2024/08/23 10:09
+ */
+public class WeaviateUtil {
+
+
+
+}

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/config/ZhipuAiConfig.java

@@ -1,22 +1,11 @@
 package com.hnac.hzims.bigmodel.zhipuai.config;
 
-import com.alibaba.fastjson.JSON;
 import com.zhipu.oapi.ClientV4;
-import io.weaviate.client.Config;
-import io.weaviate.client.WeaviateAuthClient;
-import io.weaviate.client.WeaviateClient;
-import io.weaviate.client.base.Result;
-import io.weaviate.client.v1.auth.exception.AuthException;
-import io.weaviate.client.v1.data.model.WeaviateObject;
-import io.weaviate.client.v1.misc.model.Meta;
 import lombok.AllArgsConstructor;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import java.util.HashMap;
-import java.util.List;
-
 /**
  * @Author: ypj
  * @Date: 2024/8/9 17:01
@@ -32,28 +21,4 @@ public class ZhipuAiConfig {
     public ClientV4 clientV4() {
         return new ClientV4.Builder(properties.getApiSecret()).build();
     }
-
-//    public static void main(String[] args) throws AuthException {
-//        Config config = new Config("http", "192.168.60.16:9992");
-//        WeaviateClient client = WeaviateAuthClient.apiKey(config,"123");
-//        // 新增 HZIMS_TEST
-////        Result<WeaviateObject> craeteTableTest = client.data().creator().withClassName("HZIMS_TEST").run();
-////        System.out.println(JSON.toJSONString(craeteTableTest));
-////        Result<List<WeaviateObject>> searchResult = client.data().objectsGetter().withClassName("HZIMS_TEST").run();
-////        System.out.println(JSON.toJSONString(searchResult));
-//        // 新增数据
-////        HashMap<String,Object> hashMap = new HashMap<String, Object>() {{
-////            put("id_", "13412423423412");
-////            put("name", "test");
-////        }};
-////        Result<WeaviateObject> hzimsTest = client.data().creator().withClassName("HZIMS_TEST").withProperties(hashMap).withVector(new Float[]{}).run();
-////        System.out.println(JSON.toJSONString(hzimsTest));
-////        Result<List<WeaviateObject>> searchResult = client.data().objectsGetter().withClassName("HZIMS_TEST").run();
-////        System.out.println(JSON.toJSONString(searchResult));
-//        // 删除数据
-//        Result<Boolean> hzimsTest = client.data().deleter().withClassName("HZIMS_TEST").withID("14c6847d-b758-475f-968d-64b0e193e43e").run();
-//        System.out.println(JSON.toJSONString(hzimsTest));
-//        Result<List<WeaviateObject>> searchResult = client.data().objectsGetter().withClassName("HZIMS_TEST").run();
-//        System.out.println(JSON.toJSONString(searchResult));
-//    }
 }

hzims-service/hzims-big-model/src/main/resources/template/template.yml

@@ -49,6 +49,7 @@ spring:
 
 gglm:
   url: "http://${gglm.host}:${gglm.port}"
+  vectorUrl: "http://${vector.host}:${vector.port}"
   api:
     assistantAsk: "/qa/assistant_ask"
     assistantStatus: "/qa/assistant_status"
@@ -62,6 +63,7 @@ gglm:
     smartReportGeneratePower: "/custom/smart_report_generate_power"
     assistantAnalyseAsk: "/qa/assistant_analyse_ask"
     updateKnowledge: "/kn/update_knowledge"
+    compute: "compute"
 swagger:
   base-packages: com.hnac.hzims.bigmodel