Merge remote-tracking branch 'origin/prod-5.1.3' into prod-5.1.3

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java

@@ -27,8 +27,11 @@ public class WeaviateController {
     }
 
     @GetMapping("/list")
-    public R<List<WeaviateObject>> list(@RequestParam(value = "id",required = false) String id, @RequestParam("className") String className) {
+    public R<List<WeaviateObject>> list(@RequestParam(value = "id",required = false) String id,
-        return R.data(weaviateService.list(id,className));
+                                        @RequestParam("className") String className,
+                                        @RequestParam("current") Integer current,
+                                        @RequestParam("pageSize") Integer pageSize) {
+        return R.data(weaviateService.list(id,className,current,pageSize));
     }
 
     @DeleteMapping("/removeById")

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java

@@ -8,27 +8,34 @@ import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
 import com.hnac.hzims.bigmodel.configuration.BigModelInvokeApi;
 import com.hnac.hzinfo.exception.HzServiceException;
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
 import io.weaviate.client.WeaviateClient;
 import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.auth.exception.AuthException;
 import io.weaviate.client.v1.data.api.ObjectCreator;
 import io.weaviate.client.v1.data.api.ObjectDeleter;
 import io.weaviate.client.v1.data.api.ObjectUpdater;
 import io.weaviate.client.v1.data.api.ObjectsGetter;
 import io.weaviate.client.v1.data.model.WeaviateObject;
+import io.weaviate.client.v1.graphql.model.GraphQLResponse;
+import io.weaviate.client.v1.graphql.query.Get;
+import io.weaviate.client.v1.graphql.query.fields.Field;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springblade.core.tool.api.ResultCode;
 import org.springblade.core.tool.utils.BeanUtil;
 import org.springblade.core.tool.utils.Func;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 /**
  * @Author: huangxing
@@ -92,7 +99,10 @@ public class WeaviateService {
                 // log.info("vector:{}",JSON.toJSONString(vector.get(i)));
                 Map<String, Object> properties = this.objectToMap(entities.get(i));
                 log.info("properties:{}",JSON.toJSONString(properties));
-                creator.withProperties(properties).withVectors(vector.get(i)).run();
+                Result<WeaviateObject> run = creator.withProperties(properties).withVectors(vector.get(i)).run();
+                if(run.hasErrors()) {
+                    log.error("保存失败！，保存结果为：{}",JSON.toJSONString(run));
+                }
             }
         } else {
             entities.forEach(entity -> creator.withProperties(this.objectToMap(entity)).run());
@@ -104,7 +114,13 @@ public class WeaviateService {
     private Map<String,Object> objectToMap(Object object) {
         JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(object));
         Map<String,Object> properties = new HashMap<>();
-        jsonObject.forEach((k,v) -> properties.put(k,v));
+        jsonObject.forEach((k,v) -> {
+            if("id".equalsIgnoreCase(k)) {
+                properties.put("id_",v);
+            } else {
+                properties.put(k,v);
+            }
+        });
         return properties;
     }
 
@@ -113,16 +129,22 @@ public class WeaviateService {
      * @param className 表名
      * @return 删除结果
      */
-    public Boolean delete(String id,String className) {
+    public Boolean delete(String ids,String className) {
         ObjectDeleter deleter = weaviateClient.data().deleter();
-        if(Func.isNotEmpty(id)) {
-            deleter.withID(id);
-        }
-        if(Func.isNotEmpty(className)) {
         deleter.withClassName(className);
+        if(Func.isEmpty(ids)) {
+            Result<List<WeaviateObject>> allObject = weaviateClient.data().objectsGetter().withClassName(className).withLimit(10000).run();
+            if(!allObject.hasErrors()) {
+                ids = allObject.getResult().stream().map(WeaviateObject::getId).collect(Collectors.joining(","));
             }
-        Result<Boolean> result = deleter.run();
+        }
-        return !result.hasErrors();
+        Func.toStrList(",",ids).forEach(id -> {
+            Result<Boolean> result = deleter.withID(id).run();
+            if(result.hasErrors()) {
+                throw new HzServiceException(ResultCode.FAILURE,id + "记录删除失败！");
+            }
+        });
+        return true;
     }
 
     /**
@@ -148,7 +170,7 @@ public class WeaviateService {
         return !result.hasErrors();
     }
 
-    public List<WeaviateObject> list(String id,String className) {
+    public List<WeaviateObject> list(String id,String className,Integer current,Integer pageSize) {
         ObjectsGetter objectsGetter = weaviateClient.data().objectsGetter();
         if(Func.isNotEmpty(id)) {
             objectsGetter.withID(id);
@@ -156,7 +178,7 @@ public class WeaviateService {
         if(Func.isNotEmpty(className)) {
             objectsGetter.withClassName(className);
         }
-        Result<List<WeaviateObject>> result = objectsGetter.run();
+        Result<List<WeaviateObject>> result = objectsGetter.withLimit(pageSize).withOffset((current-1) * pageSize).run();
         if(result.hasErrors()) {
             throw new HzServiceException("查询失败！");
         }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java

@@ -1,6 +1,7 @@
 package com.hnac.hzims.bigmodel.maintenance.controller;
 
 import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport;
 import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity;
 import com.hnac.hzims.bigmodel.maintenance.service.VectorParamService;
@@ -36,7 +37,7 @@ public class VectorParamController {
     @ApiOperation(value = "分页查询")
     @ApiOperationSupport(order = 1)
     public R<IPage<VectorParamEntity>> listPage(Query query, VectorParamEntity req) {
-        return R.data(vectorParamService.page(Condition.getPage(query), Condition.getQueryWrapper(req).lambda()));
+        return R.data(vectorParamService.page(Condition.getPage(query), Condition.getQueryWrapper(new VectorParamEntity(), req)));
     }
 
     @GetMapping("/detail")
@@ -79,7 +80,7 @@ public class VectorParamController {
     @ApiOperationSupport(order = 7)
     public R getUrlResponse(@RequestParam String url,
                             @RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken,
-                            @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) {
+                            @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException {
         return R.data(vectorParamService.getUrlResponse(url, bladeToken, hzinfoToken));
     }
 
@@ -92,7 +93,9 @@ public class VectorParamController {
         data.put("name", "测试数据");
         data.put("url", "https://img.zcool.cn/community/01d9d85a8f9f2fa801206e0d0e0e9a.jpg@1280w_1l_2o_100sh.jpg");
         data.put("data", "data");
-        return R.data(data);
+        List<Map> list = new java.util.ArrayList<>();
+        list.add(data);
+        return R.data(list);
     }
 
     @GetMapping("/getUrlResponseKeyList")
@@ -100,7 +103,15 @@ public class VectorParamController {
     @ApiOperationSupport(order = 9)
     public R<List<String>> getUrlResponseKeyList(@RequestParam String url,
                                                  @RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken,
-                                                 @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) {
+                                                 @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException {
         return R.data(vectorParamService.getUrlResponseKeyList(url, bladeToken, hzinfoToken));
     }
+
+    @PutMapping("/synchronization")
+    @ApiOperation(value = "同步")
+    @ApiOperationSupport(order = 10)
+    public R synchronization(@RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken,
+                             @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException {
+        return R.status(vectorParamService.synchronization(bladeToken, hzinfoToken));
+    }
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java

@@ -28,17 +28,26 @@ public class VectorParamEntity extends TenantEntity implements Serializable {
 
     @ApiModelProperty("url")
     @TableField("URL")
+    @QueryField(condition = SqlCondition.LIKE)
     private String url;
 
     @ApiModelProperty("项目前缀")
     @TableField("PROJECT_PREFIX")
+    @QueryField(condition = SqlCondition.LIKE)
     private String projectPrefix;
 
     @ApiModelProperty("数据库表名")
     @TableField("TABLE_NAME")
+    @QueryField(condition = SqlCondition.LIKE)
     private String tableName;
 
     @ApiModelProperty("向量键值对，json格式")
     @TableField("ATTRIBUTE_MAP")
+    @QueryField(condition = SqlCondition.LIKE)
     private String attributeMap;
+
+    @ApiModelProperty("备注")
+    @TableField("REMARK")
+    @QueryField(condition = SqlCondition.LIKE)
+    private String remark;
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java

@@ -2,9 +2,9 @@ package com.hnac.hzims.bigmodel.maintenance.service;
 
 import com.baomidou.mybatisplus.extension.service.IService;
 import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity;
+import com.hnac.hzims.bigmodel.maintenance.vo.VectorUrlResponse;
 
 import java.util.List;
-import java.util.Map;
 
 /**
  * @Author: ypj
@@ -14,7 +14,9 @@ import java.util.Map;
 public interface VectorParamService extends IService<VectorParamEntity> {
     String getAuthorization();
 
-    Map getUrlResponse(String url, String bladeToken ,String hzinfoToken);
+    VectorUrlResponse getUrlResponse(String url, String bladeToken, String hzinfoToken);
 
     List<String> getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken);
+
+    Boolean synchronization(String bladeToken, String hzinfoToken);
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java

@@ -2,20 +2,26 @@ package com.hnac.hzims.bigmodel.maintenance.service.impl;
 
 import cn.hutool.http.HttpRequest;
 import cn.hutool.http.HttpResponse;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.hnac.hzims.bigmodel.database.service.WeaviateService;
 import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity;
 import com.hnac.hzims.bigmodel.maintenance.mapper.VectorParamMapper;
 import com.hnac.hzims.bigmodel.maintenance.service.VectorParamService;
+import com.hnac.hzims.bigmodel.maintenance.vo.VectorUrlResponse;
+import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springblade.core.tool.jackson.JsonUtil;
+import org.springblade.core.tool.utils.CollectionUtil;
 import org.springframework.stereotype.Service;
 
 import javax.servlet.http.HttpServletResponse;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Base64;
-import java.util.List;
-import java.util.Map;
 
 /**
  * @Author: ypj
@@ -23,37 +29,126 @@ import java.util.Map;
  */
 @Service
 @Slf4j
+@AllArgsConstructor
 public class VectorParamServiceImpl extends ServiceImpl<VectorParamMapper, VectorParamEntity> implements VectorParamService {
 
     private static final String CLIENT_SIGN = "vector_param";
 
+    private final WeaviateService weaviateService;
+
     @Override
     public String getAuthorization() {
         return "Basic " + Base64.getEncoder().encodeToString(CLIENT_SIGN.getBytes(StandardCharsets.UTF_8));
     }
 
     @Override
-    public Map getUrlResponse(String url, String bladeToken, String hzinfoToken) {
+    public VectorUrlResponse getUrlResponse(String url, String bladeToken, String hzinfoToken) {
         HttpResponse response = HttpRequest.get(url)
                 .header("Authorization", getAuthorization())
                 .header("Blade-Auth", bladeToken)
                 .header("Hzinfo-Auth", hzinfoToken).execute();
         if (response.getStatus() == HttpServletResponse.SC_OK) {
-            return JsonUtil.toMap(response.body());
+            String body = response.body();
+            log.info("getUrlResponse:" + body);
+            JSONObject jsonObject = JSON.parseObject(body);
+            VectorUrlResponse result = new VectorUrlResponse();
+            result.setOriginalData(JSONArray.parseArray(jsonObject.getString("data")).toJavaList(JSONObject.class));
+//            try {
+//                ObjectMapper mapper = new ObjectMapper();
+//                JsonNode rootNode = mapper.readTree(body);
+//                if (rootNode.isArray()) {
+//                    return getUrlResponseByArray(rootNode);
+//                }
+//                JsonNode dataNode = rootNode.findValue("data");
+//                if (dataNode != null && dataNode.isArray()) {
+//                    return getUrlResponseByArray(dataNode);
+//                }
+//                if (dataNode != null) {
+//                    return getUrlResponseByObject(dataNode);
+//                }
+//                if (rootNode != null) {
+//                    return getUrlResponseByObject(rootNode);
+//                }
+//            } catch (JsonProcessingException e) {
+//                log.error("getUrlResponse error", e);
+//            }
         }
         return null;
     }
 
+//    private VectorUrlResponse getUrlResponseByArray(JsonNode rootNode) throws JsonProcessingException {
+//        VectorUrlResponse result = new VectorUrlResponse();
+//        result.setOriginalData(rootNode);
+//        JsonNode item = rootNode.get(0);
+//        if (item != null) {
+//            List<String> keyList = new ArrayList<>();
+//            Iterator<String> iterator = item.fieldNames();
+//            while (iterator.hasNext()) {
+//                keyList.add(iterator.next());
+//            }
+//            result.setKeyList(keyList);
+//        }
+//        return result;
+//    }
+
+//    private VectorUrlResponse getUrlResponseByObject(JsonNode rootNode) throws JsonProcessingException {
+//        VectorUrlResponse result = new VectorUrlResponse();
+//        result.setOriginalData(rootNode);
+//        List<String> keyList = new ArrayList<>();
+//        Iterator<String> iterator = rootNode.fieldNames();
+//        while (iterator.hasNext()) {
+//            keyList.add(iterator.next());
+//        }
+//        result.setKeyList(keyList);
+//        return result;
+//    }
+
     @Override
     public List<String> getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken) {
-        List<String> result = new ArrayList<>();
+        VectorUrlResponse response = getUrlResponse(url, bladeToken, hzinfoToken);
-        Map map = getUrlResponse(url, bladeToken, hzinfoToken);
+        return response.getKeyList();
-        Map<String, Object> data = ((Map<String, Object>) map.get("data"));
+    }
-        if (data != null) {
+
-            result.addAll(data.keySet());
+    @Override
+    public Boolean synchronization(String bladeToken, String hzinfoToken) {
+        List<VectorParamEntity> list = this.list();
+        if (CollectionUtil.isEmpty(list)) {
+            return false;
+        }
+        for (VectorParamEntity entity : list) {
+            VectorUrlResponse response = getUrlResponse(entity.getUrl(), bladeToken, hzinfoToken);
+            if (response != null) {
+                ObjectMapper mapper = new ObjectMapper();
+                JsonNode rootNode = null;
+                try {
+                    rootNode = mapper.readTree(entity.getAttributeMap());
+                } catch (JsonProcessingException e) {
+                    e.printStackTrace();
+                }
+                Map<String, String> attrMap = new HashMap<>();
+                Iterator<String> iterator = rootNode.fieldNames();
+                while (iterator.hasNext()) {
+                    String key = iterator.next();
+                    attrMap.put(key, rootNode.findValue(key).textValue());
+                }
+                List<Object> dataList = new ArrayList();
+                JsonNode dataObject = (JsonNode) response.getOriginalData();
+                if (dataObject.isArray()) {
+                    Iterator<JsonNode> dataIterator = dataObject.iterator();
+                    while (dataIterator.hasNext()) {
+                        dataList.add(toJsonObject(dataIterator.next()));
+                    }
                 } else {
-            result.addAll(map.keySet());
+                    dataList.add(toJsonObject(dataObject));
+                }
+                weaviateService.saveBatch(response.getOriginalData(), entity.getTableName(), attrMap);
             }
-        return result;
+        }
+        return true;
+    }
+
+    private JSONObject toJsonObject(JsonNode node) {
+        JSONObject jsonObject = JSON.parseObject(node.toString());
+        return jsonObject;
     }
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java

@@ -0,0 +1,16 @@
+package com.hnac.hzims.bigmodel.maintenance.vo;
+
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @Author: ypj
+ * @Date: 2024/9/5 21:12
+ */
+@Data
+public class VectorUrlResponse {
+    private List<String> keyList;
+
+    private List originalData;
+}

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java

@@ -1,9 +1,7 @@
 package com.hnac.hzims.bigmodel.zhipuai.service;
 
-import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.service.IService;
 import com.hnac.hzims.bigmodel.zhipuai.entity.ZhipuAnalysisInfoEntity;
-import org.springblade.core.mp.support.Query;
 
 /**
  * @Author: ypj