add:向量设置同步

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java

@@ -1,6 +1,7 @@
 package com.hnac.hzims.bigmodel.maintenance.controller;
 
 import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport;
 import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity;
 import com.hnac.hzims.bigmodel.maintenance.service.VectorParamService;
@@ -36,7 +37,7 @@ public class VectorParamController {
     @ApiOperation(value = "分页查询")
     @ApiOperationSupport(order = 1)
     public R<IPage<VectorParamEntity>> listPage(Query query, VectorParamEntity req) {
-        return R.data(vectorParamService.page(Condition.getPage(query), Condition.getQueryWrapper(req).lambda()));
+        return R.data(vectorParamService.page(Condition.getPage(query), Condition.getQueryWrapper(new VectorParamEntity(), req)));
     }
 
     @GetMapping("/detail")
@@ -79,7 +80,7 @@ public class VectorParamController {
     @ApiOperationSupport(order = 7)
     public R getUrlResponse(@RequestParam String url,
                             @RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken,
-                            @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) {
+                            @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException {
         return R.data(vectorParamService.getUrlResponse(url, bladeToken, hzinfoToken));
     }
 
@@ -92,7 +93,9 @@ public class VectorParamController {
         data.put("name", "测试数据");
         data.put("url", "https://img.zcool.cn/community/01d9d85a8f9f2fa801206e0d0e0e9a.jpg@1280w_1l_2o_100sh.jpg");
         data.put("data", "data");
-        return R.data(data);
+        List<Map> list = new java.util.ArrayList<>();
+        list.add(data);
+        return R.data(list);
     }
 
     @GetMapping("/getUrlResponseKeyList")
@@ -100,7 +103,15 @@ public class VectorParamController {
     @ApiOperationSupport(order = 9)
     public R<List<String>> getUrlResponseKeyList(@RequestParam String url,
                                                  @RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken,
-                                                 @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) {
+                                                 @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException {
         return R.data(vectorParamService.getUrlResponseKeyList(url, bladeToken, hzinfoToken));
     }
+
+    @PutMapping("/synchronization")
+    @ApiOperation(value = "同步")
+    @ApiOperationSupport(order = 10)
+    public R synchronization(@RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken,
+                             @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException {
+        return R.status(vectorParamService.synchronization(bladeToken, hzinfoToken));
+    }
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java

@@ -28,17 +28,26 @@ public class VectorParamEntity extends TenantEntity implements Serializable {
 
     @ApiModelProperty("url")
     @TableField("URL")
+    @QueryField(condition = SqlCondition.LIKE)
     private String url;
 
     @ApiModelProperty("项目前缀")
     @TableField("PROJECT_PREFIX")
+    @QueryField(condition = SqlCondition.LIKE)
     private String projectPrefix;
 
     @ApiModelProperty("数据库表名")
     @TableField("TABLE_NAME")
+    @QueryField(condition = SqlCondition.LIKE)
     private String tableName;
 
     @ApiModelProperty("向量键值对，json格式")
     @TableField("ATTRIBUTE_MAP")
+    @QueryField(condition = SqlCondition.LIKE)
     private String attributeMap;
+
+    @ApiModelProperty("备注")
+    @TableField("REMARK")
+    @QueryField(condition = SqlCondition.LIKE)
+    private String remark;
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java

@@ -2,9 +2,9 @@ package com.hnac.hzims.bigmodel.maintenance.service;
 
 import com.baomidou.mybatisplus.extension.service.IService;
 import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity;
+import com.hnac.hzims.bigmodel.maintenance.vo.VectorUrlResponse;
 
 import java.util.List;
-import java.util.Map;
 
 /**
  * @Author: ypj
@@ -14,7 +14,9 @@ import java.util.Map;
 public interface VectorParamService extends IService<VectorParamEntity> {
     String getAuthorization();
 
-    Map getUrlResponse(String url, String bladeToken ,String hzinfoToken);
+    VectorUrlResponse getUrlResponse(String url, String bladeToken, String hzinfoToken);
 
     List<String> getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken);
+
+    Boolean synchronization(String bladeToken, String hzinfoToken);
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java

@@ -2,20 +2,26 @@ package com.hnac.hzims.bigmodel.maintenance.service.impl;
 
 import cn.hutool.http.HttpRequest;
 import cn.hutool.http.HttpResponse;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.hnac.hzims.bigmodel.database.service.WeaviateService;
 import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity;
 import com.hnac.hzims.bigmodel.maintenance.mapper.VectorParamMapper;
 import com.hnac.hzims.bigmodel.maintenance.service.VectorParamService;
+import com.hnac.hzims.bigmodel.maintenance.vo.VectorUrlResponse;
+import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springblade.core.tool.jackson.JsonUtil;
+import org.springblade.core.tool.utils.CollectionUtil;
 import org.springframework.stereotype.Service;
 
 import javax.servlet.http.HttpServletResponse;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Base64;
-import java.util.List;
-import java.util.Map;
 
 /**
  * @Author: ypj
@@ -23,37 +29,126 @@ import java.util.Map;
  */
 @Service
 @Slf4j
+@AllArgsConstructor
 public class VectorParamServiceImpl extends ServiceImpl<VectorParamMapper, VectorParamEntity> implements VectorParamService {
 
     private static final String CLIENT_SIGN = "vector_param";
 
+    private final WeaviateService weaviateService;
+
     @Override
     public String getAuthorization() {
         return "Basic " + Base64.getEncoder().encodeToString(CLIENT_SIGN.getBytes(StandardCharsets.UTF_8));
     }
 
     @Override
-    public Map getUrlResponse(String url, String bladeToken, String hzinfoToken) {
+    public VectorUrlResponse getUrlResponse(String url, String bladeToken, String hzinfoToken) {
         HttpResponse response = HttpRequest.get(url)
                 .header("Authorization", getAuthorization())
                 .header("Blade-Auth", bladeToken)
                 .header("Hzinfo-Auth", hzinfoToken).execute();
         if (response.getStatus() == HttpServletResponse.SC_OK) {
-            return JsonUtil.toMap(response.body());
+            String body = response.body();
+            log.info("getUrlResponse:" + body);
+            JSONObject jsonObject = JSON.parseObject(body);
+            VectorUrlResponse result = new VectorUrlResponse();
+            result.setOriginalData(JSONArray.parseArray(jsonObject.getString("data")).toJavaList(JSONObject.class));
+//            try {
+//                ObjectMapper mapper = new ObjectMapper();
+//                JsonNode rootNode = mapper.readTree(body);
+//                if (rootNode.isArray()) {
+//                    return getUrlResponseByArray(rootNode);
+//                }
+//                JsonNode dataNode = rootNode.findValue("data");
+//                if (dataNode != null && dataNode.isArray()) {
+//                    return getUrlResponseByArray(dataNode);
+//                }
+//                if (dataNode != null) {
+//                    return getUrlResponseByObject(dataNode);
+//                }
+//                if (rootNode != null) {
+//                    return getUrlResponseByObject(rootNode);
+//                }
+//            } catch (JsonProcessingException e) {
+//                log.error("getUrlResponse error", e);
+//            }
         }
         return null;
     }
 
+//    private VectorUrlResponse getUrlResponseByArray(JsonNode rootNode) throws JsonProcessingException {
+//        VectorUrlResponse result = new VectorUrlResponse();
+//        result.setOriginalData(rootNode);
+//        JsonNode item = rootNode.get(0);
+//        if (item != null) {
+//            List<String> keyList = new ArrayList<>();
+//            Iterator<String> iterator = item.fieldNames();
+//            while (iterator.hasNext()) {
+//                keyList.add(iterator.next());
+//            }
+//            result.setKeyList(keyList);
+//        }
+//        return result;
+//    }
+
+//    private VectorUrlResponse getUrlResponseByObject(JsonNode rootNode) throws JsonProcessingException {
+//        VectorUrlResponse result = new VectorUrlResponse();
+//        result.setOriginalData(rootNode);
+//        List<String> keyList = new ArrayList<>();
+//        Iterator<String> iterator = rootNode.fieldNames();
+//        while (iterator.hasNext()) {
+//            keyList.add(iterator.next());
+//        }
+//        result.setKeyList(keyList);
+//        return result;
+//    }
+
     @Override
     public List<String> getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken) {
-        List<String> result = new ArrayList<>();
+        VectorUrlResponse response = getUrlResponse(url, bladeToken, hzinfoToken);
-        Map map = getUrlResponse(url, bladeToken, hzinfoToken);
+        return response.getKeyList();
-        Map<String, Object> data = ((Map<String, Object>) map.get("data"));
+    }
-        if (data != null) {
+
-            result.addAll(data.keySet());
+    @Override
-        } else {
+    public Boolean synchronization(String bladeToken, String hzinfoToken) {
-            result.addAll(map.keySet());
+        List<VectorParamEntity> list = this.list();
+        if (CollectionUtil.isEmpty(list)) {
+            return false;
+        }
+        for (VectorParamEntity entity : list) {
+            VectorUrlResponse response = getUrlResponse(entity.getUrl(), bladeToken, hzinfoToken);
+            if (response != null) {
+                ObjectMapper mapper = new ObjectMapper();
+                JsonNode rootNode = null;
+                try {
+                    rootNode = mapper.readTree(entity.getAttributeMap());
+                } catch (JsonProcessingException e) {
+                    e.printStackTrace();
+                }
+                Map<String, String> attrMap = new HashMap<>();
+                Iterator<String> iterator = rootNode.fieldNames();
+                while (iterator.hasNext()) {
+                    String key = iterator.next();
+                    attrMap.put(key, rootNode.findValue(key).textValue());
+                }
+                List<Object> dataList = new ArrayList();
+                JsonNode dataObject = (JsonNode) response.getOriginalData();
+                if (dataObject.isArray()) {
+                    Iterator<JsonNode> dataIterator = dataObject.iterator();
+                    while (dataIterator.hasNext()) {
+                        dataList.add(toJsonObject(dataIterator.next()));
+                    }
+                } else {
+                    dataList.add(toJsonObject(dataObject));
+                }
+                weaviateService.saveBatch(response.getOriginalData(), entity.getTableName(), attrMap);
+            }
         }
-        return result;
+        return true;
+    }
+
+    private JSONObject toJsonObject(JsonNode node) {
+        JSONObject jsonObject = JSON.parseObject(node.toString());
+        return jsonObject;
     }
 }

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java

@@ -0,0 +1,16 @@
+package com.hnac.hzims.bigmodel.maintenance.vo;
+
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @Author: ypj
+ * @Date: 2024/9/5 21:12
+ */
+@Data
+public class VectorUrlResponse {
+    private List<String> keyList;
+
+    private List originalData;
+}

hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java

@@ -1,9 +1,7 @@
 package com.hnac.hzims.bigmodel.zhipuai.service;
 
-import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.service.IService;
 import com.hnac.hzims.bigmodel.zhipuai.entity.ZhipuAnalysisInfoEntity;
-import org.springblade.core.mp.support.Query;
 
 /**
  * @Author: ypj