From 19db5e59f22cbd4ffa7ec52c1ab1e3bd96d5c348 Mon Sep 17 00:00:00 2001 From: luyie Date: Fri, 6 Sep 2024 11:35:53 +0800 Subject: [PATCH] =?UTF-8?q?add:=E5=90=91=E9=87=8F=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/VectorParamController.java | 19 +++- .../maintenance/entity/VectorParamEntity.java | 9 ++ .../maintenance/service/VectorParamService.java | 6 +- .../service/impl/VectorParamServiceImpl.java | 125 ++++++++++++++++++--- .../bigmodel/maintenance/vo/VectorUrlResponse.java | 16 +++ .../zhipuai/service/ZhipuAnalysisInfoService.java | 2 - 6 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java index 13b0475..ec681f1 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java @@ -1,6 +1,7 @@ package com.hnac.hzims.bigmodel.maintenance.controller; import com.baomidou.mybatisplus.core.metadata.IPage; +import com.fasterxml.jackson.core.JsonProcessingException; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity; import com.hnac.hzims.bigmodel.maintenance.service.VectorParamService; @@ -36,7 +37,7 @@ public class VectorParamController { @ApiOperation(value = "分页查询") @ApiOperationSupport(order = 1) public R> listPage(Query query, VectorParamEntity req) { - return R.data(vectorParamService.page(Condition.getPage(query), Condition.getQueryWrapper(req).lambda())); + return R.data(vectorParamService.page(Condition.getPage(query), Condition.getQueryWrapper(new VectorParamEntity(), req))); } @GetMapping("/detail") @@ -79,7 +80,7 @@ public class VectorParamController { @ApiOperationSupport(order = 7) public R getUrlResponse(@RequestParam String url, @RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken, - @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) { + @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException { return R.data(vectorParamService.getUrlResponse(url, bladeToken, hzinfoToken)); } @@ -92,7 +93,9 @@ public class VectorParamController { data.put("name", "测试数据"); data.put("url", "https://img.zcool.cn/community/01d9d85a8f9f2fa801206e0d0e0e9a.jpg@1280w_1l_2o_100sh.jpg"); data.put("data", "data"); - return R.data(data); + List list = new java.util.ArrayList<>(); + list.add(data); + return R.data(list); } @GetMapping("/getUrlResponseKeyList") @@ -100,7 +103,15 @@ public class VectorParamController { @ApiOperationSupport(order = 9) public R> getUrlResponseKeyList(@RequestParam String url, @RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken, - @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) { + @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException { return R.data(vectorParamService.getUrlResponseKeyList(url, bladeToken, hzinfoToken)); } + + @PutMapping("/synchronization") + @ApiOperation(value = "同步") + @ApiOperationSupport(order = 10) + public R synchronization(@RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken, + @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException { + return R.status(vectorParamService.synchronization(bladeToken, hzinfoToken)); + } } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java index f38826f..6553578 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java @@ -28,17 +28,26 @@ public class VectorParamEntity extends TenantEntity implements Serializable { @ApiModelProperty("url") @TableField("URL") + @QueryField(condition = SqlCondition.LIKE) private String url; @ApiModelProperty("项目前缀") @TableField("PROJECT_PREFIX") + @QueryField(condition = SqlCondition.LIKE) private String projectPrefix; @ApiModelProperty("数据库表名") @TableField("TABLE_NAME") + @QueryField(condition = SqlCondition.LIKE) private String tableName; @ApiModelProperty("向量键值对，json格式") @TableField("ATTRIBUTE_MAP") + @QueryField(condition = SqlCondition.LIKE) private String attributeMap; + + @ApiModelProperty("备注") + @TableField("REMARK") + @QueryField(condition = SqlCondition.LIKE) + private String remark; } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java index 4fac0be..20cf06f 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java @@ -2,9 +2,9 @@ package com.hnac.hzims.bigmodel.maintenance.service; import com.baomidou.mybatisplus.extension.service.IService; import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity; +import com.hnac.hzims.bigmodel.maintenance.vo.VectorUrlResponse; import java.util.List; -import java.util.Map; /** * @Author: ypj @@ -14,7 +14,9 @@ import java.util.Map; public interface VectorParamService extends IService { String getAuthorization(); - Map getUrlResponse(String url, String bladeToken ,String hzinfoToken); + VectorUrlResponse getUrlResponse(String url, String bladeToken, String hzinfoToken); List getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken); + + Boolean synchronization(String bladeToken, String hzinfoToken); } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java index 0878002..a350a57 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java @@ -2,20 +2,26 @@ package com.hnac.hzims.bigmodel.maintenance.service.impl; import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.hnac.hzims.bigmodel.database.service.WeaviateService; import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity; import com.hnac.hzims.bigmodel.maintenance.mapper.VectorParamMapper; import com.hnac.hzims.bigmodel.maintenance.service.VectorParamService; +import com.hnac.hzims.bigmodel.maintenance.vo.VectorUrlResponse; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springblade.core.tool.jackson.JsonUtil; +import org.springblade.core.tool.utils.CollectionUtil; import org.springframework.stereotype.Service; import javax.servlet.http.HttpServletResponse; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; +import java.util.*; /** * @Author: ypj @@ -23,37 +29,126 @@ import java.util.Map; */ @Service @Slf4j +@AllArgsConstructor public class VectorParamServiceImpl extends ServiceImpl implements VectorParamService { private static final String CLIENT_SIGN = "vector_param"; + private final WeaviateService weaviateService; + @Override public String getAuthorization() { return "Basic " + Base64.getEncoder().encodeToString(CLIENT_SIGN.getBytes(StandardCharsets.UTF_8)); } @Override - public Map getUrlResponse(String url, String bladeToken, String hzinfoToken) { + public VectorUrlResponse getUrlResponse(String url, String bladeToken, String hzinfoToken) { HttpResponse response = HttpRequest.get(url) .header("Authorization", getAuthorization()) .header("Blade-Auth", bladeToken) .header("Hzinfo-Auth", hzinfoToken).execute(); if (response.getStatus() == HttpServletResponse.SC_OK) { - return JsonUtil.toMap(response.body()); + String body = response.body(); + log.info("getUrlResponse:" + body); + JSONObject jsonObject = JSON.parseObject(body); + VectorUrlResponse result = new VectorUrlResponse(); + result.setOriginalData(JSONArray.parseArray(jsonObject.getString("data")).toJavaList(JSONObject.class)); +// try { +// ObjectMapper mapper = new ObjectMapper(); +// JsonNode rootNode = mapper.readTree(body); +// if (rootNode.isArray()) { +// return getUrlResponseByArray(rootNode); +// } +// JsonNode dataNode = rootNode.findValue("data"); +// if (dataNode != null && dataNode.isArray()) { +// return getUrlResponseByArray(dataNode); +// } +// if (dataNode != null) { +// return getUrlResponseByObject(dataNode); +// } +// if (rootNode != null) { +// return getUrlResponseByObject(rootNode); +// } +// } catch (JsonProcessingException e) { +// log.error("getUrlResponse error", e); +// } } return null; } +// private VectorUrlResponse getUrlResponseByArray(JsonNode rootNode) throws JsonProcessingException { +// VectorUrlResponse result = new VectorUrlResponse(); +// result.setOriginalData(rootNode); +// JsonNode item = rootNode.get(0); +// if (item != null) { +// List keyList = new ArrayList<>(); +// Iterator iterator = item.fieldNames(); +// while (iterator.hasNext()) { +// keyList.add(iterator.next()); +// } +// result.setKeyList(keyList); +// } +// return result; +// } + +// private VectorUrlResponse getUrlResponseByObject(JsonNode rootNode) throws JsonProcessingException { +// VectorUrlResponse result = new VectorUrlResponse(); +// result.setOriginalData(rootNode); +// List keyList = new ArrayList<>(); +// Iterator iterator = rootNode.fieldNames(); +// while (iterator.hasNext()) { +// keyList.add(iterator.next()); +// } +// result.setKeyList(keyList); +// return result; +// } + @Override public List getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken) { - List result = new ArrayList<>(); - Map map = getUrlResponse(url, bladeToken, hzinfoToken); - Map data = ((Map) map.get("data")); - if (data != null) { - result.addAll(data.keySet()); - } else { - result.addAll(map.keySet()); + VectorUrlResponse response = getUrlResponse(url, bladeToken, hzinfoToken); + return response.getKeyList(); + } + + @Override + public Boolean synchronization(String bladeToken, String hzinfoToken) { + List list = this.list(); + if (CollectionUtil.isEmpty(list)) { + return false; + } + for (VectorParamEntity entity : list) { + VectorUrlResponse response = getUrlResponse(entity.getUrl(), bladeToken, hzinfoToken); + if (response != null) { + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = null; + try { + rootNode = mapper.readTree(entity.getAttributeMap()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + Map attrMap = new HashMap<>(); + Iterator iterator = rootNode.fieldNames(); + while (iterator.hasNext()) { + String key = iterator.next(); + attrMap.put(key, rootNode.findValue(key).textValue()); + } + List dataList = new ArrayList(); + JsonNode dataObject = (JsonNode) response.getOriginalData(); + if (dataObject.isArray()) { + Iterator dataIterator = dataObject.iterator(); + while (dataIterator.hasNext()) { + dataList.add(toJsonObject(dataIterator.next())); + } + } else { + dataList.add(toJsonObject(dataObject)); + } + weaviateService.saveBatch(response.getOriginalData(), entity.getTableName(), attrMap); + } } - return result; + return true; + } + + private JSONObject toJsonObject(JsonNode node) { + JSONObject jsonObject = JSON.parseObject(node.toString()); + return jsonObject; } } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java new file mode 100644 index 0000000..4728c0e --- /dev/null +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java @@ -0,0 +1,16 @@ +package com.hnac.hzims.bigmodel.maintenance.vo; + +import lombok.Data; + +import java.util.List; + +/** + * @Author: ypj + * @Date: 2024/9/5 21:12 + */ +@Data +public class VectorUrlResponse { + private List keyList; + + private List originalData; +} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java index a7cb75b..823b870 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java @@ -1,9 +1,7 @@ package com.hnac.hzims.bigmodel.zhipuai.service; -import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.service.IService; import com.hnac.hzims.bigmodel.zhipuai.entity.ZhipuAnalysisInfoEntity; -import org.springblade.core.mp.support.Query; /** * @Author: ypj