From 19db5e59f22cbd4ffa7ec52c1ab1e3bd96d5c348 Mon Sep 17 00:00:00 2001 From: luyie Date: Fri, 6 Sep 2024 11:35:53 +0800 Subject: [PATCH 1/3] =?UTF-8?q?add:=E5=90=91=E9=87=8F=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/VectorParamController.java | 19 +++- .../maintenance/entity/VectorParamEntity.java | 9 ++ .../maintenance/service/VectorParamService.java | 6 +- .../service/impl/VectorParamServiceImpl.java | 125 ++++++++++++++++++--- .../bigmodel/maintenance/vo/VectorUrlResponse.java | 16 +++ .../zhipuai/service/ZhipuAnalysisInfoService.java | 2 - 6 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java index 13b0475..ec681f1 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/controller/VectorParamController.java @@ -1,6 +1,7 @@ package com.hnac.hzims.bigmodel.maintenance.controller; import com.baomidou.mybatisplus.core.metadata.IPage; +import com.fasterxml.jackson.core.JsonProcessingException; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity; import com.hnac.hzims.bigmodel.maintenance.service.VectorParamService; @@ -36,7 +37,7 @@ public class VectorParamController { @ApiOperation(value = "分页查询") @ApiOperationSupport(order = 1) public R> listPage(Query query, VectorParamEntity req) { - return R.data(vectorParamService.page(Condition.getPage(query), Condition.getQueryWrapper(req).lambda())); + return R.data(vectorParamService.page(Condition.getPage(query), Condition.getQueryWrapper(new VectorParamEntity(), req))); } @GetMapping("/detail") @@ -79,7 +80,7 @@ public class VectorParamController { @ApiOperationSupport(order = 7) public R getUrlResponse(@RequestParam String url, @RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken, - @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) { + @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException { return R.data(vectorParamService.getUrlResponse(url, bladeToken, hzinfoToken)); } @@ -92,7 +93,9 @@ public class VectorParamController { data.put("name", "测试数据"); data.put("url", "https://img.zcool.cn/community/01d9d85a8f9f2fa801206e0d0e0e9a.jpg@1280w_1l_2o_100sh.jpg"); data.put("data", "data"); - return R.data(data); + List list = new java.util.ArrayList<>(); + list.add(data); + return R.data(list); } @GetMapping("/getUrlResponseKeyList") @@ -100,7 +103,15 @@ public class VectorParamController { @ApiOperationSupport(order = 9) public R> getUrlResponseKeyList(@RequestParam String url, @RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken, - @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) { + @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException { return R.data(vectorParamService.getUrlResponseKeyList(url, bladeToken, hzinfoToken)); } + + @PutMapping("/synchronization") + @ApiOperation(value = "同步") + @ApiOperationSupport(order = 10) + public R synchronization(@RequestHeader(name = "Blade-Auth", value = "Blade-Auth", required = false) String bladeToken, + @RequestHeader(name = "Hzinfo-Auth", value = "Hzinfo-Auth", required = false) String hzinfoToken) throws JsonProcessingException { + return R.status(vectorParamService.synchronization(bladeToken, hzinfoToken)); + } } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java index f38826f..6553578 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/entity/VectorParamEntity.java @@ -28,17 +28,26 @@ public class VectorParamEntity extends TenantEntity implements Serializable { @ApiModelProperty("url") @TableField("URL") + @QueryField(condition = SqlCondition.LIKE) private String url; @ApiModelProperty("项目前缀") @TableField("PROJECT_PREFIX") + @QueryField(condition = SqlCondition.LIKE) private String projectPrefix; @ApiModelProperty("数据库表名") @TableField("TABLE_NAME") + @QueryField(condition = SqlCondition.LIKE) private String tableName; @ApiModelProperty("向量键值对，json格式") @TableField("ATTRIBUTE_MAP") + @QueryField(condition = SqlCondition.LIKE) private String attributeMap; + + @ApiModelProperty("备注") + @TableField("REMARK") + @QueryField(condition = SqlCondition.LIKE) + private String remark; } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java index 4fac0be..20cf06f 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/VectorParamService.java @@ -2,9 +2,9 @@ package com.hnac.hzims.bigmodel.maintenance.service; import com.baomidou.mybatisplus.extension.service.IService; import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity; +import com.hnac.hzims.bigmodel.maintenance.vo.VectorUrlResponse; import java.util.List; -import java.util.Map; /** * @Author: ypj @@ -14,7 +14,9 @@ import java.util.Map; public interface VectorParamService extends IService { String getAuthorization(); - Map getUrlResponse(String url, String bladeToken ,String hzinfoToken); + VectorUrlResponse getUrlResponse(String url, String bladeToken, String hzinfoToken); List getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken); + + Boolean synchronization(String bladeToken, String hzinfoToken); } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java index 0878002..a350a57 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java @@ -2,20 +2,26 @@ package com.hnac.hzims.bigmodel.maintenance.service.impl; import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.hnac.hzims.bigmodel.database.service.WeaviateService; import com.hnac.hzims.bigmodel.maintenance.entity.VectorParamEntity; import com.hnac.hzims.bigmodel.maintenance.mapper.VectorParamMapper; import com.hnac.hzims.bigmodel.maintenance.service.VectorParamService; +import com.hnac.hzims.bigmodel.maintenance.vo.VectorUrlResponse; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springblade.core.tool.jackson.JsonUtil; +import org.springblade.core.tool.utils.CollectionUtil; import org.springframework.stereotype.Service; import javax.servlet.http.HttpServletResponse; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; +import java.util.*; /** * @Author: ypj @@ -23,37 +29,126 @@ import java.util.Map; */ @Service @Slf4j +@AllArgsConstructor public class VectorParamServiceImpl extends ServiceImpl implements VectorParamService { private static final String CLIENT_SIGN = "vector_param"; + private final WeaviateService weaviateService; + @Override public String getAuthorization() { return "Basic " + Base64.getEncoder().encodeToString(CLIENT_SIGN.getBytes(StandardCharsets.UTF_8)); } @Override - public Map getUrlResponse(String url, String bladeToken, String hzinfoToken) { + public VectorUrlResponse getUrlResponse(String url, String bladeToken, String hzinfoToken) { HttpResponse response = HttpRequest.get(url) .header("Authorization", getAuthorization()) .header("Blade-Auth", bladeToken) .header("Hzinfo-Auth", hzinfoToken).execute(); if (response.getStatus() == HttpServletResponse.SC_OK) { - return JsonUtil.toMap(response.body()); + String body = response.body(); + log.info("getUrlResponse:" + body); + JSONObject jsonObject = JSON.parseObject(body); + VectorUrlResponse result = new VectorUrlResponse(); + result.setOriginalData(JSONArray.parseArray(jsonObject.getString("data")).toJavaList(JSONObject.class)); +// try { +// ObjectMapper mapper = new ObjectMapper(); +// JsonNode rootNode = mapper.readTree(body); +// if (rootNode.isArray()) { +// return getUrlResponseByArray(rootNode); +// } +// JsonNode dataNode = rootNode.findValue("data"); +// if (dataNode != null && dataNode.isArray()) { +// return getUrlResponseByArray(dataNode); +// } +// if (dataNode != null) { +// return getUrlResponseByObject(dataNode); +// } +// if (rootNode != null) { +// return getUrlResponseByObject(rootNode); +// } +// } catch (JsonProcessingException e) { +// log.error("getUrlResponse error", e); +// } } return null; } +// private VectorUrlResponse getUrlResponseByArray(JsonNode rootNode) throws JsonProcessingException { +// VectorUrlResponse result = new VectorUrlResponse(); +// result.setOriginalData(rootNode); +// JsonNode item = rootNode.get(0); +// if (item != null) { +// List keyList = new ArrayList<>(); +// Iterator iterator = item.fieldNames(); +// while (iterator.hasNext()) { +// keyList.add(iterator.next()); +// } +// result.setKeyList(keyList); +// } +// return result; +// } + +// private VectorUrlResponse getUrlResponseByObject(JsonNode rootNode) throws JsonProcessingException { +// VectorUrlResponse result = new VectorUrlResponse(); +// result.setOriginalData(rootNode); +// List keyList = new ArrayList<>(); +// Iterator iterator = rootNode.fieldNames(); +// while (iterator.hasNext()) { +// keyList.add(iterator.next()); +// } +// result.setKeyList(keyList); +// return result; +// } + @Override public List getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken) { - List result = new ArrayList<>(); - Map map = getUrlResponse(url, bladeToken, hzinfoToken); - Map data = ((Map) map.get("data")); - if (data != null) { - result.addAll(data.keySet()); - } else { - result.addAll(map.keySet()); + VectorUrlResponse response = getUrlResponse(url, bladeToken, hzinfoToken); + return response.getKeyList(); + } + + @Override + public Boolean synchronization(String bladeToken, String hzinfoToken) { + List list = this.list(); + if (CollectionUtil.isEmpty(list)) { + return false; + } + for (VectorParamEntity entity : list) { + VectorUrlResponse response = getUrlResponse(entity.getUrl(), bladeToken, hzinfoToken); + if (response != null) { + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = null; + try { + rootNode = mapper.readTree(entity.getAttributeMap()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + Map attrMap = new HashMap<>(); + Iterator iterator = rootNode.fieldNames(); + while (iterator.hasNext()) { + String key = iterator.next(); + attrMap.put(key, rootNode.findValue(key).textValue()); + } + List dataList = new ArrayList(); + JsonNode dataObject = (JsonNode) response.getOriginalData(); + if (dataObject.isArray()) { + Iterator dataIterator = dataObject.iterator(); + while (dataIterator.hasNext()) { + dataList.add(toJsonObject(dataIterator.next())); + } + } else { + dataList.add(toJsonObject(dataObject)); + } + weaviateService.saveBatch(response.getOriginalData(), entity.getTableName(), attrMap); + } } - return result; + return true; + } + + private JSONObject toJsonObject(JsonNode node) { + JSONObject jsonObject = JSON.parseObject(node.toString()); + return jsonObject; } } diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java new file mode 100644 index 0000000..4728c0e --- /dev/null +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/maintenance/vo/VectorUrlResponse.java @@ -0,0 +1,16 @@ +package com.hnac.hzims.bigmodel.maintenance.vo; + +import lombok.Data; + +import java.util.List; + +/** + * @Author: ypj + * @Date: 2024/9/5 21:12 + */ +@Data +public class VectorUrlResponse { + private List keyList; + + private List originalData; +} diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java index a7cb75b..823b870 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/zhipuai/service/ZhipuAnalysisInfoService.java @@ -1,9 +1,7 @@ package com.hnac.hzims.bigmodel.zhipuai.service; -import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.service.IService; import com.hnac.hzims.bigmodel.zhipuai.entity.ZhipuAnalysisInfoEntity; -import org.springblade.core.mp.support.Query; /** * @Author: ypj From 663bb18677289dc36699e4c82ff5931758e5c1d5 Mon Sep 17 00:00:00 2001 From: haungxing <1203316822@qq.com> Date: Fri, 6 Sep 2024 13:41:45 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=E5=AD=98=E5=85=A5=E5=90=91=E9=87=8F?= =?UTF-8?q?=E5=BA=93=E6=95=B0=E6=8D=AE=E5=AD=98=E5=9C=A8ID=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E5=BC=95=E8=B5=B7=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hzims/bigmodel/database/service/WeaviateService.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java index cc43acf..2af50cd 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java @@ -92,7 +92,10 @@ public class WeaviateService { // log.info("vector:{}",JSON.toJSONString(vector.get(i))); Map properties = this.objectToMap(entities.get(i)); log.info("properties:{}",JSON.toJSONString(properties)); - creator.withProperties(properties).withVectors(vector.get(i)).run(); + Result run = creator.withProperties(properties).withVectors(vector.get(i)).run(); + if(run.hasErrors()) { + log.error("保存失败！，保存结果为：{}",JSON.toJSONString(run)); + } } } else { entities.forEach(entity -> creator.withProperties(this.objectToMap(entity)).run()); @@ -104,7 +107,13 @@ public class WeaviateService { private Map objectToMap(Object object) { JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(object)); Map properties = new HashMap<>(); - jsonObject.forEach((k,v) -> properties.put(k,v)); + jsonObject.forEach((k,v) -> { + if("id".equalsIgnoreCase(k)) { + properties.put("id_",v); + } else { + properties.put(k,v); + } + }); return properties; } From cc903588abf9b24cf58e3b6ff8dcb49e6ed32a8d Mon Sep 17 00:00:00 2001 From: haungxing <1203316822@qq.com> Date: Fri, 6 Sep 2024 14:54:15 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=E5=88=86=E9=A1=B5=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E5=90=91=E9=87=8F=E5=BA=93=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../database/controller/WeaviateController.java | 7 +++-- .../bigmodel/database/service/WeaviateService.java | 35 +++++++++++++++------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java index 7683887..620950d 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/controller/WeaviateController.java @@ -27,8 +27,11 @@ public class WeaviateController { } @GetMapping("/list") - public R> list(@RequestParam(value = "id",required = false) String id, @RequestParam("className") String className) { - return R.data(weaviateService.list(id,className)); + public R> list(@RequestParam(value = "id",required = false) String id, + @RequestParam("className") String className, + @RequestParam("current") Integer current, + @RequestParam("pageSize") Integer pageSize) { + return R.data(weaviateService.list(id,className,current,pageSize)); } @DeleteMapping("/removeById") diff --git a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java index 2af50cd..76d308c 100644 --- a/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java +++ b/hzims-service/hzims-big-model/src/main/java/com/hnac/hzims/bigmodel/database/service/WeaviateService.java @@ -8,27 +8,34 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.hnac.hzims.bigmodel.configuration.BigModelInvokeApi; import com.hnac.hzinfo.exception.HzServiceException; +import io.weaviate.client.Config; +import io.weaviate.client.WeaviateAuthClient; import io.weaviate.client.WeaviateClient; import io.weaviate.client.base.Result; +import io.weaviate.client.v1.auth.exception.AuthException; import io.weaviate.client.v1.data.api.ObjectCreator; import io.weaviate.client.v1.data.api.ObjectDeleter; import io.weaviate.client.v1.data.api.ObjectUpdater; import io.weaviate.client.v1.data.api.ObjectsGetter; import io.weaviate.client.v1.data.model.WeaviateObject; +import io.weaviate.client.v1.graphql.model.GraphQLResponse; +import io.weaviate.client.v1.graphql.query.Get; +import io.weaviate.client.v1.graphql.query.fields.Field; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springblade.core.tool.api.ResultCode; import org.springblade.core.tool.utils.BeanUtil; import org.springblade.core.tool.utils.Func; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; /** * @Author: huangxing @@ -122,16 +129,22 @@ public class WeaviateService { * @param className 表名 * @return 删除结果 */ - public Boolean delete(String id,String className) { + public Boolean delete(String ids,String className) { ObjectDeleter deleter = weaviateClient.data().deleter(); - if(Func.isNotEmpty(id)) { - deleter.withID(id); - } - if(Func.isNotEmpty(className)) { - deleter.withClassName(className); + deleter.withClassName(className); + if(Func.isEmpty(ids)) { + Result> allObject = weaviateClient.data().objectsGetter().withClassName(className).withLimit(10000).run(); + if(!allObject.hasErrors()) { + ids = allObject.getResult().stream().map(WeaviateObject::getId).collect(Collectors.joining(",")); + } } - Result result = deleter.run(); - return !result.hasErrors(); + Func.toStrList(",",ids).forEach(id -> { + Result result = deleter.withID(id).run(); + if(result.hasErrors()) { + throw new HzServiceException(ResultCode.FAILURE,id + "记录删除失败！"); + } + }); + return true; } /** @@ -157,7 +170,7 @@ public class WeaviateService { return !result.hasErrors(); } - public List list(String id,String className) { + public List list(String id,String className,Integer current,Integer pageSize) { ObjectsGetter objectsGetter = weaviateClient.data().objectsGetter(); if(Func.isNotEmpty(id)) { objectsGetter.withID(id); @@ -165,7 +178,7 @@ public class WeaviateService { if(Func.isNotEmpty(className)) { objectsGetter.withClassName(className); } - Result> result = objectsGetter.run(); + Result> result = objectsGetter.withLimit(pageSize).withOffset((current-1) * pageSize).run(); if(result.hasErrors()) { throw new HzServiceException("查询失败！"); }