From f1c76ce1cb042e07c5b9a755088124ef8c2eeda9 Mon Sep 17 00:00:00 2001 From: haungxing <1203316822@qq.com> Date: Wed, 25 Sep 2024 15:07:11 +0800 Subject: [PATCH] =?UTF-8?q?add:=20=E5=90=91=E9=87=8F=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E6=A0=B9=E6=8D=AE=E6=9D=A1=E4=BB=B6=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E6=95=B0=E6=8D=AEapi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../database/controller/WeaviateController.java | 8 ++++ .../bigmodel/database/service/WeaviateService.java | 51 +++++++++++++++++++--- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java index 224514f..b16899a 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java @@ -9,6 +9,7 @@ import org.springblade.core.tool.api.R; import org.springframework.web.bind.annotation.*; import java.util.List; +import java.util.Map; /** * @Author: huangxing @@ -44,4 +45,11 @@ public class WeaviateController { public R query(@RequestBody WeaviateQueryDTO query) { return R.data(weaviateService.query(query.getResultFields(),query.getClassName(),query.getQuery())); } + + @DeleteMapping("/deleteCondition") + public R deleteCondition(@RequestBody Map request) { + String className = (String) request.get("className"); + Map condition = (Map) request.get("condition"); + return R.status(weaviateService.deleteCondition(className,condition)); + } } diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java index d419337..f83fb3b 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java @@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; import cn.hutool.json.JSONUtil; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; import com.alibaba.fastjson.serializer.SerializerFeature; @@ -11,6 +12,7 @@ import com.google.common.collect.Lists; import com.hnac.gglm.bigmodel.BigModelConstants; import com.hnac.gglm.bigmodel.configuration.BigModelInvokeApi; import com.hnac.gglm.bigmodel.utils.RequestClientUtil; +import com.hnac.hzims.fdp.constants.ScheduledConstant; import com.hnac.hzinfo.exception.HzServiceException; import io.weaviate.client.Config; import io.weaviate.client.WeaviateAuthClient; @@ -100,7 +102,7 @@ public class WeaviateService { List entitiesList = splitList(entities, 1000); int total = 0; for (List entityList : entitiesList) { - Integer insert = this.insert(entityList, className, attrsMap, params); + Integer insert = this.insert(entityList, attrsMap, params); total += insert; } // 查询weaviate 中该表的数据量 @@ -108,6 +110,36 @@ public class WeaviateService { } /** + * 根据条件删除数据 + * @param className 表名 + * @param condition 查询条件 + * @return 删除结果 + */ + public Boolean deleteCondition(String className,Map condition) { + // 查询到相关数据 + Object query = this.query(null, className, condition); + if(Func.isEmpty(query)) { + throw new HzServiceException("暂无数据，删除失败！"); + } + JSONObject queryJson = JSONObject.parseObject(JSON.toJSONString(query)); + JSONArray data = Optional.ofNullable(queryJson).map(json -> json.getJSONObject("Get")) + .map(json -> json.getJSONArray(className)).orElse(null); + if(Func.isNotEmpty(data)) { + List ids = data.stream().map(item -> { + JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(item)); + return Optional.ofNullable(jsonObject) + .map(json -> json.getJSONObject("_additional")) + .map(json -> json.getString("id")) + .orElse(""); + }).filter(Func::isNotEmpty).collect(Collectors.toList()); + if(Func.isNotEmpty(ids)) { + this.delete(ids.stream().collect(Collectors.joining(",")), className); + } + } + return true; + } + + /** * 将list按size截断为多个list * @param list 待截断的list * @param size 截断大小 @@ -122,14 +154,21 @@ public class WeaviateService { return parts; } - private Integer insert(List entities,String className, Map attrsMap, Map params) { + /** + * 批量插入数据 + * @param entities 待插入的数据 + * @param attrsMap 向量計算Map + * @param params 向量計算参数 + * @return 插入数量 + */ + private Integer insert(List entities, Map attrsMap, Map params) { List> data = new ArrayList<>(); entities.forEach(entity -> { // 将entity转换为Map JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity, SerializerFeature.WriteMapNullValue)); - Map map = new HashMap<>(); + Map map = new HashMap<>(); data.add(this.getVectorData(map,attrsMap)); - jsonObject.forEach((k,v) -> map.put(k,jsonObject.getString(k))); + jsonObject.forEach((k,v) -> map.put(k,jsonObject.get(k))); }); params.put("data",data); String url = gglmUrl + invokeApi.getInsertVectors(); @@ -138,14 +177,14 @@ public class WeaviateService { return (Integer) stringIntegerMap.get("total"); } - private Map getVectorData(Map entity,Map attrsMap) { + private Map getVectorData(Map entity,Map attrsMap) { Map result = new HashMap<>(2); result.put("object", entity); List> vectors = new ArrayList<>(); attrsMap.forEach((k,fields) -> { Map vector = new HashMap<>(); vector.put("key",k); - String value = Func.toStrList(",", fields).stream().map(field -> entity.get(field)).collect(Collectors.joining(" ")); + String value = Func.toStrList(",", fields).stream().map(field -> entity.get(field)).map(String::valueOf).collect(Collectors.joining(" ")); vector.put("content", value); vectors.add(vector); });