|
|
|
|
@ -4,10 +4,15 @@ import cn.hutool.http.HttpRequest;
|
|
|
|
|
import cn.hutool.http.HttpResponse; |
|
|
|
|
import cn.hutool.json.JSONUtil; |
|
|
|
|
import com.alibaba.fastjson.JSON; |
|
|
|
|
import com.alibaba.fastjson.JSONArray; |
|
|
|
|
import com.alibaba.fastjson.JSONObject; |
|
|
|
|
import com.alibaba.fastjson.TypeReference; |
|
|
|
|
import com.alibaba.fastjson.serializer.SerializerFeature; |
|
|
|
|
import com.google.common.collect.Lists; |
|
|
|
|
import com.hnac.gglm.bigmodel.BigModelConstants; |
|
|
|
|
import com.hnac.gglm.bigmodel.configuration.BigModelInvokeApi; |
|
|
|
|
import com.hnac.gglm.bigmodel.utils.RequestClientUtil; |
|
|
|
|
import com.hnac.hzims.fdp.constants.ScheduledConstant; |
|
|
|
|
import com.hnac.hzinfo.exception.HzServiceException; |
|
|
|
|
import io.weaviate.client.Config; |
|
|
|
|
import io.weaviate.client.WeaviateAuthClient; |
|
|
|
|
@ -30,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import org.springblade.core.tool.api.ResultCode; |
|
|
|
|
import org.springblade.core.tool.utils.BeanUtil; |
|
|
|
|
import org.springblade.core.tool.utils.Func; |
|
|
|
|
import org.springblade.system.user.entity.User; |
|
|
|
|
import org.springframework.beans.factory.annotation.Value; |
|
|
|
|
import org.springframework.stereotype.Service; |
|
|
|
|
|
|
|
|
|
@ -76,50 +82,9 @@ public class WeaviateService {
|
|
|
|
|
return !result.hasErrors(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// /**
|
|
|
|
|
// * 对象批量保存向量数据库
|
|
|
|
|
// * @param entities 保存对象列表
|
|
|
|
|
// * @param className 保存表名
|
|
|
|
|
// * @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔
|
|
|
|
|
// * @return 保存操作结果
|
|
|
|
|
// */
|
|
|
|
|
// public Boolean saveBatch(List entities,String className, Map<String,String> attrsMap) {
|
|
|
|
|
// entities = entities.subList(0, 1);
|
|
|
|
|
// ObjectCreator creator = weaviateClient.data().creator().withClassName(className);
|
|
|
|
|
// List<String> vectorStrs = Lists.newArrayList();
|
|
|
|
|
// List<String> attrs = Lists.newArrayList();
|
|
|
|
|
// if(Func.isNotEmpty(attrsMap)) {
|
|
|
|
|
// // 格式化数据
|
|
|
|
|
// attrsMap.forEach((k,v) -> attrs.add(v));
|
|
|
|
|
// // 解析待计算的向量字段
|
|
|
|
|
// entities.forEach(entity -> {
|
|
|
|
|
// List<String> vectorStr = attrs.stream().map(fields -> this.getFieldValue(fields, entity)).filter(Func::isNotEmpty).collect(Collectors.toList());
|
|
|
|
|
// vectorStrs.addAll(vectorStr);
|
|
|
|
|
// });
|
|
|
|
|
// }
|
|
|
|
|
// if(Func.isNotEmpty(vectorStrs)) {
|
|
|
|
|
// // 若解析出来的向量存在值
|
|
|
|
|
// Float[] vectors = this.compute(vectorStrs);
|
|
|
|
|
// List<Map<String, Float[]>> vector = this.splitVector(entities.size(), attrsMap, vectors);
|
|
|
|
|
// for(int i = 0; i < entities.size(); i++) {
|
|
|
|
|
// // log.info("vector:{}",JSON.toJSONString(vector.get(i)));
|
|
|
|
|
// Map<String, Object> properties = this.objectToMap(entities.get(i));
|
|
|
|
|
// log.info("properties:{}",JSON.toJSONString(properties));
|
|
|
|
|
// Result<WeaviateObject> run = creator.withProperties(properties).withVectors(vector.get(i)).run();
|
|
|
|
|
// if(run.hasErrors()) {
|
|
|
|
|
// log.error("保存失败！，保存结果为：{}",JSON.toJSONString(run));
|
|
|
|
|
// }
|
|
|
|
|
// }
|
|
|
|
|
// } else {
|
|
|
|
|
// entities.forEach(entity -> creator.withProperties(this.objectToMap(entity)).run());
|
|
|
|
|
// return true;
|
|
|
|
|
// }
|
|
|
|
|
// return false;
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
|
|
public Boolean saveBatch(List entities,String className, Map<String,String> attrsMap) { |
|
|
|
|
// 查询Weaviate 4.7.0表是否存在 若不存则新建表
|
|
|
|
|
Result<Boolean> existResult = weaviateClient.schema().exists().withClassName("Hzn_lm_" + className).run(); |
|
|
|
|
public String saveBatch(List entities,String className, Map<String,String> attrsMap) { |
|
|
|
|
// 查询表是否存在 若不存则新建表
|
|
|
|
|
Result<Boolean> existResult = weaviateClient.schema().exists().withClassName(BigModelConstants.PREFIX + "_" + className).run(); |
|
|
|
|
if(existResult.hasErrors() || !existResult.getResult()) { |
|
|
|
|
Map<java.lang.String,Object> createTableParams = new HashMap<>(2); |
|
|
|
|
Map<String,String> deleteTableParams = new HashMap<>(1); |
|
|
|
|
@ -133,23 +98,94 @@ public class WeaviateService {
|
|
|
|
|
} |
|
|
|
|
Map<String,Object> params = new HashMap<>(2); |
|
|
|
|
params.put("table_name", className); |
|
|
|
|
// 将entities按size截断为1000个一组
|
|
|
|
|
List<List> entitiesList = splitList(entities, 1000); |
|
|
|
|
int total = 0; |
|
|
|
|
for (List entityList : entitiesList) { |
|
|
|
|
Integer insert = this.insert(entityList, attrsMap, params); |
|
|
|
|
total += insert; |
|
|
|
|
} |
|
|
|
|
// 查询weaviate 中该表的数据量
|
|
|
|
|
return String.format("传入数据总量为：%s 保存成功数量为：%s", entities.size(), total); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* 根据条件删除数据 |
|
|
|
|
* @param className 表名 |
|
|
|
|
* @param condition 查询条件 |
|
|
|
|
* @return 删除结果 |
|
|
|
|
*/ |
|
|
|
|
public Boolean deleteCondition(String className,Map<String,String> condition) { |
|
|
|
|
// 查询到相关数据
|
|
|
|
|
Object query = this.query(null, className, condition); |
|
|
|
|
if(Func.isEmpty(query)) { |
|
|
|
|
throw new HzServiceException("暂无数据，删除失败！"); |
|
|
|
|
} |
|
|
|
|
JSONObject queryJson = JSONObject.parseObject(JSON.toJSONString(query)); |
|
|
|
|
JSONArray data = Optional.ofNullable(queryJson).map(json -> json.getJSONObject("Get")) |
|
|
|
|
.map(json -> json.getJSONArray(className)).orElse(null); |
|
|
|
|
if(Func.isNotEmpty(data)) { |
|
|
|
|
List<String> ids = data.stream().map(item -> { |
|
|
|
|
JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(item)); |
|
|
|
|
return Optional.ofNullable(jsonObject) |
|
|
|
|
.map(json -> json.getJSONObject("_additional")) |
|
|
|
|
.map(json -> json.getString("id")) |
|
|
|
|
.orElse(""); |
|
|
|
|
}).filter(Func::isNotEmpty).collect(Collectors.toList()); |
|
|
|
|
if(Func.isNotEmpty(ids)) { |
|
|
|
|
this.delete(ids.stream().collect(Collectors.joining(",")), className); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
return true; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* 将list按size截断为多个list |
|
|
|
|
* @param list 待截断的list |
|
|
|
|
* @param size 截断大小 |
|
|
|
|
* @return |
|
|
|
|
* @param <T> |
|
|
|
|
*/ |
|
|
|
|
public static <T> List<List<T>> splitList(List<T> list, int size) { |
|
|
|
|
List<List<T>> parts = new ArrayList<>(); |
|
|
|
|
for (int i = 0; i < list.size(); i += size) { |
|
|
|
|
parts.add(list.subList(i, Math.min(list.size(), i + size))); |
|
|
|
|
} |
|
|
|
|
return parts; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* 批量插入数据 |
|
|
|
|
* @param entities 待插入的数据 |
|
|
|
|
* @param attrsMap 向量計算Map |
|
|
|
|
* @param params 向量計算参数 |
|
|
|
|
* @return 插入数量 |
|
|
|
|
*/ |
|
|
|
|
private Integer insert(List entities, Map<String,String> attrsMap, Map<String,Object> params) { |
|
|
|
|
List<Map<String, Object>> data = new ArrayList<>(); |
|
|
|
|
entities.forEach(entity -> data.add(this.getVectorData(entity,attrsMap))); |
|
|
|
|
log.info("data:{}",JSON.toJSONString(data)); |
|
|
|
|
entities.forEach(entity -> { |
|
|
|
|
// 将entity转换为Map<String,String>
|
|
|
|
|
JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity, SerializerFeature.WriteMapNullValue)); |
|
|
|
|
Map<String,Object> map = new HashMap<>(); |
|
|
|
|
data.add(this.getVectorData(map,attrsMap)); |
|
|
|
|
jsonObject.forEach((k,v) -> map.put(k,jsonObject.get(k))); |
|
|
|
|
}); |
|
|
|
|
params.put("data",data); |
|
|
|
|
String url = gglmUrl + invokeApi.getInsertVectors(); |
|
|
|
|
RequestClientUtil.postCall(url,params); |
|
|
|
|
return true; |
|
|
|
|
Map<String, Object> stringIntegerMap = RequestClientUtil.postCall(url, params, new TypeReference<Map<String, Object>>() { |
|
|
|
|
}); |
|
|
|
|
return (Integer) stringIntegerMap.get("total"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private Map<String,Object> getVectorData(Object entity,Map<String,String> attrsMap) { |
|
|
|
|
private Map<String,Object> getVectorData(Map<String,Object> entity,Map<String,String> attrsMap) { |
|
|
|
|
Map<String,Object> result = new HashMap<>(2); |
|
|
|
|
result.put("object", entity); |
|
|
|
|
List<Map<String,String>> vectors = new ArrayList<>(); |
|
|
|
|
attrsMap.forEach((k,fields) -> { |
|
|
|
|
Map<String,String> vector = new HashMap<>(); |
|
|
|
|
vector.put("key",k); |
|
|
|
|
vector.put("content", this.getFieldValue(fields, entity)); |
|
|
|
|
String value = Func.toStrList(",", fields).stream().map(field -> entity.get(field)).map(String::valueOf).collect(Collectors.joining(" ")); |
|
|
|
|
vector.put("content", value); |
|
|
|
|
vectors.add(vector); |
|
|
|
|
}); |
|
|
|
|
result.put("vector", vectors); |
|
|
|
|
@ -178,7 +214,7 @@ public class WeaviateService {
|
|
|
|
|
if(Func.isEmpty(ids) && Func.isNotEmpty(className)) { |
|
|
|
|
// 删除className
|
|
|
|
|
Map<String,String> deleteTableParams = new HashMap<>(1); |
|
|
|
|
deleteTableParams.put("table_name",className.replace("Hzn_lm_","")); |
|
|
|
|
deleteTableParams.put("table_name",className.replace(BigModelConstants.PREFIX + "_","")); |
|
|
|
|
RequestClientUtil.postCall(gglmUrl + invokeApi.getDeleteTable(),deleteTableParams); |
|
|
|
|
} else { |
|
|
|
|
// 删除记录
|
|
|
|
|
|
