yangshj
/
zhongwei
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge remote-tracking branch 'origin/prod-5.1.3' into prod-5.1.3

zhongwei
yang_shj 4 months ago
parent
844fed8af4 b95e31b51c
commit
b8e633ef87
10 changed files with 112 additions and 45 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      hzims-service-api/big-model-api/src/main/java/com/hnac/gglm/bigmodel/BigModelConstants.java
  2. 2
      hzims-service-api/equipment-api/src/main/java/com/hnac/hzims/fdp/constants/ScheduledConstant.java
  3. 2
      hzims-service/equipment/src/main/java/com/hnac/hzims/equipment/scheduled/DeviceLedgerScheduledTask.java
  4. 14
      hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/configuration/WeaviateProperties.java
  5. 7
      hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java
  6. 102
      hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java
  7. 3
      hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/interactive/service/impl/ChoiceAnswerResolveServiceImpl.java
  8. 9
      hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/interactive/service/impl/HznlmInvokeServiceImpl.java
  9. 7
      hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/schedule/FrontEndInteractiveSchedule.java
  10. 9
      hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/websocket/server/InteractiveWsServer.java

2
hzims-service-api/big-model-api/src/main/java/com/hnac/gglm/bigmodel/BigModelConstants.java
Unescape View File

@ -2,7 +2,7 @@ package com.hnac.gglm.bigmodel;
/**
* @Author: huangxing
* @Date: 2024/04/26 10:15
* @Date: 2024/04/26 10:15listKnowledgeData
*/
public interface BigModelConstants {

2
hzims-service-api/equipment-api/src/main/java/com/hnac/hzims/fdp/constants/ScheduledConstant.java
Unescape View File

@ -33,5 +33,5 @@ public interface ScheduledConstant {
String DEVICE_LEDGER_DATA_GENERATE = "deviceLedgerDataGenerate";
/**设备台账数据存入向量数据库表名**/
String DEVICE_LEDGER_CLASS_NAME = "Hzims_deviceLedger_data";
String DEVICE_LEDGER_CLASS_NAME = "deviceLedger_data";
}

2
hzims-service/equipment/src/main/java/com/hnac/hzims/equipment/scheduled/DeviceLedgerScheduledTask.java
Unescape View File

@ -77,7 +77,7 @@ public class DeviceLedgerScheduledTask {
// 批量插入数据
WeaviateInsertDTO weaviateInsertDTO = new WeaviateInsertDTO();
weaviateInsertDTO.setEntities(deviceLedgerList);
weaviateInsertDTO.setClassName(ScheduledConstant.DEVICE_LEDGER_CLASS_NAME.replace(BigModelConstants.PREFIX + "_",""));
weaviateInsertDTO.setClassName(ScheduledConstant.DEVICE_LEDGER_CLASS_NAME);
Map<String,String> attrMap = new HashMap<>();
attrMap.put("dataSummary","dataSummary");
weaviateInsertDTO.setAttrsMap(attrMap);

14
hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/configuration/WeaviateProperties.java
Unescape View File

@ -13,13 +13,27 @@ import org.springframework.stereotype.Component;
@ConfigurationProperties(prefix = "weaviate.datasource")
public class WeaviateProperties {
/**
* schema 分为 http 和 https
*/
private String schema;
/**
* 主机
*/
private String host;
/**
* 端口
*/
private String port;
/**
* 数据库名 实际为数据库名前缀
*/
private String database;
/**
* 登录认证KEY
*/
private String apiKey;

7
hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java
Unescape View File

@ -1,10 +1,12 @@
package com.hnac.gglm.bigmodel.database.controller;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.hnac.gglm.bigmodel.api.dto.WeaviateQueryDTO;
import com.hnac.gglm.bigmodel.database.dto.WeaviateSaveDTO;
import com.hnac.gglm.bigmodel.database.service.WeaviateService;
import io.weaviate.client.v1.data.model.WeaviateObject;
import lombok.AllArgsConstructor;
import org.springblade.core.mp.support.Query;
import org.springblade.core.tool.api.R;
import org.springframework.web.bind.annotation.*;
@ -52,4 +54,9 @@ public class WeaviateController {
Map<String,String> condition = (Map<String, String>) request.get("condition");
return R.status(weaviateService.deleteCondition(className,condition));
}
@GetMapping("/page")
public R<IPage<WeaviateObject>> page(String className, Query query) {
return R.data(weaviateService.page(className,query));
}
}

102
hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java
Unescape View File

@ -8,9 +8,12 @@ import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.google.common.collect.Lists;
import com.hnac.gglm.bigmodel.BigModelConstants;
import com.hnac.gglm.bigmodel.configuration.BigModelInvokeApi;
import com.hnac.gglm.bigmodel.configuration.WeaviateProperties;
import com.hnac.gglm.bigmodel.utils.RequestClientUtil;
import com.hnac.hzims.fdp.constants.ScheduledConstant;
import com.hnac.hzinfo.exception.HzServiceException;
@ -32,6 +35,7 @@ import io.weaviate.client.v1.graphql.query.Get;
import io.weaviate.client.v1.graphql.query.fields.Field;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springblade.core.mp.support.Query;
import org.springblade.core.tool.api.ResultCode;
import org.springblade.core.tool.utils.BeanUtil;
import org.springblade.core.tool.utils.Func;
@ -57,39 +61,36 @@ public class WeaviateService {
private final WeaviateClient weaviateClient;
private final BigModelInvokeApi invokeApi;
private final WeaviateProperties weaviateProperties;
@Value("${gglm.vectorUrl}")
private String vectorUrl;
@Value("${gglm.url}")
private String gglmUrl;
private static final String QUERY_TEMPLATE = "{Aggregate {%s {meta {count}}}}";
/**
* 对象保存向量数据库
* @param entity 保存对象
* @param className 保存表名
* @param attrs 待计算的列信息
* @param attrsMap
* @return 保存操作结果
*/
public Boolean save(Object entity, String className, List<String> attrs) {
ObjectCreator creator = weaviateClient.data().creator().withClassName(className);
if(Func.isNotEmpty(attrs)) {
JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity));
List<String> vectors = attrs.stream().map(attr -> jsonObject.getString(attr)).collect(Collectors.toList());
Float[] compute = this.compute(vectors);
creator.withVector(compute);
}
Result<WeaviateObject> result = creator.withProperties(BeanUtil.toMap(entity)).run();
return !result.hasErrors();
public String save(Object entity, String className, Map<String,String> attrsMap) {
return this.saveBatch(Lists.newArrayList(entity),className, attrsMap);
}
public String saveBatch(List entities,String className, Map<String,String> attrsMap) {
String tableName = getRealTableName(className);
String realClassName = this.getRealClassName(className);
// 查询表是否存在 若不存则新建表
Result<Boolean> existResult = weaviateClient.schema().exists().withClassName(BigModelConstants.PREFIX + "_" + className).run();
Result<Boolean> existResult = weaviateClient.schema().exists().withClassName(tableName).run();
if(existResult.hasErrors() || !existResult.getResult()) {
Map<java.lang.String,Object> createTableParams = new HashMap<>(2);
Map<String,String> deleteTableParams = new HashMap<>(1);
deleteTableParams.put("table_name",className);
createTableParams.put("table_name",className);
deleteTableParams.put("table_name", realClassName);
createTableParams.put("table_name", realClassName);
List<String> vectorStr = Lists.newArrayList();
attrsMap.keySet().forEach(key -> vectorStr.add(key));
createTableParams.put("vector_names",vectorStr.toArray(new String[vectorStr.size()]));
@ -97,18 +98,26 @@ public class WeaviateService {
RequestClientUtil.postCall(gglmUrl + invokeApi.getCreateTable(),createTableParams);
}
Map<String,Object> params = new HashMap<>(2);
params.put("table_name", className);
params.put("table_name", realClassName);
// 将entities按size截断为1000个一组
List<List> entitiesList = splitList(entities, 1000);
int total = 0;
for (List entityList : entitiesList) {
Integer insert = this.insert(entityList, attrsMap, params);
total += insert;
total = insert;
}
// 查询weaviate 中该表的数据量
return String.format("传入数据总量为：%s 保存成功数量为：%s", entities.size(), total);
}
private String getRealTableName(String className) {
return className.contains(weaviateProperties.getDatabase()) ? className : weaviateProperties.getDatabase() + "_" + className;
}
private String getRealClassName(String className) {
return className.contains(weaviateProperties.getDatabase()) ? className.replace(weaviateProperties.getDatabase() + "_","") : className;
}
/**
* 根据条件删除数据
* @param className 表名
@ -116,15 +125,18 @@ public class WeaviateService {
* @return 删除结果
*/
public Boolean deleteCondition(String className,Map<String,String> condition) {
// 获取到实际表名
String tableName = this.getRealTableName(className);
// 查询到相关数据
Object query = this.query(null, className, condition);
Object query = this.query(null, tableName, condition);
if(Func.isEmpty(query)) {
throw new HzServiceException("暂无数据，删除失败！");
}
JSONObject queryJson = JSONObject.parseObject(JSON.toJSONString(query));
JSONArray data = Optional.ofNullable(queryJson).map(json -> json.getJSONObject("Get"))
.map(json -> json.getJSONArray(className)).orElse(null);
.map(json -> json.getJSONArray(tableName)).orElse(null);
if(Func.isNotEmpty(data)) {
// 获取到主键id，根据id删除数据
List<String> ids = data.stream().map(item -> {
JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(item));
return Optional.ofNullable(jsonObject)
@ -133,7 +145,7 @@ public class WeaviateService {
.orElse("");
}).filter(Func::isNotEmpty).collect(Collectors.toList());
if(Func.isNotEmpty(ids)) {
this.delete(ids.stream().collect(Collectors.joining(",")), className);
this.delete(ids.stream().collect(Collectors.joining(",")), tableName);
}
}
return true;
@ -164,7 +176,7 @@ public class WeaviateService {
private Integer insert(List entities, Map<String,String> attrsMap, Map<String,Object> params) {
List<Map<String, Object>> data = new ArrayList<>();
entities.forEach(entity -> {
// 将entity转换为Map<String,String>
// 将entity转换为 Map<String,String> 根据数据以及向量字段对向量进行拼接后调用大模型接口进行保存操作
JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity, SerializerFeature.WriteMapNullValue));
Map<String,Object> map = new HashMap<>();
jsonObject.forEach((k,v) -> map.put(k,jsonObject.get(k)));
@ -211,15 +223,17 @@ public class WeaviateService {
* @return 删除结果
*/
public Boolean delete(String ids,String className) {
if(Func.isEmpty(ids) && Func.isNotEmpty(className)) {
String tableName = this.getRealTableName(className);
String realClassName = this.getRealClassName(className);
if(Func.isEmpty(ids)) {
// 删除className
Map<String,String> deleteTableParams = new HashMap<>(1);
deleteTableParams.put("table_name",className.replace(BigModelConstants.PREFIX + "_",""));
deleteTableParams.put("table_name", realClassName);
RequestClientUtil.postCall(gglmUrl + invokeApi.getDeleteTable(),deleteTableParams);
} else {
// 删除记录
ObjectDeleter deleter = weaviateClient.data().deleter();
deleter.withClassName(className);
deleter.withClassName(tableName);
Func.toStrList(",",ids).forEach(id -> {
Result<Boolean> result = deleter.withID(id).run();
if(result.hasErrors()) {
@ -235,6 +249,7 @@ public class WeaviateService {
* @param id 向量数据库ID
* @return 更新结果
*/
@Deprecated
public Boolean updateById(String id, Object entity, String className, Map<String,String> attrMap) {
ObjectUpdater updater = weaviateClient.data().updater().withClassName(className).withID(id).withProperties(BeanUtil.toMap(entity));
// 计算向量
@ -253,13 +268,49 @@ public class WeaviateService {
return !result.hasErrors();
}
public IPage<WeaviateObject> page(String className, Query query) {
IPage<WeaviateObject> result = new Page<>();
Integer offset = (query.getCurrent() - 1) * query.getSize();
ObjectsGetter objectsGetter = weaviateClient.data().objectsGetter();
String realTableName = this.getRealTableName(className);
// 查询数据库表数据总量
Integer classTotal = this.getClassTotal(className);
Result<List<WeaviateObject>> run = objectsGetter.withClassName(realTableName).withLimit(query.getSize()).withOffset(offset).run();
if(run.getResult() == null && run.hasErrors()) {
throw new HzServiceException("数据库暂无" + realTableName + "数据表信息，同步数据后查询");
}
result.setRecords(run.getResult());
result.setTotal(classTotal);
result.setSize(query.getSize());
result.setCurrent(query.getCurrent());
return result;
}
public Integer getClassTotal(String className) {
String realTableName = this.getRealTableName(className);
String url = weaviateProperties.getSchema() + "://" + weaviateProperties.getHost() + ":" + weaviateProperties.getPort() + "/v1/graphql";
String query = "{Aggregate {" + realTableName + " {meta {count}}}}";
Map<String,Object> params = new HashMap<>();
params.put("query",query);
String authorization = "Bearer " + weaviateProperties.getApiKey();
try {
String body = HttpRequest.post(url).header("Authorization", authorization).body(JSON.toJSONString(params, SerializerFeature.WriteMapNullValue)).execute().body();
JSONArray dataArray = JSONObject.parseObject(body).getJSONObject("data").getJSONObject("Aggregate").getJSONArray(realTableName);
return Optional.ofNullable(dataArray).map(array -> array.getJSONObject(0).getJSONObject("meta").getInteger("count")).orElse(0);
} catch (Exception e) {
e.printStackTrace();
throw new HzServiceException("获取" + className + "数据总量失败，查询失败！");
}
}
public List<WeaviateObject> list(String id,String className,Integer current,Integer pageSize) {
String tableName = this.getRealTableName(className);
ObjectsGetter objectsGetter = weaviateClient.data().objectsGetter();
if(Func.isNotEmpty(id)) {
objectsGetter.withID(id);
}
if(Func.isNotEmpty(className)) {
objectsGetter.withClassName(className);
objectsGetter.withClassName(tableName);
}
Result<List<WeaviateObject>> result = objectsGetter.withLimit(pageSize).withOffset((current-1) * pageSize).run();
if(result.hasErrors()) {
@ -361,9 +412,10 @@ public class WeaviateService {
* @return 查询结果
*/
public Object query(String resultFields,String className,Map<String,String> query) {
String realTableName = this.getRealTableName(className);
List<String> fieldList = Func.toStrList(",", resultFields);
Get get = weaviateClient.graphQL().get();
get.withClassName(className);
get.withClassName(realTableName);
List<Field> fields = fieldList.stream().map(fieldStr -> Field.builder().name(fieldStr).build()).collect(Collectors.toList());
Field additionalId = Field.builder().name("_additional { id }").build();
fields.add(additionalId);

3
hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/interactive/service/impl/ChoiceAnswerResolveServiceImpl.java
Unescape View File

@ -3,6 +3,7 @@ package com.hnac.gglm.bigmodel.interactive.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.hnac.gglm.bigmodel.interactive.constants.FuncRouteEnum;
import com.hnac.gglm.bigmodel.interactive.factory.AnswerResolveFactory;
import com.hnac.gglm.bigmodel.interactive.service.IAnswerResolveService;
@ -35,7 +36,7 @@ public class ChoiceAnswerResolveServiceImpl implements IAnswerResolveService {
@Override
public ExtraVO getExtra(JSONObject originExtra) {
ExtraVO result = JSONObject.parseObject(JSON.toJSONString(originExtra),ExtraVO.class);
ExtraVO result = JSONObject.parseObject(JSON.toJSONString(originExtra, SerializerFeature.WriteMapNullValue),ExtraVO.class);
result.setSpecial(true);
result.setType(FuncRouteEnum.getEnumByFuncCode(result.getFunc()).getType().getType());
return result;

9
hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/interactive/service/impl/HznlmInvokeServiceImpl.java
Unescape View File

@ -3,6 +3,7 @@ package com.hnac.gglm.bigmodel.interactive.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import com.google.common.collect.Lists;
import com.hnac.gglm.bigmodel.business.dto.RunReportAnalyseRequest;
@ -85,14 +86,14 @@ public class HznlmInvokeServiceImpl implements IHznlmInvokeService {
params.put("extra",extra);
Map<String, String[]> authDataIds = this.getAuthDataIds(userId);
params.putAll(authDataIds);
log.info("调用大模型接口：{},传参为：{}",gglmUrl + bigModelInvokeApi.getAssistantSpecialAsk(),JSON.toJSONString(params));
log.info("调用大模型接口：{},传参为：{}",gglmUrl + bigModelInvokeApi.getAssistantSpecialAsk(),JSON.toJSONString(params, SerializerFeature.WriteMapNullValue));
RequestClientUtil.postCall(gglmUrl + bigModelInvokeApi.getAssistantSpecialAsk(), params);
sessionRedisManager.addSessionId(sessionId);
}
@Override
public void specialAsk(QuestionDTO question) {
JSONObject params = JSONObject.parseObject(JSON.toJSONString(question));
JSONObject params = JSONObject.parseObject(JSON.toJSONString(question, SerializerFeature.WriteMapNullValue));
params.put("q_id", UUID.randomUUID().toString());
RequestClientUtil.postCall(gglmUrl + bigModelInvokeApi.getAssistantSpecialAsk(), params);
sessionRedisManager.addSessionId(question.getChatId());
@ -114,7 +115,7 @@ public class HznlmInvokeServiceImpl implements IHznlmInvokeService {
@Override
public void knowledgeAsk(QuestionDTO question) {
JSONObject params = JSONObject.parseObject(JSON.toJSONString(question));
JSONObject params = JSONObject.parseObject(JSON.toJSONString(question, SerializerFeature.WriteMapNullValue));
params.put("q_id", UUID.randomUUID().toString());
RequestClientUtil.postCall(gglmUrl + bigModelInvokeApi.getAssistantKnowledgeAsk(), params);
sessionRedisManager.addSessionId(question.getChatId());
@ -122,7 +123,7 @@ public class HznlmInvokeServiceImpl implements IHznlmInvokeService {
@Override
public void analyseAsk(QuestionDTO question) {
JSONObject params = JSONObject.parseObject(JSON.toJSONString(question));
JSONObject params = JSONObject.parseObject(JSON.toJSONString(question, SerializerFeature.WriteMapNullValue));
params.put("q_id", UUID.randomUUID().toString());
RequestClientUtil.postCall(gglmUrl + bigModelInvokeApi.getAssistantAnalyseAsk(), params);
sessionRedisManager.addSessionId(question.getChatId());

7
hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/schedule/FrontEndInteractiveSchedule.java
Unescape View File

@ -2,6 +2,7 @@ package com.hnac.gglm.bigmodel.schedule;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.hnac.gglm.bigmodel.interactive.factory.AnswerResolveFactory;
import com.hnac.gglm.bigmodel.interactive.service.IAnswerResolveService;
import com.hnac.gglm.bigmodel.interactive.service.IHznlmInvokeService;
@ -62,18 +63,18 @@ public class FrontEndInteractiveSchedule {
Object[] resolveExtras = Arrays.stream(extras).map(extra -> {
IAnswerResolveService answerResolveService = AnswerResolveFactory.getResolveService(extra);
if (Func.isNotEmpty(answerResolveService)) {
JSONObject extraObject = JSONObject.parseObject(JSON.toJSONString(extra));
JSONObject extraObject = JSONObject.parseObject(JSON.toJSONString(extra, SerializerFeature.WriteMapNullValue));
return answerResolveService.getExtra(extraObject);
}
return extra;
}).map(JSON::toJSONString).toArray();
}).map(extra -> JSON.toJSONString(extra,SerializerFeature.WriteMapNullValue)).toArray();
answerVO.setExtras(resolveExtras);
} catch(Exception e) {
log.error("An error occurred",e);
answerVO = AnswerVO.error(answerVO.getChatId(), answerVO.getUserId(), answerVO.getQuery());
}
}
TextMessage message = InteractiveSessionManager.getTextMessage("1",JSON.toJSONString(answerVO));
TextMessage message = InteractiveSessionManager.getTextMessage("1",JSON.toJSONString(answerVO,SerializerFeature.WriteMapNullValue));
wsService.sendMessage(answerVO.getChatId(),message);
}
}

9
hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/websocket/server/InteractiveWsServer.java
Unescape View File

@ -1,9 +0,0 @@
package com.hnac.gglm.bigmodel.websocket.server;
/**
* @Author: huangxing
* @Date: 2024/04/28 13:38
*/
public class InteractiveWsServer {
}
Write Preview
Loading…
Cancel
Save