From eb73cc5cc9954e51e6d4d2e3384e8511fc497a61 Mon Sep 17 00:00:00 2001 From: haungxing <1203316822@qq.com> Date: Wed, 25 Sep 2024 14:07:45 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=90=91=E9=87=8F=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E6=95=B0=E6=8D=AE=E5=90=8C=E6=AD=A5=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gglm/bigmodel/api/feign/WeaviatesClient.java | 2 +- .../service/impl/VectorDataServiceImpl.java | 2 +- .../database/controller/WeaviateController.java | 2 +- .../bigmodel/database/service/WeaviateService.java | 86 +++++++++------------- .../bigmodel/maintenance/dto/DeviceAttrDTO.java | 3 + .../bigmodel/maintenance/dto/DeviceFuncDTO.java | 3 + .../maintenance/service/DataInstructService.java | 7 ++ .../maintenance/service/DataRecordService.java | 29 ++------ 8 files changed, 58 insertions(+), 76 deletions(-) diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/api/feign/WeaviatesClient.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/api/feign/WeaviatesClient.java index bacaf79..2142405 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/api/feign/WeaviatesClient.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/api/feign/WeaviatesClient.java @@ -32,6 +32,6 @@ public class WeaviatesClient implements IWeaviatesClient { @Override @PostMapping(SAVE_BATCH) public R saveBatch(@RequestBody WeaviateInsertDTO insertDTO) { - return R.status(weaviateService.saveBatch(insertDTO.getEntities(), insertDTO.getClassName(), insertDTO.getAttrsMap())); + return R.data(weaviateService.saveBatch(insertDTO.getEntities(), insertDTO.getClassName(), insertDTO.getAttrsMap())); } } diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/business/service/impl/VectorDataServiceImpl.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/business/service/impl/VectorDataServiceImpl.java index abb4d19..ea1f6e6 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/business/service/impl/VectorDataServiceImpl.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/business/service/impl/VectorDataServiceImpl.java @@ -84,7 +84,7 @@ public class VectorDataServiceImpl implements VectorDataService { if (null != rule) { item.setRule(rule); } - DeviceAttrDTO deviceAttrDTO = BeanUtil.copy(item, DeviceAttrDTO.class); + DeviceAttrDTO deviceAttrDTO = new DeviceAttrDTO(); deviceAttrDTO.setDeviceId(item.getDeviceId()); deviceAttrDTO.setDeviceName(item.getDeviceName()); deviceAttrDTO.setStationName(item.getProjectName()); diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java index 108a655..224514f 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java @@ -22,7 +22,7 @@ public class WeaviateController { private final WeaviateService weaviateService; @PostMapping("/saveBatch") - public R saveBatch(@RequestBody WeaviateSaveDTO req) { + public R saveBatch(@RequestBody WeaviateSaveDTO req) { weaviateService.saveBatch(req.getEntities(), req.getClassName(), req.getAttrsMap()); return R.success("操作成功！"); } diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java index 623bc47..d419337 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java @@ -5,6 +5,7 @@ import cn.hutool.http.HttpResponse; import cn.hutool.json.JSONUtil; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.TypeReference; import com.alibaba.fastjson.serializer.SerializerFeature; import com.google.common.collect.Lists; import com.hnac.gglm.bigmodel.BigModelConstants; @@ -79,53 +80,8 @@ public class WeaviateService { return !result.hasErrors(); } -// /** -// * 对象批量保存向量数据库 -// * @param entities 保存对象列表 -// * @param className 保存表名 -// * @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔 -// * @return 保存操作结果 -// */ -// public Boolean saveBatch(List entities,String className, Map attrsMap) { -// entities = entities.subList(0, 1); -// ObjectCreator creator = weaviateClient.data().creator().withClassName(className); -// List vectorStrs = Lists.newArrayList(); -// List attrs = Lists.newArrayList(); -// if(Func.isNotEmpty(attrsMap)) { -// // 格式化数据 -// attrsMap.forEach((k,v) -> attrs.add(v)); -// // 解析待计算的向量字段 -// entities.forEach(entity -> { -// List vectorStr = attrs.stream().map(fields -> this.getFieldValue(fields, entity)).filter(Func::isNotEmpty).collect(Collectors.toList()); -// vectorStrs.addAll(vectorStr); -// }); -// } -// if(Func.isNotEmpty(vectorStrs)) { -// // 若解析出来的向量存在值 -// Float[] vectors = this.compute(vectorStrs); -// List> vector = this.splitVector(entities.size(), attrsMap, vectors); -// for(int i = 0; i < entities.size(); i++) { -// // log.info("vector:{}",JSON.toJSONString(vector.get(i))); -// Map properties = this.objectToMap(entities.get(i)); -// log.info("properties:{}",JSON.toJSONString(properties)); -// Result run = creator.withProperties(properties).withVectors(vector.get(i)).run(); -// if(run.hasErrors()) { -// log.error("保存失败！，保存结果为：{}",JSON.toJSONString(run)); -// } -// } -// } else { -// entities.forEach(entity -> creator.withProperties(this.objectToMap(entity)).run()); -// return true; -// } -// return false; -// } - - public Boolean saveBatch(List entities,String className, Map attrsMap) { - Optional modelId = entities.stream().filter(entity -> { - JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity)); - return "1442295246932828161".equals(jsonObject.getString("modelId")); - }).findFirst(); - // 查询Weaviate 4.7.0表是否存在 若不存则新建表 + public String saveBatch(List entities,String className, Map attrsMap) { + // 查询表是否存在 若不存则新建表 Result existResult = weaviateClient.schema().exists().withClassName(BigModelConstants.PREFIX + "_" + className).run(); if(existResult.hasErrors() || !existResult.getResult()) { Map createTableParams = new HashMap<>(2); @@ -140,22 +96,48 @@ public class WeaviateService { } Map params = new HashMap<>(2); params.put("table_name", className); + // 将entities按size截断为1000个一组 + List entitiesList = splitList(entities, 1000); + int total = 0; + for (List entityList : entitiesList) { + Integer insert = this.insert(entityList, className, attrsMap, params); + total += insert; + } + // 查询weaviate 中该表的数据量 + return String.format("传入数据总量为：%s 保存成功数量为：%s", entities.size(), total); + } + + /** + * 将list按size截断为多个list + * @param list 待截断的list + * @param size 截断大小 + * @return + * @param + */ + public static List> splitList(List list, int size) { + List> parts = new ArrayList<>(); + for (int i = 0; i < list.size(); i += size) { + parts.add(list.subList(i, Math.min(list.size(), i + size))); + } + return parts; + } + + private Integer insert(List entities,String className, Map attrsMap, Map params) { List> data = new ArrayList<>(); entities.forEach(entity -> { // 将entity转换为Map JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity, SerializerFeature.WriteMapNullValue)); Map map = new HashMap<>(); - jsonObject.forEach((k,v) -> map.put(k,Optional.ofNullable(jsonObject.getString(k)).orElse(""))); data.add(this.getVectorData(map,attrsMap)); + jsonObject.forEach((k,v) -> map.put(k,jsonObject.getString(k))); }); - log.info("data:{}",JSON.toJSONString(data)); params.put("data",data); String url = gglmUrl + invokeApi.getInsertVectors(); - RequestClientUtil.postCall(url,params); - return true; + Map stringIntegerMap = RequestClientUtil.postCall(url, params, new TypeReference>() { + }); + return (Integer) stringIntegerMap.get("total"); } - private Map getVectorData(Map entity,Map attrsMap) { Map result = new HashMap<>(2); result.put("object", entity); diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceAttrDTO.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceAttrDTO.java index 4330048..7057a1a 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceAttrDTO.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceAttrDTO.java @@ -39,4 +39,7 @@ public class DeviceAttrDTO { @ApiModelProperty("取数规则") @JsonProperty("rule") private String rule; + + @ApiModelProperty("数据类型") + private String type = "平台"; } diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceFuncDTO.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceFuncDTO.java index dedfabe..1df8b54 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceFuncDTO.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceFuncDTO.java @@ -36,4 +36,7 @@ public class DeviceFuncDTO { @JsonProperty("yk_name") private String ykName; + @ApiModelProperty("遥控名称") + private String type = "平台"; + } diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataInstructService.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataInstructService.java index 76fb1cc..f12b37a 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataInstructService.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataInstructService.java @@ -3,6 +3,7 @@ package com.hnac.gglm.bigmodel.maintenance.service; import com.baomidou.dynamic.datasource.annotation.DS; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.google.common.collect.Lists; import com.hnac.gglm.bigmodel.business.service.VectorDataService; import com.hnac.gglm.bigmodel.maintenance.dto.DeviceFuncDTO; import com.hnac.gglm.bigmodel.maintenance.entity.DataInstructEntity; @@ -12,6 +13,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springblade.core.mp.base.BaseServiceImpl; import org.springblade.core.tool.utils.BeanUtil; +import org.springblade.core.tool.utils.Func; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -52,8 +54,13 @@ public class DataInstructService extends BaseServiceImpl getDataInstructList() { + List result = Lists.newArrayList(); List list = baseMapper.selectList(null); List deviceFuncDTOS = BeanUtil.copyProperties(list, DeviceFuncDTO.class); + if(Func.isNotEmpty(deviceFuncDTOS)) { + deviceFuncDTOS.forEach(deviceFuncDTO -> deviceFuncDTO.setType("业务")); + result.addAll(deviceFuncDTOS); + } List deviceFuncList = vectorDataService.getDeviceFunc(); if (deviceFuncList != null) { for (DeviceFuncInfoDTO deviceFunc : deviceFuncList) { diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataRecordService.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataRecordService.java index 74a26bb..66eaac9 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataRecordService.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataRecordService.java @@ -3,6 +3,7 @@ package com.hnac.gglm.bigmodel.maintenance.service; import com.baomidou.dynamic.datasource.annotation.DS; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.google.common.collect.Lists; import com.hnac.gglm.bigmodel.business.service.VectorDataService; import com.hnac.gglm.bigmodel.maintenance.dto.DeviceAttrDTO; import com.hnac.gglm.bigmodel.maintenance.dto.DeviceFuncDTO; @@ -80,31 +81,17 @@ public class DataRecordService extends BaseServiceImpl getDataRecordList() { + List result = Lists.newArrayList(); List list = baseMapper.selectList(null); List deviceAttrDTOS = BeanUtil.copyProperties(list, DeviceAttrDTO.class); + if(Func.isNotEmpty(deviceAttrDTOS)) { + deviceAttrDTOS.forEach(deviceAttrDTO -> deviceAttrDTO.setType("业务")); + result.addAll(deviceAttrDTOS); + } List deviceAttrList = vectorDataService.getDeviceAttr(); if(Func.isNotEmpty(deviceAttrList)) { - deviceAttrDTOS.addAll(deviceAttrList); + result.addAll(deviceAttrList); } -// if (deviceAttrList != null) { -// for (DeviceAttrInfoDTO deviceAttr : deviceAttrList) { -// DeviceAttrDTO deviceAttrDTO = new DeviceAttrDTO(); -// deviceAttrDTO.setDeviceId(deviceAttr.getDeviceId()); -// deviceAttrDTO.setDeviceName(deviceAttr.getDeviceName()); -// deviceAttrDTO.setStationName(deviceAttr.getProjectName()); -// deviceAttrDTO.setStationId(deviceAttr.getProjectId()); -// deviceAttrDTO.setRecordName(deviceAttr.getName()); -// deviceAttrDTO.setItemId(deviceAttr.getSignage()); -// String stationName = deviceAttrDTO.getStationName() == null ? "" : deviceAttrDTO.getStationName(); -// String deviceName = deviceAttrDTO.getDeviceName() == null ? "" : deviceAttrDTO.getDeviceName(); -// String recordName = deviceAttrDTO.getRecordName() == null ? "" : deviceAttrDTO.getRecordName(); -// deviceAttrDTO.setItemName(new StringBuffer(stationName) -// .append(" ").append(deviceName) -// .append(" ").append(recordName).toString()); -// deviceAttrDTO.setRule(deviceAttr.getRule()); -// deviceAttrDTOS.add(deviceAttrDTO); -// } -// } - return deviceAttrDTOS; + return result; } }