fix: 向量数据库数据同步接口优化

hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/api/feign/WeaviatesClient.java

@@ -32,6 +32,6 @@ public class WeaviatesClient implements IWeaviatesClient {
     @Override
     @PostMapping(SAVE_BATCH)
     public R saveBatch(@RequestBody WeaviateInsertDTO insertDTO) {
-        return R.status(weaviateService.saveBatch(insertDTO.getEntities(), insertDTO.getClassName(), insertDTO.getAttrsMap()));
+        return R.data(weaviateService.saveBatch(insertDTO.getEntities(), insertDTO.getClassName(), insertDTO.getAttrsMap()));
     }
 }

hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/business/service/impl/VectorDataServiceImpl.java

@@ -84,7 +84,7 @@ public class VectorDataServiceImpl implements VectorDataService {
                 if (null != rule) {
                     item.setRule(rule);
                 }
-                DeviceAttrDTO deviceAttrDTO = BeanUtil.copy(item, DeviceAttrDTO.class);
+                DeviceAttrDTO deviceAttrDTO = new DeviceAttrDTO();
                 deviceAttrDTO.setDeviceId(item.getDeviceId());
                 deviceAttrDTO.setDeviceName(item.getDeviceName());
                 deviceAttrDTO.setStationName(item.getProjectName());

hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/controller/WeaviateController.java

@@ -22,7 +22,7 @@ public class WeaviateController {
     private final WeaviateService weaviateService;
 
     @PostMapping("/saveBatch")
-    public R<Boolean> saveBatch(@RequestBody WeaviateSaveDTO req) {
+    public R<String> saveBatch(@RequestBody WeaviateSaveDTO req) {
         weaviateService.saveBatch(req.getEntities(), req.getClassName(), req.getAttrsMap());
         return R.success("操作成功！");
     }

hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/database/service/WeaviateService.java

@@ -5,6 +5,7 @@ import cn.hutool.http.HttpResponse;
 import cn.hutool.json.JSONUtil;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
 import com.alibaba.fastjson.serializer.SerializerFeature;
 import com.google.common.collect.Lists;
 import com.hnac.gglm.bigmodel.BigModelConstants;
@@ -79,53 +80,8 @@ public class WeaviateService {
         return !result.hasErrors();
     }
 
-//    /**
+    public String saveBatch(List entities,String className, Map<String,String> attrsMap) {
-//     * 对象批量保存向量数据库
+        // 查询表是否存在 若不存则新建表
-//     * @param entities 保存对象列表
-//     * @param className 保存表名
-//     * @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔
-//     * @return 保存操作结果
-//     */
-//    public Boolean saveBatch(List entities,String className, Map<String,String> attrsMap) {
-//        entities = entities.subList(0, 1);
-//        ObjectCreator creator = weaviateClient.data().creator().withClassName(className);
-//        List<String> vectorStrs = Lists.newArrayList();
-//        List<String> attrs = Lists.newArrayList();
-//        if(Func.isNotEmpty(attrsMap)) {
-//            // 格式化数据
-//            attrsMap.forEach((k,v) -> attrs.add(v));
-//            // 解析待计算的向量字段
-//            entities.forEach(entity -> {
-//                List<String> vectorStr = attrs.stream().map(fields -> this.getFieldValue(fields, entity)).filter(Func::isNotEmpty).collect(Collectors.toList());
-//                vectorStrs.addAll(vectorStr);
-//            });
-//        }
-//        if(Func.isNotEmpty(vectorStrs)) {
-//            // 若解析出来的向量存在值
-//            Float[] vectors = this.compute(vectorStrs);
-//            List<Map<String, Float[]>> vector = this.splitVector(entities.size(), attrsMap, vectors);
-//            for(int i = 0; i < entities.size(); i++) {
-//                // log.info("vector:{}",JSON.toJSONString(vector.get(i)));
-//                Map<String, Object> properties = this.objectToMap(entities.get(i));
-//                log.info("properties:{}",JSON.toJSONString(properties));
-//                Result<WeaviateObject> run = creator.withProperties(properties).withVectors(vector.get(i)).run();
-//                if(run.hasErrors()) {
-//                    log.error("保存失败！，保存结果为：{}",JSON.toJSONString(run));
-//                }
-//            }
-//        } else {
-//            entities.forEach(entity -> creator.withProperties(this.objectToMap(entity)).run());
-//            return true;
-//        }
-//        return false;
-//    }
-
-    public Boolean saveBatch(List entities,String className, Map<String,String> attrsMap) {
-        Optional modelId = entities.stream().filter(entity -> {
-            JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity));
-            return "1442295246932828161".equals(jsonObject.getString("modelId"));
-        }).findFirst();
-        // 查询Weaviate 4.7.0表是否存在 若不存则新建表
         Result<Boolean> existResult = weaviateClient.schema().exists().withClassName(BigModelConstants.PREFIX + "_" + className).run();
         if(existResult.hasErrors() || !existResult.getResult()) {
             Map<java.lang.String,Object> createTableParams = new HashMap<>(2);
@@ -140,22 +96,48 @@ public class WeaviateService {
         }
         Map<String,Object> params = new HashMap<>(2);
         params.put("table_name", className);
+        // 将entities按size截断为1000个一组
+        List<List> entitiesList = splitList(entities, 1000);
+        int total = 0;
+        for (List entityList : entitiesList) {
+            Integer insert = this.insert(entityList, className, attrsMap, params);
+            total += insert;
+        }
+        // 查询weaviate 中该表的数据量
+        return String.format("传入数据总量为：%s 保存成功数量为：%s", entities.size(), total);
+    }
+
+    /**
+     * 将list按size截断为多个list
+     * @param list 待截断的list
+     * @param size 截断大小
+     * @return
+     * @param <T>
+     */
+    public static <T> List<List<T>> splitList(List<T> list, int size) {
+        List<List<T>> parts = new ArrayList<>();
+        for (int i = 0; i < list.size(); i += size) {
+            parts.add(list.subList(i, Math.min(list.size(), i + size)));
+        }
+        return parts;
+    }
+
+    private Integer insert(List entities,String className, Map<String,String> attrsMap, Map<String,Object> params) {
         List<Map<String, Object>> data = new ArrayList<>();
         entities.forEach(entity -> {
             // 将entity转换为Map<String,String>
             JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity, SerializerFeature.WriteMapNullValue));
             Map<String,String> map = new HashMap<>();
-            jsonObject.forEach((k,v) -> map.put(k,Optional.ofNullable(jsonObject.getString(k)).orElse("")));
             data.add(this.getVectorData(map,attrsMap));
+            jsonObject.forEach((k,v) -> map.put(k,jsonObject.getString(k)));
         });
-        log.info("data:{}",JSON.toJSONString(data));
         params.put("data",data);
         String url = gglmUrl + invokeApi.getInsertVectors();
-        RequestClientUtil.postCall(url,params);
+        Map<String, Object> stringIntegerMap = RequestClientUtil.postCall(url, params, new TypeReference<Map<String, Object>>() {
-        return true;
+        });
+        return (Integer) stringIntegerMap.get("total");
     }
 
-
     private Map<String,Object> getVectorData(Map<String,String> entity,Map<String,String> attrsMap) {
         Map<String,Object> result = new HashMap<>(2);
         result.put("object", entity);

hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceAttrDTO.java

@@ -39,4 +39,7 @@ public class DeviceAttrDTO {
     @ApiModelProperty("取数规则")
     @JsonProperty("rule")
     private String rule;
+
+    @ApiModelProperty("数据类型")
+    private String type = "平台";
 }

hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/dto/DeviceFuncDTO.java

@@ -36,4 +36,7 @@ public class DeviceFuncDTO {
     @JsonProperty("yk_name")
     private String ykName;
 
+    @ApiModelProperty("遥控名称")
+    private String type = "平台";
+
 }

hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataInstructService.java

@@ -3,6 +3,7 @@ package com.hnac.gglm.bigmodel.maintenance.service;
 import com.baomidou.dynamic.datasource.annotation.DS;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.google.common.collect.Lists;
 import com.hnac.gglm.bigmodel.business.service.VectorDataService;
 import com.hnac.gglm.bigmodel.maintenance.dto.DeviceFuncDTO;
 import com.hnac.gglm.bigmodel.maintenance.entity.DataInstructEntity;
@@ -12,6 +13,7 @@ import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springblade.core.mp.base.BaseServiceImpl;
 import org.springblade.core.tool.utils.BeanUtil;
+import org.springblade.core.tool.utils.Func;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
@@ -52,8 +54,13 @@ public class DataInstructService extends BaseServiceImpl<DataInstructMapper, Dat
     }
 
     public List<DeviceFuncDTO> getDataInstructList() {
+        List<DeviceFuncDTO> result = Lists.newArrayList();
         List<DataInstructEntity> list = baseMapper.selectList(null);
         List<DeviceFuncDTO> deviceFuncDTOS = BeanUtil.copyProperties(list, DeviceFuncDTO.class);
+        if(Func.isNotEmpty(deviceFuncDTOS)) {
+            deviceFuncDTOS.forEach(deviceFuncDTO -> deviceFuncDTO.setType("业务"));
+            result.addAll(deviceFuncDTOS);
+        }
         List<DeviceFuncInfoDTO> deviceFuncList = vectorDataService.getDeviceFunc();
         if (deviceFuncList != null) {
             for (DeviceFuncInfoDTO deviceFunc : deviceFuncList) {

hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/DataRecordService.java

@@ -3,6 +3,7 @@ package com.hnac.gglm.bigmodel.maintenance.service;
 import com.baomidou.dynamic.datasource.annotation.DS;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.google.common.collect.Lists;
 import com.hnac.gglm.bigmodel.business.service.VectorDataService;
 import com.hnac.gglm.bigmodel.maintenance.dto.DeviceAttrDTO;
 import com.hnac.gglm.bigmodel.maintenance.dto.DeviceFuncDTO;
@@ -80,31 +81,17 @@ public class DataRecordService extends BaseServiceImpl<DataRecordMapper, DataRec
     }
 
     public List<DeviceAttrDTO> getDataRecordList() {
+        List<DeviceAttrDTO> result = Lists.newArrayList();
         List<DataRecordEntity> list = baseMapper.selectList(null);
         List<DeviceAttrDTO> deviceAttrDTOS = BeanUtil.copyProperties(list, DeviceAttrDTO.class);
+        if(Func.isNotEmpty(deviceAttrDTOS)) {
+            deviceAttrDTOS.forEach(deviceAttrDTO -> deviceAttrDTO.setType("业务"));
+            result.addAll(deviceAttrDTOS);
+        }
         List<DeviceAttrDTO> deviceAttrList = vectorDataService.getDeviceAttr();
         if(Func.isNotEmpty(deviceAttrList)) {
-            deviceAttrDTOS.addAll(deviceAttrList);
+            result.addAll(deviceAttrList);
         }
-//        if (deviceAttrList != null) {
+        return result;
-//            for (DeviceAttrInfoDTO deviceAttr : deviceAttrList) {
-//                DeviceAttrDTO deviceAttrDTO = new DeviceAttrDTO();
-//                deviceAttrDTO.setDeviceId(deviceAttr.getDeviceId());
-//                deviceAttrDTO.setDeviceName(deviceAttr.getDeviceName());
-//                deviceAttrDTO.setStationName(deviceAttr.getProjectName());
-//                deviceAttrDTO.setStationId(deviceAttr.getProjectId());
-//                deviceAttrDTO.setRecordName(deviceAttr.getName());
-//                deviceAttrDTO.setItemId(deviceAttr.getSignage());
-//                String stationName = deviceAttrDTO.getStationName() == null ? "" : deviceAttrDTO.getStationName();
-//                String deviceName = deviceAttrDTO.getDeviceName() == null ? "" : deviceAttrDTO.getDeviceName();
-//                String recordName = deviceAttrDTO.getRecordName() == null ? "" : deviceAttrDTO.getRecordName();
-//                deviceAttrDTO.setItemName(new StringBuffer(stationName)
-//                        .append(" ").append(deviceName)
-//                        .append(" ").append(recordName).toString());
-//                deviceAttrDTO.setRule(deviceAttr.getRule());
-//                deviceAttrDTOS.add(deviceAttrDTO);
-//            }
-//        }
-        return deviceAttrDTOS;
     }
 }