From e7b3676abc943827dd0f2ec90612853305ae545c Mon Sep 17 00:00:00 2001 From: luyie Date: Mon, 23 Sep 2024 12:04:37 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E5=90=91=E9=87=8F=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=90=8C=E6=AD=A5=E6=97=B6=EF=BC=8C=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E4=B8=8D=E5=88=B7=E6=96=B0=E6=9B=B4=E6=96=B0=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../maintenance/mapper/VectorParamMapper.java | 3 + .../maintenance/service/VectorParamService.java | 2 + .../service/impl/VectorParamServiceImpl.java | 66 ++++++++++++++-------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/mapper/VectorParamMapper.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/mapper/VectorParamMapper.java index 6dc3e63..9fbfb79 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/mapper/VectorParamMapper.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/mapper/VectorParamMapper.java @@ -1,5 +1,6 @@ package com.hnac.gglm.bigmodel.maintenance.mapper; +import com.baomidou.dynamic.datasource.annotation.DS; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.hnac.gglm.bigmodel.maintenance.entity.VectorParamEntity; @@ -7,5 +8,7 @@ import com.hnac.gglm.bigmodel.maintenance.entity.VectorParamEntity; * @Author: ypj * @Date: 2024/09/02 15:12 */ + +@DS("hznlm") public interface VectorParamMapper extends BaseMapper { } diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/VectorParamService.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/VectorParamService.java index 8553964..760447c 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/VectorParamService.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/VectorParamService.java @@ -19,4 +19,6 @@ public interface VectorParamService extends IService { List getUrlResponseKeyList(String url, String bladeToken, String hzinfoToken); Boolean synchronization(List id ,String bladeToken, String hzinfoToken); + + Boolean synchronization(Long id, String bladeToken, String hzinfoToken); } diff --git a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java index 86e4921..bc836a2 100644 --- a/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java +++ b/hzims-service/gglm-big-model/src/main/java/com/hnac/gglm/bigmodel/maintenance/service/impl/VectorParamServiceImpl.java @@ -12,10 +12,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.hnac.gglm.bigmodel.database.service.WeaviateService; -import com.hnac.gglm.bigmodel.maintenance.vo.VectorUrlResponse; import com.hnac.gglm.bigmodel.maintenance.entity.VectorParamEntity; import com.hnac.gglm.bigmodel.maintenance.mapper.VectorParamMapper; import com.hnac.gglm.bigmodel.maintenance.service.VectorParamService; +import com.hnac.gglm.bigmodel.maintenance.vo.VectorUrlResponse; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -23,6 +23,9 @@ import org.springframework.stereotype.Service; import javax.servlet.http.HttpServletResponse; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * @Author: ypj @@ -115,32 +118,47 @@ public class VectorParamServiceImpl extends ServiceImpl attrMap = new HashMap<>(); - Iterator iterator = rootNode.fieldNames(); - while (iterator.hasNext()) { - String key = iterator.next(); - attrMap.put(key, rootNode.findValue(key).textValue()); + ThreadPoolExecutor executor = new ThreadPoolExecutor(5, ids.size() > 5 ? ids.size() : 5, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue()); + ids.forEach(id -> { + executor.execute(() -> { + if (synchronization(id, bladeToken, hzinfoToken)) { + log.info("synchronization vector param id:{} success", id); + } else { + log.error("synchronization vector param id:{} fail", id); } - String tableName = entity.getTableName().replace(entity.getProjectPrefix() + "_",""); - weaviateService.saveBatch(response.getOriginalData(), tableName, attrMap); - this.update(Wrappers.lambdaUpdate().eq(VectorParamEntity::getId, id).set(VectorParamEntity::getUpdateTime, new Date())); + }); + }); + executor.shutdown(); + return true; + } + + + @Override + public Boolean synchronization(Long id, String bladeToken, String hzinfoToken) { + VectorParamEntity entity = baseMapper.selectById(id); + if (null == entity) { + return false; + } + log.info("synchronization vector param id:{}", id); + VectorUrlResponse response = getUrlResponse(entity.getUrl(), bladeToken, hzinfoToken); + if (response != null) { + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = null; + try { + rootNode = mapper.readTree(entity.getAttributeMap()); + } catch (JsonProcessingException e) { + log.error("getUrlResponseKeyList error", e); + } + Map attrMap = new HashMap<>(); + Iterator iterator = rootNode.fieldNames(); + while (iterator.hasNext()) { + String key = iterator.next(); + attrMap.put(key, rootNode.findValue(key).textValue()); } + String tableName = entity.getTableName().replace(entity.getProjectPrefix() + "_", ""); + weaviateService.saveBatch(response.getOriginalData(), tableName, attrMap); + return this.update(Wrappers.lambdaUpdate().eq(VectorParamEntity::getId, id).set(VectorParamEntity::getUpdateTime, new Date())); } return true; } - }