32 changed files with 693 additions and 159 deletions
@ -0,0 +1,21 @@
|
||||
package com.hnac.hzims.bigmodel.api.wrapper; |
||||
|
||||
import com.hnac.hzims.bigmodel.api.dto.SyncDTO; |
||||
|
||||
import java.util.List; |
||||
import java.util.stream.Collectors; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/20 09:23 |
||||
* @Describe 对象转换为同步对象基础类 |
||||
*/ |
||||
public abstract class BaseSyncWrapper<O,S extends SyncDTO> { |
||||
|
||||
public abstract S convert(O obj); |
||||
|
||||
public List<S> listVO(List<O> list) { |
||||
return list.stream().map(this::convert).collect(Collectors.toList()); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,23 @@
|
||||
package com.hnac.hzims.bigmodel.database.entity; |
||||
|
||||
import com.alibaba.fastjson.annotation.JSONField; |
||||
import lombok.Data; |
||||
import lombok.EqualsAndHashCode; |
||||
|
||||
import java.io.Serializable; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/22 19:35 |
||||
*/ |
||||
@Data |
||||
@EqualsAndHashCode |
||||
public class WeaviateEntity implements Serializable { |
||||
|
||||
@JSONField(name = "item_id") |
||||
private String itemId; |
||||
|
||||
@JSONField(name = "item_name") |
||||
private String itemName; |
||||
|
||||
} |
||||
@ -1,4 +1,4 @@
|
||||
package com.hnac.hzims.bigmodel.business.control; |
||||
package com.hnac.hzims.bigmodel.business.controller; |
||||
|
||||
import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; |
||||
import com.hnac.hzims.bigmodel.business.service.DataSourceService; |
||||
@ -0,0 +1,21 @@
|
||||
package com.hnac.hzims.bigmodel.business.wrapper; |
||||
|
||||
import com.hnac.hzims.bigmodel.api.dto.CanvasSyncDTO; |
||||
import com.hnac.hzims.bigmodel.api.wrapper.BaseSyncWrapper; |
||||
import com.hnac.hzinfo.sdk.v5.scada.vo.CanvasVO; |
||||
import lombok.AllArgsConstructor; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/20 16:49 |
||||
*/ |
||||
@Component |
||||
@AllArgsConstructor |
||||
public class CanvasSyncWrapper extends BaseSyncWrapper<CanvasVO, CanvasSyncDTO> { |
||||
|
||||
@Override |
||||
public CanvasSyncDTO convert(CanvasVO obj) { |
||||
return null; |
||||
} |
||||
} |
||||
@ -0,0 +1,56 @@
|
||||
package com.hnac.hzims.bigmodel.configuration; |
||||
|
||||
import io.weaviate.client.Config; |
||||
import io.weaviate.client.WeaviateAuthClient; |
||||
import io.weaviate.client.WeaviateClient; |
||||
import io.weaviate.client.v1.auth.exception.AuthException; |
||||
import io.weaviate.client.v1.data.api.ObjectCreator; |
||||
import io.weaviate.client.v1.data.api.ObjectDeleter; |
||||
import io.weaviate.client.v1.data.api.ObjectUpdater; |
||||
import io.weaviate.client.v1.data.api.ObjectsGetter; |
||||
import org.springframework.context.annotation.Bean; |
||||
import org.springframework.context.annotation.Configuration; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/22 18:38 |
||||
*/ |
||||
@Configuration |
||||
public class WeaviateConfigure { |
||||
|
||||
private final WeaviateProperties weaviateProperties; |
||||
|
||||
public WeaviateConfigure(WeaviateProperties weaviateProperties) { |
||||
this.weaviateProperties = weaviateProperties; |
||||
} |
||||
|
||||
@Bean |
||||
public WeaviateClient weaviateClient() throws AuthException { |
||||
Config config = new Config(this.weaviateProperties.getSchema(), this.weaviateProperties.getHost() + ":" + this.weaviateProperties.getPort()); |
||||
return WeaviateAuthClient.apiKey(config,this.weaviateProperties.getApiKey()); |
||||
} |
||||
|
||||
@Bean |
||||
public ObjectsGetter objectsGetter() throws AuthException { |
||||
WeaviateClient weaviateClient = weaviateClient(); |
||||
return weaviateClient.data().objectsGetter(); |
||||
} |
||||
|
||||
@Bean |
||||
public ObjectCreator objectCreator() throws AuthException { |
||||
WeaviateClient weaviateClient = weaviateClient(); |
||||
return weaviateClient.data().creator(); |
||||
} |
||||
|
||||
@Bean |
||||
public ObjectDeleter deleter() throws AuthException { |
||||
WeaviateClient weaviateClient = weaviateClient(); |
||||
return weaviateClient.data().deleter(); |
||||
} |
||||
|
||||
@Bean |
||||
public ObjectUpdater updater() throws AuthException { |
||||
WeaviateClient weaviateClient = weaviateClient(); |
||||
return weaviateClient.data().updater(); |
||||
} |
||||
} |
||||
@ -0,0 +1,32 @@
|
||||
package com.hnac.hzims.bigmodel.configuration; |
||||
|
||||
import lombok.Data; |
||||
import org.springframework.boot.context.properties.ConfigurationProperties; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/21 15:22 |
||||
*/ |
||||
@Data |
||||
@Component |
||||
@ConfigurationProperties(prefix = "weaviate.datasource") |
||||
public class WeaviateProperties { |
||||
|
||||
private String schema; |
||||
|
||||
private String host; |
||||
|
||||
private String port; |
||||
|
||||
/** |
||||
* 登录认证KEY |
||||
*/ |
||||
private String apiKey; |
||||
|
||||
/** |
||||
* 数据库表名前缀 |
||||
*/ |
||||
private String classNamePrefix; |
||||
|
||||
} |
||||
@ -0,0 +1,252 @@
|
||||
package com.hnac.hzims.bigmodel.database.service; |
||||
|
||||
import cn.hutool.http.HttpRequest; |
||||
import cn.hutool.http.HttpResponse; |
||||
import cn.hutool.json.JSONUtil; |
||||
import com.alibaba.fastjson.JSON; |
||||
import com.alibaba.fastjson.JSONObject; |
||||
import com.google.common.collect.Lists; |
||||
import com.hnac.hzims.bigmodel.configuration.BigModelInvokeApi; |
||||
import com.hnac.hzinfo.exception.HzServiceException; |
||||
import io.weaviate.client.base.Result; |
||||
import io.weaviate.client.v1.data.api.ObjectCreator; |
||||
import io.weaviate.client.v1.data.api.ObjectDeleter; |
||||
import io.weaviate.client.v1.data.api.ObjectUpdater; |
||||
import io.weaviate.client.v1.data.api.ObjectsGetter; |
||||
import io.weaviate.client.v1.data.model.WeaviateObject; |
||||
import lombok.RequiredArgsConstructor; |
||||
import lombok.extern.slf4j.Slf4j; |
||||
import org.springblade.core.tool.utils.BeanUtil; |
||||
import org.springblade.core.tool.utils.Func; |
||||
import org.springframework.beans.factory.annotation.Value; |
||||
import org.springframework.stereotype.Service; |
||||
|
||||
import java.lang.reflect.Field; |
||||
import java.nio.ByteBuffer; |
||||
import java.nio.ByteOrder; |
||||
import java.util.*; |
||||
import java.util.concurrent.atomic.AtomicInteger; |
||||
import java.util.stream.Collectors; |
||||
import java.util.stream.IntStream; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/22 19:17 |
||||
*/ |
||||
@RequiredArgsConstructor |
||||
@Service |
||||
@Slf4j |
||||
public class WeaviateService { |
||||
|
||||
private final ObjectCreator objectCreator; |
||||
private final ObjectUpdater objectUpdater; |
||||
private final ObjectDeleter objectDeleter; |
||||
private final ObjectsGetter objectsGetter; |
||||
private final BigModelInvokeApi invokeApi; |
||||
|
||||
@Value("${gglm.vectorUrl}") |
||||
private String vectorUrl; |
||||
|
||||
/** |
||||
* 对象保存向量数据库 |
||||
* @param entity 保存对象 |
||||
* @param className 保存表名 |
||||
* @param attrs 待计算的列信息 |
||||
* @return 保存操作结果 |
||||
*/ |
||||
public Boolean save(Object entity, String className, List<String> attrs) { |
||||
ObjectCreator creator = objectCreator.withClassName(className); |
||||
if(Func.isNotEmpty(attrs)) { |
||||
JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(entity)); |
||||
List<String> vectors = attrs.stream().map(attr -> jsonObject.getString(attr)).collect(Collectors.toList()); |
||||
Float[] compute = this.compute(vectors); |
||||
creator.withVector(compute); |
||||
} |
||||
Result<WeaviateObject> result = creator.withProperties(BeanUtil.toMap(entity)).run(); |
||||
return !result.hasErrors(); |
||||
} |
||||
|
||||
/** |
||||
* 对象批量保存向量数据库 |
||||
* @param entities 保存对象列表 |
||||
* @param className 保存表名 |
||||
* @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔 |
||||
* @return 保存操作结果 |
||||
*/ |
||||
public Boolean saveBatch(List entities,String className, Map<String,String> attrsMap) { |
||||
ObjectCreator creator = objectCreator.withClassName(className); |
||||
List<String> vectorStrs = Lists.newArrayList(); |
||||
List<String> attrs = Lists.newArrayList(); |
||||
if(Func.isNotEmpty(attrsMap)) { |
||||
// 格式化数据
|
||||
attrsMap.forEach((k,v) -> attrs.add(v)); |
||||
// 解析待计算的向量字段
|
||||
entities.forEach(entity -> { |
||||
List<String> vectorStr = attrs.stream().map(fields -> this.getFieldValue(fields, entity)).filter(Func::isNotEmpty).collect(Collectors.toList()); |
||||
vectorStrs.addAll(vectorStr); |
||||
}); |
||||
} |
||||
if(Func.isNotEmpty(vectorStrs)) { |
||||
// 若解析出来的向量存在值
|
||||
Float[] vectors = this.compute(vectorStrs); |
||||
List<Map<String, Float[]>> vector = this.splitVector(entities.size(), attrsMap, vectors); |
||||
for(int i = 0; i < entities.size(); i++) { |
||||
creator.withProperties(BeanUtil.toMap(entities.get(i))).withVectors(vector.get(i)).run(); |
||||
} |
||||
} else { |
||||
entities.forEach(entity -> creator.withProperties(BeanUtil.toMap(entity)).run()); |
||||
return true; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/** |
||||
* 删除向量数据库（表名） |
||||
* @param className 表名 |
||||
* @return 删除结果 |
||||
*/ |
||||
public Boolean deleteByClassName(String className) { |
||||
Result<Boolean> result = objectDeleter.withClassName(className).run(); |
||||
return !result.hasErrors(); |
||||
} |
||||
|
||||
/** |
||||
* 删除向量数据库（ID） |
||||
* @param id 向量数据库ID |
||||
* @return 删除结果 |
||||
*/ |
||||
public Boolean deleteById(String id) { |
||||
Result<Boolean> result = objectDeleter.withID(id).run(); |
||||
return !result.hasErrors(); |
||||
} |
||||
|
||||
/** |
||||
* 更新数据库（通过ID） |
||||
* @param id 向量数据库ID |
||||
* @return 更新结果 |
||||
*/ |
||||
public Boolean updateById(String id, Object entity, String className, Map<String,String> attrMap) { |
||||
ObjectUpdater updater = objectUpdater.withClassName(className).withID(id).withProperties(BeanUtil.toMap(entity)); |
||||
// 计算向量
|
||||
Map<String, Float[]> vector = new HashMap<>(); |
||||
if(Func.isNotEmpty(attrMap)) { |
||||
attrMap.forEach((k,v) -> { |
||||
String fieldValue = this.getFieldValue(v, entity); |
||||
Float[] compute = this.compute(Lists.newArrayList(fieldValue)); |
||||
vector.put(k,compute); |
||||
}); |
||||
} |
||||
if(Func.isNotEmpty(vector)) { |
||||
updater.withVectors(vector); |
||||
} |
||||
Result<Boolean> result = updater.run(); |
||||
return !result.hasErrors(); |
||||
} |
||||
|
||||
public List<Map> list(String id,String className) { |
||||
if(Func.isNotEmpty(id)) { |
||||
objectsGetter.withID(id); |
||||
} |
||||
if(Func.isNotEmpty(className)) { |
||||
objectsGetter.withClassName(className); |
||||
} |
||||
Result<List<WeaviateObject>> result = objectsGetter.run(); |
||||
if(result.hasErrors()) { |
||||
throw new HzServiceException("查询失败！"); |
||||
} |
||||
return result.getResult().stream().map(WeaviateObject::getProperties).collect(Collectors.toList()); |
||||
} |
||||
|
||||
/** |
||||
* 拆解计算出来的向量Float[] |
||||
* @param entitySize 对象列表size |
||||
* @param attrsMap 待计算的列信息 key-向量名 value-实体类对象属性,多个按逗号分隔 |
||||
* @param vectorTotal 计算出的向量总量 |
||||
* @return 拆解结果 |
||||
*/ |
||||
private List<Map<String,Float[]>> splitVector(Integer entitySize,Map<String,String> attrsMap,Float[] vectorTotal) { |
||||
List<Map<String,Float[]>> result = Lists.newArrayList(); |
||||
List<Float> vectorTotalList = Lists.newArrayList(vectorTotal); |
||||
// 获取待切割的下标
|
||||
List<Integer> indexes = this.getSplitIndex(vectorTotal.length, entitySize); |
||||
int step = vectorTotal.length / entitySize; |
||||
indexes.forEach(index -> { |
||||
List<Float> vectors = vectorTotalList.subList(index, index + step); |
||||
Map<String,Float[]> vectorMap = new HashMap<>(); |
||||
List<Integer> splitIndex = this.getSplitIndex(vectors.size(), attrsMap.size()); |
||||
AtomicInteger i = new AtomicInteger(); |
||||
attrsMap.forEach((k,v) -> { |
||||
List<Float> vector = vectors.subList(splitIndex.get(i.get()), splitIndex.get(i.get() + (vectors.size() / attrsMap.size()))); |
||||
vectorMap.put(k, vector.toArray(new Float[vector.size()])); |
||||
i.getAndIncrement(); |
||||
}); |
||||
}); |
||||
return result; |
||||
} |
||||
|
||||
/** |
||||
* 获取将list等量分隔成若干份的列表下标 |
||||
* @param size 总数 |
||||
* @param splitNum 分隔数量 |
||||
* @return 下标集合 |
||||
*/ |
||||
private List<Integer> getSplitIndex(int size,int splitNum) { |
||||
if(size % splitNum != 0) { |
||||
throw new HzServiceException("向量计算失败，无法根据同步对象进行等量分隔！"); |
||||
} |
||||
return IntStream.iterate(0, index -> index + 1) |
||||
.limit(splitNum) |
||||
.mapToObj(index -> index * (size / splitNum)) |
||||
.collect(Collectors.toList()); |
||||
} |
||||
|
||||
|
||||
private String getFieldValue(String fields,Object object) { |
||||
Class clazz = object.getClass(); |
||||
return Func.toStrList(",", fields).stream().map(field -> { |
||||
try { |
||||
Field declaredField = clazz.getDeclaredField(field); |
||||
declaredField.setAccessible(true); |
||||
return declaredField.get(object).toString(); |
||||
} catch (NoSuchFieldException | IllegalAccessException e) { |
||||
return null; |
||||
} |
||||
}).collect(Collectors.joining(" ")); |
||||
} |
||||
|
||||
/** |
||||
* 计算向量值 |
||||
* @param vectors 待计算的向量 |
||||
* @return 向量值Float[] |
||||
*/ |
||||
private Float[] compute(List<String> vectors) { |
||||
// 向量计算
|
||||
String url = vectorUrl + invokeApi.getCompute(); |
||||
String jsonData = JSONUtil.toJsonStr(vectors); |
||||
HttpResponse response = HttpRequest.post(url) |
||||
.header("Content-Type", "application/json; charset=utf-8") |
||||
.body(jsonData) |
||||
.execute(); |
||||
byte[] bytes = response.bodyBytes(); |
||||
if (bytes.length % 4 != 0) { |
||||
throw new HzServiceException("向量计算失败！响应数据长度不是4的倍数"); |
||||
} |
||||
List<byte[]> chunks = new ArrayList<>(); |
||||
int range = bytes.length / 4; |
||||
IntStream.range(0, range) |
||||
.forEach(index -> { |
||||
byte[] chunk = new byte[4]; |
||||
int page = index * 4; |
||||
chunk[0] = bytes[page]; |
||||
chunk[1] = bytes[page + 1]; |
||||
chunk[2] = bytes[page + 2]; |
||||
chunk[3] = bytes[page + 3]; |
||||
chunks.add(chunk); |
||||
}); |
||||
List<Float> floats = chunks.stream().map(b -> { |
||||
ByteBuffer buffer = ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN); |
||||
return buffer.getFloat(); |
||||
}).collect(Collectors.toList()); |
||||
return floats.toArray(new Float[floats.size()]); |
||||
} |
||||
} |
||||
@ -0,0 +1,11 @@
|
||||
package com.hnac.hzims.bigmodel.database.util; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/23 10:09 |
||||
*/ |
||||
public class WeaviateUtil { |
||||
|
||||
|
||||
|
||||
} |
||||
@ -0,0 +1,40 @@
|
||||
package com.hnac.hzims.bigmodel.maintenance.entity; |
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableName; |
||||
import io.swagger.annotations.ApiModel; |
||||
import io.swagger.annotations.ApiModelProperty; |
||||
import lombok.Data; |
||||
import lombok.EqualsAndHashCode; |
||||
import org.springblade.core.tenant.mp.TenantEntity; |
||||
|
||||
import java.io.Serializable; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/21 08:33 |
||||
*/ |
||||
@Data |
||||
@EqualsAndHashCode |
||||
@ApiModel(value = "同步向量库基础配置",description = "同步向量库基础配置") |
||||
@TableName("") |
||||
public class SyncPropertyEntity extends TenantEntity implements Serializable { |
||||
|
||||
@ApiModelProperty("配置编号") |
||||
private String code; |
||||
|
||||
@ApiModelProperty("数据源") |
||||
private String datasource; |
||||
|
||||
@ApiModelProperty("数据源名称") |
||||
private String datasourceName; |
||||
|
||||
@ApiModelProperty("数据库表名") |
||||
private String tableName; |
||||
|
||||
@ApiModelProperty("向量数据库表名,由小写字母以及_组成") |
||||
private String vectorTableName; |
||||
|
||||
@ApiModelProperty("表描述") |
||||
private String tableDesc; |
||||
|
||||
} |
||||
@ -0,0 +1,38 @@
|
||||
package com.hnac.hzims.operational.station.wrapper; |
||||
|
||||
import com.hnac.hzims.bigmodel.api.dto.VideoSyncDTO; |
||||
import com.hnac.hzims.bigmodel.api.wrapper.BaseSyncWrapper; |
||||
import com.hnac.hzims.operational.station.entity.StationVideoTypeEntity; |
||||
import com.hnac.hzims.operational.station.service.IStationService; |
||||
import com.hnac.hzims.operational.station.vo.StationVO; |
||||
import lombok.AllArgsConstructor; |
||||
import lombok.NoArgsConstructor; |
||||
import org.springblade.core.tool.utils.BeanUtil; |
||||
import org.springblade.core.tool.utils.SpringUtil; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
import java.util.Optional; |
||||
|
||||
/** |
||||
* @Author: huangxing |
||||
* @Date: 2024/08/20 11:21 |
||||
*/ |
||||
@Component |
||||
@AllArgsConstructor |
||||
public class VideoSyncWrapper extends BaseSyncWrapper<StationVideoTypeEntity, VideoSyncDTO> { |
||||
|
||||
private final IStationService stationService; |
||||
|
||||
@Override |
||||
public VideoSyncDTO convert(StationVideoTypeEntity entity) { |
||||
VideoSyncDTO videoSyncDTO = BeanUtil.copy(entity,VideoSyncDTO.class); |
||||
StationVO station = stationService.getStationByCode(entity.getStationId()); |
||||
String stationName = Optional.ofNullable(station).map(StationVO::getName).orElse(""); |
||||
videoSyncDTO.setId(String.valueOf(entity.getId())); |
||||
videoSyncDTO.setItemId(String.valueOf(entity.getId())); |
||||
videoSyncDTO.setStationName(stationName); |
||||
videoSyncDTO.setItemName(stationName + " " + videoSyncDTO.getName()); |
||||
return videoSyncDTO; |
||||
} |
||||
} |
||||
Loading…
Reference in new issue