feat:上传到阿里云oss
This commit is contained in:
dzc committed
1 parent
73d5a13762
commit
3252f88079
17 files changed
+1499
-51
No files matched your search
@@ -72,6 +72,22 @@ https://www.yuque.com/u1475064/imk5n2/qtezcg32q1d0dr29#SbvXq
|
||||
- redis记录了每个用户的websocket连接的是哪个im-server,当用户发送消息时，im-platform将根据所连接的im-server的id,决定将消息推向哪个queue
|
||||
|
||||
|
||||
#### 文件存储：MinIO 与阿里云 OSS
|
||||
|
||||
`im-platform/src/main/resources/application-dev.yml` 和 `application-prod.yml` 分别保留原 `minio` 配置，并新增独立的 `oss` 配置。`oss.enabled: true` 时使用阿里云原生 SDK 上传；设为 `false` 或没有 `oss` 节点时使用原 MinIO。修改配置后需要重新部署或重启 im-platform。
|
||||
|
||||
阿里云的 `endpoint` 填 Bucket 所在地域的服务端节点，`domain` 填 Bucket 公网域名或绑定的 CDN 域名，`bucketName` 填桶名；`accessKey`、`secretKey` 填对应阿里云凭据。OSS 地址格式为 `domain/对象路径`，不会额外拼接 Bucket；MinIO 保留 `domain/bucketName/对象路径` 格式。当前独立 OSS Bucket 为 `htys-chat`。
|
||||
|
||||
切换存储只影响新上传，历史 MinIO 文件和消息地址仍然保留，不自动搬迁。重复上传按当前存储匹配文件，避免命中旧 MinIO 记录而跳过 OSS 上传。启用 OSS 时，过期清理按已配置的 OSS/MinIO 域名和目录识别原图归属，使用各自的保留天数；未知地址会跳过，缩略图仍按原规则保留。关闭 OSS 后不清理 OSS 中的历史对象。OSS Bucket 及访问权限由已有配置管理，应用不自动创建桶或更改权限。
|
||||
|
||||
从项目根目录执行真实图片上传测试（默认读取 dev）：
|
||||
|
||||
```sh
|
||||
mvn -pl im-platform -am -Dtest=StorageUploadSmokeTest -Dsurefire.failIfNoSpecifiedTests=false -Doss.upload.test=true test
|
||||
```
|
||||
|
||||
测试自动生成一张小 PNG，通过实际上传代码上传，读回比对内容并匿名检查最终图片地址。测试图片保留在 `image/_upload-test-UUID/` 下，控制台输出其地址；测试不连接数据库或 Redis。可追加 `-Doss.upload.profile=prod` 测试 prod 配置。普通测试不会自动执行真实上传。
|
||||
|
||||
#### 本地启动
|
||||
1.安装运行环境
|
||||
- 安装node:v18.19.0
|
||||
|
||||
@@ -75,6 +75,11 @@
|
||||
<artifactId>minio</artifactId>
|
||||
<version>${minio.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.aliyun.oss</groupId>
|
||||
<artifactId>aliyun-sdk-oss</artifactId>
|
||||
<version>3.17.3</version>
|
||||
</dependency>
|
||||
<!--thumbnailator图片处理-->
|
||||
<dependency>
|
||||
<groupId>net.coobird</groupId>
|
||||
|
||||
@@ -4,11 +4,13 @@ import com.bx.implatform.config.props.MinioProperties;
|
||||
import io.minio.MinioClient;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
|
||||
@Configuration
|
||||
public class MinIoClientConfig {
|
||||
|
||||
@Bean
|
||||
@Lazy
|
||||
public MinioClient minioClient(MinioProperties minioProps) {
|
||||
// 注入minio 客户端
|
||||
return MinioClient.builder()
|
||||
@@ -16,4 +18,4 @@ public class MinIoClientConfig {
|
||||
.credentials(minioProps.getAccessKey(), minioProps.getSecretKey())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.bx.implatform.config;
|
||||
|
||||
import com.aliyun.oss.ClientBuilderConfiguration;
|
||||
import com.aliyun.oss.OSS;
|
||||
import com.aliyun.oss.OSSClientBuilder;
|
||||
import com.aliyun.oss.common.comm.Protocol;
|
||||
import com.bx.implatform.config.props.OssProperties;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
@Configuration
|
||||
@ConditionalOnProperty(prefix = "oss", name = "enabled", havingValue = "true")
|
||||
public class OssClientConfig {
|
||||
|
||||
@Bean(destroyMethod = "shutdown")
|
||||
public OSS ossClient(OssProperties properties) {
|
||||
Assert.hasText(properties.getEndpoint(), "oss.endpoint 不能为空");
|
||||
Assert.hasText(properties.getAccessKey(), "oss.access-key 不能为空");
|
||||
Assert.hasText(properties.getSecretKey(), "oss.secret-key 不能为空");
|
||||
Assert.hasText(properties.getBucketName(), "oss.bucket-name 不能为空");
|
||||
ClientBuilderConfiguration configuration = new ClientBuilderConfiguration();
|
||||
configuration.setProtocol(Protocol.HTTPS);
|
||||
configuration.setConnectionTimeout(15_000);
|
||||
configuration.setConnectionRequestTimeout(15_000);
|
||||
configuration.setSocketTimeout(30_000);
|
||||
configuration.setRequestTimeout(60_000);
|
||||
configuration.setRequestTimeoutEnabled(true);
|
||||
configuration.setMaxErrorRetry(2);
|
||||
// Construct the client locally; no bucket checks, creation or ACL changes.
|
||||
return new OSSClientBuilder().build(properties.getEndpoint(), properties.getAccessKey(),
|
||||
properties.getSecretKey(), configuration);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.bx.implatform.config.props;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** 阿里云 OSS 配置。避免生成包含访问凭据的 toString。 */
|
||||
@Getter
|
||||
@Setter
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "oss")
|
||||
public class OssProperties {
|
||||
|
||||
private boolean enabled = false;
|
||||
|
||||
private String endpoint;
|
||||
|
||||
private String domain;
|
||||
|
||||
private String accessKey;
|
||||
|
||||
private String secretKey;
|
||||
|
||||
private String bucketName;
|
||||
|
||||
private String imagePath = "image";
|
||||
|
||||
private String filePath = "file";
|
||||
|
||||
private String videoPath = "video";
|
||||
|
||||
private Integer expireIn = 180;
|
||||
}
|
||||
@@ -1,10 +1,8 @@
|
||||
package com.bx.implatform.service.impl;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
||||
import com.bx.implatform.config.props.MinioProperties;
|
||||
import com.bx.implatform.contant.Constant;
|
||||
import com.bx.implatform.entity.FileInfo;
|
||||
import com.bx.implatform.enums.FileType;
|
||||
@@ -13,7 +11,7 @@ import com.bx.implatform.exception.GlobalException;
|
||||
import com.bx.implatform.mapper.FileInfoMapper;
|
||||
import com.bx.implatform.service.FileService;
|
||||
import com.bx.implatform.session.SessionContext;
|
||||
import com.bx.implatform.thirdparty.MinioService;
|
||||
import com.bx.implatform.thirdparty.FileStorageService;
|
||||
import com.bx.implatform.util.FileUtil;
|
||||
import com.bx.implatform.util.ImageUtil;
|
||||
import com.bx.implatform.vo.UploadImageVO;
|
||||
@@ -42,18 +40,11 @@ import java.util.Objects;
|
||||
@RequiredArgsConstructor
|
||||
public class FileServiceImpl extends ServiceImpl<FileInfoMapper, FileInfo> implements FileService {
|
||||
|
||||
private final MinioService minioSerivce;
|
||||
|
||||
private final MinioProperties minioProps;
|
||||
private final FileStorageService storage;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
if (!minioSerivce.bucketExists(minioProps.getBucketName())) {
|
||||
// 创建bucket
|
||||
minioSerivce.makeBucket(minioProps.getBucketName());
|
||||
// 公开bucket
|
||||
minioSerivce.setBucketPublic(minioProps.getBucketName());
|
||||
}
|
||||
storage.init();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -77,7 +68,7 @@ public class FileServiceImpl extends ServiceImpl<FileInfoMapper, FileInfo> imple
|
||||
return fileInfo.getFilePath();
|
||||
}
|
||||
// 上传
|
||||
String fileName = minioSerivce.upload(minioProps.getBucketName(), minioProps.getFilePath(), file);
|
||||
String fileName = storage.upload(FileType.FILE, file);
|
||||
if (StringUtils.isEmpty(fileName)) {
|
||||
throw new GlobalException(ResultCode.PROGRAM_ERROR, "文件上传失败");
|
||||
}
|
||||
@@ -128,7 +119,7 @@ public class FileServiceImpl extends ServiceImpl<FileInfoMapper, FileInfo> imple
|
||||
return vo;
|
||||
}
|
||||
// 上传原图
|
||||
String fileName = minioSerivce.upload(minioProps.getBucketName(), minioProps.getImagePath(), file);
|
||||
String fileName = storage.upload(FileType.IMAGE, file);
|
||||
if (StringUtils.isEmpty(fileName)) {
|
||||
throw new GlobalException(ResultCode.PROGRAM_ERROR, "图片上传失败");
|
||||
}
|
||||
@@ -136,8 +127,7 @@ public class FileServiceImpl extends ServiceImpl<FileInfoMapper, FileInfo> imple
|
||||
if (file.getSize() > thumbSize * 1024) {
|
||||
// 大于50K的文件需上传缩略图
|
||||
byte[] imageByte = ImageUtil.compressForScale(file.getBytes(), thumbSize);
|
||||
String thumbFileName = minioSerivce.upload(minioProps.getBucketName(), minioProps.getImagePath(),
|
||||
file.getOriginalFilename(), imageByte, file.getContentType());
|
||||
String thumbFileName = storage.upload(FileType.IMAGE, file.getOriginalFilename(), imageByte, file.getContentType());
|
||||
if (StringUtils.isEmpty(thumbFileName)) {
|
||||
throw new GlobalException(ResultCode.PROGRAM_ERROR, "图片上传失败");
|
||||
}
|
||||
@@ -159,23 +149,22 @@ public class FileServiceImpl extends ServiceImpl<FileInfoMapper, FileInfo> imple
|
||||
}
|
||||
|
||||
private String generUrl(FileType fileType, String fileName) {
|
||||
return StrUtil.join("/", minioProps.getDomain(), minioProps.getBucketName(), getBucketPath(fileType), fileName);
|
||||
}
|
||||
|
||||
private String getBucketPath(FileType fileType) {
|
||||
return switch (fileType) {
|
||||
case FILE -> minioProps.getFilePath();
|
||||
case IMAGE -> minioProps.getImagePath();
|
||||
case VIDEO -> minioProps.getVideoPath();
|
||||
};
|
||||
return storage.url(fileType, fileName);
|
||||
}
|
||||
|
||||
private FileInfo findByMd5(String md5, Integer fileType) {
|
||||
LambdaQueryWrapper<FileInfo> wrapper = Wrappers.lambdaQuery();
|
||||
wrapper.eq(FileInfo::getMd5, md5);
|
||||
wrapper.eq(FileInfo::getFileType, fileType);
|
||||
// A previously uploaded MinIO file must not bypass a newly selected OSS
|
||||
// upload simply because its MD5 matches. Keep old records/URLs intact.
|
||||
wrapper.likeRight(FileInfo::getFilePath, storage.baseUrl() + "/");
|
||||
wrapper.last("limit 1");
|
||||
return getOne(wrapper);
|
||||
FileInfo existing = getOne(wrapper);
|
||||
if (existing == null || !storage.isCurrentUrl(existing.getFilePath())) return null;
|
||||
if (existing.getCompressedPath() != null && !existing.getCompressedPath().isBlank()
|
||||
&& !storage.isCurrentUrl(existing.getCompressedPath())) return null;
|
||||
return existing;
|
||||
}
|
||||
|
||||
private void saveImageFileInfo(MultipartFile file, String md5, String filePath, String compressedPath,
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
package com.bx.implatform.task.schedule;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import com.bx.implatform.annotation.RedisLock;
|
||||
import com.bx.implatform.config.props.MinioProperties;
|
||||
import com.bx.implatform.contant.RedisKey;
|
||||
import com.bx.implatform.entity.FileInfo;
|
||||
import com.bx.implatform.service.FileService;
|
||||
import com.bx.implatform.thirdparty.MinioService;
|
||||
import com.bx.implatform.thirdparty.FileStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.time.DateUtils;
|
||||
@@ -17,6 +15,7 @@ import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.OptionalInt;
|
||||
|
||||
/**
|
||||
* 过期文件清理任务
|
||||
@@ -30,46 +29,62 @@ import java.util.List;
|
||||
public class FileExpireTask {
|
||||
|
||||
private final FileService fileService;
|
||||
private final MinioService minioService;
|
||||
private final MinioProperties minioProps;
|
||||
private final FileStorageService fileStorageService;
|
||||
|
||||
@RedisLock(prefixKey = RedisKey.IM_LOCK_FILE_TASK)
|
||||
@Scheduled(cron = "0 0 3 * * ?")
|
||||
public void run() {
|
||||
log.info("【定时任务】过期文件处理...");
|
||||
int minimumExpireIn = fileStorageService.getMinimumExpireIn();
|
||||
if (minimumExpireIn < 0) {
|
||||
log.warn("文件保留天数无效，跳过本次过期清理");
|
||||
return;
|
||||
}
|
||||
int batchSize = 100;
|
||||
List<FileInfo> files = loadBatch(batchSize);
|
||||
Date now = new Date();
|
||||
Date scanBefore = DateUtils.addDays(now, -minimumExpireIn);
|
||||
Long cursor = null;
|
||||
while (true) {
|
||||
List<FileInfo> files = loadBatch(batchSize, cursor, scanBefore);
|
||||
for (FileInfo fileInfo : files) {
|
||||
String url = fileInfo.getFilePath();
|
||||
String relativePath = url.substring(fileInfo.getFilePath().indexOf(minioProps.getBucketName()));
|
||||
String[] arr = relativePath.split("/");
|
||||
String bucket = minioProps.getBucketName();
|
||||
String path = arr[1];
|
||||
String fileNme = StrUtil.join("/", arr[2], arr[3]);
|
||||
if (minioService.isExist(bucket, path, fileNme)) {
|
||||
if (!minioService.remove(bucket, path, fileNme)) {
|
||||
// 删除失败，不再往下执行
|
||||
log.error("删除过期文件异常, id:{},文件名:{}", fileInfo.getId(), fileInfo.getFileName());
|
||||
return;
|
||||
// Skipped/failed rows must not keep the first full page repeating.
|
||||
cursor = fileInfo.getId();
|
||||
try {
|
||||
if (!Boolean.FALSE.equals(fileInfo.getIsPermanent()) || fileInfo.getUploadTime() == null) {
|
||||
continue;
|
||||
}
|
||||
// 删除文件信息
|
||||
fileService.removeById(fileInfo.getId());
|
||||
OptionalInt expireIn = fileStorageService.expireInForUrl(fileInfo.getFilePath());
|
||||
if (expireIn.isEmpty() || expireIn.getAsInt() < 0) {
|
||||
continue;
|
||||
}
|
||||
Date expiresBefore = DateUtils.addDays(now, -expireIn.getAsInt());
|
||||
if (fileInfo.getUploadTime().after(expiresBefore)) {
|
||||
continue;
|
||||
}
|
||||
// Only the original object is removed; thumbnails stay intact.
|
||||
// The storage facade treats an already missing object as success.
|
||||
if (!fileStorageService.removeByUrl(fileInfo.getFilePath())) {
|
||||
log.warn("删除过期文件未完成, id:{}", fileInfo.getId());
|
||||
continue;
|
||||
}
|
||||
if (!fileService.removeById(fileInfo.getId())) {
|
||||
log.warn("过期文件已删除但文件记录未删除, id:{}", fileInfo.getId());
|
||||
}
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("清理过期文件失败, id:{}", fileInfo.getId(), e);
|
||||
}
|
||||
}
|
||||
if (files.size() < batchSize) {
|
||||
break;
|
||||
}
|
||||
// 下一批
|
||||
files = loadBatch(batchSize);
|
||||
}
|
||||
}
|
||||
|
||||
List<FileInfo> loadBatch(int size) {
|
||||
Date minDate = DateUtils.addDays(new Date(), -minioProps.getExpireIn());
|
||||
List<FileInfo> loadBatch(int size, Long cursor, Date scanBefore) {
|
||||
LambdaQueryWrapper<FileInfo> wrapper = Wrappers.lambdaQuery();
|
||||
wrapper.eq(FileInfo::getIsPermanent, false);
|
||||
wrapper.le(FileInfo::getUploadTime, minDate);
|
||||
wrapper.le(FileInfo::getUploadTime, scanBefore);
|
||||
wrapper.gt(cursor != null, FileInfo::getId, cursor);
|
||||
wrapper.orderByAsc(FileInfo::getId);
|
||||
wrapper.last("limit " + size);
|
||||
return fileService.list(wrapper);
|
||||
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
package com.bx.implatform.thirdparty;
|
||||
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.aliyun.oss.OSS;
|
||||
import com.aliyun.oss.OSSException;
|
||||
import com.aliyun.oss.model.ObjectMetadata;
|
||||
import com.aliyun.oss.model.PutObjectRequest;
|
||||
import com.bx.implatform.util.DateTimeUtils;
|
||||
import com.bx.implatform.util.FileUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.Date;
|
||||
|
||||
/** 原生 OSS 操作，返回值与现有 MinioService 的相对文件名约定一致。 */
|
||||
@Slf4j
|
||||
@Service
|
||||
@ConditionalOnProperty(prefix = "oss", name = "enabled", havingValue = "true")
|
||||
public class AliyunOssService {
|
||||
|
||||
private final OSS ossClient;
|
||||
|
||||
public AliyunOssService(OSS ossClient) {
|
||||
this.ossClient = ossClient;
|
||||
}
|
||||
|
||||
public String upload(String bucketName, String path, MultipartFile file) {
|
||||
try {
|
||||
String objectName = createObjectName(file.getOriginalFilename());
|
||||
try (InputStream stream = file.getInputStream()) {
|
||||
upload(bucketName, objectKey(path, objectName), stream, file.getSize(), file.getContentType());
|
||||
}
|
||||
return objectName;
|
||||
} catch (Exception exception) {
|
||||
log.warn("OSS 文件上传失败，错误类型：{}", failureCode(exception));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public String upload(String bucketName, String path, String name, byte[] content, String contentType) {
|
||||
try {
|
||||
String objectName = createObjectName(name);
|
||||
try (InputStream stream = new ByteArrayInputStream(content)) {
|
||||
upload(bucketName, objectKey(path, objectName), stream, content.length, contentType);
|
||||
}
|
||||
return objectName;
|
||||
} catch (Exception exception) {
|
||||
log.warn("OSS 文件上传失败，错误类型：{}", failureCode(exception));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean remove(String bucketName, String path, String fileName) {
|
||||
if (StringUtils.isBlank(fileName)) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
ossClient.deleteObject(bucketName, objectKey(path, fileName));
|
||||
return true;
|
||||
} catch (Exception exception) {
|
||||
// DeleteObject normally succeeds for absent objects; tolerate an
|
||||
// explicit NoSuchKey response without issuing an existence check.
|
||||
if (exception instanceof OSSException ossException && "NoSuchKey".equals(ossException.getErrorCode())) {
|
||||
return true;
|
||||
}
|
||||
log.warn("OSS 文件删除失败，错误类型：{}", failureCode(exception));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void upload(String bucketName, String key, InputStream stream, long size, String contentType) {
|
||||
ObjectMetadata metadata = new ObjectMetadata();
|
||||
metadata.setContentLength(size);
|
||||
metadata.setContentType(StringUtils.isBlank(contentType) ? "application/octet-stream" : contentType);
|
||||
ossClient.putObject(new PutObjectRequest(bucketName, key, stream, metadata));
|
||||
}
|
||||
|
||||
private String createObjectName(String originalName) {
|
||||
if (StringUtils.isBlank(originalName)) {
|
||||
throw new IllegalArgumentException("文件名不能为空");
|
||||
}
|
||||
String fileName = IdUtil.getSnowflakeNextIdStr() + "." + FileUtil.getFileExtension(originalName);
|
||||
return DateTimeUtils.getFormatDate(new Date(), DateTimeUtils.PARTDATEFORMAT) + "/" + fileName;
|
||||
}
|
||||
|
||||
private String objectKey(String path, String objectName) {
|
||||
String prefix = StringUtils.strip(path, "/");
|
||||
return StringUtils.isBlank(prefix) ? objectName : prefix + "/" + objectName;
|
||||
}
|
||||
|
||||
private String failureCode(Exception exception) {
|
||||
if (exception instanceof OSSException ossException) {
|
||||
String code = ossException.getErrorCode();
|
||||
if (code != null && code.matches("[A-Za-z][A-Za-z0-9]{0,63}")) {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
// SDK exception messages may contain signed requests or credentials.
|
||||
return exception.getClass().getSimpleName();
|
||||
}
|
||||
}
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
package com.bx.implatform.thirdparty;
|
||||
|
||||
import com.bx.implatform.config.props.MinioProperties;
|
||||
import com.bx.implatform.config.props.OssProperties;
|
||||
import com.bx.implatform.enums.FileType;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.util.UriUtils;
|
||||
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Optional;
|
||||
import java.util.OptionalInt;
|
||||
|
||||
/** Selects new uploads and resolves historical URLs without guessing buckets. */
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class FileStorageService {
|
||||
private final MinioProperties minio;
|
||||
private final OssProperties oss;
|
||||
private final ObjectProvider<MinioService> minioServices;
|
||||
private final ObjectProvider<AliyunOssService> ossServices;
|
||||
|
||||
public void init() {
|
||||
// OSS buckets and their access policies are managed outside the app.
|
||||
baseUrl();
|
||||
if (oss.isEnabled()) {
|
||||
ossService();
|
||||
} else {
|
||||
MinioService service = minioService();
|
||||
if (!service.bucketExists(minio.getBucketName())) {
|
||||
service.makeBucket(minio.getBucketName());
|
||||
service.setBucketPublic(minio.getBucketName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String upload(FileType type, MultipartFile file) {
|
||||
return oss.isEnabled()
|
||||
? ossService().upload(oss.getBucketName(), path(type, true), file)
|
||||
: minioService().upload(minio.getBucketName(), path(type, false), file);
|
||||
}
|
||||
|
||||
public String upload(FileType type, String name, byte[] bytes, String contentType) {
|
||||
return oss.isEnabled()
|
||||
? ossService().upload(oss.getBucketName(), path(type, true), name, bytes, contentType)
|
||||
: minioService().upload(minio.getBucketName(), path(type, false), name, bytes, contentType);
|
||||
}
|
||||
|
||||
public String url(FileType type, String relativeName) {
|
||||
String key = path(type, oss.isEnabled()) + "/" + required(relativeName, "文件名");
|
||||
if (!validKey(key)) throw new IllegalArgumentException("文件路径不合法");
|
||||
return baseUrl() + "/" + UriUtils.encodePath(key, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public String baseUrl() {
|
||||
return providerBase(oss.isEnabled());
|
||||
}
|
||||
|
||||
public int getExpireIn() {
|
||||
return expiry(oss.isEnabled());
|
||||
}
|
||||
|
||||
public int getMinimumExpireIn() {
|
||||
int minimum = getExpireIn();
|
||||
OptionalInt legacy = retention(false);
|
||||
if (oss.isEnabled() && hasMinioLocation() && legacy.isPresent()) minimum = Math.min(minimum, legacy.getAsInt());
|
||||
return minimum;
|
||||
}
|
||||
|
||||
public OptionalInt expireInForUrl(String url) {
|
||||
Optional<Location> location = locate(url);
|
||||
return location.isPresent() ? retention(location.get().oss()) : OptionalInt.empty();
|
||||
}
|
||||
|
||||
public boolean isCurrentUrl(String url) {
|
||||
return locate(url).filter(location -> location.oss() == oss.isEnabled()).isPresent();
|
||||
}
|
||||
|
||||
public boolean removeByUrl(String url) {
|
||||
Optional<Location> located = locate(url);
|
||||
if (located.isEmpty()) return false;
|
||||
Location location = located.get();
|
||||
int slash = location.key().lastIndexOf('/');
|
||||
String path = location.key().substring(0, slash);
|
||||
String name = location.key().substring(slash + 1);
|
||||
try {
|
||||
return location.oss()
|
||||
? ossService().remove(oss.getBucketName(), path, name)
|
||||
: minioService().remove(minio.getBucketName(), path, name);
|
||||
} catch (RuntimeException error) {
|
||||
// Do not log signed requests, credentials or full SDK exceptions.
|
||||
log.warn("清理文件失败，存储类型={}，错误类型={}", location.oss() ? "oss" : "minio", error.getClass().getSimpleName());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private Optional<Location> locate(String url) {
|
||||
if (url == null || url.isBlank()) return Optional.empty();
|
||||
Location ossLocation = oss.isEnabled() ? matchSafely(url, true) : null;
|
||||
Location minioLocation = hasMinioLocation() ? matchSafely(url, false) : null;
|
||||
// Ambiguous configuration must never send a delete to a guessed backend.
|
||||
if (ossLocation != null && minioLocation != null) return Optional.empty();
|
||||
return Optional.ofNullable(ossLocation != null ? ossLocation : minioLocation);
|
||||
}
|
||||
|
||||
private Location matchSafely(String value, boolean useOss) {
|
||||
try { return match(value, useOss); }
|
||||
catch (IllegalArgumentException error) { return null; }
|
||||
}
|
||||
|
||||
private Location match(String value, boolean useOss) {
|
||||
URI actual = URI.create(value);
|
||||
URI base = URI.create(providerBase(useOss));
|
||||
if (actual.getHost() == null || actual.getUserInfo() != null || actual.getQuery() != null
|
||||
|| actual.getFragment() != null || !base.getScheme().equalsIgnoreCase(actual.getScheme())
|
||||
|| !base.getHost().equalsIgnoreCase(actual.getHost()) || port(base) != port(actual)) return null;
|
||||
String prefix = base.getRawPath() + "/";
|
||||
if (!actual.getRawPath().startsWith(prefix)) return null;
|
||||
String key = UriUtils.decode(actual.getRawPath().substring(prefix.length()), StandardCharsets.UTF_8);
|
||||
if (!validKey(key)) return null;
|
||||
boolean owned = Arrays.stream(FileType.values()).anyMatch(type -> key.startsWith(path(type, useOss) + "/"));
|
||||
return owned ? new Location(useOss, key) : null;
|
||||
}
|
||||
|
||||
private String providerBase(boolean useOss) {
|
||||
String bucket = required(useOss ? oss.getBucketName() : minio.getBucketName(), "Bucket 名称");
|
||||
String domain = useOss ? oss.getDomain() : minio.getDomain();
|
||||
if (useOss && (domain == null || domain.isBlank())) {
|
||||
URI endpoint = URI.create(required(oss.getEndpoint(), "OSS endpoint"));
|
||||
String host = endpoint.getHost();
|
||||
if (host == null || !host.endsWith(".aliyuncs.com")) {
|
||||
throw new IllegalArgumentException("自定义 OSS endpoint 需要同时配置公开 domain");
|
||||
}
|
||||
domain = "https://" + bucket + "." + host.replace("-internal.aliyuncs.com", ".aliyuncs.com");
|
||||
}
|
||||
String root = stripTrailingSlash(required(domain, "存储访问域名"));
|
||||
if (!useOss) root += "/" + bucket;
|
||||
URI uri = URI.create(root);
|
||||
if (uri.getHost() == null || !("https".equalsIgnoreCase(uri.getScheme()) || "http".equalsIgnoreCase(uri.getScheme()))
|
||||
|| uri.getUserInfo() != null || uri.getQuery() != null || uri.getFragment() != null) {
|
||||
throw new IllegalArgumentException("存储访问域名必须是完整的 HTTP(S) 地址，不能带凭据、查询参数或片段");
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
private String path(FileType type, boolean useOss) {
|
||||
String configured = switch (type) {
|
||||
case FILE -> useOss ? oss.getFilePath() : minio.getFilePath();
|
||||
case IMAGE -> useOss ? oss.getImagePath() : minio.getImagePath();
|
||||
case VIDEO -> useOss ? oss.getVideoPath() : minio.getVideoPath();
|
||||
};
|
||||
String path = stripTrailingSlash(required(configured, "文件目录"));
|
||||
if (!validKey(path)) throw new IllegalArgumentException("文件目录不合法");
|
||||
return path;
|
||||
}
|
||||
|
||||
private int expiry(boolean useOss) {
|
||||
return retention(useOss).orElseThrow(() -> new IllegalArgumentException("文件过期天数必须为正整数"));
|
||||
}
|
||||
|
||||
private OptionalInt retention(boolean useOss) {
|
||||
Integer days = useOss ? oss.getExpireIn() : minio.getExpireIn();
|
||||
return days == null || days < 1 ? OptionalInt.empty() : OptionalInt.of(days);
|
||||
}
|
||||
|
||||
private boolean hasMinioLocation() {
|
||||
return minio.getDomain() != null && !minio.getDomain().isBlank()
|
||||
&& minio.getBucketName() != null && !minio.getBucketName().isBlank();
|
||||
}
|
||||
|
||||
private MinioService minioService() {
|
||||
MinioService service = minioServices.getIfAvailable();
|
||||
if (service == null) throw new IllegalStateException("MinIO 客户端未配置");
|
||||
return service;
|
||||
}
|
||||
|
||||
private AliyunOssService ossService() {
|
||||
AliyunOssService service = ossServices.getIfAvailable();
|
||||
if (service == null) throw new IllegalStateException("已启用 OSS，但 OSS 客户端未配置");
|
||||
return service;
|
||||
}
|
||||
|
||||
private static String required(String value, String name) {
|
||||
if (value == null || value.isBlank()) throw new IllegalArgumentException(name + "不能为空");
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private static String stripTrailingSlash(String value) {
|
||||
return value.replaceAll("/+$", "");
|
||||
}
|
||||
|
||||
private static int port(URI uri) {
|
||||
return uri.getPort() >= 0 ? uri.getPort() : "https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80;
|
||||
}
|
||||
|
||||
private static boolean validKey(String key) {
|
||||
return key != null && !key.isEmpty() && !key.contains("\\") && !key.contains("\0")
|
||||
&& Arrays.stream(key.split("/", -1)).noneMatch(part -> part.isEmpty() || ".".equals(part) || "..".equals(part));
|
||||
}
|
||||
|
||||
private record Location(boolean oss, String key) {}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
@@ -17,6 +18,7 @@ import java.util.Date;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
@Lazy
|
||||
@RequiredArgsConstructor
|
||||
public class MinioService {
|
||||
|
||||
|
||||
@@ -22,6 +22,19 @@ minio:
|
||||
videoPath: video
|
||||
expireIn: 180 # 文件过期时间,单位:天
|
||||
|
||||
# 阿里云 OSS 独立配置
|
||||
oss:
|
||||
enabled: true # true 使用阿里云 OSS；false 使用原有 MinIO 配置
|
||||
endpoint: https://oss-cn-hangzhou.aliyuncs.com #服务端访问地址（公网）
|
||||
domain: https://htys-chat.oss-cn-hangzhou.aliyuncs.com #客户端公开访问域名，不追加 Bucket 名
|
||||
accessKey: LTAI5t6ey9ia44LGqLnGdHuf
|
||||
secretKey: 1d8OFbypsc206M6G2bRkLtCudr28cL
|
||||
bucketName: htys-chat
|
||||
imagePath: image
|
||||
filePath: file
|
||||
videoPath: video
|
||||
expireIn: 180 # 文件过期时间,单位:天
|
||||
|
||||
webrtc:
|
||||
max-channel: 9 # 多人通话最大通道数量，最大不能超过16,建议值:4,9,16
|
||||
iceServers:
|
||||
|
||||
@@ -22,6 +22,19 @@ minio:
|
||||
videoPath: video
|
||||
expireIn: 180 # 文件过期时间,单位:天
|
||||
|
||||
# 阿里云 OSS 独立配置
|
||||
oss:
|
||||
enabled: true # true 使用阿里云 OSS；false 使用原有 MinIO 配置
|
||||
endpoint: https://oss-cn-hangzhou.aliyuncs.com #服务端访问地址（公网）
|
||||
domain: https://htys-chat.oss-cn-hangzhou.aliyuncs.com #客户端公开访问域名，不追加 Bucket 名
|
||||
accessKey: LTAI5t6ey9ia44LGqLnGdHuf
|
||||
secretKey: 1d8OFbypsc206M6G2bRkLtCudr28cL
|
||||
bucketName: htys-chat
|
||||
imagePath: image
|
||||
filePath: file
|
||||
videoPath: video
|
||||
expireIn: 180 # 文件过期时间,单位:天
|
||||
|
||||
webrtc:
|
||||
max-channel: 9 # 多人通话最大通道数量，最大不能超过16,建议值:4,9,16
|
||||
iceServers: #coturn配置
|
||||
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
package com.bx.implatform.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.bx.implatform.entity.FileInfo;
|
||||
import com.bx.implatform.enums.FileType;
|
||||
import com.bx.implatform.service.impl.FileServiceImpl;
|
||||
import com.bx.implatform.thirdparty.FileStorageService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class FileServiceStorageSelectionTest {
|
||||
private static final String MD5 = "bd7e610677b8e1c6f8354547effb7910";
|
||||
private static final String OSS_BASE = "https://chat-bucket.oss-cn-hangzhou.aliyuncs.com";
|
||||
private static final String OSS_IMAGE = OSS_BASE + "/image/20261105/original.png";
|
||||
private static final String OSS_THUMB = OSS_BASE + "/image/20261105/thumb.png";
|
||||
private static final String MINIO_IMAGE = "https://minio.example/old-bucket/image/20261105/original.png";
|
||||
private static final String MINIO_THUMB = "https://minio.example/old-bucket/image/20261105/thumb.png";
|
||||
|
||||
private FileStorageService storage;
|
||||
private FileServiceImpl service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableMetadata() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), "file-storage-selection-test"), FileInfo.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
storage = mock(FileStorageService.class);
|
||||
service = spy(new FileServiceImpl(storage));
|
||||
when(storage.baseUrl()).thenReturn(OSS_BASE);
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
void currentFileWithoutThumbnailIsReusableAndLookupIsScopedToActiveStorage() {
|
||||
String fileUrl = OSS_BASE + "/file/20261105/document.pdf";
|
||||
FileInfo current = file(FileType.FILE, fileUrl, null);
|
||||
doReturn(current).when(service).getOne(any());
|
||||
when(storage.isCurrentUrl(fileUrl)).thenReturn(true);
|
||||
|
||||
assertSame(current, findByMd5(FileType.FILE));
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<FileInfo>> query = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
||||
verify(service).getOne(query.capture());
|
||||
String predicate = query.getValue().getSqlSegment();
|
||||
assertTrue(predicate.contains("md5 ="), predicate);
|
||||
assertTrue(predicate.contains("file_type ="), predicate);
|
||||
assertTrue(predicate.contains("file_path LIKE"), predicate);
|
||||
assertTrue(query.getValue().getParamNameValuePairs().containsValue(MD5));
|
||||
assertTrue(query.getValue().getParamNameValuePairs().containsValue(FileType.FILE.code()));
|
||||
assertTrue(query.getValue().getParamNameValuePairs().containsValue(OSS_BASE + "/%"),
|
||||
"MD5 lookup must exclude old storage before selecting a reusable record");
|
||||
verify(storage, times(1)).isCurrentUrl(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyMinioImageIsNotReusableEvenWhenItsMd5Matches() {
|
||||
FileInfo legacy = file(FileType.IMAGE, MINIO_IMAGE, MINIO_THUMB);
|
||||
doReturn(legacy).when(service).getOne(any());
|
||||
when(storage.isCurrentUrl(MINIO_IMAGE)).thenReturn(false);
|
||||
|
||||
assertNull(findByMd5(FileType.IMAGE), "old MinIO content must not bypass a new OSS upload");
|
||||
|
||||
assertEquals(MINIO_IMAGE, legacy.getFilePath(), "historical URL must remain unchanged");
|
||||
assertEquals(MINIO_THUMB, legacy.getCompressedPath());
|
||||
verify(storage, never()).isCurrentUrl(MINIO_THUMB);
|
||||
}
|
||||
|
||||
@Test
|
||||
void currentOssImageAndThumbnailAreReusable() {
|
||||
FileInfo current = file(FileType.IMAGE, OSS_IMAGE, OSS_THUMB);
|
||||
doReturn(current).when(service).getOne(any());
|
||||
when(storage.isCurrentUrl(OSS_IMAGE)).thenReturn(true);
|
||||
when(storage.isCurrentUrl(OSS_THUMB)).thenReturn(true);
|
||||
|
||||
assertSame(current, findByMd5(FileType.IMAGE));
|
||||
|
||||
verify(storage).isCurrentUrl(OSS_IMAGE);
|
||||
verify(storage).isCurrentUrl(OSS_THUMB);
|
||||
}
|
||||
|
||||
@Test
|
||||
void currentOssImageWithLegacyThumbnailMustBeUploadedAgain() {
|
||||
FileInfo mixed = file(FileType.IMAGE, OSS_IMAGE, MINIO_THUMB);
|
||||
doReturn(mixed).when(service).getOne(any());
|
||||
when(storage.isCurrentUrl(OSS_IMAGE)).thenReturn(true);
|
||||
when(storage.isCurrentUrl(MINIO_THUMB)).thenReturn(false);
|
||||
|
||||
assertNull(findByMd5(FileType.IMAGE), "a legacy thumbnail must not leak into a new OSS upload response");
|
||||
|
||||
verify(storage).isCurrentUrl(OSS_IMAGE);
|
||||
verify(storage).isCurrentUrl(MINIO_THUMB);
|
||||
}
|
||||
|
||||
private FileInfo findByMd5(FileType type) {
|
||||
return ReflectionTestUtils.invokeMethod(service, "findByMd5", MD5, type.code());
|
||||
}
|
||||
|
||||
private static FileInfo file(FileType type, String original, String thumbnail) {
|
||||
FileInfo file = new FileInfo();
|
||||
file.setId(101L);
|
||||
file.setMd5(MD5);
|
||||
file.setFileType(type.code());
|
||||
file.setFilePath(original);
|
||||
file.setCompressedPath(thumbnail);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package com.bx.implatform.task.schedule;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.bx.implatform.entity.FileInfo;
|
||||
import com.bx.implatform.service.FileService;
|
||||
import com.bx.implatform.thirdparty.FileStorageService;
|
||||
import org.apache.commons.lang3.time.DateUtils;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.OptionalInt;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class FileExpireTaskTest {
|
||||
|
||||
@Mock
|
||||
private FileService fileService;
|
||||
@Mock
|
||||
private FileStorageService fileStorageService;
|
||||
private FileExpireTask task;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableMetadata() {
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), "file-expire-test"), FileInfo.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
task = new FileExpireTask(fileService, fileStorageService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void usesEachProvidersRetentionAndDeletesOnlyRecognizedExpiredOriginals() {
|
||||
FileInfo minio = file(1, "https://old.example/bucket/image/a.png", 10);
|
||||
FileInfo oss = file(2, "https://cdn.example/image/b.png", 35);
|
||||
FileInfo youngerOss = file(3, "https://cdn.example/image/c.png", 10);
|
||||
FileInfo unknown = file(4, "https://outside.example/image.png", 90);
|
||||
FileInfo permanent = file(5, "https://cdn.example/image/permanent.png", 90);
|
||||
permanent.setIsPermanent(true);
|
||||
when(fileStorageService.getMinimumExpireIn()).thenReturn(7);
|
||||
when(fileService.list(ArgumentMatchers.<LambdaQueryWrapper<FileInfo>>any())).thenReturn(List.of(minio, oss, youngerOss, unknown, permanent));
|
||||
when(fileStorageService.expireInForUrl(minio.getFilePath())).thenReturn(OptionalInt.of(7));
|
||||
when(fileStorageService.expireInForUrl(oss.getFilePath())).thenReturn(OptionalInt.of(30));
|
||||
when(fileStorageService.expireInForUrl(youngerOss.getFilePath())).thenReturn(OptionalInt.of(30));
|
||||
when(fileStorageService.expireInForUrl(unknown.getFilePath())).thenReturn(OptionalInt.empty());
|
||||
when(fileStorageService.removeByUrl(minio.getFilePath())).thenReturn(true);
|
||||
when(fileStorageService.removeByUrl(oss.getFilePath())).thenReturn(true);
|
||||
when(fileService.removeById(1L)).thenReturn(true);
|
||||
when(fileService.removeById(2L)).thenReturn(true);
|
||||
|
||||
task.run();
|
||||
|
||||
verify(fileStorageService).removeByUrl(minio.getFilePath());
|
||||
verify(fileStorageService).removeByUrl(oss.getFilePath());
|
||||
verify(fileStorageService, never()).removeByUrl(youngerOss.getFilePath());
|
||||
verify(fileStorageService, never()).removeByUrl(unknown.getFilePath());
|
||||
verify(fileStorageService, never()).expireInForUrl(permanent.getFilePath());
|
||||
verify(fileStorageService, never()).removeByUrl(minio.getCompressedPath());
|
||||
verify(fileStorageService, never()).removeByUrl(oss.getCompressedPath());
|
||||
verify(fileService).removeById(1L);
|
||||
verify(fileService).removeById(2L);
|
||||
verify(fileService, times(2)).removeById(any(Long.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
void cursorMovesPastAFullSkippedOrFailedPageAndProcessesTheNextPage() {
|
||||
List<FileInfo> firstPage = new ArrayList<>();
|
||||
for (long id = 1; id <= 100; id++) firstPage.add(file(id, "https://unknown.example/" + id + ".png", 50));
|
||||
FileInfo failed = firstPage.get(0);
|
||||
FileInfo next = file(101, "https://cdn.example/next.png", 50);
|
||||
when(fileStorageService.getMinimumExpireIn()).thenReturn(7);
|
||||
when(fileService.list(ArgumentMatchers.<LambdaQueryWrapper<FileInfo>>any())).thenReturn(firstPage, List.of(next));
|
||||
when(fileStorageService.expireInForUrl(anyString())).thenReturn(OptionalInt.empty());
|
||||
when(fileStorageService.expireInForUrl(failed.getFilePath())).thenReturn(OptionalInt.of(7));
|
||||
when(fileStorageService.expireInForUrl(next.getFilePath())).thenReturn(OptionalInt.of(7));
|
||||
when(fileStorageService.removeByUrl(failed.getFilePath())).thenReturn(false);
|
||||
when(fileStorageService.removeByUrl(next.getFilePath())).thenReturn(true);
|
||||
when(fileService.removeById(101L)).thenReturn(true);
|
||||
|
||||
task.run();
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<FileInfo>> queries = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
||||
verify(fileService, times(2)).list(queries.capture());
|
||||
LambdaQueryWrapper<FileInfo> first = queries.getAllValues().get(0);
|
||||
LambdaQueryWrapper<FileInfo> second = queries.getAllValues().get(1);
|
||||
assertFalse(first.getSqlSegment().contains("id >"));
|
||||
assertTrue(second.getSqlSegment().contains("id >"));
|
||||
assertTrue(second.getParamNameValuePairs().containsValue(100L));
|
||||
Date firstCutoff = first.getParamNameValuePairs().values().stream().filter(Date.class::isInstance).map(Date.class::cast).findFirst().orElseThrow();
|
||||
assertTrue(second.getParamNameValuePairs().containsValue(firstCutoff), "all pages use the same time boundary");
|
||||
verify(fileService, never()).removeById(failed.getId());
|
||||
verify(fileService).removeById(101L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneProviderFailureDoesNotPreventTheFollowingFileFromBeingRemoved() {
|
||||
FileInfo failed = file(1, "https://old.example/bucket/a.png", 40);
|
||||
FileInfo next = file(2, "https://cdn.example/b.png", 40);
|
||||
when(fileStorageService.getMinimumExpireIn()).thenReturn(7);
|
||||
when(fileService.list(ArgumentMatchers.<LambdaQueryWrapper<FileInfo>>any())).thenReturn(List.of(failed, next));
|
||||
when(fileStorageService.expireInForUrl(anyString())).thenReturn(OptionalInt.of(7));
|
||||
when(fileStorageService.removeByUrl(failed.getFilePath())).thenThrow(new IllegalStateException("simulated provider failure"));
|
||||
when(fileStorageService.removeByUrl(next.getFilePath())).thenReturn(true);
|
||||
when(fileService.removeById(2L)).thenReturn(true);
|
||||
|
||||
assertDoesNotThrow(task::run);
|
||||
|
||||
verify(fileService, never()).removeById(1L);
|
||||
verify(fileService).removeById(2L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingObjectSuccessAllowsMetadataCleanupAndMetadataFailureDoesNotStopTheBatch() {
|
||||
FileInfo alreadyMissing = file(1, "https://cdn.example/missing.png", 40);
|
||||
FileInfo next = file(2, "https://cdn.example/next.png", 40);
|
||||
when(fileStorageService.getMinimumExpireIn()).thenReturn(7);
|
||||
when(fileService.list(ArgumentMatchers.<LambdaQueryWrapper<FileInfo>>any())).thenReturn(List.of(alreadyMissing, next));
|
||||
when(fileStorageService.expireInForUrl(anyString())).thenReturn(OptionalInt.of(7));
|
||||
when(fileStorageService.removeByUrl(anyString())).thenReturn(true);
|
||||
when(fileService.removeById(1L)).thenReturn(false);
|
||||
when(fileService.removeById(2L)).thenReturn(true);
|
||||
|
||||
task.run();
|
||||
|
||||
verify(fileService).removeById(1L);
|
||||
verify(fileService).removeById(2L);
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
void loadBatchFiltersNonPermanentCandidatesByFixedCutoffAndAscendingCursor() {
|
||||
Date cutoff = new Date(1_700_000_000_000L);
|
||||
when(fileService.list(ArgumentMatchers.<LambdaQueryWrapper<FileInfo>>any())).thenReturn(List.of());
|
||||
|
||||
task.loadBatch(100, 200L, cutoff);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<FileInfo>> query = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
||||
verify(fileService).list(query.capture());
|
||||
String sql = query.getValue().getSqlSegment();
|
||||
assertTrue(sql.contains("is_permanent ="));
|
||||
assertTrue(sql.contains("upload_time <="));
|
||||
assertTrue(sql.contains("id >"));
|
||||
assertTrue(sql.contains("ORDER BY id ASC"));
|
||||
assertTrue(sql.endsWith("limit 100"));
|
||||
assertTrue(query.getValue().getParamNameValuePairs().containsValue(false));
|
||||
assertTrue(query.getValue().getParamNameValuePairs().containsValue(cutoff));
|
||||
assertTrue(query.getValue().getParamNameValuePairs().containsValue(200L));
|
||||
verifyNoInteractions(fileStorageService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidMinimumRetentionStopsBeforeScanningOrDeleting() {
|
||||
when(fileStorageService.getMinimumExpireIn()).thenReturn(-1);
|
||||
|
||||
task.run();
|
||||
|
||||
verifyNoInteractions(fileService);
|
||||
verify(fileStorageService, never()).removeByUrl(any());
|
||||
}
|
||||
|
||||
private static FileInfo file(long id, String url, int ageDays) {
|
||||
FileInfo file = new FileInfo();
|
||||
file.setId(id);
|
||||
file.setFileName("image-" + id + ".png");
|
||||
file.setFilePath(url);
|
||||
file.setCompressedPath(url + ".thumbnail");
|
||||
file.setCoverPath(url + ".cover");
|
||||
file.setUploadTime(DateUtils.addDays(new Date(), -ageDays));
|
||||
file.setIsPermanent(false);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
+272
@@ -0,0 +1,272 @@
|
||||
package com.bx.implatform.thirdparty;
|
||||
|
||||
import ch.qos.logback.classic.Logger;
|
||||
import ch.qos.logback.classic.spi.ILoggingEvent;
|
||||
import ch.qos.logback.core.read.ListAppender;
|
||||
import com.aliyun.oss.ClientBuilderConfiguration;
|
||||
import com.aliyun.oss.OSS;
|
||||
import com.aliyun.oss.OSSClientBuilder;
|
||||
import com.aliyun.oss.OSSException;
|
||||
import com.aliyun.oss.common.comm.Protocol;
|
||||
import com.aliyun.oss.model.PutObjectRequest;
|
||||
import com.aliyun.oss.model.PutObjectResult;
|
||||
import com.bx.implatform.config.OssClientConfig;
|
||||
import com.bx.implatform.config.props.OssProperties;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockedConstruction;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AliyunOssServiceTest {
|
||||
|
||||
@Mock
|
||||
private OSS ossClient;
|
||||
|
||||
private AliyunOssService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new AliyunOssService(ossClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void multipartUploadPreservesBytesMimeAndRelativeDateSnowflakeName() throws IOException {
|
||||
byte[] bytes = {0, 1, -1, 42, 77};
|
||||
AtomicReference<byte[]> received = new AtomicReference<>();
|
||||
when(ossClient.putObject(any(PutObjectRequest.class))).thenAnswer(invocation -> {
|
||||
PutObjectRequest request = invocation.getArgument(0);
|
||||
received.set(request.getInputStream().readAllBytes());
|
||||
return new PutObjectResult();
|
||||
});
|
||||
String result = service.upload("chat-bucket", "image", new MockMultipartFile("file", "头像.PNG", "image/png", bytes));
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.matches("\\d{8}/\\d+\\.PNG"));
|
||||
assertFalse(result.contains("://"));
|
||||
assertFalse(result.startsWith("image/"));
|
||||
ArgumentCaptor<PutObjectRequest> captor = ArgumentCaptor.forClass(PutObjectRequest.class);
|
||||
verify(ossClient).putObject(captor.capture());
|
||||
assertEquals("chat-bucket", captor.getValue().getBucketName());
|
||||
assertEquals("image/" + result, captor.getValue().getKey());
|
||||
assertEquals(bytes.length, captor.getValue().getMetadata().getContentLength());
|
||||
assertEquals("image/png", captor.getValue().getMetadata().getContentType());
|
||||
assertArrayEquals(bytes, received.get());
|
||||
verifyNoMoreInteractions(ossClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void byteArrayUploadPreservesThumbnailContentAndNormalizesOnlyDirectorySlashes() throws IOException {
|
||||
byte[] bytes = {3, 1, 4, 1, 5};
|
||||
AtomicReference<byte[]> received = new AtomicReference<>();
|
||||
when(ossClient.putObject(any(PutObjectRequest.class))).thenAnswer(invocation -> {
|
||||
PutObjectRequest request = invocation.getArgument(0);
|
||||
received.set(request.getInputStream().readAllBytes());
|
||||
return new PutObjectResult();
|
||||
});
|
||||
String result = service.upload("chat-bucket", "/image/", "thumb.jpg", bytes, "image/jpeg");
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.matches("\\d{8}/\\d+\\.jpg"));
|
||||
ArgumentCaptor<PutObjectRequest> captor = ArgumentCaptor.forClass(PutObjectRequest.class);
|
||||
verify(ossClient).putObject(captor.capture());
|
||||
assertEquals("image/" + result, captor.getValue().getKey());
|
||||
assertEquals("image/jpeg", captor.getValue().getMetadata().getContentType());
|
||||
assertEquals(bytes.length, captor.getValue().getMetadata().getContentLength());
|
||||
assertArrayEquals(bytes, received.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingMimeUsesBinaryContentTypeWithoutChangingBytes() {
|
||||
String result = service.upload("chat-bucket", "", "document.bin", new byte[]{7}, null);
|
||||
ArgumentCaptor<PutObjectRequest> captor = ArgumentCaptor.forClass(PutObjectRequest.class);
|
||||
verify(ossClient).putObject(captor.capture());
|
||||
assertEquals(result, captor.getValue().getKey());
|
||||
assertEquals("application/octet-stream", captor.getValue().getMetadata().getContentType());
|
||||
}
|
||||
|
||||
@Test
|
||||
void multipartStreamsAreClosedWhenUploadFails() throws IOException {
|
||||
TrackingInputStream stream = new TrackingInputStream(new byte[]{1, 2});
|
||||
MultipartFile file = mock(MultipartFile.class);
|
||||
when(file.getOriginalFilename()).thenReturn("photo.png");
|
||||
when(file.getInputStream()).thenReturn(stream);
|
||||
when(file.getSize()).thenReturn(2L);
|
||||
when(file.getContentType()).thenReturn("image/png");
|
||||
when(ossClient.putObject(any(PutObjectRequest.class))).thenThrow(new IllegalStateException("sensitive request"));
|
||||
|
||||
assertNull(service.upload("chat-bucket", "image", file));
|
||||
assertTrue(stream.closed);
|
||||
verify(file, never()).getBytes();
|
||||
}
|
||||
|
||||
@Test
|
||||
void inputReadFailuresReturnNullWithoutCallingOss() throws IOException {
|
||||
MultipartFile file = mock(MultipartFile.class);
|
||||
when(file.getOriginalFilename()).thenReturn("photo.png");
|
||||
when(file.getInputStream()).thenThrow(new IOException("local read failed"));
|
||||
assertNull(service.upload("chat-bucket", "image", file));
|
||||
verifyNoInteractions(ossClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void blankNamesAndNullBytesReturnNullWithoutAnUpload() {
|
||||
assertNull(service.upload("chat-bucket", "image", "", new byte[]{1}, "image/png"));
|
||||
assertNull(service.upload("chat-bucket", "image", "a.png", null, "image/png"));
|
||||
assertNull(service.upload("chat-bucket", "image", new MockMultipartFile("file", "", "image/png", new byte[]{1})));
|
||||
verifyNoInteractions(ossClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void nativeDeleteIsIdempotentAndDoesNotProbeTheBucket() {
|
||||
assertTrue(service.remove("chat-bucket", "image", "20261105/123.png"));
|
||||
assertTrue(service.remove("chat-bucket", "image", "20261105/123.png"));
|
||||
verify(ossClient, times(2)).deleteObject("chat-bucket", "image/20261105/123.png");
|
||||
verifyNoMoreInteractions(ossClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteTreatsMissingObjectsAsSuccessAndOtherErrorsAsFailure() {
|
||||
doThrow(ossException("NoSuchKey")).when(ossClient).deleteObject("chat-bucket", "image/missing.png");
|
||||
doThrow(ossException("AccessDenied")).when(ossClient).deleteObject("chat-bucket", "image/denied.png");
|
||||
assertTrue(service.remove("chat-bucket", "image", "missing.png"));
|
||||
assertFalse(service.remove("chat-bucket", "image", "denied.png"));
|
||||
assertFalse(service.remove("chat-bucket", "image", " "));
|
||||
verify(ossClient, times(2)).deleteObject(anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void sdkErrorsLogOnlyASafeCodeAndNeverSignedRequestsOrCredentials() {
|
||||
Logger logger = (Logger) LoggerFactory.getLogger(AliyunOssService.class);
|
||||
ListAppender<ILoggingEvent> appender = new ListAppender<>();
|
||||
appender.start();
|
||||
logger.addAppender(appender);
|
||||
try {
|
||||
when(ossClient.putObject(any(PutObjectRequest.class))).thenThrow(ossException("AccessDenied"));
|
||||
assertNull(service.upload("chat-bucket", "image", "photo.png", new byte[]{1}, "image/png"));
|
||||
assertEquals(1, appender.list.size());
|
||||
ILoggingEvent event = appender.list.get(0);
|
||||
assertTrue(event.getFormattedMessage().contains("AccessDenied"));
|
||||
assertFalse(event.getFormattedMessage().contains("secret-marker"));
|
||||
assertFalse(event.getFormattedMessage().contains("Authorization"));
|
||||
assertNull(event.getThrowableProxy());
|
||||
} finally {
|
||||
logger.detachAppender(appender);
|
||||
appender.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void propertiesDefaultToDisabledAndDoNotRenderCredentials() {
|
||||
OssProperties properties = properties();
|
||||
assertFalse(properties.isEnabled());
|
||||
assertEquals("image", properties.getImagePath());
|
||||
assertEquals("file", properties.getFilePath());
|
||||
assertEquals("video", properties.getVideoPath());
|
||||
assertEquals(180, properties.getExpireIn());
|
||||
assertFalse(properties.toString().contains(properties.getAccessKey()));
|
||||
assertFalse(properties.toString().contains(properties.getSecretKey()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void disabledOssDoesNotCreateAClientOrService() {
|
||||
new ApplicationContextRunner()
|
||||
.withUserConfiguration(OssClientConfig.class, OssProperties.class, AliyunOssService.class)
|
||||
.run(context -> {
|
||||
assertNull(context.getStartupFailure());
|
||||
assertEquals(0, context.getBeansOfType(OSS.class).size());
|
||||
assertEquals(0, context.getBeansOfType(AliyunOssService.class).size());
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledServiceUsesTheProvidedNativeClient() {
|
||||
new ApplicationContextRunner()
|
||||
.withPropertyValues("oss.enabled=true")
|
||||
.withBean(OSS.class, () -> ossClient)
|
||||
.withUserConfiguration(AliyunOssService.class)
|
||||
.run(context -> {
|
||||
assertNull(context.getStartupFailure());
|
||||
assertEquals(1, context.getBeansOfType(AliyunOssService.class).size());
|
||||
verifyNoInteractions(ossClient);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void clientConstructionSetsBoundedTimeoutsAndPerformsNoRemoteOperations() throws NoSuchMethodException {
|
||||
OssProperties properties = properties();
|
||||
try (MockedConstruction<OSSClientBuilder> builders = mockConstruction(OSSClientBuilder.class,
|
||||
(builder, context) -> when(builder.build(anyString(), anyString(), anyString(), any(ClientBuilderConfiguration.class))).thenReturn(ossClient))) {
|
||||
OSS created = new OssClientConfig().ossClient(properties);
|
||||
assertSame(ossClient, created);
|
||||
ArgumentCaptor<ClientBuilderConfiguration> captor = ArgumentCaptor.forClass(ClientBuilderConfiguration.class);
|
||||
verify(builders.constructed().get(0)).build(eq(properties.getEndpoint()), eq(properties.getAccessKey()), eq(properties.getSecretKey()), captor.capture());
|
||||
ClientBuilderConfiguration options = captor.getValue();
|
||||
assertEquals(15_000, options.getConnectionTimeout());
|
||||
assertEquals(15_000, options.getConnectionRequestTimeout());
|
||||
assertEquals(30_000, options.getSocketTimeout());
|
||||
assertEquals(60_000, options.getRequestTimeout());
|
||||
assertTrue(options.isRequestTimeoutEnabled());
|
||||
assertEquals(Protocol.HTTPS, options.getProtocol());
|
||||
verifyNoInteractions(ossClient);
|
||||
}
|
||||
Bean bean = OssClientConfig.class.getMethod("ossClient", OssProperties.class).getAnnotation(Bean.class);
|
||||
assertEquals("shutdown", bean.destroyMethod());
|
||||
}
|
||||
|
||||
@Test
|
||||
void incompleteEnabledConfigurationFailsBeforeClientConstruction() {
|
||||
OssProperties properties = properties();
|
||||
properties.setSecretKey(" ");
|
||||
try (MockedConstruction<OSSClientBuilder> builders = mockConstruction(OSSClientBuilder.class)) {
|
||||
IllegalArgumentException failure = assertThrows(IllegalArgumentException.class,
|
||||
() -> new OssClientConfig().ossClient(properties));
|
||||
assertTrue(failure.getMessage().contains("oss.secret-key"));
|
||||
assertEquals(0, builders.constructed().size());
|
||||
}
|
||||
}
|
||||
|
||||
private static OssProperties properties() {
|
||||
OssProperties properties = new OssProperties();
|
||||
properties.setEndpoint("https://oss-cn-hangzhou.aliyuncs.com");
|
||||
properties.setAccessKey("unit-test-access-key");
|
||||
properties.setSecretKey("unit-test-secret-key");
|
||||
properties.setBucketName("unit-test-bucket");
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static OSSException ossException(String code) {
|
||||
return new OSSException("secret-marker signed request", code, "request-id", "host",
|
||||
"Authorization: secret-marker", "object", "PUT");
|
||||
}
|
||||
|
||||
private static class TrackingInputStream extends ByteArrayInputStream {
|
||||
private boolean closed;
|
||||
|
||||
TrackingInputStream(byte[] bytes) {
|
||||
super(bytes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
closed = true;
|
||||
super.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+181
@@ -0,0 +1,181 @@
|
||||
package com.bx.implatform.thirdparty;
|
||||
|
||||
import com.bx.implatform.config.MinIoClientConfig;
|
||||
import com.bx.implatform.config.props.MinioProperties;
|
||||
import com.bx.implatform.config.props.OssProperties;
|
||||
import com.bx.implatform.enums.FileType;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.support.StaticListableBeanFactory;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
|
||||
import java.util.OptionalInt;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class FileStorageServiceTest {
|
||||
private MinioProperties minio;
|
||||
private OssProperties oss;
|
||||
private MinioService minioClient;
|
||||
private AliyunOssService ossClient;
|
||||
private FileStorageService storage;
|
||||
|
||||
@BeforeEach
|
||||
void setup() {
|
||||
minio = new MinioProperties();
|
||||
minio.setDomain("https://minio.test/files/");
|
||||
minio.setBucketName("old-bucket");
|
||||
minio.setImagePath("image");
|
||||
minio.setFilePath("file");
|
||||
minio.setVideoPath("video");
|
||||
minio.setExpireIn(180);
|
||||
oss = new OssProperties();
|
||||
oss.setEnabled(true);
|
||||
oss.setEndpoint("https://oss-cn-hangzhou.aliyuncs.com");
|
||||
oss.setDomain("https://htys-chat.oss-cn-hangzhou.aliyuncs.com/");
|
||||
oss.setBucketName("htys-chat");
|
||||
oss.setExpireIn(30);
|
||||
minioClient = mock(MinioService.class);
|
||||
ossClient = mock(AliyunOssService.class);
|
||||
StaticListableBeanFactory factory = new StaticListableBeanFactory();
|
||||
factory.addBean("minioService", minioClient);
|
||||
factory.addBean("ossService", ossClient);
|
||||
storage = new FileStorageService(minio, oss, factory.getBeanProvider(MinioService.class),
|
||||
factory.getBeanProvider(AliyunOssService.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ossUploadsBothOriginalAndThumbnailToItsOwnBucketAndUsesBucketDomainOnlyOnce() {
|
||||
byte[] bytes = {1, 2, 3};
|
||||
MockMultipartFile file = new MockMultipartFile("file", "photo.png", "image/png", bytes);
|
||||
when(ossClient.upload("htys-chat", "image", file)).thenReturn("20261105/1.png");
|
||||
when(ossClient.upload("htys-chat", "image", "photo.png", bytes, "image/png")).thenReturn("20261105/2.png");
|
||||
assertEquals("20261105/1.png", storage.upload(FileType.IMAGE, file));
|
||||
assertEquals("20261105/2.png", storage.upload(FileType.IMAGE, "photo.png", bytes, "image/png"));
|
||||
assertEquals("https://htys-chat.oss-cn-hangzhou.aliyuncs.com/image/20261105/1.png",
|
||||
storage.url(FileType.IMAGE, "20261105/1.png"));
|
||||
verifyNoInteractions(minioClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void minioModePreservesExistingUploadAndBucketPathLayout() {
|
||||
oss.setEnabled(false);
|
||||
MockMultipartFile file = new MockMultipartFile("file", "a.txt", "text/plain", new byte[]{1});
|
||||
when(minioClient.upload("old-bucket", "file", file)).thenReturn("20261105/1.txt");
|
||||
assertEquals("20261105/1.txt", storage.upload(FileType.FILE, file));
|
||||
assertEquals("https://minio.test/files/old-bucket/file/20261105/1.txt",
|
||||
storage.url(FileType.FILE, "20261105/1.txt"));
|
||||
verifyNoInteractions(ossClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void ossInitializationDoesNotCreateBucketsOrChangePermissions() {
|
||||
storage.init();
|
||||
verifyNoInteractions(ossClient, minioClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void originalMinioInitializationRemainsAvailableWhenOssIsDisabled() {
|
||||
oss.setEnabled(false);
|
||||
when(minioClient.bucketExists("old-bucket")).thenReturn(false);
|
||||
storage.init();
|
||||
verify(minioClient).makeBucket("old-bucket");
|
||||
verify(minioClient).setBucketPublic("old-bucket");
|
||||
verifyNoInteractions(ossClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void oldMinioAndNewOssDeletionUseSeparateCredentialsBucketsAndRetention() {
|
||||
String oldUrl = "https://minio.test/files/old-bucket/image/20261105/old.png";
|
||||
String newUrl = "https://htys-chat.oss-cn-hangzhou.aliyuncs.com/image/20261105/new.png";
|
||||
when(minioClient.remove("old-bucket", "image/20261105", "old.png")).thenReturn(true);
|
||||
when(ossClient.remove("htys-chat", "image/20261105", "new.png")).thenReturn(true);
|
||||
assertTrue(storage.removeByUrl(oldUrl));
|
||||
assertTrue(storage.removeByUrl(newUrl));
|
||||
assertFalse(storage.isCurrentUrl(oldUrl));
|
||||
assertTrue(storage.isCurrentUrl(newUrl));
|
||||
assertEquals(OptionalInt.of(180), storage.expireInForUrl(oldUrl));
|
||||
assertEquals(OptionalInt.of(30), storage.expireInForUrl(newUrl));
|
||||
assertEquals(30, storage.getMinimumExpireIn());
|
||||
verify(minioClient).remove("old-bucket", "image/20261105", "old.png");
|
||||
verify(ossClient).remove("htys-chat", "image/20261105", "new.png");
|
||||
}
|
||||
|
||||
@Test
|
||||
void customCdnPrefixAndEncodedObjectNamesAreDecodedExactlyOnceForDeletion() {
|
||||
oss.setDomain("https://cdn.test/chat");
|
||||
String relative = "20261105/中文+100%.png";
|
||||
String url = storage.url(FileType.IMAGE, relative);
|
||||
assertTrue(url.startsWith("https://cdn.test/chat/image/"));
|
||||
assertTrue(storage.isCurrentUrl(url));
|
||||
when(ossClient.remove("htys-chat", "image/20261105", "中文+100%.png")).thenReturn(true);
|
||||
assertTrue(storage.removeByUrl(url));
|
||||
verify(ossClient).remove("htys-chat", "image/20261105", "中文+100%.png");
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownOrAmbiguousUrlsCannotDeleteAnything() {
|
||||
for (String url : new String[]{
|
||||
"https://outside.test/image/a.png", "not-a-url",
|
||||
"https://htys-chat.oss-cn-hangzhou.aliyuncs.com.evil.test/image/a.png",
|
||||
"https://htys-chat.oss-cn-hangzhou.aliyuncs.com/private/a.png",
|
||||
"https://htys-chat.oss-cn-hangzhou.aliyuncs.com/image/%2e%2e/a.png",
|
||||
"https://htys-chat.oss-cn-hangzhou.aliyuncs.com/image/a.png?signature=abc",
|
||||
"https://minio.test/files/old-bucket-extra/image/a.png",
|
||||
"https://minio.test/files/old-bucket/image/a.png#fragment"}) {
|
||||
assertFalse(storage.removeByUrl(url), url);
|
||||
assertTrue(storage.expireInForUrl(url).isEmpty(), url);
|
||||
}
|
||||
oss.setDomain("https://minio.test/files/old-bucket");
|
||||
assertFalse(storage.removeByUrl("https://minio.test/files/old-bucket/image/a.png"));
|
||||
verifyNoInteractions(minioClient, ossClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
void providerFailureDoesNotReportSuccessfulDeletion() {
|
||||
String url = storage.url(FileType.IMAGE, "20261105/a.png");
|
||||
when(ossClient.remove("htys-chat", "image/20261105", "a.png")).thenReturn(false);
|
||||
assertFalse(storage.removeByUrl(url));
|
||||
when(ossClient.remove("htys-chat", "image/20261105", "a.png")).thenThrow(new IllegalStateException("failure"));
|
||||
assertFalse(storage.removeByUrl(url));
|
||||
}
|
||||
|
||||
@Test
|
||||
void publicDomainDefaultsToBucketHostWithoutTheInternalEndpointSuffix() {
|
||||
oss.setDomain(null);
|
||||
oss.setEndpoint("https://oss-cn-hangzhou-internal.aliyuncs.com");
|
||||
assertEquals("https://htys-chat.oss-cn-hangzhou.aliyuncs.com", storage.baseUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
void malformedLegacyConfigurationDoesNotDisableValidOssUrls() {
|
||||
minio.setDomain("bad-domain");
|
||||
assertTrue(storage.isCurrentUrl(storage.url(FileType.IMAGE, "20261105/a.png")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingLegacyRetentionDoesNotBlockOssExpirationOrAssumeADeletionDate() {
|
||||
minio.setExpireIn(null);
|
||||
assertEquals(30, storage.getMinimumExpireIn());
|
||||
assertEquals(OptionalInt.of(30), storage.expireInForUrl(storage.url(FileType.IMAGE, "20261105/a.png")));
|
||||
assertTrue(storage.expireInForUrl("https://minio.test/files/old-bucket/image/20261105/a.png").isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void ossCanStartWithoutAnyMinioParametersOrClientConstruction() {
|
||||
new ApplicationContextRunner()
|
||||
.withPropertyValues("oss.enabled=true")
|
||||
.withUserConfiguration(MinIoClientConfig.class, MinioService.class, FileStorageService.class)
|
||||
.withBean(MinioProperties.class, MinioProperties::new)
|
||||
.withBean(OssProperties.class, () -> oss)
|
||||
.withBean(AliyunOssService.class, () -> ossClient)
|
||||
.run(context -> {
|
||||
assertNull(context.getStartupFailure());
|
||||
context.getBean(FileStorageService.class).init();
|
||||
assertFalse(context.getBeanFactory().containsSingleton("minioClient"));
|
||||
assertFalse(context.getBeanFactory().containsSingleton("minioService"));
|
||||
});
|
||||
}
|
||||
}
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
package com.bx.implatform.thirdparty;
|
||||
|
||||
import ch.qos.logback.classic.Level;
|
||||
import ch.qos.logback.classic.Logger;
|
||||
import ch.qos.logback.classic.LoggerContext;
|
||||
import com.aliyun.oss.ClientException;
|
||||
import com.aliyun.oss.OSS;
|
||||
import com.aliyun.oss.OSSException;
|
||||
import com.aliyun.oss.model.OSSObject;
|
||||
import com.bx.implatform.config.MinIoClientConfig;
|
||||
import com.bx.implatform.config.OssClientConfig;
|
||||
import com.bx.implatform.config.props.MinioProperties;
|
||||
import com.bx.implatform.config.props.OssProperties;
|
||||
import com.bx.implatform.enums.FileType;
|
||||
import io.minio.GetObjectArgs;
|
||||
import io.minio.GetObjectResponse;
|
||||
import io.minio.MinioClient;
|
||||
import io.minio.errors.ErrorResponseException;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import okhttp3.ResponseBody;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
|
||||
import org.springframework.beans.factory.support.StaticListableBeanFactory;
|
||||
import org.springframework.boot.context.properties.bind.Bindable;
|
||||
import org.springframework.boot.context.properties.bind.Binder;
|
||||
import org.springframework.core.env.PropertiesPropertySource;
|
||||
import org.springframework.core.env.StandardEnvironment;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.URI;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* 手动真实上传测试：读取项目 YAML，根据 oss.enabled 复用实际存储门面和选中客户端。
|
||||
* 不启动 Spring，不连接 MySQL/Redis，不调用初始化、建桶或修改权限。
|
||||
* 在 imagePath/_upload-test-UUID/ 下上传一张小 PNG；对象保留用于验收或排查后续校验失败。
|
||||
*
|
||||
* 在 box-im-master 目录执行（默认 dev）：
|
||||
* mvn -pl im-platform -am -Dtest=StorageUploadSmokeTest
|
||||
* -Dsurefire.failIfNoSpecifiedTests=false -Doss.upload.test=true test
|
||||
* 可用 -Doss.upload.profile=prod 切换配置；普通 mvn test 不会进行真实上传。
|
||||
*/
|
||||
@EnabledIfSystemProperty(named = "oss.upload.test", matches = "true")
|
||||
class StorageUploadSmokeTest {
|
||||
|
||||
@Test
|
||||
void uploadImageUsingProjectConfiguration() {
|
||||
System.setProperty("java.awt.headless", "true");
|
||||
String stage = "读取项目配置";
|
||||
OSS ossClient = null;
|
||||
String objectKey = null;
|
||||
try (QuietStorageLogs ignored = new QuietStorageLogs()) {
|
||||
try {
|
||||
String profile = System.getProperty("oss.upload.profile", "dev");
|
||||
assertTrue(profile.matches("[A-Za-z0-9_-]+"), "配置环境名称不合法");
|
||||
StorageProperties properties = loadProperties(profile);
|
||||
MinioProperties minio = properties.minio();
|
||||
OssProperties oss = properties.oss();
|
||||
boolean useOss = oss.isEnabled();
|
||||
String imagePath = testImagePath(useOss ? oss.getImagePath() : minio.getImagePath());
|
||||
if (useOss) {
|
||||
oss.setImagePath(imagePath);
|
||||
} else {
|
||||
minio.setImagePath(imagePath);
|
||||
}
|
||||
|
||||
byte[] png = createPng();
|
||||
stage = "创建存储客户端";
|
||||
StaticListableBeanFactory beans = new StaticListableBeanFactory();
|
||||
MinioClient minioClient = null;
|
||||
if (useOss) {
|
||||
ossClient = new OssClientConfig().ossClient(oss);
|
||||
beans.addBean("aliyunOssService", new AliyunOssService(ossClient));
|
||||
} else {
|
||||
minioClient = new MinIoClientConfig().minioClient(minio);
|
||||
minioClient.setTimeout(15_000, 30_000, 30_000);
|
||||
beans.addBean("minioService", new MinioService(minioClient));
|
||||
}
|
||||
FileStorageService storage = new FileStorageService(minio, oss,
|
||||
beans.getBeanProvider(MinioService.class), beans.getBeanProvider(AliyunOssService.class));
|
||||
System.out.printf("上传测试：环境=%s，存储=%s，目录=%s，大小=%d 字节%n",
|
||||
profile, useOss ? "阿里云 OSS" : "MinIO", imagePath, png.length);
|
||||
|
||||
stage = "通过项目存储门面上传图片";
|
||||
String relativeName = storage.upload(FileType.IMAGE, "storage-smoke.png", png, "image/png");
|
||||
assertNotNull(relativeName, "存储门面未返回上传文件名");
|
||||
assertFalse(relativeName.isBlank(), "存储门面返回空文件名");
|
||||
objectKey = imagePath + "/" + relativeName;
|
||||
System.out.println("上传成功，key=" + objectKey);
|
||||
|
||||
stage = "通过存储客户端读回图片";
|
||||
if (useOss) {
|
||||
try (OSSObject stored = ossClient.getObject(oss.getBucketName(), objectKey)) {
|
||||
assertArrayEquals(png, stored.getObjectContent().readNBytes(png.length + 1),
|
||||
"OSS 中的图片与上传内容不一致");
|
||||
}
|
||||
} else {
|
||||
try (GetObjectResponse stored = minioClient.getObject(GetObjectArgs.builder()
|
||||
.bucket(minio.getBucketName()).object(objectKey).build())) {
|
||||
assertArrayEquals(png, stored.readNBytes(png.length + 1),
|
||||
"MinIO 中的图片与上传内容不一致");
|
||||
}
|
||||
}
|
||||
|
||||
stage = "生成项目图片访问地址";
|
||||
String publicUrl = storage.url(FileType.IMAGE, relativeName);
|
||||
assertNotNull(publicUrl, "项目没有生成图片访问地址");
|
||||
URI parsed = URI.create(publicUrl);
|
||||
assertTrue("https".equalsIgnoreCase(parsed.getScheme())
|
||||
|| "http".equalsIgnoreCase(parsed.getScheme()), "图片地址必须使用 HTTP 或 HTTPS");
|
||||
assertTrue(parsed.getRawUserInfo() == null, "图片公开地址不能包含凭据");
|
||||
assertTrue(parsed.getRawQuery() == null, "图片公开地址应为稳定地址，不能包含签名参数");
|
||||
assertTrue(parsed.getRawFragment() == null, "图片公开地址不能包含片段");
|
||||
System.out.println("项目生成的图片地址：" + publicUrl);
|
||||
|
||||
stage = "匿名访问项目生成的图片地址";
|
||||
verifyPublicImage(publicUrl, png);
|
||||
System.out.println("验证通过：上传、存储读回、公开地址访问的内容完全一致。");
|
||||
} catch (OSSException error) {
|
||||
fail(stage + "失败：OSS " + safeCode(error.getErrorCode()));
|
||||
} catch (ClientException error) {
|
||||
fail(stage + "失败：OSS 客户端 " + safeCode(error.getErrorCode()));
|
||||
} catch (ErrorResponseException error) {
|
||||
fail(stage + "失败：HTTP " + error.response().code() + "，"
|
||||
+ safeCode(error.errorResponse().code()));
|
||||
} catch (Exception error) {
|
||||
// SDK 原异常和配置绑定错误可能包含凭据或签名请求，不输出 message/cause。
|
||||
fail(stage + "失败：" + error.getClass().getSimpleName());
|
||||
} finally {
|
||||
if (objectKey != null) {
|
||||
System.out.println("测试图片已保留用于验收或排查，key=" + objectKey);
|
||||
}
|
||||
if (ossClient != null) {
|
||||
try {
|
||||
ossClient.shutdown();
|
||||
} catch (RuntimeException error) {
|
||||
System.err.println("OSS 客户端关闭失败（已隐藏详细异常）");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void verifyPublicImage(String publicUrl, byte[] expected) throws Exception {
|
||||
OkHttpClient http = new OkHttpClient.Builder()
|
||||
.connectTimeout(15, TimeUnit.SECONDS)
|
||||
.readTimeout(30, TimeUnit.SECONDS)
|
||||
.callTimeout(45, TimeUnit.SECONDS)
|
||||
.followRedirects(false)
|
||||
.followSslRedirects(false)
|
||||
.retryOnConnectionFailure(false)
|
||||
.build();
|
||||
try (Response response = http.newCall(new Request.Builder().url(publicUrl).get().build()).execute()) {
|
||||
assertEquals(200, response.code(), "图片地址无法公开访问");
|
||||
ResponseBody body = response.body();
|
||||
assertNotNull(body, "图片地址返回空响应");
|
||||
assertTrue(body.contentLength() == -1 || body.contentLength() == expected.length,
|
||||
"公开地址返回的图片长度不一致");
|
||||
assertArrayEquals(expected, body.byteStream().readNBytes(expected.length + 1),
|
||||
"公开地址返回的图片与上传内容不一致");
|
||||
} finally {
|
||||
http.connectionPool().evictAll();
|
||||
http.dispatcher().executorService().shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
private static StorageProperties loadProperties(String profile) {
|
||||
YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean();
|
||||
yaml.setResources(new ClassPathResource("application.yml"),
|
||||
new ClassPathResource("application-" + profile + ".yml"));
|
||||
Properties loaded = yaml.getObject();
|
||||
assertNotNull(loaded, "无法读取项目配置");
|
||||
StandardEnvironment environment = new StandardEnvironment();
|
||||
environment.getPropertySources().addLast(new PropertiesPropertySource("upload-test-yaml", loaded));
|
||||
Binder binder = Binder.get(environment);
|
||||
return new StorageProperties(
|
||||
binder.bind("minio", Bindable.of(MinioProperties.class)).orElseGet(MinioProperties::new),
|
||||
binder.bind("oss", Bindable.of(OssProperties.class)).orElseGet(OssProperties::new));
|
||||
}
|
||||
|
||||
private static String testImagePath(String imagePath) {
|
||||
assertNotNull(imagePath, "当前存储缺少 imagePath");
|
||||
String root = imagePath.replaceAll("^/+|/+$", "");
|
||||
assertFalse(root.isBlank(), "当前存储 imagePath 不能为空");
|
||||
return root + "/_upload-test-" + UUID.randomUUID();
|
||||
}
|
||||
|
||||
private static byte[] createPng() throws Exception {
|
||||
BufferedImage image = new BufferedImage(96, 96, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D graphics = image.createGraphics();
|
||||
try {
|
||||
graphics.setColor(new Color(38, 112, 232));
|
||||
graphics.fillRect(0, 0, 96, 96);
|
||||
graphics.setColor(Color.WHITE);
|
||||
graphics.fillRoundRect(22, 22, 52, 52, 12, 12);
|
||||
graphics.setColor(new Color(38, 112, 232));
|
||||
graphics.fillOval(34, 34, 28, 28);
|
||||
} finally {
|
||||
graphics.dispose();
|
||||
}
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
assertTrue(ImageIO.write(image, "png", output), "无法生成 PNG 测试图片");
|
||||
return output.toByteArray();
|
||||
}
|
||||
|
||||
private static String safeCode(String code) {
|
||||
return code != null && code.matches("[A-Za-z][A-Za-z0-9_.-]{0,79}") ? code : "UnknownError";
|
||||
}
|
||||
|
||||
private record StorageProperties(MinioProperties minio, OssProperties oss) {
|
||||
}
|
||||
|
||||
/** 防止客户端或原 MinioService 日志在失败时输出原始签名请求。 */
|
||||
private static final class QuietStorageLogs implements AutoCloseable {
|
||||
private final Map<Logger, Level> levels = new LinkedHashMap<>();
|
||||
|
||||
private QuietStorageLogs() {
|
||||
LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
|
||||
for (String name : new String[]{"com.aliyun.oss", "io.minio", "org.apache.http",
|
||||
MinioService.class.getName()}) {
|
||||
Logger logger = context.getLogger(name);
|
||||
levels.put(logger, logger.getLevel());
|
||||
logger.setLevel(Level.OFF);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
levels.forEach(Logger::setLevel);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user