refactor(pigx-common-oss): 将OSS操作从S3AsyncClient迁移至同步S3Client

This commit is contained in:
PIGCLOUD committed 2026-05-18 19:03:23 +08:00
1 parent 87a1ea1ae0
commit c796af6c5f
18 files changed
+599 -136

No files matched your search

@@ -0,0 +1,35 @@
package com.pig4cloud.pigx.common.file.core;
import org.springframework.boot.context.properties.bind.Binder;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.env.Environment;
import org.springframework.core.type.AnnotatedTypeMetadata;
/**
* 文件存储类型条件。
*
* @author lengleng
*/
abstract class AbstractFileTypeCondition implements Condition {
@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
Environment environment = context.getEnvironment();
Binder binder = Binder.get(environment);
String type = binder.bind("file.type", String.class).orElse(null);
if (type != null) {
return matchesType(type);
}
return matchesLegacySwitch(binder);
}
protected abstract boolean matchesType(String type);
protected abstract boolean matchesLegacySwitch(Binder binder);
protected boolean isTrue(Binder binder, String name) {
return binder.bind(name, Boolean.class).orElse(Boolean.FALSE);
}
}
@@ -0,0 +1,27 @@
package com.pig4cloud.pigx.common.file.core;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.Instant;
/**
* 文件存储桶信息。
*
* @author lengleng
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class FileBucket {
private String name;
private Instant creationDate;
public FileBucket(String name) {
this.name = name;
}
}
@@ -0,0 +1,36 @@
package com.pig4cloud.pigx.common.file.core;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import java.io.InputStream;
/**
* 文件对象读取结果。
*
* @author lengleng
*/
@Getter
@RequiredArgsConstructor
public class FileObject implements AutoCloseable {
private final String bucketName;
private final String objectName;
private final String contentType;
private final Long contentLength;
private final InputStream objectContent;
@Override
@SneakyThrows
public void close() {
if (objectContent != null) {
objectContent.close();
}
}
}
@@ -0,0 +1,31 @@
package com.pig4cloud.pigx.common.file.core;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.Instant;
/**
* 文件对象元数据。
*
* @author lengleng
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class FileObjectInfo {
private String bucketName;
private String objectName;
private String contentType;
private Long contentLength;
private String eTag;
private Instant lastModified;
}
@@ -0,0 +1,27 @@
package com.pig4cloud.pigx.common.file.core;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.Instant;
/**
* 文件对象摘要。
*
* @author lengleng
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class FileObjectSummary {
private String bucketName;
private String key;
private Long size;
private Instant lastModified;
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2026, lengleng All rights reserved.
* Copyright (c) 2018-2025, lengleng All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -39,16 +39,21 @@ public class FileProperties {
*/
private String bucketName = "local";
/**
* 文件存储类型
*/
private FileType type = FileType.LOCAL;
/**
* 本地文件配置信息
*/
@NestedConfigurationProperty
private LocalFileProperties local;
private LocalFileProperties local = new LocalFileProperties();
/**
* oss 文件配置信息
*/
@NestedConfigurationProperty
private OssProperties oss;
private OssProperties oss = new OssProperties();
}
@@ -1,8 +1,5 @@
package com.pig4cloud.pigx.common.file.core;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.springframework.beans.factory.InitializingBean;
import java.io.InputStream;
@@ -28,7 +25,7 @@ public interface FileTemplate extends InitializingBean {
*
* API Documentation</a>
*/
List<Bucket> getAllBuckets();
List<FileBucket> getAllBuckets();
/**
* @param bucketName bucket名称
@@ -75,7 +72,7 @@ public interface FileTemplate extends InitializingBean {
* @param objectName 文件名称
* @return 二进制流 API Documentation</a>
*/
S3Object getObject(String bucketName, String dir, String objectName);
FileObject getObject(String bucketName, String dir, String objectName);
/**
* 获取文件
@@ -83,7 +80,7 @@ public interface FileTemplate extends InitializingBean {
* @param objectName 文件名称
* @return 二进制流 API Documentation</a>
*/
S3Object getObject(String bucketName, String objectName);
FileObject getObject(String bucketName, String objectName);
void removeObject(String bucketName, String objectName) throws Exception;
@@ -99,10 +96,10 @@ public interface FileTemplate extends InitializingBean {
* @param bucketName bucket名称
* @param prefix 前缀
* @param recursive 是否递归查询
* @return S3ObjectSummary 列表
* @return FileObjectSummary 列表
* @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/ListObjects">AWS
* API Documentation</a>
*/
List<S3ObjectSummary> getAllObjectsByPrefix(String bucketName, String prefix, boolean recursive);
List<FileObjectSummary> getAllObjectsByPrefix(String bucketName, String prefix, boolean recursive);
}
@@ -0,0 +1,20 @@
package com.pig4cloud.pigx.common.file.core;
/**
* 文件存储类型。
*
* @author lengleng
*/
public enum FileType {
/**
* 本地文件系统
*/
LOCAL,
/**
* S3 兼容对象存储
*/
OSS
}
@@ -0,0 +1,25 @@
package com.pig4cloud.pigx.common.file.core;
import org.springframework.boot.context.properties.bind.Binder;
/**
* 本地文件存储装配条件。
*
* @author lengleng
*/
public class LocalFileTypeCondition extends AbstractFileTypeCondition {
@Override
protected boolean matchesType(String type) {
return FileType.LOCAL.name().equalsIgnoreCase(type);
}
@Override
protected boolean matchesLegacySwitch(Binder binder) {
if (isTrue(binder, "file.oss.enable")) {
return false;
}
return binder.bind("file.local.enable", Boolean.class).orElse(Boolean.TRUE);
}
}
@@ -0,0 +1,22 @@
package com.pig4cloud.pigx.common.file.core;
import org.springframework.boot.context.properties.bind.Binder;
/**
* OSS 文件存储装配条件。
*
* @author lengleng
*/
public class OssFileTypeCondition extends AbstractFileTypeCondition {
@Override
protected boolean matchesType(String type) {
return FileType.OSS.name().equalsIgnoreCase(type);
}
@Override
protected boolean matchesLegacySwitch(Binder binder) {
return isTrue(binder, "file.oss.enable");
}
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2026, lengleng All rights reserved.
* Copyright (c) 2018-2025, lengleng All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -19,10 +19,11 @@ package com.pig4cloud.pigx.common.file.local;
import com.pig4cloud.pigx.common.file.core.FileProperties;
import com.pig4cloud.pigx.common.file.core.FileTemplate;
import com.pig4cloud.pigx.common.file.core.LocalFileTypeCondition;
import lombok.AllArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
/**
* aws 自动配置类
@@ -37,7 +38,7 @@ public class LocalFileAutoConfiguration {
@Bean
@ConditionalOnMissingBean(LocalFileTemplate.class)
@ConditionalOnProperty(name = "file.local.enable", havingValue = "true", matchIfMissing = true)
@Conditional(LocalFileTypeCondition.class)
public FileTemplate localFileTemplate() {
return new LocalFileTemplate(properties);
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2026, lengleng All rights reserved.
* Copyright (c) 2018-2025, lengleng All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -19,6 +19,7 @@ package com.pig4cloud.pigx.common.file.local;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.DeprecatedConfigurationProperty;
/**
* 本地文件 配置信息
@@ -34,6 +35,7 @@ public class LocalFileProperties {
/**
* 是否开启
*/
@Deprecated
private boolean enable;
/**
@@ -41,4 +43,10 @@ public class LocalFileProperties {
*/
private String basePath;
@Deprecated
@DeprecatedConfigurationProperty(reason = "使用 file.type=local 替代 file.local.enable", replacement = "file.type")
public boolean isEnable() {
return enable;
}
}
@@ -2,16 +2,13 @@ package com.pig4cloud.pigx.common.file.local;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.StrUtil;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.pig4cloud.pigx.common.file.core.FileProperties;
import com.pig4cloud.pigx.common.file.core.FileTemplate;
import com.pig4cloud.pigx.common.file.core.*;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import java.io.File;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.List;
@@ -42,11 +39,11 @@ public class LocalFileTemplate implements FileTemplate {
* API Documentation</a>
*/
@Override
public List<Bucket> getAllBuckets() {
public List<FileBucket> getAllBuckets() {
return Arrays.stream(FileUtil.ls(properties.getLocal().getBasePath()))
.filter(FileUtil::isDirectory)
.map(dir -> new Bucket(dir.getName()))
.toList();
.map(dir -> new FileBucket(dir.getName()))
.toList();
}
/**
@@ -104,11 +101,11 @@ public class LocalFileTemplate implements FileTemplate {
*/
@Override
@SneakyThrows
public S3Object getObject(String bucketName, String objectName) {
public FileObject getObject(String bucketName, String objectName) {
String dir = properties.getLocal().getBasePath() + FileUtil.FILE_SEPARATOR + bucketName;
S3Object s3Object = new S3Object();
s3Object.setObjectContent(FileUtil.getInputStream(dir + FileUtil.FILE_SEPARATOR + objectName));
return s3Object;
File file = FileUtil.file(dir + FileUtil.FILE_SEPARATOR + objectName);
return new FileObject(bucketName, objectName, Files.probeContentType(file.toPath()), file.length(),
FileUtil.getInputStream(file));
}
/**
@@ -119,7 +116,7 @@ public class LocalFileTemplate implements FileTemplate {
* @return 二进制流 API Documentation</a>
*/
@Override
public S3Object getObject(String bucketName, String dir, String objectName) {
public FileObject getObject(String bucketName, String dir, String objectName) {
if (StrUtil.isNotBlank(dir)) {
bucketName = bucketName + FileUtil.FILE_SEPARATOR + dir;
}
@@ -154,19 +151,18 @@ public class LocalFileTemplate implements FileTemplate {
* @param bucketName bucket名称
* @param prefix 前缀
* @param recursive 是否递归查询
* @return S3ObjectSummary 列表
* @return FileObjectSummary 列表
* @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/ListObjects">AWS
* API Documentation</a>
*/
@Override
public List<S3ObjectSummary> getAllObjectsByPrefix(String bucketName, String prefix, boolean recursive) {
public List<FileObjectSummary> getAllObjectsByPrefix(String bucketName, String prefix, boolean recursive) {
String dir = properties.getLocal().getBasePath() + FileUtil.FILE_SEPARATOR + bucketName;
return Arrays.stream(FileUtil.ls(dir)).filter(file -> file.getName().startsWith(prefix)).map(file -> {
S3ObjectSummary summary = new S3ObjectSummary();
summary.setKey(file.getName());
return summary;
}).toList();
return Arrays.stream(FileUtil.ls(dir))
.filter(file -> file.getName().startsWith(prefix))
.map(file -> new FileObjectSummary(bucketName, file.getName(), file.length(), null))
.toList();
}
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2026, lengleng All rights reserved.
* Copyright (c) 2018-2025, lengleng All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -19,12 +19,14 @@ package com.pig4cloud.pigx.common.file.oss;
import com.pig4cloud.pigx.common.file.core.FileProperties;
import com.pig4cloud.pigx.common.file.core.FileTemplate;
import com.pig4cloud.pigx.common.file.core.OssFileTypeCondition;
import com.pig4cloud.pigx.common.file.oss.http.OssEndpoint;
import com.pig4cloud.pigx.common.file.oss.service.OssTemplate;
import lombok.AllArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Primary;
/**
@@ -41,7 +43,7 @@ public class OssAutoConfiguration {
@Bean
@Primary
@ConditionalOnMissingBean(OssTemplate.class)
@ConditionalOnProperty(name = "file.oss.enable", havingValue = "true")
@Conditional(OssFileTypeCondition.class)
public FileTemplate ossTemplate() {
return new OssTemplate(properties);
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2026, lengleng All rights reserved.
* Copyright (c) 2018-2025, lengleng All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -19,6 +19,7 @@ package com.pig4cloud.pigx.common.file.oss;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.DeprecatedConfigurationProperty;
/**
* aws 配置信息
@@ -35,6 +36,12 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "oss")
public class OssProperties {
/**
* 是否开启
*/
@Deprecated
private Boolean enable;
/**
* 对象存储服务的URL
*/
@@ -80,6 +87,17 @@ public class OssProperties {
/**
* 跳过 MD5 检查
*/
private boolean skipMd5Check;
private boolean skipMd5Check = true;
/**
* 是否启用 AWS S3 streaming chunked encoding,S3 兼容服务默认关闭
*/
private Boolean chunkedEncodingEnabled = false;
@Deprecated
@DeprecatedConfigurationProperty(reason = "使用 file.type=oss 替代 file.oss.enable", replacement = "file.type")
public Boolean getEnable() {
return enable;
}
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2026, lengleng All rights reserved.
* Copyright (c) 2018-2025, lengleng All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -17,9 +17,9 @@
package com.pig4cloud.pigx.common.file.oss.http;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.pig4cloud.pigx.common.file.core.FileBucket;
import com.pig4cloud.pigx.common.file.core.FileObjectInfo;
import com.pig4cloud.pigx.common.file.core.FileObjectSummary;
import com.pig4cloud.pigx.common.file.oss.service.OssTemplate;
import lombok.AllArgsConstructor;
import lombok.Cleanup;
@@ -55,7 +55,7 @@ public class OssEndpoint {
*/
@SneakyThrows
@PostMapping("/bucket/{bucketName}")
public Bucket createBucker(@PathVariable String bucketName) {
public FileBucket createBucker(@PathVariable String bucketName) {
template.createBucket(bucketName);
return template.getBucket(bucketName).get();
@@ -64,13 +64,13 @@ public class OssEndpoint {
@SneakyThrows
@GetMapping("/bucket")
public List<Bucket> getBuckets() {
public List<FileBucket> getBuckets() {
return template.getAllBuckets();
}
@SneakyThrows
@GetMapping("/bucket/{bucketName}")
public Bucket getBucket(@PathVariable String bucketName) {
public FileBucket getBucket(@PathVariable String bucketName) {
return template.getBucket(bucketName).orElseThrow(() -> new IllegalArgumentException("Bucket Name not found!"));
}
@@ -86,7 +86,7 @@ public class OssEndpoint {
*/
@SneakyThrows
@PostMapping("/object/{bucketName}")
public S3Object createObject(@RequestBody MultipartFile object, @PathVariable String bucketName) {
public FileObjectInfo createObject(@RequestBody MultipartFile object, @PathVariable String bucketName) {
String name = object.getOriginalFilename();
@Cleanup
InputStream inputStream = object.getInputStream();
@@ -97,7 +97,7 @@ public class OssEndpoint {
@SneakyThrows
@PostMapping("/object/{bucketName}/{objectName}")
public S3Object createObject(@RequestBody MultipartFile object, @PathVariable String bucketName,
public FileObjectInfo createObject(@RequestBody MultipartFile object, @PathVariable String bucketName,
@PathVariable String objectName) {
@Cleanup
InputStream inputStream = object.getInputStream();
@@ -108,7 +108,7 @@ public class OssEndpoint {
@SneakyThrows
@GetMapping("/object/{bucketName}/{objectName}")
public List<S3ObjectSummary> filterObject(@PathVariable String bucketName, @PathVariable String objectName) {
public List<FileObjectSummary> filterObject(@PathVariable String bucketName, @PathVariable String objectName) {
return template.getAllObjectsByPrefix(bucketName, objectName, true);
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2026, lengleng All rights reserved.
* Copyright (c) 2018-2025, lengleng All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -17,32 +17,40 @@
package com.pig4cloud.pigx.common.file.oss.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.StrUtil;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.util.IOUtils;
import com.pig4cloud.pigx.common.file.core.FileProperties;
import com.pig4cloud.pigx.common.file.core.FileTemplate;
import lombok.Cleanup;
import com.pig4cloud.pigx.common.file.core.*;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.util.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
/**
* aws-s3 通用存储操作 支持所有兼容s3协议的云存储: {阿里云OSS,腾讯云COS,七牛云,京东云,minio 等}
* S3 通用存储操作,支持所有兼容 S3 协议的云存储。
*
* @author lengleng
* @author 858695266
@@ -50,21 +58,25 @@ import java.util.*;
* @since 1.0
*/
@RequiredArgsConstructor
public class OssTemplate implements InitializingBean, FileTemplate {
public class OssTemplate implements InitializingBean, DisposableBean, FileTemplate {
private static final String DEFAULT_REGION = "us-east-1";
private final FileProperties properties;
private AmazonS3 amazonS3;
private S3Client s3Client;
private S3Presigner s3Presigner;
/**
* 创建bucket
*
* @param bucketName bucket名称
*/
@SneakyThrows
@Override
public void createBucket(String bucketName) {
if (!amazonS3.doesBucketExistV2(bucketName)) {
amazonS3.createBucket((bucketName));
if (!bucketExists(bucketName)) {
s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
}
}
@@ -75,9 +87,9 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/ListBuckets">AWS
* API Documentation</a>
*/
@SneakyThrows
public List<Bucket> getAllBuckets() {
return amazonS3.listBuckets();
@Override
public List<FileBucket> getAllBuckets() {
return s3Client.listBuckets().buckets().stream().map(this::toFileBucket).toList();
}
/**
@@ -85,9 +97,8 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/ListBuckets">AWS
* API Documentation</a>
*/
@SneakyThrows
public Optional<Bucket> getBucket(String bucketName) {
return amazonS3.listBuckets().stream().filter(b -> b.getName().equals(bucketName)).findFirst();
public Optional<FileBucket> getBucket(String bucketName) {
return getAllBuckets().stream().filter(b -> b.getName().equals(bucketName)).findFirst();
}
/**
@@ -96,9 +107,9 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* "http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/DeleteBucket">AWS API
* Documentation</a>
*/
@SneakyThrows
@Override
public void removeBucket(String bucketName) {
amazonS3.deleteBucket(bucketName);
s3Client.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
}
/**
@@ -107,14 +118,18 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @param bucketName bucket名称
* @param prefix 前缀
* @param recursive 是否递归查询
* @return S3ObjectSummary 列表
* @return FileObjectSummary 列表
* @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/ListObjects">AWS
* API Documentation</a>
*/
@SneakyThrows
public List<S3ObjectSummary> getAllObjectsByPrefix(String bucketName, String prefix, boolean recursive) {
ObjectListing objectListing = amazonS3.listObjects(bucketName, prefix);
return new ArrayList<>(objectListing.getObjectSummaries());
@Override
public List<FileObjectSummary> getAllObjectsByPrefix(String bucketName, String prefix, boolean recursive) {
ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix);
if (!recursive) {
builder.delimiter(StrUtil.SLASH);
}
ListObjectsV2Response response = s3Client.listObjectsV2(builder.build());
return response.contents().stream().map(object -> toFileObjectSummary(bucketName, object)).toList();
}
/**
@@ -124,15 +139,15 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @param objectName 文件名称
* @param expires 过期时间 <=7
* @return url
* @see AmazonS3#generatePresignedUrl(String bucketName, String key, Date expiration)
*/
@SneakyThrows
public String getObjectURL(String bucketName, String objectName, Integer expires) {
Date date = new Date();
Calendar calendar = new GregorianCalendar();
calendar.setTime(date);
calendar.add(Calendar.DAY_OF_MONTH, expires);
URL url = amazonS3.generatePresignedUrl(bucketName, objectName, calendar.getTime());
GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(objectName).build();
GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder()
.signatureDuration(Duration.ofDays(expires))
.getObjectRequest(getObjectRequest)
.build();
PresignedGetObjectRequest presignedGetObjectRequest = s3Presigner.presignGetObject(presignRequest);
URL url = presignedGetObjectRequest.url();
return url.toString();
}
@@ -145,12 +160,12 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/GetObject">AWS
* API Documentation</a>
*/
@SneakyThrows
public S3Object getObject(String bucketName, String objectName) {
if (properties.getOss().isSkipMd5Check()) {
System.setProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation", "false");
}
return amazonS3.getObject(bucketName, objectName);
@Override
public FileObject getObject(String bucketName, String objectName) {
GetObjectRequest request = GetObjectRequest.builder().bucket(bucketName).key(objectName).build();
ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(request, ResponseTransformer.toInputStream());
GetObjectResponse response = stream.response();
return new FileObject(bucketName, objectName, response.contentType(), response.contentLength(), stream);
}
/**
@@ -162,9 +177,9 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @return 二进制流 API Documentation</a>
*/
@Override
public S3Object getObject(String bucketName, String dir, String objectName) {
public FileObject getObject(String bucketName, String dir, String objectName) {
if (StrUtil.isNotBlank(dir)) {
objectName = dir + "/" + objectName;
objectName = dir + StrUtil.SLASH + objectName;
}
return getObject(bucketName, objectName);
}
@@ -177,6 +192,7 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @param stream 文件流
* @throws Exception
*/
@Override
public void putObject(String bucketName, String objectName, InputStream stream) throws Exception {
putObject(bucketName, objectName, stream, stream.available(), "application/octet-stream");
}
@@ -190,6 +206,7 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @param contextType 文件类型
* @throws Exception
*/
@Override
public void putObject(String bucketName, String objectName, InputStream stream, String contextType)
throws Exception {
putObject(bucketName, objectName, stream, stream.available(), contextType);
@@ -209,10 +226,8 @@ public class OssTemplate implements InitializingBean, FileTemplate {
public void putObject(String bucketName, String dir, String objectName, InputStream stream, String contextType)
throws Exception {
if (StrUtil.isNotBlank(dir)) {
// dir 路径为 a/b
objectName = dir + FileUtil.FILE_SEPARATOR + objectName;
objectName = dir + StrUtil.SLASH + objectName;
}
putObject(bucketName, objectName, stream, stream.available(), contextType);
}
@@ -228,17 +243,22 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/PutObject">AWS
* API Documentation</a>
*/
public PutObjectResult putObject(String bucketName, String objectName, InputStream stream, long size,
String contextType) throws Exception {
// String fileName = getFileName(objectName);
byte[] bytes = IOUtils.toByteArray(stream);
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(size);
objectMetadata.setContentType(contextType);
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
// 上传
return amazonS3.putObject(bucketName, objectName, byteArrayInputStream, objectMetadata);
public FileObjectInfo putObject(String bucketName, String objectName, InputStream stream, long size,
String contextType) throws Exception {
PutObjectRequest.Builder request = PutObjectRequest.builder().bucket(bucketName).key(objectName);
if (StrUtil.isNotBlank(contextType)) {
request.contentType(contextType);
}
Path tempFile = Files.createTempFile("pigx-oss-", ".tmp");
try {
Files.copy(stream, tempFile, java.nio.file.StandardCopyOption.REPLACE_EXISTING);
long contentLength = Files.size(tempFile);
request.contentLength(contentLength);
PutObjectResponse response = s3Client.putObject(request.build(), RequestBody.fromFile(tempFile));
return new FileObjectInfo(bucketName, objectName, contextType, contentLength, response.eTag(), null);
} finally {
deleteTempFile(tempFile);
}
}
/**
@@ -249,15 +269,15 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/GetObject">AWS
* API Documentation</a>
*/
public S3Object getObjectInfo(String bucketName, String objectName) throws Exception {
@Cleanup
S3Object object = amazonS3.getObject(bucketName, objectName);
return object;
public FileObjectInfo getObjectInfo(String bucketName, String objectName) {
HeadObjectResponse response = s3Client
.headObject(HeadObjectRequest.builder().bucket(bucketName).key(objectName).build());
return new FileObjectInfo(bucketName, objectName, response.contentType(), response.contentLength(),
response.eTag(), response.lastModified());
}
/**
* 删除文件
*
* @param bucketName bucket名称
* @param objectName 文件名称
* @throws Exception
@@ -265,28 +285,79 @@ public class OssTemplate implements InitializingBean, FileTemplate {
* "http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/DeleteObject">AWS API
* Documentation</a>
*/
@Override
public void removeObject(String bucketName, String objectName) throws Exception {
amazonS3.deleteObject(bucketName, objectName);
s3Client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectName).build());
}
@Override
public void afterPropertiesSet() {
System.setProperty("aws.java.v1.disableDeprecationAnnouncement", "true");
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setMaxConnections(properties.getOss().getMaxConnections());
AwsClientBuilder.EndpointConfiguration endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(
properties.getOss().getEndpoint(), properties.getOss().getRegion());
AWSCredentials awsCredentials = new BasicAWSCredentials(properties.getOss().getAccessKey(),
properties.getOss().getSecretKey());
AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
this.amazonS3 = AmazonS3Client.builder()
.withEndpointConfiguration(endpointConfiguration)
.withClientConfiguration(clientConfiguration)
.withCredentials(awsCredentialsProvider)
.disableChunkedEncoding()
.withPathStyleAccessEnabled(properties.getOss().getPathStyleAccess())
Region region = Region.of(StrUtil.blankToDefault(properties.getOss().getRegion(), DEFAULT_REGION));
AwsCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(
AwsBasicCredentials.create(properties.getOss().getAccessKey(), properties.getOss().getSecretKey()));
S3Configuration serviceConfiguration = S3Configuration.builder()
.pathStyleAccessEnabled(properties.getOss().getPathStyleAccess())
.chunkedEncodingEnabled(properties.getOss().getChunkedEncodingEnabled())
.build();
RequestChecksumCalculation requestChecksumCalculation = properties.getOss().isSkipMd5Check()
? RequestChecksumCalculation.WHEN_REQUIRED
: RequestChecksumCalculation.WHEN_SUPPORTED;
ResponseChecksumValidation responseChecksumValidation = properties.getOss().isSkipMd5Check()
? ResponseChecksumValidation.WHEN_REQUIRED
: ResponseChecksumValidation.WHEN_SUPPORTED;
S3ClientBuilder clientBuilder = S3Client.builder()
.region(region)
.credentialsProvider(credentialsProvider)
.serviceConfiguration(serviceConfiguration)
.requestChecksumCalculation(requestChecksumCalculation)
.responseChecksumValidation(responseChecksumValidation);
S3Presigner.Builder presignerBuilder = S3Presigner.builder()
.region(region)
.credentialsProvider(credentialsProvider)
.serviceConfiguration(serviceConfiguration);
if (StrUtil.isNotBlank(properties.getOss().getEndpoint())) {
URI endpoint = URI.create(properties.getOss().getEndpoint());
clientBuilder.endpointOverride(endpoint);
presignerBuilder.endpointOverride(endpoint);
}
this.s3Client = clientBuilder.build();
this.s3Presigner = presignerBuilder.build();
}
@Override
public void destroy() {
if (s3Presigner != null) {
s3Presigner.close();
}
if (s3Client != null) {
s3Client.close();
}
}
private boolean bucketExists(String bucketName) {
try {
s3Client.headBucket(HeadBucketRequest.builder().bucket(bucketName).build());
return true;
} catch (NoSuchBucketException ex) {
return false;
} catch (S3Exception ex) {
if (ex.statusCode() == 404) {
return false;
}
throw ex;
}
}
private FileBucket toFileBucket(Bucket bucket) {
return new FileBucket(bucket.name(), bucket.creationDate());
}
private FileObjectSummary toFileObjectSummary(String bucketName, S3Object object) {
return new FileObjectSummary(bucketName, object.key(), object.size(), object.lastModified());
}
private void deleteTempFile(Path tempFile) throws IOException {
Files.deleteIfExists(tempFile);
}
}
@@ -0,0 +1,142 @@
{
"groups": [
{
"name": "file",
"type": "com.pig4cloud.pigx.common.file.core.FileProperties",
"sourceType": "com.pig4cloud.pigx.common.file.core.FileProperties"
},
{
"name": "file.local",
"type": "com.pig4cloud.pigx.common.file.local.LocalFileProperties",
"sourceType": "com.pig4cloud.pigx.common.file.core.FileProperties"
},
{
"name": "file.oss",
"type": "com.pig4cloud.pigx.common.file.oss.OssProperties",
"sourceType": "com.pig4cloud.pigx.common.file.core.FileProperties"
}
],
"properties": [
{
"name": "file.type",
"type": "com.pig4cloud.pigx.common.file.core.FileType",
"description": "文件存储类型,可选 local 或 oss。",
"sourceType": "com.pig4cloud.pigx.common.file.core.FileProperties",
"defaultValue": "local"
},
{
"name": "file.bucket-name",
"type": "java.lang.String",
"description": "默认的存储桶名称。",
"sourceType": "com.pig4cloud.pigx.common.file.core.FileProperties",
"defaultValue": "local"
},
{
"name": "file.local.base-path",
"type": "java.lang.String",
"description": "本地文件存储根路径。",
"sourceType": "com.pig4cloud.pigx.common.file.local.LocalFileProperties"
},
{
"name": "file.local.enable",
"type": "java.lang.Boolean",
"description": "旧版本地文件开关,已由 file.type=local 替代。",
"sourceType": "com.pig4cloud.pigx.common.file.local.LocalFileProperties",
"deprecated": true,
"deprecation": {
"reason": "使用 file.type=local 替代 file.local.enable",
"replacement": "file.type"
}
},
{
"name": "file.oss.enable",
"type": "java.lang.Boolean",
"description": "旧版 OSS 文件开关,已由 file.type=oss 替代。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties",
"deprecated": true,
"deprecation": {
"reason": "使用 file.type=oss 替代 file.oss.enable",
"replacement": "file.type"
}
},
{
"name": "file.oss.endpoint",
"type": "java.lang.String",
"description": "对象存储服务 Endpoint,例如 http://127.0.0.1:9000。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties"
},
{
"name": "file.oss.custom-domain",
"type": "java.lang.String",
"description": "对象访问自定义域名。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties"
},
{
"name": "file.oss.app-id",
"type": "java.lang.String",
"description": "对象存储应用 ID。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties"
},
{
"name": "file.oss.path-style-access",
"type": "java.lang.Boolean",
"description": "是否使用 path-style 访问模式。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties",
"defaultValue": true
},
{
"name": "file.oss.region",
"type": "java.lang.String",
"description": "对象存储区域。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties"
},
{
"name": "file.oss.access-key",
"type": "java.lang.String",
"description": "对象存储 Access Key。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties"
},
{
"name": "file.oss.secret-key",
"type": "java.lang.String",
"description": "对象存储 Secret Key。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties"
},
{
"name": "file.oss.max-connections",
"type": "java.lang.Integer",
"description": "对象存储客户端最大并发连接数。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties",
"defaultValue": 100
},
{
"name": "file.oss.skip-md5-check",
"type": "java.lang.Boolean",
"description": "是否跳过 MD5 校验。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties",
"defaultValue": false
},
{
"name": "file.oss.chunked-encoding-enabled",
"type": "java.lang.Boolean",
"description": "是否启用 AWS S3 streaming chunked encoding,S3 兼容服务建议关闭。",
"sourceType": "com.pig4cloud.pigx.common.file.oss.OssProperties",
"defaultValue": false
}
],
"hints": [
{
"name": "file.type",
"values": [
{
"value": "local",
"description": "使用本地文件系统。"
},
{
"value": "oss",
"description": "使用 S3 兼容对象存储。"
}
]
}
]
}