From 26d968ec55bb669412429fca19b388ae8bdc10b4 Mon Sep 17 00:00:00 2001 From: dzc <2388224167@qq.com> Date: Thu, 29 Oct 2026 15:46:08 +0800 Subject: [PATCH] =?UTF-8?q?feat:=E6=97=A5=E5=BF=97=E6=95=8F=E6=84=9F?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/file-access/README.md | 57 +++- docs/file-access/config.example.yml | 5 + docs/file-access/verification.md | 23 ++ .../core/file/FileAccessProperties.java | 15 + .../common/core/file/FileAccessSignature.java | 7 +- .../common/core/file/FileAccessSigner.java | 16 + .../common/core/file/FileUploadSignature.java | 24 ++ .../common/core/util/AuditLogSanitizer.java | 318 ++++++++++++++++++ .../core/util/AuditLogSanitizerTest.java | 176 ++++++++++ pjcloud-common/pjcloud-common-excel/README.md | 6 +- pjcloud-common/pjcloud-common-log/README.md | 14 + .../common/log/LogAutoConfiguration.java | 4 +- .../common/log/aspect/SysLogAspect.java | 12 +- .../log/config/PjcloudLogProperties.java | 7 +- .../common/log/event/SysLogListener.java | 53 +-- .../pjcloud/common/log/util/SysLogUtils.java | 73 +++- .../common/log/SysLogSanitizationTest.java | 252 ++++++++++++++ .../config/FileAccessFeignConfiguration.java | 13 + .../pjcloud/admin/api/vo/SysLogExcelVO.java | 71 ++++ .../admin/controller/SysFileController.java | 20 +- .../file/FileUploadExceptionHandler.java | 17 + .../admin/service/file/FileUploadGuard.java | 83 +++++ .../service/file/FileUploadNonceStore.java | 8 + .../admin/service/file/FileUploadPolicy.java | 203 +++++++++++ .../file/FileUploadRejectedException.java | 7 + .../file/RedisFileUploadNonceStore.java | 17 + .../service/impl/SysFileServiceImpl.java | 19 +- .../impl/SysLogBatchExportService.java | 57 ++-- .../admin/service/impl/SysLogServiceImpl.java | 9 +- .../admin/service/log/SysLogSanitizer.java | 51 +++ .../utils/excel/SysLogExportStyleHandler.java | 152 +++++++++ .../admin/service/file/FileAccessTest.java | 2 +- .../admin/service/file/FileReadTest.java | 6 +- .../file/FileUploadControllerTest.java | 104 ++++++ .../service/file/FileUploadGuardTest.java | 99 ++++++ .../service/file/FileUploadPolicyTest.java | 95 ++++++ .../service/file/FileUploadTestSupport.java | 43 +++ .../service/impl/SysLogBatchExportTest.java | 175 ++++++++-- .../impl/SysLogExportControllerTest.java | 40 ++- .../service/impl/SysLogSanitizationTest.java | 167 +++++++++ .../impl/SysSystemConfigLogIsolationTest.java | 47 +++ .../test/resources/file-upload-vectors.json | 32 ++ scripts/generate-file-access-config.py | 11 + .../tests/test_generate_file_access_config.py | 20 ++ test-vectors/file-upload.json | 32 ++ 45 files changed, 2515 insertions(+), 147 deletions(-) create mode 100644 pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileUploadSignature.java create mode 100644 pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/util/AuditLogSanitizer.java create mode 100644 pjcloud-common/pjcloud-common-core/src/test/java/com/bsd/pjcloud/common/core/util/AuditLogSanitizerTest.java create mode 100644 pjcloud-common/pjcloud-common-log/README.md create mode 100644 pjcloud-common/pjcloud-common-log/src/test/java/com/bsd/pjcloud/common/log/SysLogSanitizationTest.java create mode 100644 pjcloud-upms/pjcloud-upms-api/src/main/java/com/bsd/pjcloud/admin/api/vo/SysLogExcelVO.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadExceptionHandler.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadGuard.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadNonceStore.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadPolicy.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadRejectedException.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/RedisFileUploadNonceStore.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/log/SysLogSanitizer.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/utils/excel/SysLogExportStyleHandler.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadControllerTest.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadGuardTest.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadPolicyTest.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadTestSupport.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogSanitizationTest.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysSystemConfigLogIsolationTest.java create mode 100644 pjcloud-upms/pjcloud-upms-biz/src/test/resources/file-upload-vectors.json create mode 100644 test-vectors/file-upload.json diff --git a/docs/file-access/README.md b/docs/file-access/README.md index 730092889..2339639dc 100644 --- a/docs/file-access/README.md +++ b/docs/file-access/README.md @@ -1,7 +1,9 @@ -# 文件访问签名接入与发布 +# 文件读取与上传签名接入及发布 本次保护 `/admin/sys-file/oss/file` 的全部文件。默认 `observe` 只记录签名校验结果；只有加载完整配置并切到 `enforce` 后，普通裸链接才会被拒绝。旧 Dart 图片兼容是独立开关。客户端密钥可提取、UA 可伪造、有效接口地址仍可被模拟请求使用，这是一层应用级校验，不是用户或租户文件授权。 +上传入口 `/admin/sys-file/upload`、`/admin/sys-file/inner-upload`、`/admin/sys-file/qn/upload` 同步使用应用签名和服务端文件白名单。上传复用这套配置和密钥，不增加环境密钥组；读取签名不能用于上传。 + ## 配置准备 服务端配置见 `config.example.yml`。密钥为至少 32 字节随机数据的 Base64 字符串，HMAC 使用 **Base64 解码后的字节**。不要复用 OAuth 客户端信息或密码加密密钥。 @@ -62,6 +64,51 @@ Base64URL无填充编码的UTF-8文件名 后台会截获本系统文件 `` 链接，通过已有 Blob 下载工具处理；H5/小程序富文本文件链接改走图片预览或下载后打开本地文件。复制出来的接口地址不再承诺可直接在浏览器打开。Flutter 原生下载、Feign 等没有浏览器 Fetch Metadata 的请求仍执行签名/旧 Dart 规则；不支持这些请求头的旧浏览器同样不能靠本规则识别。这项规则用于限制普通浏览器导航，不能防止脚本携带有效签名并模拟请求，也不能阻止已经获准显示的图片被保存。 +## 上传身份与文件类型 + +现有登录鉴权及 `@Inner` 内部调用限制保留。每次 POST multipart 上传只允许一个名为 `file` 的文件部分，通过下列请求头传递身份，不把上传凭据放进 URL 或业务字段： + +| 请求头 | 内容 | +| --- | --- | +| `X-File-App-Id` | `admin/h5/mini/flutter/service`，`package` 不可上传 | +| `X-File-Key-Id` | 现有 kid，如 `v1` | +| `X-File-Timestamp` | Unix 秒级时间戳 | +| `X-File-Nonce` | 16～64 位字母、数字、`_`、`-`，每次请求重新生成 | +| `X-File-Signature` | HMAC-SHA256 的 64 位小写十六进制 | + +签名原文按 LF 连接，无末尾换行；密钥仍使用 Base64 解码后的字节： + +```text +v1 +oss-file-upload +POST +target +appId +kid +ts +nonce +``` + +`target` 分别为 `upload`、`inner-upload`、`qn-upload`，三种签名不能互换。`inner-upload` 仅接受 `service`。签名用于应用身份和本次上传操作校验，文件内容由服务端独立检查。默认有效 300 秒，允许客户端时间领先最多 60 秒；复用现有匿名校时接口。严格模式下 Redis 原子 `SET NX` 消耗 nonce，重复请求返回 403；Redis 或配置不可用返回 500，不自动放行。不增加 Dart UA 上传兼容。 + +`file-access.upload` 默认策略如下，省略配置时 Java 使用相同默认值： + +| 文件类别 | 白名单后缀 | +| --- | --- | +| 图片 | jpg/jpeg/png/gif/webp/bmp/ico/svg | +| 文档和安装包 | pdf/apk/doc/docx/xls/xlsx/ppt/pptx | +| 文本和压缩包 | txt/csv/zip | +| 音视频 | mp4/mov/webm/avi/mp3/wav/m4a/aac/ogg/amr/flac | +| 仅内部上传额外允许 | log/gz，供终端日志使用 | + +默认单文件最大 200 MiB，仍受现有网关、反向代理和 Servlet 更低限制约束。SVG 最大 1 MiB，只接受静态内容，拒绝脚本、事件、外部引用和实体。服务层在写存储或文件记录前统一校验后缀、文件头及适用的图片尺寸/ZIP 容器结构；APK 检查二进制 AndroidManifest，Office 新格式检查对应容器入口。存储后缀和 MIME 由服务端规范化，不信任客户端 Content-Type。这里的格式检查不等于病毒或文档宏扫描。 + +缺少或错误身份、过期、重放为 403；重复或畸形头、多个文件部分为 400；不支持类型或内容不符为 415；超过本策略大小为 413。错误响应禁止缓存。文件类型检查在 `observe` 下仍执行，观察模式只放宽应用签名准入。监控为 `file.upload.requests`，标签 `outcome/app/mode`，不记录密钥、签名和登录 Token。 + +后台 Axios、通用 el-upload 和富文本上传，H5/小程序的 `uni.uploadFile` 拦截器，以及 Flutter `HttpManage` 为本系统上传路径生成新请求头；导入接口和第三方上传地址不追加本协议。`RemoteFileService` 专属 Feign 拦截器为每次内部上传生成 service 签名。上传成功返回值和保存的文件引用继续为稳定原始地址。 + +**已有读取严格模式的环境，先发布客户端及调用文件服务的 Java 服务，再发布 UPMS。** 旧 UPMS 忽略新增头；新版 UPMS 在 `enforce` 下会拒绝没有上传签名的旧网页/服务请求。Java 调用方须随公共依赖一同更新，避免只升级 UPMS。旧 Flutter 当前通过硬件业务接口上传取证和日志，再由 Java 调用文件中心，这条链路无需旧设备立即升级；不要把这一结论扩展到其他直接调用文件中心的旧客户端。必要时切回 `observe` 恢复旧身份请求，文件白名单仍然生效。 + ## 入口与数据约束 | 部分 | 接入位置与行为 | @@ -101,17 +148,17 @@ Base64URL无填充编码的UTF-8文件名 三个前端分别执行： ```bash -node --test tests/file-access.test.cjs +node --test tests/file-access.test.cjs tests/file-upload.test.cjs ``` -后台执行 `pnpm build:prod`，H5 执行 `pnpm build:h5`，小程序执行 `pnpm build:mp-weixin`。跨语言固定向量在各项目 `test-vectors/file-access.json`，其中的密钥是公开测试数据，不能用于部署。 +后台执行 `pnpm build:prod`，H5 执行 `pnpm build:h5`，小程序执行 `pnpm build:mp-weixin`。跨语言固定向量在各项目 `test-vectors/file-access.json` 和 `test-vectors/file-upload.json`，其中的密钥是公开测试数据，不能用于部署。配置生成器回归执行 `python3 -m unittest discover -s scripts/tests`。 Java 定向回归： ```bash mvn -pl pjcloud-upms/pjcloud-upms-biz,pjcloud-integrated/pjcloud-integrated-biz,pjcloud-wallet/pjcloud-wallet-biz,pjcloud-hardware/pjcloud-hardware-biz,pjcloud-canteen/pjcloud-canteen-biz \ -am test \ - -Dtest=FileAccessTest,FileReadTest,AppVersionFileAccessTest,WalletManageBatchExportTest,WalletLedgerBatchExportTest,HardBatchExportFileTest,HardBatchExportQueryTest \ + -Dtest=FileAccessTest,FileReadTest,FileUploadGuardTest,FileUploadPolicyTest,FileUploadControllerTest,AppVersionFileAccessTest,WalletManageBatchExportTest,WalletLedgerBatchExportTest,HardBatchExportFileTest,HardBatchExportQueryTest \ -Dsurefire.failIfNoSpecifiedTests=false ``` @@ -119,7 +166,7 @@ Flutter 使用项目 FVM 3.27.4： ```bash fvm flutter test --no-pub --dart-define-from-file=.file-access/config.json \ - test/file_access_test.dart test/package_download_test.dart \ + test/file_access_test.dart test/file_upload_test.dart test/package_download_test.dart \ test/face_sync_plan_test.dart test/face_sync_pipeline_test.dart \ test/face_sync_failure_store_test.dart test/face_readiness_cache_test.dart \ test/customer_display_balance_test.dart test/customer_display_balance_widget_test.dart diff --git a/docs/file-access/config.example.yml b/docs/file-access/config.example.yml index 5cb5d1fed..12e7d7509 100644 --- a/docs/file-access/config.example.yml +++ b/docs/file-access/config.example.yml @@ -11,6 +11,11 @@ file-access: legacy-dart: enabled: true extensions: [jpg, jpeg, png, gif, webp, bmp, svg, ico] + upload: + ttl-seconds: 300 + max-file-bytes: 209715200 # 200 MiB; gateway/servlet limits may be lower + allowed-extensions: [jpg, jpeg, png, gif, webp, bmp, ico, svg, pdf, apk, doc, docx, xls, xlsx, ppt, pptx, txt, csv, zip, mp4, mov, webm, avi, mp3, wav, m4a, aac, ogg, amr, flac] + internal-extensions: [log, gz] # service-signed inner-upload only clients: admin: active-kid: v1 diff --git a/docs/file-access/verification.md b/docs/file-access/verification.md index af84ef0d7..c497f94ce 100644 --- a/docs/file-access/verification.md +++ b/docs/file-access/verification.md @@ -92,3 +92,26 @@ Java 定向测试及各端构建命令见 [接入与发布说明](README.md)。 - 后台生产、H5、微信小程序构建通过；Flutter 使用统一配置执行签名和安装包定向测试，8 项通过。 后续构建与配置准备以 README 中的统一路径为准。本文前面提到的分环境配置路径属于历史验证记录，不再作为当前操作指引。 + +## 上传身份与文件白名单补充 + +2026-10-29 在现有读取协议和同一套固定密钥上增加上传身份校验。普通、内部、七牛三个文件中心上传入口均接入，读取签名不能用于上传；上传签名绑定 POST、目标入口、appId、kid、时间和一次性 nonce。默认 5 分钟有效，严格模式通过 Redis 原子占用 nonce 防止重放；内部上传仅接受 service 身份，Dart UA 不作为上传凭据。 + +文件白名单覆盖图片、PDF、APK、Office、TXT/CSV、ZIP 和常用音视频；仅内部上传额外允许 LOG/GZ。服务层在存储前检查后缀、文件头及适用的容器结构、图片尺寸和静态 SVG 内容，并规范存储扩展名与 MIME。默认最大 200 MiB，SVG 最大 1 MiB，现有网关/Servlet 更低限制仍适用。观察模式只放宽签名准入，文件类型检查始终执行。格式识别不代表病毒或文档宏扫描。 + +| 验证 | 本次结果 | +| --- | --- | +| Java 上传测试 | 17 项通过，覆盖跨语言向量、入口隔离、重复头、过期、Dart UA 不放行、40 个并发请求只有一次 nonce 成功、Redis/错误配置失败关闭、实际 Feign multipart 编码、MockMvc 存储前拒绝及观察模式白名单 | +| 文件格式 | 覆盖常见图片、PDF、文本、Office 容器、ZIP、APK；拒绝改后缀/伪造 MIME、路径穿越、恶意 SVG、外部实体及过深 XML；最新构建的实际 Flutter Release APK 通过格式校验 | +| Java 相关回归 | 52 项通过，覆盖原文件读取、版本地址、日志导出、钱包导出、硬件导出；UPMS、wallet、hardware、integrated、canteen 及所需公共模块编译通过 | +| 后台 | 31 项通过，执行实际 Axios 拦截器、通用上传和富文本上传 setup；保留登录/租户头和 multipart 业务字段，失败文件不阻塞同批成功回填，签名不写进保存的富文本 | +| H5、小程序 | 各 21 项通过，执行实际 uni.uploadFile 拦截器；覆盖上传向量、每次请求新 nonce、校时及第三方/导入路径不签名 | +| Flutter | 签名、上传、安装包和兼容模式定向回归 21 项通过、2 项条件跳过；本次修改的文件工具、HTTP 管理器和上传测试定向 analyze 无诊断 | +| 构建 | 后台生产、H5、微信小程序及 Android Release 均通过；Flutter 使用 FVM 3.27.4 和 `.file-access/config.json`，APK 约 101.9 MB | +| 配置生成器 | 4 项通过，确认只有一套环境共用密钥、原密钥不变、默认上传策略补齐、已有定制白名单和大小限制不被覆盖 | +| TypeScript | 后台全量仍有原有 6 条解析错误，H5 仍有原有 4 条解析错误，小程序仍有原有 17 条 TS6504；后台上传相关组件及其依赖与 HEAD 基线比较均为 115 条既有诊断，无新增诊断，不能表述为全量类型检查通过 | +| 源码与密钥 | 五个项目 Git 候选文件未发现当前活动密钥，保留并发的日志导出、Flutter 调试兼容等改动；Diff 空白检查通过 | + +日志保存在本机 `/tmp/pj-upload-security-work/`。本次上传改造没有发布到运行中的 Java 服务，也没有修改 Nacos 或更换密钥。发布后的真实 multipart 联调、微信真机和旧 Flutter 设备验证仍需执行；MockMvc、内存 Feign 和本地构建不能代替现场验收。 + +已有 `enforce` 环境先发布客户端及携带新公共依赖的 Java 调用服务，再发布 UPMS，避免仅更新 UPMS 后拒绝旧上传请求。当前旧 Flutter 取证/日志先经过硬件业务接口，再由 Java 调用文件中心，服务端补签后不要求这些旧设备立即升级。`observe` 会允许旧上传身份请求，切到 `enforce` 才真正拦截未签名上传；白名单在两种模式下均有效。发布配置与详细头协议见 README 的“上传身份与文件类型”。 diff --git a/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessProperties.java b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessProperties.java index f0736eb82..688e46700 100644 --- a/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessProperties.java +++ b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessProperties.java @@ -18,6 +18,17 @@ public class FileAccessProperties implements InitializingBean { private List allowedOrigins = new ArrayList<>(); private Map clients = new LinkedHashMap<>(); private LegacyDart legacyDart = new LegacyDart(); + private Upload upload = new Upload(); + + @Data + public static class Upload { + private long ttlSeconds = 300; + private long maxFileBytes = 200L * 1024 * 1024; + private Set allowedExtensions = new HashSet<>(List.of("jpg", "jpeg", "png", "gif", "webp", "bmp", "ico", "svg", + "pdf", "apk", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "txt", "csv", "zip", + "mp4", "mov", "webm", "avi", "mp3", "wav", "m4a", "aac", "ogg", "amr", "flac")); + private Set internalExtensions = new HashSet<>(List.of("log", "gz")); + } @Data public static class Client { @@ -48,6 +59,10 @@ public class FileAccessProperties implements InitializingBean { || futureSkewSeconds < 0 || futureSkewSeconds > 300) { throw new IllegalStateException("Invalid file-access policy"); } + if (upload == null || upload.ttlSeconds < 60 || upload.ttlSeconds > 1800 || upload.maxFileBytes <= 0 + || upload.maxFileBytes > 1024L * 1024 * 1024 || upload.allowedExtensions == null || upload.internalExtensions == null) { + throw new IllegalStateException("Invalid file-access upload policy"); + } if (enforcing() || !publicBaseUrl.isBlank()) { try { java.net.URI uri = java.net.URI.create(publicBaseUrl); diff --git a/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessSignature.java b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessSignature.java index d78803aeb..33d885f1f 100644 --- a/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessSignature.java +++ b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessSignature.java @@ -17,13 +17,16 @@ public final class FileAccessSignature { } public static String sign(String fileName, String appId, String kid, long timestamp, String base64Key) { + return hmac(canonical(fileName, appId, kid, timestamp), base64Key); + } + + public static String hmac(String canonical, String base64Key) { try { byte[] key = Base64.getDecoder().decode(base64Key); if (key.length < 32) throw new IllegalArgumentException("File access key must contain at least 32 bytes"); Mac mac = Mac.getInstance("HmacSHA256"); mac.init(new SecretKeySpec(key, "HmacSHA256")); - return HexFormat.of().formatHex(mac.doFinal(canonical(fileName, appId, kid, timestamp) - .getBytes(StandardCharsets.UTF_8))); + return HexFormat.of().formatHex(mac.doFinal(canonical.getBytes(StandardCharsets.UTF_8))); } catch (IllegalArgumentException e) { throw e; } catch (Exception e) { diff --git a/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessSigner.java b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessSigner.java index da9760da0..cb6ab6300 100644 --- a/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessSigner.java +++ b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileAccessSigner.java @@ -31,6 +31,22 @@ public class FileAccessSigner { return result; } + public Map uploadHeaders(String target) { + FileAccessProperties.Client client = properties.getClients().get("service"); + if (client == null) throw new IllegalStateException("Missing file-access service signer configuration"); + String kid = client.getActiveKid(), key = properties.key("service", kid); + if (key == null || key.isBlank()) throw new IllegalStateException("Missing file-access service signer key"); + long ts = clock.instant().getEpochSecond(); + String nonce = UUID.randomUUID().toString().replace("-", ""); + Map headers = new LinkedHashMap<>(); + headers.put(FileUploadSignature.APP, "service"); + headers.put(FileUploadSignature.KID, kid); + headers.put(FileUploadSignature.TIMESTAMP, Long.toString(ts)); + headers.put(FileUploadSignature.NONCE, nonce); + headers.put(FileUploadSignature.SIGNATURE, FileUploadSignature.sign(target, "service", kid, ts, nonce, key)); + return headers; + } + public String signUrl(String source, String appId) { if (source == null || source.isBlank()) return source; String value = source.trim(); diff --git a/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileUploadSignature.java b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileUploadSignature.java new file mode 100644 index 000000000..cad09cd34 --- /dev/null +++ b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/file/FileUploadSignature.java @@ -0,0 +1,24 @@ +package com.bsd.pjcloud.common.core.file; + +import java.util.Set; + +/** Upload authorization is purpose-separated from read signatures and is never put in URLs. */ +public final class FileUploadSignature { + public static final String APP = "X-File-App-Id"; + public static final String KID = "X-File-Key-Id"; + public static final String TIMESTAMP = "X-File-Timestamp"; + public static final String NONCE = "X-File-Nonce"; + public static final String SIGNATURE = "X-File-Signature"; + public static final Set TARGETS = Set.of("upload", "inner-upload", "qn-upload"); + + private FileUploadSignature() { } + + public static String canonical(String target, String appId, String kid, long timestamp, String nonce) { + if (!TARGETS.contains(target)) throw new IllegalArgumentException("Invalid file upload target"); + return String.join("\n", "v1", "oss-file-upload", "POST", target, appId, kid, Long.toString(timestamp), nonce); + } + + public static String sign(String target, String appId, String kid, long timestamp, String nonce, String key) { + return FileAccessSignature.hmac(canonical(target, appId, kid, timestamp, nonce), key); + } +} diff --git a/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/util/AuditLogSanitizer.java b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/util/AuditLogSanitizer.java new file mode 100644 index 000000000..7e7ca3f7d --- /dev/null +++ b/pjcloud-common/pjcloud-common-core/src/main/java/com/bsd/pjcloud/common/core/util/AuditLogSanitizer.java @@ -0,0 +1,318 @@ +package com.bsd.pjcloud.common.core.util; + +import com.bsd.pjcloud.common.core.jackson.PjcloudJavaTimeModule; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamWriteConstraints; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import org.springframework.core.io.Resource; +import org.springframework.validation.BindingResult; +import org.springframework.web.multipart.MultipartFile; + +import java.io.*; +import java.lang.reflect.Array; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.*; +import java.util.regex.Pattern; + +/** 仅用于审计日志的副本过滤，不修改业务对象、业务 ObjectMapper 或系统配置。 */ +public final class AuditLogSanitizer { + public static final String HIDDEN = "参数已隐藏"; + private static final int MAX_DEPTH = 32; + private static final int MAX_NODES = 10000; + private static final ObjectMapper MAPPER = new ObjectMapper(JsonFactory.builder() + .streamReadConstraints(StreamReadConstraints.builder().maxNestingDepth(MAX_DEPTH).build()) + .streamWriteConstraints(StreamWriteConstraints.builder().maxNestingDepth(MAX_DEPTH).build()).build()) + .registerModule(new PjcloudJavaTimeModule()).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + private static final Set CREDENTIALS = Set.of( + "pass", "password", "passwd", "pwd", "credential", "credentials", "salt", + "captcha", "captchacode", "captchaverification", "captchaid", "verifycode", "verificationcode", + "validatecode", "smscode", "imagecode", "otp", "otpcode", "randomstr", "authcode", + "token", "tokenid", "authtoken", "accesstoken", "refreshtoken", "idtoken", "resettoken", "authorization", "authorizationcode", + "clientsecret", "appsecret", "accesssecret", "accesskeysecret", "secret", "secretkey", + "privatekey", "apikey", "accesskey", "sign", "signature", "cookie", "setcookie", "sessionid", "jsessionid", + "密码", "口令", "验证码", "短信验证码", "图形验证码", "令牌", "访问令牌", "刷新令牌", "密钥", "私钥"); + private static final Set AUTH_CONTEXT_FIELDS = Set.of("randomstr", "captcha", "captchaid", "captchaverification", + "captchacode", "verifycode", "verificationcode", "validatecode", "smscode", "otp", "otpcode", "granttype"); + private static final Pattern PASSWORD = Pattern.compile( + "(?:old|new|current|confirm|confirmation|repeat|re|login|pay|trade|transaction|user|admin|initial|original|temporary|encoded|encrypted|hashed)*(?:password|passwd|pwd|pass)(?:s|confirm|confirmation|repeat|again|hash|digest)?\\d*"); + private static final Pattern CREDENTIAL_FAMILY = Pattern.compile( + "[\\p{L}\\p{N}]*(?:(?:password|passwd|pwd)(?:s|confirm|confirmation|repeat|again|hash|digest)?" + + "|secret|secretkey|privatekey|token|tokenid|tokenkey|apikey|accesskey|(?:captcha|verify|verification|validate|sms|otp|auth)code)\\d*"); + private static final Pattern AUTH_PATH = Pattern.compile( + "(?i)(?:^|/)(?:oauth2?|login|applogin|captcha|verifycode|smscode)(?:/|$)" + + "|/register/(?:password|default-user)(?:/|$)|/user/(?:personal/password|fix-phone-as-password|check)(?:/|$)" + + "|/code/(?:image|check|verify)(?:/|$)"); + private static final Pattern LABEL = Pattern.compile("(? excluded; + + public AuditLogSanitizer(Collection additionalFields) { + Set fields = new HashSet<>(CREDENTIALS); + if (additionalFields != null) additionalFields.stream().filter(Objects::nonNull).map(AuditLogSanitizer::normalize) + .forEach(fields::add); + excluded = Set.copyOf(fields); + } + + public boolean isAuthenticationContext(String uri) { + return uri != null && AUTH_PATH.matcher(uri.split("[?#]", 2)[0]).find(); + } + + public boolean isSensitiveField(String name, boolean authentication) { + if (name == null) return false; + if (name.length() > 256) throw new IllegalArgumentException("Unsupported audit field length"); + String decoded = decode(name); + for (String part : decoded.split("[.\\[\\]/]+")) { + String key = normalize(part); + if (excluded.contains(key) || PASSWORD.matcher(key).matches() || CREDENTIAL_FAMILY.matcher(key).matches() + || (authentication && "code".equals(key))) return true; + } + return false; + } + + /** 历史 params 只接受可识别的 JSON 或表单，匿名值与损坏报文不回退为原文。 */ + public String sanitizeParameters(String value, String uri) { + if (value == null || value.isBlank() || HIDDEN.equals(value)) return value; + try { + String input = structuredInput(value); + if (jsonLike(input)) return MAPPER.writeValueAsString(clean(MAPPER.readTree(input), isAuthenticationContext(uri), false, 0, new Budget())); + if (formLike(input)) return form(input, isAuthenticationContext(uri), 0, new Budget()); + } catch (Exception ignored) { + // 异常原文可能包含请求值，不能输出异常消息或回退到原始内容。 + } + return HIDDEN; + } + + /** 方法参数和 DTO 立即转成过滤后的字符串快照，异步事件不携带原始业务对象。 */ + public String sanitizePayload(Object value, String uri) { + if (value instanceof CharSequence sequence) return sanitizeParameters(sequence.toString(), uri); + try { + return MAPPER.writeValueAsString(clean(value, isAuthenticationContext(uri), false, 0, new Budget())); + } catch (Exception ignored) { + return HIDDEN; + } + } + + /** 保留 URL 表单及重复参数的语义，操作副本而非 servlet 参数映射。 */ + public String sanitizeRequestParameters(Map parameters, String uri) { + try { + List pairs = new ArrayList<>(); + for (var entry : parameters.entrySet()) { + if (entry.getValue() == null) continue; + for (String value : entry.getValue()) pairs.add(encode(entry.getKey()) + "=" + encode(value == null ? "" : value)); + } + return form(String.join("&", pairs), isAuthenticationContext(uri), 0, new Budget()); + } catch (Exception ignored) { + return HIDDEN; + } + } + + /** 普通错误文本保留；含可识别凭据或损坏结构化内容时隐藏，不尝试猜测秘密值边界。 */ + public String sanitizeText(String value, String uri) { + if (value == null || value.isBlank() || HIDDEN.equals(value)) return value; + try { + String input = structuredInput(value); + if (jsonLike(input) || formLike(input)) return sanitizeParameters(input, uri); + if (unsafeText(input, isAuthenticationContext(uri))) return HIDDEN; + return value; + } catch (Exception ignored) { + return HIDDEN; + } + } + + public String sanitizeUri(String uri) { + return sanitizeUri(uri, Map.of()); + } + + /** 新请求利用路径参数名称过滤；历史记录覆盖已知的认证路径。 */ + public String sanitizeUri(String uri, Map pathVariables) { + if (uri == null || uri.isBlank() || HIDDEN.equals(uri)) return uri; + try { + int hash = uri.indexOf('#'); + String fragment = hash < 0 ? "" : uri.substring(hash + 1); + String main = hash < 0 ? uri : uri.substring(0, hash); + int question = main.indexOf('?'); + String path = question < 0 ? main : main.substring(0, question); + boolean authentication = isAuthenticationContext(path); + String[] segments = path.split("/", -1); + for (int index = 0; index < segments.length; index++) { + String segment = decode(segments[index]); + for (var variable : pathVariables.entrySet()) { + if (isSensitiveField(variable.getKey(), authentication) && Objects.equals(segment, variable.getValue())) segments[index] = HIDDEN; + } + } + path = String.join("/", segments).replaceAll("(?i)(/appLogin/(?:zzdsso|getAuthToken)/)[^/?#]+", "$1" + HIDDEN) + .replaceAll("(?i)(https?://)[^/@\\s]+@", "$1" + HIDDEN + "@"); + if (unsafeText(path, authentication)) return HIDDEN; + if (question >= 0) { + String query = safeUriForm(main.substring(question + 1), authentication); + if (!query.isBlank()) path += "?" + query; + } + if (!fragment.isBlank()) { + String safeFragment = formLike(fragment) ? safeUriForm(fragment, authentication) + : authentication || unsafeText(fragment, false) ? HIDDEN : fragment; + path += "#" + safeFragment; + } + return path; + } catch (Exception ignored) { + return HIDDEN; + } + } + + private JsonNode clean(Object value, boolean authentication, boolean named, int depth, Budget budget) throws Exception { + budget.visit(depth); + if (value == null || value instanceof JsonNode node && node.isNull()) return MAPPER.nullNode(); + if (isUnloggable(value)) throw new UnsafePayload(); + if (value instanceof Map map) { + ObjectNode result = MAPPER.createObjectNode(); + boolean auth = authentication || authenticationMap(map); + for (var entry : map.entrySet()) { + if (!(entry.getKey() instanceof String key)) throw new UnsafePayload(); + if (!isSensitiveField(key, auth) && !isUnloggable(entry.getValue())) result.set(key, clean(entry.getValue(), auth, true, depth + 1, budget)); + } + return result; + } + if (value instanceof JsonNode node && node.isObject()) { + Map entries = new LinkedHashMap<>(); + node.fields().forEachRemaining(entry -> entries.put(entry.getKey(), entry.getValue())); + return clean(entries, authentication, named, depth + 1, budget); + } + if (value instanceof JsonNode node && node.isArray() + || !(value instanceof JsonNode) && value instanceof Iterable || value.getClass().isArray()) { + ArrayNode result = MAPPER.createArrayNode(); + Iterable items = value instanceof Iterable iterable ? iterable : arrayItems(value); + for (Object item : items) if (!isUnloggable(item)) result.add(clean(item, authentication, named, depth + 1, budget)); + return result; + } + if (value instanceof CharSequence || value instanceof JsonNode node && node.isTextual()) { + String text = value instanceof JsonNode node ? node.textValue() : value.toString(); + if (!named) throw new UnsafePayload(); + String input = structuredInput(text); + if (jsonLike(input)) return MAPPER.getNodeFactory().textNode(MAPPER.writeValueAsString(clean(MAPPER.readTree(input), authentication, false, depth + 1, budget))); + if (formLike(input)) return MAPPER.getNodeFactory().textNode(form(input, authentication, depth + 1, budget)); + return MAPPER.getNodeFactory().textNode(unsafeText(input, authentication) ? HIDDEN : text); + } + if (value instanceof JsonNode node && node.isValueNode() || value instanceof Number || value instanceof Boolean || value instanceof Character || value instanceof Enum) { + if (!named) throw new UnsafePayload(); + return value instanceof JsonNode node ? node.deepCopy() : MAPPER.valueToTree(value); + } + return clean(MAPPER.valueToTree(value), authentication, named, depth + 1, budget); + } + + public static boolean isUnloggable(Object value) { + return value instanceof ServletRequest || value instanceof ServletResponse || value instanceof BindingResult + || value instanceof MultipartFile || value instanceof Resource || value instanceof InputStream + || value instanceof OutputStream || value instanceof Reader || value instanceof Writer + || value instanceof File || value instanceof Path; + } + + private boolean authenticationMap(Map map) { + for (var entry : map.entrySet()) { + if (!(entry.getKey() instanceof String field)) continue; + String key = normalize(decode(field)); + if (AUTH_CONTEXT_FIELDS.contains(key)) return true; + Object value = entry.getValue(); + String text = value instanceof JsonNode node && node.isTextual() ? node.textValue() + : value instanceof CharSequence sequence ? sequence.toString() + : value instanceof Enum enumeration ? enumeration.name() : null; + if ("bizcode".equals(key) && "SMS_NORMAL_CODE".equalsIgnoreCase(text)) return true; + } + return false; + } + + private String form(String value, boolean authentication, int depth, Budget budget) throws Exception { + budget.visit(depth); + if (value.isBlank()) return ""; + List pairs = new ArrayList<>(); + Map context = new LinkedHashMap<>(); + for (String part : value.split("&", -1)) { + if (part.isBlank()) continue; + int equals = part.indexOf('='); + String key = decode(equals < 0 ? part : part.substring(0, equals)); + if (key.isBlank() || !FORM_KEY.matcher(key).matches()) throw new UnsafePayload(); + String text = equals < 0 ? "" : decode(part.substring(equals + 1)); + pairs.add(new String[]{key, text}); + context.put(key, text); + } + boolean auth = authentication || authenticationMap(context); + List safe = new ArrayList<>(); + for (String[] pair : pairs) if (!isSensitiveField(pair[0], auth)) { + JsonNode cleaned = clean(pair[1], auth, true, depth + 1, budget); + safe.add(encode(pair[0]) + "=" + encode(cleaned.textValue())); + } + return String.join("&", safe); + } + + private boolean unsafeText(String value, boolean authentication) { + if (AUTH_VALUE.matcher(value).find()) return true; + var matcher = LABEL.matcher(value); + while (matcher.find()) if (isSensitiveField(matcher.group(1), authentication)) return true; + return false; + } + + private String safeUriForm(String value, boolean authentication) { + try { + return form(value, authentication, 0, new Budget()); + } catch (Exception ignored) { + return HIDDEN; + } + } + + private static String structuredInput(String value) { + String input = value.trim(); + if (!jsonLike(input) && !formLike(input) && input.matches("(?s).*%[0-9a-fA-F]{2}.*")) input = decode(input); + return input; + } + + private static boolean jsonLike(String value) { + return value.startsWith("{") || value.startsWith("[") || value.startsWith("\""); + } + + private static boolean formLike(String value) { + int equals = value.indexOf('='); + return equals > 0 && FORM_KEY.matcher(value.substring(0, equals)).matches(); + } + + private static String normalize(String value) { + return value.toLowerCase(Locale.ROOT).replaceAll("[^\\p{L}\\p{N}]", ""); + } + + private static String decode(String value) { + String decoded = value; + for (int attempt = 0; attempt < 3; attempt++) { + String next = URLDecoder.decode(decoded, StandardCharsets.UTF_8); + if (next.equals(decoded) || !next.matches("(?s).*%[0-9a-fA-F]{2}.*")) return next; + decoded = next; + } + throw new IllegalArgumentException("Unsupported parameter encoding"); + } + + private static String encode(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } + + private static Iterable arrayItems(Object array) { + List items = new ArrayList<>(); + for (int index = 0; index < Array.getLength(array); index++) items.add(Array.get(array, index)); + return items; + } + + private static final class Budget { + private int nodes; + private void visit(int depth) throws UnsafePayload { + if (depth > MAX_DEPTH || ++nodes > MAX_NODES) throw new UnsafePayload(); + } + } + + private static final class UnsafePayload extends Exception {} +} diff --git a/pjcloud-common/pjcloud-common-core/src/test/java/com/bsd/pjcloud/common/core/util/AuditLogSanitizerTest.java b/pjcloud-common/pjcloud-common-core/src/test/java/com/bsd/pjcloud/common/core/util/AuditLogSanitizerTest.java new file mode 100644 index 000000000..361498e76 --- /dev/null +++ b/pjcloud-common/pjcloud-common-core/src/test/java/com/bsd/pjcloud/common/core/util/AuditLogSanitizerTest.java @@ -0,0 +1,176 @@ +package com.bsd.pjcloud.common.core.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayInputStream; +import java.time.LocalDateTime; +import java.time.Duration; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +class AuditLogSanitizerTest { + private final AuditLogSanitizer sanitizer = new AuditLogSanitizer(List.of()); + private final ObjectMapper json = new ObjectMapper(); + + @ParameterizedTest + @ValueSource(strings = {"password", "old_password", "New-Password", "newpassword1", "newpassword2", + "ConfirmNewPassword", "password_confirm", "PWD", "passwd", "pass", "captchaVerification", "sms_code", + "验证码", "Authorization", "client_secret", "appSecret", "private-key", "access_token", "refresh_token", + "accessKeySecret", "COOKIE", "data.password", "users[0].password", "p%61ssword", "passwordHash", "encryptedPassword", + "rsaPrivateKey", "selfAppSecret", "selfTokenKey", "selfTokenSecret", "smtpPassword", "X-Auth-Token"}) + void dropsCredentialKeysButPreservesBusinessValues(String field) throws Exception { + var original = new LinkedHashMap(); + original.put(field, "credential-value"); + original.put("username", "operator"); + original.put("roleCode", "ADMIN"); + JsonNode result = json.readTree(sanitizer.sanitizePayload(original, "/admin/user")); + assertEquals(json.readTree("{\"username\":\"operator\",\"roleCode\":\"ADMIN\"}"), result); + assertEquals("credential-value", original.get(field)); + } + + @Test + void recursivelyFiltersMapsBeansArraysAndEmbeddedJsonWithoutChangingInputs() throws Exception { + Map child = new LinkedHashMap<>(Map.of("password", "child-secret", "id", 7)); + Map original = new LinkedHashMap<>(); + original.put("items", List.of(child)); + original.put("ids", new long[]{1, 2}); + original.put("payload", "{\"newpassword1\":\"embedded-secret\",\"code\":\"BUS-42\",\"rows\":[{\"token\":\"embedded-token\",\"id\":9}]}"); + original.put("bean", new LoginBean()); + original.put("created", LocalDateTime.of(2026, 10, 29, 12, 30)); + original.put("stream", new ByteArrayInputStream(new byte[]{1, 2})); + + String output = sanitizer.sanitizePayload(original, "/orders/update"); + JsonNode result = json.readTree(output); + assertEquals(7, result.at("/items/0/id").intValue()); + assertFalse(result.at("/items/0").has("password")); + assertEquals(json.readTree("[1,2]"), result.get("ids")); + assertEquals("2026-10-29 12:30:00", result.get("created").textValue()); + assertEquals("BUS-42", json.readTree(result.get("payload").textValue()).get("code").textValue()); + assertEquals("operator", result.at("/bean/username").textValue()); + assertFalse(result.has("stream")); + for (String secret : List.of("child-secret", "embedded-secret", "embedded-token", "bean-secret")) assertFalse(output.contains(secret)); + assertEquals("child-secret", child.get("password")); + assertTrue(original.get("payload").toString().contains("embedded-secret")); + } + + @Test + void formFilteringHandlesEncodedKeysRepeatedValuesAndNestedPayloads() { + String original = "username=operator&password=one&code=BUS-42&roleCode=ADMIN&id=1&id=2&p%61ssword=two" + + "&payload=%7B%22new_password%22%3A%22three%22%2C%22id%22%3A9%7D"; + assertEquals("username=operator&code=BUS-42&roleCode=ADMIN&id=1&id=2&payload=%7B%22id%22%3A9%7D", + sanitizer.sanitizeParameters(original, "/orders/update")); + assertTrue(original.contains("password=one")); + String[] ids = {"1", "2"}; + Map request = new LinkedHashMap<>(); + request.put("password", new String[]{"raw-secret"}); + request.put("ids", ids); + request.put("note", new String[]{"a+b 50% 中文"}); + assertEquals("ids=1&ids=2¬e=a%2Bb+50%25+%E4%B8%AD%E6%96%87", sanitizer.sanitizeRequestParameters(request, "/orders")); + assertSame(ids, request.get("ids")); + assertEquals("raw-secret", request.get("password")[0]); + } + + @Test + void genericCodeIsRemovedOnlyInAuthenticationAndVerificationContexts() throws Exception { + String parameters = "code=123456&bizCode=BUS-42&roleCode=ADMIN&passwordExpireFlag=0"; + assertEquals(parameters, sanitizer.sanitizeParameters(parameters, "/role/update")); + for (String uri : List.of("/oauth2/token", "/auth/oauth2/token", "/api/admin/oauth2/token", "/login", "/register/password")) { + assertEquals("bizCode=BUS-42&roleCode=ADMIN&passwordExpireFlag=0", sanitizer.sanitizeParameters(parameters, uri)); + } + JsonNode sms = json.readTree(sanitizer.sanitizeParameters( + "{\"bizCode\":\"SMS_NORMAL_CODE\",\"params\":{\"code\":\"123456\",\"orderId\":42}}", "/message/send/sms")); + assertFalse(sms.get("params").has("code")); + assertEquals(42, sms.at("/params/orderId").intValue()); + assertEquals("SMS_NORMAL_CODE", sms.get("bizCode").textValue()); + assertEquals("username=operator", sanitizer.sanitizeParameters("code=123456&randomStr=proof&username=operator", "/custom")); + } + + @ParameterizedTest + @ValueSource(strings = {"opaque-secret", "\"opaque-secret\"", "[\"opaque-secret\"]", "[1,2]", + "{\"password\":\"truncated-secret", "{\"id\":1} {\"password\":\"tail-secret\"}", + "password=%ZZ", "p%ZZassword=broken", "Bearer opaque-token"}) + void damagedOrAnonymousParametersFailClosed(String original) { + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizeParameters(original, "/orders")); + } + + @Test + void emptyValuesAreStableAndSanitizationIsIdempotent() { + assertNull(sanitizer.sanitizeParameters(null, "/orders")); + assertEquals("", sanitizer.sanitizeParameters("", "/orders")); + String result = sanitizer.sanitizeParameters("[{\"password\":\"secret\",\"id\":1},null]", "/orders"); + assertEquals("[{\"id\":1},null]", result); + assertEquals(result, sanitizer.sanitizeParameters(result, "/orders")); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizeParameters(AuditLogSanitizer.HIDDEN, "/orders")); + } + + @Test + void urlsAndExceptionsDoNotEchoRecognizableCredentials() { + assertEquals("https://example.test/auth/oauth2/token?username=operator&grant_type=password", + sanitizer.sanitizeUri("https://example.test/auth/oauth2/token?username=operator&code=123456&password=secret&grant_type=password")); + assertEquals("/orders?code=BUS-42", sanitizer.sanitizeUri("/orders?code=BUS-42&client_secret=secret")); + assertEquals("/orders?" + AuditLogSanitizer.HIDDEN, sanitizer.sanitizeUri("/orders?password=%ZZ")); + assertEquals("/custom/" + AuditLogSanitizer.HIDDEN, sanitizer.sanitizeUri("/custom/path-secret", Map.of("token", "path-secret"))); + assertEquals("/appLogin/getAuthToken/" + AuditLogSanitizer.HIDDEN, sanitizer.sanitizeUri("/appLogin/getAuthToken/path-secret")); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizeText("failed password=plain-secret", "/orders")); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizeText("upstream returned Bearer abc123", "/orders")); + assertEquals("java.lang.IllegalStateException: 用户不存在\n at example.User.update(User.java:42)", + sanitizer.sanitizeText("java.lang.IllegalStateException: 用户不存在\n at example.User.update(User.java:42)", "/orders")); + } + + @Test + void malformedObjectsCyclesAndSerializationFailuresDoNotEscapeOrFallBackToRawValues() { + Map cycle = new HashMap<>(); + cycle.put("next", cycle); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizePayload(cycle, "/orders")); + Map indirect = new HashMap<>(); + indirect.put("parent", cycle); + cycle.put("bizCode", indirect); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizePayload(cycle, "/orders")); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizePayload(new ExplodingBean(), "/orders")); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizeParameters("{\"nested\":\"{broken-password=secret\"}", "/orders")); + } + + @Test + void doubleEncodedNestedJsonIsDecodedAndFilteredWithoutChangingBusinessJsonStrings() throws Exception { + String input = "{\"payload\":\"%257B%2522password%2522%253A%2522encoded-secret%2522%252C%2522id%2522%253A42%257D\",\"code\":\"BUS-42\"}"; + JsonNode result = json.readTree(sanitizer.sanitizeParameters(input, "/orders")); + assertEquals(json.readTree("{\"id\":42}"), json.readTree(result.get("payload").textValue())); + assertEquals("BUS-42", result.get("code").textValue()); + JsonNode form = json.readTree(sanitizer.sanitizeParameters("{\"payload\":\"password%253Dform-secret%2526id%253D42\"}", "/orders")); + assertEquals("id=42", form.get("payload").textValue()); + assertEquals("/orders?note=a%26id%3D3", sanitizer.sanitizeUri("/orders?note=a%26id%3D3&token=secret")); + assertEquals("", sanitizer.sanitizeText("password%3Dencoded-error-secret", "/orders")); + } + + @Test + void longBusinessValuesArePreservedWithoutUnboundedCredentialLabelMatching() { + String description = "a".repeat(100000); + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + String result = sanitizer.sanitizePayload(Map.of("description", description), "/orders"); + assertTrue(result.contains(description)); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.sanitizePayload(Map.of("new".repeat(200) + "password", "secret"), "/orders")); + }); + } + + @Test + void configuredAdditionalFieldsCannotDisableMandatoryCredentialFiltering() throws Exception { + var customized = new AuditLogSanitizer(List.of("phone", "custom-field")); + JsonNode result = json.readTree(customized.sanitizeParameters( + "{\"phone\":\"123\",\"customField\":\"private\",\"PASSWORD\":\"secret\",\"passwordExpireFlag\":0,\"publicKey\":\"public\",\"clientId\":\"web\"}", "/orders")); + assertEquals(json.readTree("{\"passwordExpireFlag\":0,\"publicKey\":\"public\",\"clientId\":\"web\"}"), result); + } + + public static class LoginBean { + public String getUsername() { return "operator"; } + public String getNewpassword1() { return "bean-secret"; } + } + + public static class ExplodingBean { + public String getValue() { throw new IllegalStateException("getter-error-containing-private-secret"); } + } +} diff --git a/pjcloud-common/pjcloud-common-excel/README.md b/pjcloud-common/pjcloud-common-excel/README.md index 121b57f16..ec74c1489 100644 --- a/pjcloud-common/pjcloud-common-excel/README.md +++ b/pjcloud-common/pjcloud-common-excel/README.md @@ -55,4 +55,8 @@ builder.registerWriteHandler(new ExcelExportStyleHandler(excelConfig.getExportSt 每次导出创建独立处理器，在提交任务时固定当前 token。处理器只支持 XLSX，缓存工作簿中的字体与单元格样式，条纹按实际行号计算，适用于分批流式写入。现有转换器、单元格类型、数据格式及导出列保持原样。固定格式模板应继续使用自身样式，不自动注册此处理器。 -日志导出沿用原有列，支持日志类型、创建时间范围、应用标识及原接口的日志编号筛选。每批按编号倒序读取 200 条，仅首批统计总数，后续使用编号游标；新增的更大编号不会挤偏后续批次。旧 GET 接口仍直接返回附件，新 POST `/admin/log/export-async` 返回任务信息，使用 `/admin/async-task/progress` 和命名空间 `admin:export:log` 查询进度。两个接口均要求 `sys_log_export` 权限，超出单工作表行数限制时提示缩小筛选范围。 +日志导出使用独立的 `SysLogExcelVO`，保留原有信息并按创建时间、类型、标题、操作人、应用、请求方式、请求地址、耗时、IP、提交数据、异常、用户代理、编号的顺序展示。类型显示“正常/错误”，请求地址和用户代理使用明确的中文表头；创建时间保存为可排序的日期单元格，耗时为数字，编号为保留完整精度的文本。报告顶部展示创建时间范围，表头固定为一行，长文本自动换行并适度增加行高，原文保留供展开查看。横向滚动时冻结时间和类型，纵向滚动时冻结报告上下文及表头。 + +日志导出必须选择完整的创建时间范围，结束时间不得早于开始时间，也不得超过开始时间加一个自然月；前端和同步、异步服务端接口均校验，未通过时不查询日志或创建任务。自然月按月底截断规则计算，例如 1 月 31 日 08:30 到 2 月 28 日 08:30 允许，超过一秒即拒绝。列表查询本身不受导出时间限制。 + +每批按编号倒序读取 200 条，仅首批统计总数，后续使用编号游标；新增的更大编号不会挤偏后续批次。旧 GET `/admin/log/export` 仍直接返回附件，新 POST `/admin/log/export-async` 返回任务信息，使用 `/admin/async-task/progress` 和命名空间 `admin:export:log` 查询进度。两个接口均要求 `sys_log_export` 权限，超出单工作表行数限制时提示缩小筛选范围。 diff --git a/pjcloud-common/pjcloud-common-log/README.md b/pjcloud-common/pjcloud-common-log/README.md new file mode 100644 index 000000000..5c523e684 --- /dev/null +++ b/pjcloud-common/pjcloud-common-log/README.md @@ -0,0 +1,14 @@ +# SysLog 认证参数过滤 + +`AuditLogSanitizer` 是独立的日志过滤器，不修改业务请求、全局 ObjectMapper 或系统配置使用的 `JacksonSensitiveFieldUtil`。 + +- 密码、新旧及确认密码、验证码、OTP、`randomStr`、令牌、Authorization、客户端密钥、应用密钥、私钥等字段从结构化日志中删除。字段名匹配忽略大小写、下划线及连字符，支持嵌套字段、JSON 字符串和 URL 编码。 +- 通用 `code` 只在登录认证请求、带验证码上下文的参数或 `SMS_NORMAL_CODE` 短信模板中去除。普通业务 `code`、`bizCode`、`roleCode` 和密码过期标记等字段保留。 +- `pjcloud.log.exclude-fields` 继续作为附加排除规则。即使配置为空，也不会关闭认证凭据的必需过滤。原有日志开关和请求体采集开关继续生效。 +- 新请求采集方法参数名并立即生成过滤后的副本，跳过 Cookie 参数、请求、响应、文件、流和 BindingResult；异步事件不携带原始请求体。监听器兼容旧事件，并在远程写入前再次过滤。 +- 解析失败、截断、匿名参数值、循环引用、序列化失败、超过结构遍历限制或过滤后仍超过 `max-length` 的报文显示“参数已隐藏”，不回退为原文，也不打印可能带有秘密的解析异常消息。JSON 遍历深度上限为 32，节点上限为 10000。 +- URL 查询参数和已知认证路径、异常文本中的可识别凭据也会过滤。正常错误说明和堆栈保留；无法安全判断边界时隐藏对应内容。 + +UPMS 的单条写入、前端错误日志批量写入、分页查询以及同步和异步 Excel 导出共用这组规则。查询与导出操作副本，不更新历史数据库；已有导出的文件不会被改写。数据库结构、API 结构及导出布局不变。 + +发布时先更新 UPMS，保护入库与历史日志读取，再更新认证及依赖 common-log 的日志生产服务。本实现只针对 SysLog，不修改其他日志表或应用运行日志。 diff --git a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/LogAutoConfiguration.java b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/LogAutoConfiguration.java index 21eb95f02..c349096f3 100644 --- a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/LogAutoConfiguration.java +++ b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/LogAutoConfiguration.java @@ -51,8 +51,8 @@ public class LogAutoConfiguration { } @Bean - public SysLogAspect sysLogAspect(KeyStrResolver resolver) { - return new SysLogAspect(resolver); + public SysLogAspect sysLogAspect(KeyStrResolver resolver, PjcloudLogProperties properties) { + return new SysLogAspect(resolver, properties); } } diff --git a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/aspect/SysLogAspect.java b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/aspect/SysLogAspect.java index 2e47038da..9b20da1f4 100644 --- a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/aspect/SysLogAspect.java +++ b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/aspect/SysLogAspect.java @@ -20,7 +20,9 @@ import cn.hutool.core.util.StrUtil; import com.bsd.pjcloud.admin.api.dto.SysLogDTO; import com.bsd.pjcloud.common.core.util.KeyStrResolver; import com.bsd.pjcloud.common.core.util.SpringContextHolder; +import com.bsd.pjcloud.common.core.util.AuditLogSanitizer; import com.bsd.pjcloud.common.log.annotation.SysLog; +import com.bsd.pjcloud.common.log.config.PjcloudLogProperties; import com.bsd.pjcloud.common.log.event.SysLogEvent; import com.bsd.pjcloud.common.log.util.LogTypeEnum; import com.bsd.pjcloud.common.log.util.SysLogUtils; @@ -44,6 +46,7 @@ import org.springframework.expression.EvaluationContext; public class SysLogAspect { private final KeyStrResolver tenantKeyStrResolver; + private final PjcloudLogProperties logProperties; @Around("@annotation(sysLog)") @SneakyThrows @@ -71,8 +74,9 @@ public class SysLogAspect { SysLogDTO logVo = SysLogUtils.getSysLog(); logVo.setTitle(value); // 获取请求body参数 - if (StrUtil.isBlank(logVo.getParams())) { - logVo.setBody(point.getArgs()); + if (logProperties.isRequestEnabled() && StrUtil.isBlank(logVo.getParams())) { + logVo.setParams(SysLogUtils.captureParameters(((MethodSignature) point.getSignature()).getMethod(), + point.getArgs(), logVo.getRequestUri(), logProperties)); } // 发送异步日志事件 Long startTime = System.currentTimeMillis(); @@ -90,6 +94,10 @@ public class SysLogAspect { Long endTime = System.currentTimeMillis(); logVo.setTime(endTime - startTime); logVo.setTenantId(Long.parseLong(tenantKeyStrResolver.key())); + AuditLogSanitizer sanitizer = new AuditLogSanitizer(logProperties.getExcludeFields()); + logVo.setTitle(sanitizer.sanitizeText(logVo.getTitle(), logVo.getRequestUri())); + logVo.setException(sanitizer.sanitizeText(logVo.getException(), logVo.getRequestUri())); + logVo.setBody(null); SpringContextHolder.publishEvent(new SysLogEvent(logVo)); } diff --git a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/config/PjcloudLogProperties.java b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/config/PjcloudLogProperties.java index d8d99c9c4..413c99315 100644 --- a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/config/PjcloudLogProperties.java +++ b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/config/PjcloudLogProperties.java @@ -34,6 +34,7 @@ import java.util.List; public class PjcloudLogProperties { public static final String PREFIX = "pjcloud.log"; + public static final String DEFAULT_EXCLUDE_FIELDS = "password,mobile,idcard,phone,accessSecret,tokenId,sign"; /** * 开启日志记录 @@ -46,10 +47,10 @@ public class PjcloudLogProperties { private boolean requestEnabled = true; /** - * 放行字段，password,mobile,idcard,phone + * 日志附加排除字段。认证凭据的必需过滤规则由 AuditLogSanitizer 独立执行。 */ - @Value("${pjcloud.log.exclude-fields:password,mobile,idcard,phone,accessSecret,tokenId,sign}") - private List excludeFields; + @Value("${pjcloud.log.exclude-fields:" + DEFAULT_EXCLUDE_FIELDS + "}") + private List excludeFields = List.of(DEFAULT_EXCLUDE_FIELDS.split(",")); /** * 请求报文最大存储长度 diff --git a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/event/SysLogListener.java b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/event/SysLogListener.java index 7f04227db..027b2cee1 100644 --- a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/event/SysLogListener.java +++ b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/event/SysLogListener.java @@ -16,77 +16,54 @@ package com.bsd.pjcloud.common.log.event; -import cn.hutool.core.collection.CollUtil; -import cn.hutool.core.util.StrUtil; import com.bsd.pjcloud.admin.api.dto.SysLogDTO; import com.bsd.pjcloud.admin.api.feign.RemoteLogService; import com.bsd.pjcloud.common.core.jackson.PjcloudJavaTimeModule; +import com.bsd.pjcloud.common.core.util.AuditLogSanitizer; import com.bsd.pjcloud.common.log.config.PjcloudLogProperties; import com.bsd.pjcloud.common.log.util.JacksonSensitiveFieldUtil; -import jakarta.servlet.ServletRequest; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.InitializingBean; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.scheduling.annotation.Async; -import org.springframework.validation.BindingResult; - -import java.util.Arrays; -import java.util.List; -import java.util.Objects; /** * @author pjcloud 异步监听日志事件 */ -@Slf4j @RequiredArgsConstructor public class SysLogListener implements InitializingBean { - /** - * 忽略序列化的对象类型 - */ - private final static Class[] ignoreClass = {ServletRequest.class, BindingResult.class}; - - /** - * new 一个 避免日志脱敏策略影响全局ObjectMapper - */ - private final RemoteLogService remoteLogService; private final PjcloudLogProperties logProperties; - @SneakyThrows @Async @Order @EventListener(SysLogEvent.class) public void saveSysLog(SysLogEvent event) { SysLogDTO source = (SysLogDTO) event.getSource(); - // json 格式刷参数放在异步中处理，提升性能 - if (Objects.nonNull(source.getBody()) && logProperties.isRequestEnabled()) { - Object[] args = (Object[]) source.getBody(); - List list = CollUtil.toList(args); - // 删除部分无法序列化的参数 - list.removeIf(obj -> Arrays.stream(ignoreClass).anyMatch(clazz -> clazz.isAssignableFrom(obj.getClass()))); - - try { - // 序列化参数 - String params = JacksonSensitiveFieldUtil.getObjectMapper().writeValueAsString(list); - source.setParams(StrUtil.subPre(params, logProperties.getMaxLength())); - } catch (Exception e) { - log.error("请求参数序列化异常:{}", e.getMessage()); - } - } - + AuditLogSanitizer sanitizer = new AuditLogSanitizer(logProperties.getExcludeFields()); + String uri = source.getRequestUri(); + String params = source.getBody() != null && logProperties.isRequestEnabled() + ? sanitizer.sanitizePayload(source.getBody(), uri) : sanitizer.sanitizeParameters(source.getParams(), uri); + int maximum = logProperties.getMaxLength() == null ? 2000 : Math.max(0, logProperties.getMaxLength()); + // 截断后仍保持 fail-closed，不能把损坏报文作为未过滤的原文返回。 + if (params != null && params.length() > maximum) params = AuditLogSanitizer.HIDDEN; + source.setParams(params); + source.setRequestUri(sanitizer.sanitizeUri(uri)); + source.setException(sanitizer.sanitizeText(source.getException(), uri)); + source.setTitle(sanitizer.sanitizeText(source.getTitle(), uri)); + source.setUserAgent(sanitizer.sanitizeText(source.getUserAgent(), uri)); source.setBody(null); remoteLogService.saveLog(source); } @Override public void afterPropertiesSet() { - String[] ignorableFieldNames = logProperties.getExcludeFields().toArray(new String[0]); + String[] ignorableFieldNames = logProperties.getExcludeFields() == null ? new String[0] + : logProperties.getExcludeFields().toArray(new String[0]); JacksonSensitiveFieldUtil.configureSensitiveFields(ignorableFieldNames); JacksonSensitiveFieldUtil.registerCustomModule(new PjcloudJavaTimeModule()); } diff --git a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/util/SysLogUtils.java b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/util/SysLogUtils.java index ff4cc3471..db6a8d0c9 100644 --- a/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/util/SysLogUtils.java +++ b/pjcloud-common/pjcloud-common-log/src/main/java/com/bsd/pjcloud/common/log/util/SysLogUtils.java @@ -17,13 +17,12 @@ package com.bsd.pjcloud.common.log.util; import cn.hutool.core.map.MapUtil; -import cn.hutool.core.util.ArrayUtil; import cn.hutool.core.util.URLUtil; import cn.hutool.extra.servlet.JakartaServletUtil; import cn.hutool.extra.spring.SpringUtil; -import cn.hutool.http.HttpUtil; import com.bsd.pjcloud.admin.api.dto.SysLogDTO; import com.bsd.pjcloud.common.core.constant.SecurityConstants; +import com.bsd.pjcloud.common.core.util.AuditLogSanitizer; import com.bsd.pjcloud.common.log.config.PjcloudLogProperties; import jakarta.servlet.http.HttpServletRequest; import lombok.experimental.UtilityClass; @@ -38,9 +37,16 @@ import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.oauth2.core.OAuth2AuthenticatedPrincipal; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.CookieValue; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.servlet.HandlerMapping; import java.lang.reflect.Method; -import java.util.HashMap; +import java.lang.reflect.Parameter; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -57,7 +63,7 @@ public class SysLogUtils { .requireNonNull(RequestContextHolder.getRequestAttributes())).getRequest(); SysLogDTO sysLog = new SysLogDTO(); sysLog.setLogType(LogTypeEnum.NORMAL.getType()); - sysLog.setRequestUri(URLUtil.getPath(request.getRequestURI())); + String uri = URLUtil.getPath(request.getRequestURI()); sysLog.setMethod(request.getMethod()); sysLog.setRemoteAddr(JakartaServletUtil.getClientIP(request)); sysLog.setUserAgent(request.getHeader(HttpHeaders.USER_AGENT)); @@ -65,16 +71,63 @@ public class SysLogUtils { // 获取服务名称 sysLog.setServiceId(SpringUtil.getProperty("spring.application.name")); - // get 参数脱敏 PjcloudLogProperties logProperties = SpringUtil.getBean(PjcloudLogProperties.class); - - // 修复request.getParameterMap()其他容器不允许改，tomcat会报错，当然undertow默认可以的 - Map paramsMap = MapUtil.removeAny(new HashMap<>(request.getParameterMap()), - ArrayUtil.toArray(logProperties.getExcludeFields(), String.class)); - sysLog.setParams(HttpUtil.toParams(paramsMap)); + AuditLogSanitizer sanitizer = new AuditLogSanitizer(logProperties.getExcludeFields()); + sysLog.setUserAgent(sanitizer.sanitizeText(sysLog.getUserAgent(), uri)); + Object variables = request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE); + Map pathVariables = new LinkedHashMap<>(); + if (variables instanceof Map map) map.forEach((key, value) -> { + if (key instanceof String name && value instanceof String text) pathVariables.put(name, text); + }); + sysLog.setRequestUri(sanitizer.sanitizeUri(uri, pathVariables)); + sysLog.setParams(sanitizer.sanitizeRequestParameters(request.getParameterMap(), uri)); return sysLog; } + /** 将命名参数记录为副本；原始 DTO、数组、文件和请求对象不进入异步事件。 */ + public String captureParameters(Method method, Object[] arguments, String uri, PjcloudLogProperties properties) { + try { + return captureNamedParameters(method, arguments, uri, properties); + } catch (Exception ignored) { + return AuditLogSanitizer.HIDDEN; + } + } + + private String captureNamedParameters(Method method, Object[] arguments, String uri, PjcloudLogProperties properties) { + AuditLogSanitizer sanitizer = new AuditLogSanitizer(properties.getExcludeFields()); + Map named = new LinkedHashMap<>(); + Parameter[] parameters = method.getParameters(); + for (int index = 0; index < arguments.length; index++) { + Object argument = arguments[index]; + if (AuditLogSanitizer.isUnloggable(argument)) continue; + if (index >= parameters.length) return AuditLogSanitizer.HIDDEN; + Parameter parameter = parameters[index]; + // Cookie 参数可能承载不固定名称的会话凭据，不进入操作参数快照。 + if (parameter.isAnnotationPresent(CookieValue.class)) continue; + String name = parameterName(parameter); + if (name == null) return AuditLogSanitizer.HIDDEN; + if (sanitizer.isSensitiveField(name, sanitizer.isAuthenticationContext(uri))) continue; + if (argument instanceof CharSequence text && parameter.isAnnotationPresent(RequestBody.class)) { + argument = sanitizer.sanitizeParameters(text.toString(), uri); + } + named.put(name, argument); + } + return sanitizer.sanitizePayload(named, uri); + } + + private String parameterName(Parameter parameter) { + RequestParam query = parameter.getAnnotation(RequestParam.class); + if (query != null && !query.name().isBlank()) return query.name(); + if (query != null && !query.value().isBlank()) return query.value(); + PathVariable path = parameter.getAnnotation(PathVariable.class); + if (path != null && !path.name().isBlank()) return path.name(); + if (path != null && !path.value().isBlank()) return path.value(); + RequestHeader header = parameter.getAnnotation(RequestHeader.class); + if (header != null && !header.name().isBlank()) return header.name(); + if (header != null && !header.value().isBlank()) return header.value(); + return parameter.isNamePresent() ? parameter.getName() : null; + } + /** * 获取用户名称 * @return username diff --git a/pjcloud-common/pjcloud-common-log/src/test/java/com/bsd/pjcloud/common/log/SysLogSanitizationTest.java b/pjcloud-common/pjcloud-common-log/src/test/java/com/bsd/pjcloud/common/log/SysLogSanitizationTest.java new file mode 100644 index 000000000..0f82f4259 --- /dev/null +++ b/pjcloud-common/pjcloud-common-log/src/test/java/com/bsd/pjcloud/common/log/SysLogSanitizationTest.java @@ -0,0 +1,252 @@ +package com.bsd.pjcloud.common.log; + +import cn.hutool.extra.spring.SpringUtil; +import com.bsd.pjcloud.admin.api.dto.SysLogDTO; +import com.bsd.pjcloud.admin.api.feign.RemoteLogService; +import com.bsd.pjcloud.common.core.util.AuditLogSanitizer; +import com.bsd.pjcloud.common.core.util.KeyStrResolver; +import com.bsd.pjcloud.common.core.util.SpringContextHolder; +import com.bsd.pjcloud.common.log.annotation.SysLog; +import com.bsd.pjcloud.common.log.aspect.SysLogAspect; +import com.bsd.pjcloud.common.log.config.PjcloudLogProperties; +import com.bsd.pjcloud.common.log.event.SysLogEvent; +import com.bsd.pjcloud.common.log.event.SysLogListener; +import com.bsd.pjcloud.common.log.util.JacksonSensitiveFieldUtil; +import com.bsd.pjcloud.common.log.util.SysLogUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.http.HttpServletRequest; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.reflect.MethodSignature; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.validation.BindingResult; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.CookieValue; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; +import org.springframework.web.servlet.HandlerMapping; + +import java.io.ByteArrayInputStream; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +class SysLogSanitizationTest { + private GenericApplicationContext context; + private final PjcloudLogProperties properties = new PjcloudLogProperties(); + private final MockHttpServletRequest request = new MockHttpServletRequest(); + private final List events = new ArrayList<>(); + + @BeforeEach + void setup() { + context = new GenericApplicationContext(); + context.getBeanFactory().registerSingleton("logProperties", properties); + context.refresh(); + SpringUtil spring = new SpringUtil(); + spring.postProcessBeanFactory(context.getBeanFactory()); + spring.setApplicationContext(context); + new SpringContextHolder().setApplicationContext(context); + context.addApplicationListener(event -> { + if (event instanceof SysLogEvent logEvent) events.add((SysLogDTO) logEvent.getSource()); + }); + request.setRequestURI("/user/update"); + request.setMethod("POST"); + RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(request)); + } + + @AfterEach + void cleanup() { + RequestContextHolder.resetRequestAttributes(); + SecurityContextHolder.clearContext(); + SpringContextHolder.clearHolder(); + SpringUtil spring = new SpringUtil(); + spring.postProcessBeanFactory(null); + spring.setApplicationContext(null); + context.close(); + } + + @Test + void requestParametersAndNamedPathCredentialsAreFilteredWithoutChangingTheRequest() { + request.addParameter("PASSWORD", "request-secret"); + request.addParameter("username", "operator"); + request.addParameter("code", "BUS-42"); + request.setRequestURI("/custom/path-secret"); + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, Map.of("token", "path-secret")); + SysLogDTO record = SysLogUtils.getSysLog(); + assertFalse(record.getParams().contains("request-secret")); + assertTrue(record.getParams().contains("username=operator")); + assertTrue(record.getParams().contains("code=BUS-42")); + assertEquals("/custom/" + AuditLogSanitizer.HIDDEN, record.getRequestUri()); + assertEquals("request-secret", request.getParameter("PASSWORD")); + request.setRequestURI("/auth/oauth2/token"); + assertFalse(SysLogUtils.getSysLog().getParams().contains("code=")); + } + + @Test + void namedArgumentsDtosAndIgnoredRequestAndFileObjectsAreCapturedAsASafeSnapshot() throws Exception { + var method = Sample.class.getMethod("named", String.class, SecretBean.class, HttpServletRequest.class); + var bean = new SecretBean(); + String record = SysLogUtils.captureParameters(method, new Object[]{"named-secret", bean, request}, "/user/update", properties); + var values = new ObjectMapper().readTree(record); + assertFalse(values.has("newpassword1")); + assertFalse(values.has("request")); + assertEquals("operator", values.at("/body/username").textValue()); + assertFalse(record.contains("body-secret")); + bean.username = "changed"; + assertEquals("operator", values.at("/body/username").textValue()); + assertEquals("body-secret", bean.getPassword()); + } + + @Test + void cookieCredentialsAreSkippedEvenWhenTheSessionCookieHasACustomName() throws Exception { + var method = Sample.class.getMethod("cookie", String.class, String.class); + assertEquals("{\"code\":\"BUS-42\"}", SysLogUtils.captureParameters(method, + new Object[]{"cookie-secret", "BUS-42"}, "/orders", properties)); + } + + @Test + void aspectPublishesFilteredSnapshotsBeforeBusinessObjectsCanChange() throws Throwable { + var method = Sample.class.getMethod("update", SecretBean.class); + var body = new SecretBean(); + var signature = mock(MethodSignature.class); + when(signature.getMethod()).thenReturn(method); + when(signature.getName()).thenReturn("update"); + var point = mock(ProceedingJoinPoint.class); + when(point.getTarget()).thenReturn(new Sample()); + when(point.getSignature()).thenReturn(signature); + when(point.getArgs()).thenReturn(new Object[]{body}); + when(point.proceed()).thenAnswer(call -> { + assertEquals("body-secret", body.getPassword()); + body.username = "changed-by-business"; + return "success"; + }); + KeyStrResolver tenant = mock(KeyStrResolver.class); + when(tenant.key()).thenReturn("7"); + assertEquals("success", new SysLogAspect(tenant, properties).around(point, method.getAnnotation(SysLog.class))); + assertEquals(1, events.size()); + assertNull(events.get(0).getBody()); + assertFalse(events.get(0).getParams().contains("body-secret")); + assertTrue(events.get(0).getParams().contains("operator")); + assertFalse(events.get(0).getParams().contains("changed-by-business")); + assertEquals("changed-by-business", body.username); + } + + @Test + void collectorDisabledDoesNotSerializeRequestBodyOrChangeBusinessResults() throws Throwable { + properties.setRequestEnabled(false); + var method = Sample.class.getMethod("update", SecretBean.class); + var signature = mock(MethodSignature.class); + when(signature.getMethod()).thenReturn(method); + when(signature.getName()).thenReturn("update"); + var point = mock(ProceedingJoinPoint.class); + when(point.getTarget()).thenReturn(new Sample()); + when(point.getSignature()).thenReturn(signature); + when(point.getArgs()).thenReturn(new Object[]{new ExplodingBean()}); + when(point.proceed()).thenReturn("success"); + KeyStrResolver tenant = mock(KeyStrResolver.class); + when(tenant.key()).thenReturn("7"); + assertEquals("success", new SysLogAspect(tenant, properties).around(point, method.getAnnotation(SysLog.class))); + assertEquals("", events.get(0).getParams()); + assertNull(events.get(0).getBody()); + } + + @Test + void aspectKeepsTheOriginalBusinessExceptionButDoesNotPublishItsCredentialText() throws Throwable { + var method = Sample.class.getMethod("update", SecretBean.class); + var signature = mock(MethodSignature.class); + when(signature.getMethod()).thenReturn(method); + when(signature.getName()).thenReturn("update"); + var point = mock(ProceedingJoinPoint.class); + when(point.getTarget()).thenReturn(new Sample()); + when(point.getSignature()).thenReturn(signature); + when(point.getArgs()).thenReturn(new Object[]{new SecretBean()}); + var original = new IllegalStateException("password=business-exception-secret"); + when(point.proceed()).thenThrow(original); + KeyStrResolver tenant = mock(KeyStrResolver.class); + when(tenant.key()).thenReturn("7"); + assertSame(original, assertThrows(IllegalStateException.class, + () -> new SysLogAspect(tenant, properties).around(point, method.getAnnotation(SysLog.class)))); + assertEquals("9", events.get(0).getLogType()); + assertEquals("", events.get(0).getException()); + assertFalse(events.get(0).getParams().contains("body-secret")); + assertEquals("password=business-exception-secret", original.getMessage()); + } + + @Test + void listenerFiltersLegacyEventsAndClearsIgnoredObjectsBeforeRemoteWrite() throws Exception { + var remote = mock(RemoteLogService.class); + var listener = new SysLogListener(remote, properties); + var record = new SysLogDTO(); + record.setRequestUri("/user/update?token=uri-token"); + record.setBody(new Object[]{new SecretBean(), null, request, new MockHttpServletResponse(), + mock(BindingResult.class), new MockMultipartFile("file", new byte[]{1}), new ByteArrayInputStream(new byte[]{2})}); + record.setException("upstream Authorization: Bearer error-token"); + listener.saveSysLog(new SysLogEvent(record)); + assertNull(record.getBody()); + assertEquals("/user/update", record.getRequestUri()); + assertFalse(record.getParams().contains("body-secret")); + assertTrue(record.getParams().contains("operator")); + assertEquals(AuditLogSanitizer.HIDDEN, record.getException()); + verify(remote).saveLog(record); + } + + @Test + void serializationFailureAnonymousBodyAndLengthLimitsNeverFallBackToUnfilteredParameters() { + var listener = new SysLogListener(mock(RemoteLogService.class), properties); + for (Object body : List.of(new ExplodingBean(), new Object[]{"anonymous-secret"})) { + var record = new SysLogDTO(); + record.setRequestUri("/orders"); + record.setBody(body); + record.setParams("password=raw-fallback-secret"); + listener.saveSysLog(new SysLogEvent(record)); + assertEquals(AuditLogSanitizer.HIDDEN, record.getParams()); + assertNull(record.getBody()); + } + properties.setMaxLength(10); + var record = new SysLogDTO(); + record.setParams("{\"username\":\"long-business-value\",\"password\":\"secret\"}"); + listener.saveSysLog(new SysLogEvent(record)); + assertEquals(AuditLogSanitizer.HIDDEN, record.getParams()); + } + + @Test + void logFilteringDoesNotExpandTheSerializerUsedBySystemConfiguration() throws Exception { + var listener = new SysLogListener(mock(RemoteLogService.class), properties); + listener.afterPropertiesSet(); + String configuration = "{\"appSecret\":\"config-app-secret\",\"clientSecret\":\"config-client-secret\",\"username\":\"operator\"}"; + String before = JacksonSensitiveFieldUtil.readStr(configuration); + var record = new SysLogDTO(); + record.setParams(configuration); + listener.saveSysLog(new SysLogEvent(record)); + assertEquals("{\"username\":\"operator\"}", record.getParams()); + assertEquals(before, JacksonSensitiveFieldUtil.readStr(configuration)); + assertTrue(before.contains("config-app-secret")); + assertTrue(before.contains("config-client-secret")); + } + + public static class SecretBean { + private String username = "operator"; + public String getUsername() { return username; } + public String getPassword() { return "body-secret"; } + } + + public static class ExplodingBean { + public String getValue() { throw new IllegalStateException("exception-message-containing-a-secret"); } + } + + public static class Sample { + public void named(@RequestParam("newpassword1") String password, @RequestBody SecretBean body, HttpServletRequest request) {} + public void cookie(@CookieValue("CUSTOM_SESSION") String session, @RequestParam("code") String code) {} + @SysLog("修改用户") + public Object update(@RequestBody SecretBean body) { return null; } + } +} diff --git a/pjcloud-upms/pjcloud-upms-api/src/main/java/com/bsd/pjcloud/admin/api/config/FileAccessFeignConfiguration.java b/pjcloud-upms/pjcloud-upms-api/src/main/java/com/bsd/pjcloud/admin/api/config/FileAccessFeignConfiguration.java index f77f76cca..5f74f1c0f 100644 --- a/pjcloud-upms/pjcloud-upms-api/src/main/java/com/bsd/pjcloud/admin/api/config/FileAccessFeignConfiguration.java +++ b/pjcloud-upms/pjcloud-upms-api/src/main/java/com/bsd/pjcloud/admin/api/config/FileAccessFeignConfiguration.java @@ -13,6 +13,19 @@ public class FileAccessFeignConfiguration { @Bean public RequestInterceptor fileAccessRequestInterceptor(FileAccessSigner signer) { return template -> { + if ("POST".equals(template.method())) { + String target = switch (template.path()) { + case "/sys-file/upload" -> "upload"; + case "/sys-file/inner-upload" -> "inner-upload"; + case "/sys-file/qn/upload" -> "qn-upload"; + default -> null; + }; + if (target != null) signer.uploadHeaders(target).forEach((key, value) -> { + template.header(key, (String[]) null); + template.header(key, value); + }); + return; + } if (!"GET".equals(template.method()) || !"/sys-file/oss/file".equals(template.path())) return; Collection names = template.queries().get("fileName"); if (names == null || names.size() != 1) throw new IllegalArgumentException("Exactly one fileName is required"); diff --git a/pjcloud-upms/pjcloud-upms-api/src/main/java/com/bsd/pjcloud/admin/api/vo/SysLogExcelVO.java b/pjcloud-upms/pjcloud-upms-api/src/main/java/com/bsd/pjcloud/admin/api/vo/SysLogExcelVO.java new file mode 100644 index 000000000..4f5b88f8e --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-api/src/main/java/com/bsd/pjcloud/admin/api/vo/SysLogExcelVO.java @@ -0,0 +1,71 @@ +package com.bsd.pjcloud.admin.api.vo; + +import com.alibaba.excel.annotation.ExcelIgnoreUnannotated; +import com.alibaba.excel.annotation.ExcelProperty; +import com.alibaba.excel.annotation.format.DateTimeFormat; +import com.alibaba.excel.annotation.format.NumberFormat; +import com.alibaba.excel.annotation.write.style.ColumnWidth; +import com.alibaba.excel.converters.localdatetime.LocalDateTimeNumberConverter; +import com.alibaba.excel.converters.longconverter.LongNumberConverter; +import lombok.Data; + +import java.time.LocalDateTime; + +/** 日志导出专用列，常用信息在前，长文本与精确编号在后。 */ +@Data +@ExcelIgnoreUnannotated +public class SysLogExcelVO { + @ExcelProperty(value = "创建时间", index = 0, converter = LocalDateTimeNumberConverter.class) + @DateTimeFormat("yyyy-mm-dd hh:mm:ss") + @ColumnWidth(32) + private LocalDateTime createTime; + + @ExcelProperty(value = "日志类型", index = 1) + @ColumnWidth(12) + private String logType; + + @ExcelProperty(value = "日志标题", index = 2) + @ColumnWidth(28) + private String title; + + @ExcelProperty(value = "操作人", index = 3) + @ColumnWidth(18) + private String createBy; + + @ExcelProperty(value = "所属应用", index = 4) + @ColumnWidth(28) + private String serviceId; + + @ExcelProperty(value = "请求方式", index = 5) + @ColumnWidth(12) + private String method; + + @ExcelProperty(value = "请求地址", index = 6) + @ColumnWidth(42) + private String requestUri; + + @ExcelProperty(value = "耗时（毫秒）", index = 7, converter = LongNumberConverter.class) + @NumberFormat("#,##0") + @ColumnWidth(16) + private Long time; + + @ExcelProperty(value = "IP 地址", index = 8) + @ColumnWidth(42) + private String remoteAddr; + + @ExcelProperty(value = "提交数据", index = 9) + @ColumnWidth(64) + private String params; + + @ExcelProperty(value = "异常详情", index = 10) + @ColumnWidth(64) + private String exception; + + @ExcelProperty(value = "用户代理", index = 11) + @ColumnWidth(48) + private String userAgent; + + @ExcelProperty(value = "日志编号", index = 12) + @ColumnWidth(32) + private String id; +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/controller/SysFileController.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/controller/SysFileController.java index 3864128e3..fbe1149c9 100644 --- a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/controller/SysFileController.java +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/controller/SysFileController.java @@ -36,6 +36,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.servlet.http.HttpServletResponse; import jakarta.servlet.http.HttpServletRequest; import com.bsd.pjcloud.admin.service.file.FileAccessGuard; +import com.bsd.pjcloud.admin.service.file.FileUploadGuard; import com.bsd.pjcloud.common.core.file.FileAccessProperties; import java.io.IOException; import lombok.AllArgsConstructor; @@ -65,13 +66,16 @@ public class SysFileController { private final SysFileService sysFileService; private final FileAccessGuard fileAccessGuard; private final FileAccessProperties fileAccessProperties; + private final FileUploadGuard fileUploadGuard; /* ************************************* start 远程调用 start ************************************* */ @Inner - @PostMapping(value = "/inner-upload") + @PostMapping(value = "/inner-upload", consumes = "multipart/form-data") public R innerUpload(@RequestPart("file") MultipartFile file, @RequestParam(value = "dir", required = false) String dir, @RequestParam(value = "groupId", required = false) Long groupId, - @RequestParam(value = "type", required = false) String type) { + @RequestParam(value = "type", required = false) String type, + HttpServletRequest request, HttpServletResponse response) throws IOException { + if (!fileUploadGuard.authorize(request, response, "inner-upload")) return null; return sysFileService.innerUpload(file, dir, groupId, type); } /* ************************************* end 远程调用 end ************************************* */ @@ -122,18 +126,22 @@ public class SysFileController { * @param dir 文件夹 * @return R(/ admin / bucketName / filename) */ - @PostMapping(value = "/upload") + @PostMapping(value = "/upload", consumes = "multipart/form-data") public R upload(@RequestPart("file") MultipartFile file, @RequestParam(value = "dir", required = false) String dir, @RequestParam(value = "groupId", required = false) Long groupId, - @RequestParam(value = "type", required = false) String type) { + @RequestParam(value = "type", required = false) String type, + HttpServletRequest request, HttpServletResponse response) throws IOException { + if (!fileUploadGuard.authorize(request, response, "upload")) return null; return sysFileService.uploadFile(file, dir, groupId, type); } /** * 七牛云上传 */ - @PostMapping(value = "/qn/upload") - public R> qnUpload(@RequestPart("file") MultipartFile file, @RequestParam(value = "dir", required = false) String dir, @RequestParam(value = "groupId", required = false) Long groupId, @RequestParam(value = "type", required = false) String type) { + @PostMapping(value = "/qn/upload", consumes = "multipart/form-data") + public R> qnUpload(@RequestPart("file") MultipartFile file, @RequestParam(value = "dir", required = false) String dir, @RequestParam(value = "groupId", required = false) Long groupId, @RequestParam(value = "type", required = false) String type, + HttpServletRequest request, HttpServletResponse response) throws IOException { + if (!fileUploadGuard.authorize(request, response, "qn-upload")) return null; return sysFileService.qnUpload(file, dir, groupId, type); } diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadExceptionHandler.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadExceptionHandler.java new file mode 100644 index 000000000..013c6d71d --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadExceptionHandler.java @@ -0,0 +1,17 @@ +package com.bsd.pjcloud.admin.service.file; + +import com.bsd.pjcloud.admin.controller.SysFileController; +import com.bsd.pjcloud.common.core.util.R; +import org.springframework.core.annotation.Order; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +@Order(-100) +@RestControllerAdvice(assignableTypes = SysFileController.class) +public class FileUploadExceptionHandler { + @ExceptionHandler(FileUploadRejectedException.class) + public ResponseEntity> rejected(FileUploadRejectedException error) { + return ResponseEntity.status(error.getStatus()).header("Cache-Control", "no-store").body(R.failed(error.getMessage())); + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadGuard.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadGuard.java new file mode 100644 index 000000000..c5f7e0786 --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadGuard.java @@ -0,0 +1,83 @@ +package com.bsd.pjcloud.admin.service.file; + +import com.bsd.pjcloud.common.core.file.FileAccessProperties; +import com.bsd.pjcloud.common.core.file.FileAccessSignature; +import com.bsd.pjcloud.common.core.file.FileUploadSignature; +import io.micrometer.core.instrument.MeterRegistry; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.stereotype.Component; +import java.io.IOException; +import java.time.Clock; +import java.time.Duration; +import java.util.*; + +@Component +public class FileUploadGuard { + private static final Set APPS = Set.of("admin", "h5", "mini", "flutter", "service"); + private static final List HEADERS = List.of(FileUploadSignature.APP, FileUploadSignature.KID, + FileUploadSignature.TIMESTAMP, FileUploadSignature.NONCE, FileUploadSignature.SIGNATURE); + private final FileAccessProperties properties; + private final FileUploadNonceStore nonces; + private final MeterRegistry meters; + private final Clock clock; + + @org.springframework.beans.factory.annotation.Autowired + public FileUploadGuard(FileAccessProperties properties, FileUploadNonceStore nonces, ObjectProvider meters) { + this(properties, nonces, meters.getIfAvailable(), Clock.systemUTC()); + } + public FileUploadGuard(FileAccessProperties properties, FileUploadNonceStore nonces, MeterRegistry meters, Clock clock) { + this.properties = properties; this.nonces = nonces; this.meters = meters; this.clock = clock; + } + public record Decision(int status, String reason) { } + + public Decision evaluate(HttpServletRequest request, String target) { + properties.validate(); + if (!"POST".equals(request.getMethod()) || !FileUploadSignature.TARGETS.contains(target)) return new Decision(400, "invalid_upload_target"); + if (request instanceof org.springframework.web.multipart.MultipartHttpServletRequest multipart + && (multipart.getMultiFileMap().values().stream().mapToLong(List::size).sum() != 1 + || !multipart.getMultiFileMap().containsKey("file"))) return new Decision(400, "invalid_upload_part_count"); + for (String name : HEADERS) { + List values = Collections.list(request.getHeaders(name)); + if (values.isEmpty()) return new Decision(403, "missing_upload_signature"); + if (values.size() != 1) return new Decision(400, "duplicate_upload_header"); + } + String app = request.getHeader(FileUploadSignature.APP), kid = request.getHeader(FileUploadSignature.KID), + ts = request.getHeader(FileUploadSignature.TIMESTAMP), nonce = request.getHeader(FileUploadSignature.NONCE), + signature = request.getHeader(FileUploadSignature.SIGNATURE); + if (!APPS.contains(app)) return new Decision(403, "unknown_upload_client"); + if (!kid.matches("[A-Za-z0-9_-]{1,32}") || !ts.matches("(?:0|[1-9][0-9]{0,11})") + || !nonce.matches("[A-Za-z0-9_-]{16,64}") || !signature.matches("[0-9a-f]{64}")) return new Decision(400, "malformed_upload_signature"); + if (target.equals("inner-upload") && !app.equals("service")) return new Decision(403, "internal_upload_only"); + String key = properties.key(app, kid); + if (key == null || key.isBlank()) return new Decision(403, "unknown_upload_key"); + long issued = Long.parseLong(ts), now = clock.instant().getEpochSecond(), ttl = properties.getUpload().getTtlSeconds(); + if (issued > now + Math.min(60, properties.getFutureSkewSeconds())) return new Decision(403, "upload_clock_skew"); + if (now - issued >= ttl) return new Decision(403, "upload_signature_expired"); + if (!FileAccessSignature.matches(FileUploadSignature.sign(target, app, kid, issued, nonce, key), signature)) return new Decision(403, "invalid_upload_signature"); + if (properties.enforcing() && !nonces.claim(app, kid, nonce, Duration.ofSeconds(issued + ttl - now + 1))) { + return new Decision(403, "upload_signature_replayed"); + } + return new Decision(200, "upload_signed"); + } + + public boolean authorize(HttpServletRequest request, HttpServletResponse response, String target) throws IOException { + Decision decision; + try { decision = evaluate(request, target); } + catch (RuntimeException error) { decision = new Decision(500, "upload_authorization_unavailable"); } + String app = request.getHeader(FileUploadSignature.APP); + String mode = Objects.toString(properties.getMode(), "invalid"); + if (!Set.of("observe", "enforce").contains(mode)) mode = "invalid"; + if (meters != null) meters.counter("file.upload.requests", "outcome", decision.reason(), "app", APPS.contains(Objects.toString(app, "")) ? app : "unknown", + "mode", mode).increment(); + response.setHeader("Cache-Control", "no-store"); + if (decision.status() != 200 && (properties.enforcing() || decision.status() == 500)) { + response.setStatus(decision.status()); + response.setContentType("application/json;charset=UTF-8"); + response.getWriter().write("{\"code\":" + decision.status() + ",\"msg\":\"上传身份校验失败\",\"reason\":\"" + decision.reason() + "\"}"); + return false; + } + return true; + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadNonceStore.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadNonceStore.java new file mode 100644 index 000000000..3a39f533b --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadNonceStore.java @@ -0,0 +1,8 @@ +package com.bsd.pjcloud.admin.service.file; + +import java.time.Duration; + +@FunctionalInterface +public interface FileUploadNonceStore { + boolean claim(String appId, String kid, String nonce, Duration lifetime); +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadPolicy.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadPolicy.java new file mode 100644 index 000000000..4d168333e --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadPolicy.java @@ -0,0 +1,203 @@ +package com.bsd.pjcloud.admin.service.file; + +import com.bsd.pjcloud.common.core.file.FileAccessProperties; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.stream.MemoryCacheImageInputStream; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.*; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** Checks the server-side allowlist and actual bytes before any storage or file log is written. */ +@Component +@RequiredArgsConstructor +public class FileUploadPolicy { + private final FileAccessProperties properties; + public record Accepted(String extension, String contentType) { } + private static final Pattern UNSAFE_CSS = Pattern.compile("(?is)@import|expression\\s*\\("); + private static final Pattern CSS_URL = Pattern.compile("(?is)url\\s*\\(([^)]*)\\)"); + + public Accepted validate(MultipartFile file, String dir, boolean internal) { + if (file == null || file.isEmpty() || file.getSize() <= 0) reject(400, "上传文件不能为空"); + String name = file.getOriginalFilename(); + if (name == null || name.isBlank() || name.length() > 512 || name.chars().anyMatch(c -> c < 32 || c == 127) + || name.contains("/") || name.contains("\\")) reject(400, "文件名不合法"); + if (dir != null && (dir.length() > 256 || dir.chars().anyMatch(c -> c < 32 || c == 127) || dir.contains("\\") + || Arrays.stream(dir.split("/", -1)).anyMatch(s -> s.equals(".") || s.equals("..")))) reject(400, "文件目录不合法"); + int dot = name.lastIndexOf('.'); + String ext = dot < 1 ? "" : name.substring(dot + 1).toLowerCase(Locale.ROOT); + var policy = properties.getUpload(); + if (!policy.getAllowedExtensions().contains(ext) && !(internal && policy.getInternalExtensions().contains(ext))) reject(415, "不支持上传此文件类型"); + if (file.getSize() > policy.getMaxFileBytes()) reject(413, "文件超过服务器上传大小限制"); + try (InputStream input = file.getInputStream()) { + byte[] head = input.readNBytes(4096); + boolean valid = switch (ext) { + case "jpg", "jpeg" -> starts(head, 0xff, 0xd8, 0xff) && imageDimensions(file); + case "png" -> starts(head, 137, 80, 78, 71, 13, 10, 26, 10) && imageDimensions(file); + case "gif" -> (ascii(head, 0, "GIF87a") || ascii(head, 0, "GIF89a")) && imageDimensions(file); + case "bmp" -> ascii(head, 0, "BM") && imageDimensions(file); + case "webp" -> ascii(head, 0, "RIFF") && ascii(head, 8, "WEBP"); + case "ico" -> starts(head, 0, 0, 1, 0) && head.length >= 6 && ((head[4] & 255) | (head[5] & 255) << 8) > 0; + case "svg" -> safeSvg(file); + case "pdf" -> ascii(head, 0, "%PDF-"); + case "apk", "zip", "docx", "xlsx", "pptx" -> starts(head, 80, 75, 3, 4) && validArchive(file, ext); + case "doc", "xls", "ppt" -> starts(head, 0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1); + case "txt", "csv", "log" -> validText(file); + case "gz" -> starts(head, 0x1f, 0x8b, 8); + case "mp4", "mov", "m4a" -> ascii(head, 4, "ftyp"); + case "avi" -> ascii(head, 0, "RIFF") && ascii(head, 8, "AVI "); + case "wav" -> ascii(head, 0, "RIFF") && ascii(head, 8, "WAVE"); + case "webm" -> starts(head, 0x1a, 0x45, 0xdf, 0xa3); + case "mp3" -> ascii(head, 0, "ID3") || (head.length > 1 && (head[0] & 255) == 255 && (head[1] & 0xe0) == 0xe0); + case "aac" -> head.length > 1 && (head[0] & 255) == 255 && (head[1] & 0xf6) == 0xf0; + case "ogg" -> ascii(head, 0, "OggS"); + case "amr" -> ascii(head, 0, "#!AMR\n") || ascii(head, 0, "#!AMR-WB\n"); + case "flac" -> ascii(head, 0, "fLaC"); + default -> false; + }; + if (!valid) reject(415, "文件内容与后缀类型不符，或文件格式不安全"); + return new Accepted(ext, contentType(ext)); + } catch (FileUploadRejectedException error) { throw error; } + catch (Exception error) { throw new FileUploadRejectedException(415, "无法识别文件内容或文件已损坏"); } + } + + private boolean imageDimensions(MultipartFile file) throws IOException { + try (var stream = new MemoryCacheImageInputStream(file.getInputStream())) { + Iterator readers = ImageIO.getImageReaders(stream); + if (!readers.hasNext()) return false; + ImageReader reader = readers.next(); + try { + reader.setInput(stream, true, true); + int width = reader.getWidth(0), height = reader.getHeight(0); + return width > 0 && height > 0 && width <= 30000 && height <= 30000 && (long) width * height <= 100_000_000; + } finally { reader.dispose(); } + } + } + + private boolean validText(MultipartFile file) throws IOException { + var decoder = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT); + try (Reader reader = new InputStreamReader(file.getInputStream(), decoder)) { + char[] buffer = new char[8192]; int count; + while ((count = reader.read(buffer)) >= 0) for (int i = 0; i < count; i++) { + char c = buffer[i]; + if (c < 32 && c != '\r' && c != '\n' && c != '\t' && c != '\f' && c != 27) return false; + } + return true; + } + } + + private boolean validArchive(MultipartFile file, String extension) throws IOException { + Path temporary = Files.createTempFile("file-upload-check-", ".zip"); + try { + try (InputStream input = file.getInputStream(); OutputStream output = Files.newOutputStream(temporary)) { + byte[] buffer = new byte[8192]; long copied = 0; int count; + while ((count = input.read(buffer)) >= 0) { + copied += count; + if (copied > properties.getUpload().getMaxFileBytes()) reject(413, "文件超过服务器上传大小限制"); + output.write(buffer, 0, count); + } + } + try (ZipFile archive = new ZipFile(temporary.toFile())) { + long expanded = 0; int entries = 0; + for (Enumeration e = archive.entries(); e.hasMoreElements();) { + ZipEntry entry = e.nextElement(); String name = entry.getName().replace('\\', '/'); + if (++entries > 50000 || name.startsWith("/") || name.matches("^[A-Za-z]:.*") + || Arrays.asList(name.split("/")).contains("..")) return false; + expanded += Math.max(0, entry.getSize()); + if (expanded > 1024L * 1024 * 1024) return false; + } + return switch (extension) { + case "zip" -> entries > 0; + case "docx" -> archive.getEntry("[Content_Types].xml") != null && archive.getEntry("word/document.xml") != null; + case "xlsx" -> archive.getEntry("[Content_Types].xml") != null && archive.getEntry("xl/workbook.xml") != null; + case "pptx" -> archive.getEntry("[Content_Types].xml") != null && archive.getEntry("ppt/presentation.xml") != null; + case "apk" -> { + ZipEntry manifest = archive.getEntry("AndroidManifest.xml"); + if (manifest == null) yield false; + try (InputStream input = archive.getInputStream(manifest)) { yield starts(input.readNBytes(4), 3, 0, 8, 0); } + } + default -> false; + }; + } + } finally { Files.deleteIfExists(temporary); } + } + + private boolean safeSvg(MultipartFile file) throws Exception { + if (file.getSize() > 1024 * 1024) reject(413, "SVG 文件不能超过 1MB"); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); factory.setXIncludeAware(false); factory.setExpandEntityReferences(false); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + factory.setAttribute("http://www.oracle.com/xml/jaxp/properties/maxElementDepth", "256"); + factory.setAttribute("http://www.oracle.com/xml/jaxp/properties/elementAttributeLimit", "256"); + try (InputStream input = file.getInputStream()) { + var document = factory.newDocumentBuilder().parse(input); + Element root = document.getDocumentElement(); + if (!"svg".equals(root.getLocalName()) || !"http://www.w3.org/2000/svg".equals(root.getNamespaceURI())) return false; + Deque nodes = new ArrayDeque<>(); nodes.push(document); int count = 0; + while (!nodes.isEmpty()) { + Node node = nodes.pop(); + if (++count > 20000 || node.getNodeType() == Node.PROCESSING_INSTRUCTION_NODE) return false; + String local = Objects.toString(node.getLocalName(), "").toLowerCase(Locale.ROOT); + if (Set.of("script", "foreignobject", "iframe", "object", "embed", "animate", "set", "animatetransform", "animatemotion").contains(local)) return false; + if (local.equals("style") && unsafeCss(node.getTextContent())) return false; + if (node.hasAttributes()) for (int i = 0; i < node.getAttributes().getLength(); i++) { + Node attr = node.getAttributes().item(i); + String name = Objects.toString(attr.getLocalName(), attr.getNodeName()).toLowerCase(Locale.ROOT), value = attr.getNodeValue().trim(); + if (name.startsWith("on") || name.equals("base") || unsafeCss(value)) return false; + if (name.equals("href") && !value.startsWith("#") + && !value.matches("(?is)data:image/(?:png|jpeg|gif|webp);base64,[a-z0-9+/=\\s]+")) return false; + } + for (Node child = node.getFirstChild(); child != null; child = child.getNextSibling()) nodes.push(child); + } + return true; + } + } + + private static boolean unsafeCss(String value) { + if (value.contains("\\") || UNSAFE_CSS.matcher(value).find()) return true; + var urls = CSS_URL.matcher(value); + while (urls.find()) { + String reference = urls.group(1).trim().replaceAll("^['\"]|['\"]$", ""); + if (!reference.startsWith("#")) return true; + } + return false; + } + + private static boolean starts(byte[] value, int... prefix) { + if (value.length < prefix.length) return false; + for (int i = 0; i < prefix.length; i++) if ((value[i] & 255) != prefix[i]) return false; + return true; + } + private static boolean ascii(byte[] value, int offset, String marker) { + return value.length >= offset + marker.length() && new String(value, offset, marker.length(), StandardCharsets.US_ASCII).equals(marker); + } + private static void reject(int status, String message) { throw new FileUploadRejectedException(status, message); } + private static String contentType(String ext) { + return switch (ext) { + case "jpg", "jpeg" -> "image/jpeg"; case "png" -> "image/png"; case "gif" -> "image/gif"; + case "bmp" -> "image/bmp"; case "webp" -> "image/webp"; case "ico" -> "image/x-icon"; case "svg" -> "image/svg+xml"; + case "pdf" -> "application/pdf"; case "apk" -> "application/vnd.android.package-archive"; + case "zip" -> "application/zip"; case "gz" -> "application/gzip"; + case "txt", "log" -> "text/plain;charset=UTF-8"; case "csv" -> "text/csv;charset=UTF-8"; + case "mp4", "mov" -> "video/mp4"; case "webm" -> "video/webm"; case "avi" -> "video/x-msvideo"; + case "mp3" -> "audio/mpeg"; case "wav" -> "audio/wav"; case "m4a" -> "audio/mp4"; case "aac" -> "audio/aac"; + case "ogg" -> "audio/ogg"; case "amr" -> "audio/amr"; case "flac" -> "audio/flac"; + default -> "application/octet-stream"; + }; + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadRejectedException.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadRejectedException.java new file mode 100644 index 000000000..b8c83a6ab --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/FileUploadRejectedException.java @@ -0,0 +1,7 @@ +package com.bsd.pjcloud.admin.service.file; + +public class FileUploadRejectedException extends RuntimeException { + private final int status; + public FileUploadRejectedException(int status, String message) { super(message); this.status = status; } + public int getStatus() { return status; } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/RedisFileUploadNonceStore.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/RedisFileUploadNonceStore.java new file mode 100644 index 000000000..2a3d05047 --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/file/RedisFileUploadNonceStore.java @@ -0,0 +1,17 @@ +package com.bsd.pjcloud.admin.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; +import java.time.Duration; + +@Component +@RequiredArgsConstructor +public class RedisFileUploadNonceStore implements FileUploadNonceStore { + private final StringRedisTemplate redis; + + @Override + public boolean claim(String appId, String kid, String nonce, Duration lifetime) { + return Boolean.TRUE.equals(redis.opsForValue().setIfAbsent("file-upload:nonce:" + appId + ':' + kid + ':' + nonce, "1", lifetime)); + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysFileServiceImpl.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysFileServiceImpl.java index 7aec783df..ab8f58ada 100644 --- a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysFileServiceImpl.java +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysFileServiceImpl.java @@ -38,6 +38,7 @@ import com.bsd.pjcloud.admin.mapper.SysFileGroupMapper; import com.bsd.pjcloud.admin.mapper.SysFileMapper; import com.bsd.pjcloud.admin.service.QnOssService; import com.bsd.pjcloud.admin.service.SysFileService; +import com.bsd.pjcloud.admin.service.file.FileUploadPolicy; import com.bsd.pjcloud.common.core.constant.CommonConstants; import com.bsd.pjcloud.common.core.util.R; import com.bsd.pjcloud.common.data.tenant.TenantContextHolder; @@ -76,6 +77,7 @@ public class SysFileServiceImpl extends ServiceImpl impl private final QnOssConfig qnOssConfig; private final QnOssService qnOssService; + private final FileUploadPolicy uploadPolicy; /** * 上传文件 @@ -88,14 +90,15 @@ public class SysFileServiceImpl extends ServiceImpl impl */ @Override public R uploadFile(MultipartFile file, String dir, Long groupId, String type) { - String fileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()); + var accepted = uploadPolicy.validate(file, dir, false); + String fileName = IdUtil.simpleUUID() + StrUtil.DOT + accepted.extension(); Map resultMap = new HashMap<>(4); resultMap.put("bucketName", properties.getBucketName()); resultMap.put("fileName", fileName); resultMap.put("url", String.format("/admin/sys-file/oss/file?fileName=%s", fileName)); try (InputStream inputStream = file.getInputStream()) { - fileTemplate.putObject(properties.getBucketName(), dir, fileName, inputStream, file.getContentType()); + fileTemplate.putObject(properties.getBucketName(), dir, fileName, inputStream, accepted.contentType()); // 文件管理数据记录,收集管理追踪文件 fileLog(file, dir, fileName, groupId, type); } catch (Exception e) { @@ -108,13 +111,14 @@ public class SysFileServiceImpl extends ServiceImpl impl @Override public R innerUpload(MultipartFile file, String dir, Long groupId, String type) { - String fileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()); + var accepted = uploadPolicy.validate(file, dir, true); + String fileName = IdUtil.simpleUUID() + StrUtil.DOT + accepted.extension(); Map resultMap = new HashMap<>(5); resultMap.put("bucketName", properties.getBucketName()); resultMap.put("fileName", fileName); resultMap.put("url", String.format("/admin/sys-file/oss/file?fileName=%s", fileName)); try (InputStream inputStream = file.getInputStream()) { - fileTemplate.putObject(properties.getBucketName(), dir, fileName, inputStream, file.getContentType()); + fileTemplate.putObject(properties.getBucketName(), dir, fileName, inputStream, accepted.contentType()); // 文件管理数据记录,收集管理追踪文件 SysFile sysFile = innerFileLog(file, dir, fileName, groupId, type); // 内部上传调用方需要文件中心主键执行后续替换、清理和保留周期管理。 @@ -130,7 +134,8 @@ public class SysFileServiceImpl extends ServiceImpl impl */ @Override public R> qnUpload(MultipartFile file, String dir, Long groupId, String type) { - String fileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()); + var accepted = uploadPolicy.validate(file, dir, false); + String fileName = IdUtil.simpleUUID() + StrUtil.DOT + accepted.extension(); Map resultMap = new HashMap<>(4); QnUploadVo upload = qnOssService.upload(file, fileName); @@ -307,7 +312,7 @@ public class SysFileServiceImpl extends ServiceImpl impl sysFile.setType(type); sysFile.setGroupId(groupId); - sysFile.setHash(SecureUtil.md5(file.getInputStream())); + try (InputStream hashInput = file.getInputStream()) { sysFile.setHash(SecureUtil.md5(hashInput)); } // 调用save方法保存SysFile对象 this.save(sysFile); } @@ -329,7 +334,7 @@ public class SysFileServiceImpl extends ServiceImpl impl sysFile.setType(type); sysFile.setGroupId(groupId); - sysFile.setHash(SecureUtil.md5(file.getInputStream())); + try (InputStream hashInput = file.getInputStream()) { sysFile.setHash(SecureUtil.md5(hashInput)); } sysFile.setCreateBy("SYSTEM"); sysFile.setUpdateBy("SYSTEM"); // 调用save方法保存SysFile对象 diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysLogBatchExportService.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysLogBatchExportService.java index 6656b5b36..6b12045f0 100644 --- a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysLogBatchExportService.java +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysLogBatchExportService.java @@ -10,50 +10,45 @@ import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.bsd.pjcloud.admin.api.dto.SysLogExportDTO; import com.bsd.pjcloud.admin.api.entity.SysLog; +import com.bsd.pjcloud.admin.api.vo.SysLogExcelVO; import com.bsd.pjcloud.admin.mapper.SysLogMapper; import com.bsd.pjcloud.admin.service.SysFileService; +import com.bsd.pjcloud.admin.service.log.SysLogSanitizer; +import com.bsd.pjcloud.admin.utils.excel.SysLogExportStyleHandler; import com.bsd.pjcloud.common.asynctask.service.*; import com.bsd.pjcloud.common.asynctask.vo.AsyncTaskProgressVO; import com.bsd.pjcloud.common.core.util.R; import com.bsd.pjcloud.common.excel.config.ExcelConfigProperties; -import com.bsd.pjcloud.common.excel.converters.LocalDateTimeStringConverter; -import com.bsd.pjcloud.common.excel.converters.LongStringConverter; import com.bsd.pjcloud.common.excel.stream.BatchExcelExportTemplate; import com.bsd.pjcloud.common.excel.stream.ExportMultipartFile; import com.bsd.pjcloud.common.excel.style.ExcelExportStyleHandler; import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.springframework.beans.BeanUtils; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Objects; /** 日志按主键游标分批读取，同步和异步导出共用原有列及统一 Excel 样式。 */ @Service @RequiredArgsConstructor public class SysLogBatchExportService { private static final String FILE_NAME = "操作日志.xlsx"; - private static final Map COLUMN_WIDTHS = Map.ofEntries( - Map.entry("id", 24D), Map.entry("logType", 28D), Map.entry("title", 32D), - Map.entry("createBy", 20D), Map.entry("createTime", 24D), Map.entry("remoteAddr", 20D), - Map.entry("userAgent", 40D), Map.entry("requestUri", 40D), Map.entry("method", 16D), - Map.entry("params", 50D), Map.entry("time", 16D), Map.entry("exception", 60D), - Map.entry("serviceId", 24D)); private final SysLogMapper mapper; private final SysFileService files; private final AsyncTaskExecutorService tasks; private final ExcelConfigProperties excelConfig; + private final SysLogSanitizer logSanitizer; public AsyncTaskProgressVO submit(SysLogExportDTO dto) { SysLogExportDTO query = snapshot(dto); - ExcelExportStyleHandler style = new ExcelExportStyleHandler(excelConfig.getExportStyle(), COLUMN_WIDTHS); + ExcelExportStyleHandler style = new SysLogExportStyleHandler(excelConfig.getExportStyle(), query.getCreateTime()); return tasks.submitWithResult("admin:export:log", "日志导出", 1, reporter -> generate(query, reporter, style, this::upload)); } @@ -61,21 +56,23 @@ public class SysLogBatchExportService { /** 文件完整生成后才写出响应，失败时仍可返回正常的错误提示。 */ public void export(HttpServletResponse response, SysLogExportDTO dto) throws Exception { SysLogExportDTO query = snapshot(dto); - ExcelExportStyleHandler style = new ExcelExportStyleHandler(excelConfig.getExportStyle(), COLUMN_WIDTHS); + ExcelExportStyleHandler style = new SysLogExportStyleHandler(excelConfig.getExportStyle(), query.getCreateTime()); generate(query, null, style, file -> BatchExportSupport.respond(file, FILE_NAME, response)); } private SysLogExportDTO snapshot(SysLogExportDTO dto) { - SysLogExportDTO query = BeanUtil.toBean(dto, SysLogExportDTO.class); - LocalDateTime[] range = dto.getCreateTime(); - if (range != null && range.length > 0 && Arrays.stream(range).anyMatch(Objects::nonNull)) { - if (range.length != 2 || range[0] == null || range[1] == null || range[0].isAfter(range[1])) { - throw new AsyncTaskException("请选择有效的日志创建时间范围"); - } - query.setCreateTime(range.clone()); - } else { - query.setCreateTime(null); + LocalDateTime[] range = dto == null ? null : dto.getCreateTime(); + if (range == null || range.length != 2 || range[0] == null || range[1] == null) { + throw new IllegalArgumentException("导出前请选择完整的日志创建时间范围"); } + if (range[0].isAfter(range[1])) { + throw new IllegalArgumentException("日志创建时间的开始时间不能晚于结束时间"); + } + if (range[1].isAfter(range[0].plusMonths(1))) { + throw new IllegalArgumentException("日志导出时间范围不能超过一个月"); + } + SysLogExportDTO query = BeanUtil.toBean(dto, SysLogExportDTO.class); + query.setCreateTime(range.clone()); query.setIds(dto.getIds() == null ? null : dto.getIds().clone()); return query; } @@ -85,7 +82,7 @@ public class SysLogBatchExportService { Page first = fetch(query, null); long total = first.getTotal(); if (total == 0) throw new AsyncTaskException("当前筛选条件没有可导出的日志"); - int maximum = BatchExcelExportTemplate.calculateMaxDetailRowCount(1, 0); + int maximum = BatchExcelExportTemplate.calculateMaxDetailRowCount(SysLogExportStyleHandler.DATA_START_ROW, 0); if (total > maximum) throw new AsyncTaskException("导出数据超过单工作表支持的 " + maximum + " 条，请缩小筛选范围"); if (reporter != null) { reporter.updateTotal(BatchExcelExportTemplate.calculateProgressTotal(total, BatchExportSupport.BATCH_SIZE, 2), @@ -94,16 +91,15 @@ public class SysLogBatchExportService { Path output = Files.createTempFile("log-export-", ".xlsx"); try { long written = 0; - try (ExcelWriter writer = EasyExcel.write(output.toFile()).head(SysLog.class) - .registerConverter(LocalDateTimeStringConverter.INSTANCE) - .registerConverter(LongStringConverter.INSTANCE) + try (ExcelWriter writer = EasyExcel.write(output.toFile()).head(SysLogExcelVO.class) + .relativeHeadRowIndex(SysLogExportStyleHandler.HEADER_ROW) .registerWriteHandler(style).build()) { WriteSheet sheet = EasyExcel.writerSheet("操作日志").build(); List rows = first.getRecords(); while (!rows.isEmpty()) { written += rows.size(); if (written > total) throw new AsyncTaskException("导出期间日志数据发生变化，请重试"); - writer.write(rows, sheet); + writer.write(rows.stream().map(this::convert).toList(), sheet); if (reporter != null) reporter.advance("已写入 " + written + " / " + total + " 条日志"); rows = fetch(query, rows.get(rows.size() - 1).getId()).getRecords(); } @@ -116,6 +112,15 @@ public class SysLogBatchExportService { } } + private SysLogExcelVO convert(SysLog log) { + SysLogExcelVO row = new SysLogExcelVO(); + BeanUtils.copyProperties(logSanitizer.copy(log), row); + row.setId(log.getId() == null ? null : log.getId().toString()); + if ("0".equals(log.getLogType())) row.setLogType("正常"); + else if ("9".equals(log.getLogType())) row.setLogType("错误"); + return row; + } + private Page fetch(SysLogExportDTO query, Long lastId) { // 各批次都只取游标后的前 200 条；新增的更大编号不会造成重复或页码偏移。 Page page = BatchExportSupport.page(1); diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysLogServiceImpl.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysLogServiceImpl.java index 16ee94190..55a5ed7ef 100644 --- a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysLogServiceImpl.java +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/impl/SysLogServiceImpl.java @@ -32,12 +32,14 @@ import com.bsd.pjcloud.admin.api.entity.SysLog; import com.bsd.pjcloud.admin.api.vo.PreLogVO; import com.bsd.pjcloud.admin.mapper.SysLogMapper; import com.bsd.pjcloud.admin.service.SysLogService; +import com.bsd.pjcloud.admin.service.log.SysLogSanitizer; import com.bsd.pjcloud.common.data.tenant.TenantBroker; import com.bsd.pjcloud.common.data.tenant.TenantContextHolder; import com.bsd.pjcloud.common.log.util.LogTypeEnum; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import lombok.RequiredArgsConstructor; import java.time.LocalDateTime; import java.util.Comparator; @@ -54,8 +56,11 @@ import java.util.stream.Collectors; * @since 2017-11-20 */ @Service +@RequiredArgsConstructor public class SysLogServiceImpl extends ServiceImpl implements SysLogService { + private final SysLogSanitizer logSanitizer; + /** * 批量插入前端错误日志 * @@ -73,6 +78,7 @@ public class SysLogServiceImpl extends ServiceImpl impleme log.setCreateTime(LocalDateTime.now()); log.setRequestUri(pre.getUrl()); log.setCreateBy(pre.getUser()); + logSanitizer.apply(log); return log; }).collect(Collectors.toList()); return this.saveBatch(sysLogs); @@ -92,7 +98,7 @@ public class SysLogServiceImpl extends ServiceImpl impleme } wrapper.eq(StrUtil.isNotBlank(sysLog.getServiceId()), SysLog::getServiceId, sysLog.getServiceId()); - return baseMapper.selectPage(page, wrapper); + return logSanitizer.copyPage(baseMapper.selectPage(page, wrapper)); } /** @@ -108,6 +114,7 @@ public class SysLogServiceImpl extends ServiceImpl impleme TenantContextHolder.setTenantId(tenantId); SysLog log = new SysLog(); BeanUtils.copyProperties(sysLog, log, "createTime"); + logSanitizer.apply(log); return baseMapper.insert(log); }); return Boolean.TRUE; diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/log/SysLogSanitizer.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/log/SysLogSanitizer.java new file mode 100644 index 000000000..029283d46 --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/service/log/SysLogSanitizer.java @@ -0,0 +1,51 @@ +package com.bsd.pjcloud.admin.service.log; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.bsd.pjcloud.admin.api.entity.SysLog; +import com.bsd.pjcloud.common.core.util.AuditLogSanitizer; +import com.bsd.pjcloud.common.log.config.PjcloudLogProperties; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.ArrayList; + +/** UPMS 写入与历史记录读取的统一边界；不依赖日志采集开关。 */ +@Component +public class SysLogSanitizer { + private final AuditLogSanitizer sanitizer; + + public SysLogSanitizer(@Value("${pjcloud.log.exclude-fields:" + PjcloudLogProperties.DEFAULT_EXCLUDE_FIELDS + "}") + List additionalFields) { + sanitizer = new AuditLogSanitizer(additionalFields); + } + + public SysLog copy(SysLog original) { + SysLog copy = new SysLog(); + BeanUtils.copyProperties(original, copy); + apply(copy); + return copy; + } + + /** 仅对准备入库的新实体或返回副本操作，绝不回写历史记录。 */ + public void apply(SysLog log) { + String uri = log.getRequestUri(); + log.setParams(sanitizer.sanitizeParameters(log.getParams(), uri)); + log.setException(sanitizer.sanitizeText(log.getException(), uri)); + log.setTitle(sanitizer.sanitizeText(log.getTitle(), uri)); + log.setUserAgent(sanitizer.sanitizeText(log.getUserAgent(), uri)); + log.setRequestUri(sanitizer.sanitizeUri(uri)); + } + + public Page copyPage(Page original) { + Page copy = new Page<>(original.getCurrent(), original.getSize(), original.getTotal(), original.searchCount()); + copy.setMaxLimit(original.maxLimit()); + copy.setCountId(original.countId()); + copy.setOptimizeCountSql(original.optimizeCountSql()); + copy.setOptimizeJoinOfCountSql(original.optimizeJoinOfCountSql()); + copy.setOrders(new ArrayList<>(original.orders())); + copy.setRecords(original.getRecords().stream().map(this::copy).toList()); + return copy; + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/utils/excel/SysLogExportStyleHandler.java b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/utils/excel/SysLogExportStyleHandler.java new file mode 100644 index 000000000..19de5dcdc --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/main/java/com/bsd/pjcloud/admin/utils/excel/SysLogExportStyleHandler.java @@ -0,0 +1,152 @@ +package com.bsd.pjcloud.admin.utils.excel; + +import com.alibaba.excel.write.handler.context.CellWriteHandlerContext; +import com.alibaba.excel.write.metadata.holder.WriteSheetHolder; +import com.alibaba.excel.write.metadata.holder.WriteWorkbookHolder; +import com.bsd.pjcloud.common.excel.style.ExcelExportStyleHandler; +import com.bsd.pjcloud.common.excel.style.ExcelStyleTokens; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.ss.util.CellRangeAddress; +import org.springframework.beans.BeanUtils; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** 日志报告布局。样式按种类复用，行高仅检查当前写入的单元格，不扫描全部日志。 */ +public class SysLogExportStyleHandler extends ExcelExportStyleHandler { + public static final int HEADER_ROW = 3; + public static final int DATA_START_ROW = HEADER_ROW + 1; + private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final Map COLUMN_WIDTHS = Map.ofEntries( + Map.entry("createTime", 32D), Map.entry("logType", 12D), Map.entry("title", 28D), + Map.entry("createBy", 18D), Map.entry("serviceId", 28D), Map.entry("method", 12D), + Map.entry("requestUri", 42D), Map.entry("time", 16D), Map.entry("remoteAddr", 42D), + Map.entry("params", 64D), Map.entry("exception", 64D), Map.entry("userAgent", 48D), + Map.entry("id", 32D)); + private static final Set WRAPPED_FIELDS = Set.of("title", "createBy", "serviceId", "requestUri", + "remoteAddr", "params", "exception", "userAgent"); + private static final Set CENTERED_FIELDS = Set.of("logType", "method"); + + private final String period; + private final String fontName; + private final Map bodyStyles = new HashMap<>(); + + public SysLogExportStyleHandler(ExcelStyleTokens source, LocalDateTime[] range) { + super(logTokens(source), COLUMN_WIDTHS); + period = "时间范围：" + range[0].format(TIME_FORMAT) + " 至 " + range[1].format(TIME_FORMAT); + fontName = source.getBody().getFontName(); + } + + /** 只调整本次日志报告，不修改全局配置或其他业务的导出。 */ + private static ExcelStyleTokens logTokens(ExcelStyleTokens source) { + ExcelStyleTokens tokens = new ExcelStyleTokens(); + BeanUtils.copyProperties(source, tokens); + var header = new ExcelStyleTokens.CellTokens(); + var body = new ExcelStyleTokens.CellTokens(); + BeanUtils.copyProperties(source.getHeader(), header); + BeanUtils.copyProperties(source.getBody(), body); + header.setRowHeight(32); + header.setWrapText(false); + body.setRowHeight(32); + body.setWrapText(true); + tokens.setHeader(header); + tokens.setBody(body); + return tokens; + } + + @Override + public void afterSheetCreate(WriteWorkbookHolder workbook, WriteSheetHolder holder) { + super.afterSheetCreate(workbook, holder); + Sheet sheet = holder.getSheet(); + sheet.setZoom(90); + Cell title = sheet.createRow(0).createCell(0); + title.setCellValue("操作日志"); + title.getRow().setHeightInPoints(32); + title.setCellStyle(textStyle(workbook.getWorkbook(), (short) 16, true, IndexedColors.DARK_BLUE)); + Cell scope = sheet.createRow(1).createCell(0); + scope.setCellValue(period); + scope.getRow().setHeightInPoints(22); + scope.setCellStyle(textStyle(workbook.getWorkbook(), (short) 10, false, IndexedColors.GREY_50_PERCENT)); + sheet.addMergedRegion(new CellRangeAddress(0, 0, 0, COLUMN_WIDTHS.size() - 1)); + sheet.addMergedRegion(new CellRangeAddress(1, 1, 0, COLUMN_WIDTHS.size() - 1)); + sheet.createRow(2).setHeightInPoints(8); + } + + private CellStyle textStyle(Workbook workbook, short size, boolean bold, IndexedColors color) { + Font font = workbook.createFont(); + font.setFontName(fontName); + font.setFontHeightInPoints(size); + font.setBold(bold); + font.setColor(color.getIndex()); + CellStyle style = workbook.createCellStyle(); + style.setFont(font); + style.setAlignment(HorizontalAlignment.CENTER); + style.setVerticalAlignment(VerticalAlignment.CENTER); + return style; + } + + @Override + public void afterCellDispose(CellWriteHandlerContext context) { + super.afterCellDispose(context); + if (!Boolean.FALSE.equals(context.getHead()) || context.getHeadData() == null) return; + Cell cell = context.getCell(); + String field = context.getHeadData().getFieldName(); + boolean wrap = WRAPPED_FIELDS.contains(field); + HorizontalAlignment alignment = "time".equals(field) ? HorizontalAlignment.RIGHT + : CENTERED_FIELDS.contains(field) ? HorizontalAlignment.CENTER : HorizontalAlignment.LEFT; + String numberFormat = "createTime".equals(field) ? "yyyy-mm-dd hh:mm:ss" : "id".equals(field) ? "@" : null; + StyleKey key = new StyleKey(cell.getCellStyle().getIndex(), alignment, wrap, numberFormat); + cell.setCellStyle(bodyStyles.computeIfAbsent(key, ignored -> { + CellStyle style = cell.getSheet().getWorkbook().createCellStyle(); + style.cloneStyleFrom(cell.getCellStyle()); + style.setAlignment(alignment); + style.setVerticalAlignment(VerticalAlignment.TOP); + style.setWrapText(wrap); + if (numberFormat != null) { + style.setDataFormat(cell.getSheet().getWorkbook().createDataFormat().getFormat(numberFormat)); + } + return style; + })); + if (wrap && cell.getCellType() == CellType.STRING) { + int lines = visibleLines(cell.getStringCellValue(), cell.getSheet().getColumnWidth(cell.getColumnIndex()) / 256D); + cell.getRow().setHeightInPoints(Math.max(cell.getRow().getHeightInPoints(), Math.min(108, 16 * lines + 12))); + } + } + + /** 为换行内容增加适量行高，极长原文保留在单元格中供展开查看。 */ + private int visibleLines(String value, double width) { + int lines = 1; + double used = 0; + double capacity = Math.max(8, width - 3); + for (int index = 0; index < value.length() && lines < 6; index++) { + char character = value.charAt(index); + if (character == '\n') { + lines++; + used = 0; + } else if (character != '\r') { + used += character > 255 ? 2 : 1; + if (used > capacity) { + lines++; + used = character > 255 ? 2 : 1; + } + } + } + return lines; + } + + @Override + public void afterWorkbookDispose(WriteWorkbookHolder holder) { + try { + super.afterWorkbookDispose(holder); + // 横向滚动时保留时间和类型，纵向滚动时保留报告上下文及表头。 + holder.getWorkbook().getSheetAt(0).createFreezePane(2, DATA_START_ROW); + } finally { + bodyStyles.clear(); + } + } + + private record StyleKey(short baseStyle, HorizontalAlignment alignment, boolean wrap, String numberFormat) {} +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileAccessTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileAccessTest.java index 6835a3390..c101fb8a2 100644 --- a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileAccessTest.java +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileAccessTest.java @@ -118,7 +118,7 @@ class FileAccessTest { } @Test void controllerGatesGetAndHeadBeforeStorageAndSupportsAnonymousClock() throws Exception { var p = properties("enforce"); var service = mock(SysFileService.class); var guard = new FileAccessGuard(p, null, clock); - var mvc = MockMvcBuilders.standaloneSetup(new SysFileController(service, guard, p)).build(); + var mvc = MockMvcBuilders.standaloneSetup(new SysFileController(service, guard, p, mock(FileUploadGuard.class))).build(); mvc.perform(get("/sys-file/oss/file").param("fileName", "a.jpg").header("Authorization", "Bearer arbitrary")).andExpect(status().isForbidden()); mvc.perform(head("/sys-file/oss/file").param("fileName", "a.jpg")).andExpect(status().isForbidden()); verifyZeroInteractions(service); diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileReadTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileReadTest.java index 73b5ee419..8b1681c08 100644 --- a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileReadTest.java +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileReadTest.java @@ -17,7 +17,7 @@ import static org.mockito.Mockito.*; class FileReadTest { @Test void missingFilesAre404AndTenantScopeIsRestored() { var template = mock(FileTemplate.class); var mapper = mock(SysFileMapper.class); - var service = new SysFileServiceImpl(template, null, null, null, null); ReflectionTestUtils.setField(service, "baseMapper", mapper); + var service = new SysFileServiceImpl(template, null, null, null, null, null); ReflectionTestUtils.setField(service, "baseMapper", mapper); TenantContextHolder.setTenantId(42L); TenantContextHolder.setTenantSkip(false); try { var response = new MockHttpServletResponse(); service.getFile("missing.jpg", response); @@ -27,7 +27,7 @@ class FileReadTest { } @Test void bytesAndReadFailuresHaveCorrectStatus() throws Exception { var template = mock(FileTemplate.class); var mapper = mock(SysFileMapper.class); - var service = new SysFileServiceImpl(template, null, null, null, null); ReflectionTestUtils.setField(service, "baseMapper", mapper); + var service = new SysFileServiceImpl(template, null, null, null, null, null); ReflectionTestUtils.setField(service, "baseMapper", mapper); var file = new SysFile(); file.setBucketName("bucket"); file.setDir("dir"); file.setOriginal("a.jpg"); file.setHash("hash"); when(mapper.selectOne(any())).thenReturn(file); var object = new S3Object(); object.setObjectContent(new ByteArrayInputStream(new byte[]{1,2,3})); @@ -39,7 +39,7 @@ class FileReadTest { } @Test void databaseFailureAlsoRestoresAnAlreadySkippedTenantScope() { var mapper = mock(SysFileMapper.class); - var service = new SysFileServiceImpl(mock(FileTemplate.class), null, null, null, null); + var service = new SysFileServiceImpl(mock(FileTemplate.class), null, null, null, null, null); ReflectionTestUtils.setField(service, "baseMapper", mapper); when(mapper.selectOne(any())).thenThrow(new RuntimeException("database unavailable")); TenantContextHolder.setTenantId(42L); TenantContextHolder.setTenantSkip(true); diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadControllerTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadControllerTest.java new file mode 100644 index 000000000..5c27b28de --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadControllerTest.java @@ -0,0 +1,104 @@ +package com.bsd.pjcloud.admin.service.file; + +import com.bsd.pjcloud.admin.api.config.QnOssConfig; +import com.bsd.pjcloud.admin.api.entity.SysFile; +import com.bsd.pjcloud.admin.api.vo.third.QnUploadVo; +import com.bsd.pjcloud.admin.controller.SysFileController; +import com.bsd.pjcloud.admin.mapper.SysFileMapper; +import com.bsd.pjcloud.admin.service.QnOssService; +import com.bsd.pjcloud.admin.service.impl.SysFileServiceImpl; +import com.bsd.pjcloud.common.core.file.FileAccessProperties; +import com.bsd.pjcloud.common.file.core.FileProperties; +import com.bsd.pjcloud.common.file.core.FileTemplate; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import java.io.InputStream; +import java.util.*; +import static com.bsd.pjcloud.admin.service.file.FileUploadTestSupport.*; +import static org.hamcrest.Matchers.matchesPattern; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +class FileUploadControllerTest { + static class Harness { + final FileAccessProperties properties = properties("enforce"); + final FileTemplate storage = mock(FileTemplate.class); + final SysFileMapper mapper = mock(SysFileMapper.class); + final QnOssService qn = mock(QnOssService.class); + final MockMvc mvc; + Harness() { + var settings = mock(FileProperties.class); when(settings.getBucketName()).thenReturn("bucket"); + var qnSettings = mock(QnOssConfig.class, RETURNS_DEEP_STUBS); when(qnSettings.getOss().getBucketName()).thenReturn("qn-bucket"); + when(mapper.insert(any(SysFile.class))).thenReturn(1); + when(qn.upload(any(), anyString())).thenAnswer(call -> { var v = new QnUploadVo(); v.setFullUrl("https://qn.example/a.png"); return v; }); + var service = new SysFileServiceImpl(storage, null, settings, qnSettings, qn, new FileUploadPolicy(properties)); + ReflectionTestUtils.setField(service, "baseMapper", mapper); + Set used = new HashSet<>(); + var guard = new FileUploadGuard(properties, (app, kid, nonce, ttl) -> used.add(app + kid + nonce), null, CLOCK); + mvc = MockMvcBuilders.standaloneSetup(new SysFileController(service, new FileAccessGuard(properties, null, CLOCK), properties, guard)) + .setControllerAdvice(new FileUploadExceptionHandler()).build(); + } + } + MockMultipartHttpServletRequestBuilder request(String path, String signedTarget, String app, String nonce, MockMultipartFile file) { + var builder = multipart(path).file(file); + var identity = signed(signedTarget, app, NOW, nonce); + for (var names = identity.getHeaderNames(); names.hasMoreElements();) { String name = names.nextElement(); builder.header(name, identity.getHeader(name)); } + return builder; + } + @Test void missingIdentityAndMultipleFilesNeverReachStorage() throws Exception { + var h = new Harness(); var file = new MockMultipartFile("file", "photo.png", "image/png", image("png")); + h.mvc.perform(multipart("/sys-file/upload").file(file)).andExpect(status().isForbidden()); + h.mvc.perform(request("/sys-file/upload", "upload", "admin", NONCE, file).file(file)).andExpect(status().isBadRequest()); + verifyZeroInteractions(h.storage, h.mapper, h.qn); + } + @Test void validUploadsKeepRawReferencesAndAReplayedRequestCannotWriteAgain() throws Exception { + var h = new Harness(); var file = new MockMultipartFile("file", "photo.PNG", "text/html", image("png")); + h.mvc.perform(request("/sys-file/upload", "upload", "admin", NONCE, file)).andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)).andExpect(jsonPath("$.data.fileName", matchesPattern("[a-f0-9]{32}\\.png"))) + .andExpect(jsonPath("$.data.url", org.hamcrest.Matchers.startsWith("/admin/sys-file/oss/file?fileName="))); + h.mvc.perform(request("/sys-file/upload", "upload", "admin", NONCE, file)).andExpect(status().isForbidden()) + .andExpect(jsonPath("$.reason").value("upload_signature_replayed")); + verify(h.storage, times(1)).putObject(eq("bucket"), isNull(), anyString(), any(InputStream.class), eq("image/png")); + verify(h.mapper, times(1)).insert(any(SysFile.class)); + } + @Test void badExtensionSpoofedContentAndOversizedFilesReturnUsefulHttpErrors() throws Exception { + var h = new Harness(); + h.mvc.perform(request("/sys-file/upload", "upload", "admin", NONCE, + new MockMultipartFile("file", "code.jsp", "image/png", image("png")))) + .andExpect(status().isUnsupportedMediaType()).andExpect(header().string("Cache-Control", "no-store")); + h.mvc.perform(request("/sys-file/upload", "upload", "admin", NONCE + "1", + new MockMultipartFile("file", "fake.png", "image/png", "".getBytes()))) + .andExpect(status().isUnsupportedMediaType()); + h.properties.getUpload().setMaxFileBytes(1); + h.mvc.perform(request("/sys-file/upload", "upload", "admin", NONCE + "2", + new MockMultipartFile("file", "photo.png", "image/png", image("png")))) + .andExpect(status().isPayloadTooLarge()); + verifyZeroInteractions(h.storage, h.mapper, h.qn); + } + @Test void qiniuAndInternalRoutesCannotBeUsedAsUnsignedAlternatives() throws Exception { + var h = new Harness(); var file = new MockMultipartFile("file", "photo.png", "image/png", image("png")); + h.mvc.perform(multipart("/sys-file/qn/upload").file(file)).andExpect(status().isForbidden()); + h.mvc.perform(multipart("/sys-file/inner-upload").file(file).header("from", "Y")).andExpect(status().isForbidden()); + h.mvc.perform(request("/sys-file/inner-upload", "inner-upload", "admin", NONCE, file)).andExpect(status().isForbidden()); + h.mvc.perform(request("/sys-file/qn/upload", "upload", "admin", NONCE, file)).andExpect(status().isForbidden()); + verifyZeroInteractions(h.storage, h.mapper, h.qn); + h.mvc.perform(request("/sys-file/qn/upload", "qn-upload", "admin", NONCE, file)).andExpect(status().isOk()).andExpect(jsonPath("$.code").value(0)); + verify(h.qn, times(1)).upload(any(), anyString()); + } + @Test void observationAllowsOldIdentityRequestsButStillRejectsUnsafeFiles() throws Exception { + var h = new Harness(); h.properties.setMode("observe"); + h.mvc.perform(multipart("/sys-file/upload").file(new MockMultipartFile("file", "fake.png", "image/png", "not an image".getBytes()))) + .andExpect(status().isUnsupportedMediaType()); + verifyZeroInteractions(h.storage, h.mapper, h.qn); + h.mvc.perform(multipart("/sys-file/upload").file(new MockMultipartFile("file", "photo.png", "image/png", image("png")))) + .andExpect(status().isOk()).andExpect(jsonPath("$.code").value(0)); + verify(h.storage, times(1)).putObject(eq("bucket"), isNull(), anyString(), any(InputStream.class), eq("image/png")); + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadGuardTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadGuardTest.java new file mode 100644 index 000000000..4891464d7 --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadGuardTest.java @@ -0,0 +1,99 @@ +package com.bsd.pjcloud.admin.service.file; + +import com.bsd.pjcloud.admin.api.config.FileAccessFeignConfiguration; +import com.bsd.pjcloud.admin.api.feign.RemoteFileService; +import com.bsd.pjcloud.common.core.file.*; +import com.bsd.pjcloud.common.core.util.R; +import com.fasterxml.jackson.databind.ObjectMapper; +import feign.Feign; +import feign.Request; +import feign.Response; +import feign.form.spring.SpringFormEncoder; +import org.junit.jupiter.api.Test; +import org.springframework.cloud.openfeign.support.SpringMvcContract; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.mock.web.MockMultipartFile; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; +import static com.bsd.pjcloud.admin.service.file.FileUploadTestSupport.*; +import static org.junit.jupiter.api.Assertions.*; + +class FileUploadGuardTest { + FileUploadGuard guard(String mode) { + Set used = ConcurrentHashMap.newKeySet(); + return new FileUploadGuard(properties(mode), (app, kid, nonce, ttl) -> used.add(app + kid + nonce), null, CLOCK); + } + @Test void sharedVectorsAndReadSignaturesNeverAuthorizeUploads() throws Exception { + var vectors = new ObjectMapper().readTree(getClass().getResourceAsStream("/file-upload-vectors.json")); + for (var v : vectors) { + assertEquals(v.get("canonical").asText(), FileUploadSignature.canonical(v.get("target").asText(), v.get("appId").asText(), v.get("kid").asText(), v.get("ts").asLong(), v.get("nonce").asText())); + assertEquals(v.get("sign").asText(), FileUploadSignature.sign(v.get("target").asText(), v.get("appId").asText(), v.get("kid").asText(), v.get("ts").asLong(), v.get("nonce").asText(), v.get("key").asText())); + } + var request = signed("upload", "admin", NOW, NONCE); + request.removeHeader(FileUploadSignature.SIGNATURE); request.addHeader(FileUploadSignature.SIGNATURE, FileAccessSignature.sign("a.jpg", "admin", "v1", NOW, KEY)); + assertEquals("invalid_upload_signature", guard("enforce").evaluate(request, "upload").reason()); + assertEquals("invalid_upload_signature", guard("enforce").evaluate(signed("upload", "service", NOW, NONCE), "inner-upload").reason()); + } + @Test void onlyServiceCanUseInternalUploadAndDartUaIsNotAnUploadCredential() { + assertEquals(200, guard("enforce").evaluate(signed("inner-upload", "service", NOW, NONCE), "inner-upload").status()); + for (String app : List.of("admin", "h5", "mini", "flutter", "package")) { + assertEquals(403, guard("enforce").evaluate(signed("inner-upload", app, NOW, NONCE), "inner-upload").status()); + } + var missing = new MockHttpServletRequest("POST", "/sys-file/upload"); missing.addHeader("User-Agent", "Dart/3.6 (dart:io)"); + assertEquals(403, guard("enforce").evaluate(missing, "upload").status()); + } + @Test void malformedDuplicateUnknownAndExpiredRequestsAreRejected() { + var guard = guard("enforce"); + for (String header : List.of(FileUploadSignature.APP, FileUploadSignature.KID, FileUploadSignature.TIMESTAMP, FileUploadSignature.NONCE, FileUploadSignature.SIGNATURE)) { + var request = signed("upload", "admin", NOW, NONCE); request.addHeader(header, request.getHeader(header)); + assertEquals(400, guard.evaluate(request, "upload").status()); + } + assertEquals(403, guard.evaluate(signed("upload", "admin", NOW - 300, NONCE), "upload").status()); + assertEquals(403, guard.evaluate(signed("upload", "admin", NOW + 61, NONCE), "upload").status()); + assertEquals(200, guard.evaluate(signed("upload", "admin", NOW + 60, NONCE), "upload").status()); + var request = signed("upload", "admin", NOW, NONCE); request.removeHeader(FileUploadSignature.KID); request.addHeader(FileUploadSignature.KID, "missing"); + assertEquals(403, guard.evaluate(request, "upload").status()); + } + @Test void nonceHasExactlyOneWinnerAcrossConcurrentRequests() throws Exception { + var guard = guard("enforce"); var executor = Executors.newFixedThreadPool(8); + try { + List> attempts = new ArrayList<>(); + for (int i = 0; i < 40; i++) attempts.add(() -> guard.evaluate(signed("upload", "admin", NOW, NONCE), "upload").status()); + var results = executor.invokeAll(attempts); + assertEquals(1, results.stream().filter(f -> { try { return f.get() == 200; } catch (Exception e) { throw new RuntimeException(e); } }).count()); + assertEquals("upload_signature_replayed", guard.evaluate(signed("upload", "admin", NOW, NONCE), "upload").reason()); + } finally { executor.shutdownNow(); } + } + @Test void observationDoesNotConsumeNoncesAndEnforcementFailsClosedWhenRedisFails() throws Exception { + var p = properties("observe"); var guard = new FileUploadGuard(p, (a, k, n, ttl) -> { throw new IllegalStateException("redis offline"); }, null, CLOCK); + assertEquals(200, guard.evaluate(signed("upload", "admin", NOW, NONCE), "upload").status()); + p.setMode("enforce"); var response = new MockHttpServletResponse(); + assertFalse(guard.authorize(signed("upload", "admin", NOW, NONCE), response, "upload")); + assertEquals(500, response.getStatus()); assertEquals("no-store", response.getHeader("Cache-Control")); + p.setMode(null); response = new MockHttpServletResponse(); + var meters = new io.micrometer.core.instrument.simple.SimpleMeterRegistry(); + var invalid = new FileUploadGuard(p, (a, k, n, ttl) -> true, meters, CLOCK); + assertFalse(invalid.authorize(signed("upload", "admin", NOW, NONCE), response, "upload")); + assertEquals(500, response.getStatus()); + assertEquals(1.0, meters.get("file.upload.requests").tag("mode", "invalid").counter().count()); + } + @Test void realFeignMultipartUploadsReceiveFreshServiceHeadersForEachRequest() throws Exception { + var signer = new FileAccessSigner(properties("enforce"), CLOCK); var captured = new AtomicReference(); + RemoteFileService client = Feign.builder().contract(new SpringMvcContract()).encoder(new SpringFormEncoder()) + .decoder((response, type) -> R.ok()) + .requestInterceptor(new FileAccessFeignConfiguration().fileAccessRequestInterceptor(signer)) + .client((request, options) -> { captured.set(request); return Response.builder().request(request).status(200).reason("OK").headers(Map.of()).build(); }) + .target(RemoteFileService.class, "https://files.example"); + var file = new MockMultipartFile("file", "qr.png", "image/png", image("png")); + client.upload(file, null, null, null); var first = captured.get(); + client.upload(file, null, null, null); var second = captured.get(); + String n1 = first.headers().get(FileUploadSignature.NONCE).iterator().next(), n2 = second.headers().get(FileUploadSignature.NONCE).iterator().next(); + assertNotEquals(n1, n2); assertEquals(1, second.headers().get(FileUploadSignature.SIGNATURE).size()); + assertEquals(FileUploadSignature.sign("upload", "service", "v1", NOW, n2, KEY), second.headers().get(FileUploadSignature.SIGNATURE).iterator().next()); + client.innerUpload(file, null, null, null); + var request = captured.get(); String nonce = request.headers().get(FileUploadSignature.NONCE).iterator().next(); + assertEquals(FileUploadSignature.sign("inner-upload", "service", "v1", NOW, nonce, KEY), request.headers().get(FileUploadSignature.SIGNATURE).iterator().next()); + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadPolicyTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadPolicyTest.java new file mode 100644 index 000000000..cfd8bebff --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadPolicyTest.java @@ -0,0 +1,95 @@ +package com.bsd.pjcloud.admin.service.file; + +import com.bsd.pjcloud.admin.api.entity.SysFile; +import com.bsd.pjcloud.admin.mapper.SysFileMapper; +import com.bsd.pjcloud.admin.service.QnOssService; +import com.bsd.pjcloud.admin.service.impl.SysFileServiceImpl; +import com.bsd.pjcloud.common.file.core.FileProperties; +import com.bsd.pjcloud.common.file.core.FileTemplate; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.util.ReflectionTestUtils; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import static com.bsd.pjcloud.admin.service.file.FileUploadTestSupport.*; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class FileUploadPolicyTest { + FileUploadPolicy policy() { return new FileUploadPolicy(properties("enforce")); } + MockMultipartFile file(String name, byte[] bytes) { return new MockMultipartFile("file", name, "application/octet-stream", bytes); } + @Test void commonBusinessFormatsAndUppercaseImageNamesRemainAllowed() throws Exception { + for (String type : new String[]{"png", "jpeg", "gif", "bmp"}) assertEquals(type, policy().validate(file("photo." + type.toUpperCase(), image(type)), null, false).extension()); + assertEquals("application/pdf", policy().validate(file("proof.pdf", "%PDF-1.4\n%%EOF".getBytes()), null, false).contentType()); + assertEquals("txt", policy().validate(file("note.txt", "中文文本\nhello".getBytes(StandardCharsets.UTF_8)), null, false).extension()); + assertEquals("zip", policy().validate(file("archive.zip", zip(Map.of("readme.txt", "hello".getBytes()))), null, false).extension()); + for (var entry : Map.of("docx", "word/document.xml", "xlsx", "xl/workbook.xml", "pptx", "ppt/presentation.xml").entrySet()) { + byte[] bytes = zip(Map.of("[Content_Types].xml", "".getBytes(), entry.getValue(), "".getBytes())); + assertEquals(entry.getKey(), policy().validate(file("report." + entry.getKey(), bytes), null, false).extension()); + } + assertEquals("apk", policy().validate(file("terminal.apk", zip(Map.of("AndroidManifest.xml", new byte[]{3,0,8,0,8,0,0,0}))), null, false).extension()); + } + @Test void scriptExecutablesAndRenamedContentsAreRejectedIndependentlyOfClientMime() throws Exception { + for (String name : new String[]{"run.exe", "x.jsp", "script.js", "x.html", "run.sh", "x.jar", "image.png.exe"}) { + var error = assertThrows(FileUploadRejectedException.class, () -> policy().validate(file(name, image("png")), null, false)); + assertEquals(415, error.getStatus()); + } + var disguised = new MockMultipartFile("file", "logo.jpg", "image/jpeg", "".getBytes()); + assertEquals(415, assertThrows(FileUploadRejectedException.class, () -> policy().validate(disguised, null, false)).getStatus()); + byte[] archive = zip(Map.of("payload.exe", "MZ".getBytes())); + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("fake.apk", archive), null, false)); + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("fake.xlsx", archive), null, false)); + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("unsafe.zip", zip(Map.of("../escape.txt", "x".getBytes()))), null, false)); + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("fake.txt", new byte[]{1,0,0,0}), null, false)); + } + @Test void activeSvgContentAndExternalEntitiesAreRejectedButStaticLogosAreAllowed() { + String safe = ""; + assertEquals("svg", policy().validate(file("logo.svg", safe.getBytes()), null, false).extension()); + for (String body : new String[]{"", "", "", "", "", ""}) { + String svg = "" + body + ""; + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("unsafe.svg", svg.getBytes()), null, false)); + } + String xxe = "]>&e;"; + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("xxe.svg", xxe.getBytes()), null, false)); + String deep = "" + "".repeat(300) + "".repeat(300) + ""; + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("deep.svg", deep.getBytes()), null, false)); + } + @Test void logFilesAreInternalOnlyAndSizeAndNamesAreCheckedBeforeStorage() { + byte[] log = "日志\n\u001b[32mmessage\u001b[0m".getBytes(StandardCharsets.UTF_8); + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("terminal.log", log), null, false)); + assertEquals("log", policy().validate(file("terminal.log", log), "terminal-log", true).extension()); + assertEquals(400, assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("empty.pdf", new byte[0]), null, false)).getStatus()); + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("../image.png", log), null, false)); + assertThrows(FileUploadRejectedException.class, () -> policy().validate(file("note.txt", log), "../outside", false)); + var p = properties("enforce"); p.getUpload().setMaxFileBytes(2); + assertEquals(413, assertThrows(FileUploadRejectedException.class, () -> new FileUploadPolicy(p).validate(file("note.txt", log), null, false)).getStatus()); + } + @Test void allServiceEntryPointsValidateBeforeAnyExternalWriteAndValidImagesUseCanonicalMime() throws Exception { + var storage = mock(FileTemplate.class); var mapper = mock(SysFileMapper.class); var qn = mock(QnOssService.class); + var settings = mock(FileProperties.class); when(settings.getBucketName()).thenReturn("bucket"); + var service = new SysFileServiceImpl(storage, null, settings, null, qn, policy()); ReflectionTestUtils.setField(service, "baseMapper", mapper); + var bad = file("photo.png", "MZbinary".getBytes()); + assertThrows(FileUploadRejectedException.class, () -> service.uploadFile(bad, null, null, null)); + assertThrows(FileUploadRejectedException.class, () -> service.innerUpload(bad, null, null, null)); + assertThrows(FileUploadRejectedException.class, () -> service.qnUpload(bad, null, null, null)); + verifyZeroInteractions(storage, mapper, qn); + when(mapper.insert(any(SysFile.class))).thenReturn(1); + var response = service.uploadFile(new MockMultipartFile("file", "photo.PNG", "text/html", image("png")), "images", null, null); + assertTrue(response.isOk()); + verify(storage).putObject(eq("bucket"), eq("images"), matches("[a-f0-9]{32}\\.png"), any(InputStream.class), eq("image/png")); + verify(mapper).insert(any(SysFile.class)); + } + @Test + @EnabledIfSystemProperty(named = "upload.apk.fixture", matches = ".+") + void actualFlutterReleasePackagePassesTheFormatPolicy() throws Exception { + Path path = Path.of(System.getProperty("upload.apk.fixture")); + try (InputStream input = Files.newInputStream(path)) { + assertEquals("apk", policy().validate(new MockMultipartFile("file", "terminal.apk", "application/octet-stream", input), null, false).extension()); + } + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadTestSupport.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadTestSupport.java new file mode 100644 index 000000000..a2c4760db --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/file/FileUploadTestSupport.java @@ -0,0 +1,43 @@ +package com.bsd.pjcloud.admin.service.file; + +import com.bsd.pjcloud.common.core.file.*; +import org.springframework.mock.web.MockHttpServletRequest; +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.time.*; +import java.util.*; +import java.util.zip.*; + +final class FileUploadTestSupport { + static final long NOW = 1793232000L; + static final String KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; + static final Clock CLOCK = Clock.fixed(Instant.ofEpochSecond(NOW), ZoneOffset.UTC); + static final String NONCE = "00112233445566778899aabbccddeeff"; + static FileAccessProperties properties(String mode) { + var p = new FileAccessProperties(); p.setMode(mode); p.setPublicBaseUrl("https://files.example/api"); + for (String app : List.of("admin", "h5", "mini", "flutter", "service", "package")) { + var client = new FileAccessProperties.Client(); client.getKeys().put("v1", KEY); p.getClients().put(app, client); + } + return p; + } + static MockHttpServletRequest signed(String target, String app, long ts, String nonce) { + var request = new MockHttpServletRequest("POST", "/sys-file/" + target); + request.addHeader(FileUploadSignature.APP, app); request.addHeader(FileUploadSignature.KID, "v1"); + request.addHeader(FileUploadSignature.TIMESTAMP, Long.toString(ts)); request.addHeader(FileUploadSignature.NONCE, nonce); + request.addHeader(FileUploadSignature.SIGNATURE, FileUploadSignature.sign(target, app, "v1", ts, nonce, KEY)); + return request; + } + static byte[] image(String format) throws Exception { + var out = new ByteArrayOutputStream(); + if (!ImageIO.write(new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB), format, out)) throw new IllegalArgumentException(format); + return out.toByteArray(); + } + static byte[] zip(Map files) throws Exception { + var out = new ByteArrayOutputStream(); + try (var zip = new ZipOutputStream(out)) { + for (var item : files.entrySet()) { zip.putNextEntry(new ZipEntry(item.getKey())); zip.write(item.getValue()); zip.closeEntry(); } + } + return out.toByteArray(); + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogBatchExportTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogBatchExportTest.java index 9c7a45427..c145e6569 100644 --- a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogBatchExportTest.java +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogBatchExportTest.java @@ -8,6 +8,8 @@ import com.bsd.pjcloud.admin.api.dto.SysLogExportDTO; import com.bsd.pjcloud.admin.api.entity.SysLog; import com.bsd.pjcloud.admin.mapper.SysLogMapper; import com.bsd.pjcloud.admin.service.SysFileService; +import com.bsd.pjcloud.admin.service.log.SysLogSanitizer; +import com.bsd.pjcloud.admin.utils.excel.SysLogExportStyleHandler; import com.bsd.pjcloud.common.asynctask.service.*; import com.bsd.pjcloud.common.core.util.R; import com.bsd.pjcloud.common.excel.config.ExcelConfigProperties; @@ -22,9 +24,12 @@ import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayInputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.LocalDateTime; import java.util.*; import java.util.regex.Pattern; +import java.util.function.Consumer; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.*; @@ -35,6 +40,39 @@ class SysLogBatchExportTest { private static final LocalDateTime START = LocalDateTime.of(2026, 10, 29, 0, 0); private static final LocalDateTime END = START.plusDays(1).minusSeconds(1); + @Test + void synchronousAndAsyncExportsFilterHistoryWithoutMutatingMapperRecords() throws Exception { + for (boolean async : List.of(false, true)) { + Harness harness = new Harness(2); + List originalRows = new ArrayList<>(); + harness.customize = row -> { + row.setParams("{\"password\":\"excel-secret\",\"refreshToken\":\"excel-token\",\"username\":\"operator\"}"); + row.setRequestUri("/auth/oauth2/token?code=123456&username=operator"); + row.setException("failed Authorization: Bearer exception-token"); + originalRows.add(row); + }; + byte[] bytes; + if (async) { + harness.submit(query()).execute(mock(AsyncTaskProgressReporter.class)); + bytes = harness.uploaded; + } else { + var response = new MockHttpServletResponse(); + harness.service.export(response, query()); + bytes = response.getContentAsByteArray(); + } + try (var workbook = new XSSFWorkbook(new ByteArrayInputStream(bytes))) { + var sheet = workbook.getSheetAt(0); + for (int index = 4; index <= 5; index++) { + assertEquals("{\"username\":\"operator\"}", sheet.getRow(index).getCell(9).getStringCellValue()); + assertEquals("参数已隐藏", sheet.getRow(index).getCell(10).getStringCellValue()); + assertEquals("/auth/oauth2/token?username=operator", sheet.getRow(index).getCell(6).getStringCellValue()); + } + } + assertTrue(originalRows.stream().allMatch(row -> row.getParams().contains("excel-secret"))); + assertTrue(originalRows.stream().allMatch(row -> row.getRequestUri().contains("123456"))); + } + } + @BeforeAll static void tableInfo() { TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), SysLog.class); @@ -81,12 +119,12 @@ class SysLogBatchExportTest { } @Test - void legacyExportUsesTheSameBatchesAndStylesAndBlankFiltersExportAllTypes() throws Exception { + void legacyExportUsesTheSameReportAndRequiresDatesWhileTypeAndServiceRemainOptional() throws Exception { Harness harness = new Harness(201); var query = new SysLogExportDTO(); query.setLogType(""); query.setServiceId(""); - query.setCreateTime(new LocalDateTime[]{null}); // GET createTime= + query.setCreateTime(new LocalDateTime[]{START, END}); query.setIds(new Long[0]); var response = new MockHttpServletResponse(); harness.service.export(response, query); @@ -94,9 +132,15 @@ class SysLogBatchExportTest { assertEquals(BatchExportSupport.CONTENT_TYPE, response.getContentType()); assertTrue(response.getHeader("Content-Disposition").endsWith(".xlsx")); assertEquals(3, harness.pages.size()); - assertEquals("ORDER BY id DESC", harness.queries.get(0).getSqlSegment().trim()); + assertFalse(harness.queries.get(0).getSqlSegment().contains("log_type")); + assertFalse(harness.queries.get(0).getSqlSegment().contains("service_id")); + assertFalse(harness.queries.get(0).getSqlSegment().contains(" IN ")); + assertEquals(START, parameter(harness.queries.get(0), "create_time", ">=")); + assertEquals(END, parameter(harness.queries.get(0), "create_time", "<=")); verifyNoMoreInteractions(harness.tasks, harness.files); assertWorkbook(response.getContentAsByteArray(), 201); + String preview = System.getProperty("log.export.preview"); + if (preview != null) Files.write(Path.of(preview), response.getContentAsByteArray()); } @Test @@ -104,14 +148,14 @@ class SysLogBatchExportTest { Harness empty = new Harness(0); var response = new MockHttpServletResponse(); assertEquals("当前筛选条件没有可导出的日志", assertThrows(AsyncTaskException.class, - () -> empty.service.export(response, new SysLogExportDTO())).getMessage()); + () -> empty.service.export(response, query())).getMessage()); assertEquals(0, response.getContentAsByteArray().length); assertFalse(response.isCommitted()); verifyNoMoreInteractions(empty.files); - Harness oversized = new Harness((long) BatchExcelExportTemplate.calculateMaxDetailRowCount(1, 0) + 1); + Harness oversized = new Harness((long) BatchExcelExportTemplate.calculateMaxDetailRowCount(SysLogExportStyleHandler.DATA_START_ROW, 0) + 1); assertTrue(assertThrows(AsyncTaskException.class, - () -> oversized.service.export(response, new SysLogExportDTO())).getMessage().contains("超过单工作表")); + () -> oversized.service.export(response, query())).getMessage().contains("超过单工作表")); assertEquals(0, response.getContentAsByteArray().length); verifyNoMoreInteractions(oversized.files); } @@ -122,56 +166,110 @@ class SysLogBatchExportTest { changed.availableRows = 200; var response = new MockHttpServletResponse(); assertEquals("导出期间日志数据发生变化，请重试", assertThrows(AsyncTaskException.class, - () -> changed.service.export(response, new SysLogExportDTO())).getMessage()); + () -> changed.service.export(response, query())).getMessage()); assertEquals(0, response.getContentAsByteArray().length); Harness failedUpload = new Harness(1); when(failedUpload.files.uploadFile(any(), any(), any(), any())).thenReturn(R.failed("保存失败")); - var action = failedUpload.submit(new SysLogExportDTO()); + var action = failedUpload.submit(query()); assertEquals("导出文件保存失败，请重试", assertThrows(AsyncTaskException.class, () -> action.execute(mock(AsyncTaskProgressReporter.class))).getMessage()); } @Test - void invalidRangesFailBeforeSubmittingOrReadingLogs() { + void missingRangesFailOnSyncAndAsyncBeforeSubmittingOrReadingLogs() { Harness harness = new Harness(1); - for (LocalDateTime[] range : List.of(new LocalDateTime[]{START}, - new LocalDateTime[]{START, null}, new LocalDateTime[]{END, START})) { + for (LocalDateTime[] range : Arrays.asList(null, new LocalDateTime[0], new LocalDateTime[]{null}, + new LocalDateTime[]{START}, new LocalDateTime[]{START, null}, new LocalDateTime[]{null, END}, + new LocalDateTime[]{null, null})) { var query = new SysLogExportDTO(); query.setCreateTime(range); - assertEquals("请选择有效的日志创建时间范围", assertThrows(AsyncTaskException.class, - () -> harness.service.submit(query)).getMessage()); + assertRangeRejected(harness, query, "导出前请选择完整的日志创建时间范围"); } verifyNoMoreInteractions(harness.tasks, harness.mapper, harness.files); } + @Test + void reversedOrMoreThanOneCalendarMonthRangesFailOnBothRoutes() { + Harness harness = new Harness(1); + var reversed = query(); + reversed.setCreateTime(new LocalDateTime[]{END, START}); + assertRangeRejected(harness, reversed, "日志创建时间的开始时间不能晚于结束时间"); + for (LocalDateTime start : List.of(START, LocalDateTime.of(2026, 1, 31, 8, 30), + LocalDateTime.of(2024, 1, 31, 8, 30))) { + var query = query(); + query.setCreateTime(new LocalDateTime[]{start, start.plusMonths(1).plusSeconds(1)}); + assertRangeRejected(harness, query, "日志导出时间范围不能超过一个月"); + } + verifyNoMoreInteractions(harness.tasks, harness.mapper, harness.files); + } + + @Test + void exactCalendarMonthAndMonthEndAndLeapYearBoundariesAreAllowed() { + for (LocalDateTime start : List.of(START, LocalDateTime.of(2026, 1, 31, 8, 30), + LocalDateTime.of(2024, 1, 31, 8, 30))) { + Harness harness = new Harness(1); + var query = query(); + query.setCreateTime(new LocalDateTime[]{start, start.plusMonths(1)}); + harness.submit(query); + verifyNoMoreInteractions(harness.mapper, harness.files); + assertEquals(36, harness.properties.getExportStyle().getHeader().getRowHeight()); + assertFalse(harness.properties.getExportStyle().getBody().isWrapText()); + } + } + + private void assertRangeRejected(Harness harness, SysLogExportDTO query, String message) { + var response = new MockHttpServletResponse(); + assertEquals(message, assertThrows(IllegalArgumentException.class, () -> harness.service.submit(query)).getMessage()); + assertEquals(message, assertThrows(IllegalArgumentException.class, () -> harness.service.export(response, query)).getMessage()); + assertEquals(0, response.getContentAsByteArray().length); + assertFalse(response.isCommitted()); + } + private void assertWorkbook(byte[] bytes, int rows) throws Exception { try (var workbook = new XSSFWorkbook(new ByteArrayInputStream(bytes))) { var sheet = workbook.getSheetAt(0); List headers = new ArrayList<>(); - for (Cell cell : sheet.getRow(0)) headers.add(cell.getStringCellValue()); - assertEquals(List.of("日志编号", "日志类型（0-正常 9-错误）", "日志标题", "创建人", "创建时间", - "操作ip地址", "userAgent", "浏览器", "操作方式", "提交数据", "执行时间", "异常信息", "应用标识"), headers); - assertEquals(rows, sheet.getLastRowNum()); - assertEquals(Long.toString(FIRST_ID), sheet.getRow(1).getCell(0).getStringCellValue()); - assertEquals(Long.toString(FIRST_ID - rows + 1), sheet.getRow(rows).getCell(0).getStringCellValue()); - assertEquals("2026-10-29 00:00:00", sheet.getRow(1).getCell(4).getStringCellValue()); - assertEquals("{\"message\":\"中文日志\"}", sheet.getRow(1).getCell(9).getStringCellValue()); - var header = sheet.getRow(0).getCell(0).getCellStyle(); + assertEquals("操作日志", sheet.getRow(0).getCell(0).getStringCellValue()); + assertTrue(sheet.getRow(1).getCell(0).getStringCellValue().contains("2026-10-29 00:00:00 至 2026-10-29 23:59:59")); + assertEquals(List.of("A1:M1", "A2:M2"), sheet.getMergedRegions().stream() + .map(org.apache.poi.ss.util.CellRangeAddress::formatAsString).toList()); + for (Cell cell : sheet.getRow(3)) headers.add(cell.getStringCellValue()); + assertEquals(List.of("创建时间", "日志类型", "日志标题", "操作人", "所属应用", "请求方式", "请求地址", + "耗时（毫秒）", "IP 地址", "提交数据", "异常详情", "用户代理", "日志编号"), headers); + assertEquals(rows + 3, sheet.getLastRowNum()); + assertEquals(Long.toString(FIRST_ID), sheet.getRow(4).getCell(12).getStringCellValue()); + assertEquals("@", sheet.getRow(4).getCell(12).getCellStyle().getDataFormatString()); + assertEquals(Long.toString(FIRST_ID - rows + 1), sheet.getRow(rows + 3).getCell(12).getStringCellValue()); + assertEquals(START, sheet.getRow(4).getCell(0).getLocalDateTimeCellValue()); + assertTrue(org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(sheet.getRow(4).getCell(0))); + assertEquals("yyyy-mm-dd hh:mm:ss", sheet.getRow(4).getCell(0).getCellStyle().getDataFormatString()); + assertEquals("正常", sheet.getRow(4).getCell(1).getStringCellValue()); + assertEquals("错误", sheet.getRow(5).getCell(1).getStringCellValue()); + assertEquals(1116D, sheet.getRow(4).getCell(7).getNumericCellValue()); + assertEquals("{\"message\":\"中文日志\"}", sheet.getRow(4).getCell(9).getStringCellValue()); + assertTrue(sheet.getRow(5).getCell(10).getStringCellValue().contains("IllegalStateException")); + assertTrue(sheet.getRow(5).getCell(10).getCellStyle().getWrapText()); + assertTrue(sheet.getRow(5).getHeightInPoints() > 32); + var header = sheet.getRow(3).getCell(0).getCellStyle(); assertTrue(header.getFont().getBold()); assertEquals("Microsoft YaHei", header.getFont().getFontName()); assertEquals("1F4E78", rgb(header.getFillForegroundXSSFColor().getRGB())); - assertEquals(36F, sheet.getRow(0).getHeightInPoints()); - assertEquals(26F, sheet.getRow(1).getHeightInPoints()); - assertEquals("F3F6FA", rgb(sheet.getRow(200).getCell(0).getCellStyle().getFillForegroundXSSFColor().getRGB())); - assertEquals("FFFFFF", rgb(sheet.getRow(201).getCell(0).getCellStyle().getFillForegroundXSSFColor().getRGB())); - assertEquals(24 * 256, sheet.getColumnWidth(0)); - assertEquals(50 * 256, sheet.getColumnWidth(9)); - assertEquals(60 * 256, sheet.getColumnWidth(11)); + assertEquals(32F, sheet.getRow(3).getHeightInPoints()); + assertFalse(header.getWrapText()); + assertEquals(32F, sheet.getRow(4).getHeightInPoints()); + assertEquals("F3F6FA", rgb(sheet.getRow(203).getCell(0).getCellStyle().getFillForegroundXSSFColor().getRGB())); + assertEquals("FFFFFF", rgb(sheet.getRow(204).getCell(0).getCellStyle().getFillForegroundXSSFColor().getRGB())); + assertEquals(32 * 256, sheet.getColumnWidth(0)); + assertEquals(64 * 256, sheet.getColumnWidth(9)); + assertEquals(64 * 256, sheet.getColumnWidth(10)); + assertEquals(32 * 256, sheet.getColumnWidth(12)); assertTrue(sheet.getPaneInformation().isFreezePane()); - assertEquals("A1:M" + (rows + 1), sheet.getCTWorksheet().getAutoFilter().getRef()); + assertEquals(4, sheet.getPaneInformation().getHorizontalSplitPosition()); + assertEquals(2, sheet.getPaneInformation().getVerticalSplitPosition()); + assertEquals("A4:M" + (rows + 4), sheet.getCTWorksheet().getAutoFilter().getRef()); assertFalse(sheet.isDisplayGridlines()); - assertTrue(workbook.getNumCellStyles() < 20); + assertTrue(workbook.getNumCellStyles() < 40, "样式必须按种类复用"); } } @@ -199,11 +297,12 @@ class SysLogBatchExportTest { private final SysFileService files = mock(SysFileService.class); private final AsyncTaskExecutorService tasks = mock(AsyncTaskExecutorService.class); private final ExcelConfigProperties properties = new ExcelConfigProperties(); - private final SysLogBatchExportService service = new SysLogBatchExportService(mapper, files, tasks, properties); + private final SysLogBatchExportService service = new SysLogBatchExportService(mapper, files, tasks, properties, new SysLogSanitizer(List.of())); private final List> pages = new ArrayList<>(); private final List> queries = new ArrayList<>(); private long availableRows; private byte[] uploaded; + private Consumer customize = row -> {}; private Harness(long total) { availableRows = total; @@ -218,11 +317,19 @@ class SysLogBatchExportTest { for (long index = start; index < Math.min(start + page.getSize(), availableRows); index++) { var row = new SysLog(); row.setId(FIRST_ID - index); - row.setLogType("0"); - row.setTitle("日志" + index); + row.setLogType(index == 1 ? "9" : "0"); + row.setTitle(index == 1 ? "修改用户失败" : "登录成功"); + row.setCreateBy("admin"); row.setCreateTime(START); + row.setMethod("POST"); + row.setRequestUri("/admin/user/login"); + row.setRemoteAddr("192.168.10.23"); + row.setTime(1116L + index); + row.setUserAgent("Mozilla/5.0 Chrome/130.0.0.0 Safari/537.36"); row.setParams("{\"message\":\"中文日志\"}"); + if (index == 1) row.setException("java.lang.IllegalStateException: 用户状态不可用\n at example.UserService.update(UserService.java:42)\n at example.UserController.update(UserController.java:27)"); row.setServiceId("sms"); + customize.accept(row); rows.add(row); } return new Page(1, page.getSize()).setTotal(total).setRecords(rows); diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogExportControllerTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogExportControllerTest.java index fbc30ffb6..0c620dd3b 100644 --- a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogExportControllerTest.java +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogExportControllerTest.java @@ -3,8 +3,14 @@ package com.bsd.pjcloud.admin.service.impl; import com.bsd.pjcloud.admin.api.dto.SysLogExportDTO; import com.bsd.pjcloud.admin.controller.SysLogController; import com.bsd.pjcloud.admin.service.SysLogService; +import com.bsd.pjcloud.admin.service.SysFileService; +import com.bsd.pjcloud.admin.mapper.SysLogMapper; +import com.bsd.pjcloud.admin.service.log.SysLogSanitizer; +import com.bsd.pjcloud.common.asynctask.service.AsyncTaskExecutorService; import com.bsd.pjcloud.common.core.jackson.PjcloudJavaTimeModule; +import com.bsd.pjcloud.common.excel.config.ExcelConfigProperties; import com.bsd.pjcloud.common.security.annotation.HasPermission; +import com.bsd.pjcloud.common.sentinel.handle.GlobalBizExceptionHandler; import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.servlet.http.HttpServletResponse; import org.junit.jupiter.api.BeforeEach; @@ -22,6 +28,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; class SysLogExportControllerTest { private final SysLogBatchExportService exports = mock(SysLogBatchExportService.class); @@ -35,12 +42,7 @@ class SysLogExportControllerTest { } @Test - void originalGetSupportsEmptyFiltersAndTimeRangeAndIds() throws Exception { - mvc.perform(get("/log/export").param("logType", "").param("createTime", "").param("serviceId", "")) - .andExpect(status().isOk()); - verify(exports).export(any(HttpServletResponse.class), any(SysLogExportDTO.class)); - - reset(exports); + void originalGetBindsTimeRangeAndIds() throws Exception { mvc.perform(get("/log/export").param("logType", "9").param("serviceId", "sms") .param("createTime", "2026-10-29 00:00:00", "2026-10-29 23:59:59") .param("ids", "90071992547409930", "90071992547409929")) @@ -54,6 +56,32 @@ class SysLogExportControllerTest { assertArrayEquals(new Long[]{90071992547409930L, 90071992547409929L}, captured.getValue().getIds()); } + @Test + void bothRoutesRejectMissingOrExcessiveRangesWithReadableBadRequestMessages() throws Exception { + SysLogMapper mapper = mock(SysLogMapper.class); + SysFileService files = mock(SysFileService.class); + AsyncTaskExecutorService tasks = mock(AsyncTaskExecutorService.class); + var service = new SysLogBatchExportService(mapper, files, tasks, new ExcelConfigProperties(), new SysLogSanitizer(java.util.List.of())); + ObjectMapper json = new ObjectMapper().registerModule(new PjcloudJavaTimeModule()); + MockMvc guarded = MockMvcBuilders.standaloneSetup(new SysLogController(mock(SysLogService.class), service)) + .setControllerAdvice(new GlobalBizExceptionHandler()) + .setMessageConverters(new MappingJackson2HttpMessageConverter(json)).build(); + guarded.perform(get("/log/export").param("logType", "").param("createTime", "").param("serviceId", "")) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.msg").value("导出前请选择完整的日志创建时间范围")); + guarded.perform(post("/log/export-async").contentType(MediaType.APPLICATION_JSON).content("{}")) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.msg").value("导出前请选择完整的日志创建时间范围")); + guarded.perform(get("/log/export").param("createTime", "2026-10-01 00:00:00", "2026-11-01 00:00:01")) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.msg").value("日志导出时间范围不能超过一个月")); + guarded.perform(post("/log/export-async").contentType(MediaType.APPLICATION_JSON).content(""" + {"createTime":["2026-01-31 08:30:00","2026-02-28 08:30:01"]} + """)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.msg").value("日志导出时间范围不能超过一个月")); + verifyNoMoreInteractions(mapper, files, tasks); + } + @Test void asyncAcceptsListDateFormatAndBothRoutesKeepExportPermission() throws Exception { mvc.perform(post("/log/export-async").contentType(MediaType.APPLICATION_JSON).content(""" diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogSanitizationTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogSanitizationTest.java new file mode 100644 index 000000000..39c3e161b --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysLogSanitizationTest.java @@ -0,0 +1,167 @@ +package com.bsd.pjcloud.admin.service.impl; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.baomidou.mybatisplus.core.metadata.OrderItem; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.bsd.pjcloud.admin.api.dto.SysLogDTO; +import com.bsd.pjcloud.admin.api.entity.SysLog; +import com.bsd.pjcloud.admin.api.vo.PreLogVO; +import com.bsd.pjcloud.admin.mapper.SysLogMapper; +import com.bsd.pjcloud.admin.service.log.SysLogSanitizer; +import com.bsd.pjcloud.common.core.jackson.PjcloudJavaTimeModule; +import com.bsd.pjcloud.common.core.util.AuditLogSanitizer; +import com.bsd.pjcloud.common.data.tenant.TenantContextHolder; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.test.context.support.TestPropertySourceUtils; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class SysLogSanitizationTest { + private final SysLogSanitizer sanitizer = new SysLogSanitizer(List.of()); + private final SysLogMapper mapper = mock(SysLogMapper.class); + private final SysLogServiceImpl service = new SysLogServiceImpl(sanitizer); + + @BeforeAll + static void metadata() { + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), SysLog.class); + } + + @AfterEach + void cleanup() { TenantContextHolder.clear(); } + + @Test + void directFeignWriteIsFilteredBeforeInsertWithoutChangingTheSubmittedDtoOrTenantContext() { + ReflectionTestUtils.setField(service, "baseMapper", mapper); + when(mapper.insert(any(SysLog.class))).thenAnswer(call -> { + assertEquals(7L, TenantContextHolder.getTenantId()); + return 1; + }); + TenantContextHolder.setTenantId(9L); + var dto = new SysLogDTO(); + dto.setTenantId(7L); + dto.setTitle("更新业务记录"); + dto.setLogType("0"); + dto.setParams("{\"newpassword1\":\"direct-secret\",\"access_token\":\"direct-token\",\"id\":42}"); + dto.setRequestUri("/orders?code=BUS-42&client_secret=url-secret"); + dto.setException("upstream failed Authorization: Bearer error-token"); + assertTrue(service.saveLog(dto)); + + var inserted = ArgumentCaptor.forClass(SysLog.class); + verify(mapper).insert(inserted.capture()); + assertEquals("{\"id\":42}", inserted.getValue().getParams()); + assertEquals("/orders?code=BUS-42", inserted.getValue().getRequestUri()); + assertEquals(AuditLogSanitizer.HIDDEN, inserted.getValue().getException()); + assertTrue(dto.getParams().contains("direct-secret")); + assertTrue(dto.getRequestUri().contains("url-secret")); + assertEquals(9L, TenantContextHolder.getTenantId()); + } + + @Test + void frontendErrorsAreFilteredBeforeBatchWrite() { + var spy = spy(service); + doReturn(true).when(spy).saveBatch(anyCollection()); + PreLogVO error = new PreLogVO(); + error.setInfo("前端请求失败"); + error.setUrl("/auth/oauth2/token?code=123456&username=operator"); + error.setUser("operator"); + error.setMessage("code=123456&password=frontend-secret&username=operator"); + error.setStack("request failed password=error-secret"); + assertTrue(spy.saveBatchLogs(List.of(error))); + + var batch = ArgumentCaptor.forClass(Collection.class); + verify(spy).saveBatch(batch.capture()); + SysLog inserted = (SysLog) batch.getValue().iterator().next(); + assertEquals("username=operator", inserted.getParams()); + assertEquals("/auth/oauth2/token?username=operator", inserted.getRequestUri()); + assertEquals(AuditLogSanitizer.HIDDEN, inserted.getException()); + assertTrue(error.getMessage().contains("frontend-secret")); + } + + @Test + void historicalPageReturnsFilteredCopiesAndPreservesDatabaseRowsAndPagination() throws Exception { + ReflectionTestUtils.setField(service, "baseMapper", mapper); + SysLog original = history(); + Page stored = new Page(3, 10, 51).setRecords(List.of(original)); + stored.setMaxLimit(200L); + stored.setSearchCount(false); + stored.setOptimizeCountSql(false); + stored.setOptimizeJoinOfCountSql(false); + stored.setCountId("customLogCount"); + stored.addOrder(OrderItem.desc("create_time")); + when(mapper.selectPage(any(Page.class), any())).thenReturn(stored); + Page result = service.getLogByPage(new Page<>(3, 10), new SysLogDTO()); + + assertNotSame(stored, result); + assertNotSame(original, result.getRecords().get(0)); + assertEquals(3, result.getCurrent()); + assertEquals(10, result.getSize()); + assertEquals(51, result.getTotal()); + assertEquals(200L, result.maxLimit()); + assertFalse(result.searchCount()); + assertFalse(result.optimizeCountSql()); + assertFalse(result.optimizeJoinOfCountSql()); + assertEquals("customLogCount", result.countId()); + assertEquals("create_time", result.orders().get(0).getColumn()); + assertNotSame(stored.orders(), result.orders()); + assertEquals("{\"id\":42,\"code\":\"BUS-42\"}", result.getRecords().get(0).getParams()); + String response = new ObjectMapper().registerModule(new PjcloudJavaTimeModule()).writeValueAsString(result); + assertFalse(response.contains("history-secret")); + assertFalse(response.contains("history-token")); + assertFalse(response.contains("url-secret")); + assertEquals(original.getCreateTime(), result.getRecords().get(0).getCreateTime()); + assertTrue(original.getParams().contains("history-secret")); + assertTrue(original.getRequestUri().contains("url-secret")); + verify(mapper).selectPage(any(Page.class), any()); + verifyNoMoreInteractions(mapper); + } + + @Test + void additionalFieldsAndFailClosedHistoricalParametersUseTheSameReadBoundary() { + SysLog original = history(); + original.setParams("{\"phone\":\"123\",\"username\":\"operator\",\"password\":\"secret\"}"); + SysLog filtered = new SysLogSanitizer(List.of("phone")).copy(original); + assertEquals("{\"username\":\"operator\"}", filtered.getParams()); + original.setParams("[\"anonymous-password\"]"); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.copy(original).getParams()); + original.setParams("{\"password\":\"truncated"); + assertEquals(AuditLogSanitizer.HIDDEN, sanitizer.copy(original).getParams()); + } + + @Test + void upmsProtectionAndConfiguredExclusionsRemainActiveWhenCollectionIsDisabled() { + try (var context = new AnnotationConfigApplicationContext()) { + TestPropertySourceUtils.addInlinedPropertiesToEnvironment(context, + "pjcloud.log.enabled=false", "pjcloud.log.exclude-fields=phone"); + context.register(SysLogSanitizer.class); + context.refresh(); + SysLog source = history(); + source.setParams("{\"phone\":\"123\",\"newpassword1\":\"secret\",\"username\":\"operator\"}"); + assertEquals("{\"username\":\"operator\"}", context.getBean(SysLogSanitizer.class).copy(source).getParams()); + } + } + + private SysLog history() { + var row = new SysLog(); + row.setId(42L); + row.setLogType("0"); + row.setTitle("历史业务操作"); + row.setCreateTime(LocalDateTime.of(2026, 10, 29, 12, 30)); + row.setParams("{\"password\":\"history-secret\",\"refreshToken\":\"history-token\",\"id\":42,\"code\":\"BUS-42\"}"); + row.setRequestUri("/orders?code=BUS-42&app_secret=url-secret"); + return row; + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysSystemConfigLogIsolationTest.java b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysSystemConfigLogIsolationTest.java new file mode 100644 index 000000000..5bfe22bf1 --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/java/com/bsd/pjcloud/admin/service/impl/SysSystemConfigLogIsolationTest.java @@ -0,0 +1,47 @@ +package com.bsd.pjcloud.admin.service.impl; + +import com.bsd.pjcloud.admin.api.entity.SysSystemConfigEntity; +import com.bsd.pjcloud.admin.mapper.SysSystemConfigMapper; +import com.bsd.pjcloud.common.core.util.AuditLogSanitizer; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +class SysSystemConfigLogIsolationTest { + @Test + void filteringTheAuditCopyDoesNotRemoveSecretsFromTheActualConfigurationSave() throws Exception { + var mapper = mock(SysSystemConfigMapper.class); + var service = new SysSystemConfigServiceImpl(mock(SysMessageServiceImpl.class)); + ReflectionTestUtils.setField(service, "baseMapper", mapper); + var stored = new SysSystemConfigEntity(); + stored.setId(1L); + stored.setConfigType("email"); + stored.setConfigValue("{\"appSecret\":\"old-app-secret\",\"clientSecret\":\"old-client-secret\",\"smtpServer\":\"old-host\"}"); + when(mapper.selectById(1L)).thenReturn(stored); + when(mapper.updateById(any(SysSystemConfigEntity.class))).thenReturn(1); + var update = new SysSystemConfigEntity(); + update.setId(1L); + update.setConfigType("email"); + update.setConfigValue("{\"appSecret\":\"new-app-secret\",\"clientSecret\":\"new-client-secret\",\"password\":\"smtp-secret\",\"smtpServer\":\"new-host\"}"); + String audit = new AuditLogSanitizer(List.of()).sanitizePayload(update, "/system-config"); + assertFalse(audit.contains("new-app-secret")); + assertFalse(audit.contains("new-client-secret")); + assertFalse(audit.contains("smtp-secret")); + assertTrue(service.updateSystemConfig(update).isOk()); + + var saved = ArgumentCaptor.forClass(SysSystemConfigEntity.class); + verify(mapper).updateById(saved.capture()); + var result = new ObjectMapper().readTree(saved.getValue().getConfigValue()); + assertEquals("new-app-secret", result.get("appSecret").textValue()); + assertEquals("new-client-secret", result.get("clientSecret").textValue()); + assertEquals("smtp-secret", result.get("password").textValue()); + assertEquals("new-host", result.get("smtpServer").textValue()); + } +} diff --git a/pjcloud-upms/pjcloud-upms-biz/src/test/resources/file-upload-vectors.json b/pjcloud-upms/pjcloud-upms-biz/src/test/resources/file-upload-vectors.json new file mode 100644 index 000000000..bcab05184 --- /dev/null +++ b/pjcloud-upms/pjcloud-upms-biz/src/test/resources/file-upload-vectors.json @@ -0,0 +1,32 @@ +[ + { + "target": "upload", + "appId": "admin", + "kid": "v1", + "ts": 1793232000, + "nonce": "00112233445566778899aabbccddeeff", + "key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "canonical": "v1\noss-file-upload\nPOST\nupload\nadmin\nv1\n1793232000\n00112233445566778899aabbccddeeff", + "sign": "75fde91ab548250139f260680aeb2e51f831d29d8e4fb3cfb28898528fd3a2a8" + }, + { + "target": "qn-upload", + "appId": "h5", + "kid": "v1", + "ts": 1793232000, + "nonce": "ffeeddccbbaa99887766554433221100", + "key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "canonical": "v1\noss-file-upload\nPOST\nqn-upload\nh5\nv1\n1793232000\nffeeddccbbaa99887766554433221100", + "sign": "01cef848210b1bc31f5fe7679428b36d8e0f15b9f5517472a023d4d2c008086c" + }, + { + "target": "inner-upload", + "appId": "service", + "kid": "v1", + "ts": 1793232000, + "nonce": "0123456789abcdef0123456789abcdef", + "key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "canonical": "v1\noss-file-upload\nPOST\ninner-upload\nservice\nv1\n1793232000\n0123456789abcdef0123456789abcdef", + "sign": "0d49bb992e5138cdb7a3f2bd7c83dfd60aedeae2b205898c4a6211c794f35aae" + } +] diff --git a/scripts/generate-file-access-config.py b/scripts/generate-file-access-config.py index 3ac30585d..7135d9f0f 100644 --- a/scripts/generate-file-access-config.py +++ b/scripts/generate-file-access-config.py @@ -57,6 +57,17 @@ def main(): 'legacy-dart': {'enabled': True, 'extensions': ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg', 'ico']}} for name, value in defaults.items(): policy.setdefault(name, value) + upload = policy.setdefault('upload', {}) + for name, value in { + 'ttl-seconds': 300, + 'max-file-bytes': 200 * 1024 * 1024, + 'allowed-extensions': ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'ico', 'svg', + 'pdf', 'apk', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', + 'txt', 'csv', 'zip', 'mp4', 'mov', 'webm', 'avi', 'mp3', + 'wav', 'm4a', 'aac', 'ogg', 'amr', 'flac'], + 'internal-extensions': ['log', 'gz'], + }.items(): + upload.setdefault(name, value) if args.mode: policy['mode'] = args.mode policy['public-base-url'] = args.public_base_url.rstrip('/') diff --git a/scripts/tests/test_generate_file_access_config.py b/scripts/tests/test_generate_file_access_config.py index 6de5d0f82..61b68fe8c 100644 --- a/scripts/tests/test_generate_file_access_config.py +++ b/scripts/tests/test_generate_file_access_config.py @@ -77,6 +77,26 @@ class SharedFileAccessConfigurationTest(unittest.TestCase): policy = json.loads((self.secret_dir / 'config.yml').read_text())['file-access'] self.assertEqual(policy['mode'], 'observe') self.assertTrue(policy['block-browser-navigation']) + self.assertEqual(policy['upload']['ttl-seconds'], 300) + self.assertEqual(policy['upload']['max-file-bytes'], 200 * 1024 * 1024) + self.assertTrue({'jpg', 'pdf', 'apk', 'docx', 'xlsx', 'zip', 'mp4', 'txt'}.issubset(policy['upload']['allowed-extensions'])) + self.assertEqual(policy['upload']['internal-extensions'], ['log', 'gz']) + + def test_existing_custom_upload_policy_is_preserved_without_changing_keys(self): + self.run_script() + registry = (self.secret_dir / 'keys.json').read_bytes() + path = self.secret_dir / 'config.yml' + config = json.loads(path.read_text()) + config['file-access']['upload'] = {'ttl-seconds': 120, 'max-file-bytes': 1024, + 'allowed-extensions': ['png', 'pdf']} + path.write_text(json.dumps(config)) + self.run_script() + policy = json.loads(path.read_text())['file-access']['upload'] + self.assertEqual(policy['ttl-seconds'], 120) + self.assertEqual(policy['max-file-bytes'], 1024) + self.assertEqual(policy['allowed-extensions'], ['png', 'pdf']) + self.assertEqual(policy['internal-extensions'], ['log', 'gz']) + self.assertEqual((self.secret_dir / 'keys.json').read_bytes(), registry) def test_invalid_existing_keys_are_never_silently_replaced(self): original = {app: 'not-a-valid-base64-key' for app in APPS} diff --git a/test-vectors/file-upload.json b/test-vectors/file-upload.json new file mode 100644 index 000000000..bcab05184 --- /dev/null +++ b/test-vectors/file-upload.json @@ -0,0 +1,32 @@ +[ + { + "target": "upload", + "appId": "admin", + "kid": "v1", + "ts": 1793232000, + "nonce": "00112233445566778899aabbccddeeff", + "key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "canonical": "v1\noss-file-upload\nPOST\nupload\nadmin\nv1\n1793232000\n00112233445566778899aabbccddeeff", + "sign": "75fde91ab548250139f260680aeb2e51f831d29d8e4fb3cfb28898528fd3a2a8" + }, + { + "target": "qn-upload", + "appId": "h5", + "kid": "v1", + "ts": 1793232000, + "nonce": "ffeeddccbbaa99887766554433221100", + "key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "canonical": "v1\noss-file-upload\nPOST\nqn-upload\nh5\nv1\n1793232000\nffeeddccbbaa99887766554433221100", + "sign": "01cef848210b1bc31f5fe7679428b36d8e0f15b9f5517472a023d4d2c008086c" + }, + { + "target": "inner-upload", + "appId": "service", + "kid": "v1", + "ts": 1793232000, + "nonce": "0123456789abcdef0123456789abcdef", + "key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "canonical": "v1\noss-file-upload\nPOST\ninner-upload\nservice\nv1\n1793232000\n0123456789abcdef0123456789abcdef", + "sign": "0d49bb992e5138cdb7a3f2bd7c83dfd60aedeae2b205898c4a6211c794f35aae" + } +]