175 lines
9.5 KiB
JavaScript
175 lines
9.5 KiB
JavaScript
import { createHash } from 'node:crypto';
|
||
|
||
export const sha256 = value => createHash('sha256').update(value).digest('hex');
|
||
export const quote = name => '`' + String(name).replaceAll('`', '``') + '`';
|
||
export const qualified = (schema, table) => `${quote(schema)}.${quote(table)}`;
|
||
export const bit = value => Buffer.isBuffer(value) ? value.some(byte => byte !== 0) : value === true || value === 1 || value === '1';
|
||
export const errorText = error => String(error.code || error.message || error.name || '未知错误').replace(/https?:\/\/\S+/g, '[URL]').slice(0, 300);
|
||
|
||
export function id(value) {
|
||
if (typeof value === 'number' && !Number.isSafeInteger(value)) throw new Error('配置编号超出安全整数范围，请使用字符串');
|
||
if (!/^[1-9]\d*$/.test(String(value))) throw new Error('配置编号必须为正整数，不能使用存储类型代替配置编号');
|
||
return String(value);
|
||
}
|
||
|
||
export function baseUrl(value) {
|
||
const parsed = new URL(value);
|
||
if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || parsed.search || parsed.hash) {
|
||
throw new Error('访问基础地址必须是无账号、查询参数和片段的 HTTP(S) URL');
|
||
}
|
||
return parsed.href.replace(/\/+$/, '');
|
||
}
|
||
|
||
export function encodePath(path) {
|
||
if (!path || path.startsWith('/') || path.includes('\\') || path.split('/').some(part => ['.', '..', ''].includes(part))) {
|
||
throw new Error('文件路径为空或含不支持的路径段，需单独处理');
|
||
}
|
||
return path.split('/').map(encodeURIComponent).join('/');
|
||
}
|
||
|
||
export const downloadUrl = (base, configId, path) => `${baseUrl(base)}/${id(configId)}/get/${encodePath(path)}`;
|
||
|
||
export function normalizeConfig(input) {
|
||
const config = {
|
||
sourceMode: 'http', targetUrlMode: 'direct', objectPrefix: 'migration/db-images',
|
||
imagesOnly: true, batchSize: 100, timeoutMs: 60000, maxFileBytes: 64 * 1024 * 1024,
|
||
retries: 3, oldDomains: [], excludeReferenceTables: [], ...input
|
||
};
|
||
if (!/^[a-zA-Z0-9_-]{1,64}$/.test(config.runId || '')) throw new Error('runId 必须是 1～64 位字母、数字、横线或下划线');
|
||
if (!config.mysql?.database || !config.mysql?.user) throw new Error('请填写 mysql.database 和 mysql.user');
|
||
if (!Array.isArray(config.sourceConfigIds) || !config.sourceConfigIds.length) throw new Error('请填写 sourceConfigIds（原数据库存储配置编号）');
|
||
config.sourceConfigIds = [...new Set(config.sourceConfigIds.map(id))];
|
||
config.targetConfigId = id(config.targetConfigId);
|
||
if (config.sourceConfigIds.includes(config.targetConfigId)) throw new Error('源配置和 OSS 目标配置不能相同');
|
||
if (!['http', 'database'].includes(config.sourceMode)) throw new Error('sourceMode 只能为 http 或 database');
|
||
if (!['direct', 'backend'].includes(config.targetUrlMode)) throw new Error('targetUrlMode 只能为 direct 或 backend');
|
||
if (config.sourceMode === 'http' || config.targetUrlMode === 'backend') config.downloadBaseUrl = baseUrl(config.downloadBaseUrl);
|
||
if (!/^[a-zA-Z0-9][a-zA-Z0-9/_-]{0,119}$/.test(config.objectPrefix) || config.objectPrefix.endsWith('/')) throw new Error('objectPrefix 只能包含字母、数字、横线、下划线和目录分隔符');
|
||
for (const key of ['batchSize', 'timeoutMs', 'maxFileBytes', 'retries']) {
|
||
if (!Number.isSafeInteger(config[key]) || config[key] < 1) throw new Error(`${key} 必须为正整数`);
|
||
}
|
||
if (config.batchSize > 1000 || config.retries > 10) throw new Error('batchSize 上限为 1000，retries 上限为 10');
|
||
config.oldDomains = config.oldDomains.map(baseUrl);
|
||
config.referenceSchemas = [...new Set([config.mysql.database, ...(config.referenceSchemas || [])])];
|
||
return config;
|
||
}
|
||
|
||
export function parseStorage(row) {
|
||
return typeof row.config === 'string' ? JSON.parse(row.config) : row.config;
|
||
}
|
||
|
||
export function ossSettings(row, config) {
|
||
if (Number(row.storage) !== 20) throw new Error('目标配置不是 S3/OSS 存储（storage=20）');
|
||
const value = parseStorage(row);
|
||
const endpoint = new URL(/^https?:\/\//.test(value.endpoint) ? value.endpoint : `https://${value.endpoint}`);
|
||
if (!/^oss-[a-z0-9-]+\.aliyuncs\.com$/.test(endpoint.hostname) || endpoint.pathname !== '/' || endpoint.search || endpoint.hash || endpoint.username || endpoint.password) {
|
||
throw new Error('目标 endpoint 必须为阿里云 OSS 区域/加速节点，不能是其他 S3 服务或 Bucket 域名');
|
||
}
|
||
if (!value.bucket || !value.accessKey || !value.accessSecret) throw new Error('OSS 配置缺少 bucket/accessKey/accessSecret');
|
||
const publicDomain = baseUrl(config.targetPublicDomain || value.domain || `https://${value.bucket}.${endpoint.hostname.replace('-internal.', '.')}`);
|
||
return { endpoint: endpoint.origin, bucket: value.bucket, accessKeyId: value.accessKey, accessKeySecret: value.accessSecret, publicDomain };
|
||
}
|
||
|
||
export function objectKey(config, item) {
|
||
const extension = /\.([a-zA-Z0-9]{1,10})$/.exec(item.path)?.[1]?.toLowerCase();
|
||
return `${config.objectPrefix}/${item.configId}/${sha256(item.path)}/${item.sha256}${extension ? '.' + extension : ''}`;
|
||
}
|
||
|
||
export function aliases(item, rows, sourceDomain, config) {
|
||
const result = new Set(rows.map(row => row.url).filter(Boolean));
|
||
const domains = [...new Set([sourceDomain, ...config.oldDomains].filter(Boolean).map(baseUrl))];
|
||
const paths = [...new Set([item.path, ...rows.map(row => row.path)])];
|
||
const sourcePaths = new Set(paths);
|
||
const addPathAliases = (prefix, path) => {
|
||
const encoded = encodePath(path);
|
||
result.add(prefix + path);
|
||
result.add(prefix + encoded);
|
||
// FileController uses form-style decoding: a literal '+' means a space.
|
||
result.add(prefix + encoded.replaceAll('%20', '+'));
|
||
};
|
||
for (const domain of domains) for (const path of paths) {
|
||
const prefix = `${domain}/admin-api/infra/file/${item.configId}/get/`;
|
||
addPathAliases(prefix, path);
|
||
}
|
||
if (config.downloadBaseUrl) {
|
||
const prefix = baseUrl(config.downloadBaseUrl) + '/' + id(item.configId) + '/get/';
|
||
for (const path of paths) addPathAliases(prefix, path);
|
||
}
|
||
return [...result].filter(value => {
|
||
try {
|
||
// Parse the actual request path: raw '?' / '#' end the path, and the
|
||
// browser encodes spaces/Unicode before FileController sees requestURI.
|
||
const parsed = new URL(value, 'https://migration-alias.invalid');
|
||
const route = parsed.pathname.match(/\/infra\/file\/([^/]+)\/get(?:\/|$)/);
|
||
if (!route) return true; // Keep pre-existing URLs from other file routes.
|
||
if (BigInt(decodeURIComponent(route[1])) !== BigInt(item.configId)) return false;
|
||
const marker = parsed.pathname.indexOf('/get/');
|
||
if (marker < 0) return false;
|
||
const path = decodeURIComponent(parsed.pathname.slice(marker + 5).replaceAll('+', ' '));
|
||
return sourcePaths.has(path);
|
||
} catch {
|
||
// Unverifiable aliases remain untouched and are reported by the caller.
|
||
return false;
|
||
}
|
||
});
|
||
}
|
||
|
||
export function mimeType(path, records) {
|
||
const stored = records.map(row => row.type).find(type => /^image\//i.test(type || ''));
|
||
if (stored) return stored;
|
||
const ext = path.split('.').pop().toLowerCase();
|
||
const types = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', svg: 'image/svg+xml', bmp: 'image/bmp', ico: 'image/x-icon', tif: 'image/tiff', tiff: 'image/tiff', avif: 'image/avif', heic: 'image/heic', heif: 'image/heif' };
|
||
return types[ext] || records.find(row => row.type)?.type || 'application/octet-stream';
|
||
}
|
||
|
||
export async function readHttp(url, { timeoutMs, maxFileBytes }, expectedSize) {
|
||
const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs), redirect: 'error' });
|
||
if (!response.ok) { await response.body?.cancel(); throw new Error(`文件访问返回 HTTP ${response.status}`); }
|
||
const declared = Number(response.headers.get('content-length'));
|
||
if (declared > maxFileBytes || (expectedSize !== undefined && declared > expectedSize)) {
|
||
await response.body?.cancel(); throw new Error('HTTP 文件长度超过限制');
|
||
}
|
||
let size = 0;
|
||
const chunks = [];
|
||
for await (const chunk of response.body) {
|
||
size += chunk.length;
|
||
if (size > maxFileBytes || (expectedSize !== undefined && size > expectedSize)) throw new Error('HTTP 文件长度超过限制');
|
||
chunks.push(chunk);
|
||
}
|
||
return Buffer.concat(chunks);
|
||
}
|
||
|
||
export function verifyContent(bytes, expected) {
|
||
if (bytes.length !== Number(expected.size) || sha256(bytes) !== expected.sha256) throw new Error('文件长度或 SHA-256 不一致');
|
||
}
|
||
|
||
export async function ensureObject(oss, key, bytes, item) {
|
||
let exists = true;
|
||
try {
|
||
const result = await oss.head(key);
|
||
if (Number(result.res.headers['content-length']) !== Number(item.size)) throw new Error('OSS 同名对象长度不同，禁止覆盖');
|
||
} catch (error) {
|
||
if (error.code === 'NoSuchKey' || error.status === 404) exists = false;
|
||
else throw error;
|
||
}
|
||
if (!exists) {
|
||
try {
|
||
await oss.put(key, bytes, { headers: { 'Content-Type': item.type, 'x-oss-forbid-overwrite': 'true' } });
|
||
} catch (error) {
|
||
if (error.code !== 'FileAlreadyExists' && error.status !== 409) throw error;
|
||
}
|
||
}
|
||
const result = await oss.head(key);
|
||
if (Number(result.res.headers['content-length']) !== Number(item.size)) throw new Error('OSS 文件长度不一致');
|
||
verifyContent((await oss.get(key)).content, item);
|
||
}
|
||
|
||
export async function retry(action, attempts, wait = ms => new Promise(resolve => setTimeout(resolve, ms))) {
|
||
for (let i = 0; ; i++) {
|
||
try { return await action(); } catch (error) {
|
||
if (i + 1 >= attempts) throw error;
|
||
await wait(Math.min(1000 * 2 ** i, 5000));
|
||
}
|
||
}
|
||
}
|