import test, { mock } from 'node:test'; import assert from 'node:assert/strict'; import { createServer } from 'node:http'; import { once } from 'node:events'; import { aliases, baseUrl, bit, downloadUrl, encodePath, ensureObject, errorText, id, mimeType, normalizeConfig, objectKey, ossSettings, parseStorage, qualified, quote, readHttp, retry, sha256, verifyContent, } from '../lib/core.mjs'; const IMAGE = Buffer.from('89504e470d0a1a0a0001020304050607', 'hex'); const ITEM = { configId: '9007199254740993', path: '头像/张 三+100%.PNG', size: IMAGE.length, sha256: sha256(IMAGE), type: 'image/png', }; const HTTP_OPTIONS = { timeoutMs: 1000, maxFileBytes: 1024 }; const sourceBase = 'http://127.0.0.1:48080/admin-api/infra/file'; const baseConfig = overrides => ({ runId: 'unit-test', mysql: { database: 'unit_test', user: 'test-only' }, sourceConfigIds: ['1'], targetConfigId: '20', downloadBaseUrl: sourceBase, ...overrides, }); const storageRow = overrides => ({ storage: 20, config: { endpoint: 'oss-cn-hangzhou.aliyuncs.com', bucket: 'test-only-bucket', accessKey: 'test-only-access-key', accessSecret: 'test-only-access-secret', ...overrides, }, }); const noSuchKey = () => Object.assign(new Error('missing'), { code: 'NoSuchKey', status: 404 }); const head = bytes => ({ res: { headers: { 'content-length': String(bytes.length) } } }); const storedObject = bytes => ({ head: mock.fn(async () => head(bytes)), get: mock.fn(async () => ({ content: bytes })), put: mock.fn(async () => { throw new Error('unexpected write'); }), }); async function localServer(t, handler) { const server = createServer(handler); server.listen(0, '127.0.0.1'); await once(server, 'listening'); t.after(async () => { const closed = new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); server.closeAllConnections(); await closed; }); return 'http://127.0.0.1:' + server.address().port; } test('download URL preserves Chinese, space, plus, percent, question mark and fragment characters', () => { const path = '头像/张 三+100%?#.png'; const encoded = '%E5%A4%B4%E5%83%8F/%E5%BC%A0%20%E4%B8%89%2B100%25%3F%23.png'; assert.equal(encodePath(path), encoded); const url = downloadUrl(sourceBase + '///', ITEM.configId, path); assert.equal(url, sourceBase + '/' + ITEM.configId + '/get/' + encoded); assert.equal(decodeURIComponent(new URL(url).pathname.split('/get/')[1]), path); assert.equal(new URL(url).search, ''); assert.equal(new URL(url).hash, ''); }); test('encoded-looking path text is encoded exactly once without decoding the source path', () => { assert.equal(encodePath('%2F/%20+%.png'), '%252F/%2520%2B%25.png'); assert.equal(decodeURIComponent(encodePath('%2F/%20+%.png')), '%2F/%20+%.png'); }); test('ambiguous or normalized path segments are rejected', () => { for (const path of ['', '/image.png', 'a//b.png', 'a/', './a.png', 'a/../b.png', 'a/./b.png', 'a\\b.png']) { assert.throws(() => encodePath(path), /路径/, path); } }); test('URL base validates scheme and rejects credentials, queries and fragments', () => { assert.equal(baseUrl('https://images.example.test/prefix///'), 'https://images.example.test/prefix'); for (const url of ['ftp://example.test', 'file:///tmp/image', 'https://user:pass@example.test', 'https://example.test/?key=value', 'https://example.test/#part']) { assert.throws(() => baseUrl(url), /HTTP/); } }); test('bigint IDs remain exact strings and unsafe numeric IDs are rejected', () => { assert.equal(id('9007199254740993'), '9007199254740993'); assert.equal(id(9007199254740993n), '9007199254740993'); assert.equal(id(42), '42'); assert.throws(() => id(Number('9007199254740993')), /安全整数/); for (const value of [0, -1, 1.5, '01', '1e3', ' 1', '', null, undefined, true]) { assert.throws(() => id(value), /正整数|安全整数/, String(value)); } }); test('MySQL BIT values distinguish zero buffers from true values', () => { for (const value of [Buffer.from([0]), Buffer.alloc(0), false, 0, '0', null, undefined]) assert.equal(bit(value), false); for (const value of [Buffer.from([1]), true, 1, '1']) assert.equal(bit(value), true); }); test('configuration normalizes IDs and schemas without changing bigint precision', () => { const result = normalizeConfig(baseConfig({ sourceConfigIds: [1, '1', '9007199254740993'], referenceSchemas: ['chat', 'unit_test', 'chat'], oldDomains: ['https://old.example.test///'], })); assert.deepEqual(result.sourceConfigIds, ['1', '9007199254740993']); assert.deepEqual(result.referenceSchemas, ['unit_test', 'chat']); assert.deepEqual(result.oldDomains, ['https://old.example.test']); assert.equal(result.downloadBaseUrl, sourceBase); }); test('configuration rejects same source and target, invalid modes and unsafe batch limits', () => { for (const overrides of [ { targetConfigId: '1' }, { sourceMode: 'unknown' }, { targetUrlMode: 'unknown' }, { batchSize: 0 }, { batchSize: 1001 }, { retries: 0 }, { retries: 11 }, { maxFileBytes: 1.5 }, { timeoutMs: 0 }, { objectPrefix: '../unsafe' }, ]) assert.throws(() => normalizeConfig(baseConfig(overrides))); }); test('database source and direct target do not require a download address', () => { const result = normalizeConfig(baseConfig({ sourceMode: 'database', downloadBaseUrl: undefined })); assert.equal(result.sourceMode, 'database'); }); test('OSS settings use the actual Java configuration field names and support JSON storage', () => { const row = storageRow(); row.config = JSON.stringify(row.config); assert.equal(parseStorage(row).accessKey, 'test-only-access-key'); assert.deepEqual(ossSettings(row, {}), { endpoint: 'https://oss-cn-hangzhou.aliyuncs.com', bucket: 'test-only-bucket', accessKeyId: 'test-only-access-key', accessKeySecret: 'test-only-access-secret', publicDomain: 'https://test-only-bucket.oss-cn-hangzhou.aliyuncs.com', }); }); test('OSS internal endpoint gets a public fallback domain and explicit domain takes precedence', () => { const row = storageRow({ endpoint: 'https://oss-cn-hangzhou-internal.aliyuncs.com' }); const result = ossSettings(row, {}); assert.equal(result.endpoint, 'https://oss-cn-hangzhou-internal.aliyuncs.com'); assert.equal(result.publicDomain, 'https://test-only-bucket.oss-cn-hangzhou.aliyuncs.com'); assert.equal(ossSettings(storageRow({ domain: 'https://cdn.example.test/' }), {}).publicDomain, 'https://cdn.example.test'); assert.equal(ossSettings(row, { targetPublicDomain: 'https://override.example.test/assets/' }).publicDomain, 'https://override.example.test/assets'); }); test('OSS settings reject wrong storage type, non-OSS services, bucket endpoints and missing fields', () => { assert.throws(() => ossSettings({ ...storageRow(), storage: 1 }, {}), /storage=20/); for (const endpoint of [ 'http://127.0.0.1:9000', 's3.amazonaws.com', 'bucket.oss-cn-hangzhou.aliyuncs.com', 'oss-cn-hangzhou.aliyuncs.com.evil.test', 'https://oss-cn-hangzhou.aliyuncs.com/path', 'https://user:pass@oss-cn-hangzhou.aliyuncs.com', 'https://oss-cn-hangzhou.aliyuncs.com/?x=1', ]) assert.throws(() => ossSettings(storageRow({ endpoint }), {}), /endpoint/); for (const field of ['bucket', 'accessKey', 'accessSecret']) { assert.throws(() => ossSettings(storageRow({ [field]: '' }), {}), /缺少/); } }); test('object keys separate config, source path and content while keeping a safe extension', () => { const config = { objectPrefix: 'migration/test' }; const key = objectKey(config, ITEM); assert.equal(key, 'migration/test/' + ITEM.configId + '/' + sha256(ITEM.path) + '/' + ITEM.sha256 + '.png'); assert.notEqual(key, objectKey(config, { ...ITEM, configId: '2' })); assert.notEqual(key, objectKey(config, { ...ITEM, path: 'other.PNG' })); assert.notEqual(key, objectKey(config, { ...ITEM, sha256: 'b'.repeat(64) })); }); test('aliases retain unrelated stored URLs and only route variants resolving to the source path', () => { const row = { path: ITEM.path, url: 'https://special.example.test/persisted-image' }; const result = aliases(ITEM, [row, row], 'https://old.example.test', { oldDomains: ['https://older.example.test'], downloadBaseUrl: sourceBase, }); assert.equal(result.filter(value => value === row.url).length, 1); assert.ok(!result.includes('https://old.example.test/admin-api/infra/file/' + ITEM.configId + '/get/' + ITEM.path)); assert.ok(result.includes('https://older.example.test/admin-api/infra/file/' + ITEM.configId + '/get/' + encodePath(ITEM.path))); assert.ok(result.includes(downloadUrl(sourceBase, ITEM.configId, ITEM.path))); }); test('plus and space source files cannot claim each other through raw plus aliases', () => { const domain = 'https://old.example.test'; const prefix = domain + '/admin-api/infra/file/1/get/'; const config = { oldDomains: [], downloadBaseUrl: sourceBase }; const plus = aliases({ configId: '1', path: 'a+b.png' }, [ { path: 'a+b.png', url: prefix + 'a+b.png' }, { path: 'a+b.png', url: prefix + 'a%2Bb.png' }, ], domain, config); const space = aliases({ configId: '1', path: 'a b.png' }, [ { path: 'a b.png', url: prefix + 'a+b.png' }, ], domain, config); assert.ok(!plus.includes(prefix + 'a+b.png')); assert.ok(plus.includes(prefix + 'a%2Bb.png')); assert.ok(space.includes(prefix + 'a+b.png')); assert.ok(space.includes(prefix + 'a b.png')); assert.ok(space.includes(prefix + 'a%20b.png')); assert.ok(!space.includes(prefix + 'a%2Bb.png')); assert.ok(space.includes(sourceBase + '/1/get/a+b.png')); assert.deepEqual(plus.filter(url => space.includes(url)), []); }); test('percent-encoded-looking filenames keep their meaning after exactly one route decode', () => { const domain = 'https://old.example.test'; const prefix = domain + '/admin-api/infra/file/1/get/'; const config = { oldDomains: [] }; const literal = aliases({ configId: '1', path: 'a%2Fb.png' }, [ { path: 'a%2Fb.png', url: prefix + 'a%2Fb.png' }, ], domain, config); const directory = aliases({ configId: '1', path: 'a/b.png' }, [ { path: 'a/b.png', url: prefix + 'a%2Fb.png' }, ], domain, config); assert.ok(!literal.includes(prefix + 'a%2Fb.png')); assert.ok(literal.includes(prefix + 'a%252Fb.png')); assert.ok(directory.includes(prefix + 'a%2Fb.png')); assert.ok(directory.includes(prefix + 'a/b.png')); assert.deepEqual(literal.filter(url => directory.includes(url)), []); }); test('raw query, fragment and unverifiable percent paths are excluded from source aliases', () => { const domain = 'https://old.example.test'; const prefix = domain + '/admin-api/infra/file/1/get/'; for (const path of ['question?.png', 'fragment#.png', 'literal%.png', 'a%20b.png']) { const result = aliases({ configId: '1', path }, [{ path, url: prefix + path }], domain, { oldDomains: [] }); assert.ok(!result.includes(prefix + path), path); assert.ok(result.includes(prefix + encodePath(path)), path); } }); test('route validation accepts SQL-equivalent spellings and ignores genuine query parameters', () => { const domain = 'https://old.example.test'; const prefix = domain + '/admin-api/infra/file/1/get/'; const alternate = 'photo a.JPG'; const result = aliases({ configId: '1', path: 'Photo A.jpg' }, [ { path: alternate, url: prefix + 'photo+a.JPG?download=1#preview' }, { path: alternate, url: domain + '/admin-api/infra/file/2/get/photo+a.JPG' }, { path: alternate, url: prefix + 'unrelated.png' }, ], domain, { oldDomains: [] }); assert.ok(result.includes(prefix + 'photo+a.JPG?download=1#preview')); assert.ok(result.includes(prefix + 'photo+a.JPG')); assert.ok(result.includes(prefix + 'Photo+A.jpg')); assert.ok(!result.includes(domain + '/admin-api/infra/file/2/get/photo+a.JPG')); assert.ok(!result.includes(prefix + 'unrelated.png')); }); test('MIME detection prefers stored image type, then extension, then other stored type', () => { assert.equal(mimeType('renamed.jpg', [{ type: 'image/png' }]), 'image/png'); assert.equal(mimeType('image.WEBP', []), 'image/webp'); assert.equal(mimeType('no-extension', [{ type: 'application/pdf' }]), 'application/pdf'); assert.equal(mimeType('no-extension', []), 'application/octet-stream'); }); test('SQL identifiers are quoted and error URLs are redacted', () => { assert.equal(quote('strange\u0060name'), '\u0060strange\u0060\u0060name\u0060'); assert.equal(qualified('schema', 'table'), '\u0060schema\u0060.\u0060table\u0060'); const redacted = errorText(new Error('failed https://example.test/?secret=private afterward')); assert.equal(redacted, 'failed [URL] afterward'); assert.ok(errorText(new Error('x'.repeat(500))).length <= 300); }); test('HTTP download sends the intended encoded route and preserves binary bytes', async t => { let observedUrl; const base = await localServer(t, (request, response) => { observedUrl = request.url; response.writeHead(200, { 'Content-Type': 'image/png', 'Content-Length': IMAGE.length }); response.end(IMAGE); }); const route = downloadUrl(base + '/admin-api/infra/file', ITEM.configId, ITEM.path); const bytes = await readHttp(route, HTTP_OPTIONS, IMAGE.length); assert.deepEqual(bytes, IMAGE); assert.equal(observedUrl, '/admin-api/infra/file/' + ITEM.configId + '/get/' + encodePath(ITEM.path)); assert.doesNotThrow(() => verifyContent(bytes, ITEM)); }); test('HTTP errors and redirects are rejected rather than uploaded', async t => { const base = await localServer(t, (request, response) => { if (request.url === '/redirect') response.writeHead(302, { Location: '/image' }); else response.writeHead(404); response.end(); }); await assert.rejects(readHttp(base + '/missing', HTTP_OPTIONS), /HTTP 404/); await assert.rejects(readHttp(base + '/redirect', HTTP_OPTIONS)); }); test('HTTP 200 HTML or JSON with the same byte count fails hash verification', async t => { const base = await localServer(t, (request, response) => { const fake = Buffer.alloc(IMAGE.length, ' '); fake.write(request.url === '/html' ? 'error' : '{"error":true}'); response.writeHead(200, { 'Content-Type': request.url === '/html' ? 'text/html' : 'application/json' }); response.end(fake); }); for (const path of ['/html', '/json']) { const bytes = await readHttp(base + path, HTTP_OPTIONS, IMAGE.length); assert.equal(bytes.length, IMAGE.length); assert.throws(() => verifyContent(bytes, ITEM), /SHA-256/); } }); test('short downloads fail expected length validation', async t => { const base = await localServer(t, (_request, response) => response.end(IMAGE.subarray(0, 4))); const bytes = await readHttp(base, HTTP_OPTIONS, IMAGE.length); assert.throws(() => verifyContent(bytes, ITEM), /长度/); }); test('HTTP declared length exceeding either cap is rejected', async t => { const base = await localServer(t, (_request, response) => { response.writeHead(200, { 'Content-Length': IMAGE.length }); response.end(IMAGE); }); await assert.rejects(readHttp(base, { ...HTTP_OPTIONS, maxFileBytes: 4 }), /长度超过限制/); await assert.rejects(readHttp(base, HTTP_OPTIONS, 4), /长度超过限制/); }); test('chunked HTTP bodies cannot bypass actual byte limits', async t => { const base = await localServer(t, (_request, response) => { response.writeHead(200, { 'Transfer-Encoding': 'chunked' }); response.write(IMAGE.subarray(0, 8)); response.end(IMAGE.subarray(8)); }); await assert.rejects(readHttp(base, { ...HTTP_OPTIONS, maxFileBytes: 4 }), /长度超过限制/); await assert.rejects(readHttp(base, HTTP_OPTIONS, 4), /长度超过限制/); }); test('HTTP requests time out when the source never responds', async t => { const base = await localServer(t, () => {}); await assert.rejects(readHttp(base, { ...HTTP_OPTIONS, timeoutMs: 30 })); }); test('content verification checks both length and hash', () => { assert.doesNotThrow(() => verifyContent(IMAGE, { ...ITEM, size: String(IMAGE.length) })); assert.throws(() => verifyContent(IMAGE, { ...ITEM, size: IMAGE.length + 1 }), /长度/); assert.throws(() => verifyContent(IMAGE, { ...ITEM, sha256: '0'.repeat(64) }), /SHA-256/); }); test('existing identical OSS objects are read and verified without upload', async () => { const oss = storedObject(IMAGE); await ensureObject(oss, 'target/key.png', IMAGE, ITEM); assert.equal(oss.put.mock.callCount(), 0); assert.equal(oss.get.mock.callCount(), 1); assert.equal(oss.get.mock.calls[0].arguments[0], 'target/key.png'); }); test('existing OSS objects with a different length are not overwritten', async () => { const oss = storedObject(Buffer.from('wrong length')); await assert.rejects(ensureObject(oss, 'target/key.png', IMAGE, ITEM), /禁止覆盖/); assert.equal(oss.put.mock.callCount(), 0); assert.equal(oss.get.mock.callCount(), 0); }); test('existing same-length OSS objects with different bytes are not overwritten', async () => { const oss = storedObject(Buffer.alloc(IMAGE.length, 1)); await assert.rejects(ensureObject(oss, 'target/key.png', IMAGE, ITEM), /SHA-256/); assert.equal(oss.put.mock.callCount(), 0); }); test('missing OSS objects upload exact bytes with MIME and overwrite prevention', async () => { let saved; const oss = { head: mock.fn(async () => { if (!saved) throw noSuchKey(); return head(saved); }), put: mock.fn(async (_key, bytes) => { saved = Buffer.from(bytes); }), get: mock.fn(async () => ({ content: saved })), }; await ensureObject(oss, 'target/key.png', IMAGE, ITEM); assert.equal(oss.put.mock.callCount(), 1); assert.deepEqual(oss.put.mock.calls[0].arguments, [ 'target/key.png', IMAGE, { headers: { 'Content-Type': 'image/png', 'x-oss-forbid-overwrite': 'true' } }, ]); assert.deepEqual(saved, IMAGE); }); test('OSS HEAD authorization failures do not cause a speculative upload', async () => { const denied = Object.assign(new Error('denied'), { code: 'AccessDenied', status: 403 }); const oss = { head: mock.fn(async () => { throw denied; }), put: mock.fn(), get: mock.fn() }; await assert.rejects(ensureObject(oss, 'key', IMAGE, ITEM), error => error === denied); assert.equal(oss.put.mock.callCount(), 0); }); test('upload competition is accepted only after verifying the winner content', async () => { for (const sameContent of [true, false]) { const winner = sameContent ? IMAGE : Buffer.alloc(IMAGE.length, 2); let appeared = false; const oss = { head: mock.fn(async () => { if (!appeared) throw noSuchKey(); return head(winner); }), put: mock.fn(async () => { appeared = true; throw Object.assign(new Error('another upload won'), { code: 'FileAlreadyExists', status: 409 }); }), get: mock.fn(async () => ({ content: winner })), }; if (sameContent) await ensureObject(oss, 'key', IMAGE, ITEM); else await assert.rejects(ensureObject(oss, 'key', IMAGE, ITEM), /SHA-256/); assert.equal(oss.put.mock.callCount(), 1); assert.equal(oss.get.mock.callCount(), 1); } }); test('failed or corrupted OSS uploads never report success', async () => { const offline = Object.assign(new Error('offline'), { code: 'ConnectionTimeout' }); const failed = { head: mock.fn(async () => { throw noSuchKey(); }), put: mock.fn(async () => { throw offline; }), get: mock.fn(), }; await assert.rejects(ensureObject(failed, 'key', IMAGE, ITEM), error => error === offline); assert.equal(failed.get.mock.callCount(), 0); let uploaded = false; const corrupt = { head: mock.fn(async () => { if (!uploaded) throw noSuchKey(); return head(IMAGE); }), put: mock.fn(async () => { uploaded = true; }), get: mock.fn(async () => ({ content: Buffer.alloc(IMAGE.length) })), }; await assert.rejects(ensureObject(corrupt, 'key', IMAGE, ITEM), /SHA-256/); }); test('retry stops on success and uses capped exponential waits', async () => { let calls = 0; const waits = []; const value = await retry(async () => { if (++calls < 7) throw new Error('transient'); return 'done'; }, 7, async ms => { waits.push(ms); }); assert.equal(value, 'done'); assert.equal(calls, 7); assert.deepEqual(waits, [1000, 2000, 4000, 5000, 5000, 5000]); }); test('retry preserves the last failure and never exceeds the configured attempt count', async () => { let calls = 0; const waits = []; const failure = new Error('permanent'); await assert.rejects(retry(async () => { calls++; throw failure; }, 3, async ms => waits.push(ms)), error => error === failure); assert.equal(calls, 3); assert.deepEqual(waits, [1000, 2000]); const first = mock.fn(async () => 'first'); const wait = mock.fn(); assert.equal(await retry(first, 3, wait), 'first'); assert.equal(first.mock.callCount(), 1); assert.equal(wait.mock.callCount(), 0); });