Friendship-and-chat-api/script/migrate-db-files-to-oss/test/core.test.mjs

import test, { mock } from 'node:test';
import assert from 'node:assert/strict';
import { createServer } from 'node:http';
import { once } from 'node:events';
import {
  aliases, baseUrl, bit, downloadUrl, encodePath, ensureObject, errorText, id,
  mimeType, normalizeConfig, objectKey, ossSettings, parseStorage, qualified,
  quote, readHttp, retry, sha256, verifyContent,
} from '../lib/core.mjs';

const IMAGE = Buffer.from('89504e470d0a1a0a0001020304050607', 'hex');
const ITEM = {
  configId: '9007199254740993', path: '头像/张 三+100%.PNG',
  size: IMAGE.length, sha256: sha256(IMAGE), type: 'image/png',
};
const HTTP_OPTIONS = { timeoutMs: 1000, maxFileBytes: 1024 };
const sourceBase = 'http://127.0.0.1:48080/admin-api/infra/file';
const baseConfig = overrides => ({
  runId: 'unit-test', mysql: { database: 'unit_test', user: 'test-only' },
  sourceConfigIds: ['1'], targetConfigId: '20', downloadBaseUrl: sourceBase,
  ...overrides,
});
const storageRow = overrides => ({
  storage: 20,
  config: {
    endpoint: 'oss-cn-hangzhou.aliyuncs.com', bucket: 'test-only-bucket',
    accessKey: 'test-only-access-key', accessSecret: 'test-only-access-secret',
    ...overrides,
  },
});
const noSuchKey = () => Object.assign(new Error('missing'), { code: 'NoSuchKey', status: 404 });
const head = bytes => ({ res: { headers: { 'content-length': String(bytes.length) } } });
const storedObject = bytes => ({
  head: mock.fn(async () => head(bytes)),
  get: mock.fn(async () => ({ content: bytes })),
  put: mock.fn(async () => { throw new Error('unexpected write'); }),
});

async function localServer(t, handler) {
  const server = createServer(handler);
  server.listen(0, '127.0.0.1');
  await once(server, 'listening');
  t.after(async () => {
    const closed = new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
    server.closeAllConnections();
    await closed;
  });
  return 'http://127.0.0.1:' + server.address().port;
}

test('download URL preserves Chinese, space, plus, percent, question mark and fragment characters', () => {
  const path = '头像/张 三+100%?#.png';
  const encoded = '%E5%A4%B4%E5%83%8F/%E5%BC%A0%20%E4%B8%89%2B100%25%3F%23.png';
  assert.equal(encodePath(path), encoded);
  const url = downloadUrl(sourceBase + '///', ITEM.configId, path);
  assert.equal(url, sourceBase + '/' + ITEM.configId + '/get/' + encoded);
  assert.equal(decodeURIComponent(new URL(url).pathname.split('/get/')[1]), path);
  assert.equal(new URL(url).search, '');
  assert.equal(new URL(url).hash, '');
});

test('encoded-looking path text is encoded exactly once without decoding the source path', () => {
  assert.equal(encodePath('%2F/%20+%.png'), '%252F/%2520%2B%25.png');
  assert.equal(decodeURIComponent(encodePath('%2F/%20+%.png')), '%2F/%20+%.png');
});

test('ambiguous or normalized path segments are rejected', () => {
  for (const path of ['', '/image.png', 'a//b.png', 'a/', './a.png', 'a/../b.png', 'a/./b.png', 'a\\b.png']) {
    assert.throws(() => encodePath(path), /路径/, path);
  }
});

test('URL base validates scheme and rejects credentials, queries and fragments', () => {
  assert.equal(baseUrl('https://images.example.test/prefix///'), 'https://images.example.test/prefix');
  for (const url of ['ftp://example.test', 'file:///tmp/image', 'https://user:pass@example.test', 'https://example.test/?key=value', 'https://example.test/#part']) {
    assert.throws(() => baseUrl(url), /HTTP/);
  }
});

test('bigint IDs remain exact strings and unsafe numeric IDs are rejected', () => {
  assert.equal(id('9007199254740993'), '9007199254740993');
  assert.equal(id(9007199254740993n), '9007199254740993');
  assert.equal(id(42), '42');
  assert.throws(() => id(Number('9007199254740993')), /安全整数/);
  for (const value of [0, -1, 1.5, '01', '1e3', ' 1', '', null, undefined, true]) {
    assert.throws(() => id(value), /正整数|安全整数/, String(value));
  }
});

test('MySQL BIT values distinguish zero buffers from true values', () => {
  for (const value of [Buffer.from([0]), Buffer.alloc(0), false, 0, '0', null, undefined]) assert.equal(bit(value), false);
  for (const value of [Buffer.from([1]), true, 1, '1']) assert.equal(bit(value), true);
});

test('configuration normalizes IDs and schemas without changing bigint precision', () => {
  const result = normalizeConfig(baseConfig({
    sourceConfigIds: [1, '1', '9007199254740993'],
    referenceSchemas: ['chat', 'unit_test', 'chat'],
    oldDomains: ['https://old.example.test///'],
  }));
  assert.deepEqual(result.sourceConfigIds, ['1', '9007199254740993']);
  assert.deepEqual(result.referenceSchemas, ['unit_test', 'chat']);
  assert.deepEqual(result.oldDomains, ['https://old.example.test']);
  assert.equal(result.downloadBaseUrl, sourceBase);
});

test('configuration rejects same source and target, invalid modes and unsafe batch limits', () => {
  for (const overrides of [
    { targetConfigId: '1' }, { sourceMode: 'unknown' }, { targetUrlMode: 'unknown' },
    { batchSize: 0 }, { batchSize: 1001 }, { retries: 0 }, { retries: 11 },
    { maxFileBytes: 1.5 }, { timeoutMs: 0 }, { objectPrefix: '../unsafe' },
  ]) assert.throws(() => normalizeConfig(baseConfig(overrides)));
});

test('database source and direct target do not require a download address', () => {
  const result = normalizeConfig(baseConfig({ sourceMode: 'database', downloadBaseUrl: undefined }));
  assert.equal(result.sourceMode, 'database');
});

test('OSS settings use the actual Java configuration field names and support JSON storage', () => {
  const row = storageRow();
  row.config = JSON.stringify(row.config);
  assert.equal(parseStorage(row).accessKey, 'test-only-access-key');
  assert.deepEqual(ossSettings(row, {}), {
    endpoint: 'https://oss-cn-hangzhou.aliyuncs.com', bucket: 'test-only-bucket',
    accessKeyId: 'test-only-access-key', accessKeySecret: 'test-only-access-secret',
    publicDomain: 'https://test-only-bucket.oss-cn-hangzhou.aliyuncs.com',
  });
});

test('OSS internal endpoint gets a public fallback domain and explicit domain takes precedence', () => {
  const row = storageRow({ endpoint: 'https://oss-cn-hangzhou-internal.aliyuncs.com' });
  const result = ossSettings(row, {});
  assert.equal(result.endpoint, 'https://oss-cn-hangzhou-internal.aliyuncs.com');
  assert.equal(result.publicDomain, 'https://test-only-bucket.oss-cn-hangzhou.aliyuncs.com');
  assert.equal(ossSettings(storageRow({ domain: 'https://cdn.example.test/' }), {}).publicDomain, 'https://cdn.example.test');
  assert.equal(ossSettings(row, { targetPublicDomain: 'https://override.example.test/assets/' }).publicDomain, 'https://override.example.test/assets');
});

test('OSS settings reject wrong storage type, non-OSS services, bucket endpoints and missing fields', () => {
  assert.throws(() => ossSettings({ ...storageRow(), storage: 1 }, {}), /storage=20/);
  for (const endpoint of [
    'http://127.0.0.1:9000', 's3.amazonaws.com', 'bucket.oss-cn-hangzhou.aliyuncs.com',
    'oss-cn-hangzhou.aliyuncs.com.evil.test', 'https://oss-cn-hangzhou.aliyuncs.com/path',
    'https://user:pass@oss-cn-hangzhou.aliyuncs.com', 'https://oss-cn-hangzhou.aliyuncs.com/?x=1',
  ]) assert.throws(() => ossSettings(storageRow({ endpoint }), {}), /endpoint/);
  for (const field of ['bucket', 'accessKey', 'accessSecret']) {
    assert.throws(() => ossSettings(storageRow({ [field]: '' }), {}), /缺少/);
  }
});

test('object keys separate config, source path and content while keeping a safe extension', () => {
  const config = { objectPrefix: 'migration/test' };
  const key = objectKey(config, ITEM);
  assert.equal(key, 'migration/test/' + ITEM.configId + '/' + sha256(ITEM.path) + '/' + ITEM.sha256 + '.png');
  assert.notEqual(key, objectKey(config, { ...ITEM, configId: '2' }));
  assert.notEqual(key, objectKey(config, { ...ITEM, path: 'other.PNG' }));
  assert.notEqual(key, objectKey(config, { ...ITEM, sha256: 'b'.repeat(64) }));
});

test('aliases retain unrelated stored URLs and only route variants resolving to the source path', () => {
  const row = { path: ITEM.path, url: 'https://special.example.test/persisted-image' };
  const result = aliases(ITEM, [row, row], 'https://old.example.test', {
    oldDomains: ['https://older.example.test'], downloadBaseUrl: sourceBase,
  });
  assert.equal(result.filter(value => value === row.url).length, 1);
  assert.ok(!result.includes('https://old.example.test/admin-api/infra/file/' + ITEM.configId + '/get/' + ITEM.path));
  assert.ok(result.includes('https://older.example.test/admin-api/infra/file/' + ITEM.configId + '/get/' + encodePath(ITEM.path)));
  assert.ok(result.includes(downloadUrl(sourceBase, ITEM.configId, ITEM.path)));
});

test('plus and space source files cannot claim each other through raw plus aliases', () => {
  const domain = 'https://old.example.test';
  const prefix = domain + '/admin-api/infra/file/1/get/';
  const config = { oldDomains: [], downloadBaseUrl: sourceBase };
  const plus = aliases({ configId: '1', path: 'a+b.png' }, [
    { path: 'a+b.png', url: prefix + 'a+b.png' },
    { path: 'a+b.png', url: prefix + 'a%2Bb.png' },
  ], domain, config);
  const space = aliases({ configId: '1', path: 'a b.png' }, [
    { path: 'a b.png', url: prefix + 'a+b.png' },
  ], domain, config);
  assert.ok(!plus.includes(prefix + 'a+b.png'));
  assert.ok(plus.includes(prefix + 'a%2Bb.png'));
  assert.ok(space.includes(prefix + 'a+b.png'));
  assert.ok(space.includes(prefix + 'a b.png'));
  assert.ok(space.includes(prefix + 'a%20b.png'));
  assert.ok(!space.includes(prefix + 'a%2Bb.png'));
  assert.ok(space.includes(sourceBase + '/1/get/a+b.png'));
  assert.deepEqual(plus.filter(url => space.includes(url)), []);
});

test('percent-encoded-looking filenames keep their meaning after exactly one route decode', () => {
  const domain = 'https://old.example.test';
  const prefix = domain + '/admin-api/infra/file/1/get/';
  const config = { oldDomains: [] };
  const literal = aliases({ configId: '1', path: 'a%2Fb.png' }, [
    { path: 'a%2Fb.png', url: prefix + 'a%2Fb.png' },
  ], domain, config);
  const directory = aliases({ configId: '1', path: 'a/b.png' }, [
    { path: 'a/b.png', url: prefix + 'a%2Fb.png' },
  ], domain, config);
  assert.ok(!literal.includes(prefix + 'a%2Fb.png'));
  assert.ok(literal.includes(prefix + 'a%252Fb.png'));
  assert.ok(directory.includes(prefix + 'a%2Fb.png'));
  assert.ok(directory.includes(prefix + 'a/b.png'));
  assert.deepEqual(literal.filter(url => directory.includes(url)), []);
});

test('raw query, fragment and unverifiable percent paths are excluded from source aliases', () => {
  const domain = 'https://old.example.test';
  const prefix = domain + '/admin-api/infra/file/1/get/';
  for (const path of ['question?.png', 'fragment#.png', 'literal%.png', 'a%20b.png']) {
    const result = aliases({ configId: '1', path }, [{ path, url: prefix + path }], domain, { oldDomains: [] });
    assert.ok(!result.includes(prefix + path), path);
    assert.ok(result.includes(prefix + encodePath(path)), path);
  }
});

test('route validation accepts SQL-equivalent spellings and ignores genuine query parameters', () => {
  const domain = 'https://old.example.test';
  const prefix = domain + '/admin-api/infra/file/1/get/';
  const alternate = 'photo a.JPG';
  const result = aliases({ configId: '1', path: 'Photo A.jpg' }, [
    { path: alternate, url: prefix + 'photo+a.JPG?download=1#preview' },
    { path: alternate, url: domain + '/admin-api/infra/file/2/get/photo+a.JPG' },
    { path: alternate, url: prefix + 'unrelated.png' },
  ], domain, { oldDomains: [] });
  assert.ok(result.includes(prefix + 'photo+a.JPG?download=1#preview'));
  assert.ok(result.includes(prefix + 'photo+a.JPG'));
  assert.ok(result.includes(prefix + 'Photo+A.jpg'));
  assert.ok(!result.includes(domain + '/admin-api/infra/file/2/get/photo+a.JPG'));
  assert.ok(!result.includes(prefix + 'unrelated.png'));
});

test('MIME detection prefers stored image type, then extension, then other stored type', () => {
  assert.equal(mimeType('renamed.jpg', [{ type: 'image/png' }]), 'image/png');
  assert.equal(mimeType('image.WEBP', []), 'image/webp');
  assert.equal(mimeType('no-extension', [{ type: 'application/pdf' }]), 'application/pdf');
  assert.equal(mimeType('no-extension', []), 'application/octet-stream');
});

test('SQL identifiers are quoted and error URLs are redacted', () => {
  assert.equal(quote('strange\u0060name'), '\u0060strange\u0060\u0060name\u0060');
  assert.equal(qualified('schema', 'table'), '\u0060schema\u0060.\u0060table\u0060');
  const redacted = errorText(new Error('failed https://example.test/?secret=private afterward'));
  assert.equal(redacted, 'failed [URL] afterward');
  assert.ok(errorText(new Error('x'.repeat(500))).length <= 300);
});

test('HTTP download sends the intended encoded route and preserves binary bytes', async t => {
  let observedUrl;
  const base = await localServer(t, (request, response) => {
    observedUrl = request.url;
    response.writeHead(200, { 'Content-Type': 'image/png', 'Content-Length': IMAGE.length });
    response.end(IMAGE);
  });
  const route = downloadUrl(base + '/admin-api/infra/file', ITEM.configId, ITEM.path);
  const bytes = await readHttp(route, HTTP_OPTIONS, IMAGE.length);
  assert.deepEqual(bytes, IMAGE);
  assert.equal(observedUrl, '/admin-api/infra/file/' + ITEM.configId + '/get/' + encodePath(ITEM.path));
  assert.doesNotThrow(() => verifyContent(bytes, ITEM));
});

test('HTTP errors and redirects are rejected rather than uploaded', async t => {
  const base = await localServer(t, (request, response) => {
    if (request.url === '/redirect') response.writeHead(302, { Location: '/image' });
    else response.writeHead(404);
    response.end();
  });
  await assert.rejects(readHttp(base + '/missing', HTTP_OPTIONS), /HTTP 404/);
  await assert.rejects(readHttp(base + '/redirect', HTTP_OPTIONS));
});

test('HTTP 200 HTML or JSON with the same byte count fails hash verification', async t => {
  const base = await localServer(t, (request, response) => {
    const fake = Buffer.alloc(IMAGE.length, ' ');
    fake.write(request.url === '/html' ? '<html>error' : '{"error":true}');
    response.writeHead(200, { 'Content-Type': request.url === '/html' ? 'text/html' : 'application/json' });
    response.end(fake);
  });
  for (const path of ['/html', '/json']) {
    const bytes = await readHttp(base + path, HTTP_OPTIONS, IMAGE.length);
    assert.equal(bytes.length, IMAGE.length);
    assert.throws(() => verifyContent(bytes, ITEM), /SHA-256/);
  }
});

test('short downloads fail expected length validation', async t => {
  const base = await localServer(t, (_request, response) => response.end(IMAGE.subarray(0, 4)));
  const bytes = await readHttp(base, HTTP_OPTIONS, IMAGE.length);
  assert.throws(() => verifyContent(bytes, ITEM), /长度/);
});

test('HTTP declared length exceeding either cap is rejected', async t => {
  const base = await localServer(t, (_request, response) => {
    response.writeHead(200, { 'Content-Length': IMAGE.length });
    response.end(IMAGE);
  });
  await assert.rejects(readHttp(base, { ...HTTP_OPTIONS, maxFileBytes: 4 }), /长度超过限制/);
  await assert.rejects(readHttp(base, HTTP_OPTIONS, 4), /长度超过限制/);
});

test('chunked HTTP bodies cannot bypass actual byte limits', async t => {
  const base = await localServer(t, (_request, response) => {
    response.writeHead(200, { 'Transfer-Encoding': 'chunked' });
    response.write(IMAGE.subarray(0, 8));
    response.end(IMAGE.subarray(8));
  });
  await assert.rejects(readHttp(base, { ...HTTP_OPTIONS, maxFileBytes: 4 }), /长度超过限制/);
  await assert.rejects(readHttp(base, HTTP_OPTIONS, 4), /长度超过限制/);
});

test('HTTP requests time out when the source never responds', async t => {
  const base = await localServer(t, () => {});
  await assert.rejects(readHttp(base, { ...HTTP_OPTIONS, timeoutMs: 30 }));
});

test('content verification checks both length and hash', () => {
  assert.doesNotThrow(() => verifyContent(IMAGE, { ...ITEM, size: String(IMAGE.length) }));
  assert.throws(() => verifyContent(IMAGE, { ...ITEM, size: IMAGE.length + 1 }), /长度/);
  assert.throws(() => verifyContent(IMAGE, { ...ITEM, sha256: '0'.repeat(64) }), /SHA-256/);
});

test('existing identical OSS objects are read and verified without upload', async () => {
  const oss = storedObject(IMAGE);
  await ensureObject(oss, 'target/key.png', IMAGE, ITEM);
  assert.equal(oss.put.mock.callCount(), 0);
  assert.equal(oss.get.mock.callCount(), 1);
  assert.equal(oss.get.mock.calls[0].arguments[0], 'target/key.png');
});

test('existing OSS objects with a different length are not overwritten', async () => {
  const oss = storedObject(Buffer.from('wrong length'));
  await assert.rejects(ensureObject(oss, 'target/key.png', IMAGE, ITEM), /禁止覆盖/);
  assert.equal(oss.put.mock.callCount(), 0);
  assert.equal(oss.get.mock.callCount(), 0);
});

test('existing same-length OSS objects with different bytes are not overwritten', async () => {
  const oss = storedObject(Buffer.alloc(IMAGE.length, 1));
  await assert.rejects(ensureObject(oss, 'target/key.png', IMAGE, ITEM), /SHA-256/);
  assert.equal(oss.put.mock.callCount(), 0);
});

test('missing OSS objects upload exact bytes with MIME and overwrite prevention', async () => {
  let saved;
  const oss = {
    head: mock.fn(async () => { if (!saved) throw noSuchKey(); return head(saved); }),
    put: mock.fn(async (_key, bytes) => { saved = Buffer.from(bytes); }),
    get: mock.fn(async () => ({ content: saved })),
  };
  await ensureObject(oss, 'target/key.png', IMAGE, ITEM);
  assert.equal(oss.put.mock.callCount(), 1);
  assert.deepEqual(oss.put.mock.calls[0].arguments, [
    'target/key.png', IMAGE, { headers: { 'Content-Type': 'image/png', 'x-oss-forbid-overwrite': 'true' } },
  ]);
  assert.deepEqual(saved, IMAGE);
});

test('OSS HEAD authorization failures do not cause a speculative upload', async () => {
  const denied = Object.assign(new Error('denied'), { code: 'AccessDenied', status: 403 });
  const oss = { head: mock.fn(async () => { throw denied; }), put: mock.fn(), get: mock.fn() };
  await assert.rejects(ensureObject(oss, 'key', IMAGE, ITEM), error => error === denied);
  assert.equal(oss.put.mock.callCount(), 0);
});

test('upload competition is accepted only after verifying the winner content', async () => {
  for (const sameContent of [true, false]) {
    const winner = sameContent ? IMAGE : Buffer.alloc(IMAGE.length, 2);
    let appeared = false;
    const oss = {
      head: mock.fn(async () => { if (!appeared) throw noSuchKey(); return head(winner); }),
      put: mock.fn(async () => {
        appeared = true;
        throw Object.assign(new Error('another upload won'), { code: 'FileAlreadyExists', status: 409 });
      }),
      get: mock.fn(async () => ({ content: winner })),
    };
    if (sameContent) await ensureObject(oss, 'key', IMAGE, ITEM);
    else await assert.rejects(ensureObject(oss, 'key', IMAGE, ITEM), /SHA-256/);
    assert.equal(oss.put.mock.callCount(), 1);
    assert.equal(oss.get.mock.callCount(), 1);
  }
});

test('failed or corrupted OSS uploads never report success', async () => {
  const offline = Object.assign(new Error('offline'), { code: 'ConnectionTimeout' });
  const failed = {
    head: mock.fn(async () => { throw noSuchKey(); }),
    put: mock.fn(async () => { throw offline; }), get: mock.fn(),
  };
  await assert.rejects(ensureObject(failed, 'key', IMAGE, ITEM), error => error === offline);
  assert.equal(failed.get.mock.callCount(), 0);
  let uploaded = false;
  const corrupt = {
    head: mock.fn(async () => { if (!uploaded) throw noSuchKey(); return head(IMAGE); }),
    put: mock.fn(async () => { uploaded = true; }),
    get: mock.fn(async () => ({ content: Buffer.alloc(IMAGE.length) })),
  };
  await assert.rejects(ensureObject(corrupt, 'key', IMAGE, ITEM), /SHA-256/);
});

test('retry stops on success and uses capped exponential waits', async () => {
  let calls = 0;
  const waits = [];
  const value = await retry(async () => {
    if (++calls < 7) throw new Error('transient');
    return 'done';
  }, 7, async ms => { waits.push(ms); });
  assert.equal(value, 'done');
  assert.equal(calls, 7);
  assert.deepEqual(waits, [1000, 2000, 4000, 5000, 5000, 5000]);
});

test('retry preserves the last failure and never exceeds the configured attempt count', async () => {
  let calls = 0;
  const waits = [];
  const failure = new Error('permanent');
  await assert.rejects(retry(async () => { calls++; throw failure; }, 3, async ms => waits.push(ms)), error => error === failure);
  assert.equal(calls, 3);
  assert.deepEqual(waits, [1000, 2000]);
  const first = mock.fn(async () => 'first');
  const wait = mock.fn();
  assert.equal(await retry(first, 3, wait), 'first');
  assert.equal(first.mock.callCount(), 1);
  assert.equal(wait.mock.callCount(), 0);
});