108 lines
4.9 KiB
JavaScript
108 lines
4.9 KiB
JavaScript
import { id, qualified } from './core.mjs';
|
|
|
|
const compareIds = (left, right) => BigInt(left) < BigInt(right) ? -1 : BigInt(left) > BigInt(right) ? 1 : 0;
|
|
|
|
function boundary(value) {
|
|
if ((typeof value === 'number' && !Number.isSafeInteger(value)) || !/^\d+$/.test(String(value))) {
|
|
throw new Error('文件清单上界必须是非负整数；大整数请使用字符串');
|
|
}
|
|
return BigInt(value).toString();
|
|
}
|
|
|
|
function pathText(value) {
|
|
if (Buffer.isBuffer(value)) return value.toString('utf8');
|
|
if (typeof value !== 'string') throw new Error('文件路径不是有效的文本值');
|
|
return value;
|
|
}
|
|
|
|
function fileMetadata(row) {
|
|
return {
|
|
id: id(row.id), config_id: id(row.config_id), path: pathText(row.path),
|
|
url: row.url, type: row.type, size: row.size
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Build a metadata-only index in three SELECTs. MySQL performs all path
|
|
* grouping/comparisons with the columns' collation; JavaScript only associates
|
|
* records by their latest numeric ID. No file bytes or digests are selected.
|
|
*/
|
|
export async function loadSourceIndex(db, config, { maxId, progress } = {}) {
|
|
if (!config?.mysql?.database || !Array.isArray(config.sourceConfigIds) || !config.sourceConfigIds.length) {
|
|
throw new Error('文件清单需要数据库名称和至少一个源配置编号');
|
|
}
|
|
const sourceIds = [...new Set(config.sourceConfigIds.map(id))].sort(compareIds);
|
|
const upperBound = maxId === undefined ? undefined : boundary(maxId);
|
|
const contentTable = qualified(config.mysql.database, 'infra_file_content');
|
|
const fileTable = qualified(config.mysql.database, 'infra_file');
|
|
const markers = sourceIds.map(() => '?').join(',');
|
|
// Keep the bound outside this grouping: a newer active row above the bound
|
|
// must suppress the whole group, not make an older duplicate current again.
|
|
const latest = `SELECT config_id,path,MAX(id) AS id FROM ${contentTable}
|
|
WHERE deleted=0 AND config_id IN (${markers}) GROUP BY config_id,path`;
|
|
const boundClause = upperBound === undefined ? '' : ' WHERE g.id<=?';
|
|
const boundParams = upperBound === undefined ? [] : [upperBound];
|
|
|
|
progress?.stage?.('读取数据库文件分组（只读取元数据，不计算图片哈希）');
|
|
const [latestRows] = await db.execute(
|
|
`SELECT c.id AS content_id,c.config_id,c.path FROM ${contentTable} c
|
|
JOIN (${latest}) g ON g.id=c.id${boundClause} ORDER BY c.id`,
|
|
[...sourceIds, ...boundParams]
|
|
);
|
|
const byContentId = new Map();
|
|
let observedMaximum = '0';
|
|
for (const row of latestRows) {
|
|
const contentId = id(row.content_id);
|
|
if (upperBound !== undefined && compareIds(contentId, upperBound) > 0) continue;
|
|
const group = {
|
|
contentId, configId: id(row.config_id), path: pathText(row.path), files: [], spellings: []
|
|
};
|
|
byContentId.set(contentId, group);
|
|
if (compareIds(contentId, observedMaximum) > 0) observedMaximum = contentId;
|
|
}
|
|
|
|
progress?.stage?.('批量匹配文件记录并检查缺失内容');
|
|
const [fileRows] = await db.execute(
|
|
`SELECT f.id,f.config_id,f.path,f.url,f.type,f.size,g.id AS content_id
|
|
FROM ${fileTable} f LEFT JOIN (${latest}) g
|
|
ON g.config_id=f.config_id AND g.path=f.path
|
|
WHERE f.deleted=0 AND f.config_id IN (${markers}) ORDER BY f.id`,
|
|
[...sourceIds, ...sourceIds]
|
|
);
|
|
const missingFiles = [];
|
|
for (const row of fileRows) {
|
|
const file = fileMetadata(row);
|
|
if (row.content_id === null || row.content_id === undefined) {
|
|
missingFiles.push(file);
|
|
continue;
|
|
}
|
|
// The join deliberately has no upper bound: a file whose latest content
|
|
// lies beyond this batch is not a missing-content file.
|
|
byContentId.get(id(row.content_id))?.files.push(file);
|
|
}
|
|
|
|
progress?.stage?.('批量收集原路径拼写，保留历史图片地址兼容性');
|
|
const [spellingRows] = await db.execute(
|
|
`SELECT DISTINCT g.id AS content_id,BINARY c.path AS path
|
|
FROM ${contentTable} c JOIN (${latest}) g
|
|
ON g.config_id=c.config_id AND g.path=c.path
|
|
WHERE c.deleted=0${upperBound === undefined ? '' : ' AND g.id<=?'} ORDER BY content_id`,
|
|
[...sourceIds, ...boundParams]
|
|
);
|
|
const spellingSets = new Map();
|
|
for (const row of spellingRows) {
|
|
const contentId = id(row.content_id);
|
|
if (!byContentId.has(contentId)) continue;
|
|
if (!spellingSets.has(contentId)) spellingSets.set(contentId, new Set());
|
|
spellingSets.get(contentId).add(pathText(row.path));
|
|
}
|
|
const groups = [...byContentId.values()].sort((left, right) => compareIds(left.contentId, right.contentId));
|
|
for (const group of groups) {
|
|
group.files.sort((left, right) => compareIds(left.id, right.id));
|
|
group.spellings = [...(spellingSets.get(group.contentId) || [])]
|
|
.sort((left, right) => Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')));
|
|
}
|
|
missingFiles.sort((left, right) => compareIds(left.id, right.id));
|
|
return { maxId: upperBound ?? observedMaximum, groups, missingFiles };
|
|
}
|