import { id, qualified } from './core.mjs'; const compareIds = (left, right) => BigInt(left) < BigInt(right) ? -1 : BigInt(left) > BigInt(right) ? 1 : 0; function boundary(value) { if ((typeof value === 'number' && !Number.isSafeInteger(value)) || !/^\d+$/.test(String(value))) { throw new Error('文件清单上界必须是非负整数;大整数请使用字符串'); } return BigInt(value).toString(); } function pathText(value) { if (Buffer.isBuffer(value)) return value.toString('utf8'); if (typeof value !== 'string') throw new Error('文件路径不是有效的文本值'); return value; } function fileMetadata(row) { return { id: id(row.id), config_id: id(row.config_id), path: pathText(row.path), url: row.url, type: row.type, size: row.size }; } /** * Build a metadata-only index in three SELECTs. MySQL performs all path * grouping/comparisons with the columns' collation; JavaScript only associates * records by their latest numeric ID. No file bytes or digests are selected. */ export async function loadSourceIndex(db, config, { maxId, progress } = {}) { if (!config?.mysql?.database || !Array.isArray(config.sourceConfigIds) || !config.sourceConfigIds.length) { throw new Error('文件清单需要数据库名称和至少一个源配置编号'); } const sourceIds = [...new Set(config.sourceConfigIds.map(id))].sort(compareIds); const upperBound = maxId === undefined ? undefined : boundary(maxId); const contentTable = qualified(config.mysql.database, 'infra_file_content'); const fileTable = qualified(config.mysql.database, 'infra_file'); const markers = sourceIds.map(() => '?').join(','); // Keep the bound outside this grouping: a newer active row above the bound // must suppress the whole group, not make an older duplicate current again. const latest = `SELECT config_id,path,MAX(id) AS id FROM ${contentTable} WHERE deleted=0 AND config_id IN (${markers}) GROUP BY config_id,path`; const boundClause = upperBound === undefined ? '' : ' WHERE g.id<=?'; const boundParams = upperBound === undefined ? [] : [upperBound]; progress?.stage?.('读取数据库文件分组(只读取元数据,不计算图片哈希)'); const [latestRows] = await db.execute( `SELECT c.id AS content_id,c.config_id,c.path FROM ${contentTable} c JOIN (${latest}) g ON g.id=c.id${boundClause} ORDER BY c.id`, [...sourceIds, ...boundParams] ); const byContentId = new Map(); let observedMaximum = '0'; for (const row of latestRows) { const contentId = id(row.content_id); if (upperBound !== undefined && compareIds(contentId, upperBound) > 0) continue; const group = { contentId, configId: id(row.config_id), path: pathText(row.path), files: [], spellings: [] }; byContentId.set(contentId, group); if (compareIds(contentId, observedMaximum) > 0) observedMaximum = contentId; } progress?.stage?.('批量匹配文件记录并检查缺失内容'); const [fileRows] = await db.execute( `SELECT f.id,f.config_id,f.path,f.url,f.type,f.size,g.id AS content_id FROM ${fileTable} f LEFT JOIN (${latest}) g ON g.config_id=f.config_id AND g.path=f.path WHERE f.deleted=0 AND f.config_id IN (${markers}) ORDER BY f.id`, [...sourceIds, ...sourceIds] ); const missingFiles = []; for (const row of fileRows) { const file = fileMetadata(row); if (row.content_id === null || row.content_id === undefined) { missingFiles.push(file); continue; } // The join deliberately has no upper bound: a file whose latest content // lies beyond this batch is not a missing-content file. byContentId.get(id(row.content_id))?.files.push(file); } progress?.stage?.('批量收集原路径拼写,保留历史图片地址兼容性'); const [spellingRows] = await db.execute( `SELECT DISTINCT g.id AS content_id,BINARY c.path AS path FROM ${contentTable} c JOIN (${latest}) g ON g.config_id=c.config_id AND g.path=c.path WHERE c.deleted=0${upperBound === undefined ? '' : ' AND g.id<=?'} ORDER BY content_id`, [...sourceIds, ...boundParams] ); const spellingSets = new Map(); for (const row of spellingRows) { const contentId = id(row.content_id); if (!byContentId.has(contentId)) continue; if (!spellingSets.has(contentId)) spellingSets.set(contentId, new Set()); spellingSets.get(contentId).add(pathText(row.path)); } const groups = [...byContentId.values()].sort((left, right) => compareIds(left.contentId, right.contentId)); for (const group of groups) { group.files.sort((left, right) => compareIds(left.id, right.id)); group.spellings = [...(spellingSets.get(group.contentId) || [])] .sort((left, right) => Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8'))); } missingFiles.sort((left, right) => compareIds(left.id, right.id)); return { maxId: upperBound ?? observedMaximum, groups, missingFiles }; }