273 lines
7.9 KiB
JavaScript
273 lines
7.9 KiB
JavaScript
/**
|
||
* scripts/grid-db/sync-to-drive.js
|
||
*
|
||
* 功能：将 grid-db/ 中的关键文件同步到 Google Drive 镜像目录
|
||
*
|
||
* 同步范围：
|
||
* grid-db/memory/DEV-XXX/*.json → Drive/mirror/memory/DEV-XXX/
|
||
* grid-db/outbox/latest/*.json → Drive/mirror/outbox/
|
||
* grid-db/rules/*.json → Drive/mirror/rules/
|
||
* grid-db/drive-index/*.json → Drive/mirror/（每个 DEV 的 index.json）
|
||
*
|
||
* 同步策略：
|
||
* - 比较文件内容 MD5 hash，仅同步有变更的文件（节省 API 配额）
|
||
* - 文件不存在则创建，已存在则覆盖更新
|
||
* - Drive 中多出的文件不删除（防误删）
|
||
*
|
||
* 注意：
|
||
* - 不同步 interactions/ 和 training-lake/（数据量大 + 无需 Gemini 读取）
|
||
* - 不同步 inbox/ 和 processing/（系统内部流转区）
|
||
*
|
||
* 环境变量：
|
||
* - GDRIVE_CLIENT_ID: OAuth 客户端 ID
|
||
* - GDRIVE_CLIENT_SECRET: OAuth 客户端密钥
|
||
* - GDRIVE_REFRESH_TOKEN: 长效刷新令牌
|
||
* - DRIVE_FOLDER_ID: Drive 镜像根文件夹 ID
|
||
*
|
||
* 守护: PER-ZY001 铸渊
|
||
* 系统: SYS-GLW-0001
|
||
*/
|
||
|
||
const { google } = require('googleapis');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const crypto = require('crypto');
|
||
|
||
const GRID_DB_ROOT = path.join(__dirname, '../../grid-db');
|
||
|
||
// 同步目录映射：本地相对路径 → Drive 镜像子目录
|
||
const SYNC_DIRS = [
|
||
{ local: 'memory', drivePrefix: 'mirror/memory', recursive: true },
|
||
{ local: 'outbox/latest', drivePrefix: 'mirror/outbox', recursive: false },
|
||
{ local: 'rules', drivePrefix: 'mirror/rules', recursive: false },
|
||
{ local: 'drive-index', drivePrefix: 'mirror', recursive: false }
|
||
];
|
||
|
||
/**
|
||
* 计算文件内容的 MD5 哈希
|
||
*/
|
||
function md5(content) {
|
||
return crypto.createHash('md5').update(content).digest('hex');
|
||
}
|
||
|
||
/**
|
||
* 递归收集目录下的所有 JSON 文件
|
||
*/
|
||
function collectFiles(dirPath, recursive) {
|
||
const results = [];
|
||
if (!fs.existsSync(dirPath)) return results;
|
||
|
||
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
||
for (const entry of entries) {
|
||
const fullPath = path.join(dirPath, entry.name);
|
||
if (entry.isDirectory() && recursive) {
|
||
results.push(...collectFiles(fullPath, true));
|
||
} else if (entry.isFile() && entry.name.endsWith('.json')) {
|
||
results.push(fullPath);
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/**
|
||
* 转义 Drive API 查询中的单引号
|
||
*/
|
||
function escapeQuery(str) {
|
||
return str.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
|
||
}
|
||
|
||
/**
|
||
* 在 Drive 中查找或创建文件夹（按路径逐级创建）
|
||
*/
|
||
async function getOrCreateDriveFolder(drive, parentId, folderPath) {
|
||
const parts = folderPath.split('/').filter(Boolean);
|
||
let currentParentId = parentId;
|
||
|
||
for (const part of parts) {
|
||
const safePart = escapeQuery(part);
|
||
const query = `name='${safePart}' and '${currentParentId}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false`;
|
||
const res = await drive.files.list({
|
||
q: query,
|
||
fields: 'files(id, name)',
|
||
pageSize: 1
|
||
});
|
||
|
||
if (res.data.files && res.data.files.length > 0) {
|
||
currentParentId = res.data.files[0].id;
|
||
} else {
|
||
const createRes = await drive.files.create({
|
||
requestBody: {
|
||
name: part,
|
||
mimeType: 'application/vnd.google-apps.folder',
|
||
parents: [currentParentId]
|
||
},
|
||
fields: 'id'
|
||
});
|
||
currentParentId = createRes.data.id;
|
||
console.log(`[sync-to-drive] Created folder: ${part}`);
|
||
}
|
||
}
|
||
|
||
return currentParentId;
|
||
}
|
||
|
||
/**
|
||
* 在 Drive 目标文件夹中查找同名文件
|
||
*/
|
||
async function findFileInDrive(drive, folderId, fileName) {
|
||
const safeFileName = escapeQuery(fileName);
|
||
const query = `name='${safeFileName}' and '${folderId}' in parents and trashed=false`;
|
||
const res = await drive.files.list({
|
||
q: query,
|
||
fields: 'files(id, name, md5Checksum)',
|
||
pageSize: 1
|
||
});
|
||
|
||
if (res.data.files && res.data.files.length > 0) {
|
||
return res.data.files[0];
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 上传或更新单个文件到 Drive
|
||
*/
|
||
async function uploadFile(drive, folderId, fileName, content) {
|
||
const existingFile = await findFileInDrive(drive, folderId, fileName);
|
||
const { Readable } = require('stream');
|
||
|
||
const media = {
|
||
mimeType: 'application/json',
|
||
body: Readable.from([content])
|
||
};
|
||
|
||
if (existingFile) {
|
||
// 检查内容是否变更（Drive 对上传文件自动计算 MD5）
|
||
const localHash = md5(content);
|
||
if (existingFile.md5Checksum === localHash) {
|
||
return { action: 'skipped', fileName };
|
||
}
|
||
|
||
await drive.files.update({
|
||
fileId: existingFile.id,
|
||
media
|
||
});
|
||
return { action: 'updated', fileName };
|
||
} else {
|
||
await drive.files.create({
|
||
requestBody: {
|
||
name: fileName,
|
||
parents: [folderId]
|
||
},
|
||
media,
|
||
fields: 'id'
|
||
});
|
||
return { action: 'created', fileName };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 主同步流程
|
||
*/
|
||
async function main() {
|
||
// 验证环境变量
|
||
const rootFolderId = process.env.DRIVE_FOLDER_ID;
|
||
|
||
if (!rootFolderId) {
|
||
console.log('[sync-to-drive] DRIVE_FOLDER_ID not set, skipping');
|
||
return;
|
||
}
|
||
|
||
// OAuth2 认证（统一入口）
|
||
let drive;
|
||
try {
|
||
const { getDriveClient } = require('./drive-auth');
|
||
drive = getDriveClient();
|
||
console.log('[sync-to-drive] ✅ OAuth2 credentials configured');
|
||
} catch (err) {
|
||
console.error(`[sync-to-drive] 🔴 OAuth2 auth failed: ${err.message}`);
|
||
process.exit(1);
|
||
}
|
||
|
||
// 预生成 Drive 索引文件
|
||
try {
|
||
const { main: generateIndex } = require('./generate-drive-index');
|
||
generateIndex();
|
||
} catch (err) {
|
||
console.log('[sync-to-drive] Index generation skipped (non-fatal):', err.message);
|
||
}
|
||
|
||
console.log('[sync-to-drive] Starting Grid-DB → Drive mirror sync');
|
||
|
||
let totalCreated = 0;
|
||
let totalUpdated = 0;
|
||
let totalSkipped = 0;
|
||
let totalErrors = 0;
|
||
|
||
// 遍历每个同步目录
|
||
for (const syncDir of SYNC_DIRS) {
|
||
const localDir = path.join(GRID_DB_ROOT, syncDir.local);
|
||
const files = collectFiles(localDir, syncDir.recursive);
|
||
|
||
if (files.length === 0) {
|
||
console.log(`[sync-to-drive] ${syncDir.local}: no files to sync`);
|
||
continue;
|
||
}
|
||
|
||
console.log(`[sync-to-drive] ${syncDir.local}: found ${files.length} file(s)`);
|
||
|
||
for (const filePath of files) {
|
||
try {
|
||
// 计算 Drive 中的目标路径
|
||
const relativePath = path.relative(localDir, filePath);
|
||
const dirParts = path.dirname(relativePath);
|
||
const fileName = path.basename(filePath);
|
||
|
||
// 构建 Drive 文件夹路径
|
||
let driveFolderPath = syncDir.drivePrefix;
|
||
if (dirParts !== '.') {
|
||
driveFolderPath = `${syncDir.drivePrefix}/${dirParts}`;
|
||
}
|
||
|
||
// 确保 Drive 文件夹存在
|
||
const targetFolderId = await getOrCreateDriveFolder(
|
||
drive, rootFolderId, driveFolderPath
|
||
);
|
||
|
||
// 读取文件内容并上传
|
||
const content = fs.readFileSync(filePath, 'utf8');
|
||
const result = await uploadFile(drive, targetFolderId, fileName, content);
|
||
|
||
if (result.action === 'created') {
|
||
totalCreated++;
|
||
console.log(`[sync-to-drive] + ${relativePath} (created)`);
|
||
} else if (result.action === 'updated') {
|
||
totalUpdated++;
|
||
console.log(`[sync-to-drive] ~ ${relativePath} (updated)`);
|
||
} else {
|
||
totalSkipped++;
|
||
}
|
||
} catch (err) {
|
||
totalErrors++;
|
||
console.error(`[sync-to-drive] ! ${filePath}: ${err.message}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 输出统计
|
||
console.log('[sync-to-drive] Sync complete:');
|
||
console.log(` Created: ${totalCreated}`);
|
||
console.log(` Updated: ${totalUpdated}`);
|
||
console.log(` Skipped (unchanged): ${totalSkipped}`);
|
||
console.log(` Errors: ${totalErrors}`);
|
||
|
||
if (totalErrors > 0) {
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
main().catch(err => {
|
||
console.error('[sync-to-drive] Fatal error:', err.message);
|
||
process.exit(1);
|
||
});
|