guanghulab/scripts/grid-db/sync-to-drive.js

/**
 * scripts/grid-db/sync-to-drive.js
 *
 * 功能：将 grid-db/ 中的关键文件同步到 Google Drive 镜像目录
 *
 * 同步范围：
 *   grid-db/memory/DEV-XXX/*.json  →  Drive/mirror/memory/DEV-XXX/
 *   grid-db/outbox/latest/*.json    →  Drive/mirror/outbox/
 *   grid-db/rules/*.json            →  Drive/mirror/rules/
 *   grid-db/drive-index/*.json      →  Drive/mirror/（每个 DEV 的 index.json）
 *
 * 同步策略：
 *   - 比较文件内容 MD5 hash，仅同步有变更的文件（节省 API 配额）
 *   - 文件不存在则创建，已存在则覆盖更新
 *   - Drive 中多出的文件不删除（防误删）
 *
 * 注意：
 *   - 不同步 interactions/ 和 training-lake/（数据量大 + 无需 Gemini 读取）
 *   - 不同步 inbox/ 和 processing/（系统内部流转区）
 *
 * 环境变量：
 *   - GDRIVE_CLIENT_ID: OAuth 客户端 ID
 *   - GDRIVE_CLIENT_SECRET: OAuth 客户端密钥
 *   - GDRIVE_REFRESH_TOKEN: 长效刷新令牌
 *   - DRIVE_FOLDER_ID: Drive 镜像根文件夹 ID
 *
 * 守护: PER-ZY001 铸渊
 * 系统: SYS-GLW-0001
 */

const { google } = require('googleapis');
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');

const GRID_DB_ROOT = path.join(__dirname, '../../grid-db');

// 同步目录映射：本地相对路径 → Drive 镜像子目录
const SYNC_DIRS = [
  { local: 'memory',        drivePrefix: 'mirror/memory',  recursive: true },
  { local: 'outbox/latest', drivePrefix: 'mirror/outbox',  recursive: false },
  { local: 'rules',         drivePrefix: 'mirror/rules',   recursive: false },
  { local: 'drive-index',   drivePrefix: 'mirror',         recursive: false }
];

/**
 * 计算文件内容的 MD5 哈希
 */
function md5(content) {
  return crypto.createHash('md5').update(content).digest('hex');
}

/**
 * 递归收集目录下的所有 JSON 文件
 */
function collectFiles(dirPath, recursive) {
  const results = [];
  if (!fs.existsSync(dirPath)) return results;

  const entries = fs.readdirSync(dirPath, { withFileTypes: true });
  for (const entry of entries) {
    const fullPath = path.join(dirPath, entry.name);
    if (entry.isDirectory() && recursive) {
      results.push(...collectFiles(fullPath, true));
    } else if (entry.isFile() && entry.name.endsWith('.json')) {
      results.push(fullPath);
    }
  }
  return results;
}

/**
 * 转义 Drive API 查询中的单引号
 */
function escapeQuery(str) {
  return str.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
}

/**
 * 在 Drive 中查找或创建文件夹（按路径逐级创建）
 */
async function getOrCreateDriveFolder(drive, parentId, folderPath) {
  const parts = folderPath.split('/').filter(Boolean);
  let currentParentId = parentId;

  for (const part of parts) {
    const safePart = escapeQuery(part);
    const query = `name='${safePart}' and '${currentParentId}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false`;
    const res = await drive.files.list({
      q: query,
      fields: 'files(id, name)',
      pageSize: 1
    });

    if (res.data.files && res.data.files.length > 0) {
      currentParentId = res.data.files[0].id;
    } else {
      const createRes = await drive.files.create({
        requestBody: {
          name: part,
          mimeType: 'application/vnd.google-apps.folder',
          parents: [currentParentId]
        },
        fields: 'id'
      });
      currentParentId = createRes.data.id;
      console.log(`[sync-to-drive] Created folder: ${part}`);
    }
  }

  return currentParentId;
}

/**
 * 在 Drive 目标文件夹中查找同名文件
 */
async function findFileInDrive(drive, folderId, fileName) {
  const safeFileName = escapeQuery(fileName);
  const query = `name='${safeFileName}' and '${folderId}' in parents and trashed=false`;
  const res = await drive.files.list({
    q: query,
    fields: 'files(id, name, md5Checksum)',
    pageSize: 1
  });

  if (res.data.files && res.data.files.length > 0) {
    return res.data.files[0];
  }
  return null;
}

/**
 * 上传或更新单个文件到 Drive
 */
async function uploadFile(drive, folderId, fileName, content) {
  const existingFile = await findFileInDrive(drive, folderId, fileName);
  const { Readable } = require('stream');

  const media = {
    mimeType: 'application/json',
    body: Readable.from([content])
  };

  if (existingFile) {
    // 检查内容是否变更（Drive 对上传文件自动计算 MD5）
    const localHash = md5(content);
    if (existingFile.md5Checksum === localHash) {
      return { action: 'skipped', fileName };
    }

    await drive.files.update({
      fileId: existingFile.id,
      media
    });
    return { action: 'updated', fileName };
  } else {
    await drive.files.create({
      requestBody: {
        name: fileName,
        parents: [folderId]
      },
      media,
      fields: 'id'
    });
    return { action: 'created', fileName };
  }
}

/**
 * 主同步流程
 */
async function main() {
  // 验证环境变量
  const rootFolderId = process.env.DRIVE_FOLDER_ID;

  if (!rootFolderId) {
    console.log('[sync-to-drive] DRIVE_FOLDER_ID not set, skipping');
    return;
  }

  // OAuth2 认证（统一入口）
  let drive;
  try {
    const { getDriveClient } = require('./drive-auth');
    drive = getDriveClient();
    console.log('[sync-to-drive] ✅ OAuth2 credentials configured');
  } catch (err) {
    console.error(`[sync-to-drive] 🔴 OAuth2 auth failed: ${err.message}`);
    process.exit(1);
  }

  // 预生成 Drive 索引文件
  try {
    const { main: generateIndex } = require('./generate-drive-index');
    generateIndex();
  } catch (err) {
    console.log('[sync-to-drive] Index generation skipped (non-fatal):', err.message);
  }

  console.log('[sync-to-drive] Starting Grid-DB → Drive mirror sync');

  let totalCreated = 0;
  let totalUpdated = 0;
  let totalSkipped = 0;
  let totalErrors = 0;

  // 遍历每个同步目录
  for (const syncDir of SYNC_DIRS) {
    const localDir = path.join(GRID_DB_ROOT, syncDir.local);
    const files = collectFiles(localDir, syncDir.recursive);

    if (files.length === 0) {
      console.log(`[sync-to-drive] ${syncDir.local}: no files to sync`);
      continue;
    }

    console.log(`[sync-to-drive] ${syncDir.local}: found ${files.length} file(s)`);

    for (const filePath of files) {
      try {
        // 计算 Drive 中的目标路径
        const relativePath = path.relative(localDir, filePath);
        const dirParts = path.dirname(relativePath);
        const fileName = path.basename(filePath);

        // 构建 Drive 文件夹路径
        let driveFolderPath = syncDir.drivePrefix;
        if (dirParts !== '.') {
          driveFolderPath = `${syncDir.drivePrefix}/${dirParts}`;
        }

        // 确保 Drive 文件夹存在
        const targetFolderId = await getOrCreateDriveFolder(
          drive, rootFolderId, driveFolderPath
        );

        // 读取文件内容并上传
        const content = fs.readFileSync(filePath, 'utf8');
        const result = await uploadFile(drive, targetFolderId, fileName, content);

        if (result.action === 'created') {
          totalCreated++;
          console.log(`[sync-to-drive]   + ${relativePath} (created)`);
        } else if (result.action === 'updated') {
          totalUpdated++;
          console.log(`[sync-to-drive]   ~ ${relativePath} (updated)`);
        } else {
          totalSkipped++;
        }
      } catch (err) {
        totalErrors++;
        console.error(`[sync-to-drive]   ! ${filePath}: ${err.message}`);
      }
    }
  }

  // 输出统计
  console.log('[sync-to-drive] Sync complete:');
  console.log(`  Created: ${totalCreated}`);
  console.log(`  Updated: ${totalUpdated}`);
  console.log(`  Skipped (unchanged): ${totalSkipped}`);
  console.log(`  Errors: ${totalErrors}`);

  if (totalErrors > 0) {
    process.exit(1);
  }
}

main().catch(err => {
  console.error('[sync-to-drive] Fatal error:', err.message);
  process.exit(1);
});