guanghulab/scripts/cos-bridge/sg-fetch-push.js

#!/usr/bin/env node
/**
 * ════════════════════════════════════════════════════════════════
 * 光湖 · COS 中转桥 · 新加坡侧拉取 + 推送
 * Sovereign: TCS-0002∞ · ICE-GL∞ · 国作登字-2026-A-00037559
 * 守护: 铸渊 · ICE-GL-ZY001
 * ════════════════════════════════════════════════════════════════
 *
 * 设计理念 (cc-003 动态适配 + cc-004 系统强制自主):
 *
 *   国内服务器下载国外开源软件慢 → 新加坡出站快 (35Mbps)
 *   铸渊触发下载 → 新加坡拉 → 打包推 COS → 广州从 COS 拉
 *
 *   铸渊自己串整条链路，冰朔不需要手动做任何事。
 *
 * 用法:
 *   # 拉取单个文件/仓库
 *   node sg-fetch-push.js --type npm --name express --version latest
 *   node sg-fetch-push.js --type git --url https://github.com/user/repo.git
 *   node sg-fetch-push.js --type file --url https://example.com/package.tar.gz
 *
 *   # 批量拉取（从配置文件）
 *   node sg-fetch-push.js --config ./fetch-manifest.json
 *
 * 凭据:
 *   通过环境变量: TENCENT_COS_SECRET_ID / TENCENT_COS_SECRET_KEY
 *   COS 配置: COS_BUCKET / COS_REGION (默认 sy-finetune-corpus-1317346199 / ap-guangzhou)
 *
 * 存储路径:
 *   COS: cos-bridge/{type}/{name}/{version}/{filename}
 *   例: cos-bridge/npm/express/4.18.2/express-4.18.2.tgz
 *       cos-bridge/git/gitea/1.23.7/gitea-src.tar.gz
 *       cos-bridge/file/some-package/1.0/package.tar.gz
 */

'use strict';

const { execSync } = require('child_process');
const fs = require('fs');
const path = require('path');
const os = require('os');
const crypto = require('crypto');

// ─── 配置 ─────────────────────────────────────────────────────
const CONFIG = {
  cosBucket: process.env.COS_BUCKET || 'sy-finetune-corpus-1317346199',
  cosRegion: process.env.COS_REGION || 'ap-guangzhou',
  cosSecretId: process.env.TENCENT_COS_SECRET_ID,
  cosSecretKey: process.env.TENCENT_COS_SECRET_KEY,
  cosPrefix: 'cos-bridge',
  tmpDir: process.env.COS_BRIDGE_TMP || path.join(os.tmpdir(), 'cos-bridge'),
  dryRun: !process.env.TENCENT_COS_SECRET_ID,
};

const COPYRIGHT = {
  registration: '国作登字-2026-A-00037559',
  sovereign: '冰朔 · TCS-0002∞',
  system_root: 'SYS-GLW-0001',
  guard: '铸渊 · ICE-GL-ZY001',
};

// ─── 日志 ─────────────────────────────────────────────────────
function log(level, msg) {
  const ts = new Date().toISOString();
  const prefix = { info: '📋', ok: '✅', warn: '⚠️', err: '❌', bridge: '🌉' };
  console.log(`${prefix[level] || '·'} [${ts}] ${msg}`);
}

// ─── 工具 ─────────────────────────────────────────────────────
function ensureDir(dir) {
  fs.mkdirSync(dir, { recursive: true });
}

function sha256File(filePath) {
  const hash = crypto.createHash('sha256');
  hash.update(fs.readFileSync(filePath));
  return hash.digest('hex');
}

function exec(cmd, opts = {}) {
  log('info', `执行: ${cmd}`);
  try {
    return execSync(cmd, { encoding: 'utf-8', timeout: 300000, ...opts });
  } catch (e) {
    log('err', `命令失败: ${e.message}`);
    throw e;
  }
}

// ─── 拉取逻辑 ─────────────────────────────────────────────────

/**
 * npm 包拉取
 */
function fetchNpm(name, version = 'latest') {
  const workDir = path.join(CONFIG.tmpDir, 'npm', name);
  ensureDir(workDir);

  log('bridge', `🇸🇬 新加坡 → npm 拉取 ${name}@${version}`);

  // 用 npm pack 拉取 tgz
  const target = exec(`npm pack ${name}@${version} --pack-destination="${workDir}"`).trim();
  const localFile = path.join(workDir, target);

  // 获取实际版本号
  const resolvedVersion = target.replace(/^[^-]+-/, '').replace(/\.tgz$/, '');

  log('ok', `拉取完成: ${target} (${(fs.statSync(localFile).size / 1024).toFixed(1)}KB)`);

  return {
    type: 'npm',
    name,
    version: resolvedVersion,
    localFile,
    cosKey: `${CONFIG.cosPrefix}/npm/${name}/${resolvedVersion}/${target}`,
  };
}

/**
 * Git 仓库拉取
 */
function fetchGit(url, branch = 'main', name = null) {
  const repoName = name || url.split('/').pop().replace('.git', '');
  const workDir = path.join(CONFIG.tmpDir, 'git', repoName);
  ensureDir(workDir);

  log('bridge', `🇸🇬 新加坡 → git clone ${url} (branch: ${branch})`);

  // 浅克隆
  exec(`git clone --depth=1 --branch ${branch} "${url}" "${workDir}/src"`);

  // 打包
  const archiveFile = path.join(CONFIG.tmpDir, 'git', `${repoName}-${branch}.tar.gz`);
  exec(`cd "${workDir}/src" && git archive --format=tar.gz --prefix="${repoName}/" -o "${archiveFile}" HEAD`);

  // 获取 commit hash
  const commitHash = exec(`cd "${workDir}/src" && git rev-parse --short HEAD`).trim();

  log('ok', `打包完成: ${repoName}-${branch}.tar.gz (${(fs.statSync(archiveFile).size / 1024 / 1024).toFixed(1)}MB) @${commitHash}`);

  return {
    type: 'git',
    name: repoName,
    version: commitHash,
    branch,
    localFile: archiveFile,
    cosKey: `${CONFIG.cosPrefix}/git/${repoName}/${branch}/${repoName}-${commitHash}.tar.gz`,
  };
}

/**
 * 通用文件拉取
 */
function fetchFile(url, name = null) {
  const fileName = name || url.split('/').pop().split('?')[0];
  const workDir = path.join(CONFIG.tmpDir, 'file');
  ensureDir(workDir);

  log('bridge', `🇸🇬 新加坡 → wget ${url}`);

  const localFile = path.join(workDir, fileName);
  exec(`wget -q -O "${localFile}" "${url}"`);

  log('ok', `下载完成: ${fileName} (${(fs.statSync(localFile).size / 1024 / 1024).toFixed(1)}MB)`);

  return {
    type: 'file',
    name: fileName,
    version: new Date().toISOString().split('T')[0],
    localFile,
    cosKey: `${CONFIG.cosPrefix}/file/${fileName}/${new Date().toISOString().split('T')[0]}/${fileName}`,
  };
}

// ─── COS 推送 ──────────────────────────────────────────────────

/**
 * 推送到 COS
 * 优先使用 coscli（如果安装了），否则用 COS Node SDK
 */
async function pushToCos(item) {
  if (CONFIG.dryRun) {
    log('warn', `[DRY-RUN] 将推送 ${item.localFile} → cos://${CONFIG.cosBucket}/${item.cosKey}`);
    return { ...item, cosUrl: `cos://${CONFIG.cosBucket}/${item.cosKey}`, dryRun: true };
  }

  const sha256 = sha256File(item.localFile);
  const fileSize = fs.statSync(item.localFile).size;

  log('bridge', `☁️ 推送 → COS: ${item.cosKey} (${(fileSize / 1024 / 1024).toFixed(1)}MB, sha256=${sha256.slice(0, 12)}...)`);

  // 尝试用 coscli
  try {
    exec(`coscli cp "${item.localFile}" "cos://${CONFIG.cosBucket}/${item.cosKey}"`, { timeout: 600000 });
    log('ok', `COS 推送成功 (coscli)`);
  } catch (e) {
    // coscli 不可用，尝试 coscmd
    try {
      exec(`coscmd upload "${item.localFile}" "${item.cosKey}"`, { timeout: 600000 });
      log('ok', `COS 推送成功 (coscmd)`);
    } catch (e2) {
      log('warn', `coscli 和 coscmd 均不可用，尝试 Node SDK...`);
      await pushViaNodeSdk(item);
    }
  }

  // 同时推送 sha256 校验文件
  const shaFile = item.localFile + '.sha256';
  fs.writeFileSync(shaFile, `${sha256}  ${path.basename(item.localFile)}`);

  try {
    exec(`coscli cp "${shaFile}" "cos://${CONFIG.cosBucket}/${item.cosKey}.sha256"`, { timeout: 60000 });
  } catch (_) {
    // 校验文件推送失败不影响主流程
  }

  return {
    ...item,
    sha256,
    fileSize,
    cosUrl: `https://${CONFIG.cosBucket}.cos.${CONFIG.cosRegion}.myqcloud.com/${item.cosKey}`,
    pushedAt: new Date().toISOString(),
  };
}

/**
 * Node SDK 推送（兜底）
 */
async function pushViaNodeSdk(item) {
  try {
    // 动态加载 cos-nodejs-sdk-v5
    const COS = require('cos-nodejs-sdk-v5');
    const cos = new COS({
      SecretId: CONFIG.cosSecretId,
      SecretKey: CONFIG.cosSecretKey,
    });

    await new Promise((resolve, reject) => {
      cos.sliceUploadFile({
        Bucket: CONFIG.cosBucket,
        Region: CONFIG.cosRegion,
        Key: item.cosKey,
        FilePath: item.localFile,
      }, (err, data) => {
        if (err) reject(err);
        else resolve(data);
      });
    });
    log('ok', `COS 推送成功 (Node SDK)`);
  } catch (e) {
    log('err', `COS 推送全部失败: ${e.message}`);
    throw new Error('无法推送到 COS: coscli/coscmd/SDK 均不可用');
  }
}

// ─── 清理临时文件 ──────────────────────────────────────────────
function cleanup() {
  if (process.env.COS_BRIDGE_KEEP_TMP !== '1') {
    log('info', '清理临时文件...');
    try {
      exec(`rm -rf "${CONFIG.tmpDir}"`);
    } catch (_) {}
  }
}

// ─── 主流程 ────────────────────────────────────────────────────
async function main() {
  const args = process.argv.slice(2);
  const type = getArg(args, '--type');
  const name = getArg(args, '--name');
  const version = getArg(args, '--version') || 'latest';
  const url = getArg(args, '--url');
  const branch = getArg(args, '--branch') || 'main';
  const configPath = getArg(args, '--config');

  log('bridge', '═══ 光湖 COS 中转桥 · 新加坡侧 ═══');
  log('info', `COS: ${CONFIG.cosBucket} / ${CONFIG.cosRegion}`);
  log('info', `模式: ${CONFIG.dryRun ? 'DRY-RUN (无凭据)' : '正式'}`);

  let items = [];

  try {
    if (configPath) {
      // 批量模式
      const manifest = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
      log('info', `批量模式: ${manifest.items.length} 个任务`);
      for (const task of manifest.items) {
        items.push(fetchItem(task));
      }
    } else if (type === 'npm') {
      items.push(fetchNpm(name || 'express', version));
    } else if (type === 'git') {
      items.push(fetchGit(url, branch, name));
    } else if (type === 'file') {
      items.push(fetchFile(url, name));
    } else {
      console.error('用法: node sg-fetch-push.js --type npm|git|file --name NAME [--url URL] [--version VER] [--branch BRANCH]');
      console.error('批量: node sg-fetch-push.js --config ./fetch-manifest.json');
      process.exit(1);
    }

    // 逐个推送到 COS
    const results = [];
    for (const item of items) {
      const result = await pushToCos(item);
      results.push(result);
      log('ok', `✅ ${result.name}@${result.version} → ${result.cosKey}`);
    }

    // 输出清单（给广州侧用）
    const manifest = {
      generated_at: new Date().toISOString(),
      generated_by: '铸渊 · ICE-GL-ZY001',
      sovereign: 'TCS-0002∞ 冰朔',
      bridge: '新加坡 → COS → 广州',
      items: results.map(r => ({
        type: r.type,
        name: r.name,
        version: r.version,
        cosKey: r.cosKey,
        sha256: r.sha256,
        fileSize: r.fileSize,
        cosUrl: r.cosUrl,
      })),
    };

    const manifestPath = path.join(CONFIG.tmpDir || os.tmpdir(), 'cos-bridge-manifest.json');
    ensureDir(path.dirname(manifestPath));
    fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
    log('ok', `清单已写入: ${manifestPath}`);

    // 清单也推到 COS
    if (!CONFIG.dryRun) {
      try {
        const today = new Date().toISOString().split('T')[0];
        exec(`coscli cp "${manifestPath}" "cos://${CONFIG.cosBucket}/${CONFIG.cosPrefix}/manifests/${today}-manifest.json"`);
        log('ok', `清单已推到 COS: cos-bridge/manifests/${today}-manifest.json`);
      } catch (_) {}
    }

  } finally {
    cleanup();
  }
}

function fetchItem(task) {
  switch (task.type) {
    case 'npm': return fetchNpm(task.name, task.version);
    case 'git': return fetchGit(task.url, task.branch || 'main', task.name);
    case 'file': return fetchFile(task.url, task.name);
    default: throw new Error(`未知类型: ${task.type}`);
  }
}

function getArg(args, flag) {
  const idx = args.indexOf(flag);
  return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : null;
}

main().catch(e => {
  log('err', `中转桥失败: ${e.message}`);
  process.exit(1);
});