guanghulab/scripts/cos-bridge/cn-pull-from-cos.sh

#!/bin/bash
# ════════════════════════════════════════════════════════════════
# 光湖 · COS 中转桥 · 广州侧拉取
# Sovereign: TCS-0002∞ · ICE-GL∞ · 国作登字-2026-A-00037559
# 守护: 铸渊 · ICE-GL-ZY001
# ════════════════════════════════════════════════════════════════
#
# 设计理念 (cc-003 动态适配 + cc-004 系统强制自主):
#
#   新加坡拉完推到 COS → 广州从 COS 内网拉 → 免费流量
#   铸渊自己跑，冰朔不需要手动做任何事。
#
# 用法:
#   ./cn-pull-from-cos.sh                          # 拉取最新清单
#   ./cn-pull-from-cos.sh --key cos-bridge/npm/express/4.18.2/express-4.18.2.tgz
#   ./cn-pull-from-cos.sh --manifest ./my-manifest.json
#   ./cn-pull-from-cos.sh --type npm --name express --version 4.18.2
#
# 凭据:
#   环境变量: TENCENT_COS_SECRET_ID / TENCENT_COS_SECRET_KEY
#   或者走 vault: curl http://127.0.0.1:8080/internal/fetch/TENCENT_COS_SECRET_ID
#
# 存储:
#   默认拉到 /data/guanghulab/cos-bridge/ 目录
#   通过 --dest 指定其他目录
# ────────────────────────────────────────────────────────────────

set -euo pipefail

# ─── 配置 ─────────────────────────────────────────────────────
COS_BUCKET="${COS_BUCKET:-sy-finetune-corpus-1317346199}"
COS_REGION="${COS_REGION:-ap-guangzhou}"
COS_SECRET_ID="${TENCENT_COS_SECRET_ID:-}"
COS_SECRET_KEY="${TENCENT_COS_SECRET_KEY:-}"
DEST="${COS_BRIDGE_DEST:-/data/guanghulab/cos-bridge}"
PREFIX="cos-bridge"
DRY_RUN=0

# ─── 颜色日志 ─────────────────────────────────────────────────
log_info()  { echo "📋 [$(date -Iseconds)] $*"; }
log_ok()    { echo "✅ [$(date -Iseconds)] $*"; }
log_warn()  { echo "⚠️  [$(date -Iseconds)] $*"; }
log_err()   { echo "❌ [$(date -Iseconds)] $*"; }
log_bridge(){ echo "🌉 [$(date -Iseconds)] $*"; }

# ─── 尝试从 vault 获取凭据 ─────────────────────────────────────
try_vault() {
  if [ -z "$COS_SECRET_ID" ] && curl -sf http://127.0.0.1:8080/admin/__healthz >/dev/null 2>&1; then
    log_info "从 Secrets Vault 获取 COS 凭据..."
    COS_SECRET_ID=$(curl -sf http://127.0.0.1:8080/internal/fetch/TENCENT_COS_SECRET_ID 2>/dev/null || echo "")
    COS_SECRET_KEY=$(curl -sf http://127.0.0.1:8080/internal/fetch/TENCENT_COS_SECRET_KEY 2>/dev/null || echo "")
  fi
}

# ─── 下载单个 key ─────────────────────────────────────────────
pull_key() {
  local cos_key="$1"
  local local_file="${DEST}/${cos_key}"

  mkdir -p "$(dirname "$local_file")"

  log_bridge "☁️ 广州从COS拉取: ${cos_key}"

  # 优先用 coscli（内网endpoint免费流量）
  if command -v coscli >/dev/null 2>&1; then
    coscli cp "cos://${COS_BUCKET}/${cos_key}" "$local_file" \
      --region "$COS_REGION" \
      ${COS_SECRET_ID:+--secret-id "$COS_SECRET_ID"} \
      ${COS_SECRET_KEY:+--secret-key "$COS_SECRET_KEY"}
    log_ok "拉取成功 (coscli): ${local_file}"
  # 兜底用 coscmd
  elif command -v coscmd >/dev/null 2>&1; then
    coscmd download "$cos_key" "$local_file"
    log_ok "拉取成功 (coscmd): ${local_file}"
  else
    log_err "coscli 和 coscmd 均未安装"
    log_info "安装方式: pip install coscli  或  pip install coscmd"
    return 1
  fi

  # 校验 sha256（如果存在）
  local sha_key="${cos_key}.sha256"
  local sha_file="${DEST}/${sha_key}"
  if curl -sf "https://${COS_BUCKET}.cos.${COS_REGION}.myqcloud.com/${sha_key}" -o "$sha_file" 2>/dev/null; then
    local expected=$(awk '{print $1}' "$sha_file")
    local actual=$(sha256sum "$local_file" | awk '{print $1}')
    if [ "$expected" = "$actual" ]; then
      log_ok "SHA256 校验通过 ✓"
    else
      log_err "SHA256 校验失败! 期望=${expected} 实际=${actual}"
      return 1
    fi
  fi
}

# ─── 拉取清单 ─────────────────────────────────────────────────
pull_manifest() {
  local today=$(date +%Y-%m-%d)
  local manifest_key="${PREFIX}/manifests/${today}-manifest.json"
  local manifest_file="${DEST}/manifests/${today}-manifest.json"

  mkdir -p "$(dirname "$manifest_file")"

  log_info "拉取今日清单: ${manifest_key}"
  pull_key "$manifest_key"

  if [ -f "$manifest_file" ]; then
    log_ok "清单拉取成功: ${manifest_file}"
    echo ""
    echo "今日中转内容:"
    python3 -c "
import json
with open('${manifest_file}') as f:
    m = json.load(f)
for item in m.get('items', []):
    print(f\"  {item['type']:6s} {item['name']}@{item['version']} → {item['cosKey']}\")
" 2>/dev/null || cat "$manifest_file"
  else
    log_warn "今日无中转清单"
  fi
}

# ─── 从清单批量拉取 ─────────────────────────────────────────────
pull_from_manifest() {
  local manifest_file="$1"

  if [ ! -f "$manifest_file" ]; then
    log_err "清单文件不存在: ${manifest_file}"
    return 1
  fi

  log_info "从清单批量拉取: ${manifest_file}"

  python3 -c "
import json, subprocess, sys

with open('${manifest_file}') as f:
    m = json.load(f)

for item in m.get('items', []):
    key = item.get('cosKey', '')
    if key:
        print(f'🌉 拉取: {item[\"type\"]} {item[\"name\"]}@{item[\"version\"]}')
        result = subprocess.run(['bash', '$0', '--key', key], capture_output=True, text=True)
        if result.returncode != 0:
            print(f'❌ 失败: {key}', file=sys.stderr)
        else:
            print(f'✅ 完成: {key}')
" 2>&1
}

# ─── 主流程 ────────────────────────────────────────────────────
main() {
  log_bridge "═══ 光湖 COS 中转桥 · 广州侧 ═══"

  # 尝试获取凭据
  try_vault

  if [ -z "$COS_SECRET_ID" ]; then
    log_warn "无 COS 凭据，进入 DRY-RUN 模式"
    DRY_RUN=1
  fi

  mkdir -p "$DEST"

  # 解析参数
  while [ $# -gt 0 ]; do
    case "$1" in
      --key)
        pull_key "$2"
        shift 2
        ;;
      --manifest)
        pull_from_manifest "$2"
        shift 2
        ;;
      --latest)
        pull_manifest
        shift
        ;;
      --dest)
        DEST="$2"
        shift 2
        ;;
      *)
        log_err "未知参数: $1"
        echo "用法: $0 [--key COS_KEY] [--manifest FILE] [--latest] [--dest DIR]"
        exit 1
        ;;
    esac
  done

  # 无参数时默认拉取今日清单
  if [ $# -eq 0 ]; then
    pull_manifest
  fi
}

main "$@"