/** * ═══════════════════════════════════════════════════════════ * 上传引擎 · Upload Engine Service * ═══════════════════════════════════════════════════════════ * * 智库节点 Phase 4 · ZY-SVR-006 * 守护: 铸渊 · ICE-GL-ZY001 * 版权: 国作登字-2026-A-00037559 * * 功能: * - 接收用户上传的小说文件 * - 自动检测文件格式 (TXT/EPUB/HTML/MD/DOCX) * - 转换为统一 TXT 格式存储 * - 自动触发分章引擎 * - 支持多种编码自动检测 * * 支持格式: * - .txt (纯文本 · 自动检测编码) * - .epub (电子书 · 提取正文) * - .html (网页 · 剥离标签) * - .md (Markdown · 转纯文本) * - .docx (Word文档 · 提取正文,需额外解析) * * ═══════════════════════════════════════════════════════════ */ 'use strict'; const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const BOOKS_DIR = process.env.BOOKS_DIR || path.join(__dirname, '..', 'data', 'books'); const UPLOAD_DIR = path.join(__dirname, '..', 'data', 'uploads'); const UPLOAD_FILE = path.join(__dirname, '..', 'data', 'uploads-index.json'); // 支持的格式及 MIME 类型 const SUPPORTED_FORMATS = { '.txt': { mime: ['text/plain'], name: '纯文本' }, '.epub': { mime: ['application/epub+zip'], name: '电子书' }, '.html': { mime: ['text/html'], name: '网页' }, '.htm': { mime: ['text/html'], name: '网页' }, '.md': { mime: ['text/markdown', 'text/plain'], name: 'Markdown' }, '.docx': { mime: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'], name: 'Word文档' } }; const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB let uploadsIndex = []; // ─── 初始化 ─── function init() { if (!fs.existsSync(UPLOAD_DIR)) fs.mkdirSync(UPLOAD_DIR, { recursive: true }); if (!fs.existsSync(BOOKS_DIR)) fs.mkdirSync(BOOKS_DIR, { recursive: true }); loadIndex(); } function loadIndex() { try { if (fs.existsSync(UPLOAD_FILE)) { uploadsIndex = JSON.parse(fs.readFileSync(UPLOAD_FILE, 'utf8')); } } catch { uploadsIndex = []; } } function saveIndex() { try { const dir = path.dirname(UPLOAD_FILE); if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(UPLOAD_FILE, JSON.stringify(uploadsIndex, null, 2), 'utf8'); } catch (err) { console.error('[UploadEngine] 保存索引失败:', err.message); } } /** * 检测文件格式 */ function detectFormat(filename, mimeType) { const ext = path.extname(filename).toLowerCase(); if (SUPPORTED_FORMATS[ext]) { return { ext, format: SUPPORTED_FORMATS[ext].name, supported: true }; } // 尝试从 MIME 类型推断 for (const [formatExt, cfg] of Object.entries(SUPPORTED_FORMATS)) { if (cfg.mime.includes(mimeType)) { return { ext: formatExt, format: cfg.name, supported: true }; } } return { ext, format: '未知格式', supported: false }; } /** * 将 HTML 内容转为纯文本(安全剥离所有标签) */ function htmlToText(html) { let text = html; // 移除 script 和 style 标签及内容(使用 split/join 避免 ReDoS) text = removeTagWithContent(text, 'script'); text = removeTagWithContent(text, 'style'); // 段落和换行 text = text .replace(/<\/p>/gi, '\n\n') .replace(//gi, '\n') .replace(/<\/div>/gi, '\n') .replace(/<\/h[1-6]>/gi, '\n\n') .replace(/]*>/gi, '\n'); // 移除所有 HTML 标签 text = text.replace(/<[^>]+>/g, ''); // 解码 HTML 实体 text = text .replace(/ /g, ' ') .replace(/&/g, '&') .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') .replace(/'/g, "'") .replace(/'/g, "'"); // 清理多余空行 text = text.replace(/\n{3,}/g, '\n\n').trim(); return text; } /** * 安全移除指定标签及其内容(避免正则 ReDoS) */ function removeTagWithContent(html, tagName) { const openTag = '<' + tagName; const closeTag = ''; let result = html; let maxIterations = 100; while (maxIterations-- > 0) { const openIdx = result.toLowerCase().indexOf(openTag); if (openIdx === -1) break; const closeIdx = result.toLowerCase().indexOf(closeTag, openIdx); if (closeIdx === -1) { // No close tag, remove from open tag to end of string result = result.slice(0, openIdx); break; } result = result.slice(0, openIdx) + result.slice(closeIdx + closeTag.length); } return result; } /** * 将 Markdown 转为纯文本 */ function markdownToText(md) { return md // 标题 .replace(/^#{1,6}\s+/gm, '') // 加粗/斜体 .replace(/\*\*(.+?)\*\*/g, '$1') .replace(/\*(.+?)\*/g, '$1') .replace(/__(.+?)__/g, '$1') .replace(/_(.+?)_/g, '$1') // 链接 .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // 图片 .replace(/!\[([^\]]*)\]\([^)]+\)/g, '') // 代码块 .replace(/```[\s\S]*?```/g, '') .replace(/`([^`]+)`/g, '$1') // 引用 .replace(/^>\s+/gm, '') // 分隔线 .replace(/^[-*_]{3,}$/gm, '') // 清理 .replace(/\n{3,}/g, '\n\n') .trim(); } /** * 提取 EPUB 文本内容(简易版) * * EPUB 本质上是 ZIP 文件,内部包含 XHTML 文件 * 注意: 此简易实现通过字符扫描提取文本,不解压 ZIP * 对于复杂 EPUB,建议转为 TXT 后重新上传 */ function extractEpubText(buffer) { // Convert buffer to string, ignoring non-UTF8 bytes const content = buffer.toString('utf8'); // Extract text between and tags using indexOf (safe from ReDoS) const textParts = []; let searchStart = 0; const maxParts = 200; while (textParts.length < maxParts) { const bodyOpen = content.toLowerCase().indexOf('', bodyOpen); if (bodyTagEnd === -1) break; const bodyClose = content.toLowerCase().indexOf('', bodyTagEnd); if (bodyClose === -1) break; const bodyContent = content.slice(bodyTagEnd + 1, bodyClose); const text = htmlToText(bodyContent); if (text.length > 20) { textParts.push(text); } searchStart = bodyClose + 7; } if (textParts.length > 0) { return textParts.join('\n\n'); } // 降级: 提取可见纯文本行 const lines = content.split('\n').filter(line => { const trimmed = line.trim(); return trimmed.length > 10 && !trimmed.startsWith('<') && !trimmed.startsWith('PK') && !/[\x00-\x08]/.test(trimmed); }); return lines.join('\n') || '(EPUB解析受限,建议转为TXT后重新上传)'; } /** * 转换上传文件为纯文本 */ function convertToText(buffer, format) { let text; switch (format.ext) { case '.txt': text = buffer.toString('utf8'); break; case '.html': case '.htm': text = htmlToText(buffer.toString('utf8')); break; case '.md': text = markdownToText(buffer.toString('utf8')); break; case '.epub': text = extractEpubText(buffer); break; case '.docx': // DOCX 是 ZIP 文件包含 XML // 简易提取:类似 EPUB text = extractEpubText(buffer); if (text.length < 50) { text = '(DOCX格式建议先用Word另存为TXT后重新上传)'; } break; default: // 尝试作为纯文本处理 text = buffer.toString('utf8'); break; } // 统一换行符 text = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); return text; } /** * 处理上传文件 * * @param {object} file { originalname, mimetype, buffer, size } * @param {object} meta { title, author, uploader } * @returns {object} 上传结果 */ function processUpload(file, meta) { if (!file || !file.buffer) { throw new Error('没有收到文件数据'); } if (file.size > MAX_FILE_SIZE) { throw new Error(`文件过大 (${(file.size / 1024 / 1024).toFixed(1)}MB),最大支持 50MB`); } // 检测格式 const format = detectFormat(file.originalname, file.mimetype); // 转换为文本 const text = convertToText(file.buffer, format); const wordCount = text.replace(/\s/g, '').length; if (wordCount < 10) { throw new Error('文件内容为空或无法解析,请确认文件格式正确'); } // 生成安全的文件名 const baseName = (meta.title || path.basename(file.originalname, path.extname(file.originalname))) .replace(/[<>:"/\\|?*\x00-\x1F]/g, '_') .slice(0, 100); const author = (meta.author || '未知') .replace(/[<>:"/\\|?*\x00-\x1F]/g, '_') .slice(0, 50); const uploadId = `UPL-${Date.now()}-${crypto.randomBytes(3).toString('hex')}`; const targetFilename = `${baseName}_${author}_upload.txt`; const targetPath = path.join(BOOKS_DIR, targetFilename); // 保存为 TXT fs.writeFileSync(targetPath, text, 'utf8'); // 记录索引 const record = { upload_id: uploadId, original_name: file.originalname, original_format: format.format, original_ext: format.ext, original_size: file.size, converted_file: targetFilename, title: meta.title || baseName, author: meta.author || '未知', uploader: meta.uploader || 'unknown', word_count: wordCount, chapter_count: 0, created_at: new Date().toISOString() }; uploadsIndex.push(record); saveIndex(); return record; } /** * 获取上传列表 */ function listUploads(options) { let result = [...uploadsIndex]; if (options && options.uploader) { result = result.filter(u => u.uploader === options.uploader); } result.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); if (options && options.limit) { result = result.slice(0, options.limit); } return result; } /** * 获取支持的格式列表 */ function getSupportedFormats() { return Object.entries(SUPPORTED_FORMATS).map(([ext, cfg]) => ({ ext, name: cfg.name, mime: cfg.mime })); } /** * 统计 */ function getStats() { return { total_uploads: uploadsIndex.length, total_words: uploadsIndex.reduce((s, u) => s + (u.word_count || 0), 0), formats: uploadsIndex.reduce((acc, u) => { const f = u.original_format || '未知'; acc[f] = (acc[f] || 0) + 1; return acc; }, {}) }; } init(); module.exports = { processUpload, listUploads, detectFormat, getSupportedFormats, getStats };