/**
* ═══════════════════════════════════════════════════════════
* 上传引擎 · Upload Engine Service
* ═══════════════════════════════════════════════════════════
*
* 智库节点 Phase 4 · ZY-SVR-006
* 守护: 铸渊 · ICE-GL-ZY001
* 版权: 国作登字-2026-A-00037559
*
* 功能:
* - 接收用户上传的小说文件
* - 自动检测文件格式 (TXT/EPUB/HTML/MD/DOCX)
* - 转换为统一 TXT 格式存储
* - 自动触发分章引擎
* - 支持多种编码自动检测
*
* 支持格式:
* - .txt (纯文本 · 自动检测编码)
* - .epub (电子书 · 提取正文)
* - .html (网页 · 剥离标签)
* - .md (Markdown · 转纯文本)
* - .docx (Word文档 · 提取正文,需额外解析)
*
* ═══════════════════════════════════════════════════════════
*/
'use strict';
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const BOOKS_DIR = process.env.BOOKS_DIR || path.join(__dirname, '..', 'data', 'books');
const UPLOAD_DIR = path.join(__dirname, '..', 'data', 'uploads');
const UPLOAD_FILE = path.join(__dirname, '..', 'data', 'uploads-index.json');
// 支持的格式及 MIME 类型
const SUPPORTED_FORMATS = {
'.txt': { mime: ['text/plain'], name: '纯文本' },
'.epub': { mime: ['application/epub+zip'], name: '电子书' },
'.html': { mime: ['text/html'], name: '网页' },
'.htm': { mime: ['text/html'], name: '网页' },
'.md': { mime: ['text/markdown', 'text/plain'], name: 'Markdown' },
'.docx': { mime: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'], name: 'Word文档' }
};
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
let uploadsIndex = [];
// ─── 初始化 ───
function init() {
if (!fs.existsSync(UPLOAD_DIR)) fs.mkdirSync(UPLOAD_DIR, { recursive: true });
if (!fs.existsSync(BOOKS_DIR)) fs.mkdirSync(BOOKS_DIR, { recursive: true });
loadIndex();
}
function loadIndex() {
try {
if (fs.existsSync(UPLOAD_FILE)) {
uploadsIndex = JSON.parse(fs.readFileSync(UPLOAD_FILE, 'utf8'));
}
} catch {
uploadsIndex = [];
}
}
function saveIndex() {
try {
const dir = path.dirname(UPLOAD_FILE);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(UPLOAD_FILE, JSON.stringify(uploadsIndex, null, 2), 'utf8');
} catch (err) {
console.error('[UploadEngine] 保存索引失败:', err.message);
}
}
/**
* 检测文件格式
*/
function detectFormat(filename, mimeType) {
const ext = path.extname(filename).toLowerCase();
if (SUPPORTED_FORMATS[ext]) {
return {
ext,
format: SUPPORTED_FORMATS[ext].name,
supported: true
};
}
// 尝试从 MIME 类型推断
for (const [formatExt, cfg] of Object.entries(SUPPORTED_FORMATS)) {
if (cfg.mime.includes(mimeType)) {
return { ext: formatExt, format: cfg.name, supported: true };
}
}
return { ext, format: '未知格式', supported: false };
}
/**
* 将 HTML 内容转为纯文本(安全剥离所有标签)
*/
function htmlToText(html) {
let text = html;
// 移除 script 和 style 标签及内容(使用 split/join 避免 ReDoS)
text = removeTagWithContent(text, 'script');
text = removeTagWithContent(text, 'style');
// 段落和换行
text = text
.replace(/<\/p>/gi, '\n\n')
.replace(/
/gi, '\n')
.replace(/<\/div>/gi, '\n')
.replace(/<\/h[1-6]>/gi, '\n\n')
.replace(/]*>/gi, '\n');
// 移除所有 HTML 标签
text = text.replace(/<[^>]+>/g, '');
// 解码 HTML 实体
text = text
.replace(/ /g, ' ')
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/"/g, '"')
.replace(/'/g, "'")
.replace(/'/g, "'");
// 清理多余空行
text = text.replace(/\n{3,}/g, '\n\n').trim();
return text;
}
/**
* 安全移除指定标签及其内容(避免正则 ReDoS)
*/
function removeTagWithContent(html, tagName) {
const openTag = '<' + tagName;
const closeTag = '' + tagName + '>';
let result = html;
let maxIterations = 100;
while (maxIterations-- > 0) {
const openIdx = result.toLowerCase().indexOf(openTag);
if (openIdx === -1) break;
const closeIdx = result.toLowerCase().indexOf(closeTag, openIdx);
if (closeIdx === -1) {
// No close tag, remove from open tag to end of string
result = result.slice(0, openIdx);
break;
}
result = result.slice(0, openIdx) + result.slice(closeIdx + closeTag.length);
}
return result;
}
/**
* 将 Markdown 转为纯文本
*/
function markdownToText(md) {
return md
// 标题
.replace(/^#{1,6}\s+/gm, '')
// 加粗/斜体
.replace(/\*\*(.+?)\*\*/g, '$1')
.replace(/\*(.+?)\*/g, '$1')
.replace(/__(.+?)__/g, '$1')
.replace(/_(.+?)_/g, '$1')
// 链接
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
// 图片
.replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
// 代码块
.replace(/```[\s\S]*?```/g, '')
.replace(/`([^`]+)`/g, '$1')
// 引用
.replace(/^>\s+/gm, '')
// 分隔线
.replace(/^[-*_]{3,}$/gm, '')
// 清理
.replace(/\n{3,}/g, '\n\n')
.trim();
}
/**
* 提取 EPUB 文本内容(简易版)
*
* EPUB 本质上是 ZIP 文件,内部包含 XHTML 文件
* 注意: 此简易实现通过字符扫描提取文本,不解压 ZIP
* 对于复杂 EPUB,建议转为 TXT 后重新上传
*/
function extractEpubText(buffer) {
// Convert buffer to string, ignoring non-UTF8 bytes
const content = buffer.toString('utf8');
// Extract text between and tags using indexOf (safe from ReDoS)
const textParts = [];
let searchStart = 0;
const maxParts = 200;
while (textParts.length < maxParts) {
const bodyOpen = content.toLowerCase().indexOf('', bodyOpen);
if (bodyTagEnd === -1) break;
const bodyClose = content.toLowerCase().indexOf('', bodyTagEnd);
if (bodyClose === -1) break;
const bodyContent = content.slice(bodyTagEnd + 1, bodyClose);
const text = htmlToText(bodyContent);
if (text.length > 20) {
textParts.push(text);
}
searchStart = bodyClose + 7;
}
if (textParts.length > 0) {
return textParts.join('\n\n');
}
// 降级: 提取可见纯文本行
const lines = content.split('\n').filter(line => {
const trimmed = line.trim();
return trimmed.length > 10
&& !trimmed.startsWith('<')
&& !trimmed.startsWith('PK')
&& !/[\x00-\x08]/.test(trimmed);
});
return lines.join('\n') || '(EPUB解析受限,建议转为TXT后重新上传)';
}
/**
* 转换上传文件为纯文本
*/
function convertToText(buffer, format) {
let text;
switch (format.ext) {
case '.txt':
text = buffer.toString('utf8');
break;
case '.html':
case '.htm':
text = htmlToText(buffer.toString('utf8'));
break;
case '.md':
text = markdownToText(buffer.toString('utf8'));
break;
case '.epub':
text = extractEpubText(buffer);
break;
case '.docx':
// DOCX 是 ZIP 文件包含 XML
// 简易提取:类似 EPUB
text = extractEpubText(buffer);
if (text.length < 50) {
text = '(DOCX格式建议先用Word另存为TXT后重新上传)';
}
break;
default:
// 尝试作为纯文本处理
text = buffer.toString('utf8');
break;
}
// 统一换行符
text = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
return text;
}
/**
* 处理上传文件
*
* @param {object} file { originalname, mimetype, buffer, size }
* @param {object} meta { title, author, uploader }
* @returns {object} 上传结果
*/
function processUpload(file, meta) {
if (!file || !file.buffer) {
throw new Error('没有收到文件数据');
}
if (file.size > MAX_FILE_SIZE) {
throw new Error(`文件过大 (${(file.size / 1024 / 1024).toFixed(1)}MB),最大支持 50MB`);
}
// 检测格式
const format = detectFormat(file.originalname, file.mimetype);
// 转换为文本
const text = convertToText(file.buffer, format);
const wordCount = text.replace(/\s/g, '').length;
if (wordCount < 10) {
throw new Error('文件内容为空或无法解析,请确认文件格式正确');
}
// 生成安全的文件名
const baseName = (meta.title || path.basename(file.originalname, path.extname(file.originalname)))
.replace(/[<>:"/\\|?*\x00-\x1F]/g, '_')
.slice(0, 100);
const author = (meta.author || '未知')
.replace(/[<>:"/\\|?*\x00-\x1F]/g, '_')
.slice(0, 50);
const uploadId = `UPL-${Date.now()}-${crypto.randomBytes(3).toString('hex')}`;
const targetFilename = `${baseName}_${author}_upload.txt`;
const targetPath = path.join(BOOKS_DIR, targetFilename);
// 保存为 TXT
fs.writeFileSync(targetPath, text, 'utf8');
// 记录索引
const record = {
upload_id: uploadId,
original_name: file.originalname,
original_format: format.format,
original_ext: format.ext,
original_size: file.size,
converted_file: targetFilename,
title: meta.title || baseName,
author: meta.author || '未知',
uploader: meta.uploader || 'unknown',
word_count: wordCount,
chapter_count: 0,
created_at: new Date().toISOString()
};
uploadsIndex.push(record);
saveIndex();
return record;
}
/**
* 获取上传列表
*/
function listUploads(options) {
let result = [...uploadsIndex];
if (options && options.uploader) {
result = result.filter(u => u.uploader === options.uploader);
}
result.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
if (options && options.limit) {
result = result.slice(0, options.limit);
}
return result;
}
/**
* 获取支持的格式列表
*/
function getSupportedFormats() {
return Object.entries(SUPPORTED_FORMATS).map(([ext, cfg]) => ({
ext,
name: cfg.name,
mime: cfg.mime
}));
}
/**
* 统计
*/
function getStats() {
return {
total_uploads: uploadsIndex.length,
total_words: uploadsIndex.reduce((s, u) => s + (u.word_count || 0), 0),
formats: uploadsIndex.reduce((acc, u) => {
const f = u.original_format || '未知';
acc[f] = (acc[f] || 0) + 1;
return acc;
}, {})
};
}
init();
module.exports = {
processUpload,
listUploads,
detectFormat,
getSupportedFormats,
getStats
};