feat: Release v0.10 - Modular Architecture & External Config
- Refactor codebase into src/ (preprocessing, translation, assembly) - Add pipeline/ scripts for individual stages - Externalize configuration to config/config.yaml - Fix Cover Image preservation - Update documentation and manuals
This commit is contained in:
谭凯
@@ -0,0 +1,69 @@
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to sys.path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from src.preprocessing.epub_cleaner import EpubCleaner
|
||||
from src.preprocessing.text_extractor import FineGrainedExtractor
|
||||
from src.translation.manifest_manager import ManifestManager
|
||||
from src.common.data_model import BookStructure
|
||||
from src.common.utils import setup_logger, ensure_directory
|
||||
from src.common.exceptions import EpubTranslatorError
|
||||
from src.common.paths import get_work_dirs
|
||||
|
||||
logger = setup_logger("pipeline_preprocess")
|
||||
|
||||
def run_preprocess(args):
|
||||
input_path = Path(args.input_epub)
|
||||
if not input_path.exists():
|
||||
logger.error(f"Input file not found: {input_path}")
|
||||
sys.exit(1)
|
||||
|
||||
work = get_work_dirs(input_path)
|
||||
ensure_directory(work["root"])
|
||||
|
||||
structure_path = work["structure"]
|
||||
manifest_path = work["manifest"]
|
||||
|
||||
try:
|
||||
# 1. Clean / Load Structure
|
||||
if structure_path.exists() and not args.force:
|
||||
logger.info(f"Reusing existing structure: {structure_path}")
|
||||
structure = BookStructure.load(structure_path)
|
||||
else:
|
||||
logger.info("Cleaning EPUB...")
|
||||
cleaner = EpubCleaner(input_path, work["root"])
|
||||
structure_path = cleaner.clean()
|
||||
structure = BookStructure.load(structure_path)
|
||||
|
||||
# 2. Extract Text
|
||||
logger.info("Extracting text segments...")
|
||||
extractor = FineGrainedExtractor()
|
||||
entries = extractor.extract(structure)
|
||||
|
||||
# 3. Update Manifest
|
||||
logger.info(f"Updating manifest: {manifest_path}")
|
||||
manager = ManifestManager(manifest_path)
|
||||
manager.load()
|
||||
manager.add_entries(entries)
|
||||
manager.save()
|
||||
|
||||
logger.info("Preprocessing complete.")
|
||||
|
||||
except EpubTranslatorError as e:
|
||||
logger.error(f"Preprocessing failed: {e}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
logger.critical(f"Unexpected error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Step 1: Preprocessing (Clean + Extract)")
|
||||
parser.add_argument("input_epub", help="Path to input EPUB")
|
||||
parser.add_argument("--force", action="store_true", help="Force re-clean")
|
||||
|
||||
args = parser.parse_args()
|
||||
run_preprocess(args)
|
||||
@@ -0,0 +1,111 @@
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
# Add project root to sys.path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from src.translation.translator_engine import Translator
|
||||
from src.translation.llm_client import LLMClient
|
||||
from src.translation.manifest_manager import ManifestManager
|
||||
from src.preprocessing.profiler import BookProfiler
|
||||
from src.common.utils import setup_logger
|
||||
from src.common.exceptions import EpubTranslatorError
|
||||
from src.common.config import load_global_config
|
||||
from src.common.paths import get_work_dirs
|
||||
|
||||
logger = setup_logger("pipeline_translate")
|
||||
|
||||
async def run_translate(args):
|
||||
# Resolve paths
|
||||
input_path = None
|
||||
if args.input_epub:
|
||||
input_path = Path(args.input_epub)
|
||||
elif args.book_name:
|
||||
# Try to infer from work dir if book_name provided (legacy support)
|
||||
# But paths.py needs input_path to determine work dir.
|
||||
# So we really need input_epub for get_work_dirs.
|
||||
# But if the user only provides book_name, we might be in trouble with get_work_dirs logic
|
||||
# which relies on input_path.stem.
|
||||
# Let's check get_work_dirs again.
|
||||
pass
|
||||
|
||||
# Actually get_work_dirs relies on input_path.stem.
|
||||
# If the user gives only --book-name, we can't easily construct input_path
|
||||
# unless we fake it or change get_work_dirs.
|
||||
# However, pipeline instructions say input_epub is required for 02_translate.
|
||||
# checking args... 02_translate has --input-epub AND --book-name.
|
||||
|
||||
if args.input_epub:
|
||||
input_path = Path(args.input_epub)
|
||||
else:
|
||||
logger.error("Must provide --input-epub")
|
||||
sys.exit(1)
|
||||
|
||||
work = get_work_dirs(input_path)
|
||||
manifest_path = work["manifest"]
|
||||
|
||||
if not manifest_path.exists():
|
||||
logger.error(f"Manifest not found: {manifest_path}. Run Step 1 first.")
|
||||
sys.exit(1)
|
||||
|
||||
# Load Config
|
||||
config = load_global_config()
|
||||
llm_conf = config.get("llm", {})
|
||||
trans_conf = config.get("translation", {})
|
||||
|
||||
api_key = llm_conf.get("api_key")
|
||||
if not api_key:
|
||||
logger.error("OPENAI_API_KEY not found in env or config.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
# Load Manifest
|
||||
manager = ManifestManager(manifest_path)
|
||||
manager.load()
|
||||
|
||||
# Init components
|
||||
# Allow CLI args to override config if needed (not implemented yet, taking config partial priority)
|
||||
llm = LLMClient(
|
||||
api_key=api_key,
|
||||
base_url=llm_conf.get("base_url"),
|
||||
model=args.model if args.model else llm_conf.get("model", "gpt-3.5-turbo"),
|
||||
requests_per_minute=llm_conf.get("requests_per_minute", 60),
|
||||
|
||||
concurrent_requests=llm_conf.get("concurrent_requests", 5),
|
||||
chunk_dir=work["chunks"]
|
||||
)
|
||||
|
||||
# Profile
|
||||
profiler = BookProfiler(llm)
|
||||
profile = await profiler.analyze(manager.entries)
|
||||
logger.info(f"Book Profile: {profile.title} ({profile.genre})")
|
||||
|
||||
# Translate
|
||||
target_chunk_size = trans_conf.get("chunk_size", 4000)
|
||||
concurrent_reqs = llm_conf.get("concurrent_requests", 5)
|
||||
translator = Translator(llm, chunk_size=target_chunk_size, max_concurrent=concurrent_reqs)
|
||||
await translator.translate(manager.entries, profile)
|
||||
|
||||
# Save final state
|
||||
manager.save()
|
||||
await llm.close()
|
||||
|
||||
logger.info("Translation complete.")
|
||||
|
||||
except EpubTranslatorError as e:
|
||||
logger.error(f"Translation failed: {e}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
logger.critical(f"Unexpected error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Step 2: Translation")
|
||||
parser.add_argument("--input-epub", help="Path to original EPUB (to derive book name)")
|
||||
parser.add_argument("--book-name", help="Book name (folder name in work/)")
|
||||
parser.add_argument("--model", default=None, help="LLM Model (overrides config)")
|
||||
|
||||
args = parser.parse_args()
|
||||
asyncio.run(run_translate(args))
|
||||
@@ -0,0 +1,106 @@
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to sys.path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from src.assembly.backfiller import BackfillEngine
|
||||
from src.assembly.builder import BilingualBuilder
|
||||
from src.translation.manifest_manager import ManifestManager
|
||||
from src.translation.llm_client import LLMClient
|
||||
from src.common.data_model import BookStructure
|
||||
from src.common.exceptions import EpubTranslatorError
|
||||
from src.common.utils import setup_logger, ensure_directory
|
||||
from src.common.paths import get_work_dirs
|
||||
from src.common.config import load_global_config
|
||||
from dotenv import load_dotenv
|
||||
|
||||
logger = setup_logger("pipeline_assemble")
|
||||
|
||||
async def run_assemble(args):
|
||||
input_path = Path(args.input_epub)
|
||||
if not input_path.exists():
|
||||
logger.error(f"Input file not found: {input_path}")
|
||||
sys.exit(1)
|
||||
|
||||
work = get_work_dirs(input_path)
|
||||
work_root = work["root"]
|
||||
structure_path = work["structure"]
|
||||
manifest_path = work["manifest"]
|
||||
|
||||
if not structure_path.exists() or not manifest_path.exists():
|
||||
logger.error("Missing structure or manifest. Run Step 1.")
|
||||
sys.exit(1)
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
ensure_directory(output_dir)
|
||||
|
||||
# Load Config for LLM (Optional for repair)
|
||||
load_dotenv()
|
||||
config = load_global_config()
|
||||
llm_conf = config.get("llm", {})
|
||||
api_key = llm_conf.get("api_key") or os.getenv("OPENAI_API_KEY") # Ensure env priority
|
||||
|
||||
llm_client = None
|
||||
if api_key:
|
||||
logger.info("Initializing LLM Client for placeholder repair...")
|
||||
llm_client = LLMClient(
|
||||
api_key=api_key,
|
||||
base_url=llm_conf.get("base_url"),
|
||||
model=llm_conf.get("model", "gpt-3.5-turbo"),
|
||||
requests_per_minute=llm_conf.get("requests_per_minute", 60),
|
||||
concurrent_requests=llm_conf.get("concurrent_requests", 5),
|
||||
chunk_dir=work["chunks"]
|
||||
)
|
||||
else:
|
||||
logger.warning("No API Key found. Placeholder repair will be disabled.")
|
||||
|
||||
try:
|
||||
# Load Data
|
||||
structure = BookStructure.load(structure_path)
|
||||
manager = ManifestManager(manifest_path)
|
||||
manager.load()
|
||||
|
||||
# Backfill
|
||||
logger.info(f"Backfilling translations (Mode: {args.mode})...")
|
||||
backfiller = BackfillEngine(llm_client=llm_client)
|
||||
updated_structure = await backfiller.backfill(structure, manager.entries, mode=args.mode)
|
||||
|
||||
if llm_client:
|
||||
await llm_client.close()
|
||||
|
||||
# Build
|
||||
logger.info("Building EPUB...")
|
||||
builder = BilingualBuilder(work_root, original_epub_path=input_path)
|
||||
|
||||
if args.mode == "bilingual":
|
||||
output_filename = f"bilingual_{input_path.stem}.epub"
|
||||
else:
|
||||
output_filename = f"translated_{input_path.stem}.epub"
|
||||
|
||||
output_path = output_dir / output_filename
|
||||
|
||||
builder.build(updated_structure, output_path)
|
||||
logger.info(f"Assembly complete. Output: {output_path}")
|
||||
|
||||
except EpubTranslatorError as e:
|
||||
logger.error(f"Assembly failed: {e}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
logger.critical(f"Unexpected error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Step 3: Assembly (Backfill + Build)")
|
||||
parser.add_argument("input_epub", help="Path to original EPUB")
|
||||
parser.add_argument("--output-dir", default="output", help="Output directory")
|
||||
parser.add_argument("--bilingual", action="store_true", help="Output bilingual version (default target only)")
|
||||
|
||||
args = parser.parse_args()
|
||||
args.mode = "bilingual" if args.bilingual else "target_only"
|
||||
import asyncio
|
||||
asyncio.run(run_assemble(args))
|
||||
Reference in New Issue
Block a user