134 lines
7.4 KiB
Python
134 lines
7.4 KiB
Python
"""Complete-book agent translation harness; no API calls or partial final EPUBs."""
|
|
import argparse
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from src.common.data_model import BookStructure, ManifestEntry
|
|
from src.preprocessing.epub_cleaner import EpubCleaner
|
|
from src.preprocessing.text_extractor import FineGrainedExtractor
|
|
from src.preprocessing.navigation import prepare_navigation
|
|
from src.translation.glossary import BookGlossary
|
|
from src.translation.translator_engine import Translator
|
|
from src.assembly.restoration_engine import RestorationEngine
|
|
from src.assembly.backfiller import BackfillEngine
|
|
from src.assembly.builder import BilingualBuilder
|
|
from types import SimpleNamespace
|
|
|
|
|
|
def unit_digest(entry):
|
|
data = [entry.entry_id, entry.original_text, entry.placeholders, entry.context]
|
|
return hashlib.sha256(json.dumps(data, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
|
|
|
|
|
def save(path, value):
|
|
temporary = path.with_suffix('.tmp')
|
|
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2))
|
|
temporary.replace(path)
|
|
|
|
|
|
def load_entries(work):
|
|
return [ManifestEntry.model_validate(e) for e in json.loads((work/'manifest.json').read_text())]
|
|
|
|
|
|
def verify_source(source, work):
|
|
structure = BookStructure.load(work/'book_structure.json')
|
|
if structure.source_sha256 != hashlib.sha256(source.read_bytes()).hexdigest():
|
|
raise ValueError('Source changed; prepare a fresh isolated work directory')
|
|
return structure
|
|
|
|
|
|
async def run(args):
|
|
work = args.work
|
|
work.mkdir(parents=True, exist_ok=True)
|
|
if args.action == 'prepare':
|
|
previous = json.loads((work/'agent-run.json').read_text()) if (work/'agent-run.json').exists() else None
|
|
if previous and previous.get('model') != args.model:
|
|
raise ValueError('Model changed; use an isolated work directory to preserve translation provenance')
|
|
structure = BookStructure.load(EpubCleaner(args.source, work).clean())
|
|
entries = FineGrainedExtractor(complete_book=True).extract(structure)
|
|
prepare_navigation(args.source, structure, entries)
|
|
if previous and (work/'manifest.json').exists():
|
|
old = {e.entry_id: e for e in load_entries(work)}
|
|
for e in entries:
|
|
if e.entry_id in old and unit_digest(e) == unit_digest(old[e.entry_id]):
|
|
e.translated_text = old[e.entry_id].translated_text
|
|
e.validation_errors = old[e.entry_id].validation_errors
|
|
structure.save(work/'book_structure.json')
|
|
save(work/'manifest.json', [e.model_dump() for e in entries])
|
|
batches = []; current = []; size = 0
|
|
for e in entries:
|
|
if e.translated_text and not e.validation_errors:
|
|
continue
|
|
if current and size + len(e.original_text) > args.batch_chars:
|
|
batches.append(current); current = []; size = 0
|
|
current.append({'entry_id': e.entry_id, 'source_sha256': unit_digest(e),
|
|
'original_text': e.original_text, 'placeholders': e.placeholders, 'context': e.context})
|
|
size += len(e.original_text)
|
|
if current: batches.append(current)
|
|
directory = work/'batches'; directory.mkdir(exist_ok=True)
|
|
for index, batch in enumerate(batches): save(directory/f'{index:04d}.json', batch)
|
|
save(work/'agent-run.json', {'model': args.model, 'source_sha256': structure.source_sha256,
|
|
'total': len(entries), 'pending_batches': len(batches), 'batch_files': [f'{i:04d}.json' for i in range(len(batches))]})
|
|
print(f'Prepared {len(entries)} complete-book units; {len(batches)} pending batches; {work}')
|
|
return
|
|
structure = verify_source(args.source, work)
|
|
metadata = json.loads((work/'agent-run.json').read_text())
|
|
if metadata['model'] != args.model:
|
|
raise ValueError('Model differs from the prepared agent run')
|
|
entries = load_entries(work)
|
|
glossary = BookGlossary.load(args.glossary) if args.glossary else BookGlossary()
|
|
validator = Translator(SimpleNamespace(model=args.model), glossary=glossary)
|
|
if args.action == 'apply':
|
|
if not args.result: raise ValueError('apply needs --result')
|
|
by_id = {e.entry_id: e for e in entries}; results = json.loads(args.result.read_text()); seen = set()
|
|
# Check all identifiers/provenance before changing any accepted translation.
|
|
for result in results:
|
|
key = result['entry_id']
|
|
if key in seen or key not in by_id or result.get('source_sha256') != unit_digest(by_id[key]):
|
|
raise ValueError(f'Duplicate, unknown or stale result: {key}')
|
|
seen.add(key)
|
|
rejected = []
|
|
for result in results:
|
|
e = by_id[result['entry_id']]; text = result.get('translated_text')
|
|
errors = validator._errors(e, text)
|
|
if errors: rejected.append({'entry_id': e.entry_id, 'errors': errors})
|
|
# Invalid retries must not destroy an already accepted translation.
|
|
if errors and e.translated_text and not validator._errors(e, e.translated_text):
|
|
continue
|
|
e.validation_errors = errors
|
|
e.translated_text = text if not errors else None
|
|
e.translated_html = None
|
|
save(work/'manifest.json', [e.model_dump() for e in entries])
|
|
save(work/'apply-report.json', {'attempted': len(results), 'rejected': rejected,
|
|
'accepted': sum(bool(e.translated_text) and not e.validation_errors for e in entries)})
|
|
print(f'Accepted {sum(bool(e.translated_text) and not e.validation_errors for e in entries)}/{len(entries)} units')
|
|
return 1 if rejected else 0
|
|
if any(validator._errors(e, e.translated_text) for e in entries):
|
|
raise ValueError('Incomplete/invalid book; inspect manifest and apply selective repairs before building')
|
|
assert await RestorationEngine().restore_entries(entries) == len(entries)
|
|
save(work/'manifest.json', [e.model_dump() for e in entries])
|
|
output = args.output_dir or work/'output'; output.mkdir(parents=True, exist_ok=True)
|
|
for mode, prefix in [('target_only', 'translated_'), ('bilingual', 'bilingual_')]:
|
|
copy = structure.model_copy(deep=True)
|
|
await BackfillEngine().backfill(copy, entries, mode=mode)
|
|
BilingualBuilder(work, args.source).build(copy, output/(prefix+args.source.name))
|
|
save(work/'translation-provenance.json', {'model': args.model, 'source_sha256': structure.source_sha256,
|
|
'total': len(entries), 'accepted': len(entries), 'provider': 'agent-authored manifest'})
|
|
print(f'Built both full-book modes: {len(entries)}/{len(entries)}; {output}')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=['prepare', 'apply', 'build']); parser.add_argument('source', type=Path)
|
|
parser.add_argument('--work', required=True, type=Path); parser.add_argument('--result', type=Path)
|
|
parser.add_argument('--glossary', type=Path); parser.add_argument('--output-dir', type=Path)
|
|
parser.add_argument('--model', default='agent/luna'); parser.add_argument('--batch-chars', type=int, default=16000)
|
|
args = parser.parse_args()
|
|
if args.batch_chars < 1: parser.error('--batch-chars must be positive')
|
|
return asyncio.run(run(args)) or 0
|
|
|
|
if __name__ == '__main__': sys.exit(main())
|