88 lines
3.1 KiB
Python
88 lines
3.1 KiB
Python
"""
|
|
Quality Manager Module
|
|
|
|
Responsible for evaluating translation quality and deciding on re-translation.
|
|
"""
|
|
|
|
import json
|
|
import random
|
|
from typing import List, Dict, Any, Tuple
|
|
from loguru import logger
|
|
from .manifest_manager import ManifestItem
|
|
from .llm_client import LLMClient
|
|
|
|
class QualityManager:
|
|
def __init__(self, config: Dict, llm_client: LLMClient):
|
|
self.config = config
|
|
self.llm_client = llm_client
|
|
self.qc_config = config['translation'].get('quality_control', {})
|
|
self.pass_score = self.qc_config.get('pass_score', 7)
|
|
self.sample_size = self.qc_config.get('sample_size', 2)
|
|
|
|
async def evaluate_chunk(self, chunk: List[ManifestItem]) -> Tuple[bool, int, str]:
|
|
"""
|
|
Evaluate a chunk of translations.
|
|
|
|
Returns:
|
|
(passed: bool, average_score: int, reason: str)
|
|
"""
|
|
if not self.qc_config.get('enabled', False):
|
|
return True, 10, "QC Disabled"
|
|
|
|
# 1. Sample items
|
|
# Filter for items that actually have content and translations
|
|
valid_items = [item for item in chunk if item.translation and len(item.clean_text) > 20]
|
|
|
|
if not valid_items:
|
|
return True, 10, "No valid items to sample"
|
|
|
|
sample_items = random.sample(valid_items, min(len(valid_items), self.sample_size))
|
|
|
|
# 2. Build Prompt
|
|
prompt = self._build_evaluation_prompt(sample_items)
|
|
|
|
# 3. Call LLM (Smart)
|
|
try:
|
|
response = await self.llm_client.raw_chat_completion(
|
|
system_prompt="You are a professional translation editor.",
|
|
user_prompt=prompt,
|
|
model_type="smart"
|
|
)
|
|
|
|
# 4. Parse JSON
|
|
# Clean potential markdown
|
|
json_str = response.strip()
|
|
if "```json" in json_str:
|
|
json_str = json_str.split("```json")[1].split("```")[0].strip()
|
|
elif "```" in json_str:
|
|
json_str = json_str.split("```")[1].split("```")[0].strip()
|
|
|
|
result = json.loads(json_str)
|
|
score = result.get('score', 0)
|
|
reason = result.get('reason', 'No reason provided')
|
|
|
|
passed = score >= self.pass_score
|
|
return passed, score, reason
|
|
|
|
except Exception as e:
|
|
logger.error(f"QC evaluation failed: {e}")
|
|
# If QC fails, we default to PASS to avoid blocking progress, but log it
|
|
return True, 0, f"QC Error: {e}"
|
|
|
|
def _build_evaluation_prompt(self, items: List[ManifestItem]) -> str:
|
|
content = ""
|
|
for i, item in enumerate(items, 1):
|
|
content += f"Item {i}:\nOriginal: {item.clean_text}\nTranslation: {item.translation}\n\n"
|
|
|
|
return f"""Please evaluate the following translations (English to Chinese).
|
|
Focus on accuracy, fluency, and terminology consistency.
|
|
|
|
Items to evaluate:
|
|
{content}
|
|
|
|
Return a JSON object with:
|
|
- \"score\": An integer from 1 to 10 (10 being perfect).
|
|
- \"reason\": A brief explanation of the score.
|
|
|
|
JSON Output:"""
|