diff --git a/doc/rc-render-context.md b/doc/rc-render-context.md index b9a292f..b525e88 100644 --- a/doc/rc-render-context.md +++ b/doc/rc-render-context.md @@ -10,7 +10,7 @@ exist fails the render. Output is written as UTF-8 (without BOM) using LF line e | Variable | Type | Description | |-------------|-------------------|-------------| -| `languages` | array of language | One entry per language, see below. The English (`en`) entry is always present and comes first; remaining entries follow, ordered by Windows language identifier. | +| `languages` | array of language | One entry per language, see below. The English (`en`) entry is always present and comes first; remaining entries follow in the order their PO files are matched. | ## Language object diff --git a/src/metaglot/__init__.py b/src/metaglot/__init__.py index 4014dc4..c3d6df7 100644 --- a/src/metaglot/__init__.py +++ b/src/metaglot/__init__.py @@ -3,6 +3,7 @@ import sys from . import pofile, render from .cli import parse, RcOpts, RcPotOpts, RcRenderOpts +from .manifest import load_manifest def main() -> None: @@ -14,7 +15,8 @@ def main() -> None: case RcOpts() as rc_opts: match rc_opts.opts: case RcPotOpts() as rc_pot_opts: - pofile.generate(rc_pot_opts.in_manifest, rc_pot_opts.out_pot) + manifest = load_manifest(rc_pot_opts.in_manifest) + pofile.generate_pot(manifest, rc_pot_opts.out_pot) case RcRenderOpts() as rc_rdr_opts: render.render( rc_rdr_opts.in_manifest, diff --git a/src/metaglot/manifest.py b/src/metaglot/manifest.py index a1653ad..372cc46 100644 --- a/src/metaglot/manifest.py +++ b/src/metaglot/manifest.py @@ -54,9 +54,7 @@ class Manifest(BaseModel, frozen=True): if duplicates: rendered = ", ".join( f"{msgid!r}" if context is None else f"{msgid!r} (context {context!r})" - for msgid, context in sorted( - duplicates, key=lambda pair: (pair[0], pair[1] or "") - ) + for msgid, context in duplicates ) raise ValueError(f"duplicated msgid and context combination(s): {rendered}") diff --git a/src/metaglot/pofile.py b/src/metaglot/pofile.py index 0608e61..1797843 100644 --- a/src/metaglot/pofile.py +++ b/src/metaglot/pofile.py @@ -1,54 +1,122 @@ import glob import logging +from dataclasses import dataclass from pathlib import Path - +from typing import Iterable, Iterator, Optional import polib - -from .manifest import Manifest, load_manifest +from .langid import PoLang +from .manifest import Manifest _POT_HEADER = { "Project-Id-Version": "PACKAGE VERSION", + "POT-Creation-Date": "YEAR-MO-DA HO:MI+ZONE", "PO-Revision-Date": "YEAR-MO-DA HO:MI+ZONE", "Last-Translator": "FULL NAME ", "Language-Team": "LANGUAGE ", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit", + "X-Generator": "MetaGlot/polib", } -def collect_po_files(patterns: list[Path]) -> list[Path]: - files: list[Path] = [] - seen: set[Path] = set() +@dataclass(frozen=True) +class LanguagePack: + """Resolved translations of one language against a manifest.""" + + lang: PoLang + """The language of this pack.""" + + translations: dict[str, str] + """The resolved text of every manifest entry, keyed by manifest entry key.""" + + +def resolve_glob_files(patterns: Iterable[Path]) -> Iterator[Path]: + """Lazily iterate the files matched by the given glob patterns. + + A pattern matching no file only produces a warning. A file matched by + several patterns is yielded once per matching pattern. + + :param patterns: The glob patterns to match. + :return: An iterator of resolved file paths. + """ for pattern in patterns: - matched = sorted(glob.glob(str(pattern))) - if not matched: - raise ValueError(f"the pattern '{pattern}' matched no PO file") + matched = tuple(Path(p) for p in glob.glob(str(pattern))) + if len(matched) == 0: + logging.warning("Pattern %s does not match any file.", pattern) for name in matched: - file = Path(name) - resolved = file.resolve() - if resolved not in seen: - seen.add(resolved) - files.append(file) - return files + yield name.resolve() -def load_po(path: Path) -> tuple[str, dict[str, str]]: - try: - po = polib.pofile(str(path)) - except (OSError, IOError) as exc: - raise RuntimeError(f"failed to read PO file '{path}': {exc}") from exc - locale = po.metadata.get("Language") or path.stem - translations = { - entry.msgid: entry.msgstr - for entry in po - if entry.msgstr and "fuzzy" not in entry.flags - } - return locale, translations +def resolve_translations( + manifest: Manifest, po_paths: Iterator[Path] +) -> dict[PoLang, LanguagePack]: + """Resolve the translations of the given PO files against a manifest. + + Each PO file is loaded lazily and resolved to a :class:`LanguagePack` + holding the text of every manifest entry. Entries that are missing, + untranslated or marked fuzzy fall back to their source string. + + :param manifest: The manifest describing all translatable entries. + :param po_paths: The PO file paths to resolve. + :return: A mapping from each language to its resolved language pack. + :raises ValueError: If two PO files describe the same language, or a PO + file contains entries unknown to the manifest. + """ + # Collect every (msgid, context) pair the manifest defines. PO entries + # matching none of them are rejected below. + expected = {(entry.msgid, entry.context) for entry in manifest.strings.values()} + # The resolved packs being built, keyed by language. + packs: dict[PoLang, LanguagePack] = {} + # Which file introduced each language, for duplicate reporting. + sources: dict[PoLang, Path] = {} + for path in po_paths: + # Load the PO file and identify its language. + po = _load_po(path) + lang = _extract_lang(path, po) + + # One language must come from exactly one PO file. + if lang in packs: + raise ValueError( + f"duplicate language '{lang}' found in " + f"'{sources[lang].name}' and '{path.name}'" + ) + sources[lang] = path + + # Index the entries by their (msgid, msgctxt) pair, so that lookups + # and set operations share one pass over the file. Obsolete entries + # are ignored, mirroring how polib's find() treats them. + index: dict[tuple[str, Optional[str]], polib.POEntry] = { + (entry.msgid, entry.msgctxt): entry for entry in po if not entry.obsolete + } + + # Reject entries the manifest does not define. + unknown = index.keys() - expected + if unknown: + rendered = ", ".join( + f"{msgid!r}" if context is None else f"{msgid!r} (context {context!r})" + for msgid, context in unknown + ) + raise ValueError( + f"'{path.name}' contains entry(s) unknown to the manifest: {rendered}" + ) + + # Resolve every manifest entry against the index. Entries that are + # missing, untranslated or fuzzy fall back to their source string. + translations = { + key: _translated_text(index.get((entry.msgid, entry.context)), entry.msgid) + for key, entry in manifest.strings.items() + } + packs[lang] = LanguagePack(lang, translations) + return packs -def generate(manifest_path: Path, output_path: Path) -> None: - manifest: Manifest = load_manifest(manifest_path) +def generate_pot(manifest: Manifest, output_path: Path) -> None: + """Generate a POT template file from a manifest. + + :param manifest: The manifest holding the translatable strings. + :param output_path: The path of the POT file to write. + """ po = polib.POFile() po.metadata = _POT_HEADER for _, entry in manifest.strings.items(): @@ -60,6 +128,57 @@ def generate(manifest_path: Path, output_path: Path) -> None: comment=entry.comment or None, ) ) - output_path.parent.mkdir(parents=True, exist_ok=True) po.save(str(output_path)) - logging.info("wrote %d entries to %s", len(po), output_path) + logging.info("Wrote %d entries to POT file '%s'.", len(po), output_path) + + +def _load_po(path: Path) -> polib.POFile: + """Load and parse a PO file. + + :param path: The path of the PO file to load. + :return: The parsed PO file. + :raises RuntimeError: If the file cannot be read or parsed. + """ + try: + return polib.pofile(str(path)) + except (OSError, IOError) as exc: + raise RuntimeError(f"failed to read PO file '{path}': {exc}") from exc + + +def _extract_lang(path: Path, po: polib.POFile) -> PoLang: + """Extract the language of a loaded PO file. + + The ``Language`` metadata field is tried first, then the file name + without extension. + + :param path: The path of the PO file, used for fallback and reporting. + :param po: The loaded PO file. + :return: The language of the PO file. + :raises RuntimeError: If no valid language can be extracted. + """ + metadata_language = po.metadata.get("Language") + if metadata_language is not None: + try: + return PoLang(metadata_language) + except ValueError: + pass + + try: + return PoLang(path.stem) + except ValueError: + pass + + raise RuntimeError(f"can't fetch valid language id in PO file '{path}'") + + +def _translated_text(entry: Optional[polib.POEntry], fallback: str) -> str: + """Return the usable translation of an entry. + + :param entry: The PO entry to read, or ``None`` when absent. + :param fallback: The source string used when the entry is missing, + untranslated or marked fuzzy. + :return: The usable translation. + """ + if entry is None or entry.msgstr == "" or entry.fuzzy: + return fallback + return entry.msgstr diff --git a/src/metaglot/render.py b/src/metaglot/render.py index 8e53cc8..280326a 100644 --- a/src/metaglot/render.py +++ b/src/metaglot/render.py @@ -5,8 +5,9 @@ from liquid import Environment, StrictUndefined from liquid.exceptions import LiquidError from . import winlang +from .langid import PoLang from .manifest import Manifest, load_manifest -from .pofile import collect_po_files, load_po +from .pofile import LanguagePack, resolve_glob_files, resolve_translations def _rc_escape(value: str) -> str: @@ -22,10 +23,9 @@ def _create_environment() -> Environment: def _language_context( locale: str, language: winlang.WindowsLanguage, - translations: dict[str, str], + pack: LanguagePack, manifest: Manifest, ) -> dict: - entries = manifest.strings.items() return { "name": language.name, "locale": locale, @@ -37,60 +37,51 @@ def _language_context( { "key": key, "msgid": entry.msgid, - "text": translations.get(entry.msgid, entry.msgid), + "text": pack.translations[key], "context": entry.context, } - for key, entry in entries + for key, entry in manifest.strings.items() ], - "named": { - key: translations.get(entry.msgid, entry.msgid) - for key, entry in entries - }, + "named": pack.translations, } -def render(manifest_path: Path, po_patterns: list[Path], template_path: Path, output_path: Path) -> None: - manifest = load_manifest(manifest_path) - known = {entry.msgid for entry in manifest.strings.values()} +def _lookup_windows_language(lang: PoLang) -> winlang.WindowsLanguage: + try: + return winlang.lookup(lang.value) + except KeyError: + raise ValueError( + f"language '{lang}' has no Windows language mapping" + ) from None - collected = [] - locale_sources: dict[str, Path] = {} - for po_path in collect_po_files(po_patterns): - locale, translations = load_po(po_path) - unknown = sorted(set(translations) - known) - if unknown: - raise ValueError( - f"{po_path.name} contains unknown msgid(s): {', '.join(unknown)}" - ) - if locale in locale_sources: - raise ValueError( - f"duplicate locale '{locale}' found in '{locale_sources[locale].name}' and '{po_path.name}'" - ) - locale_sources[locale] = po_path - try: - language = winlang.lookup(locale) - except KeyError: - raise ValueError( - f"locale '{locale}' (from {po_path.name}) has no Windows language mapping" - ) from None - collected.append((locale, language, translations)) + +def render( + manifest_path: Path, + po_patterns: list[Path], + template_path: Path, + output_path: Path, +) -> None: + manifest = load_manifest(manifest_path) + packs = resolve_translations(manifest, resolve_glob_files(po_patterns)) english = winlang.lookup("en") - english_po = next( - (item for item in collected if item[1].langid == english.langid), None + resolved = [(pack, _lookup_windows_language(pack.lang)) for pack in packs.values()] + english_item = next( + (item for item in resolved if item[1].langid == english.langid), None ) - others = sorted( - (item for item in collected if item[1].langid != english.langid), - key=lambda item: item[1].langid, - ) - ordered = [ - ("en", english, english_po[2] if english_po else {}), - *others, - ] + others = [item for item in resolved if item[1].langid != english.langid] + if english_item is None: + english_pack = LanguagePack( + PoLang("en"), + {key: entry.msgid for key, entry in manifest.strings.items()}, + ) + ordered = [(english_pack, english), *others] + else: + ordered = [english_item, *others] languages = [ - _language_context(locale, language, translations, manifest) - for locale, language, translations in ordered + _language_context(pack.lang.value, language, pack, manifest) + for pack, language in ordered ] if not template_path.is_file():