SarasasChipWorkshop/metaglot
1
0
Fork 0
Code Activity

refactor: refactor pofile module

Browse Source
This commit is contained in:
yyc12345
2026-09-15 09:59:10 +08:00
parent cea485f85c
commit d0bca69006
5 changed files with 191 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/rc-render-context.md
+1 -1
View File
@@ -10,7 +10,7 @@ exist fails the render. Output is written as UTF-8 (without BOM) using LF line e
| Variable | Type | Description |
|-------------|-------------------|-------------|
| `languages` | array of language | One entry per language, see below. The English (`en`) entry is always present and comes first; remaining entries follow, ordered by Windows language identifier. |
| `languages` | array of language | One entry per language, see below. The English (`en`) entry is always present and comes first; remaining entries follow in the order their PO files are matched. |
## Language object
src/metaglot/__init__.py
+3 -1
View File
@@ -3,6 +3,7 @@ import sys
from . import pofile, render
from .cli import parse, RcOpts, RcPotOpts, RcRenderOpts
from .manifest import load_manifest
def main() -> None:
@@ -14,7 +15,8 @@ def main() -> None:
case RcOpts() as rc_opts:
match rc_opts.opts:
case RcPotOpts() as rc_pot_opts:
pofile.generate(rc_pot_opts.in_manifest, rc_pot_opts.out_pot)
manifest = load_manifest(rc_pot_opts.in_manifest)
pofile.generate_pot(manifest, rc_pot_opts.out_pot)
case RcRenderOpts() as rc_rdr_opts:
render.render(
rc_rdr_opts.in_manifest,
src/metaglot/manifest.py
+1 -3
View File
@@ -54,9 +54,7 @@ class Manifest(BaseModel, frozen=True):
if duplicates:
rendered = ", ".join(
f"{msgid!r}" if context is None else f"{msgid!r} (context {context!r})"
for msgid, context in sorted(
duplicates, key=lambda pair: (pair[0], pair[1] or "")
)
for msgid, context in duplicates
)
raise ValueError(f"duplicated msgid and context combination(s): {rendered}")
src/metaglot/pofile.py
+150 -31
View File
@@ -1,54 +1,122 @@
import glob
import logging
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, Iterator, Optional
import polib
from .manifest import Manifest, load_manifest
from .langid import PoLang
from .manifest import Manifest
_POT_HEADER = {
"Project-Id-Version": "PACKAGE VERSION",
"POT-Creation-Date": "YEAR-MO-DA HO:MI+ZONE",
"PO-Revision-Date": "YEAR-MO-DA HO:MI+ZONE",
"Last-Translator": "FULL NAME <EMAIL@ADDRESS>",
"Language-Team": "LANGUAGE <LL@li.org>",
"MIME-Version": "1.0",
"Content-Type": "text/plain; charset=UTF-8",
"Content-Transfer-Encoding": "8bit",
"X-Generator": "MetaGlot/polib",
}
def collect_po_files(patterns: list[Path]) -> list[Path]:
files: list[Path] = []
seen: set[Path] = set()
@dataclass(frozen=True)
class LanguagePack:
"""Resolved translations of one language against a manifest."""
lang: PoLang
"""The language of this pack."""
translations: dict[str, str]
"""The resolved text of every manifest entry, keyed by manifest entry key."""
def resolve_glob_files(patterns: Iterable[Path]) -> Iterator[Path]:
"""Lazily iterate the files matched by the given glob patterns.
A pattern matching no file only produces a warning. A file matched by
several patterns is yielded once per matching pattern.
:param patterns: The glob patterns to match.
:return: An iterator of resolved file paths.
"""
for pattern in patterns:
matched = sorted(glob.glob(str(pattern)))
if not matched:
raise ValueError(f"the pattern '{pattern}' matched no PO file")
matched = tuple(Path(p) for p in glob.glob(str(pattern)))
if len(matched) == 0:
logging.warning("Pattern %s does not match any file.", pattern)
for name in matched:
file = Path(name)
resolved = file.resolve()
if resolved not in seen:
seen.add(resolved)
files.append(file)
return files
yield name.resolve()
def load_po(path: Path) -> tuple[str, dict[str, str]]:
try:
po = polib.pofile(str(path))
except (OSError, IOError) as exc:
raise RuntimeError(f"failed to read PO file '{path}': {exc}") from exc
locale = po.metadata.get("Language") or path.stem
translations = {
entry.msgid: entry.msgstr
for entry in po
if entry.msgstr and "fuzzy" not in entry.flags
}
return locale, translations
def resolve_translations(
manifest: Manifest, po_paths: Iterator[Path]
) -> dict[PoLang, LanguagePack]:
"""Resolve the translations of the given PO files against a manifest.
Each PO file is loaded lazily and resolved to a :class:`LanguagePack`
holding the text of every manifest entry. Entries that are missing,
untranslated or marked fuzzy fall back to their source string.
:param manifest: The manifest describing all translatable entries.
:param po_paths: The PO file paths to resolve.
:return: A mapping from each language to its resolved language pack.
:raises ValueError: If two PO files describe the same language, or a PO
file contains entries unknown to the manifest.
"""
# Collect every (msgid, context) pair the manifest defines. PO entries
# matching none of them are rejected below.
expected = {(entry.msgid, entry.context) for entry in manifest.strings.values()}
# The resolved packs being built, keyed by language.
packs: dict[PoLang, LanguagePack] = {}
# Which file introduced each language, for duplicate reporting.
sources: dict[PoLang, Path] = {}
for path in po_paths:
# Load the PO file and identify its language.
po = _load_po(path)
lang = _extract_lang(path, po)
# One language must come from exactly one PO file.
if lang in packs:
raise ValueError(
f"duplicate language '{lang}' found in "
f"'{sources[lang].name}' and '{path.name}'"
)
sources[lang] = path
# Index the entries by their (msgid, msgctxt) pair, so that lookups
# and set operations share one pass over the file. Obsolete entries
# are ignored, mirroring how polib's find() treats them.
index: dict[tuple[str, Optional[str]], polib.POEntry] = {
(entry.msgid, entry.msgctxt): entry for entry in po if not entry.obsolete
}
# Reject entries the manifest does not define.
unknown = index.keys() - expected
if unknown:
rendered = ", ".join(
f"{msgid!r}" if context is None else f"{msgid!r} (context {context!r})"
for msgid, context in unknown
)
raise ValueError(
f"'{path.name}' contains entry(s) unknown to the manifest: {rendered}"
)
# Resolve every manifest entry against the index. Entries that are
# missing, untranslated or fuzzy fall back to their source string.
translations = {
key: _translated_text(index.get((entry.msgid, entry.context)), entry.msgid)
for key, entry in manifest.strings.items()
}
packs[lang] = LanguagePack(lang, translations)
return packs
def generate(manifest_path: Path, output_path: Path) -> None:
manifest: Manifest = load_manifest(manifest_path)
def generate_pot(manifest: Manifest, output_path: Path) -> None:
"""Generate a POT template file from a manifest.
:param manifest: The manifest holding the translatable strings.
:param output_path: The path of the POT file to write.
"""
po = polib.POFile()
po.metadata = _POT_HEADER
for _, entry in manifest.strings.items():
@@ -60,6 +128,57 @@ def generate(manifest_path: Path, output_path: Path) -> None:
comment=entry.comment or None,
)
)
output_path.parent.mkdir(parents=True, exist_ok=True)
po.save(str(output_path))
logging.info("wrote %d entries to %s", len(po), output_path)
logging.info("Wrote %d entries to POT file '%s'.", len(po), output_path)
def _load_po(path: Path) -> polib.POFile:
"""Load and parse a PO file.
:param path: The path of the PO file to load.
:return: The parsed PO file.
:raises RuntimeError: If the file cannot be read or parsed.
"""
try:
return polib.pofile(str(path))
except (OSError, IOError) as exc:
raise RuntimeError(f"failed to read PO file '{path}': {exc}") from exc
def _extract_lang(path: Path, po: polib.POFile) -> PoLang:
"""Extract the language of a loaded PO file.
The ``Language`` metadata field is tried first, then the file name
without extension.
:param path: The path of the PO file, used for fallback and reporting.
:param po: The loaded PO file.
:return: The language of the PO file.
:raises RuntimeError: If no valid language can be extracted.
"""
metadata_language = po.metadata.get("Language")
if metadata_language is not None:
try:
return PoLang(metadata_language)
except ValueError:
pass
try:
return PoLang(path.stem)
except ValueError:
pass
raise RuntimeError(f"can't fetch valid language id in PO file '{path}'")
def _translated_text(entry: Optional[polib.POEntry], fallback: str) -> str:
"""Return the usable translation of an entry.
:param entry: The PO entry to read, or ``None`` when absent.
:param fallback: The source string used when the entry is missing,
untranslated or marked fuzzy.
:return: The usable translation.
"""
if entry is None or entry.msgstr == "" or entry.fuzzy:
return fallback
return entry.msgstr
src/metaglot/render.py
+36 -45
View File
@@ -5,8 +5,9 @@ from liquid import Environment, StrictUndefined
from liquid.exceptions import LiquidError
from . import winlang
from .langid import PoLang
from .manifest import Manifest, load_manifest
from .pofile import collect_po_files, load_po
from .pofile import LanguagePack, resolve_glob_files, resolve_translations
def _rc_escape(value: str) -> str:
@@ -22,10 +23,9 @@ def _create_environment() -> Environment:
def _language_context(
locale: str,
language: winlang.WindowsLanguage,
translations: dict[str, str],
pack: LanguagePack,
manifest: Manifest,
) -> dict:
entries = manifest.strings.items()
return {
"name": language.name,
"locale": locale,
@@ -37,60 +37,51 @@ def _language_context(
{
"key": key,
"msgid": entry.msgid,
"text": translations.get(entry.msgid, entry.msgid),
"text": pack.translations[key],
"context": entry.context,
}
for key, entry in entries
for key, entry in manifest.strings.items()
],
"named": {
key: translations.get(entry.msgid, entry.msgid)
for key, entry in entries
},
"named": pack.translations,
}
def render(manifest_path: Path, po_patterns: list[Path], template_path: Path, output_path: Path) -> None:
manifest = load_manifest(manifest_path)
known = {entry.msgid for entry in manifest.strings.values()}
def _lookup_windows_language(lang: PoLang) -> winlang.WindowsLanguage:
try:
return winlang.lookup(lang.value)
except KeyError:
raise ValueError(
f"language '{lang}' has no Windows language mapping"
) from None
collected = []
locale_sources: dict[str, Path] = {}
for po_path in collect_po_files(po_patterns):
locale, translations = load_po(po_path)
unknown = sorted(set(translations) - known)
if unknown:
raise ValueError(
f"{po_path.name} contains unknown msgid(s): {', '.join(unknown)}"
)
if locale in locale_sources:
raise ValueError(
f"duplicate locale '{locale}' found in '{locale_sources[locale].name}' and '{po_path.name}'"
)
locale_sources[locale] = po_path
try:
language = winlang.lookup(locale)
except KeyError:
raise ValueError(
f"locale '{locale}' (from {po_path.name}) has no Windows language mapping"
) from None
collected.append((locale, language, translations))
def render(
manifest_path: Path,
po_patterns: list[Path],
template_path: Path,
output_path: Path,
) -> None:
manifest = load_manifest(manifest_path)
packs = resolve_translations(manifest, resolve_glob_files(po_patterns))
english = winlang.lookup("en")
english_po = next(
(item for item in collected if item[1].langid == english.langid), None
resolved = [(pack, _lookup_windows_language(pack.lang)) for pack in packs.values()]
english_item = next(
(item for item in resolved if item[1].langid == english.langid), None
)
others = sorted(
(item for item in collected if item[1].langid != english.langid),
key=lambda item: item[1].langid,
)
ordered = [
("en", english, english_po[2] if english_po else {}),
*others,
]
others = [item for item in resolved if item[1].langid != english.langid]
if english_item is None:
english_pack = LanguagePack(
PoLang("en"),
{key: entry.msgid for key, entry in manifest.strings.items()},
)
ordered = [(english_pack, english), *others]
else:
ordered = [english_item, *others]
languages = [
_language_context(locale, language, translations, manifest)
for locale, language, translations in ordered
_language_context(pack.lang.value, language, pack, manifest)
for pack, language in ordered
]
if not template_path.is_file():