commit 8e66c44119f8dd45d6a0b78e528a3b9dd84e7a1b
parent d45c117af6ce5cc347a162ec06e43ba71f1d736a
Author: Florian Dold <dold@taler.net>
Date: Fri, 4 Sep 2026 14:16:33 +0200
pogen: import library translation catalogues
Add catalogDependencies configuration and emit static catalogue imports.
Record message ownership metadata so composition preserves application
precedence and recomputes coverage without double-counting keys.
Correct the README description of TypeScript source extraction and
document library catalogues and locale fallback.
Diffstat:
4 files changed, 227 insertions(+), 18 deletions(-)
diff --git a/packages/pogen/README.md b/packages/pogen/README.md
@@ -6,8 +6,11 @@ emits a `strings.ts` catalogue the application imports at runtime.
## Invocation
-Run from the root of an NPM package. The input files are whatever the TypeScript
-compiler would use, taken from the package's `tsconfig.json`.
+Run from the root of an NPM package. The input files are the non-declaration,
+non-external source files reached by a TypeScript program created from the
+package's `tsconfig.json`. Normal package dependencies usually resolve to
+compiled declaration files and are therefore not extracted; use catalogue
+dependencies for strings owned by a library.
```shell
pogen extract # sources -> src/i18n/<domain>.pot
@@ -27,21 +30,66 @@ Configuration lives under `pogen` in the package's `package.json`:
{
"pogen": {
"domain": "taler-merchant-webui",
+ "catalogDependencies": ["@gnu-taler/web-util/i18n"],
"minimumCoverage": 85,
"requiredLanguages": ["de", "de-CH", "fr", "it"]
}
}
```
-`domain` names the template: `src/i18n/<domain>.pot`. `minimumCoverage` is an
-optional integer percentage used by `pogen check`; it defaults to 85, matching
-the browser language auto-selection threshold. `requiredLanguages` is an
-optional list of language tags whose catalogues must exist and be 100%
-complete; tags are matched case-insensitively and treat `_` like `-`. Paths are
-otherwise fixed — `src/i18n/*.po` in, `src/i18n/strings.ts` out. Two optional
-files are read if present: `src/i18n/poheader` (replaces the default `.pot`
-header) and `src/i18n/strings-prelude` (replaces the preamble of the emitted
-`strings.ts`).
+`domain` names the template: `src/i18n/<domain>.pot`. `catalogDependencies` is
+an optional ordered list of module specifiers. Each module must export a named
+`strings` catalogue generated by `pogen`; `emit` adds static imports and
+composes the catalogues without copying their messages into the application's
+`.po` files. Later dependencies override earlier dependencies, and the
+application catalogue always has final priority.
+
+`minimumCoverage` is an optional integer percentage used by `pogen check`; it
+defaults to 85, matching the browser language auto-selection threshold.
+`requiredLanguages` is an optional list of language tags whose catalogues must
+exist and be 100% complete; tags are matched case-insensitively and treat `_`
+like `-`. Paths are otherwise fixed — `src/i18n/*.po` in,
+`src/i18n/strings.ts` out. Two optional files are read if present:
+`src/i18n/poheader` (replaces the default `.pot` header) and
+`src/i18n/strings-prelude` (replaces the preamble of the emitted `strings.ts`).
+
+## Generated catalogue metadata
+
+Each emitted language catalogue has a `_pogen` object. It is metadata for
+catalogue composition and is not consumed by Jed:
+
+- `total` is the number of non-optional message IDs in that catalogue.
+- `translated` is the number of those IDs with a usable translation.
+- `untranslated` lists non-optional IDs without a usable translation. This
+ includes empty translations, incomplete plurals, and fuzzy entries. Pogen
+ omits fuzzy messages from `locale_data`, so this list also preserves the fact
+ that the catalogue owns them.
+- `optional` lists IDs whose gettext translations are not required. These are
+ source-controlled `i18n.fixed()` and `i18n.fixedOnly()` messages marked as
+ optional during extraction, so an empty `.po` translation does not reduce
+ catalogue completeness.
+
+The arrays contain message IDs rather than only counts because composition
+must identify collisions. Duplicate IDs from several catalogues count once,
+and the higher-priority catalogue determines whether the resulting message is
+translated or optional. In particular, an untranslated application entry
+still owns its ID and blocks a dependency translation; otherwise an older
+library translation could silently replace wording deliberately left
+untranslated by the application.
+
+Locale fallback deliberately has different precedence. After library and
+application catalogues have been composed for each locale, a missing or
+unusable regional translation may inherit the same message from a broader
+locale. Thus an untranslated application entry blocks a library entry in the
+same `de-CH` catalogue, but can still inherit the composed `de` translation.
+
+The combined `total`, `translated`, and `completeness` values are recomputed
+from these message identities, without double-counting overlapping entries.
+
+At runtime, locale tags are normalized case-insensitively and treat `_` like
+`-`. Regional and script catalogues inherit individual missing translations
+through progressively broader locales: `zh_Hant_TW` uses `zh-Hant-TW`, then
+`zh-Hant`, then `zh`. A usable more-specific translation always wins.
> Earlier versions of this document described `pofile`, `plainI18nPackage` and
> `reactI18nPackage`, and claimed extraction was scoped to strings imported from
@@ -158,9 +206,11 @@ permissive and will not tell you a catalogue is broken.
## Notes and known limitations
-- **Extraction is not scoped to a package.** `pogen` walks every source file the
- TypeScript program reaches, which in a workspace includes sibling packages. A
- package's `.pot` may therefore contain strings that belong to a dependency.
+- **Extraction follows TypeScript resolution.** Imported sibling sources can be
+ included when TypeScript resolves them directly to `.ts` or `.tsx` files.
+ Workspace and installed packages normally resolve to external `.d.ts` files,
+ which are excluded. Library-owned translations should be exposed as a
+ generated catalogue and listed in `catalogDependencies`.
- **`completeness`** in `strings.ts` is `translated / (translated + fuzzy +
untranslated)`. It measures coverage, not quality: a msgstr that merely repeats
the English counts as translated.
diff --git a/packages/pogen/src/check.ts b/packages/pogen/src/check.ts
@@ -29,7 +29,7 @@ import * as child_process from "node:child_process";
import * as fs from "node:fs";
import * as gettextParser from "gettext-parser";
import * as glob from "glob";
-import { poToStrings } from "./po2ts.js";
+import { poToStrings, resolveCatalogDependencies } from "./po2ts.js";
/**
* The default matches `web-util`'s language auto-selection threshold. Packages
@@ -190,6 +190,7 @@ export function check(): void {
requiredLanguages = resolveRequiredLanguages(
packageJson.pogen?.requiredLanguages,
);
+ resolveCatalogDependencies(packageJson.pogen?.catalogDependencies);
} catch (e) {
console.error(e instanceof Error ? e.message : e);
process.exit(1);
diff --git a/packages/pogen/src/po2ts.test.ts b/packages/pogen/src/po2ts.test.ts
@@ -19,7 +19,12 @@ import assert from "node:assert";
import * as fs from "node:fs";
import * as path from "node:path";
import { format, resolveConfig } from "prettier";
-import { CONTEXT_DELIMITER, po2ts, poToStrings } from "./po2ts.js";
+import {
+ CONTEXT_DELIMITER,
+ po2ts,
+ poToStrings,
+ resolveCatalogDependencies,
+} from "./po2ts.js";
function header(
lang: string,
@@ -96,6 +101,12 @@ msgstr "Auf Wiedersehen"
);
// 1 of 2, not 1 of 1: the fuzzy entry renders English, so it is missing.
assert.equal(s.completeness, 50);
+ assert.deepEqual(s._pogen, {
+ total: 2,
+ translated: 1,
+ untranslated: ["Goodbye"],
+ optional: [],
+ });
});
test("an empty msgstr counts in the denominator only", () => {
@@ -130,6 +141,22 @@ msgstr ""
);
assert.deepEqual(s.locale_data.messages["Declaration"], [""]);
assert.equal(s.completeness, 100);
+ assert.deepEqual(s._pogen.optional, ["Declaration"]);
+});
+
+test("catalogue dependency configuration accepts unique module specifiers", () => {
+ assert.deepEqual(resolveCatalogDependencies(undefined), []);
+ assert.deepEqual(resolveCatalogDependencies([" @gnu-taler/web-util/i18n "]), [
+ "@gnu-taler/web-util/i18n",
+ ]);
+ assert.throws(
+ () => resolveCatalogDependencies("@gnu-taler/web-util/i18n"),
+ /array of module specifiers/,
+ );
+ assert.throws(
+ () => resolveCatalogDependencies(["shared/i18n", "shared/i18n"]),
+ /duplicates/,
+ );
});
test("a plural keeps [msgid_plural, ...msgstr] and needs every form filled in", () => {
@@ -232,6 +259,10 @@ test("po2ts formats generated catalogues with the repository's Prettier rules",
"src/i18n/strings.ts\n",
);
fs.writeFileSync(
+ path.join(projectDir, "package.json"),
+ JSON.stringify({ pogen: { domain: "test" } }),
+ );
+ fs.writeFileSync(
path.join(i18nDir, "de.po"),
header("de") +
`msgid "required"
@@ -259,3 +290,51 @@ msgstr "erforderlich"
fs.rmSync(projectDir, { recursive: true, force: true });
}
});
+
+test("po2ts emits imports and application-last catalogue composition", async () => {
+ const originalCwd = process.cwd();
+ const projectDir = fs.mkdtempSync(
+ path.join(originalCwd, ".pogen-dependency-test-"),
+ );
+ const i18nDir = path.join(projectDir, "src", "i18n");
+ fs.mkdirSync(i18nDir, { recursive: true });
+ fs.writeFileSync(
+ path.join(projectDir, "package.json"),
+ JSON.stringify({
+ pogen: {
+ domain: "test",
+ catalogDependencies: ["shared-one/i18n", "shared-two/i18n"],
+ },
+ }),
+ );
+ fs.writeFileSync(
+ path.join(i18nDir, "de.po"),
+ header("de") + `msgid "Local"\nmsgstr "Lokal"\n`,
+ );
+
+ try {
+ process.chdir(projectDir);
+ po2ts();
+ process.chdir(originalCwd);
+ const generated = fs.readFileSync(
+ path.join(i18nDir, "strings.ts"),
+ "utf-8",
+ );
+ assert.match(
+ generated,
+ /import \{ strings as pogenDependencyCatalogue0 \} from "shared-one\/i18n"/,
+ );
+ assert.match(
+ generated,
+ /import \{ strings as pogenDependencyCatalogue1 \} from "shared-two\/i18n"/,
+ );
+ assert.match(
+ generated,
+ /composeTranslationCatalogues\([\s\S]*pogenDependencyCatalogue0,[\s\S]*pogenDependencyCatalogue1,[\s\S]*strings as/,
+ );
+ assert.match(generated, /Object\.assign\(strings, pogenComposedStrings\)/);
+ } finally {
+ process.chdir(originalCwd);
+ fs.rmSync(projectDir, { recursive: true, force: true });
+ }
+});
diff --git a/packages/pogen/src/po2ts.ts b/packages/pogen/src/po2ts.ts
@@ -50,6 +50,12 @@ export interface StringsType {
locale_data: {
messages: Record<string, undefined | Array<string>>;
};
+ _pogen: {
+ total: number;
+ translated: number;
+ untranslated: string[];
+ optional: string[];
+ };
}
// This prelude match the types above
@@ -62,6 +68,12 @@ export interface StringsType {
locale_data: {
messages: Record<string, unknown>;
};
+ _pogen?: {
+ total: number;
+ translated: number;
+ untranslated: string[];
+ optional: string[];
+ };
};
`;
@@ -111,6 +123,8 @@ export function poToStrings(
// concerned.
let totalKeys = 0;
let totalTranslated = 0;
+ const untranslated: string[] = [];
+ const optional: string[] = [];
// `gettext-parser` files each entry under its msgctxt, with the empty string
// for entries that have none. Every bucket has to be read: only looking at
@@ -133,13 +147,15 @@ export function poToStrings(
.some(
(comment) => comment.trim() === "pogen: optional fixed translation",
);
- if (!optionalFixed) totalKeys++;
+ if (optionalFixed) optional.push(key);
+ else totalKeys++;
// Treat fuzzy entries as untranslated (standard gettext behaviour):
// msgmerge seeds fuzzy translations from unrelated nearby strings, so
// shipping them would surface wrong-meaning text. Fall back to English.
const flags = (entry.comments && entry.comments.flag) || "";
if (flags.split(",").some((f) => f.trim() === "fuzzy")) {
+ if (!optionalFixed) untranslated.push(key);
continue;
}
@@ -149,11 +165,15 @@ export function poToStrings(
// catalogue with `msgstr[1] ""` renders English for those counts.
if (msgstr.length > 0 && msgstr.every((s) => !!s)) {
if (!optionalFixed) totalTranslated++;
+ } else if (!optionalFixed) {
+ untranslated.push(key);
}
messages[key] = [entry.msgid_plural, ...msgstr];
} else {
if (msgstr.length > 0 && !!msgstr[0]) {
if (!optionalFixed) totalTranslated++;
+ } else if (!optionalFixed) {
+ untranslated.push(key);
}
messages[key] = msgstr;
}
@@ -190,9 +210,32 @@ export function poToStrings(
plural_forms: header.plural_forms,
lang: header.lang,
completeness: Number.isNaN(completeness) ? 0 : completeness,
+ _pogen: {
+ total: totalKeys,
+ translated: totalTranslated,
+ untranslated: untranslated.sort(),
+ optional: optional.sort(),
+ },
};
}
+export function resolveCatalogDependencies(value: unknown): string[] {
+ if (value === undefined) return [];
+ if (
+ !Array.isArray(value) ||
+ value.some((entry) => typeof entry !== "string" || !entry.trim())
+ ) {
+ throw new Error(
+ "'pogen.catalogDependencies' must be an array of module specifiers",
+ );
+ }
+ const dependencies = value.map((entry) => entry.trim());
+ if (new Set(dependencies).size !== dependencies.length) {
+ throw new Error("'pogen.catalogDependencies' must not contain duplicates");
+ }
+ return dependencies;
+}
+
export function po2ts(): void {
const files = glob.sync("src/i18n/*.po");
@@ -210,7 +253,32 @@ export function po2ts(): void {
prelude = DEFAULT_STRING_PRELUDE;
}
- const chunks = [prelude];
+ const packageJson = JSON.parse(
+ fs.readFileSync("./package.json", { encoding: "utf-8" }),
+ );
+ let catalogDependencies: string[];
+ try {
+ catalogDependencies = resolveCatalogDependencies(
+ packageJson.pogen?.catalogDependencies,
+ );
+ } catch (e) {
+ console.error(e instanceof Error ? e.message : e);
+ process.exit(1);
+ }
+
+ const dependencyNames = catalogDependencies.map(
+ (_, index) => `pogenDependencyCatalogue${index}`,
+ );
+ const dependencyImports = catalogDependencies.map(
+ (dependency, index) =>
+ `import { strings as ${dependencyNames[index]} } from ${JSON.stringify(dependency)};\n`,
+ );
+ if (catalogDependencies.length > 0) {
+ dependencyImports.unshift(
+ 'import { composeTranslationCatalogues } from "@gnu-taler/taler-util";\n',
+ );
+ }
+ const chunks = [prelude, ...dependencyImports];
for (const filename of files) {
const m = filename.match(/([a-zA-Z0-9-_]+).po/);
@@ -236,6 +304,17 @@ export function po2ts(): void {
chunks.push(s);
}
+ if (catalogDependencies.length > 0) {
+ chunks.push(`
+const pogenComposedStrings = composeTranslationCatalogues(
+ ${dependencyNames.join(",\n ")},
+ strings as Parameters<typeof composeTranslationCatalogues>[0],
+);
+for (const language of Object.keys(strings)) delete strings[language];
+Object.assign(strings, pogenComposedStrings);
+`);
+ }
+
const outputPath = "src/i18n/strings.ts";
const prettierCli = fileURLToPath(
import.meta.resolve("prettier/bin/prettier.cjs"),
