Automate bilingual changelog generation before publication

tools/prepare-publication.py

@@ -7,6 +7,7 @@ import os
 from pathlib import Path
 import re
 import subprocess
+import sys
 
 ROOT = Path(__file__).resolve().parents[1]
 SOURCE_DIRS = {'adapters','android','assets','contracts','gaia','host','overrides','services','tests','tools','tundra'}
@@ -56,6 +57,8 @@ def main():
     args=parser.parse_args()
     index=args.index.resolve();normal=Path(subprocess.check_output(['git','rev-parse','--git-path','index'],cwd=ROOT,text=True).strip()).resolve()
     if index==normal:raise ValueError('Refusing to replace working index')
+    # Render both languages before selecting blobs for the publication tree.
+    subprocess.run([sys.executable, str(ROOT/'tools/render-changelog.py')], cwd=ROOT, check=True)
     selected=select();env=dict(os.environ,GIT_INDEX_FILE=str(index))
     index.parent.mkdir(parents=True,exist_ok=True)
     subprocess.run(['git','read-tree','--empty'],cwd=ROOT,env=env,check=True)

tools/render-changelog.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""Generate the public FR/EN changelogs from changelog.json (standard library only)."""
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import tempfile
+
+ROOT = Path(__file__).resolve().parents[1]
+OUTPUTS = {"fr": "CHANGELOG.md", "en": "CHANGELOG.en.md"}
+
+
+def render(source):
+    if source.get("schema") != 1 or not isinstance(source.get("entries"), list):
+        raise ValueError("Unsupported changelog schema")
+    ids = set()
+    for entry in source["entries"]:
+        identifier = entry["id"]
+        if not re.fullmatch(r"[A-Za-z0-9_.-]+", identifier) or identifier in ids:
+            raise ValueError("Invalid or duplicate entry id: " + identifier)
+        ids.add(identifier)
+    if not ids:
+        raise ValueError("The changelog must not be empty")
+    documents = {}
+    for language, filename in OUTPUTS.items():
+        preamble = source["preamble"][language].strip()
+        if not preamble.startswith("# "):
+            raise ValueError("Missing preamble for " + language)
+        sections = [preamble]
+        for entry in source["entries"]:
+            body = entry["text"][language].strip()
+            if not body.startswith("## "):
+                raise ValueError("Missing heading: " + entry["id"] + "/" + language)
+            sections.append('<a id="' + entry["id"] + '"></a>\n\n' + body)
+        documents[filename] = "\n\n".join(sections) + "\n"
+    return documents
+
+
+def generate(root=ROOT, check=False):
+    documents = render(json.loads((root / "changelog.json").read_text(encoding="utf-8")))
+    changed = []
+    for filename, content in documents.items():
+        path = root / filename
+        if path.exists() and path.read_text(encoding="utf-8") == content:
+            continue
+        changed.append(filename)
+        if check:
+            continue
+        descriptor, temporary = tempfile.mkstemp(prefix=".changelog-", dir=root)
+        try:
+            with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
+                stream.write(content)
+            os.chmod(temporary, 0o644)
+            os.replace(temporary, path)
+        finally:
+            if os.path.exists(temporary):
+                os.unlink(temporary)
+    return changed
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true", help="Fail if generated files are out of date")
+    args = parser.parse_args()
+    try:
+        changed = generate(check=args.check)
+    except (ValueError, KeyError, TypeError, AttributeError) as error:
+        parser.exit(2, "Invalid changelog: " + str(error) + "\n")
+    if changed:
+        print(("Out of date: " if args.check else "Generated: ") + ", ".join(changed))
+    return 1 if args.check and changed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/test-changelog-render.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Regression checks for generated changelogs and publication integration."""
+import copy
+import importlib.util
+import json
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location("render_changelog", ROOT / "tools/render-changelog.py")
+renderer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(renderer)
+
+
+class ChangelogTests(unittest.TestCase):
+    def setUp(self):
+        self.source = json.loads((ROOT / "changelog.json").read_text())
+
+    def test_no_translation_or_entry_is_lost(self):
+        documents = renderer.render(self.source)
+        for language, filename in renderer.OUTPUTS.items():
+            for entry in self.source["entries"]:
+                self.assertIn(entry["text"][language].strip(), documents[filename])
+                self.assertIn('<a id="' + entry["id"] + '"></a>', documents[filename])
+
+    def test_duplicate_ids_and_missing_translation_are_rejected(self):
+        duplicate = copy.deepcopy(self.source)
+        duplicate["entries"].append(duplicate["entries"][0])
+        with self.assertRaises(ValueError):
+            renderer.render(duplicate)
+        del self.source["entries"][0]["text"]["en"]
+        with self.assertRaises(KeyError):
+            renderer.render(self.source)
+
+    def test_check_mode_does_not_overwrite_files(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "changelog.json").write_text(json.dumps(self.source))
+            (root / "CHANGELOG.md").write_text("Old content")
+            self.assertEqual(len(renderer.generate(root, check=True)), 2)
+            self.assertEqual((root / "CHANGELOG.md").read_text(), "Old content")
+            renderer.generate(root)
+            self.assertEqual(renderer.generate(root, check=True), [])
+
+    def test_publication_renders_updates_without_touching_working_index(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "tools").mkdir()
+            for name in ["render-changelog.py", "prepare-publication.py"]:
+                shutil.copy2(ROOT / "tools" / name, root / "tools" / name)
+            for name in ["LICENSE", "NOTICE", "README.md", "README.fr.md"]:
+                (root / name).write_text("Publication fixture\n")
+            (root / "changelog.json").write_text(json.dumps(self.source))
+            subprocess.run(["git", "init", "-q", str(root)], check=True)
+            subprocess.run(["git", "add", "."], cwd=root, check=True)
+            before = (root / ".git/index").read_bytes()
+            self.source["entries"][0]["text"]["fr"] += "\n\nPublication integration check."
+            (root / "changelog.json").write_text(json.dumps(self.source))
+            command = [sys.executable, str(root / "tools/prepare-publication.py"),
+                       "--index", str(root / ".git/publication-index"),
+                       "--report", str(root / ".git/publication-report.json")]
+            subprocess.run(command, cwd=root, check=True, capture_output=True)
+            report = json.loads((root / ".git/publication-report.json").read_text())
+            published = subprocess.check_output(["git", "show", report["tree"] + ":CHANGELOG.md"], cwd=root, text=True)
+            self.assertIn("Publication integration check.", published)
+            self.assertIn("CHANGELOG.en.md", report["manifest"])
+            self.assertEqual(before, (root / ".git/index").read_bytes())
+
+
+if __name__ == "__main__":
+    unittest.main()