commit 1d5a56424b6fdb056cb0ff5f3f697f0218290aa7
parent aabd1f0221a2e05c4b5b6dccc2a378e904a48a7c
Author: Florian Dold <dold@taler.net>
Date: Sun, 6 Sep 2026 18:55:22 +0200
packaging/ng: skip published versions and clean successful uploads
Check testing and publications sharing its package pool before uploading,
so locally rebuilt versions already present on the server are skipped.
Clean staged package files only after publication succeeds, including
runs with no new uploads, and retain them when publishing fails.
Diffstat:
3 files changed, 429 insertions(+), 27 deletions(-)
diff --git a/packaging/ng/README.md b/packaging/ng/README.md
@@ -188,9 +188,29 @@ is created or updated.
./taler-pkg promote debian-trixie
```
-Publishing imports only packages newer than those already in testing. The
-testing repo retains all imported versions. Promotion snapshots are also kept
-so that an older release can be selected manually if a rollback is needed.
+Publishing imports only packages newer than those already in testing or in any
+publication sharing its prefix (`apt/debian` or `apt/ubuntu`) and `main`
+component, including stable. Versions are compared per package name and
+architecture. Equal versions are skipped even when the local build has
+different contents; publishing a changed build requires a newer version or
+Debian revision. `--dry` performs the same checks without changing the server.
+An incomplete server inventory aborts publishing before any files are uploaded.
+
+Packages found only in stable are skipped without copying them into testing.
+Since promotion replaces stable with the complete testing snapshot, those
+packages can disappear from stable on the next promotion. The testing repo
+retains all imported versions. Promotion snapshots are also kept so that an
+older release can be selected manually if a rollback is needed. Conflicts
+already imported into testing require a separate repair; skipping local
+uploads does not remove them.
+
+After a successful publish, including one with nothing new to upload, all
+top-level regular `.deb` and `.ddeb` files in the remote upload directory
+`/home/taler-packaging/DISTRO` are removed. Other files and subdirectories are
+preserved. Uploaded files remain there if uploading, importing, or publishing
+fails. A cleanup failure is reported separately with a nonzero exit status,
+even though publication succeeded. Run only one publisher per upload
+directory at a time.
### Bumping component versions
diff --git a/packaging/ng/taler-pkg b/packaging/ng/taler-pkg
@@ -541,6 +541,32 @@ def snapshot_packages(snapshot):
return parse_package_list(result.stdout)
+def publishing_packages(pubcfg):
+ """Include pending testing imports and publications sharing its package pool."""
+ sources = {(pubcfg.testing_repo, "local")}
+ for prefix, distribution in sorted(published_repositories()):
+ if prefix != pubcfg.prefix:
+ continue
+ publication = get_publication(distribution, prefix)
+ if (
+ publication.get("prefix") != prefix
+ or publication.get("distribution") != distribution
+ or not publication.get("sources")
+ ):
+ raise ValueError(f"invalid publication details for {prefix}/{distribution}")
+ source = publication["sources"].get("main")
+ if source is not None:
+ sources.add(source)
+
+ readers = {"local": repo_packages, "snapshot": snapshot_packages}
+ packages = set()
+ for name, kind in sorted(sources):
+ if kind not in readers:
+ raise ValueError(f"unsupported publication source {name} [{kind}]")
+ packages.update(readers[kind](name))
+ return sorted(packages)
+
+
def stable_snapshot(pubcfg):
publication = get_publication(pubcfg.codename, pubcfg.prefix)
sources = publication.get("sources", {})
@@ -680,13 +706,29 @@ def current_package_files(distro):
return current
+def cleanup_uploads(staging_dir):
+ remote_command(
+ [
+ "sh",
+ "-c",
+ (
+ 'if [ ! -e "$1" ]; then exit 0; fi\n'
+ 'find "$1" -maxdepth 1 -type f '
+ r"\( -name '*.deb' -o -name '*.ddeb' \) -delete"
+ ),
+ "taler-pkg-cleanup",
+ staging_dir,
+ ]
+ )
+
+
def publish(cfg):
distro = cfg.distro
if distro.endswith("-testing"):
print("Publish the base distro; packages always go to testing", file=sys.stderr)
sys.exit(1)
pubcfg = publishing_config(distro)
- server_packages = repo_packages(pubcfg.testing_repo)
+ server_packages = publishing_packages(pubcfg)
uploads = []
for package_file in current_package_files(distro):
package, version, architecture = package_file_identity(package_file)
@@ -708,27 +750,9 @@ def publish(cfg):
if cfg.dry:
return
+ staging_dir = f"/home/{remote_user}/{distro}"
if uploads:
- staging_dir = f"/home/{remote_user}/{distro}"
remote_command(["mkdir", "-p", staging_dir])
- remote_command(
- [
- "find",
- staging_dir,
- "-maxdepth",
- "1",
- "-type",
- "f",
- "(",
- "-name",
- "*.deb",
- "-o",
- "-name",
- "*.ddeb",
- ")",
- "-delete",
- ]
- )
local_files = [Path(f"./packages/{distro}") / name for name in uploads]
remote_files = [f"{staging_dir}/{name}" for name in uploads]
subprocess.run(
@@ -742,11 +766,18 @@ def publish(cfg):
],
check=True,
)
- remote_aptly("repo", "add", "-remove-files", pubcfg.testing_repo, *remote_files)
+ remote_aptly("repo", "add", pubcfg.testing_repo, *remote_files)
remote_aptly(
"publish", "update", pubcfg.testing_distribution, pubcfg.prefix, tty=True
)
+ try:
+ cleanup_uploads(staging_dir)
+ except (subprocess.CalledProcessError, OSError) as exc:
+ raise RuntimeError(
+ f"Publishing {pubcfg.testing_distribution} succeeded, "
+ f"but cleanup of {staging_dir} failed"
+ ) from exc
# Tag syntax variants supported by buildscripts/generic:
diff --git a/packaging/ng/testing/test_publishing.py b/packaging/ng/testing/test_publishing.py
@@ -10,8 +10,9 @@ import sys
import unittest
from io import StringIO
from pathlib import Path
+from tempfile import TemporaryDirectory
from types import SimpleNamespace
-from unittest.mock import call, patch
+from unittest.mock import Mock, call, patch
ROOT = Path(__file__).parents[1]
sys.path.insert(0, str(ROOT))
@@ -346,18 +347,136 @@ class InitializeTests(unittest.TestCase):
)
+class PublishingInventoryTests(unittest.TestCase):
+ def test_reads_each_source_once_and_ignores_other_pools(self):
+ pubcfg = TALER_PKG.publishing_config("debian-trixie")
+ publications = {
+ ("apt/debian", "trixie-testing"): {
+ "main": (pubcfg.testing_repo, "local"),
+ "contrib": ("ignored-repo", "local"),
+ },
+ ("apt/debian", "trixie"): {"main": ("stable", "snapshot")},
+ ("apt/debian", "stable-alias"): {"main": ("stable", "snapshot")},
+ ("apt/debian", "legacy"): {"main": ("legacy", "local")},
+ ("apt/debian", "extras"): {"contrib": ("ignored-repo", "local")},
+ ("apt/ubuntu", "noble"): {"main": ("ignored-snapshot", "snapshot")},
+ ("filesystem:other:apt/debian", "trixie"): {
+ "main": ("ignored-snapshot", "snapshot")
+ },
+ }
+ repos = {
+ pubcfg.testing_repo: ["testing_1_amd64", "common_1_all"],
+ "legacy": ["legacy_2_arm64"],
+ }
+
+ def get_publication(distribution, prefix):
+ return {
+ "prefix": prefix,
+ "distribution": distribution,
+ "sources": publications[prefix, distribution],
+ }
+
+ with (
+ patch.object(
+ TALER_PKG, "published_repositories", return_value=set(publications)
+ ),
+ patch.object(
+ TALER_PKG, "get_publication", side_effect=get_publication
+ ) as show,
+ patch.object(
+ TALER_PKG, "repo_packages", side_effect=repos.__getitem__
+ ) as repo,
+ patch.object(
+ TALER_PKG,
+ "snapshot_packages",
+ return_value=["stable_2_amd64", "common_1_all"],
+ ) as snapshot,
+ ):
+ packages = TALER_PKG.publishing_packages(pubcfg)
+
+ self.assertEqual(
+ ["common_1_all", "legacy_2_arm64", "stable_2_amd64", "testing_1_amd64"],
+ packages,
+ )
+ self.assertCountEqual(
+ [call(pubcfg.testing_repo), call("legacy")], repo.call_args_list
+ )
+ snapshot.assert_called_once_with("stable")
+ self.assertEqual(5, show.call_count)
+ self.assertTrue(
+ all(item.args[1] == "apt/debian" for item in show.call_args_list)
+ )
+
+ def test_inventory_errors_prevent_uploads_and_cleanup(self):
+ pubcfg = TALER_PKG.publishing_config("debian-trixie")
+ for failure in ("list", "show", "repo", "snapshot", "kind", "details"):
+ with self.subTest(failure=failure):
+ error = subprocess.CalledProcessError(1, ["aptly", failure])
+ details = TALER_PKG.parse_publication(
+ publication("apt/debian", "trixie", "stable", "snapshot")
+ )
+ if failure == "kind":
+ details["sources"]["main"] = ("stable", "unsupported")
+ elif failure == "details":
+ details = {"sources": {}}
+ with (
+ patch.object(
+ TALER_PKG,
+ "published_repositories",
+ return_value={("apt/debian", "trixie")},
+ side_effect=error if failure == "list" else None,
+ ),
+ patch.object(
+ TALER_PKG,
+ "get_publication",
+ return_value=details,
+ side_effect=error if failure == "show" else None,
+ ),
+ patch.object(
+ TALER_PKG,
+ "repo_packages",
+ return_value=[],
+ side_effect=error if failure == "repo" else None,
+ ),
+ patch.object(
+ TALER_PKG,
+ "snapshot_packages",
+ return_value=[],
+ side_effect=error if failure == "snapshot" else None,
+ ),
+ patch.object(TALER_PKG, "remote_command") as command,
+ patch.object(TALER_PKG, "remote_aptly") as aptly,
+ patch.object(TALER_PKG.subprocess, "run") as run,
+ patch.object(TALER_PKG, "cleanup_uploads") as cleanup,
+ self.assertRaises((ValueError, subprocess.CalledProcessError)),
+ ):
+ TALER_PKG.publish(SimpleNamespace(distro=pubcfg.distro, dry=False))
+
+ command.assert_not_called()
+ aptly.assert_not_called()
+ run.assert_not_called()
+ cleanup.assert_not_called()
+
+
class PublishTests(unittest.TestCase):
+ def setUp(self):
+ self.publications = self.enterContext(
+ patch.object(TALER_PKG, "published_repositories", return_value=set())
+ )
+
def test_uploads_only_newer_packages_and_updates_testing(self):
local_packages = [
"merchant_2.0_amd64.deb",
"merchant-dbgsym_2.0_amd64.ddeb",
"exchange_1.0_amd64.deb",
+ "manual_1.0_all.deb",
]
server_packages = [
"merchant_1.0_amd64",
"merchant_3.0_arm64",
"merchant-dbgsym_1.0_amd64",
"exchange_2.0_amd64",
+ "manual_1.0_all",
]
with (
patch.object(
@@ -367,7 +486,13 @@ class PublishTests(unittest.TestCase):
patch.object(TALER_PKG, "remote_command") as remote_command,
patch.object(TALER_PKG, "remote_aptly") as remote_aptly,
patch.object(TALER_PKG.subprocess, "run") as run,
+ patch.object(TALER_PKG, "cleanup_uploads") as cleanup,
):
+ operations = Mock()
+ operations.attach_mock(remote_command, "command")
+ operations.attach_mock(run, "upload")
+ operations.attach_mock(remote_aptly, "aptly")
+ operations.attach_mock(cleanup, "cleanup")
TALER_PKG.publish(SimpleNamespace(distro="debian-trixie", dry=False))
rsync = run.call_args.args[0]
@@ -376,13 +501,20 @@ class PublishTests(unittest.TestCase):
Path("packages/debian-trixie/merchant-dbgsym_2.0_amd64.ddeb"), rsync
)
self.assertNotIn(Path("packages/debian-trixie/exchange_1.0_amd64.deb"), rsync)
- self.assertEqual(2, remote_command.call_count)
+ self.assertNotIn(Path("packages/debian-trixie/manual_1.0_all.deb"), rsync)
+ remote_command.assert_called_once_with(
+ ["mkdir", "-p", "/home/taler-packaging/debian-trixie"]
+ )
+ self.assertEqual(
+ ["command", "upload", "aptly", "aptly", "cleanup"],
+ [item[0] for item in operations.mock_calls],
+ )
+ cleanup.assert_called_once_with("/home/taler-packaging/debian-trixie")
remote_aptly.assert_has_calls(
[
call(
"repo",
"add",
- "-remove-files",
"taler-debian-trixie-testing",
"/home/taler-packaging/debian-trixie/merchant_2.0_amd64.deb",
"/home/taler-packaging/debian-trixie/merchant-dbgsym_2.0_amd64.ddeb",
@@ -402,6 +534,124 @@ class PublishTests(unittest.TestCase):
)
)
+ def test_skips_equal_and_older_versions_in_published_sources(self):
+ local_packages = [
+ "merchant_2.0_amd64.deb",
+ "merchant-dbgsym_2.0_amd64.ddeb",
+ "exchange_1.0_amd64.deb",
+ "manual_1.0_all.deb",
+ ]
+ server_packages = [
+ "merchant_2.0_amd64",
+ "merchant-dbgsym_2.0_amd64",
+ "exchange_2.0_amd64",
+ "manual_1.0_all",
+ ]
+ for kind in ("local", "snapshot"):
+ for distribution in ("trixie", "legacy"):
+ for dry in (False, True):
+ with self.subTest(kind=kind, distribution=distribution, dry=dry):
+ self.publications.return_value = {("apt/debian", distribution)}
+ details = TALER_PKG.parse_publication(
+ publication("apt/debian", distribution, "published", kind)
+ )
+ with (
+ patch.object(
+ TALER_PKG, "get_publication", return_value=details
+ ),
+ patch.object(
+ TALER_PKG,
+ "repo_packages",
+ side_effect=lambda name: (
+ server_packages if name == "published" else []
+ ),
+ ),
+ patch.object(
+ TALER_PKG,
+ "snapshot_packages",
+ return_value=server_packages,
+ ),
+ patch.object(
+ TALER_PKG,
+ "current_package_files",
+ return_value=local_packages,
+ ),
+ patch.object(TALER_PKG, "remote_command") as command,
+ patch.object(TALER_PKG, "remote_aptly") as aptly,
+ patch.object(TALER_PKG.subprocess, "run") as run,
+ patch.object(TALER_PKG, "cleanup_uploads") as cleanup,
+ patch("sys.stdout", new_callable=StringIO) as stdout,
+ ):
+ TALER_PKG.publish(
+ SimpleNamespace(distro="debian-trixie", dry=dry)
+ )
+
+ command.assert_not_called()
+ run.assert_not_called()
+ for package in server_packages:
+ self.assertIn(f"server has {package}", stdout.getvalue())
+ if dry:
+ aptly.assert_not_called()
+ cleanup.assert_not_called()
+ else:
+ aptly.assert_called_once_with(
+ "publish",
+ "update",
+ "trixie-testing",
+ "apt/debian",
+ tty=True,
+ )
+ cleanup.assert_called_once_with(
+ "/home/taler-packaging/debian-trixie"
+ )
+
+ def test_newer_versions_and_other_architectures_are_uploaded(self):
+ self.publications.return_value = {("apt/debian", "trixie")}
+ local_packages = [
+ "merchant_2.0-1+trixie_amd64.deb",
+ "merchant_2.0-0+trixie_arm64.deb",
+ ]
+ with (
+ patch.object(
+ TALER_PKG,
+ "get_publication",
+ return_value=TALER_PKG.parse_publication(
+ publication("apt/debian", "trixie", "stable", "snapshot")
+ ),
+ ),
+ patch.object(TALER_PKG, "repo_packages", return_value=[]),
+ patch.object(
+ TALER_PKG,
+ "snapshot_packages",
+ return_value=["merchant_2.0-0+trixie_amd64"],
+ ),
+ patch.object(
+ TALER_PKG, "current_package_files", return_value=local_packages
+ ),
+ patch.object(TALER_PKG, "remote_command"),
+ patch.object(TALER_PKG, "remote_aptly") as aptly,
+ patch.object(TALER_PKG.subprocess, "run") as run,
+ patch.object(TALER_PKG, "cleanup_uploads"),
+ ):
+ TALER_PKG.publish(SimpleNamespace(distro="debian-trixie", dry=False))
+
+ for filename in local_packages:
+ self.assertIn(
+ Path("packages/debian-trixie") / filename, run.call_args.args[0]
+ )
+ self.assertEqual(
+ call(
+ "repo",
+ "add",
+ "taler-debian-trixie-testing",
+ *(
+ f"/home/taler-packaging/debian-trixie/{name}"
+ for name in local_packages
+ ),
+ ),
+ aptly.call_args_list[0],
+ )
+
def test_rejects_non_package_or_path_artifacts(self):
for filename in (
"merchant.changes",
@@ -423,12 +673,14 @@ class PublishTests(unittest.TestCase):
patch.object(TALER_PKG, "remote_command") as remote_command,
patch.object(TALER_PKG, "remote_aptly") as remote_aptly,
patch.object(TALER_PKG.subprocess, "run") as run,
+ patch.object(TALER_PKG, "cleanup_uploads") as cleanup,
):
TALER_PKG.publish(SimpleNamespace(distro="debian-trixie", dry=True))
remote_command.assert_not_called()
remote_aptly.assert_not_called()
run.assert_not_called()
+ cleanup.assert_not_called()
def test_empty_publish_still_refreshes_testing_metadata(self):
with (
@@ -437,6 +689,7 @@ class PublishTests(unittest.TestCase):
patch.object(TALER_PKG, "remote_command") as remote_command,
patch.object(TALER_PKG, "remote_aptly") as remote_aptly,
patch.object(TALER_PKG.subprocess, "run") as run,
+ patch.object(TALER_PKG, "cleanup_uploads") as cleanup,
):
TALER_PKG.publish(SimpleNamespace(distro="ubuntu-noble", dry=False))
@@ -445,6 +698,104 @@ class PublishTests(unittest.TestCase):
remote_aptly.assert_called_once_with(
"publish", "update", "noble-testing", "apt/ubuntu", tty=True
)
+ cleanup.assert_called_once_with("/home/taler-packaging/ubuntu-noble")
+
+ def test_failed_upload_import_or_publication_does_not_clean_staged_files(self):
+ for phase in ("upload", "import", "publish"):
+ with self.subTest(phase=phase):
+ error = subprocess.CalledProcessError(1, [phase])
+
+ with (
+ patch.object(TALER_PKG, "repo_packages", return_value=[]),
+ patch.object(
+ TALER_PKG,
+ "current_package_files",
+ return_value=["merchant_2.0_amd64.deb"],
+ ),
+ patch.object(TALER_PKG, "remote_command") as command,
+ patch.object(
+ TALER_PKG,
+ "remote_aptly",
+ side_effect=[error] if phase == "import" else [result(), error],
+ ),
+ patch.object(
+ TALER_PKG.subprocess,
+ "run",
+ side_effect=error if phase == "upload" else None,
+ ),
+ patch.object(TALER_PKG, "cleanup_uploads") as cleanup,
+ self.assertRaises(subprocess.CalledProcessError),
+ ):
+ TALER_PKG.publish(
+ SimpleNamespace(distro="debian-trixie", dry=False)
+ )
+
+ command.assert_called_once_with(
+ ["mkdir", "-p", "/home/taler-packaging/debian-trixie"]
+ )
+ cleanup.assert_not_called()
+
+ def test_cleanup_failure_reports_that_publication_succeeded(self):
+ error = subprocess.CalledProcessError(1, ["cleanup"])
+ with (
+ patch.object(TALER_PKG, "repo_packages", return_value=[]),
+ patch.object(TALER_PKG, "current_package_files", return_value=[]),
+ patch.object(TALER_PKG, "remote_aptly") as aptly,
+ patch.object(TALER_PKG, "cleanup_uploads", side_effect=error),
+ self.assertRaisesRegex(
+ RuntimeError,
+ "Publishing trixie-testing succeeded, but cleanup .* failed",
+ ),
+ ):
+ TALER_PKG.publish(SimpleNamespace(distro="debian-trixie", dry=False))
+
+ aptly.assert_called_once_with(
+ "publish", "update", "trixie-testing", "apt/debian", tty=True
+ )
+
+
+class UploadCleanupTests(unittest.TestCase):
+ def test_cleanup_only_removes_top_level_regular_package_files(self):
+ with TemporaryDirectory() as tmp:
+ staging = Path(tmp) / "upload dir 'quoted'"
+ staging.mkdir()
+ for filename in (
+ "package.deb",
+ "debug.ddeb",
+ "notes.txt",
+ "package.changes",
+ ):
+ (staging / filename).write_text("test data")
+ nested = staging / "nested"
+ nested.mkdir()
+ (nested / "package.deb").write_text("nested data")
+ outside = Path(tmp) / "outside.deb"
+ outside.write_text("outside data")
+ (staging / "link.deb").symlink_to(outside)
+ with patch.object(
+ TALER_PKG,
+ "remote_command",
+ side_effect=lambda command: subprocess.run(command, check=True),
+ ):
+ TALER_PKG.cleanup_uploads(str(staging))
+
+ self.assertEqual(
+ {"notes.txt", "package.changes", "nested", "link.deb"},
+ {item.name for item in staging.iterdir()},
+ )
+ self.assertEqual("nested data", (nested / "package.deb").read_text())
+ self.assertEqual("outside data", outside.read_text())
+
+ def test_missing_upload_directory_is_a_noop(self):
+ with TemporaryDirectory() as tmp:
+ staging = Path(tmp) / "missing"
+ with patch.object(
+ TALER_PKG,
+ "remote_command",
+ side_effect=lambda command: subprocess.run(command, check=True),
+ ):
+ TALER_PKG.cleanup_uploads(str(staging))
+ self.assertFalse(staging.exists())
class PromoteTests(unittest.TestCase):
