commit 981219adb2927d80c4f9181400291c5d304520b8
parent ddcd2db9a580dfbe9f1da76a8d683144ff76a1a5
Author: Florian Dold <dold@taler.net>
Date: Thu, 3 Sep 2026 00:40:36 +0200
packaging/ng: replace reprepro publishing with aptly
Diffstat:
13 files changed, 975 insertions(+), 237 deletions(-)
diff --git a/packaging/ng/README.md b/packaging/ng/README.md
@@ -56,7 +56,7 @@ so that this folder can be directly consumed as a trusted package source.
* `packages.toml`: Repository, package, version, builder, and dependency configuration.
* `buildscripts/*`: Build scripts used during the package build steps.
* `distros/*`: Files for building for a specific distro.
-* `server-side/*.sh`: Scripts that must be installed in taler-packaing@taler.net:$HOME
+* `server-side/`: Server repository topology and initialization notes.
## Versions
@@ -150,6 +150,44 @@ a package tag always produces a package that apt considers newer:
./taler-pkg build $DISTRO
```
+Supported publishing distributions are `debian-trixie` and `ubuntu-noble`.
+
+### Initializing the package repositories
+
+```
+./taler-pkg init
+```
+
+This idempotently initializes both supported distributions on
+`taler-packaging@taler.net`. Testing is published directly from its mutable
+aptly repo. Stable is published from an initial snapshot and is subsequently
+changed only by promotion. Existing repositories and publications are
+validated; incompatible existing state is reported rather than replaced.
+The `apt/debian` and `apt/ubuntu` prefixes use aptly's default endpoint, so
+they appear below `$rootDir/public`.
+
+Repository signing can prompt for the signing-key passphrase when a publication
+is created or updated.
+
+### Publishing and promoting packages
+
+```
+# Preview and publish locally built packages to testing.
+./taler-pkg publish --dry debian-trixie
+./taler-pkg publish debian-trixie
+
+# List the packages in the current stable snapshot.
+./taler-pkg show-published debian-trixie
+
+# Compare testing with stable, then promote testing as a new stable snapshot.
+./taler-pkg promote --dry debian-trixie
+./taler-pkg promote debian-trixie
+```
+
+Publishing imports only packages newer than those already in testing. The
+testing repo retains all imported versions. Promotion snapshots are also kept
+so that an older release can be selected manually if a rollback is needed.
+
### Bumping component versions
```
diff --git a/packaging/ng/server-side/README.md b/packaging/ng/server-side/README.md
@@ -0,0 +1,35 @@
+# Aptly repository layout
+
+The package repository on `taler-packaging@taler.net` has one mutable testing
+repo and one bootstrap repo for each supported distribution:
+
+| Build distro | Testing repo | Stable bootstrap repo | Publish prefix |
+| --- | --- | --- | --- |
+| `debian-trixie` | `taler-debian-trixie-testing` | `taler-debian-trixie-stable` | `apt/debian` |
+| `ubuntu-noble` | `taler-ubuntu-noble-testing` | `taler-ubuntu-noble-stable` | `apt/ubuntu` |
+
+These prefixes use aptly's default filesystem endpoint. Aptly therefore
+publishes them below its default publish directory as
+`$rootDir/public/apt/debian` and `$rootDir/public/apt/ubuntu`; no named
+filesystem endpoint is involved.
+
+All repositories use the `main` component and publish `amd64` and `arm64`.
+Testing distributions are `trixie-testing` and `noble-testing`; stable
+distributions are `trixie` and `noble`.
+
+Run the initialization from `packaging/ng`:
+
+```
+./taler-pkg init
+```
+
+The command creates anything missing for both distributions. It directly
+publishes each testing repo, creates an initial snapshot from each stable
+bootstrap repo, and publishes that snapshot as stable. Rerunning the command
+validates the existing topology and otherwise does nothing. It deliberately
+refuses to replace incompatible publications.
+
+Normal uploads go only to the testing repos. `taler-pkg promote DISTRO`
+snapshots the complete testing repo and atomically switches the stable
+publication to that snapshot. Testing retains all imported versions, and
+promotion snapshots are retained for manual rollback.
diff --git a/packaging/ng/server-side/confs/debian/README b/packaging/ng/server-side/confs/debian/README
@@ -1 +0,0 @@
-These files go into www/apt/debian/conf/
-\ No newline at end of file
diff --git a/packaging/ng/server-side/confs/debian/distributions b/packaging/ng/server-side/confs/debian/distributions
@@ -1,38 +0,0 @@
-Origin: GNU Taler
-Label: Taler
-Suite: stable
-Codename: bookworm
-Architectures: amd64
-Components: main
-Description: Debian bookworm package repository for GNU Taler
-SignWith: 0084993C2C6CDF471A7D7EFD26E546A5FE7E0266
-Pull: bookworm-testing
-
-Origin: GNU Taler
-Label: Taler
-Suite: testing
-Codename: bookworm-testing
-Architectures: amd64
-Components: main
-Description: Debian bookworm package repository for GNU Taler
-SignWith: 0084993C2C6CDF471A7D7EFD26E546A5FE7E0266
-
-Origin: GNU Taler
-Label: Taler
-Suite: stable
-Codename: trixie
-Architectures: amd64 arm64
-Components: main
-Description: Debian bookworm package repository for GNU Taler
-SignWith: 0084993C2C6CDF471A7D7EFD26E546A5FE7E0266
-Pull: trixie-testing
-
-Origin: GNU Taler
-Label: Taler
-Suite: testing
-Codename: trixie-testing
-Architectures: amd64 arm64
-Components: main
-Description: Debian bookworm package repository for GNU Taler
-SignWith: 0084993C2C6CDF471A7D7EFD26E546A5FE7E0266
-
diff --git a/packaging/ng/server-side/confs/debian/options b/packaging/ng/server-side/confs/debian/options
@@ -1,3 +0,0 @@
-verbose
-basedir /home/taler-packaging/www/apt/debian
-ask-passphrase
diff --git a/packaging/ng/server-side/confs/debian/pulls b/packaging/ng/server-side/confs/debian/pulls
@@ -1,14 +0,0 @@
-Name: bookworm-testing
-From: bookworm-testing
-Components: main
-Architectures: amd64
-UDebComponents:
-FilterFormula: Package (% * )
-
-
-Name: trixie-testing
-From: trixie-testing
-Components: main
-Architectures: amd64
-UDebComponents:
-FilterFormula: Package (% * )
diff --git a/packaging/ng/server-side/confs/ubuntu/README b/packaging/ng/server-side/confs/ubuntu/README
@@ -1 +0,0 @@
-These files go into www/apt/ubuntu/conf/
-\ No newline at end of file
diff --git a/packaging/ng/server-side/confs/ubuntu/distributions b/packaging/ng/server-side/confs/ubuntu/distributions
@@ -1,19 +0,0 @@
-Origin: GNU Taler
-Label: Taler
-Suite: testing
-Codename: noble-testing
-Architectures: amd64 arm64
-Components: main debug
-Description: Ubuntu Noble repository for GNU Taler
-SignWith: 0084993C2C6CDF471A7D7EFD26E546A5FE7E0266
-
-Origin: GNU Taler
-Label: Taler
-Suite: stable
-Codename: noble
-Architectures: amd64 arm64
-Components: main debug
-Description: Ubuntu Noble repository for GNU Taler
-SignWith: 0084993C2C6CDF471A7D7EFD26E546A5FE7E0266
-Pull: noble-testing
-
diff --git a/packaging/ng/server-side/confs/ubuntu/options b/packaging/ng/server-side/confs/ubuntu/options
@@ -1,3 +0,0 @@
-verbose
-basedir /home/taler-packaging/www/apt/ubuntu
-ask-passphrase
diff --git a/packaging/ng/server-side/confs/ubuntu/pulls b/packaging/ng/server-side/confs/ubuntu/pulls
@@ -1,7 +0,0 @@
-Name: noble-testing
-From: noble-testing
-Components: main
-Architectures: amd64
-UDebComponents:
-FilterFormula: Package (% * )
-
diff --git a/packaging/ng/taler-pkg b/packaging/ng/taler-pkg
@@ -5,12 +5,14 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import argparse
-import subprocess
-import platform
+import datetime
import os
+import platform
import re
+import shlex
+import subprocess
import sys
-from dataclasses import replace
+from dataclasses import dataclass, replace
from pathlib import Path
# Make local util package available
@@ -27,10 +29,76 @@ from package_config import ConfigError, load_config, write_config # noqa: E402
archs = ["arm64", "amd64"]
host = "taler.net"
+remote_user = "taler-packaging"
+signing_key = "0084993C2C6CDF471A7D7EFD26E546A5FE7E0266"
native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64"
config_path = Path(mydir) / "packages.toml"
config = load_config(config_path)
components = config.enabled_packages()
+
+
+@dataclass(frozen=True)
+class PublishingConfig:
+ distro: str
+ vendor: str
+ codename: str
+
+ @property
+ def prefix(self):
+ return f"apt/{self.vendor}"
+
+ @property
+ def stable_repo(self):
+ return f"taler-{self.vendor}-{self.codename}-stable"
+
+ @property
+ def testing_repo(self):
+ return f"taler-{self.vendor}-{self.codename}-testing"
+
+ @property
+ def testing_distribution(self):
+ return f"{self.codename}-testing"
+
+ @property
+ def initial_snapshot(self):
+ return f"{self.stable_repo}-initial"
+
+
+publishing_configs = {
+ distro: PublishingConfig(distro, vendor, codename)
+ for distro, vendor, codename in (
+ ("debian-trixie", "debian", "trixie"),
+ ("ubuntu-noble", "ubuntu", "noble"),
+ )
+}
+
+
+def publishing_config(distro):
+ try:
+ return publishing_configs[distro]
+ except KeyError:
+ raise ValueError(f"unsupported publishing distro: {distro}") from None
+
+
+def remote_command(command, *, capture_output=False, tty=False):
+ ssh_command = ["ssh"]
+ if tty:
+ ssh_command.append("-t")
+ ssh_command.extend(
+ [f"{remote_user}@{host}", shlex.join(str(arg) for arg in command)]
+ )
+ return subprocess.run(
+ ssh_command,
+ check=True,
+ capture_output=capture_output,
+ text=capture_output,
+ )
+
+
+def remote_aptly(*args, capture_output=False, tty=False):
+ return remote_command(["aptly", *args], capture_output=capture_output, tty=tty)
+
+
deps = {
name: list(package.dependencies)
for name, package in config.packages.items()
@@ -266,56 +334,262 @@ def show_order(cfg):
print("build order:", buildorder)
+def aptly_list(*args):
+ result = remote_aptly(*args, "-raw", capture_output=True)
+ return set(result.stdout.split())
-def promote(cfg):
- dry = cfg.dry
- distro = cfg.distro
- vendor, codename = distro.split("-", 1)
- if dry:
- subprocess.run(
- [
- "ssh",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ checkpull {codename}",
- ],
- check=True,
+def published_repositories():
+ result = remote_aptly("publish", "list", "-raw", capture_output=True)
+ repositories = set()
+ for line in result.stdout.splitlines():
+ fields = line.split()
+ if len(fields) != 2:
+ raise ValueError(f"unexpected aptly publish list line: {line}")
+ repositories.add(tuple(fields))
+ return repositories
+
+
+def parse_publication(output):
+ publication = {"sources": {}}
+ in_sources = False
+ for line in output.splitlines():
+ if line == "Sources:":
+ in_sources = True
+ continue
+ if in_sources and line.startswith(" "):
+ match = re.fullmatch(r"\s+([^:]+):\s+(.+)\s+\[([^]]+)]", line)
+ if match is None:
+ raise ValueError(f"unexpected aptly publication source: {line}")
+ component, name, kind = match.groups()
+ publication["sources"][component] = (name, kind)
+ continue
+ in_sources = False
+ if line.startswith("Prefix: "):
+ publication["prefix"] = line.removeprefix("Prefix: ")
+ elif line.startswith("Distribution: "):
+ publication["distribution"] = line.removeprefix("Distribution: ")
+ elif line.startswith("Architectures: "):
+ publication["architectures"] = set(
+ line.removeprefix("Architectures: ").replace(",", " ").split()
+ )
+ return publication
+
+
+def get_publication(distribution, prefix):
+ result = remote_aptly("publish", "show", distribution, prefix, capture_output=True)
+ return parse_publication(result.stdout)
+
+
+def snapshot_origin(snapshot):
+ result = remote_aptly("snapshot", "show", snapshot, capture_output=True)
+ match = re.search(r"Snapshot from local repo \[([^]]+)]", result.stdout)
+ if match is not None:
+ return match.group(1)
+ in_sources = False
+ for line in result.stdout.splitlines():
+ if line == "Sources:":
+ in_sources = True
+ continue
+ if in_sources and line.startswith(" "):
+ match = re.fullmatch(r"\s+(.+)\s+\[(?:local|repo)]", line)
+ if match is not None:
+ return match.group(1)
+ elif in_sources:
+ break
+ raise ValueError(f"cannot determine source repo for aptly snapshot {snapshot}")
+
+
+def validate_publication(pubcfg, publication, *, stable):
+ distribution = pubcfg.codename if stable else pubcfg.testing_distribution
+ sources = publication.get("sources", {})
+ if publication.get("prefix") != pubcfg.prefix:
+ raise ValueError(f"unexpected prefix for {distribution} publication")
+ if publication.get("distribution") != distribution:
+ raise ValueError(f"unexpected distribution for {distribution} publication")
+ if publication.get("architectures") != set(archs):
+ raise ValueError(f"unexpected architectures for {distribution} publication")
+ if set(sources) != {"main"}:
+ raise ValueError(f"unexpected components for {distribution} publication")
+ source = sources["main"]
+ if stable:
+ if source[1] != "snapshot":
+ raise ValueError(f"stable publication {distribution} is not a snapshot")
+ origin = snapshot_origin(source[0])
+ if origin not in {pubcfg.stable_repo, pubcfg.testing_repo}:
+ raise ValueError(
+ f"stable publication {distribution} has unexpected source {origin}"
+ )
+ elif source != (pubcfg.testing_repo, "local"):
+ raise ValueError(
+ f"testing publication {distribution} has unexpected source {source[0]}"
)
+
+
+def publish_options(distribution):
+ return [
+ f"-architectures={','.join(sorted(archs))}",
+ "-component=main",
+ f"-distribution={distribution}",
+ f"-gpg-key={signing_key}",
+ "-origin=GNU Taler",
+ "-label=Taler",
+ ]
+
+
+def initialize(cfg):
+ del cfg
+ repos = aptly_list("repo", "list")
+ snapshots = aptly_list("snapshot", "list")
+ publications = published_repositories()
+ changed = False
+
+ for pubcfg in publishing_configs.values():
+ for repo, distribution in (
+ (pubcfg.stable_repo, pubcfg.codename),
+ (pubcfg.testing_repo, pubcfg.testing_distribution),
+ ):
+ if repo not in repos:
+ remote_aptly(
+ "repo",
+ "create",
+ f"-distribution={distribution}",
+ "-component=main",
+ repo,
+ )
+ repos.add(repo)
+ changed = True
+
+ testing_key = (pubcfg.prefix, pubcfg.testing_distribution)
+ if testing_key not in publications:
+ remote_aptly(
+ "publish",
+ "repo",
+ *publish_options(pubcfg.testing_distribution),
+ pubcfg.testing_repo,
+ pubcfg.prefix,
+ tty=True,
+ )
+ publications.add(testing_key)
+ changed = True
+ else:
+ validate_publication(
+ pubcfg,
+ get_publication(pubcfg.testing_distribution, pubcfg.prefix),
+ stable=False,
+ )
+
+ stable_key = (pubcfg.prefix, pubcfg.codename)
+ if stable_key not in publications:
+ if pubcfg.initial_snapshot not in snapshots:
+ remote_aptly(
+ "snapshot",
+ "create",
+ pubcfg.initial_snapshot,
+ "from",
+ "repo",
+ pubcfg.stable_repo,
+ )
+ snapshots.add(pubcfg.initial_snapshot)
+ changed = True
+ elif snapshot_origin(pubcfg.initial_snapshot) != pubcfg.stable_repo:
+ raise ValueError(
+ f"snapshot {pubcfg.initial_snapshot} has an unexpected source"
+ )
+ remote_aptly(
+ "publish",
+ "snapshot",
+ *publish_options(pubcfg.codename),
+ pubcfg.initial_snapshot,
+ pubcfg.prefix,
+ tty=True,
+ )
+ publications.add(stable_key)
+ changed = True
+ else:
+ validate_publication(
+ pubcfg,
+ get_publication(pubcfg.codename, pubcfg.prefix),
+ stable=True,
+ )
+
+ if changed:
+ print("aptly repositories and publications initialized")
else:
- subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ pull {codename}",
- ],
- check=True,
- )
- # Always export!
- # Reprepro is weird, listed packages might actually not show
- # up in the index yet.
- subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}",
- ],
- check=True,
- )
+ print("aptly repositories and publications already initialized")
-def show_published(cfg):
- distro = cfg.distro
- vendor, codename = distro.split("-", 1)
- listfmt = "${package}_${version}_${architecture}.${$type}\n"
- subprocess.run(
- [
- "ssh",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}",
- ],
- check=True,
+
+def parse_package_list(output):
+ packages = []
+ in_packages = False
+ for line in output.splitlines():
+ if line == "Packages:":
+ in_packages = True
+ continue
+ if in_packages and line.startswith(" "):
+ packages.append(line.strip())
+ elif in_packages:
+ break
+ return packages
+
+
+def repo_packages(repo):
+ result = remote_aptly("repo", "show", "-with-packages", repo, capture_output=True)
+ return parse_package_list(result.stdout)
+
+
+def snapshot_packages(snapshot):
+ result = remote_aptly(
+ "snapshot", "show", "-with-packages", snapshot, capture_output=True
)
+ return parse_package_list(result.stdout)
+
+
+def stable_snapshot(pubcfg):
+ publication = get_publication(pubcfg.codename, pubcfg.prefix)
+ sources = publication.get("sources", {})
+ if set(sources) != {"main"} or sources["main"][1] != "snapshot":
+ raise ValueError(f"stable publication {pubcfg.codename} is not a snapshot")
+ return sources["main"][0]
+
+
+def promotion_snapshot_name(pubcfg, now=None):
+ if now is None:
+ now = datetime.datetime.now(datetime.timezone.utc)
+ timestamp = now.strftime("%Y%m%dT%H%M%S%fZ")
+ return f"{pubcfg.testing_repo}-{timestamp}"
+
+
+def promote(cfg):
+ pubcfg = publishing_config(cfg.distro)
+ if cfg.dry:
+ stable = set(snapshot_packages(stable_snapshot(pubcfg)))
+ testing = set(repo_packages(pubcfg.testing_repo))
+ for package in sorted(stable - testing):
+ print(f"- {package}")
+ for package in sorted(testing - stable):
+ print(f"+ {package}")
+ if stable == testing:
+ print("testing and stable contain the same packages")
+ return
+
+ snapshot = promotion_snapshot_name(pubcfg)
+ remote_aptly("snapshot", "create", snapshot, "from", "repo", pubcfg.testing_repo)
+ remote_aptly(
+ "publish",
+ "switch",
+ pubcfg.codename,
+ pubcfg.prefix,
+ snapshot,
+ tty=True,
+ )
+ print(f"promoted {pubcfg.testing_repo} as snapshot {snapshot}")
+
+
+def show_published(cfg):
+ pubcfg = publishing_config(cfg.distro)
+ for package in snapshot_packages(stable_snapshot(pubcfg)):
+ print(package)
def test(cfg):
@@ -359,116 +633,123 @@ def test(cfg):
check=True,
)
-def publish(cfg):
- distro = cfg.distro
- if distro.endswith("-testing"):
- print("Files are automatically published to testing", file=sys.stderr)
- sys.exit(1)
- vendor, codename = distro.split("-", 1)
- # List of .deb and .ddeb files.
- debs = []
- listfmt = "${package}_${version}_${architecture}.${$type}\n"
- server_debs_str = subprocess.check_output(
- [
- "ssh",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}",
- ],
- encoding="utf-8",
- )
- server_debs = server_debs_str.split()
+
+def package_identity(filename):
+ stem = filename
+ for suffix in (".ddeb", ".deb"):
+ if filename.endswith(suffix):
+ stem = filename.removesuffix(suffix)
+ break
+ fields = stem.rsplit("_", 2)
+ if len(fields) != 3:
+ raise ValueError(f"invalid package reference: {filename}")
+ return fields
+
+
+def package_file_identity(filename):
+ if Path(filename).name != filename or not filename.endswith((".deb", ".ddeb")):
+ raise ValueError(f"invalid package filename: {filename}")
+ return package_identity(filename)
+
+
+def newest_server_package(package, architecture, server_packages):
+ newest = None
+ newest_ref = None
+ for server_package in server_packages:
+ server_name, server_version, server_arch = package_identity(server_package)
+ if package != server_name or architecture != server_arch:
+ continue
+ if newest is None or vercomp.compare_versions(server_version, newest) > 0:
+ newest = server_version
+ newest_ref = server_package
+ return newest, newest_ref
+
+
+def current_package_files(distro):
+ current = []
+ seen = set()
for component in components:
- current = []
+ component_current = []
for arch in archs + ["all"]:
- cf = Path(f"./packages/{distro}/{component}@{arch}.built.current")
- if not cf.exists():
+ current_file = Path(f"./packages/{distro}/{component}@{arch}.built.current")
+ if not current_file.exists():
print(f"component {component}@{arch} has no current packages")
continue
- with open(cf) as f:
- current = current + f.read().split()
- print("current", current)
- for deb in current:
- if deb.endswith(".deb"):
- pkg1, ver1, arch1 = deb.removesuffix(".deb").split("_")
- elif deb.endswith(".ddeb"):
- pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_")
- else:
- raise ValueError(f"invalid deb filename: {deb}")
- fresh = True
- server_deb = None
- # If the server has the same or a later version,
- # the local version isn't fresh.
- for srvdeb in server_debs:
- pkg2, ver2, arch2 = srvdeb.removesuffix(".deb").split("_")
- if pkg1 != pkg2 or arch1 != arch2:
- continue
- if vercomp.compare_versions(ver1, ver2) <= 0:
- fresh = False
- server_deb = srvdeb
- break
- if fresh:
- debs.append(deb)
- else:
- print("package", deb, "not fresh, server has", server_deb)
- if len(debs) == 0:
- print("nothing to upload")
+ component_current.extend(current_file.read_text().split())
+ print("current", component_current)
+ for package in component_current:
+ if package not in seen:
+ current.append(package)
+ seen.add(package)
+ return current
+
+
+def publish(cfg):
+ distro = cfg.distro
+ if distro.endswith("-testing"):
+ print("Publish the base distro; packages always go to testing", file=sys.stderr)
+ sys.exit(1)
+ pubcfg = publishing_config(distro)
+ server_packages = repo_packages(pubcfg.testing_repo)
+ uploads = []
+ for package_file in current_package_files(distro):
+ package, version, architecture = package_file_identity(package_file)
+ server_version, server_ref = newest_server_package(
+ package, architecture, server_packages
+ )
+ if (
+ server_version is None
+ or vercomp.compare_versions(version, server_version) > 0
+ ):
+ uploads.append(package_file)
+ else:
+ print("package", package_file, "not fresh, server has", server_ref)
+
+ if uploads:
+ print("uploading debs", uploads)
else:
- print("uploading debs", debs)
- if cfg.dry:
- return
- debs = [Path(f"./packages/{distro}/") / x for x in debs]
- subprocess.run(
+ print("nothing to upload")
+ if cfg.dry:
+ return
+
+ if uploads:
+ staging_dir = f"/home/{remote_user}/{distro}"
+ remote_command(["mkdir", "-p", staging_dir])
+ remote_command(
[
- "ssh",
- f"taler-packaging@{host}",
- f"rm -f '/home/taler-packaging/{distro}/'*.deb '/home/taler-packaging/{distro}/'*.ddeb",
- ],
- check=True,
+ "find",
+ staging_dir,
+ "-maxdepth",
+ "1",
+ "-type",
+ "f",
+ "(",
+ "-name",
+ "*.deb",
+ "-o",
+ "-name",
+ "*.ddeb",
+ ")",
+ "-delete",
+ ]
)
+ local_files = [Path(f"./packages/{distro}") / name for name in uploads]
+ remote_files = [f"{staging_dir}/{name}" for name in uploads]
subprocess.run(
- ["rsync", "-a", "--info=progress2", "--", *debs, f"taler-packaging@{host}:{distro}/"], check=True
- )
- ret = subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.deb",
- ],
- )
- if ret.returncode != 0:
- # Usually not critical if it fails.
- print(
- "Including ddebs failed. This can happen when including packages that have been included previously"
- )
- # Almost the same, but with ddebs.
- # We explicitly need to tell reprepro
- # to ignore the extension, because it does not
- # deal well with ddebs out of the box.
- ret = subprocess.run(
[
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro --ignore=extension -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.ddeb",
+ "rsync",
+ "-a",
+ "--info=progress2",
+ "--",
+ *local_files,
+ f"{remote_user}@{host}:{staging_dir}/",
],
+ check=True,
)
- if ret.returncode != 0:
- # Usually not critical if it fails.
- print(
- "Including ddebs failed. This can happen when including packages that have been included previously"
- )
- # Always export!
- # Reprepro is weird, listed packages might actually not show
- # up in the index yet.
- subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing",
- ],
- check=True,
+ remote_aptly("repo", "add", "-remove-files", pubcfg.testing_repo, *remote_files)
+
+ remote_aptly(
+ "publish", "update", pubcfg.testing_distribution, pubcfg.prefix, tty=True
)
@@ -565,7 +846,9 @@ def upgrade(cfg):
remote_tags[giturl] = list_remote_tags(giturl)
latest = latest_tag(remote_tags[giturl], cfg.dev)
if latest is None:
- print(f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr)
+ print(
+ f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr
+ )
continue
curr = package.tag
currkey = tag_sortkey(curr)
@@ -604,6 +887,11 @@ def main():
subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND")
+ parser_init = subparsers.add_parser(
+ "init", help="Initialize aptly repositories and publications."
+ )
+ parser_init.set_defaults(func=initialize)
+
# subcommand build
parser_build = subparsers.add_parser("build", help="Build packages for distro.")
@@ -697,7 +985,10 @@ def main():
parser_promote = subparsers.add_parser("promote", help="Promote testing to stable")
parser_promote.add_argument(
- "--dry", help="Dry run (show pulls)", action="store_true", default=False
+ "--dry",
+ help="Dry run (show testing changes)",
+ action="store_true",
+ default=False,
)
parser_promote.add_argument("distro")
parser_promote.set_defaults(func=promote)
diff --git a/packaging/ng/testing/test_build_grouping.py b/packaging/ng/testing/test_build_grouping.py
@@ -13,6 +13,7 @@ from types import SimpleNamespace
from unittest.mock import patch
ROOT = Path(__file__).parents[1]
+sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "buildscripts"))
LOADER = importlib.machinery.SourceFileLoader("taler_pkg", str(ROOT / "taler-pkg"))
SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER)
diff --git a/packaging/ng/testing/test_publishing.py b/packaging/ng/testing/test_publishing.py
@@ -0,0 +1,461 @@
+#!/usr/bin/env python3
+
+# This file is in the public domain.
+
+import datetime
+import importlib.machinery
+import importlib.util
+import subprocess
+import sys
+import unittest
+from io import StringIO
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import call, patch
+
+ROOT = Path(__file__).parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "buildscripts"))
+LOADER = importlib.machinery.SourceFileLoader(
+ "taler_pkg_publishing", str(ROOT / "taler-pkg")
+)
+SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER)
+TALER_PKG = importlib.util.module_from_spec(SPEC)
+LOADER.exec_module(TALER_PKG)
+
+
+def result(stdout=""):
+ return subprocess.CompletedProcess([], 0, stdout=stdout)
+
+
+def publication(prefix, distribution, source, kind):
+ return (
+ f"Prefix: {prefix}\n"
+ f"Distribution: {distribution}\n"
+ "Architectures: amd64 arm64\n"
+ "Sources:\n"
+ f" main: {source} [{kind}]\n"
+ )
+
+
+class PublishingConfigTests(unittest.TestCase):
+ def test_distribution_mapping(self):
+ debian = TALER_PKG.publishing_config("debian-trixie")
+ ubuntu = TALER_PKG.publishing_config("ubuntu-noble")
+
+ self.assertEqual("apt/debian", debian.prefix)
+ self.assertEqual("taler-debian-trixie-testing", debian.testing_repo)
+ self.assertEqual("trixie-testing", debian.testing_distribution)
+ self.assertEqual("apt/ubuntu", ubuntu.prefix)
+ self.assertEqual("taler-ubuntu-noble-stable", ubuntu.stable_repo)
+ with self.assertRaisesRegex(ValueError, "unsupported publishing distro"):
+ TALER_PKG.publishing_config("debian-bookworm")
+
+ def test_remote_command_quotes_every_remote_argument(self):
+ with patch.object(TALER_PKG.subprocess, "run", return_value=result()) as run:
+ TALER_PKG.remote_command(
+ ["command", "argument with spaces", "*.deb"],
+ capture_output=True,
+ tty=True,
+ )
+
+ run.assert_called_once_with(
+ [
+ "ssh",
+ "-t",
+ "taler-packaging@taler.net",
+ "command 'argument with spaces' '*.deb'",
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+
+ def test_parses_publication_and_package_list(self):
+ parsed = TALER_PKG.parse_publication(
+ publication(
+ "apt/debian",
+ "trixie",
+ "taler-debian-trixie-testing-20260903T120000000000Z",
+ "snapshot",
+ )
+ )
+ packages = TALER_PKG.parse_package_list(
+ "Name: repo\nPackages:\n one_1_amd64\n two_2_all\n"
+ )
+
+ self.assertEqual("apt/debian", parsed["prefix"])
+ self.assertEqual(
+ (
+ "taler-debian-trixie-testing-20260903T120000000000Z",
+ "snapshot",
+ ),
+ parsed["sources"]["main"],
+ )
+ self.assertEqual(["one_1_amd64", "two_2_all"], packages)
+
+ def test_snapshot_names_are_utc_and_unique_to_microseconds(self):
+ pubcfg = TALER_PKG.publishing_config("debian-trixie")
+ now = datetime.datetime(
+ 2026, 9, 3, 12, 34, 56, 123456, tzinfo=datetime.timezone.utc
+ )
+
+ name = TALER_PKG.promotion_snapshot_name(pubcfg, now)
+
+ self.assertEqual("taler-debian-trixie-testing-20260903T123456123456Z", name)
+
+
+class InitializeTests(unittest.TestCase):
+ def test_initializes_all_missing_repositories_and_publications(self):
+ outputs = iter([result(), result(), result()])
+
+ def aptly(*args, **kwargs):
+ if kwargs.get("capture_output"):
+ return next(outputs)
+ return result()
+
+ with (
+ patch.object(TALER_PKG, "remote_aptly", side_effect=aptly) as remote,
+ patch("sys.stdout", new_callable=StringIO) as stdout,
+ ):
+ TALER_PKG.initialize(SimpleNamespace())
+
+ calls = remote.call_args_list
+ self.assertIn(
+ call(
+ "repo",
+ "create",
+ "-distribution=trixie-testing",
+ "-component=main",
+ "taler-debian-trixie-testing",
+ ),
+ calls,
+ )
+ self.assertIn(
+ call(
+ "publish",
+ "repo",
+ *TALER_PKG.publish_options("trixie-testing"),
+ "taler-debian-trixie-testing",
+ "apt/debian",
+ tty=True,
+ ),
+ calls,
+ )
+ self.assertIn(
+ call(
+ "snapshot",
+ "create",
+ "taler-ubuntu-noble-stable-initial",
+ "from",
+ "repo",
+ "taler-ubuntu-noble-stable",
+ ),
+ calls,
+ )
+ self.assertIn(
+ call(
+ "publish",
+ "snapshot",
+ *TALER_PKG.publish_options("noble"),
+ "taler-ubuntu-noble-stable-initial",
+ "apt/ubuntu",
+ tty=True,
+ ),
+ calls,
+ )
+ self.assertIn("initialized", stdout.getvalue())
+
+ def test_complete_initialization_is_a_noop(self):
+ repos = "\n".join(
+ repo
+ for pubcfg in TALER_PKG.publishing_configs.values()
+ for repo in (pubcfg.stable_repo, pubcfg.testing_repo)
+ )
+ snapshots = "debian-snapshot\nubuntu-snapshot\n"
+ publications = (
+ "apt/debian trixie-testing\n"
+ "apt/debian trixie\n"
+ "apt/ubuntu noble-testing\n"
+ "apt/ubuntu noble\n"
+ )
+
+ def aptly(*args, **kwargs):
+ if args == ("repo", "list", "-raw"):
+ return result(repos)
+ if args == ("snapshot", "list", "-raw"):
+ return result(snapshots)
+ if args == ("publish", "list", "-raw"):
+ return result(publications)
+ if args[:2] == ("publish", "show"):
+ distribution, prefix = args[2:]
+ pubcfg = next(
+ item
+ for item in TALER_PKG.publishing_configs.values()
+ if item.prefix == prefix
+ )
+ if distribution.endswith("-testing"):
+ return result(
+ publication(prefix, distribution, pubcfg.testing_repo, "local")
+ )
+ snapshot = (
+ "debian-snapshot"
+ if pubcfg.vendor == "debian"
+ else "ubuntu-snapshot"
+ )
+ return result(publication(prefix, distribution, snapshot, "snapshot"))
+ if args[:2] == ("snapshot", "show"):
+ pubcfg = (
+ TALER_PKG.publishing_config("debian-trixie")
+ if args[2] == "debian-snapshot"
+ else TALER_PKG.publishing_config("ubuntu-noble")
+ )
+ return result(
+ f"Description: Snapshot from local repo [{pubcfg.testing_repo}]\n"
+ )
+ self.fail(f"unexpected aptly call: {args} {kwargs}")
+
+ with (
+ patch.object(TALER_PKG, "remote_aptly", side_effect=aptly) as remote,
+ patch("sys.stdout", new_callable=StringIO) as stdout,
+ ):
+ TALER_PKG.initialize(SimpleNamespace())
+
+ self.assertFalse(
+ any(not item.kwargs.get("capture_output") for item in remote.mock_calls)
+ )
+ self.assertIn("already initialized", stdout.getvalue())
+
+ def test_reuses_initial_snapshots_when_only_stable_publish_is_missing(self):
+ repos = "\n".join(
+ repo
+ for pubcfg in TALER_PKG.publishing_configs.values()
+ for repo in (pubcfg.stable_repo, pubcfg.testing_repo)
+ )
+ snapshots = "\n".join(
+ pubcfg.initial_snapshot for pubcfg in TALER_PKG.publishing_configs.values()
+ )
+ publications = "apt/debian trixie-testing\napt/ubuntu noble-testing\n"
+
+ def aptly(*args, **kwargs):
+ if args == ("repo", "list", "-raw"):
+ return result(repos)
+ if args == ("snapshot", "list", "-raw"):
+ return result(snapshots)
+ if args == ("publish", "list", "-raw"):
+ return result(publications)
+ if args[:2] == ("publish", "show"):
+ distribution, prefix = args[2:]
+ pubcfg = next(
+ item
+ for item in TALER_PKG.publishing_configs.values()
+ if item.prefix == prefix
+ )
+ return result(
+ publication(prefix, distribution, pubcfg.testing_repo, "local")
+ )
+ if args[:2] == ("snapshot", "show"):
+ pubcfg = next(
+ item
+ for item in TALER_PKG.publishing_configs.values()
+ if item.initial_snapshot == args[2]
+ )
+ return result(
+ f"Description: Snapshot from local repo [{pubcfg.stable_repo}]\n"
+ )
+ return result()
+
+ with patch.object(TALER_PKG, "remote_aptly", side_effect=aptly) as remote:
+ TALER_PKG.initialize(SimpleNamespace())
+
+ mutations = [
+ item for item in remote.mock_calls if not item.kwargs.get("capture_output")
+ ]
+ self.assertEqual(2, len(mutations))
+ self.assertTrue(
+ all(item.args[:2] == ("publish", "snapshot") for item in mutations)
+ )
+
+ def test_rejects_existing_testing_publication_with_wrong_source(self):
+ pubcfg = TALER_PKG.publishing_config("debian-trixie")
+ wrong = publication(
+ pubcfg.prefix,
+ pubcfg.testing_distribution,
+ pubcfg.stable_repo,
+ "local",
+ )
+ with self.assertRaisesRegex(ValueError, "unexpected source"):
+ TALER_PKG.validate_publication(
+ pubcfg, TALER_PKG.parse_publication(wrong), stable=False
+ )
+
+
+class PublishTests(unittest.TestCase):
+ def test_uploads_only_newer_packages_and_updates_testing(self):
+ local_packages = [
+ "merchant_2.0_amd64.deb",
+ "merchant-dbgsym_2.0_amd64.ddeb",
+ "exchange_1.0_amd64.deb",
+ ]
+ server_packages = [
+ "merchant_1.0_amd64",
+ "merchant_3.0_arm64",
+ "merchant-dbgsym_1.0_amd64",
+ "exchange_2.0_amd64",
+ ]
+ with (
+ patch.object(
+ TALER_PKG, "current_package_files", return_value=local_packages
+ ),
+ patch.object(TALER_PKG, "repo_packages", return_value=server_packages),
+ patch.object(TALER_PKG, "remote_command") as remote_command,
+ patch.object(TALER_PKG, "remote_aptly") as remote_aptly,
+ patch.object(TALER_PKG.subprocess, "run") as run,
+ ):
+ TALER_PKG.publish(SimpleNamespace(distro="debian-trixie", dry=False))
+
+ rsync = run.call_args.args[0]
+ self.assertIn(Path("packages/debian-trixie/merchant_2.0_amd64.deb"), rsync)
+ self.assertIn(
+ Path("packages/debian-trixie/merchant-dbgsym_2.0_amd64.ddeb"), rsync
+ )
+ self.assertNotIn(Path("packages/debian-trixie/exchange_1.0_amd64.deb"), rsync)
+ self.assertEqual(2, remote_command.call_count)
+ remote_aptly.assert_has_calls(
+ [
+ call(
+ "repo",
+ "add",
+ "-remove-files",
+ "taler-debian-trixie-testing",
+ "/home/taler-packaging/debian-trixie/merchant_2.0_amd64.deb",
+ "/home/taler-packaging/debian-trixie/merchant-dbgsym_2.0_amd64.ddeb",
+ ),
+ call(
+ "publish",
+ "update",
+ "trixie-testing",
+ "apt/debian",
+ tty=True,
+ ),
+ ]
+ )
+ self.assertFalse(
+ any(
+ call.args[0:2] == ("repo", "remove") for call in remote_aptly.mock_calls
+ )
+ )
+
+ def test_rejects_non_package_or_path_artifacts(self):
+ for filename in (
+ "merchant.changes",
+ "merchant.deb",
+ "../merchant_2.0_amd64.deb",
+ ):
+ with self.subTest(filename=filename):
+ with self.assertRaisesRegex(ValueError, "invalid package"):
+ TALER_PKG.package_file_identity(filename)
+
+ def test_dry_publish_never_mutates(self):
+ with (
+ patch.object(
+ TALER_PKG,
+ "current_package_files",
+ return_value=["merchant_2.0_amd64.deb"],
+ ),
+ patch.object(TALER_PKG, "repo_packages", return_value=[]),
+ patch.object(TALER_PKG, "remote_command") as remote_command,
+ patch.object(TALER_PKG, "remote_aptly") as remote_aptly,
+ patch.object(TALER_PKG.subprocess, "run") as run,
+ ):
+ TALER_PKG.publish(SimpleNamespace(distro="debian-trixie", dry=True))
+
+ remote_command.assert_not_called()
+ remote_aptly.assert_not_called()
+ run.assert_not_called()
+
+ def test_empty_publish_still_refreshes_testing_metadata(self):
+ with (
+ patch.object(TALER_PKG, "current_package_files", return_value=[]),
+ patch.object(TALER_PKG, "repo_packages", return_value=[]),
+ patch.object(TALER_PKG, "remote_command") as remote_command,
+ patch.object(TALER_PKG, "remote_aptly") as remote_aptly,
+ patch.object(TALER_PKG.subprocess, "run") as run,
+ ):
+ TALER_PKG.publish(SimpleNamespace(distro="ubuntu-noble", dry=False))
+
+ remote_command.assert_not_called()
+ run.assert_not_called()
+ remote_aptly.assert_called_once_with(
+ "publish", "update", "noble-testing", "apt/ubuntu", tty=True
+ )
+
+
+class PromoteTests(unittest.TestCase):
+ def test_dry_promote_shows_stable_to_testing_diff(self):
+ with (
+ patch.object(TALER_PKG, "stable_snapshot", return_value="stable-snapshot"),
+ patch.object(
+ TALER_PKG,
+ "snapshot_packages",
+ return_value=["common_1_all", "removed_1_amd64"],
+ ),
+ patch.object(
+ TALER_PKG,
+ "repo_packages",
+ return_value=["common_1_all", "new_2_amd64"],
+ ),
+ patch.object(TALER_PKG, "remote_aptly") as remote_aptly,
+ patch("sys.stdout", new_callable=StringIO) as stdout,
+ ):
+ TALER_PKG.promote(SimpleNamespace(distro="debian-trixie", dry=True))
+
+ self.assertEqual("- removed_1_amd64\n+ new_2_amd64\n", stdout.getvalue())
+ remote_aptly.assert_not_called()
+
+ def test_promote_snapshots_testing_and_switches_stable(self):
+ snapshot = "taler-ubuntu-noble-testing-20260903T123456123456Z"
+ with (
+ patch.object(TALER_PKG, "promotion_snapshot_name", return_value=snapshot),
+ patch.object(TALER_PKG, "remote_aptly") as remote,
+ ):
+ TALER_PKG.promote(SimpleNamespace(distro="ubuntu-noble", dry=False))
+
+ remote.assert_has_calls(
+ [
+ call(
+ "snapshot",
+ "create",
+ snapshot,
+ "from",
+ "repo",
+ "taler-ubuntu-noble-testing",
+ ),
+ call(
+ "publish",
+ "switch",
+ "noble",
+ "apt/ubuntu",
+ snapshot,
+ tty=True,
+ ),
+ ]
+ )
+
+ def test_show_published_lists_current_stable_snapshot(self):
+ with (
+ patch.object(TALER_PKG, "stable_snapshot", return_value="stable-snapshot"),
+ patch.object(
+ TALER_PKG,
+ "snapshot_packages",
+ return_value=["one_1_amd64", "two_2_all"],
+ ),
+ patch("sys.stdout", new_callable=StringIO) as stdout,
+ ):
+ TALER_PKG.show_published(SimpleNamespace(distro="debian-trixie"))
+
+ self.assertEqual("one_1_amd64\ntwo_2_all\n", stdout.getvalue())
+
+
+if __name__ == "__main__":
+ unittest.main()
