commit d9233c157dcf6e761a7181a099b4aed50ab39828
parent efbad6d7f38ba8649e60573b8b12bb391fcc5a7e
Author: Christian Grothoff <christian@grothoff.org>
Date: Wed, 28 Oct 2026 01:57:25 +0100
bump pepsi
Diffstat:
2 files changed, 657 insertions(+), 309 deletions(-)
diff --git a/packaging/ng/packages.toml b/packaging/ng/packages.toml
@@ -77,6 +77,10 @@ builder = "generic"
url = "git://git.taler.net/taler-rust.git"
builder = "generic"
+[repositories."pepsi"]
+url = "git://git.taler.net/pepsi.git"
+builder = "generic"
+
[repositories."typescript-core"]
url = "git://git.taler.net/taler-typescript-core.git"
builder = "pnpm-workspace"
@@ -166,6 +170,13 @@ debian_path = ""
dependencies = []
enabled = true
+[packages."pepsi"]
+repository = "pepsi"
+tag = "v0.1.1"
+debian_path = ""
+dependencies = []
+enabled = true
+
[packages."sync"]
repository = "sync"
tag = "v1.6.0-dev.1"
diff --git a/packaging/ng/taler-pkg b/packaging/ng/taler-pkg
@@ -5,11 +5,14 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import argparse
-import subprocess
-import platform
+import datetime
import os
+import platform
import re
+import shlex
+import subprocess
import sys
+from dataclasses import dataclass, replace
from pathlib import Path
# Make local util package available
@@ -17,63 +20,95 @@ file = Path(__file__).resolve()
parent, root = file.parent, file.parents[1]
sys.path.append(str(root))
-from util import vercomp
+from util import vercomp # noqa: E402
mydir = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(mydir, "buildscripts"))
+
+from package_config import ConfigError, load_config, write_config # noqa: E402
archs = ["arm64", "amd64"]
host = "taler.net"
+remote_user = "taler-packaging"
+signing_key = "0084993C2C6CDF471A7D7EFD26E546A5FE7E0266"
native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64"
+config_path = Path(mydir) / "packages.toml"
+config = load_config(config_path)
+components = config.enabled_packages()
+
+
+@dataclass(frozen=True)
+class PublishingConfig:
+ distro: str
+ vendor: str
+ codename: str
+
+ @property
+ def prefix(self):
+ return f"apt/{self.vendor}"
+
+ @property
+ def testing_repo(self):
+ return f"taler-{self.vendor}-{self.codename}-testing"
+
+ @property
+ def testing_distribution(self):
+ return f"{self.codename}-testing"
+
+ @property
+ def initial_snapshot(self):
+ return f"taler-{self.vendor}-{self.codename}-stable-initial"
+
+
+publishing_configs = {
+ distro: PublishingConfig(distro, vendor, codename)
+ for distro, vendor, codename in (
+ ("debian-trixie", "debian", "trixie"),
+ ("ubuntu-noble", "ubuntu", "noble"),
+ )
+}
+
+
+def publishing_config(distro):
+ try:
+ return publishing_configs[distro]
+ except KeyError:
+ raise ValueError(f"unsupported publishing distro: {distro}") from None
+
+
+def remote_command(command, *, capture_output=False, tty=False):
+ ssh_command = ["ssh"]
+ if tty:
+ ssh_command.append("-t")
+ ssh_command.extend(
+ [f"{remote_user}@{host}", shlex.join(str(arg) for arg in command)]
+ )
+ return subprocess.run(
+ ssh_command,
+ check=True,
+ capture_output=capture_output,
+ text=capture_output,
+ )
+
+
+def remote_aptly(*args, capture_output=False, tty=False):
+ return remote_command(["aptly", *args], capture_output=capture_output, tty=tty)
-components = [
- "taler-wallet-cli",
- "taler-merchant-webui",
- "taler-exchange-kyc-webui",
- "taler-exchange-aml-webui",
- "taler-auditor-webui",
- "taler-challenger-helpers",
- "challenger-webui",
- "gnunet",
- "libeufin",
- "donau",
- "paivana-httpd",
- "challenger",
- "taler-exchange",
- "taler-harness",
- "taler-merchant",
- "taler-rust",
- "robocop",
- "pepsi",
- #"depolymerization",
- # These two packages don't have good debs yet,
- # Debian complains "No section given for ..., skipping.
- # "taler-directory",
- # "taler-mailbox",
- # We don't publish packages for these yet
- # "taler-mdb",
- # "taler-merchant-demos",
- "anastasis",
- "anastasis-gtk",
- # Currently not used anywhere
- # "sync",
-]
deps = {
- "taler-exchange": ["gnunet"],
- "anastasis": ["gnunet", "taler-merchant"],
- "anastasis-gtk": ["anastasis"],
- "taler-merchant": ["gnunet", "taler-exchange", "donau"],
- "donau": ["gnunet", "taler-exchange"],
- "challenger": ["taler-exchange"],
- # "taler-mdb": ["gnunet", "taler-exchange", "taler-merchant"],
- "sync": ["taler-merchant", "taler-exchange", "gnunet"],
- "paivana-httpd": ["taler-merchant", "taler-exchange", "gnunet"],
+ name: list(package.dependencies)
+ for name, package in config.packages.items()
+ if package.dependencies
}
# Compute reverse dependencies
rdeps = {}
for n1, d in deps.items():
+ if n1 not in components:
+ continue
for n2 in d:
+ if n2 not in components:
+ continue
rd = rdeps.setdefault(n2, [])
if n1 not in rd:
rd.append(n1)
@@ -125,7 +160,7 @@ def find_outdated(pkgdir, arch, roots):
"""Find outdated components based on tag files"""
outdated = set()
for component in roots:
- ver_requested = open(f"buildconfig/{component}.tag").read().strip()
+ ver_requested = config.packages[component].tag
built_tag_file = pkgdir / f"{component}@{arch}.built.tag"
ver_built = None
if built_tag_file.exists():
@@ -136,10 +171,58 @@ def find_outdated(pkgdir, arch, roots):
return outdated
+def component_group_key(component):
+ package = config.packages[component]
+ repository = config.repository_for(package)
+ builder = config.builder_for(package)
+ if builder != "pnpm-workspace":
+ # Single-package builders are deliberately isolated even when repositories match.
+ return repository.url, package.tag, builder, component
+ return repository.url, package.tag, builder, ""
+
+
+def group_components(selected, buildorder):
+ """Group selected components and topologically order the resulting groups."""
+ selected = set(selected)
+ groups = {}
+ component_to_group = {}
+ for component in buildorder:
+ if component not in selected:
+ continue
+ key = component_group_key(component)
+ groups.setdefault(key, []).append(component)
+ component_to_group[component] = key
+
+ group_dependencies = {key: set() for key in groups}
+ for component, key in component_to_group.items():
+ for dependency in deps.get(component, []):
+ dependency_key = component_to_group.get(dependency)
+ if dependency_key is not None and dependency_key != key:
+ group_dependencies[key].add(dependency_key)
+
+ ordered = []
+ permanent = set()
+ temporary = set()
+
+ def visit(key):
+ if key in permanent:
+ return
+ if key in temporary:
+ raise ConfigError("build groups contain a dependency cycle")
+ temporary.add(key)
+ for dependency in sorted(group_dependencies[key]):
+ visit(dependency)
+ temporary.remove(key)
+ permanent.add(key)
+ ordered.append(groups[key])
+
+ for key in groups:
+ visit(key)
+ return ordered
+
+
def build(cfg):
- transitive = False
- if cfg.transitive:
- transitive = True
+ transitive = cfg.transitive
distro = cfg.distro
vendor, codename = distro.split("-", 1)
print("building", distro)
@@ -193,14 +276,10 @@ def build(cfg):
print("outdated closure", closure)
- for component in buildorder:
- if transitive:
- if component not in closure:
- continue
- else:
- if component not in outdated:
- continue
- print("building", component)
+ selected = closure if transitive else outdated
+ for group in group_components(selected, buildorder):
+ builder = config.builder_for(config.packages[group[0]])
+ print("building", " ".join(group), f"with {builder}")
pkgdir.mkdir(parents=True, exist_ok=True)
cmd = [
"podman",
@@ -224,20 +303,20 @@ def build(cfg):
"--env",
"CARGO_BUILD_BUILD_DIR=/root/.cargo-build",
"--mount",
- f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared",
+ f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared,U=true",
"--mount",
- f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared",
+ f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared,U=true",
"--mount",
f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly",
"--mount",
- f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly",
+ f"type=bind,source={config_path},target=/packages.toml,readonly",
"--mount",
f"type=bind,source={pkgdir},target=/pkgdir",
image_tag,
- "/buildscripts/generic",
- component,
+ f"/buildscripts/{builder}",
codename,
arch,
+ *group,
]
if not cfg.dry:
subprocess.run(
@@ -251,57 +330,324 @@ def show_order(cfg):
print("build order:", buildorder)
+def aptly_list(*args):
+ result = remote_aptly(*args, "-raw", capture_output=True)
+ return set(result.stdout.split())
-def promote(cfg):
- dry = cfg.dry
- distro = cfg.distro
- vendor, codename = distro.split("-", 1)
- listfmt = "${package}_${version}_${architecture}.${$type}\n"
- if dry:
- subprocess.run(
- [
- "ssh",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ checkpull {codename}",
- ],
- check=True,
- )
+def published_repositories():
+ result = remote_aptly("publish", "list", "-raw", capture_output=True)
+ repositories = set()
+ for line in result.stdout.splitlines():
+ fields = line.split()
+ if len(fields) != 2:
+ raise ValueError(f"unexpected aptly publish list line: {line}")
+ repositories.add(tuple(fields))
+ return repositories
+
+
+def parse_publication(output):
+ publication = {"sources": {}}
+ in_sources = False
+ for line in output.splitlines():
+ if line == "Sources:":
+ in_sources = True
+ continue
+ if in_sources and line.startswith(" "):
+ match = re.fullmatch(r"\s+([^:]+):\s+(.+)\s+\[([^]]+)]", line)
+ if match is None:
+ raise ValueError(f"unexpected aptly publication source: {line}")
+ component, name, kind = match.groups()
+ publication["sources"][component] = (name, kind)
+ continue
+ in_sources = False
+ if line.startswith("Prefix: "):
+ publication["prefix"] = line.removeprefix("Prefix: ")
+ elif line.startswith("Distribution: "):
+ publication["distribution"] = line.removeprefix("Distribution: ")
+ elif line.startswith("Architectures: "):
+ publication["architectures"] = set(
+ line.removeprefix("Architectures: ").replace(",", " ").split()
+ )
+ return publication
+
+
+def get_publication(distribution, prefix):
+ result = remote_aptly("publish", "show", distribution, prefix, capture_output=True)
+ return parse_publication(result.stdout)
+
+
+def snapshot_origin(snapshot):
+ result = remote_aptly("snapshot", "show", snapshot, capture_output=True)
+ match = re.search(r"Snapshot from local repo \[([^]]+)]", result.stdout)
+ if match is not None:
+ return match.group(1)
+ in_sources = False
+ for line in result.stdout.splitlines():
+ if line == "Sources:":
+ in_sources = True
+ continue
+ if in_sources and line.startswith(" "):
+ match = re.fullmatch(r"\s+(.+)\s+\[(?:local|repo)]", line)
+ if match is not None:
+ return match.group(1)
+ elif in_sources:
+ break
+ raise ValueError(f"cannot determine source repo for aptly snapshot {snapshot}")
+
+
+def validate_publication(pubcfg, publication, *, stable):
+ distribution = pubcfg.codename if stable else pubcfg.testing_distribution
+ if publication.get("prefix") != pubcfg.prefix:
+ raise ValueError(f"unexpected prefix for {distribution} publication")
+ if publication.get("distribution") != distribution:
+ raise ValueError(f"unexpected distribution for {distribution} publication")
+ if publication.get("architectures") != set(archs):
+ raise ValueError(f"unexpected architectures for {distribution} publication")
+ snapshot = publication_snapshot(publication)
+ validate_snapshot(pubcfg, snapshot, stable=stable)
+
+
+def validate_snapshot(pubcfg, snapshot, *, stable):
+ if stable and snapshot == pubcfg.initial_snapshot:
+ if snapshot_packages(snapshot):
+ raise ValueError(f"initial snapshot {snapshot} is not empty")
else:
- subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ pull {codename}",
- ],
- check=True,
- )
- # Always export!
- # Reprepro is weird, listed packages might actually not show
- # up in the index yet.
- subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}",
- ],
- check=True,
+ origin = snapshot_origin(snapshot)
+ if origin != pubcfg.testing_repo:
+ distribution = pubcfg.codename if stable else pubcfg.testing_distribution
+ raise ValueError(
+ f"publication {distribution} has unexpected source {origin}"
+ )
+
+
+def publish_options(distribution):
+ return [
+ f"-architectures={','.join(sorted(archs))}",
+ "-component=main",
+ f"-distribution={distribution}",
+ f"-gpg-key={signing_key}",
+ "-origin=GNU Taler",
+ "-label=Taler",
+ ]
+
+
+def initialize(cfg):
+ del cfg
+ repos = aptly_list("repo", "list")
+ snapshots = aptly_list("snapshot", "list")
+ publications = published_repositories()
+ changed = False
+
+ for pubcfg in publishing_configs.values():
+ if pubcfg.testing_repo not in repos:
+ remote_aptly(
+ "repo",
+ "create",
+ f"-distribution={pubcfg.testing_distribution}",
+ "-component=main",
+ pubcfg.testing_repo,
+ )
+ repos.add(pubcfg.testing_repo)
+ changed = True
+
+ testing_key = (pubcfg.prefix, pubcfg.testing_distribution)
+ if testing_key not in publications:
+ snapshot = create_testing_snapshot(pubcfg)
+ remote_aptly(
+ "publish",
+ "snapshot",
+ *publish_options(pubcfg.testing_distribution),
+ snapshot,
+ pubcfg.prefix,
+ tty=True,
+ )
+ publications.add(testing_key)
+ changed = True
+ else:
+ validate_publication(
+ pubcfg,
+ get_publication(pubcfg.testing_distribution, pubcfg.prefix),
+ stable=False,
+ )
+
+ stable_key = (pubcfg.prefix, pubcfg.codename)
+ if stable_key not in publications:
+ if pubcfg.initial_snapshot not in snapshots:
+ remote_aptly(
+ "snapshot",
+ "create",
+ pubcfg.initial_snapshot,
+ "empty",
+ )
+ snapshots.add(pubcfg.initial_snapshot)
+ changed = True
+ else:
+ validate_snapshot(pubcfg, pubcfg.initial_snapshot, stable=True)
+ remote_aptly(
+ "publish",
+ "snapshot",
+ *publish_options(pubcfg.codename),
+ pubcfg.initial_snapshot,
+ pubcfg.prefix,
+ tty=True,
+ )
+ publications.add(stable_key)
+ changed = True
+ else:
+ validate_publication(
+ pubcfg,
+ get_publication(pubcfg.codename, pubcfg.prefix),
+ stable=True,
+ )
+
+ if changed:
+ print("aptly repositories and publications initialized")
+ else:
+ print("aptly repositories and publications already initialized")
+
+
+def parse_package_list(output):
+ packages = []
+ in_packages = False
+ for line in output.splitlines():
+ if line == "Packages:":
+ in_packages = True
+ continue
+ if in_packages and line.startswith(" "):
+ packages.append(line.strip())
+ elif in_packages:
+ break
+ return packages
+
+
+def repo_packages(repo):
+ result = remote_aptly("repo", "show", "-with-packages", repo, capture_output=True)
+ return parse_package_list(result.stdout)
+
+
+def snapshot_packages(snapshot):
+ result = remote_aptly(
+ "snapshot", "show", "-with-packages", snapshot, capture_output=True
+ )
+ return parse_package_list(result.stdout)
+
+
+def publishing_packages(pubcfg):
+ """Include pending testing imports and publications sharing its package pool."""
+ sources = {(pubcfg.testing_repo, "local")}
+ for prefix, distribution in sorted(published_repositories()):
+ if prefix != pubcfg.prefix:
+ continue
+ publication = get_publication(distribution, prefix)
+ if (
+ publication.get("prefix") != prefix
+ or publication.get("distribution") != distribution
+ or not publication.get("sources")
+ ):
+ raise ValueError(f"invalid publication details for {prefix}/{distribution}")
+ source = publication["sources"].get("main")
+ if source is not None:
+ sources.add(source)
+
+ readers = {"local": repo_packages, "snapshot": snapshot_packages}
+ packages = set()
+ for name, kind in sorted(sources):
+ if kind not in readers:
+ raise ValueError(f"unsupported publication source {name} [{kind}]")
+ packages.update(readers[kind](name))
+ return sorted(packages)
+
+
+def publication_snapshot(publication):
+ distribution = publication.get("distribution", "unknown")
+ sources = publication.get("sources", {})
+ if set(sources) != {"main"}:
+ raise ValueError(f"unexpected components for {distribution} publication")
+ snapshot, kind = sources["main"]
+ if kind != "snapshot":
+ raise ValueError(
+ f"publication {distribution} uses a {kind} source; "
+ "convert it to a snapshot publication before proceeding"
)
+ return snapshot
-def show_published(cfg):
- distro = cfg.distro
- vendor, codename = distro.split("-", 1)
- listfmt = "${package}_${version}_${architecture}.${$type}\n"
- subprocess.run(
- [
- "ssh",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}",
- ],
- check=True,
+
+def published_snapshot(pubcfg, *, stable):
+ """Return the published snapshot name and its enabled architectures."""
+ distribution = pubcfg.codename if stable else pubcfg.testing_distribution
+ publication = get_publication(distribution, pubcfg.prefix)
+ if (
+ publication.get("prefix") != pubcfg.prefix
+ or publication.get("distribution") != distribution
+ or not publication.get("architectures")
+ ):
+ raise ValueError(f"invalid publication details for {pubcfg.prefix}/{distribution}")
+ return publication_snapshot(publication), publication["architectures"]
+
+
+def testing_snapshot_name(pubcfg, now=None):
+ if now is None:
+ now = datetime.datetime.now(datetime.timezone.utc)
+ timestamp = now.strftime("%Y%m%dT%H%M%S%fZ")
+ return f"{pubcfg.testing_repo}-{timestamp}"
+
+
+def create_testing_snapshot(pubcfg):
+ snapshot = testing_snapshot_name(pubcfg)
+ remote_aptly("snapshot", "create", snapshot, "from", "repo", pubcfg.testing_repo)
+ return snapshot
+
+
+def promote(cfg):
+ pubcfg = publishing_config(cfg.distro)
+ snapshot, _ = published_snapshot(pubcfg, stable=False)
+ if cfg.dry:
+ stable_name, _ = published_snapshot(pubcfg, stable=True)
+ stable = set(snapshot_packages(stable_name))
+ testing = set(snapshot_packages(snapshot))
+ for package in sorted(stable - testing):
+ print(f"- {package}")
+ for package in sorted(testing - stable):
+ print(f"+ {package}")
+ if stable == testing:
+ print("testing and stable contain the same packages")
+ return
+
+ remote_aptly(
+ "publish",
+ "switch",
+ pubcfg.codename,
+ pubcfg.prefix,
+ snapshot,
+ tty=True,
)
+ print(f"promoted published testing snapshot {snapshot}")
+
+
+def show_published(cfg):
+ distro, _, channel = cfg.distro.rpartition("-")
+ if channel not in ("stable", "testing"):
+ distro, channel = cfg.distro, "stable"
+ pubcfg = publishing_config(distro)
+ snapshot, architectures = published_snapshot(pubcfg, stable=channel == "stable")
+ packages = {
+ package
+ for package in snapshot_packages(snapshot)
+ if package_identity(package)[2] in architectures | {"all"}
+ }
+ if cfg.latest:
+ latest = {}
+ for package in sorted(packages):
+ name, version, architecture = package_identity(package)
+ key = (name, architecture)
+ if key not in latest or vercomp.compare_versions(
+ version, latest[key][0]
+ ) > 0:
+ latest[key] = (version, package)
+ packages = {package for _, package in latest.values()}
+ for package in sorted(packages):
+ print(package)
def test(cfg):
@@ -310,7 +656,7 @@ def test(cfg):
distro = f"{vendor}-{codename}"
image_tag = f"localhost/taler-packaging-{distro}:latest"
dockerfile = f"distros/{distro}.Dockerfile"
- cachedir = Path(f"cache").absolute()
+ cachedir = Path("cache").absolute()
print("building base image")
subprocess.run(
[
@@ -345,183 +691,136 @@ def test(cfg):
check=True,
)
-def publish(cfg):
- distro = cfg.distro
- if distro.endswith("-testing"):
- print("Files are automatically published to testing", file=sys.stderr)
- sys.exit(1)
- vendor, codename = distro.split("-", 1)
- # List of .deb and .ddeb files.
- debs = []
- listfmt = "${package}_${version}_${architecture}.${$type}\n"
- server_debs_str = subprocess.check_output(
- [
- "ssh",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}",
- ],
- encoding="utf-8",
- )
- server_debs = server_debs_str.split()
- for component in components:
- current = []
- for arch in archs + ["all"]:
- cf = Path(f"./packages/{distro}/{component}@{arch}.built.current")
- if not cf.exists():
- print(f"component {component}@{arch} has no current packages")
- continue
- with open(cf) as f:
- current = current + f.read().split()
- print("current", current)
- for deb in current:
- if deb.endswith(".deb"):
- pkg1, ver1, arch1 = deb.removesuffix(".deb").split("_")
- elif deb.endswith(".ddeb"):
- pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_")
- else:
- raise Error(f"invalid deb filename: {deb}")
- fresh = True
- server_deb = None
- # If the server has the same or a later version,
- # the local version isn't fresh.
- for srvdeb in server_debs:
- pkg2, ver2, arch2 = srvdeb.removesuffix(".deb").split("_")
- if pkg1 != pkg2 or arch1 != arch2:
- continue
- if vercomp.compare_versions(ver1, ver2) <= 0:
- fresh = False
- server_deb = srvdeb
- break
- if fresh:
- debs.append(deb)
- else:
- print("package", deb, "not fresh, server has", server_deb)
- if len(debs) == 0:
- print("nothing to upload")
- else:
- print("uploading debs", debs)
- if cfg.dry:
- return
- debs = [Path(f"./packages/{distro}/") / x for x in debs]
- subprocess.run(
- [
- "ssh",
- f"taler-packaging@{host}",
- f"rm -f '/home/taler-packaging/{distro}/'*.deb '/home/taler-packaging/{distro}/'*.ddeb",
- ],
- check=True,
- )
- subprocess.run(
- ["rsync", "-a", "--info=progress2", "--", *debs, f"taler-packaging@{host}:{distro}/"], check=True
- )
- ret = subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.deb",
- ],
- )
- if ret.returncode != 0:
- # Usually not critical if it fails.
- print(
- "Including ddebs failed. This can happen when including packages that have been included previously"
- )
- # Almost the same, but with ddebs.
- # We explicitly need to tell reprepro
- # to ignore the extension, because it does not
- # deal well with ddebs out of the box.
- ret = subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro --ignore=extension -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.ddeb",
- ],
- )
- if ret.returncode != 0:
- # Usually not critical if it fails.
- print(
- "Including ddebs failed. This can happen when including packages that have been included previously"
- )
- # Always export!
- # Reprepro is weird, listed packages might actually not show
- # up in the index yet.
- subprocess.run(
- [
- "ssh",
- "-t",
- f"taler-packaging@{host}",
- f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing",
- ],
- check=True,
- )
+def package_identity(filename):
+ stem = filename
+ for suffix in (".ddeb", ".deb"):
+ if filename.endswith(suffix):
+ stem = filename.removesuffix(suffix)
+ break
+ fields = stem.rsplit("_", 2)
+ if len(fields) != 3:
+ raise ValueError(f"invalid package reference: {filename}")
+ return fields
-def get_remote_version(url):
- """Get the latest stable tag from the git repo"""
- # Construct the git command
- # We use -c versionsort.suffix=- to ensure correct semantic version sorting
- cmd = [
- "git",
- "-c",
- "versionsort.suffix=-",
- "ls-remote",
- "--exit-code",
- "--refs",
- "--sort=version:refname",
- "--tags",
- url,
- "*.*.*",
- ]
- result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+def package_file_identity(filename):
+ if Path(filename).name != filename or not filename.endswith((".deb", ".ddeb")):
+ raise ValueError(f"invalid package filename: {filename}")
+ return package_identity(filename)
- # Parse the output
- # Output format is usually: <hash>\trefs/tags/<tagname>
- lines = result.stdout.strip().split("\n")
- valid_tags = []
-
- for line in lines:
- parts = line.split()
- if len(parts) < 2:
+def newest_server_package(package, architecture, server_packages):
+ newest = None
+ newest_ref = None
+ for server_package in server_packages:
+ server_name, server_version, server_arch = package_identity(server_package)
+ if package != server_name or architecture != server_arch:
continue
+ if newest is None or vercomp.compare_versions(server_version, newest) > 0:
+ newest = server_version
+ newest_ref = server_package
+ return newest, newest_ref
- # refs/tags/v1.0.0 -> v1.0.0
- ref_path = parts[1]
- tag = ref_path.split("/")[-1]
-
- # Exclude pre-release semver versions
- if tag.startswith("v") and "-" in tag:
- continue
- valid_tags.append(tag)
-
- if valid_tags:
- return valid_tags[-1]
- return "(none)"
+def current_package_files(distro):
+ current = []
+ seen = set()
+ for component in components:
+ component_current = []
+ for arch in archs + ["all"]:
+ current_file = Path(f"./packages/{distro}/{component}@{arch}.built.current")
+ if not current_file.exists():
+ print(f"component {component}@{arch} has no current packages")
+ continue
+ component_current.extend(current_file.read_text().split())
+ print("current", component_current)
+ for package in component_current:
+ if package not in seen:
+ current.append(package)
+ seen.add(package)
+ return current
-def check_version(name, url):
- """
- Compares local buildconfig version with remote git version.
- """
- ver = get_remote_version(url)
- config_path = os.path.join("buildconfig", f"{name}.tag")
- with open(config_path, "r") as f:
- curr = f.read().strip()
- prefix = "[!] " if curr != ver else ""
- print(f"{prefix}{name} curr: {curr} latest: {ver}")
+def cleanup_uploads(staging_dir):
+ remote_command(
+ [
+ "sh",
+ "-c",
+ (
+ 'if [ ! -e "$1" ]; then exit 0; fi\n'
+ 'find "$1" -maxdepth 1 -type f '
+ r"\( -name '*.deb' -o -name '*.ddeb' \) -delete"
+ ),
+ "taler-pkg-cleanup",
+ staging_dir,
+ ]
+ )
-def print_latest(cfg):
- """Print latest upstream tag for each component"""
- for name in components:
- config_path = os.path.join("buildconfig", f"{name}.giturl")
- with open(config_path, "r") as f:
- giturl = f.read().strip()
- check_version(name, giturl)
+def publish(cfg):
+ distro = cfg.distro
+ if distro.endswith("-testing"):
+ print("Publish the base distro; packages always go to testing", file=sys.stderr)
+ sys.exit(1)
+ pubcfg = publishing_config(distro)
+ published_snapshot(pubcfg, stable=False)
+ server_packages = publishing_packages(pubcfg)
+ uploads = []
+ for package_file in current_package_files(distro):
+ package, version, architecture = package_file_identity(package_file)
+ server_version, server_ref = newest_server_package(
+ package, architecture, server_packages
+ )
+ if (
+ server_version is None
+ or vercomp.compare_versions(version, server_version) > 0
+ ):
+ uploads.append(package_file)
+ else:
+ print("package", package_file, "not fresh, server has", server_ref)
+
+ if uploads:
+ print("uploading debs", uploads)
+ else:
+ print("nothing to upload")
+ if cfg.dry:
+ return
+
+ staging_dir = f"/home/{remote_user}/{distro}"
+ if uploads:
+ remote_command(["mkdir", "-p", staging_dir])
+ local_files = [Path(f"./packages/{distro}") / name for name in uploads]
+ remote_files = [f"{staging_dir}/{name}" for name in uploads]
+ subprocess.run(
+ [
+ "rsync",
+ "-a",
+ "--info=progress2",
+ "--",
+ *local_files,
+ f"{remote_user}@{host}:{staging_dir}/",
+ ],
+ check=True,
+ )
+ remote_aptly("repo", "add", pubcfg.testing_repo, *remote_files)
+
+ snapshot = create_testing_snapshot(pubcfg)
+ remote_aptly(
+ "publish",
+ "switch",
+ pubcfg.testing_distribution,
+ pubcfg.prefix,
+ snapshot,
+ tty=True,
+ )
+ try:
+ cleanup_uploads(staging_dir)
+ except (subprocess.CalledProcessError, OSError) as exc:
+ raise RuntimeError(
+ f"Publishing {pubcfg.testing_distribution} succeeded, "
+ f"but cleanup of {staging_dir} failed"
+ ) from exc
# Tag syntax variants supported by buildscripts/generic:
@@ -585,55 +884,73 @@ def latest_tag(tags, dev):
return best
+def print_latest(cfg):
+ """Print the latest stable upstream tag for each enabled component."""
+ remote_tags = {}
+ for name in components:
+ package = config.packages[name]
+ url = config.repository_for(package).url
+ if url not in remote_tags:
+ remote_tags[url] = list_remote_tags(url)
+ latest = latest_tag(remote_tags[url], False) or "(none)"
+ prefix = "[!] " if package.tag != latest else ""
+ print(f"{prefix}{name} curr: {package.tag} latest: {latest}")
+
+
def upgrade(cfg):
- """Upgrade tag files in buildconfig to the latest upstream tag"""
+ """Upgrade package tags to the latest tag from their repositories."""
names = cfg.components
+ explicitly_selected = bool(names)
if not names:
- names = sorted(p.stem for p in Path("buildconfig").glob("*.tag"))
+ names = sorted(config.packages)
+ unknown = sorted(set(names) - set(config.packages))
+ if unknown:
+ raise ConfigError(f"unknown package(s): {', '.join(unknown)}")
# Multiple components can share a repo, only ask each remote once.
remote_tags = {}
upgraded = []
+ updated_packages = dict(config.packages)
for name in names:
- tag_file = Path("buildconfig") / f"{name}.tag"
- url_file = Path("buildconfig") / f"{name}.giturl"
- if not url_file.exists():
- print(f"[?] {name} has no giturl, skipping", file=sys.stderr)
+ package = config.packages[name]
+ if not package.auto_upgrade and not explicitly_selected:
+ print(f" {name} {package.tag} (automatic upgrades disabled)")
continue
- giturl = url_file.read_text().strip()
+ giturl = config.repository_for(package).url
if giturl not in remote_tags:
remote_tags[giturl] = list_remote_tags(giturl)
latest = latest_tag(remote_tags[giturl], cfg.dev)
if latest is None:
- print(f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr)
+ print(
+ f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr
+ )
continue
- curr = None
- if tag_file.exists():
- curr = tag_file.read_text().strip()
- currkey = tag_sortkey(curr)
- if currkey is None:
- print(
- f"[?] {name} tag {curr} has unsupported syntax, skipping",
- file=sys.stderr,
- )
- continue
- latestkey = tag_sortkey(latest)
- if currkey > latestkey:
- # Happens when the tag file pins a dev version but only
- # production tags are considered.
- print(f" {name} {curr} (newer than latest {latest})")
- continue
- if currkey == latestkey:
- print(f" {name} {curr} (up to date)")
- continue
- print(f"[!] {name} {curr or '(none)'} -> {latest}")
+ curr = package.tag
+ currkey = tag_sortkey(curr)
+ if currkey is None:
+ print(
+ f"[?] {name} tag {curr} has unsupported syntax, skipping",
+ file=sys.stderr,
+ )
+ continue
+ latestkey = tag_sortkey(latest)
+ if currkey > latestkey:
+ # Happens when a package pins a dev version but only production
+ # tags are considered.
+ print(f" {name} {curr} (newer than latest {latest})")
+ continue
+ if currkey == latestkey:
+ print(f" {name} {curr} (up to date)")
+ continue
+ print(f"[!] {name} {curr} -> {latest}")
upgraded.append(name)
if not cfg.dry:
- tag_file.write_text(latest + "\n")
+ updated_packages[name] = replace(package, tag=latest)
if not upgraded:
print("nothing to upgrade")
elif cfg.dry:
print("would upgrade:", " ".join(upgraded))
else:
+ write_config(config_path, replace(config, packages=updated_packages))
print("upgraded:", " ".join(upgraded))
@@ -644,6 +961,11 @@ def main():
subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND")
+ parser_init = subparsers.add_parser(
+ "init", help="Initialize aptly repositories and publications."
+ )
+ parser_init.set_defaults(func=initialize)
+
# subcommand build
parser_build = subparsers.add_parser("build", help="Build packages for distro.")
@@ -653,16 +975,15 @@ def main():
parser_build.add_argument(
"--no-transitive",
help="Do not build transitive deps of changed components (default)",
- action="store_true",
+ action="store_false",
dest="transitive",
- default=None,
+ default=False,
)
parser_build.add_argument(
"--transitive",
help="Build transitive deps of changed components",
- action="store_false",
+ action="store_true",
dest="transitive",
- default=None,
)
parser_build.add_argument(
"--arch",
@@ -702,7 +1023,7 @@ def main():
parser_upgrade.add_argument(
"components",
nargs="*",
- help="Components to upgrade (default: all components in buildconfig)",
+ help="Components to upgrade (default: all packages in packages.toml)",
)
parser_upgrade.add_argument(
"--dev",
@@ -722,9 +1043,22 @@ def main():
# subcommand show-published
parser_show_published = subparsers.add_parser(
- "show-published", help="Show published packages on deb.taler.net"
+ "show-published", help="Show packages in a published stable or testing snapshot"
+ )
+ parser_show_published.add_argument(
+ "--latest",
+ action="store_true",
+ help="Show only the newest version of each package per architecture",
+ )
+ parser_show_published.add_argument(
+ "distro",
+ choices=[
+ f"{distro}{suffix}"
+ for distro in publishing_configs
+ for suffix in ("", "-stable", "-testing")
+ ],
+ help="Base distro (stable by default), or distro-stable/distro-testing",
)
- parser_show_published.add_argument("distro")
parser_show_published.set_defaults(func=show_published)
# subcommand publish
@@ -738,7 +1072,10 @@ def main():
parser_promote = subparsers.add_parser("promote", help="Promote testing to stable")
parser_promote.add_argument(
- "--dry", help="Dry run (show pulls)", action="store_true", default=False
+ "--dry",
+ help="Dry run (show testing changes)",
+ action="store_true",
+ default=False,
)
parser_promote.add_argument("distro")
parser_promote.set_defaults(func=promote)
