commit efbad6d7f38ba8649e60573b8b12bb391fcc5a7e
parent 029285e5b24b7fc9fea99c88ed612b4b80a07c71
Author: Christian Grothoff <christian@grothoff.org>
Date: Tue, 27 Oct 2026 22:51:12 +0100
resolve
Diffstat:
4 files changed, 331 insertions(+), 652 deletions(-)
diff --git a/packaging/ng/buildconfig/pepsi.tag b/packaging/ng/buildconfig/pepsi.tag
@@ -0,0 +1 @@
+v0.1.0
+\ No newline at end of file
diff --git a/packaging/ng/buildconfig/taler-rust.tag b/packaging/ng/buildconfig/taler-rust.tag
@@ -0,0 +1 @@
+v1.6.2
diff --git a/packaging/ng/distros/debian-trixie.Dockerfile b/packaging/ng/distros/debian-trixie.Dockerfile
@@ -4,6 +4,7 @@ FROM debian:trixie
# Docker image to build Ubuntu packages of
# GNUnet, GNU Taler and GNU Anastasis.
+
ARG DEBIAN_FRONTEND=noninteractive
ENV AWS_LC_SYS_NO_JITTER_ENTROPY=1
@@ -14,7 +15,6 @@ RUN \
apt-get -y full-upgrade \
&& \
apt-get -y install \
- rustup \
build-essential zip jq python3 python3-pip nodejs npm \
autoconf automake libtool libltdl-dev \
libpq-dev libsqlite3-dev libunistring-dev libqrencode-dev libgcrypt-dev \
@@ -27,14 +27,27 @@ RUN \
libgtk-3-dev libgladeui-dev libmagic-dev \
dbconfig-no-thanks \
devscripts equivs \
- python3-click python3-requests \
+ python3-click python3-requests python3-sphinx-rtd-theme \
meson \
default-jdk-headless \
golang \
postgresql \
&& \
- pip install sphinx_rtd_theme --break-system-packages \
- && \
- npm install -g pnpm@10 && \
- rustup default stable
+ npm install -g pnpm@10
+
+RUN echo "Types: deb\nURIs: https://mirror.init7.net/debian/\nSuites: trixie-updates trixie-backports\nComponents: main\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" \
+ > /etc/apt/sources.list.d/trixie-backports.sources
+
+RUN echo "Types: deb\nURIs: https://mirror.init7.net/debian/\nSuites: trixie-updates trixie-updates\nComponents: main\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" \
+ > /etc/apt/sources.list.d/trixie-updates.sources
+RUN printf 'Package: *\nPin: release n=trixie-backports\nPin-Priority: 500\n' \
+ > /etc/apt/preferences.d/trixie-backports
+
+RUN printf 'Package: *\nPin: release n=trixie-updates\nPin-Priority: 550\n' \
+ > /etc/apt/preferences.d/trixie-updates
+
+RUN \
+ apt-get update \
+ && apt-get install -y --no-install-recommends \
+ -t trixie-backports rustc cargo
diff --git a/packaging/ng/taler-pkg b/packaging/ng/taler-pkg
@@ -5,14 +5,11 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import argparse
-import datetime
-import os
+import subprocess
import platform
+import os
import re
-import shlex
-import subprocess
import sys
-from dataclasses import dataclass, replace
from pathlib import Path
# Make local util package available
@@ -20,95 +17,63 @@ file = Path(__file__).resolve()
parent, root = file.parent, file.parents[1]
sys.path.append(str(root))
-from util import vercomp # noqa: E402
+from util import vercomp
mydir = os.path.dirname(os.path.realpath(__file__))
-sys.path.append(os.path.join(mydir, "buildscripts"))
-
-from package_config import ConfigError, load_config, write_config # noqa: E402
archs = ["arm64", "amd64"]
host = "taler.net"
-remote_user = "taler-packaging"
-signing_key = "0084993C2C6CDF471A7D7EFD26E546A5FE7E0266"
native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64"
-config_path = Path(mydir) / "packages.toml"
-config = load_config(config_path)
-components = config.enabled_packages()
-
-
-@dataclass(frozen=True)
-class PublishingConfig:
- distro: str
- vendor: str
- codename: str
-
- @property
- def prefix(self):
- return f"apt/{self.vendor}"
-
- @property
- def testing_repo(self):
- return f"taler-{self.vendor}-{self.codename}-testing"
-
- @property
- def testing_distribution(self):
- return f"{self.codename}-testing"
-
- @property
- def initial_snapshot(self):
- return f"taler-{self.vendor}-{self.codename}-stable-initial"
-
-
-publishing_configs = {
- distro: PublishingConfig(distro, vendor, codename)
- for distro, vendor, codename in (
- ("debian-trixie", "debian", "trixie"),
- ("ubuntu-noble", "ubuntu", "noble"),
- )
-}
-
-
-def publishing_config(distro):
- try:
- return publishing_configs[distro]
- except KeyError:
- raise ValueError(f"unsupported publishing distro: {distro}") from None
-
-
-def remote_command(command, *, capture_output=False, tty=False):
- ssh_command = ["ssh"]
- if tty:
- ssh_command.append("-t")
- ssh_command.extend(
- [f"{remote_user}@{host}", shlex.join(str(arg) for arg in command)]
- )
- return subprocess.run(
- ssh_command,
- check=True,
- capture_output=capture_output,
- text=capture_output,
- )
-
-
-def remote_aptly(*args, capture_output=False, tty=False):
- return remote_command(["aptly", *args], capture_output=capture_output, tty=tty)
+components = [
+ "taler-wallet-cli",
+ "taler-merchant-webui",
+ "taler-exchange-kyc-webui",
+ "taler-exchange-aml-webui",
+ "taler-auditor-webui",
+ "taler-challenger-helpers",
+ "challenger-webui",
+ "gnunet",
+ "libeufin",
+ "donau",
+ "paivana-httpd",
+ "challenger",
+ "taler-exchange",
+ "taler-harness",
+ "taler-merchant",
+ "taler-rust",
+ "robocop",
+ "pepsi",
+ #"depolymerization",
+ # These two packages don't have good debs yet,
+ # Debian complains "No section given for ..., skipping.
+ # "taler-directory",
+ # "taler-mailbox",
+ # We don't publish packages for these yet
+ # "taler-mdb",
+ # "taler-merchant-demos",
+ "anastasis",
+ "anastasis-gtk",
+ # Currently not used anywhere
+ # "sync",
+]
deps = {
- name: list(package.dependencies)
- for name, package in config.packages.items()
- if package.dependencies
+ "taler-exchange": ["gnunet"],
+ "anastasis": ["gnunet", "taler-merchant"],
+ "anastasis-gtk": ["anastasis"],
+ "taler-merchant": ["gnunet", "taler-exchange", "donau"],
+ "donau": ["gnunet", "taler-exchange"],
+ "challenger": ["taler-exchange"],
+ # "taler-mdb": ["gnunet", "taler-exchange", "taler-merchant"],
+ "sync": ["taler-merchant", "taler-exchange", "gnunet"],
+ "paivana-httpd": ["taler-merchant", "taler-exchange", "gnunet"],
}
# Compute reverse dependencies
rdeps = {}
for n1, d in deps.items():
- if n1 not in components:
- continue
for n2 in d:
- if n2 not in components:
- continue
rd = rdeps.setdefault(n2, [])
if n1 not in rd:
rd.append(n1)
@@ -160,7 +125,7 @@ def find_outdated(pkgdir, arch, roots):
"""Find outdated components based on tag files"""
outdated = set()
for component in roots:
- ver_requested = config.packages[component].tag
+ ver_requested = open(f"buildconfig/{component}.tag").read().strip()
built_tag_file = pkgdir / f"{component}@{arch}.built.tag"
ver_built = None
if built_tag_file.exists():
@@ -171,58 +136,10 @@ def find_outdated(pkgdir, arch, roots):
return outdated
-def component_group_key(component):
- package = config.packages[component]
- repository = config.repository_for(package)
- builder = config.builder_for(package)
- if builder != "pnpm-workspace":
- # Single-package builders are deliberately isolated even when repositories match.
- return repository.url, package.tag, builder, component
- return repository.url, package.tag, builder, ""
-
-
-def group_components(selected, buildorder):
- """Group selected components and topologically order the resulting groups."""
- selected = set(selected)
- groups = {}
- component_to_group = {}
- for component in buildorder:
- if component not in selected:
- continue
- key = component_group_key(component)
- groups.setdefault(key, []).append(component)
- component_to_group[component] = key
-
- group_dependencies = {key: set() for key in groups}
- for component, key in component_to_group.items():
- for dependency in deps.get(component, []):
- dependency_key = component_to_group.get(dependency)
- if dependency_key is not None and dependency_key != key:
- group_dependencies[key].add(dependency_key)
-
- ordered = []
- permanent = set()
- temporary = set()
-
- def visit(key):
- if key in permanent:
- return
- if key in temporary:
- raise ConfigError("build groups contain a dependency cycle")
- temporary.add(key)
- for dependency in sorted(group_dependencies[key]):
- visit(dependency)
- temporary.remove(key)
- permanent.add(key)
- ordered.append(groups[key])
-
- for key in groups:
- visit(key)
- return ordered
-
-
def build(cfg):
- transitive = cfg.transitive
+ transitive = False
+ if cfg.transitive:
+ transitive = True
distro = cfg.distro
vendor, codename = distro.split("-", 1)
print("building", distro)
@@ -276,10 +193,14 @@ def build(cfg):
print("outdated closure", closure)
- selected = closure if transitive else outdated
- for group in group_components(selected, buildorder):
- builder = config.builder_for(config.packages[group[0]])
- print("building", " ".join(group), f"with {builder}")
+ for component in buildorder:
+ if transitive:
+ if component not in closure:
+ continue
+ else:
+ if component not in outdated:
+ continue
+ print("building", component)
pkgdir.mkdir(parents=True, exist_ok=True)
cmd = [
"podman",
@@ -303,20 +224,20 @@ def build(cfg):
"--env",
"CARGO_BUILD_BUILD_DIR=/root/.cargo-build",
"--mount",
- f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared,U=true",
+ f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared",
"--mount",
- f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared,U=true",
+ f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared",
"--mount",
f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly",
"--mount",
- f"type=bind,source={config_path},target=/packages.toml,readonly",
+ f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly",
"--mount",
f"type=bind,source={pkgdir},target=/pkgdir",
image_tag,
- f"/buildscripts/{builder}",
+ "/buildscripts/generic",
+ component,
codename,
arch,
- *group,
]
if not cfg.dry:
subprocess.run(
@@ -330,324 +251,57 @@ def show_order(cfg):
print("build order:", buildorder)
-def aptly_list(*args):
- result = remote_aptly(*args, "-raw", capture_output=True)
- return set(result.stdout.split())
-
-
-def published_repositories():
- result = remote_aptly("publish", "list", "-raw", capture_output=True)
- repositories = set()
- for line in result.stdout.splitlines():
- fields = line.split()
- if len(fields) != 2:
- raise ValueError(f"unexpected aptly publish list line: {line}")
- repositories.add(tuple(fields))
- return repositories
-
-
-def parse_publication(output):
- publication = {"sources": {}}
- in_sources = False
- for line in output.splitlines():
- if line == "Sources:":
- in_sources = True
- continue
- if in_sources and line.startswith(" "):
- match = re.fullmatch(r"\s+([^:]+):\s+(.+)\s+\[([^]]+)]", line)
- if match is None:
- raise ValueError(f"unexpected aptly publication source: {line}")
- component, name, kind = match.groups()
- publication["sources"][component] = (name, kind)
- continue
- in_sources = False
- if line.startswith("Prefix: "):
- publication["prefix"] = line.removeprefix("Prefix: ")
- elif line.startswith("Distribution: "):
- publication["distribution"] = line.removeprefix("Distribution: ")
- elif line.startswith("Architectures: "):
- publication["architectures"] = set(
- line.removeprefix("Architectures: ").replace(",", " ").split()
- )
- return publication
-
-
-def get_publication(distribution, prefix):
- result = remote_aptly("publish", "show", distribution, prefix, capture_output=True)
- return parse_publication(result.stdout)
-
-
-def snapshot_origin(snapshot):
- result = remote_aptly("snapshot", "show", snapshot, capture_output=True)
- match = re.search(r"Snapshot from local repo \[([^]]+)]", result.stdout)
- if match is not None:
- return match.group(1)
- in_sources = False
- for line in result.stdout.splitlines():
- if line == "Sources:":
- in_sources = True
- continue
- if in_sources and line.startswith(" "):
- match = re.fullmatch(r"\s+(.+)\s+\[(?:local|repo)]", line)
- if match is not None:
- return match.group(1)
- elif in_sources:
- break
- raise ValueError(f"cannot determine source repo for aptly snapshot {snapshot}")
-
-
-def validate_publication(pubcfg, publication, *, stable):
- distribution = pubcfg.codename if stable else pubcfg.testing_distribution
- if publication.get("prefix") != pubcfg.prefix:
- raise ValueError(f"unexpected prefix for {distribution} publication")
- if publication.get("distribution") != distribution:
- raise ValueError(f"unexpected distribution for {distribution} publication")
- if publication.get("architectures") != set(archs):
- raise ValueError(f"unexpected architectures for {distribution} publication")
- snapshot = publication_snapshot(publication)
- validate_snapshot(pubcfg, snapshot, stable=stable)
-
-
-def validate_snapshot(pubcfg, snapshot, *, stable):
- if stable and snapshot == pubcfg.initial_snapshot:
- if snapshot_packages(snapshot):
- raise ValueError(f"initial snapshot {snapshot} is not empty")
- else:
- origin = snapshot_origin(snapshot)
- if origin != pubcfg.testing_repo:
- distribution = pubcfg.codename if stable else pubcfg.testing_distribution
- raise ValueError(
- f"publication {distribution} has unexpected source {origin}"
- )
-def publish_options(distribution):
- return [
- f"-architectures={','.join(sorted(archs))}",
- "-component=main",
- f"-distribution={distribution}",
- f"-gpg-key={signing_key}",
- "-origin=GNU Taler",
- "-label=Taler",
- ]
-
-
-def initialize(cfg):
- del cfg
- repos = aptly_list("repo", "list")
- snapshots = aptly_list("snapshot", "list")
- publications = published_repositories()
- changed = False
-
- for pubcfg in publishing_configs.values():
- if pubcfg.testing_repo not in repos:
- remote_aptly(
- "repo",
- "create",
- f"-distribution={pubcfg.testing_distribution}",
- "-component=main",
- pubcfg.testing_repo,
- )
- repos.add(pubcfg.testing_repo)
- changed = True
-
- testing_key = (pubcfg.prefix, pubcfg.testing_distribution)
- if testing_key not in publications:
- snapshot = create_testing_snapshot(pubcfg)
- remote_aptly(
- "publish",
- "snapshot",
- *publish_options(pubcfg.testing_distribution),
- snapshot,
- pubcfg.prefix,
- tty=True,
- )
- publications.add(testing_key)
- changed = True
- else:
- validate_publication(
- pubcfg,
- get_publication(pubcfg.testing_distribution, pubcfg.prefix),
- stable=False,
- )
-
- stable_key = (pubcfg.prefix, pubcfg.codename)
- if stable_key not in publications:
- if pubcfg.initial_snapshot not in snapshots:
- remote_aptly(
- "snapshot",
- "create",
- pubcfg.initial_snapshot,
- "empty",
- )
- snapshots.add(pubcfg.initial_snapshot)
- changed = True
- else:
- validate_snapshot(pubcfg, pubcfg.initial_snapshot, stable=True)
- remote_aptly(
- "publish",
- "snapshot",
- *publish_options(pubcfg.codename),
- pubcfg.initial_snapshot,
- pubcfg.prefix,
- tty=True,
- )
- publications.add(stable_key)
- changed = True
- else:
- validate_publication(
- pubcfg,
- get_publication(pubcfg.codename, pubcfg.prefix),
- stable=True,
- )
-
- if changed:
- print("aptly repositories and publications initialized")
+def promote(cfg):
+ dry = cfg.dry
+ distro = cfg.distro
+ vendor, codename = distro.split("-", 1)
+ listfmt = "${package}_${version}_${architecture}.${$type}\n"
+ if dry:
+ subprocess.run(
+ [
+ "ssh",
+ f"taler-packaging@{host}",
+ f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ checkpull {codename}",
+ ],
+ check=True,
+ )
else:
- print("aptly repositories and publications already initialized")
-
-
-def parse_package_list(output):
- packages = []
- in_packages = False
- for line in output.splitlines():
- if line == "Packages:":
- in_packages = True
- continue
- if in_packages and line.startswith(" "):
- packages.append(line.strip())
- elif in_packages:
- break
- return packages
-
-
-def repo_packages(repo):
- result = remote_aptly("repo", "show", "-with-packages", repo, capture_output=True)
- return parse_package_list(result.stdout)
-
-
-def snapshot_packages(snapshot):
- result = remote_aptly(
- "snapshot", "show", "-with-packages", snapshot, capture_output=True
- )
- return parse_package_list(result.stdout)
-
-
-def publishing_packages(pubcfg):
- """Include pending testing imports and publications sharing its package pool."""
- sources = {(pubcfg.testing_repo, "local")}
- for prefix, distribution in sorted(published_repositories()):
- if prefix != pubcfg.prefix:
- continue
- publication = get_publication(distribution, prefix)
- if (
- publication.get("prefix") != prefix
- or publication.get("distribution") != distribution
- or not publication.get("sources")
- ):
- raise ValueError(f"invalid publication details for {prefix}/{distribution}")
- source = publication["sources"].get("main")
- if source is not None:
- sources.add(source)
-
- readers = {"local": repo_packages, "snapshot": snapshot_packages}
- packages = set()
- for name, kind in sorted(sources):
- if kind not in readers:
- raise ValueError(f"unsupported publication source {name} [{kind}]")
- packages.update(readers[kind](name))
- return sorted(packages)
-
-
-def publication_snapshot(publication):
- distribution = publication.get("distribution", "unknown")
- sources = publication.get("sources", {})
- if set(sources) != {"main"}:
- raise ValueError(f"unexpected components for {distribution} publication")
- snapshot, kind = sources["main"]
- if kind != "snapshot":
- raise ValueError(
- f"publication {distribution} uses a {kind} source; "
- "convert it to a snapshot publication before proceeding"
+ subprocess.run(
+ [
+ "ssh",
+ "-t",
+ f"taler-packaging@{host}",
+ f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ pull {codename}",
+ ],
+ check=True,
+ )
+ # Always export!
+ # Reprepro is weird, listed packages might actually not show
+ # up in the index yet.
+ subprocess.run(
+ [
+ "ssh",
+ "-t",
+ f"taler-packaging@{host}",
+ f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}",
+ ],
+ check=True,
)
- return snapshot
-
-
-def published_snapshot(pubcfg, *, stable):
- """Return the published snapshot name and its enabled architectures."""
- distribution = pubcfg.codename if stable else pubcfg.testing_distribution
- publication = get_publication(distribution, pubcfg.prefix)
- if (
- publication.get("prefix") != pubcfg.prefix
- or publication.get("distribution") != distribution
- or not publication.get("architectures")
- ):
- raise ValueError(f"invalid publication details for {pubcfg.prefix}/{distribution}")
- return publication_snapshot(publication), publication["architectures"]
-
-
-def testing_snapshot_name(pubcfg, now=None):
- if now is None:
- now = datetime.datetime.now(datetime.timezone.utc)
- timestamp = now.strftime("%Y%m%dT%H%M%S%fZ")
- return f"{pubcfg.testing_repo}-{timestamp}"
-
-
-def create_testing_snapshot(pubcfg):
- snapshot = testing_snapshot_name(pubcfg)
- remote_aptly("snapshot", "create", snapshot, "from", "repo", pubcfg.testing_repo)
- return snapshot
-
-
-def promote(cfg):
- pubcfg = publishing_config(cfg.distro)
- snapshot, _ = published_snapshot(pubcfg, stable=False)
- if cfg.dry:
- stable_name, _ = published_snapshot(pubcfg, stable=True)
- stable = set(snapshot_packages(stable_name))
- testing = set(snapshot_packages(snapshot))
- for package in sorted(stable - testing):
- print(f"- {package}")
- for package in sorted(testing - stable):
- print(f"+ {package}")
- if stable == testing:
- print("testing and stable contain the same packages")
- return
-
- remote_aptly(
- "publish",
- "switch",
- pubcfg.codename,
- pubcfg.prefix,
- snapshot,
- tty=True,
- )
- print(f"promoted published testing snapshot {snapshot}")
-
def show_published(cfg):
- distro, _, channel = cfg.distro.rpartition("-")
- if channel not in ("stable", "testing"):
- distro, channel = cfg.distro, "stable"
- pubcfg = publishing_config(distro)
- snapshot, architectures = published_snapshot(pubcfg, stable=channel == "stable")
- packages = {
- package
- for package in snapshot_packages(snapshot)
- if package_identity(package)[2] in architectures | {"all"}
- }
- if cfg.latest:
- latest = {}
- for package in sorted(packages):
- name, version, architecture = package_identity(package)
- key = (name, architecture)
- if key not in latest or vercomp.compare_versions(
- version, latest[key][0]
- ) > 0:
- latest[key] = (version, package)
- packages = {package for _, package in latest.values()}
- for package in sorted(packages):
- print(package)
+ distro = cfg.distro
+ vendor, codename = distro.split("-", 1)
+ listfmt = "${package}_${version}_${architecture}.${$type}\n"
+ subprocess.run(
+ [
+ "ssh",
+ f"taler-packaging@{host}",
+ f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}",
+ ],
+ check=True,
+ )
def test(cfg):
@@ -656,7 +310,7 @@ def test(cfg):
distro = f"{vendor}-{codename}"
image_tag = f"localhost/taler-packaging-{distro}:latest"
dockerfile = f"distros/{distro}.Dockerfile"
- cachedir = Path("cache").absolute()
+ cachedir = Path(f"cache").absolute()
print("building base image")
subprocess.run(
[
@@ -691,136 +345,183 @@ def test(cfg):
check=True,
)
+def publish(cfg):
+ distro = cfg.distro
+ if distro.endswith("-testing"):
+ print("Files are automatically published to testing", file=sys.stderr)
+ sys.exit(1)
+ vendor, codename = distro.split("-", 1)
+ # List of .deb and .ddeb files.
+ debs = []
+ listfmt = "${package}_${version}_${architecture}.${$type}\n"
+ server_debs_str = subprocess.check_output(
+ [
+ "ssh",
+ f"taler-packaging@{host}",
+ f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}",
+ ],
+ encoding="utf-8",
+ )
+ server_debs = server_debs_str.split()
+ for component in components:
+ current = []
+ for arch in archs + ["all"]:
+ cf = Path(f"./packages/{distro}/{component}@{arch}.built.current")
+ if not cf.exists():
+ print(f"component {component}@{arch} has no current packages")
+ continue
+ with open(cf) as f:
+ current = current + f.read().split()
+ print("current", current)
+ for deb in current:
+ if deb.endswith(".deb"):
+ pkg1, ver1, arch1 = deb.removesuffix(".deb").split("_")
+ elif deb.endswith(".ddeb"):
+ pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_")
+ else:
+ raise Error(f"invalid deb filename: {deb}")
+ fresh = True
+ server_deb = None
+ # If the server has the same or a later version,
+ # the local version isn't fresh.
+ for srvdeb in server_debs:
+ pkg2, ver2, arch2 = srvdeb.removesuffix(".deb").split("_")
+ if pkg1 != pkg2 or arch1 != arch2:
+ continue
+ if vercomp.compare_versions(ver1, ver2) <= 0:
+ fresh = False
+ server_deb = srvdeb
+ break
+ if fresh:
+ debs.append(deb)
+ else:
+ print("package", deb, "not fresh, server has", server_deb)
+ if len(debs) == 0:
+ print("nothing to upload")
+ else:
+ print("uploading debs", debs)
+ if cfg.dry:
+ return
+ debs = [Path(f"./packages/{distro}/") / x for x in debs]
+ subprocess.run(
+ [
+ "ssh",
+ f"taler-packaging@{host}",
+ f"rm -f '/home/taler-packaging/{distro}/'*.deb '/home/taler-packaging/{distro}/'*.ddeb",
+ ],
+ check=True,
+ )
+ subprocess.run(
+ ["rsync", "-a", "--info=progress2", "--", *debs, f"taler-packaging@{host}:{distro}/"], check=True
+ )
+ ret = subprocess.run(
+ [
+ "ssh",
+ "-t",
+ f"taler-packaging@{host}",
+ f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.deb",
+ ],
+ )
+ if ret.returncode != 0:
+ # Usually not critical if it fails.
+ print(
+ "Including ddebs failed. This can happen when including packages that have been included previously"
+ )
+ # Almost the same, but with ddebs.
+ # We explicitly need to tell reprepro
+ # to ignore the extension, because it does not
+ # deal well with ddebs out of the box.
+ ret = subprocess.run(
+ [
+ "ssh",
+ "-t",
+ f"taler-packaging@{host}",
+ f"reprepro --ignore=extension -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.ddeb",
+ ],
+ )
+ if ret.returncode != 0:
+ # Usually not critical if it fails.
+ print(
+ "Including ddebs failed. This can happen when including packages that have been included previously"
+ )
+ # Always export!
+ # Reprepro is weird, listed packages might actually not show
+ # up in the index yet.
+ subprocess.run(
+ [
+ "ssh",
+ "-t",
+ f"taler-packaging@{host}",
+ f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing",
+ ],
+ check=True,
+ )
-def package_identity(filename):
- stem = filename
- for suffix in (".ddeb", ".deb"):
- if filename.endswith(suffix):
- stem = filename.removesuffix(suffix)
- break
- fields = stem.rsplit("_", 2)
- if len(fields) != 3:
- raise ValueError(f"invalid package reference: {filename}")
- return fields
+def get_remote_version(url):
+ """Get the latest stable tag from the git repo"""
+ # Construct the git command
+ # We use -c versionsort.suffix=- to ensure correct semantic version sorting
+ cmd = [
+ "git",
+ "-c",
+ "versionsort.suffix=-",
+ "ls-remote",
+ "--exit-code",
+ "--refs",
+ "--sort=version:refname",
+ "--tags",
+ url,
+ "*.*.*",
+ ]
-def package_file_identity(filename):
- if Path(filename).name != filename or not filename.endswith((".deb", ".ddeb")):
- raise ValueError(f"invalid package filename: {filename}")
- return package_identity(filename)
+ result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+ # Parse the output
+ # Output format is usually: <hash>\trefs/tags/<tagname>
+ lines = result.stdout.strip().split("\n")
-def newest_server_package(package, architecture, server_packages):
- newest = None
- newest_ref = None
- for server_package in server_packages:
- server_name, server_version, server_arch = package_identity(server_package)
- if package != server_name or architecture != server_arch:
+ valid_tags = []
+
+ for line in lines:
+ parts = line.split()
+ if len(parts) < 2:
continue
- if newest is None or vercomp.compare_versions(server_version, newest) > 0:
- newest = server_version
- newest_ref = server_package
- return newest, newest_ref
+ # refs/tags/v1.0.0 -> v1.0.0
+ ref_path = parts[1]
+ tag = ref_path.split("/")[-1]
-def current_package_files(distro):
- current = []
- seen = set()
- for component in components:
- component_current = []
- for arch in archs + ["all"]:
- current_file = Path(f"./packages/{distro}/{component}@{arch}.built.current")
- if not current_file.exists():
- print(f"component {component}@{arch} has no current packages")
- continue
- component_current.extend(current_file.read_text().split())
- print("current", component_current)
- for package in component_current:
- if package not in seen:
- current.append(package)
- seen.add(package)
- return current
+ # Exclude pre-release semver versions
+ if tag.startswith("v") and "-" in tag:
+ continue
+ valid_tags.append(tag)
-def cleanup_uploads(staging_dir):
- remote_command(
- [
- "sh",
- "-c",
- (
- 'if [ ! -e "$1" ]; then exit 0; fi\n'
- 'find "$1" -maxdepth 1 -type f '
- r"\( -name '*.deb' -o -name '*.ddeb' \) -delete"
- ),
- "taler-pkg-cleanup",
- staging_dir,
- ]
- )
+ if valid_tags:
+ return valid_tags[-1]
+ return "(none)"
-def publish(cfg):
- distro = cfg.distro
- if distro.endswith("-testing"):
- print("Publish the base distro; packages always go to testing", file=sys.stderr)
- sys.exit(1)
- pubcfg = publishing_config(distro)
- published_snapshot(pubcfg, stable=False)
- server_packages = publishing_packages(pubcfg)
- uploads = []
- for package_file in current_package_files(distro):
- package, version, architecture = package_file_identity(package_file)
- server_version, server_ref = newest_server_package(
- package, architecture, server_packages
- )
- if (
- server_version is None
- or vercomp.compare_versions(version, server_version) > 0
- ):
- uploads.append(package_file)
- else:
- print("package", package_file, "not fresh, server has", server_ref)
-
- if uploads:
- print("uploading debs", uploads)
- else:
- print("nothing to upload")
- if cfg.dry:
- return
-
- staging_dir = f"/home/{remote_user}/{distro}"
- if uploads:
- remote_command(["mkdir", "-p", staging_dir])
- local_files = [Path(f"./packages/{distro}") / name for name in uploads]
- remote_files = [f"{staging_dir}/{name}" for name in uploads]
- subprocess.run(
- [
- "rsync",
- "-a",
- "--info=progress2",
- "--",
- *local_files,
- f"{remote_user}@{host}:{staging_dir}/",
- ],
- check=True,
- )
- remote_aptly("repo", "add", pubcfg.testing_repo, *remote_files)
-
- snapshot = create_testing_snapshot(pubcfg)
- remote_aptly(
- "publish",
- "switch",
- pubcfg.testing_distribution,
- pubcfg.prefix,
- snapshot,
- tty=True,
- )
- try:
- cleanup_uploads(staging_dir)
- except (subprocess.CalledProcessError, OSError) as exc:
- raise RuntimeError(
- f"Publishing {pubcfg.testing_distribution} succeeded, "
- f"but cleanup of {staging_dir} failed"
- ) from exc
+def check_version(name, url):
+ """
+ Compares local buildconfig version with remote git version.
+ """
+ ver = get_remote_version(url)
+ config_path = os.path.join("buildconfig", f"{name}.tag")
+ with open(config_path, "r") as f:
+ curr = f.read().strip()
+ prefix = "[!] " if curr != ver else ""
+ print(f"{prefix}{name} curr: {curr} latest: {ver}")
+
+
+def print_latest(cfg):
+ """Print latest upstream tag for each component"""
+ for name in components:
+ config_path = os.path.join("buildconfig", f"{name}.giturl")
+ with open(config_path, "r") as f:
+ giturl = f.read().strip()
+ check_version(name, giturl)
# Tag syntax variants supported by buildscripts/generic:
@@ -884,73 +585,55 @@ def latest_tag(tags, dev):
return best
-def print_latest(cfg):
- """Print the latest stable upstream tag for each enabled component."""
- remote_tags = {}
- for name in components:
- package = config.packages[name]
- url = config.repository_for(package).url
- if url not in remote_tags:
- remote_tags[url] = list_remote_tags(url)
- latest = latest_tag(remote_tags[url], False) or "(none)"
- prefix = "[!] " if package.tag != latest else ""
- print(f"{prefix}{name} curr: {package.tag} latest: {latest}")
-
-
def upgrade(cfg):
- """Upgrade package tags to the latest tag from their repositories."""
+ """Upgrade tag files in buildconfig to the latest upstream tag"""
names = cfg.components
- explicitly_selected = bool(names)
if not names:
- names = sorted(config.packages)
- unknown = sorted(set(names) - set(config.packages))
- if unknown:
- raise ConfigError(f"unknown package(s): {', '.join(unknown)}")
+ names = sorted(p.stem for p in Path("buildconfig").glob("*.tag"))
# Multiple components can share a repo, only ask each remote once.
remote_tags = {}
upgraded = []
- updated_packages = dict(config.packages)
for name in names:
- package = config.packages[name]
- if not package.auto_upgrade and not explicitly_selected:
- print(f" {name} {package.tag} (automatic upgrades disabled)")
+ tag_file = Path("buildconfig") / f"{name}.tag"
+ url_file = Path("buildconfig") / f"{name}.giturl"
+ if not url_file.exists():
+ print(f"[?] {name} has no giturl, skipping", file=sys.stderr)
continue
- giturl = config.repository_for(package).url
+ giturl = url_file.read_text().strip()
if giturl not in remote_tags:
remote_tags[giturl] = list_remote_tags(giturl)
latest = latest_tag(remote_tags[giturl], cfg.dev)
if latest is None:
- print(
- f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr
- )
- continue
- curr = package.tag
- currkey = tag_sortkey(curr)
- if currkey is None:
- print(
- f"[?] {name} tag {curr} has unsupported syntax, skipping",
- file=sys.stderr,
- )
- continue
- latestkey = tag_sortkey(latest)
- if currkey > latestkey:
- # Happens when a package pins a dev version but only production
- # tags are considered.
- print(f" {name} {curr} (newer than latest {latest})")
- continue
- if currkey == latestkey:
- print(f" {name} {curr} (up to date)")
+ print(f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr)
continue
- print(f"[!] {name} {curr} -> {latest}")
+ curr = None
+ if tag_file.exists():
+ curr = tag_file.read_text().strip()
+ currkey = tag_sortkey(curr)
+ if currkey is None:
+ print(
+ f"[?] {name} tag {curr} has unsupported syntax, skipping",
+ file=sys.stderr,
+ )
+ continue
+ latestkey = tag_sortkey(latest)
+ if currkey > latestkey:
+ # Happens when the tag file pins a dev version but only
+ # production tags are considered.
+ print(f" {name} {curr} (newer than latest {latest})")
+ continue
+ if currkey == latestkey:
+ print(f" {name} {curr} (up to date)")
+ continue
+ print(f"[!] {name} {curr or '(none)'} -> {latest}")
upgraded.append(name)
if not cfg.dry:
- updated_packages[name] = replace(package, tag=latest)
+ tag_file.write_text(latest + "\n")
if not upgraded:
print("nothing to upgrade")
elif cfg.dry:
print("would upgrade:", " ".join(upgraded))
else:
- write_config(config_path, replace(config, packages=updated_packages))
print("upgraded:", " ".join(upgraded))
@@ -961,11 +644,6 @@ def main():
subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND")
- parser_init = subparsers.add_parser(
- "init", help="Initialize aptly repositories and publications."
- )
- parser_init.set_defaults(func=initialize)
-
# subcommand build
parser_build = subparsers.add_parser("build", help="Build packages for distro.")
@@ -975,15 +653,16 @@ def main():
parser_build.add_argument(
"--no-transitive",
help="Do not build transitive deps of changed components (default)",
- action="store_false",
+ action="store_true",
dest="transitive",
- default=False,
+ default=None,
)
parser_build.add_argument(
"--transitive",
help="Build transitive deps of changed components",
- action="store_true",
+ action="store_false",
dest="transitive",
+ default=None,
)
parser_build.add_argument(
"--arch",
@@ -1023,7 +702,7 @@ def main():
parser_upgrade.add_argument(
"components",
nargs="*",
- help="Components to upgrade (default: all packages in packages.toml)",
+ help="Components to upgrade (default: all components in buildconfig)",
)
parser_upgrade.add_argument(
"--dev",
@@ -1043,22 +722,9 @@ def main():
# subcommand show-published
parser_show_published = subparsers.add_parser(
- "show-published", help="Show packages in a published stable or testing snapshot"
- )
- parser_show_published.add_argument(
- "--latest",
- action="store_true",
- help="Show only the newest version of each package per architecture",
- )
- parser_show_published.add_argument(
- "distro",
- choices=[
- f"{distro}{suffix}"
- for distro in publishing_configs
- for suffix in ("", "-stable", "-testing")
- ],
- help="Base distro (stable by default), or distro-stable/distro-testing",
+ "show-published", help="Show published packages on deb.taler.net"
)
+ parser_show_published.add_argument("distro")
parser_show_published.set_defaults(func=show_published)
# subcommand publish
@@ -1072,10 +738,7 @@ def main():
parser_promote = subparsers.add_parser("promote", help="Promote testing to stable")
parser_promote.add_argument(
- "--dry",
- help="Dry run (show testing changes)",
- action="store_true",
- default=False,
+ "--dry", help="Dry run (show pulls)", action="store_true", default=False
)
parser_promote.add_argument("distro")
parser_promote.set_defaults(func=promote)
