taler-pkg (35824B)
1 #!/usr/bin/env python3 2 3 # Copyright (c) 2024 Taler Systems SA 4 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 5 # SPDX-License-Identifier: GPL-3.0-or-later 6 7 import argparse 8 import datetime 9 import os 10 import platform 11 import re 12 import shlex 13 import subprocess 14 import sys 15 from dataclasses import dataclass, replace 16 from pathlib import Path 17 18 # Make local util package available 19 file = Path(__file__).resolve() 20 parent, root = file.parent, file.parents[1] 21 sys.path.append(str(root)) 22 23 from util import vercomp # noqa: E402 24 25 mydir = os.path.dirname(os.path.realpath(__file__)) 26 sys.path.append(os.path.join(mydir, "buildscripts")) 27 28 from package_config import ConfigError, load_config, write_config # noqa: E402 29 30 archs = ["arm64", "amd64"] 31 host = "taler.net" 32 remote_user = "taler-packaging" 33 signing_key = "0084993C2C6CDF471A7D7EFD26E546A5FE7E0266" 34 native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64" 35 config_path = Path(mydir) / "packages.toml" 36 config = load_config(config_path) 37 components = config.enabled_packages() 38 39 40 @dataclass(frozen=True) 41 class PublishingConfig: 42 distro: str 43 vendor: str 44 codename: str 45 46 @property 47 def prefix(self): 48 return f"apt/{self.vendor}" 49 50 @property 51 def testing_repo(self): 52 return f"taler-{self.vendor}-{self.codename}-testing" 53 54 @property 55 def testing_distribution(self): 56 return f"{self.codename}-testing" 57 58 @property 59 def initial_snapshot(self): 60 return f"taler-{self.vendor}-{self.codename}-stable-initial" 61 62 63 publishing_configs = { 64 distro: PublishingConfig(distro, vendor, codename) 65 for distro, vendor, codename in ( 66 ("debian-trixie", "debian", "trixie"), 67 ("ubuntu-noble", "ubuntu", "noble"), 68 ) 69 } 70 71 72 def publishing_config(distro): 73 try: 74 return publishing_configs[distro] 75 except KeyError: 76 raise ValueError(f"unsupported publishing distro: {distro}") from None 77 78 79 def remote_command(command, *, capture_output=False, tty=False): 80 ssh_command = ["ssh"] 81 if tty: 82 ssh_command.append("-t") 83 ssh_command.extend( 84 [f"{remote_user}@{host}", shlex.join(str(arg) for arg in command)] 85 ) 86 return subprocess.run( 87 ssh_command, 88 check=True, 89 capture_output=capture_output, 90 text=capture_output, 91 ) 92 93 94 def remote_aptly(*args, capture_output=False, tty=False): 95 return remote_command(["aptly", *args], capture_output=capture_output, tty=tty) 96 97 98 deps = { 99 name: list(package.dependencies) 100 for name, package in config.packages.items() 101 if package.dependencies 102 } 103 104 # Compute reverse dependencies 105 rdeps = {} 106 for n1, d in deps.items(): 107 if n1 not in components: 108 continue 109 for n2 in d: 110 if n2 not in components: 111 continue 112 rd = rdeps.setdefault(n2, []) 113 if n1 not in rd: 114 rd.append(n1) 115 116 117 def buildsort(roots): 118 """Toposort transitive closure of roots based on deps""" 119 out = [] 120 stack = list(roots[::-1]) 121 pmark = set() 122 tmark = set() 123 while len(stack): 124 node = stack[-1] 125 if node in pmark: 126 stack.pop() 127 tmark.discard(node) 128 continue 129 done = True 130 for dep in deps.get(node, []): 131 if dep not in pmark: 132 if dep in tmark: 133 raise Exception("cycle") 134 stack.append(dep) 135 tmark.add(node) 136 done = False 137 if done: 138 pmark.add(node) 139 out.append(node) 140 stack.pop() 141 tmark.discard(node) 142 return out 143 144 145 def propagate_outdated(outdated): 146 """Propagate outdatedness to dependees""" 147 closure = set() 148 q = list(outdated) 149 while len(q): 150 n = q.pop() 151 closure.add(n) 152 for r in rdeps.get(n, []): 153 if r not in closure: 154 closure.add(r) 155 q.append(r) 156 return closure 157 158 159 def find_outdated(pkgdir, arch, roots): 160 """Find outdated components based on tag files""" 161 outdated = set() 162 for component in roots: 163 ver_requested = config.packages[component].tag 164 built_tag_file = pkgdir / f"{component}@{arch}.built.tag" 165 ver_built = None 166 if built_tag_file.exists(): 167 ver_built = open(built_tag_file).read().strip() 168 if ver_built != ver_requested: 169 outdated.add(component) 170 print(component, ver_built, "->", ver_requested) 171 return outdated 172 173 174 def component_group_key(component): 175 package = config.packages[component] 176 repository = config.repository_for(package) 177 builder = config.builder_for(package) 178 if builder != "pnpm-workspace": 179 # Single-package builders are deliberately isolated even when repositories match. 180 return repository.url, package.tag, builder, component 181 return repository.url, package.tag, builder, "" 182 183 184 def group_components(selected, buildorder): 185 """Group selected components and topologically order the resulting groups.""" 186 selected = set(selected) 187 groups = {} 188 component_to_group = {} 189 for component in buildorder: 190 if component not in selected: 191 continue 192 key = component_group_key(component) 193 groups.setdefault(key, []).append(component) 194 component_to_group[component] = key 195 196 group_dependencies = {key: set() for key in groups} 197 for component, key in component_to_group.items(): 198 for dependency in deps.get(component, []): 199 dependency_key = component_to_group.get(dependency) 200 if dependency_key is not None and dependency_key != key: 201 group_dependencies[key].add(dependency_key) 202 203 ordered = [] 204 permanent = set() 205 temporary = set() 206 207 def visit(key): 208 if key in permanent: 209 return 210 if key in temporary: 211 raise ConfigError("build groups contain a dependency cycle") 212 temporary.add(key) 213 for dependency in sorted(group_dependencies[key]): 214 visit(dependency) 215 temporary.remove(key) 216 permanent.add(key) 217 ordered.append(groups[key]) 218 219 for key in groups: 220 visit(key) 221 return ordered 222 223 224 def build(cfg): 225 transitive = cfg.transitive 226 distro = cfg.distro 227 vendor, codename = distro.split("-", 1) 228 print("building", distro) 229 dockerfile = f"distros/{distro}.Dockerfile" 230 image_tag = f"localhost/taler-packaging-{distro}:latest" 231 pkgdir = Path(f"packages/{distro}").absolute() 232 cachedir = Path("cache").absolute() 233 cachedir.mkdir(exist_ok=True) 234 (cachedir / "cargo-git").mkdir(exist_ok=True) 235 (cachedir / "cargo-registry").mkdir(exist_ok=True) 236 (cachedir / "cargo-build").mkdir(exist_ok=True) 237 (cachedir / "gradle").mkdir(exist_ok=True) 238 (cachedir / "pnpm").mkdir(exist_ok=True) 239 (cachedir / distro / "apt-archives").mkdir(parents=True, exist_ok=True) 240 (cachedir / distro / "apt-lists").mkdir(parents=True, exist_ok=True) 241 242 if cfg.arch is None: 243 arch_list = [native_arch] 244 else: 245 arch_list = cfg.arch.split(",") 246 247 if not cfg.dry: 248 for arch in arch_list: 249 subprocess.run( 250 [ 251 "podman", 252 "build", 253 "--arch", 254 arch, 255 "-v", 256 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 257 "-v", 258 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 259 "-t", 260 image_tag, 261 "-f", 262 dockerfile, 263 ], 264 check=True, 265 ) 266 267 # Sort components by their dependencies 268 buildorder = buildsort(components) 269 print("build order:", buildorder) 270 271 for arch in arch_list: 272 outdated = find_outdated(pkgdir, arch, buildorder) 273 274 # Propagate outdatedness to dependees 275 closure = propagate_outdated(outdated) 276 277 print("outdated closure", closure) 278 279 selected = closure if transitive else outdated 280 for group in group_components(selected, buildorder): 281 builder = config.builder_for(config.packages[group[0]]) 282 print("building", " ".join(group), f"with {builder}") 283 pkgdir.mkdir(parents=True, exist_ok=True) 284 cmd = [ 285 "podman", 286 "run", 287 "-it", 288 "--arch", 289 arch, 290 "--entrypoint=/bin/python3", 291 "--security-opt", 292 "label=disable", 293 "--mount", 294 f"type=bind,source={cachedir}/gradle,target=/root/.gradle/caches", 295 "--mount", 296 f"type=bind,source={cachedir}/pnpm,target=/root/.local/share/pnpm/store", 297 "--mount", 298 f"type=bind,source={cachedir}/cargo-registry,target=/root/.cargo/registry", 299 "--mount", 300 f"type=bind,source={cachedir}/cargo-git,target=/root/.cargo/git", 301 "--mount", 302 f"type=bind,source={cachedir}/cargo-build,target=/root/.cargo-build", 303 "--env", 304 "CARGO_BUILD_BUILD_DIR=/root/.cargo-build", 305 "--mount", 306 f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared,U=true", 307 "--mount", 308 f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared,U=true", 309 "--mount", 310 f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly", 311 "--mount", 312 f"type=bind,source={config_path},target=/packages.toml,readonly", 313 "--mount", 314 f"type=bind,source={pkgdir},target=/pkgdir", 315 image_tag, 316 f"/buildscripts/{builder}", 317 codename, 318 arch, 319 *group, 320 ] 321 if not cfg.dry: 322 subprocess.run( 323 cmd, 324 check=True, 325 ) 326 327 328 def show_order(cfg): 329 buildorder = buildsort(list(cfg.roots)) 330 print("build order:", buildorder) 331 332 333 def aptly_list(*args): 334 result = remote_aptly(*args, "-raw", capture_output=True) 335 return set(result.stdout.split()) 336 337 338 def published_repositories(): 339 result = remote_aptly("publish", "list", "-raw", capture_output=True) 340 repositories = set() 341 for line in result.stdout.splitlines(): 342 fields = line.split() 343 if len(fields) != 2: 344 raise ValueError(f"unexpected aptly publish list line: {line}") 345 repositories.add(tuple(fields)) 346 return repositories 347 348 349 def parse_publication(output): 350 publication = {"sources": {}} 351 in_sources = False 352 for line in output.splitlines(): 353 if line == "Sources:": 354 in_sources = True 355 continue 356 if in_sources and line.startswith(" "): 357 match = re.fullmatch(r"\s+([^:]+):\s+(.+)\s+\[([^]]+)]", line) 358 if match is None: 359 raise ValueError(f"unexpected aptly publication source: {line}") 360 component, name, kind = match.groups() 361 publication["sources"][component] = (name, kind) 362 continue 363 in_sources = False 364 if line.startswith("Prefix: "): 365 publication["prefix"] = line.removeprefix("Prefix: ") 366 elif line.startswith("Distribution: "): 367 publication["distribution"] = line.removeprefix("Distribution: ") 368 elif line.startswith("Architectures: "): 369 publication["architectures"] = set( 370 line.removeprefix("Architectures: ").replace(",", " ").split() 371 ) 372 return publication 373 374 375 def get_publication(distribution, prefix): 376 result = remote_aptly("publish", "show", distribution, prefix, capture_output=True) 377 return parse_publication(result.stdout) 378 379 380 def snapshot_origin(snapshot): 381 result = remote_aptly("snapshot", "show", snapshot, capture_output=True) 382 match = re.search(r"Snapshot from local repo \[([^]]+)]", result.stdout) 383 if match is not None: 384 return match.group(1) 385 in_sources = False 386 for line in result.stdout.splitlines(): 387 if line == "Sources:": 388 in_sources = True 389 continue 390 if in_sources and line.startswith(" "): 391 match = re.fullmatch(r"\s+(.+)\s+\[(?:local|repo)]", line) 392 if match is not None: 393 return match.group(1) 394 elif in_sources: 395 break 396 raise ValueError(f"cannot determine source repo for aptly snapshot {snapshot}") 397 398 399 def validate_publication(pubcfg, publication, *, stable): 400 distribution = pubcfg.codename if stable else pubcfg.testing_distribution 401 if publication.get("prefix") != pubcfg.prefix: 402 raise ValueError(f"unexpected prefix for {distribution} publication") 403 if publication.get("distribution") != distribution: 404 raise ValueError(f"unexpected distribution for {distribution} publication") 405 if publication.get("architectures") != set(archs): 406 raise ValueError(f"unexpected architectures for {distribution} publication") 407 snapshot = publication_snapshot(publication) 408 validate_snapshot(pubcfg, snapshot, stable=stable) 409 410 411 def validate_snapshot(pubcfg, snapshot, *, stable): 412 if stable and snapshot == pubcfg.initial_snapshot: 413 if snapshot_packages(snapshot): 414 raise ValueError(f"initial snapshot {snapshot} is not empty") 415 else: 416 origin = snapshot_origin(snapshot) 417 if origin != pubcfg.testing_repo: 418 distribution = pubcfg.codename if stable else pubcfg.testing_distribution 419 raise ValueError( 420 f"publication {distribution} has unexpected source {origin}" 421 ) 422 423 424 def publish_options(distribution): 425 return [ 426 f"-architectures={','.join(sorted(archs))}", 427 "-component=main", 428 f"-distribution={distribution}", 429 f"-gpg-key={signing_key}", 430 "-origin=GNU Taler", 431 "-label=Taler", 432 ] 433 434 435 def initialize(cfg): 436 del cfg 437 repos = aptly_list("repo", "list") 438 snapshots = aptly_list("snapshot", "list") 439 publications = published_repositories() 440 changed = False 441 442 for pubcfg in publishing_configs.values(): 443 if pubcfg.testing_repo not in repos: 444 remote_aptly( 445 "repo", 446 "create", 447 f"-distribution={pubcfg.testing_distribution}", 448 "-component=main", 449 pubcfg.testing_repo, 450 ) 451 repos.add(pubcfg.testing_repo) 452 changed = True 453 454 testing_key = (pubcfg.prefix, pubcfg.testing_distribution) 455 if testing_key not in publications: 456 snapshot = create_testing_snapshot(pubcfg) 457 remote_aptly( 458 "publish", 459 "snapshot", 460 *publish_options(pubcfg.testing_distribution), 461 snapshot, 462 pubcfg.prefix, 463 tty=True, 464 ) 465 publications.add(testing_key) 466 changed = True 467 else: 468 validate_publication( 469 pubcfg, 470 get_publication(pubcfg.testing_distribution, pubcfg.prefix), 471 stable=False, 472 ) 473 474 stable_key = (pubcfg.prefix, pubcfg.codename) 475 if stable_key not in publications: 476 if pubcfg.initial_snapshot not in snapshots: 477 remote_aptly( 478 "snapshot", 479 "create", 480 pubcfg.initial_snapshot, 481 "empty", 482 ) 483 snapshots.add(pubcfg.initial_snapshot) 484 changed = True 485 else: 486 validate_snapshot(pubcfg, pubcfg.initial_snapshot, stable=True) 487 remote_aptly( 488 "publish", 489 "snapshot", 490 *publish_options(pubcfg.codename), 491 pubcfg.initial_snapshot, 492 pubcfg.prefix, 493 tty=True, 494 ) 495 publications.add(stable_key) 496 changed = True 497 else: 498 validate_publication( 499 pubcfg, 500 get_publication(pubcfg.codename, pubcfg.prefix), 501 stable=True, 502 ) 503 504 if changed: 505 print("aptly repositories and publications initialized") 506 else: 507 print("aptly repositories and publications already initialized") 508 509 510 def parse_package_list(output): 511 packages = [] 512 in_packages = False 513 for line in output.splitlines(): 514 if line == "Packages:": 515 in_packages = True 516 continue 517 if in_packages and line.startswith(" "): 518 packages.append(line.strip()) 519 elif in_packages: 520 break 521 return packages 522 523 524 def repo_packages(repo): 525 result = remote_aptly("repo", "show", "-with-packages", repo, capture_output=True) 526 return parse_package_list(result.stdout) 527 528 529 def snapshot_packages(snapshot): 530 result = remote_aptly( 531 "snapshot", "show", "-with-packages", snapshot, capture_output=True 532 ) 533 return parse_package_list(result.stdout) 534 535 536 def publishing_packages(pubcfg): 537 """Include pending testing imports and publications sharing its package pool.""" 538 sources = {(pubcfg.testing_repo, "local")} 539 for prefix, distribution in sorted(published_repositories()): 540 if prefix != pubcfg.prefix: 541 continue 542 publication = get_publication(distribution, prefix) 543 if ( 544 publication.get("prefix") != prefix 545 or publication.get("distribution") != distribution 546 or not publication.get("sources") 547 ): 548 raise ValueError(f"invalid publication details for {prefix}/{distribution}") 549 source = publication["sources"].get("main") 550 if source is not None: 551 sources.add(source) 552 553 readers = {"local": repo_packages, "snapshot": snapshot_packages} 554 packages = set() 555 for name, kind in sorted(sources): 556 if kind not in readers: 557 raise ValueError(f"unsupported publication source {name} [{kind}]") 558 packages.update(readers[kind](name)) 559 return sorted(packages) 560 561 562 def publication_snapshot(publication): 563 distribution = publication.get("distribution", "unknown") 564 sources = publication.get("sources", {}) 565 if set(sources) != {"main"}: 566 raise ValueError(f"unexpected components for {distribution} publication") 567 snapshot, kind = sources["main"] 568 if kind != "snapshot": 569 raise ValueError( 570 f"publication {distribution} uses a {kind} source; " 571 "convert it to a snapshot publication before proceeding" 572 ) 573 return snapshot 574 575 576 def published_snapshot(pubcfg, *, stable): 577 """Return the published snapshot name and its enabled architectures.""" 578 distribution = pubcfg.codename if stable else pubcfg.testing_distribution 579 publication = get_publication(distribution, pubcfg.prefix) 580 if ( 581 publication.get("prefix") != pubcfg.prefix 582 or publication.get("distribution") != distribution 583 or not publication.get("architectures") 584 ): 585 raise ValueError(f"invalid publication details for {pubcfg.prefix}/{distribution}") 586 return publication_snapshot(publication), publication["architectures"] 587 588 589 def testing_snapshot_name(pubcfg, now=None): 590 if now is None: 591 now = datetime.datetime.now(datetime.timezone.utc) 592 timestamp = now.strftime("%Y%m%dT%H%M%S%fZ") 593 return f"{pubcfg.testing_repo}-{timestamp}" 594 595 596 def create_testing_snapshot(pubcfg): 597 snapshot = testing_snapshot_name(pubcfg) 598 remote_aptly("snapshot", "create", snapshot, "from", "repo", pubcfg.testing_repo) 599 return snapshot 600 601 602 def promote(cfg): 603 pubcfg = publishing_config(cfg.distro) 604 snapshot, _ = published_snapshot(pubcfg, stable=False) 605 if cfg.dry: 606 stable_name, _ = published_snapshot(pubcfg, stable=True) 607 stable = set(snapshot_packages(stable_name)) 608 testing = set(snapshot_packages(snapshot)) 609 for package in sorted(stable - testing): 610 print(f"- {package}") 611 for package in sorted(testing - stable): 612 print(f"+ {package}") 613 if stable == testing: 614 print("testing and stable contain the same packages") 615 return 616 617 remote_aptly( 618 "publish", 619 "switch", 620 pubcfg.codename, 621 pubcfg.prefix, 622 snapshot, 623 tty=True, 624 ) 625 print(f"promoted published testing snapshot {snapshot}") 626 627 628 def show_published(cfg): 629 distro, _, channel = cfg.distro.rpartition("-") 630 if channel not in ("stable", "testing"): 631 distro, channel = cfg.distro, "stable" 632 pubcfg = publishing_config(distro) 633 snapshot, architectures = published_snapshot(pubcfg, stable=channel == "stable") 634 packages = { 635 package 636 for package in snapshot_packages(snapshot) 637 if package_identity(package)[2] in architectures | {"all"} 638 } 639 if cfg.latest: 640 latest = {} 641 for package in sorted(packages): 642 name, version, architecture = package_identity(package) 643 key = (name, architecture) 644 if key not in latest or vercomp.compare_versions( 645 version, latest[key][0] 646 ) > 0: 647 latest[key] = (version, package) 648 packages = {package for _, package in latest.values()} 649 for package in sorted(packages): 650 print(package) 651 652 653 def test(cfg): 654 target = cfg.distro 655 vendor, codename, *rest = target.split("-") 656 distro = f"{vendor}-{codename}" 657 image_tag = f"localhost/taler-packaging-{distro}:latest" 658 dockerfile = f"distros/{distro}.Dockerfile" 659 cachedir = Path("cache").absolute() 660 print("building base image") 661 subprocess.run( 662 [ 663 "podman", 664 "build", 665 "-v", 666 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 667 "-v", 668 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 669 "-t", 670 image_tag, 671 "-f", 672 dockerfile, 673 ], 674 check=True, 675 ) 676 print("running test") 677 cmd = [ 678 "podman", 679 "run", 680 "-it", 681 "--entrypoint=/bin/bash", 682 "--security-opt", 683 "label=disable", 684 "--mount", 685 f"type=bind,source={mydir}/testing,target=/testing,readonly", 686 image_tag, 687 f"/testing/test-{target}", 688 ] 689 subprocess.run( 690 cmd, 691 check=True, 692 ) 693 694 695 def package_identity(filename): 696 stem = filename 697 for suffix in (".ddeb", ".deb"): 698 if filename.endswith(suffix): 699 stem = filename.removesuffix(suffix) 700 break 701 fields = stem.rsplit("_", 2) 702 if len(fields) != 3: 703 raise ValueError(f"invalid package reference: {filename}") 704 return fields 705 706 707 def package_file_identity(filename): 708 if Path(filename).name != filename or not filename.endswith((".deb", ".ddeb")): 709 raise ValueError(f"invalid package filename: {filename}") 710 return package_identity(filename) 711 712 713 def newest_server_package(package, architecture, server_packages): 714 newest = None 715 newest_ref = None 716 for server_package in server_packages: 717 server_name, server_version, server_arch = package_identity(server_package) 718 if package != server_name or architecture != server_arch: 719 continue 720 if newest is None or vercomp.compare_versions(server_version, newest) > 0: 721 newest = server_version 722 newest_ref = server_package 723 return newest, newest_ref 724 725 726 def current_package_files(distro): 727 current = [] 728 seen = set() 729 for component in components: 730 component_current = [] 731 for arch in archs + ["all"]: 732 current_file = Path(f"./packages/{distro}/{component}@{arch}.built.current") 733 if not current_file.exists(): 734 print(f"component {component}@{arch} has no current packages") 735 continue 736 component_current.extend(current_file.read_text().split()) 737 print("current", component_current) 738 for package in component_current: 739 if package not in seen: 740 current.append(package) 741 seen.add(package) 742 return current 743 744 745 def cleanup_uploads(staging_dir): 746 remote_command( 747 [ 748 "sh", 749 "-c", 750 ( 751 'if [ ! -e "$1" ]; then exit 0; fi\n' 752 'find "$1" -maxdepth 1 -type f ' 753 r"\( -name '*.deb' -o -name '*.ddeb' \) -delete" 754 ), 755 "taler-pkg-cleanup", 756 staging_dir, 757 ] 758 ) 759 760 761 def publish(cfg): 762 distro = cfg.distro 763 if distro.endswith("-testing"): 764 print("Publish the base distro; packages always go to testing", file=sys.stderr) 765 sys.exit(1) 766 pubcfg = publishing_config(distro) 767 published_snapshot(pubcfg, stable=False) 768 server_packages = publishing_packages(pubcfg) 769 uploads = [] 770 for package_file in current_package_files(distro): 771 package, version, architecture = package_file_identity(package_file) 772 server_version, server_ref = newest_server_package( 773 package, architecture, server_packages 774 ) 775 if ( 776 server_version is None 777 or vercomp.compare_versions(version, server_version) > 0 778 ): 779 uploads.append(package_file) 780 else: 781 print("package", package_file, "not fresh, server has", server_ref) 782 783 if uploads: 784 print("uploading debs", uploads) 785 else: 786 print("nothing to upload") 787 if cfg.dry: 788 return 789 790 staging_dir = f"/home/{remote_user}/{distro}" 791 if uploads: 792 remote_command(["mkdir", "-p", staging_dir]) 793 local_files = [Path(f"./packages/{distro}") / name for name in uploads] 794 remote_files = [f"{staging_dir}/{name}" for name in uploads] 795 subprocess.run( 796 [ 797 "rsync", 798 "-a", 799 "--info=progress2", 800 "--", 801 *local_files, 802 f"{remote_user}@{host}:{staging_dir}/", 803 ], 804 check=True, 805 ) 806 remote_aptly("repo", "add", pubcfg.testing_repo, *remote_files) 807 808 snapshot = create_testing_snapshot(pubcfg) 809 remote_aptly( 810 "publish", 811 "switch", 812 pubcfg.testing_distribution, 813 pubcfg.prefix, 814 snapshot, 815 tty=True, 816 ) 817 try: 818 cleanup_uploads(staging_dir) 819 except (subprocess.CalledProcessError, OSError) as exc: 820 raise RuntimeError( 821 f"Publishing {pubcfg.testing_distribution} succeeded, " 822 f"but cleanup of {staging_dir} failed" 823 ) from exc 824 825 826 # Tag syntax variants supported by buildscripts/generic: 827 # v$maj.$min.$patch => release version 828 # v$maj.$min.$patch-dev.$n => dev version 829 # deb-v$maj.$min.$patch-$revision => release version with debian revision 830 # Debian revisions of dev versions are *not* supported 831 tag_re_release = re.compile(r"v(\d+)\.(\d+)\.(\d+)") 832 tag_re_dev = re.compile(r"v(\d+)\.(\d+)\.(\d+)-dev\.(\d+)") 833 tag_re_deb = re.compile(r"deb-v(\d+)\.(\d+)\.(\d+)(?:-(\d+))?") 834 835 836 def tag_sortkey(tag): 837 """Get a sort key for a tag, or None if the tag syntax isn't supported. 838 839 Dev versions sort before the corresponding release version, debian 840 revisions sort after it. 841 """ 842 m = tag_re_release.fullmatch(tag) 843 if m: 844 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 1, 0) 845 m = tag_re_dev.fullmatch(tag) 846 if m: 847 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 0, int(m.group(4))) 848 m = tag_re_deb.fullmatch(tag) 849 if m: 850 rev = m.group(4) 851 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 1, int(rev or 0)) 852 return None 853 854 855 def list_remote_tags(url): 856 """Get all tags from the git repo""" 857 cmd = ["git", "ls-remote", "--exit-code", "--refs", "--tags", url] 858 result = subprocess.run(cmd, capture_output=True, text=True, check=True) 859 tags = [] 860 for line in result.stdout.strip().split("\n"): 861 parts = line.split() 862 if len(parts) < 2: 863 continue 864 # refs/tags/v1.0.0 -> v1.0.0 865 tags.append(parts[1].split("/")[-1]) 866 return tags 867 868 869 def latest_tag(tags, dev): 870 """Find the newest supported tag, only considering dev tags if dev is set""" 871 best = None 872 bestkey = None 873 # Iterate in sorted order, so that the result doesn't depend on 874 # the order in which the remote lists its refs. 875 for tag in sorted(tags): 876 if not dev and tag_re_dev.fullmatch(tag): 877 continue 878 key = tag_sortkey(tag) 879 if key is None: 880 continue 881 if bestkey is None or key > bestkey: 882 best = tag 883 bestkey = key 884 return best 885 886 887 def print_latest(cfg): 888 """Print the latest stable upstream tag for each enabled component.""" 889 remote_tags = {} 890 for name in components: 891 package = config.packages[name] 892 url = config.repository_for(package).url 893 if url not in remote_tags: 894 remote_tags[url] = list_remote_tags(url) 895 latest = latest_tag(remote_tags[url], False) or "(none)" 896 prefix = "[!] " if package.tag != latest else "" 897 print(f"{prefix}{name} curr: {package.tag} latest: {latest}") 898 899 900 def upgrade(cfg): 901 """Upgrade package tags to the latest tag from their repositories.""" 902 names = cfg.components 903 explicitly_selected = bool(names) 904 if not names: 905 names = sorted(config.packages) 906 unknown = sorted(set(names) - set(config.packages)) 907 if unknown: 908 raise ConfigError(f"unknown package(s): {', '.join(unknown)}") 909 # Multiple components can share a repo, only ask each remote once. 910 remote_tags = {} 911 upgraded = [] 912 updated_packages = dict(config.packages) 913 for name in names: 914 package = config.packages[name] 915 if not package.auto_upgrade and not explicitly_selected: 916 print(f" {name} {package.tag} (automatic upgrades disabled)") 917 continue 918 giturl = config.repository_for(package).url 919 if giturl not in remote_tags: 920 remote_tags[giturl] = list_remote_tags(giturl) 921 latest = latest_tag(remote_tags[giturl], cfg.dev) 922 if latest is None: 923 print( 924 f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr 925 ) 926 continue 927 curr = package.tag 928 currkey = tag_sortkey(curr) 929 if currkey is None: 930 print( 931 f"[?] {name} tag {curr} has unsupported syntax, skipping", 932 file=sys.stderr, 933 ) 934 continue 935 latestkey = tag_sortkey(latest) 936 if currkey > latestkey: 937 # Happens when a package pins a dev version but only production 938 # tags are considered. 939 print(f" {name} {curr} (newer than latest {latest})") 940 continue 941 if currkey == latestkey: 942 print(f" {name} {curr} (up to date)") 943 continue 944 print(f"[!] {name} {curr} -> {latest}") 945 upgraded.append(name) 946 if not cfg.dry: 947 updated_packages[name] = replace(package, tag=latest) 948 if not upgraded: 949 print("nothing to upgrade") 950 elif cfg.dry: 951 print("would upgrade:", " ".join(upgraded)) 952 else: 953 write_config(config_path, replace(config, packages=updated_packages)) 954 print("upgraded:", " ".join(upgraded)) 955 956 957 def main(): 958 parser = argparse.ArgumentParser( 959 prog="taler-pkg", description="Taler Packaging Helper" 960 ) 961 962 subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND") 963 964 parser_init = subparsers.add_parser( 965 "init", help="Initialize aptly repositories and publications." 966 ) 967 parser_init.set_defaults(func=initialize) 968 969 # subcommand build 970 971 parser_build = subparsers.add_parser("build", help="Build packages for distro.") 972 parser_build.set_defaults(func=build) 973 parser_build.add_argument("distro") 974 # Keep for backwards compat 975 parser_build.add_argument( 976 "--no-transitive", 977 help="Do not build transitive deps of changed components (default)", 978 action="store_false", 979 dest="transitive", 980 default=False, 981 ) 982 parser_build.add_argument( 983 "--transitive", 984 help="Build transitive deps of changed components", 985 action="store_true", 986 dest="transitive", 987 ) 988 parser_build.add_argument( 989 "--arch", 990 help="Architecture(s) to build for", 991 action="store", 992 dest="arch", 993 default=None, 994 ) 995 parser_build.add_argument( 996 "--dry", help="Dry run", action="store_true", default=False 997 ) 998 999 parser_test = subparsers.add_parser("test", help="Test packages for distro.") 1000 parser_test.set_defaults(func=test) 1001 parser_test.add_argument("distro") 1002 parser_test.add_argument( 1003 "--arch", 1004 help="Architecture(s) to test packages for", 1005 action="store", 1006 dest="arch", 1007 default=None, 1008 ) 1009 1010 # subcommand show-latest 1011 1012 parser_show_latest = subparsers.add_parser( 1013 "show-latest", help="Show latest version of packages." 1014 ) 1015 parser_show_latest.set_defaults(func=print_latest) 1016 1017 # subcommand upgrade 1018 1019 parser_upgrade = subparsers.add_parser( 1020 "upgrade", help="Upgrade component tags to the latest upstream version." 1021 ) 1022 parser_upgrade.set_defaults(func=upgrade) 1023 parser_upgrade.add_argument( 1024 "components", 1025 nargs="*", 1026 help="Components to upgrade (default: all packages in packages.toml)", 1027 ) 1028 parser_upgrade.add_argument( 1029 "--dev", 1030 help="Also consider dev tags, not just production tags", 1031 action="store_true", 1032 default=False, 1033 ) 1034 parser_upgrade.add_argument( 1035 "--dry", help="Dry run", action="store_true", default=False 1036 ) 1037 1038 # subcommand show-order 1039 1040 parser_show_order = subparsers.add_parser("show-order", help="Show build order.") 1041 parser_show_order.set_defaults(func=show_order) 1042 parser_show_order.add_argument("roots", nargs="+") 1043 1044 # subcommand show-published 1045 parser_show_published = subparsers.add_parser( 1046 "show-published", help="Show packages in a published stable or testing snapshot" 1047 ) 1048 parser_show_published.add_argument( 1049 "--latest", 1050 action="store_true", 1051 help="Show only the newest version of each package per architecture", 1052 ) 1053 parser_show_published.add_argument( 1054 "distro", 1055 choices=[ 1056 f"{distro}{suffix}" 1057 for distro in publishing_configs 1058 for suffix in ("", "-stable", "-testing") 1059 ], 1060 help="Base distro (stable by default), or distro-stable/distro-testing", 1061 ) 1062 parser_show_published.set_defaults(func=show_published) 1063 1064 # subcommand publish 1065 1066 parser_publish = subparsers.add_parser("publish", help="Publish to deb.taler.net") 1067 parser_publish.add_argument( 1068 "--dry", help="Dry run", action="store_true", default=False 1069 ) 1070 parser_publish.add_argument("distro") 1071 parser_publish.set_defaults(func=publish) 1072 1073 parser_promote = subparsers.add_parser("promote", help="Promote testing to stable") 1074 parser_promote.add_argument( 1075 "--dry", 1076 help="Dry run (show testing changes)", 1077 action="store_true", 1078 default=False, 1079 ) 1080 parser_promote.add_argument("distro") 1081 parser_promote.set_defaults(func=promote) 1082 1083 args = parser.parse_args() 1084 1085 if "func" not in args: 1086 parser.print_help() 1087 else: 1088 args.func(args) 1089 1090 1091 if __name__ == "__main__": 1092 main()