taler-pkg (23933B)
1 #!/usr/bin/env python3 2 3 # Copyright (c) 2024 Taler Systems SA 4 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 5 # SPDX-License-Identifier: GPL-3.0-or-later 6 7 import argparse 8 import subprocess 9 import platform 10 import os 11 import re 12 import sys 13 from pathlib import Path 14 15 # Make local util package available 16 file = Path(__file__).resolve() 17 parent, root = file.parent, file.parents[1] 18 sys.path.append(str(root)) 19 20 from util import vercomp 21 22 mydir = os.path.dirname(os.path.realpath(__file__)) 23 24 archs = ["arm64", "amd64"] 25 host = "taler.net" 26 native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64" 27 28 components = [ 29 "taler-wallet-cli", 30 "taler-merchant-webui", 31 "taler-exchange-kyc-webui", 32 "taler-exchange-aml-webui", 33 "taler-auditor-webui", 34 "taler-challenger-helpers", 35 "challenger-webui", 36 "gnunet", 37 "libeufin", 38 "donau", 39 "challenger", 40 "taler-exchange", 41 "taler-harness", 42 "taler-merchant", 43 "taler-rust", 44 "robocop", 45 #"depolymerization", 46 # These two packages don't have good debs yet, 47 # Debian complains "No section given for ..., skipping. 48 # "taler-directory", 49 # "taler-mailbox", 50 # We don't publish packages for these yet 51 # "taler-mdb", 52 # "taler-merchant-demos", 53 "anastasis", 54 "anastasis-gtk", 55 # Currently not used anywhere 56 # "sync", 57 ] 58 59 deps = { 60 "taler-exchange": ["gnunet"], 61 "anastasis": ["gnunet", "taler-merchant"], 62 "anastasis-gtk": ["anastasis"], 63 "taler-merchant": ["gnunet", "taler-exchange", "donau"], 64 "donau": ["gnunet", "taler-exchange"], 65 "challenger": ["taler-exchange"], 66 # "taler-mdb": ["gnunet", "taler-exchange", "taler-merchant"], 67 "sync": ["taler-merchant", "taler-exchange", "gnunet"], 68 } 69 70 # Compute reverse dependencies 71 rdeps = {} 72 for n1, d in deps.items(): 73 for n2 in d: 74 rd = rdeps.setdefault(n2, []) 75 if n1 not in rd: 76 rd.append(n1) 77 78 79 def buildsort(roots): 80 """Toposort transitive closure of roots based on deps""" 81 out = [] 82 stack = list(roots[::-1]) 83 pmark = set() 84 tmark = set() 85 while len(stack): 86 node = stack[-1] 87 if node in pmark: 88 stack.pop() 89 tmark.discard(node) 90 continue 91 done = True 92 for dep in deps.get(node, []): 93 if dep not in pmark: 94 if dep in tmark: 95 raise Exception("cycle") 96 stack.append(dep) 97 tmark.add(node) 98 done = False 99 if done: 100 pmark.add(node) 101 out.append(node) 102 stack.pop() 103 tmark.discard(node) 104 return out 105 106 107 def propagate_outdated(outdated): 108 """Propagate outdatedness to dependees""" 109 closure = set() 110 q = list(outdated) 111 while len(q): 112 n = q.pop() 113 closure.add(n) 114 for r in rdeps.get(n, []): 115 if r not in closure: 116 closure.add(r) 117 q.append(r) 118 return closure 119 120 121 def find_outdated(pkgdir, arch, roots): 122 """Find outdated components based on tag files""" 123 outdated = set() 124 for component in roots: 125 ver_requested = open(f"buildconfig/{component}.tag").read().strip() 126 built_tag_file = pkgdir / f"{component}@{arch}.built.tag" 127 ver_built = None 128 if built_tag_file.exists(): 129 ver_built = open(built_tag_file).read().strip() 130 if ver_built != ver_requested: 131 outdated.add(component) 132 print(component, ver_built, "->", ver_requested) 133 return outdated 134 135 136 def build(cfg): 137 transitive = False 138 if cfg.transitive: 139 transitive = True 140 distro = cfg.distro 141 vendor, codename = distro.split("-", 1) 142 print("building", distro) 143 dockerfile = f"distros/{distro}.Dockerfile" 144 image_tag = f"localhost/taler-packaging-{distro}:latest" 145 pkgdir = Path(f"packages/{distro}").absolute() 146 cachedir = Path("cache").absolute() 147 cachedir.mkdir(exist_ok=True) 148 (cachedir / "cargo-git").mkdir(exist_ok=True) 149 (cachedir / "cargo-registry").mkdir(exist_ok=True) 150 (cachedir / "cargo-build").mkdir(exist_ok=True) 151 (cachedir / "gradle").mkdir(exist_ok=True) 152 (cachedir / "pnpm").mkdir(exist_ok=True) 153 (cachedir / distro / "apt-archives").mkdir(parents=True, exist_ok=True) 154 (cachedir / distro / "apt-lists").mkdir(parents=True, exist_ok=True) 155 156 if cfg.arch is None: 157 arch_list = [native_arch] 158 else: 159 arch_list = cfg.arch.split(",") 160 161 if not cfg.dry: 162 for arch in arch_list: 163 subprocess.run( 164 [ 165 "podman", 166 "build", 167 "--arch", 168 arch, 169 "-v", 170 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 171 "-v", 172 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 173 "-t", 174 image_tag, 175 "-f", 176 dockerfile, 177 ], 178 check=True, 179 ) 180 181 # Sort components by their dependencies 182 buildorder = buildsort(components) 183 print("build order:", buildorder) 184 185 for arch in arch_list: 186 outdated = find_outdated(pkgdir, arch, buildorder) 187 188 # Propagate outdatedness to dependees 189 closure = propagate_outdated(outdated) 190 191 print("outdated closure", closure) 192 193 for component in buildorder: 194 if transitive: 195 if component not in closure: 196 continue 197 else: 198 if component not in outdated: 199 continue 200 print("building", component) 201 pkgdir.mkdir(parents=True, exist_ok=True) 202 cmd = [ 203 "podman", 204 "run", 205 "-it", 206 "--arch", 207 arch, 208 "--entrypoint=/bin/python3", 209 "--security-opt", 210 "label=disable", 211 "--mount", 212 f"type=bind,source={cachedir}/gradle,target=/root/.gradle/caches", 213 "--mount", 214 f"type=bind,source={cachedir}/pnpm,target=/root/.local/share/pnpm/store", 215 "--mount", 216 f"type=bind,source={cachedir}/cargo-registry,target=/root/.cargo/registry", 217 "--mount", 218 f"type=bind,source={cachedir}/cargo-git,target=/root/.cargo/git", 219 "--mount", 220 f"type=bind,source={cachedir}/cargo-build,target=/root/.cargo-build", 221 "--env", 222 "CARGO_BUILD_BUILD_DIR=/root/.cargo-build", 223 "--mount", 224 f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared", 225 "--mount", 226 f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared", 227 "--mount", 228 f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly", 229 "--mount", 230 f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly", 231 "--mount", 232 f"type=bind,source={pkgdir},target=/pkgdir", 233 image_tag, 234 "/buildscripts/generic", 235 component, 236 codename, 237 arch, 238 ] 239 if not cfg.dry: 240 subprocess.run( 241 cmd, 242 check=True, 243 ) 244 245 246 def show_order(cfg): 247 buildorder = buildsort(list(cfg.roots)) 248 print("build order:", buildorder) 249 250 251 252 253 def promote(cfg): 254 dry = cfg.dry 255 distro = cfg.distro 256 vendor, codename = distro.split("-", 1) 257 listfmt = "${package}_${version}_${architecture}.${$type}\n" 258 if dry: 259 subprocess.run( 260 [ 261 "ssh", 262 f"taler-packaging@{host}", 263 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ checkpull {codename}", 264 ], 265 check=True, 266 ) 267 else: 268 subprocess.run( 269 [ 270 "ssh", 271 "-t", 272 f"taler-packaging@{host}", 273 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ pull {codename}", 274 ], 275 check=True, 276 ) 277 # Always export! 278 # Reprepro is weird, listed packages might actually not show 279 # up in the index yet. 280 subprocess.run( 281 [ 282 "ssh", 283 "-t", 284 f"taler-packaging@{host}", 285 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}", 286 ], 287 check=True, 288 ) 289 290 def show_published(cfg): 291 distro = cfg.distro 292 vendor, codename = distro.split("-", 1) 293 listfmt = "${package}_${version}_${architecture}.${$type}\n" 294 subprocess.run( 295 [ 296 "ssh", 297 f"taler-packaging@{host}", 298 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 299 ], 300 check=True, 301 ) 302 303 304 def test(cfg): 305 target = cfg.distro 306 vendor, codename, *rest = target.split("-") 307 distro = f"{vendor}-{codename}" 308 image_tag = f"localhost/taler-packaging-{distro}:latest" 309 dockerfile = f"distros/{distro}.Dockerfile" 310 cachedir = Path(f"cache").absolute() 311 print("building base image") 312 subprocess.run( 313 [ 314 "podman", 315 "build", 316 "-v", 317 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 318 "-v", 319 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 320 "-t", 321 image_tag, 322 "-f", 323 dockerfile, 324 ], 325 check=True, 326 ) 327 print("running test") 328 cmd = [ 329 "podman", 330 "run", 331 "-it", 332 "--entrypoint=/bin/bash", 333 "--security-opt", 334 "label=disable", 335 "--mount", 336 f"type=bind,source={mydir}/testing,target=/testing,readonly", 337 image_tag, 338 f"/testing/test-{target}", 339 ] 340 subprocess.run( 341 cmd, 342 check=True, 343 ) 344 345 def publish(cfg): 346 distro = cfg.distro 347 if distro.endswith("-testing"): 348 print("Files are automatically published to testing", file=sys.stderr) 349 sys.exit(1) 350 vendor, codename = distro.split("-", 1) 351 # List of .deb and .ddeb files. 352 debs = [] 353 listfmt = "${package}_${version}_${architecture}.${$type}\n" 354 server_debs_str = subprocess.check_output( 355 [ 356 "ssh", 357 f"taler-packaging@{host}", 358 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 359 ], 360 encoding="utf-8", 361 ) 362 server_debs = server_debs_str.split() 363 for component in components: 364 current = [] 365 for arch in archs + ["all"]: 366 cf = Path(f"./packages/{distro}/{component}@{arch}.built.current") 367 if not cf.exists(): 368 print(f"component {component}@{arch} has no current packages") 369 continue 370 with open(cf) as f: 371 current = current + f.read().split() 372 print("current", current) 373 for deb in current: 374 if deb.endswith(".deb"): 375 pkg1, ver1, arch1 = deb.removesuffix(".deb").split("_") 376 elif deb.endswith(".ddeb"): 377 pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_") 378 else: 379 raise Error(f"invalid deb filename: {deb}") 380 fresh = True 381 server_deb = None 382 # If the server has the same or a later version, 383 # the local version isn't fresh. 384 for srvdeb in server_debs: 385 pkg2, ver2, arch2 = srvdeb.removesuffix(".deb").split("_") 386 if pkg1 != pkg2 or arch1 != arch2: 387 continue 388 if vercomp.compare_versions(ver1, ver2) <= 0: 389 fresh = False 390 server_deb = srvdeb 391 break 392 if fresh: 393 debs.append(deb) 394 else: 395 print("package", deb, "not fresh, server has", server_deb) 396 if len(debs) == 0: 397 print("nothing to upload") 398 else: 399 print("uploading debs", debs) 400 if cfg.dry: 401 return 402 debs = [Path(f"./packages/{distro}/") / x for x in debs] 403 subprocess.run( 404 [ 405 "ssh", 406 f"taler-packaging@{host}", 407 f"rm -f '/home/taler-packaging/{distro}/'*.deb '/home/taler-packaging/{distro}/'*.ddeb", 408 ], 409 check=True, 410 ) 411 subprocess.run( 412 ["rsync", "-a", "--info=progress2", "--", *debs, f"taler-packaging@{host}:{distro}/"], check=True 413 ) 414 ret = subprocess.run( 415 [ 416 "ssh", 417 "-t", 418 f"taler-packaging@{host}", 419 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.deb", 420 ], 421 ) 422 if ret.returncode != 0: 423 # Usually not critical if it fails. 424 print( 425 "Including ddebs failed. This can happen when including packages that have been included previously" 426 ) 427 # Almost the same, but with ddebs. 428 # We explicitly need to tell reprepro 429 # to ignore the extension, because it does not 430 # deal well with ddebs out of the box. 431 ret = subprocess.run( 432 [ 433 "ssh", 434 "-t", 435 f"taler-packaging@{host}", 436 f"reprepro --ignore=extension -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.ddeb", 437 ], 438 ) 439 if ret.returncode != 0: 440 # Usually not critical if it fails. 441 print( 442 "Including ddebs failed. This can happen when including packages that have been included previously" 443 ) 444 # Always export! 445 # Reprepro is weird, listed packages might actually not show 446 # up in the index yet. 447 subprocess.run( 448 [ 449 "ssh", 450 "-t", 451 f"taler-packaging@{host}", 452 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing", 453 ], 454 check=True, 455 ) 456 457 458 def get_remote_version(url): 459 """Get the latest stable tag from the git repo""" 460 # Construct the git command 461 # We use -c versionsort.suffix=- to ensure correct semantic version sorting 462 cmd = [ 463 "git", 464 "-c", 465 "versionsort.suffix=-", 466 "ls-remote", 467 "--exit-code", 468 "--refs", 469 "--sort=version:refname", 470 "--tags", 471 url, 472 "*.*.*", 473 ] 474 475 result = subprocess.run(cmd, capture_output=True, text=True, check=True) 476 477 # Parse the output 478 # Output format is usually: <hash>\trefs/tags/<tagname> 479 lines = result.stdout.strip().split("\n") 480 481 valid_tags = [] 482 483 for line in lines: 484 parts = line.split() 485 if len(parts) < 2: 486 continue 487 488 # refs/tags/v1.0.0 -> v1.0.0 489 ref_path = parts[1] 490 tag = ref_path.split("/")[-1] 491 492 # Exclude pre-release semver versions 493 if tag.startswith("v") and "-" in tag: 494 continue 495 496 valid_tags.append(tag) 497 498 if valid_tags: 499 return valid_tags[-1] 500 return "(none)" 501 502 503 def check_version(name, url): 504 """ 505 Compares local buildconfig version with remote git version. 506 """ 507 ver = get_remote_version(url) 508 config_path = os.path.join("buildconfig", f"{name}.tag") 509 with open(config_path, "r") as f: 510 curr = f.read().strip() 511 prefix = "[!] " if curr != ver else "" 512 print(f"{prefix}{name} curr: {curr} latest: {ver}") 513 514 515 def print_latest(cfg): 516 """Print latest upstream tag for each component""" 517 for name in components: 518 config_path = os.path.join("buildconfig", f"{name}.giturl") 519 with open(config_path, "r") as f: 520 giturl = f.read().strip() 521 check_version(name, giturl) 522 523 524 # Tag syntax variants supported by buildscripts/generic: 525 # v$maj.$min.$patch => release version 526 # v$maj.$min.$patch-dev.$n => dev version 527 # deb-v$maj.$min.$patch-$revision => release version with debian revision 528 # Debian revisions of dev versions are *not* supported 529 tag_re_release = re.compile(r"v(\d+)\.(\d+)\.(\d+)") 530 tag_re_dev = re.compile(r"v(\d+)\.(\d+)\.(\d+)-dev\.(\d+)") 531 tag_re_deb = re.compile(r"deb-v(\d+)\.(\d+)\.(\d+)(?:-(\d+))?") 532 533 534 def tag_sortkey(tag): 535 """Get a sort key for a tag, or None if the tag syntax isn't supported. 536 537 Dev versions sort before the corresponding release version, debian 538 revisions sort after it. 539 """ 540 m = tag_re_release.fullmatch(tag) 541 if m: 542 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 1, 0) 543 m = tag_re_dev.fullmatch(tag) 544 if m: 545 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 0, int(m.group(4))) 546 m = tag_re_deb.fullmatch(tag) 547 if m: 548 rev = m.group(4) 549 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 1, int(rev or 0)) 550 return None 551 552 553 def list_remote_tags(url): 554 """Get all tags from the git repo""" 555 cmd = ["git", "ls-remote", "--exit-code", "--refs", "--tags", url] 556 result = subprocess.run(cmd, capture_output=True, text=True, check=True) 557 tags = [] 558 for line in result.stdout.strip().split("\n"): 559 parts = line.split() 560 if len(parts) < 2: 561 continue 562 # refs/tags/v1.0.0 -> v1.0.0 563 tags.append(parts[1].split("/")[-1]) 564 return tags 565 566 567 def latest_tag(tags, dev): 568 """Find the newest supported tag, only considering dev tags if dev is set""" 569 best = None 570 bestkey = None 571 # Iterate in sorted order, so that the result doesn't depend on 572 # the order in which the remote lists its refs. 573 for tag in sorted(tags): 574 if not dev and tag_re_dev.fullmatch(tag): 575 continue 576 key = tag_sortkey(tag) 577 if key is None: 578 continue 579 if bestkey is None or key > bestkey: 580 best = tag 581 bestkey = key 582 return best 583 584 585 def upgrade(cfg): 586 """Upgrade tag files in buildconfig to the latest upstream tag""" 587 names = cfg.components 588 if not names: 589 names = sorted(p.stem for p in Path("buildconfig").glob("*.tag")) 590 # Multiple components can share a repo, only ask each remote once. 591 remote_tags = {} 592 upgraded = [] 593 for name in names: 594 tag_file = Path("buildconfig") / f"{name}.tag" 595 url_file = Path("buildconfig") / f"{name}.giturl" 596 if not url_file.exists(): 597 print(f"[?] {name} has no giturl, skipping", file=sys.stderr) 598 continue 599 giturl = url_file.read_text().strip() 600 if giturl not in remote_tags: 601 remote_tags[giturl] = list_remote_tags(giturl) 602 latest = latest_tag(remote_tags[giturl], cfg.dev) 603 if latest is None: 604 print(f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr) 605 continue 606 curr = None 607 if tag_file.exists(): 608 curr = tag_file.read_text().strip() 609 currkey = tag_sortkey(curr) 610 if currkey is None: 611 print( 612 f"[?] {name} tag {curr} has unsupported syntax, skipping", 613 file=sys.stderr, 614 ) 615 continue 616 latestkey = tag_sortkey(latest) 617 if currkey > latestkey: 618 # Happens when the tag file pins a dev version but only 619 # production tags are considered. 620 print(f" {name} {curr} (newer than latest {latest})") 621 continue 622 if currkey == latestkey: 623 print(f" {name} {curr} (up to date)") 624 continue 625 print(f"[!] {name} {curr or '(none)'} -> {latest}") 626 upgraded.append(name) 627 if not cfg.dry: 628 tag_file.write_text(latest + "\n") 629 if not upgraded: 630 print("nothing to upgrade") 631 elif cfg.dry: 632 print("would upgrade:", " ".join(upgraded)) 633 else: 634 print("upgraded:", " ".join(upgraded)) 635 636 637 def main(): 638 parser = argparse.ArgumentParser( 639 prog="taler-pkg", description="Taler Packaging Helper" 640 ) 641 642 subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND") 643 644 # subcommand build 645 646 parser_build = subparsers.add_parser("build", help="Build packages for distro.") 647 parser_build.set_defaults(func=build) 648 parser_build.add_argument("distro") 649 # Keep for backwards compat 650 parser_build.add_argument( 651 "--no-transitive", 652 help="Do not build transitive deps of changed components (default)", 653 action="store_true", 654 dest="transitive", 655 default=None, 656 ) 657 parser_build.add_argument( 658 "--transitive", 659 help="Build transitive deps of changed components", 660 action="store_false", 661 dest="transitive", 662 default=None, 663 ) 664 parser_build.add_argument( 665 "--arch", 666 help="Architecture(s) to build for", 667 action="store", 668 dest="arch", 669 default=None, 670 ) 671 parser_build.add_argument( 672 "--dry", help="Dry run", action="store_true", default=False 673 ) 674 675 parser_test = subparsers.add_parser("test", help="Test packages for distro.") 676 parser_test.set_defaults(func=test) 677 parser_test.add_argument("distro") 678 parser_test.add_argument( 679 "--arch", 680 help="Architecture(s) to test packages for", 681 action="store", 682 dest="arch", 683 default=None, 684 ) 685 686 # subcommand show-latest 687 688 parser_show_latest = subparsers.add_parser( 689 "show-latest", help="Show latest version of packages." 690 ) 691 parser_show_latest.set_defaults(func=print_latest) 692 693 # subcommand upgrade 694 695 parser_upgrade = subparsers.add_parser( 696 "upgrade", help="Upgrade component tags to the latest upstream version." 697 ) 698 parser_upgrade.set_defaults(func=upgrade) 699 parser_upgrade.add_argument( 700 "components", 701 nargs="*", 702 help="Components to upgrade (default: all components in buildconfig)", 703 ) 704 parser_upgrade.add_argument( 705 "--dev", 706 help="Also consider dev tags, not just production tags", 707 action="store_true", 708 default=False, 709 ) 710 parser_upgrade.add_argument( 711 "--dry", help="Dry run", action="store_true", default=False 712 ) 713 714 # subcommand show-order 715 716 parser_show_order = subparsers.add_parser("show-order", help="Show build order.") 717 parser_show_order.set_defaults(func=show_order) 718 parser_show_order.add_argument("roots", nargs="+") 719 720 # subcommand show-published 721 parser_show_published = subparsers.add_parser( 722 "show-published", help="Show published packages on deb.taler.net" 723 ) 724 parser_show_published.add_argument("distro") 725 parser_show_published.set_defaults(func=show_published) 726 727 # subcommand publish 728 729 parser_publish = subparsers.add_parser("publish", help="Publish to deb.taler.net") 730 parser_publish.add_argument( 731 "--dry", help="Dry run", action="store_true", default=False 732 ) 733 parser_publish.add_argument("distro") 734 parser_publish.set_defaults(func=publish) 735 736 parser_promote = subparsers.add_parser("promote", help="Promote testing to stable") 737 parser_promote.add_argument( 738 "--dry", help="Dry run (show pulls)", action="store_true", default=False 739 ) 740 parser_promote.add_argument("distro") 741 parser_promote.set_defaults(func=promote) 742 743 args = parser.parse_args() 744 745 if "func" not in args: 746 parser.print_help() 747 else: 748 args.func(args) 749 750 751 if __name__ == "__main__": 752 main()