taler-pkg (19089B)
1 #!/usr/bin/env python3 2 3 # Copyright (c) 2024 Taler Systems SA 4 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 5 # SPDX-License-Identifier: GPL-3.0-or-later 6 7 import argparse 8 import subprocess 9 import platform 10 import os 11 import sys 12 from pathlib import Path 13 14 # Make local util package available 15 file = Path(__file__).resolve() 16 parent, root = file.parent, file.parents[1] 17 sys.path.append(str(root)) 18 19 from util import vercomp 20 21 mydir = os.path.dirname(os.path.realpath(__file__)) 22 23 archs = ["arm64", "amd64"] 24 host = "taler.net" 25 native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64" 26 27 components = [ 28 "taler-wallet-cli", 29 "taler-merchant-webui", 30 "gnunet", 31 "libeufin", 32 "donau", 33 "challenger", 34 "taler-exchange", 35 "taler-harness", 36 "taler-merchant", 37 "taler-rust", 38 "robocop", 39 "taler-exchange-kyc-webui", 40 "taler-exchange-aml-webui", 41 "taler-auditor-webui", 42 "challenger-webui", 43 "taler-challenger-helpers", 44 #"depolymerization", 45 # These two packages don't have good debs yet, 46 # Debian complains "No section given for ..., skipping. 47 # "taler-directory", 48 # "taler-mailbox", 49 # We don't publish packages for these yet 50 # "taler-mdb", 51 # "taler-merchant-demos", 52 "anastasis", 53 "anastasis-gtk", 54 # Currently not used anywhere 55 # "sync", 56 ] 57 58 deps = { 59 "taler-exchange": ["gnunet"], 60 "anastasis": ["gnunet", "taler-merchant"], 61 "anastasis-gtk": ["anastasis"], 62 "taler-merchant": ["gnunet", "taler-exchange", "donau"], 63 "donau": ["gnunet", "taler-exchange"], 64 # "taler-mdb": ["gnunet", "taler-exchange", "taler-merchant"], 65 "sync": ["taler-merchant", "taler-exchange", "gnunet"], 66 } 67 68 # Compute reverse dependencies 69 rdeps = {} 70 for n1, d in deps.items(): 71 for n2 in d: 72 rd = rdeps.setdefault(n2, []) 73 if n1 not in rd: 74 rd.append(n1) 75 76 77 def buildsort(roots): 78 """Toposort transitive closure of roots based on deps""" 79 out = [] 80 stack = list(roots[::-1]) 81 pmark = set() 82 tmark = set() 83 while len(stack): 84 node = stack[-1] 85 if node in pmark: 86 stack.pop() 87 tmark.discard(node) 88 continue 89 done = True 90 for dep in deps.get(node, []): 91 if dep not in pmark: 92 if dep in tmark: 93 raise Exception("cycle") 94 stack.append(dep) 95 tmark.add(node) 96 done = False 97 if done: 98 pmark.add(node) 99 out.append(node) 100 stack.pop() 101 tmark.discard(node) 102 return out 103 104 105 def propagate_outdated(outdated): 106 """Propagate outdatedness to dependees""" 107 closure = set() 108 q = list(outdated) 109 while len(q): 110 n = q.pop() 111 closure.add(n) 112 for r in rdeps.get(n, []): 113 if r not in closure: 114 closure.add(r) 115 q.append(r) 116 return closure 117 118 119 def find_outdated(pkgdir, arch, roots): 120 """Find outdated components based on tag files""" 121 outdated = set() 122 for component in roots: 123 ver_requested = open(f"buildconfig/{component}.tag").read().strip() 124 built_tag_file = pkgdir / f"{component}@{arch}.built.tag" 125 ver_built = None 126 if built_tag_file.exists(): 127 ver_built = open(built_tag_file).read().strip() 128 if ver_built != ver_requested: 129 outdated.add(component) 130 print(component, ver_built, "->", ver_requested) 131 return outdated 132 133 134 def build(cfg): 135 transitive = False 136 if cfg.transitive: 137 transitive = True 138 distro = cfg.distro 139 vendor, codename = distro.split("-", 1) 140 print("building", distro) 141 dockerfile = f"distros/{distro}.Dockerfile" 142 image_tag = f"localhost/taler-packaging-{distro}:latest" 143 pkgdir = Path(f"packages/{distro}").absolute() 144 cachedir = Path("cache").absolute() 145 cachedir.mkdir(exist_ok=True) 146 (cachedir / "cargo-git").mkdir(exist_ok=True) 147 (cachedir / "cargo-registry").mkdir(exist_ok=True) 148 (cachedir / "cargo-build").mkdir(exist_ok=True) 149 (cachedir / "gradle").mkdir(exist_ok=True) 150 (cachedir / "pnpm").mkdir(exist_ok=True) 151 (cachedir / distro / "apt-archives").mkdir(parents=True, exist_ok=True) 152 (cachedir / distro / "apt-lists").mkdir(parents=True, exist_ok=True) 153 154 if cfg.arch is None: 155 arch_list = [native_arch] 156 else: 157 arch_list = cfg.arch.split(",") 158 159 if not cfg.dry: 160 for arch in arch_list: 161 subprocess.run( 162 [ 163 "podman", 164 "build", 165 "--arch", 166 arch, 167 "-v", 168 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 169 "-v", 170 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 171 "-t", 172 image_tag, 173 "-f", 174 dockerfile, 175 ], 176 check=True, 177 ) 178 179 # Sort components by their dependencies 180 buildorder = buildsort(components) 181 print("build order:", buildorder) 182 183 for arch in arch_list: 184 outdated = find_outdated(pkgdir, arch, buildorder) 185 186 # Propagate outdatedness to dependees 187 closure = propagate_outdated(outdated) 188 189 print("outdated closure", closure) 190 191 for component in buildorder: 192 if transitive: 193 if component not in closure: 194 continue 195 else: 196 if component not in outdated: 197 continue 198 print("building", component) 199 pkgdir.mkdir(parents=True, exist_ok=True) 200 cmd = [ 201 "podman", 202 "run", 203 "-it", 204 "--arch", 205 arch, 206 "--entrypoint=/bin/python3", 207 "--security-opt", 208 "label=disable", 209 "--mount", 210 f"type=bind,source={cachedir}/gradle,target=/root/.gradle/caches", 211 "--mount", 212 f"type=bind,source={cachedir}/pnpm,target=/root/.local/share/pnpm/store", 213 "--mount", 214 f"type=bind,source={cachedir}/cargo-registry,target=/root/.cargo/registry", 215 "--mount", 216 f"type=bind,source={cachedir}/cargo-git,target=/root/.cargo/git", 217 "--mount", 218 f"type=bind,source={cachedir}/cargo-build,target=/root/.cargo-build", 219 "--env", 220 "CARGO_BUILD_BUILD_DIR=/root/.cargo-build", 221 "--mount", 222 f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared", 223 "--mount", 224 f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared", 225 "--mount", 226 f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly", 227 "--mount", 228 f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly", 229 "--mount", 230 f"type=bind,source={pkgdir},target=/pkgdir", 231 image_tag, 232 "/buildscripts/generic", 233 component, 234 codename, 235 arch, 236 ] 237 if not cfg.dry: 238 subprocess.run( 239 cmd, 240 check=True, 241 ) 242 243 244 def show_order(cfg): 245 buildorder = buildsort(list(cfg.roots)) 246 print("build order:", buildorder) 247 248 249 250 251 def promote(cfg): 252 dry = cfg.dry 253 distro = cfg.distro 254 vendor, codename = distro.split("-", 1) 255 listfmt = "${package}_${version}_${architecture}.${$type}\n" 256 if dry: 257 subprocess.run( 258 [ 259 "ssh", 260 f"taler-packaging@{host}", 261 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ checkpull {codename}", 262 ], 263 check=True, 264 ) 265 else: 266 subprocess.run( 267 [ 268 "ssh", 269 "-t", 270 f"taler-packaging@{host}", 271 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ pull {codename}", 272 ], 273 check=True, 274 ) 275 # Always export! 276 # Reprepro is weird, listed packages might actually not show 277 # up in the index yet. 278 subprocess.run( 279 [ 280 "ssh", 281 "-t", 282 f"taler-packaging@{host}", 283 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}", 284 ], 285 check=True, 286 ) 287 288 def show_published(cfg): 289 distro = cfg.distro 290 vendor, codename = distro.split("-", 1) 291 listfmt = "${package}_${version}_${architecture}.${$type}\n" 292 subprocess.run( 293 [ 294 "ssh", 295 f"taler-packaging@{host}", 296 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 297 ], 298 check=True, 299 ) 300 301 302 def test(cfg): 303 target = cfg.distro 304 vendor, codename, *rest = target.split("-") 305 distro = f"{vendor}-{codename}" 306 image_tag = f"localhost/taler-packaging-{distro}:latest" 307 dockerfile = f"distros/{distro}.Dockerfile" 308 cachedir = Path(f"cache").absolute() 309 print("building base image") 310 subprocess.run( 311 [ 312 "podman", 313 "build", 314 "-v", 315 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 316 "-v", 317 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 318 "-t", 319 image_tag, 320 "-f", 321 dockerfile, 322 ], 323 check=True, 324 ) 325 print("running test") 326 cmd = [ 327 "podman", 328 "run", 329 "-it", 330 "--entrypoint=/bin/bash", 331 "--security-opt", 332 "label=disable", 333 "--mount", 334 f"type=bind,source={mydir}/testing,target=/testing,readonly", 335 image_tag, 336 f"/testing/test-{target}", 337 ] 338 subprocess.run( 339 cmd, 340 check=True, 341 ) 342 343 def publish(cfg): 344 distro = cfg.distro 345 if distro.endswith("-testing"): 346 print("Files are automatically published to testing", file=sys.stderr) 347 sys.exit(1) 348 vendor, codename = distro.split("-", 1) 349 # List of .deb and .ddeb files. 350 debs = [] 351 listfmt = "${package}_${version}_${architecture}.${$type}\n" 352 server_debs_str = subprocess.check_output( 353 [ 354 "ssh", 355 f"taler-packaging@{host}", 356 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 357 ], 358 encoding="utf-8", 359 ) 360 server_debs = server_debs_str.split() 361 for component in components: 362 current = [] 363 for arch in archs + ["all"]: 364 cf = Path(f"./packages/{distro}/{component}@{arch}.built.current") 365 if not cf.exists(): 366 print(f"component {component}@{arch} has no current packages") 367 continue 368 with open(cf) as f: 369 current = current + f.read().split() 370 print("current", current) 371 for deb in current: 372 if deb.endswith(".deb"): 373 pkg1, ver1, arch1 = deb.removesuffix(".deb").split("_") 374 elif deb.endswith(".ddeb"): 375 pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_") 376 else: 377 raise Error(f"invalid deb filename: {deb}") 378 fresh = True 379 server_deb = None 380 # If the server has the same or a later version, 381 # the local version isn't fresh. 382 for srvdeb in server_debs: 383 pkg2, ver2, arch2 = srvdeb.removesuffix(".deb").split("_") 384 if pkg1 != pkg2 or arch1 != arch2: 385 continue 386 if vercomp.compare_versions(ver1, ver2) <= 0: 387 fresh = False 388 server_deb = srvdeb 389 break 390 if fresh: 391 debs.append(deb) 392 else: 393 print("package", deb, "not fresh, server has", server_deb) 394 if len(debs) == 0: 395 print("nothing to upload") 396 else: 397 print("uploading debs", debs) 398 if cfg.dry: 399 return 400 debs = [Path(f"./packages/{distro}/") / x for x in debs] 401 subprocess.run( 402 [ 403 "ssh", 404 f"taler-packaging@{host}", 405 f"rm -f '/home/taler-packaging/{distro}/'*.deb '/home/taler-packaging/{distro}/'*.ddeb", 406 ], 407 check=True, 408 ) 409 subprocess.run( 410 ["scp", "--", *debs, f"taler-packaging@{host}:{distro}/"], check=True 411 ) 412 ret = subprocess.run( 413 [ 414 "ssh", 415 "-t", 416 f"taler-packaging@{host}", 417 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.deb", 418 ], 419 ) 420 if ret.returncode != 0: 421 # Usually not critical if it fails. 422 print( 423 "Including ddebs failed. This can happen when including packages that have been included previously" 424 ) 425 # Almost the same, but with ddebs. 426 # We explicitly need to tell reprepro 427 # to ignore the extension, because it does not 428 # deal well with ddebs out of the box. 429 ret = subprocess.run( 430 [ 431 "ssh", 432 "-t", 433 f"taler-packaging@{host}", 434 f"reprepro --ignore=extension -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.ddeb", 435 ], 436 ) 437 if ret.returncode != 0: 438 # Usually not critical if it fails. 439 print( 440 "Including ddebs failed. This can happen when including packages that have been included previously" 441 ) 442 # Always export! 443 # Reprepro is weird, listed packages might actually not show 444 # up in the index yet. 445 subprocess.run( 446 [ 447 "ssh", 448 "-t", 449 f"taler-packaging@{host}", 450 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing", 451 ], 452 check=True, 453 ) 454 455 456 def get_remote_version(url): 457 """Get the latest stable tag from the git repo""" 458 # Construct the git command 459 # We use -c versionsort.suffix=- to ensure correct semantic version sorting 460 cmd = [ 461 "git", 462 "-c", 463 "versionsort.suffix=-", 464 "ls-remote", 465 "--exit-code", 466 "--refs", 467 "--sort=version:refname", 468 "--tags", 469 url, 470 "*.*.*", 471 ] 472 473 result = subprocess.run(cmd, capture_output=True, text=True, check=True) 474 475 # Parse the output 476 # Output format is usually: <hash>\trefs/tags/<tagname> 477 lines = result.stdout.strip().split("\n") 478 479 valid_tags = [] 480 481 for line in lines: 482 parts = line.split() 483 if len(parts) < 2: 484 continue 485 486 # refs/tags/v1.0.0 -> v1.0.0 487 ref_path = parts[1] 488 tag = ref_path.split("/")[-1] 489 490 # Exclude pre-release semver versions 491 if tag.startswith("v") and "-" in tag: 492 continue 493 494 valid_tags.append(tag) 495 496 if valid_tags: 497 return valid_tags[-1] 498 return "(none)" 499 500 501 def check_version(name, url): 502 """ 503 Compares local buildconfig version with remote git version. 504 """ 505 ver = get_remote_version(url) 506 config_path = os.path.join("buildconfig", f"{name}.tag") 507 with open(config_path, "r") as f: 508 curr = f.read().strip() 509 prefix = "[!] " if curr != ver else "" 510 print(f"{prefix}{name} curr: {curr} latest: {ver}") 511 512 513 def print_latest(cfg): 514 """Print latest upstream tag for each component""" 515 for name in components: 516 config_path = os.path.join("buildconfig", f"{name}.giturl") 517 with open(config_path, "r") as f: 518 giturl = f.read().strip() 519 check_version(name, giturl) 520 521 522 def main(): 523 parser = argparse.ArgumentParser( 524 prog="taler-pkg", description="Taler Packaging Helper" 525 ) 526 527 subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND") 528 529 # subcommand build 530 531 parser_build = subparsers.add_parser("build", help="Build packages for distro.") 532 parser_build.set_defaults(func=build) 533 parser_build.add_argument("distro") 534 # Keep for backwards compat 535 parser_build.add_argument( 536 "--no-transitive", 537 help="Do not build transitive deps of changed components (default)", 538 action="store_true", 539 dest="transitive", 540 default=None, 541 ) 542 parser_build.add_argument( 543 "--transitive", 544 help="Build transitive deps of changed components", 545 action="store_false", 546 dest="transitive", 547 default=None, 548 ) 549 parser_build.add_argument( 550 "--arch", 551 help="Architecture(s) to build for", 552 action="store", 553 dest="arch", 554 default=None, 555 ) 556 parser_build.add_argument( 557 "--dry", help="Dry run", action="store_true", default=False 558 ) 559 560 parser_test = subparsers.add_parser("test", help="Test packages for distro.") 561 parser_test.set_defaults(func=test) 562 parser_test.add_argument("distro") 563 parser_test.add_argument( 564 "--arch", 565 help="Architecture(s) to test packages for", 566 action="store", 567 dest="arch", 568 default=None, 569 ) 570 571 # subcommand show-latest 572 573 parser_show_latest = subparsers.add_parser( 574 "show-latest", help="Show latest version of packages." 575 ) 576 parser_show_latest.set_defaults(func=print_latest) 577 578 # subcommand show-order 579 580 parser_show_order = subparsers.add_parser("show-order", help="Show build order.") 581 parser_show_order.set_defaults(func=show_order) 582 parser_show_order.add_argument("roots", nargs="+") 583 584 # subcommand show-published 585 parser_show_published = subparsers.add_parser( 586 "show-published", help="Show published packages on deb.taler.net" 587 ) 588 parser_show_published.add_argument("distro") 589 parser_show_published.set_defaults(func=show_published) 590 591 # subcommand publish 592 593 parser_publish = subparsers.add_parser("publish", help="Publish to deb.taler.net") 594 parser_publish.add_argument( 595 "--dry", help="Dry run", action="store_true", default=False 596 ) 597 parser_publish.add_argument("distro") 598 parser_publish.set_defaults(func=publish) 599 600 parser_promote = subparsers.add_parser("promote", help="Promote testing to stable") 601 parser_promote.add_argument( 602 "--dry", help="Dry run (show pulls)", action="store_true", default=False 603 ) 604 parser_promote.add_argument("distro") 605 parser_promote.set_defaults(func=promote) 606 607 args = parser.parse_args() 608 609 if "func" not in args: 610 parser.print_help() 611 else: 612 args.func(args) 613 614 615 if __name__ == "__main__": 616 main()