buildlib.py (10492B)
1 #!/usr/bin/env python3 2 3 # This file is in the public domain. 4 5 import json 6 import os 7 import shutil 8 import subprocess 9 import sys 10 from email.utils import formatdate 11 from pathlib import Path 12 13 from package_config import ConfigError, load_config 14 15 PKGDIR = Path("/pkgdir") 16 OVERLAYDIR = Path("/buildscripts/debian-overlays") 17 18 19 def run_cmd(cmd, shell=False, cwd=None, env=None): 20 """Run a command and stop the build if it fails.""" 21 command_env = os.environ.copy() 22 if env: 23 command_env.update(env) 24 sys.stdout.flush() 25 subprocess.check_call(cmd, shell=shell, cwd=cwd, env=command_env) 26 27 28 def get_output(cmd, shell=False, cwd=None): 29 return subprocess.check_output(cmd, shell=shell, cwd=cwd, text=True).strip() 30 31 32 def remove_stale_packages(pkgdir): 33 """Remove package artifacts not referenced by a current-build manifest.""" 34 pkgdir = Path(pkgdir) 35 current_packages = set() 36 for manifest in pkgdir.glob("*.built.current"): 37 current_packages.update(manifest.read_text().split()) 38 39 for artifact in sorted(pkgdir.iterdir()): 40 if artifact.suffix not in (".deb", ".ddeb"): 41 continue 42 if artifact.name in current_packages: 43 continue 44 print(f"Removing stale local package {artifact.name}") 45 artifact.unlink() 46 47 48 def get_tag_debver(tag): 49 """Get a Debian version string from a supported Git tag.""" 50 if tag.startswith("v"): 51 devsuff = "-dev." 52 position = tag.find(devsuff) 53 if position < 0: 54 return tag[1:] 55 return tag[1:position] + "~dev" + tag[position + len(devsuff) :] 56 if tag.startswith("deb-v"): 57 tag = tag[5:] 58 if "-" in tag: 59 version, revision = tag.split("-", 1) 60 return version + "-" + revision 61 return tag 62 raise ValueError(f"unexpected tag format: {tag}") 63 64 65 def make_codename_version(deb_version, build_codename): 66 if "-" in deb_version: 67 return f"{deb_version}+{build_codename}" 68 return f"{deb_version}-0+{build_codename}" 69 70 71 def scan_local_repository(): 72 os.chdir(PKGDIR) 73 packages_index = PKGDIR / "Packages.xz" 74 # Package versions can be identical across architectures. Keep every 75 # variant so one architecture does not disappear from the APT index. 76 run_cmd( 77 f"dpkg-scanpackages --multiversion . | xz - > {packages_index}", shell=True 78 ) 79 with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as source: 80 source.write(f"deb [trusted=yes] file:{PKGDIR} ./\n") 81 run_cmd(["apt-get", "update"]) 82 83 84 def _validate_group(config, package_names, expected_builder): 85 if not package_names: 86 raise ConfigError("no packages specified") 87 if len(set(package_names)) != len(package_names): 88 raise ConfigError("a package was specified more than once") 89 90 packages = [] 91 keys = set() 92 for name in package_names: 93 try: 94 package = config.packages[name] 95 except KeyError as exc: 96 raise ConfigError(f"unknown package {name!r}") from exc 97 repository = config.repository_for(package) 98 builder = config.builder_for(package) 99 keys.add((repository.url, package.tag, builder)) 100 packages.append(package) 101 if len(keys) != 1: 102 raise ConfigError("grouped packages must have the same repository, tag, and builder") 103 repository_url, tag, builder = keys.pop() 104 if builder != expected_builder: 105 raise ConfigError(f"expected builder {expected_builder!r}, got {builder!r}") 106 if expected_builder != "pnpm-workspace" and len(packages) != 1: 107 raise ConfigError(f"the {expected_builder} builder accepts exactly one package") 108 return packages, repository_url, tag 109 110 111 def _apply_debian_overlay(package, package_path): 112 overlay = OVERLAYDIR / package.name / "debian" 113 target = package_path / "debian" 114 if not overlay.is_dir(): 115 raise ConfigError(f"missing Debian overlay for {package.name}: {overlay}") 116 if target.exists(): 117 raise ConfigError(f"refusing to replace existing Debian packaging: {target}") 118 shutil.copytree(overlay, target) 119 120 121 def _install_build_dependencies(source_dir, package_paths): 122 tool = "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" 123 controls = [str(package_path / "debian" / "control") for package_path in package_paths] 124 run_cmd( 125 ["mk-build-deps", "--install", f"--tool={tool}", *controls], 126 cwd=source_dir, 127 ) 128 129 130 def _prepare_pnpm_workspace(source_dir, package_paths): 131 filters = [] 132 for package_path in package_paths: 133 package_json = package_path / "package.json" 134 with package_json.open(encoding="utf-8") as package_file: 135 workspace_name = json.load(package_file).get("name") 136 if not isinstance(workspace_name, str) or not workspace_name: 137 raise ConfigError(f"{package_json} has no package name") 138 filters.extend(["--filter", f"{workspace_name}..."]) 139 140 run_cmd(["pnpm", "install", "--frozen-lockfile", *filters], cwd=source_dir) 141 run_cmd(["pnpm", "run", *filters, "build"], cwd=source_dir) 142 143 144 def _write_changelog(package_path, package_name, version): 145 debian_date = formatdate(localtime=True) 146 changelog = f"""\ 147 {package_name} ({version}) unstable; urgency=low 148 149 * Release {version}. 150 151 -- Taler Packaging Team <deb@taler.net> {debian_date} 152 """ 153 (package_path / "debian" / "changelog").write_text(changelog) 154 155 156 def _package_artifacts(output_dir): 157 return { 158 artifact.resolve() 159 for pattern in ("*.deb", "*.ddeb") 160 for artifact in output_dir.glob(pattern) 161 } 162 163 164 def _check_installed_binaries(deb_files): 165 for deb in deb_files: 166 contents = get_output(["dpkg", "--contents", str(deb)]) 167 for line in contents.splitlines(): 168 parts = line.split() 169 if len(parts) < 6: 170 raise RuntimeError(f"failed to read package contents from {deb}") 171 filename = parts[5] 172 if "bin" not in filename: 173 continue 174 if filename.startswith("./"): 175 filename = filename[2:] 176 if not filename.startswith("/"): 177 filename = "/" + filename 178 file_info = get_output(["file", filename]) 179 if "ELF" in file_info and "executable" in file_info: 180 print(f"checking {filename}") 181 try: 182 run_cmd(["ldd", filename]) 183 except subprocess.CalledProcessError as exc: 184 raise RuntimeError( 185 f"installed binary {filename} has a linker issue" 186 ) from exc 187 188 189 def _build_package(package, package_path, tag, codename, arch, prebuilt): 190 version = make_codename_version(get_tag_debver(tag), codename) 191 print(f"Building {package.name} as version {version}", file=sys.stderr) 192 _write_changelog(package_path, package.name, version) 193 194 output_dir = package_path.parent 195 before = _package_artifacts(output_dir) 196 environment = {"DEB_BUILD_MAINT_OPTIONS": "debug"} 197 if prebuilt: 198 environment["TALER_PACKAGING_PREBUILT"] = "1" 199 200 debug_repository = package_path / "debian" / ".debhelper" 201 debug_repository.mkdir(parents=True, exist_ok=True) 202 (debug_repository / "debian-symbols-pool").touch() 203 environment["DEB_DBG_SYMBOLS_REPO"] = "debian/.debhelper/" 204 205 run_cmd( 206 ["dpkg-buildpackage", "-rfakeroot", "-b", "-uc", "-us"], 207 cwd=package_path, 208 env=environment, 209 ) 210 artifacts = _package_artifacts(output_dir) - before 211 deb_files = sorted(path for path in artifacts if path.suffix == ".deb") 212 if not deb_files: 213 raise RuntimeError(f"{package.name} did not produce a Debian package") 214 215 print(f"Installing built packages from {output_dir}", file=sys.stderr) 216 run_cmd(["apt", "install", "-y", *map(str, deb_files)]) 217 _check_installed_binaries(deb_files) 218 219 for artifact in artifacts: 220 shutil.copy(artifact, PKGDIR) 221 manifest = PKGDIR / f"{package.name}@{arch}.built.current" 222 manifest.write_text("".join(f"{artifact.name}\n" for artifact in sorted(artifacts))) 223 (PKGDIR / f"{package.name}@{arch}.built.tag").write_text(tag + "\n") 224 remove_stale_packages(PKGDIR) 225 scan_local_repository() 226 227 228 def build_packages(codename, arch, package_names, expected_builder): 229 if "LD_LIBRARY_PATH" in os.environ: 230 del os.environ["LD_LIBRARY_PATH"] 231 232 config = load_config("/packages.toml") 233 packages, repository_url, tag = _validate_group( 234 config, package_names, expected_builder 235 ) 236 print( 237 f"Building {' '.join(package_names)} with {expected_builder} build logic", 238 file=sys.stderr, 239 ) 240 241 scan_local_repository() 242 source_dir = Path("/build/source") 243 source_dir.parent.mkdir(parents=True, exist_ok=True) 244 run_cmd(["git", "config", "--global", "advice.detachedHead", "false"]) 245 run_cmd( 246 [ 247 "git", 248 "clone", 249 "--depth=1", 250 f"--branch={tag}", 251 repository_url, 252 str(source_dir), 253 ] 254 ) 255 run_cmd(["./bootstrap"], cwd=source_dir) 256 257 package_paths = [source_dir / package.debian_path for package in packages] 258 if expected_builder == "debian-overlay": 259 _apply_debian_overlay(packages[0], package_paths[0]) 260 for package, package_path in zip(packages, package_paths): 261 control = package_path / "debian" / "control" 262 if not control.is_file(): 263 raise ConfigError( 264 f"{package.name} has no debian/control at tag {tag}: {control}" 265 ) 266 (package_path / ".version").write_text(get_tag_debver(tag) + "\n") 267 268 _install_build_dependencies(source_dir, package_paths) 269 if expected_builder == "pnpm-workspace": 270 _prepare_pnpm_workspace(source_dir, package_paths) 271 272 for package, package_path in zip(packages, package_paths): 273 _build_package( 274 package, 275 package_path, 276 tag, 277 codename, 278 arch, 279 prebuilt=expected_builder == "pnpm-workspace", 280 ) 281 282 283 def main(expected_builder): 284 if len(sys.argv) < 4: 285 print( 286 f"Usage: {Path(sys.argv[0]).name} <CODENAME> <ARCH> <PACKAGE>...", 287 file=sys.stderr, 288 ) 289 sys.exit(1) 290 try: 291 build_packages(sys.argv[1], sys.argv[2], sys.argv[3:], expected_builder) 292 except ConfigError as exc: 293 print(f"configuration error: {exc}", file=sys.stderr) 294 sys.exit(1)