#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-or-later """Controller-side standard Debian build, validation and local repository import.""" import email.parser import hashlib import io import json import os import re import shlex import subprocess import sys import tarfile from pathlib import Path W = Path("/work") assert os.geteuid() != 0 and Path.cwd() == W module = sys.argv[1] assert re.fullmatch("[a-z0-9][a-z0-9+.-]*", module) plan = json.loads((W / "input/modules" / (module + ".json")).read_text()) release = json.loads((W / "input/release.json").read_text()) arch = release["architecture"] suite = release["build_suite"] prefix = release["prefix"].removeprefix("/") source = W / plan["target"] version = plan["version"] name = plan["source"] def digest(p): with p.open("rb") as f: return hashlib.file_digest(f, "sha256").hexdigest() def tree(root): return { str(p.relative_to(root)): ( {"link": os.readlink(p)} if p.is_symlink() else {"sha256": digest(p), "mode": p.stat().st_mode & 0o777} ) for p in sorted(root.rglob("*")) if p.is_symlink() or p.is_file() } def output(*args): return subprocess.check_output(args, text=True).strip() def cache_setup_command(url): if not re.fullmatch(r"http://127\.0\.0\.1:[1-9][0-9]{0,4}", url): raise ValueError("Run builds through the apt-cache wrapper.") rewrite = "s#https://#" + url + "/HTTPS///#g" return shlex.join( [ "find", "/etc/apt", "-maxdepth", "2", "-type", "f", "(", "-name", "*.sources", "-o", "-name", "*.list", ")", "-exec", "sed", "-i", rewrite, "{}", "+", ] ) statepath = W / "state" / (module + ".json") statepath.parent.mkdir(exist_ok=True) identity = hashlib.sha256(json.dumps(plan, sort_keys=True).encode()).hexdigest() old = json.loads(statepath.read_text()) if statepath.exists() else None if old and old["inputs"] == identity: for filename, sha in old["artifacts"].items(): assert digest(W / "results" / filename) == sha, filename print(module + ": accepted build reused", flush=True) raise SystemExit() if old and old["version"] == version: raise ValueError( "Recipe/build inputs changed without a Debian revision bump: " + module ) assert tree(source / "debian") == plan["prepared"]["prepared_debian"], ( "Prepared recipe changed" ) subprocess.run( ["sha256sum", "--check", str(W / "records/build-root.sha256")], check=True, stdout=subprocess.DEVNULL, ) logdir = W / "records/build-logs" logdir.mkdir(exist_ok=True) logpath = logdir / (module + ".log") with logpath.open("a") as log: subprocess.run( ["dpkg-source", "--build", str(source)], cwd=W / "sources", stdout=log, stderr=subprocess.STDOUT, check=True, ) # APT needs package indexes even before the repository has its first package. subprocess.run( ["reprepro", "--basedir", str(W / "repository"), "export", suite], stdout=log, stderr=subprocess.STDOUT, check=True, ) args = [ "sbuild", "--dist=" + release["distribution"], "--jobs=" + str(plan["jobs"]), "--force-orig-source", ] cache_url = os.environ.get("PLASMA_NEXT_APT_CACHE_URL") if cache_url is not None: args.append("--chroot-setup-commands=" + cache_setup_command(cache_url)) if plan["architecture_all"]: args += ["--arch-all", "--no-arch-any"] result = subprocess.run( args + [str(W / "sources" / f"{name}_{version}.dsc")], stdout=log, stderr=subprocess.STDOUT, ) if result.returncode: raise RuntimeError( f"Build failed. Inspect {logpath} and results/{name}_{version}_{arch}.build" ) buildlog = (W / "results" / f"{name}_{version}_{arch}.build").read_text() successful = bool(re.search(r"^Status: successful$", buildlog, re.M)) clean = bool(re.search(r"^Lintian: pass$", buildlog, re.M)) if not (successful and clean): allowpath = source / "debian/lintian-warning-allowlist.json" allowed = ( json.loads(allowpath.read_text())["warnings"] if allowpath.exists() else [] ) lintian_section = buildlog.partition("\nRunning lintian...\n")[2].partition( "\n+---" )[0] warnings = [ line for line in lintian_section.splitlines() if line.startswith(("E:", "W:")) ] assert ( successful and lintian_section and allowed and sorted(warnings) == sorted(allowed) and all( line.startswith("W:") and any( ": " + tag in line for tag in [ "package-has-long-file-name ", "source-package-component-has-long-file-name ", ] ) for line in warnings ) ), "Build/Lintian acceptance failed: " + str(logpath) changes = list((W / "results").glob(f"{name}_{version}_*.changes")) assert len(changes) == 1 record = email.parser.Parser().parsestr(changes[0].read_text()) artifacts = {} for line in record["Checksums-Sha256"].splitlines(): if not line.strip(): continue sha, size, filename = line.split() assert Path(filename).name == filename path = W / "results" / filename assert path.stat().st_size == int(size) and digest(path) == sha, filename artifacts[filename] = sha for filename, sha in artifacts.items(): if ".orig" in filename and ".tar." in filename: assert plan["prepared"]["inputs"]["source_files"][filename] == sha, ( "Original source archive changed" ) artifacts[changes[0].name] = digest(changes[0]) owned = set() for package in plan["packages"]: debs = list((W / "results").glob(f"{package}_{version}_*.deb")) assert len(debs) == 1, package data = subprocess.check_output(["dpkg-deb", "--fsys-tarfile", str(debs[0])]) with tarfile.open(fileobj=io.BytesIO(data)) as tar: paths = {m.name.removeprefix("./") for m in tar if not m.isdir()} assert paths and not paths & owned, (module, "overlap") owned |= paths if module != "session-integration": assert all( path.startswith( ( prefix + "/", prefix + "-build/", "usr/share/doc/", "usr/share/lintian/overrides/", ) ) for path in paths ), "Unexpected global package payload" if not package.endswith("-dev"): assert not any( path.startswith(prefix + "/include/") for path in paths ), "Headers in runtime package" dependencies = output("dpkg-deb", "--field", str(debs[0]), "Depends") assert not re.search(r"\blibqt6", dependencies), ( module, "native Qt dependency", ) subprocess.run( [ "reprepro", "--basedir", str(W / "repository"), "--ignore=wrongdistribution", "include", suite, str(changes[0]), ], stdout=log, stderr=subprocess.STDOUT, check=True, ) for action in ["check", "checkpool"]: subprocess.run( ["reprepro", "--basedir", str(W / "repository"), action], stdout=log, stderr=subprocess.STDOUT, check=True, ) tmp = statepath.with_suffix(".json.tmp") tmp.write_text( json.dumps( {"inputs": identity, "version": version, "artifacts": artifacts}, indent=2 ) + "\n" ) tmp.replace(statepath) print(module + ": built, checked and imported", flush=True)