251 lines
8.1 KiB
Python
251 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Controller-side standard Debian build, validation and local repository import."""
|
|
|
|
import email.parser
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
from pathlib import Path
|
|
|
|
W = Path("/work")
|
|
assert os.geteuid() != 0 and Path.cwd() == W
|
|
module = sys.argv[1]
|
|
assert re.fullmatch("[a-z0-9][a-z0-9+.-]*", module)
|
|
plan = json.loads((W / "input/modules" / (module + ".json")).read_text())
|
|
release = json.loads((W / "input/release.json").read_text())
|
|
arch = release["architecture"]
|
|
suite = release["build_suite"]
|
|
prefix = release["prefix"].removeprefix("/")
|
|
source = W / plan["target"]
|
|
version = plan["version"]
|
|
name = plan["source"]
|
|
|
|
|
|
def digest(p):
|
|
with p.open("rb") as f:
|
|
return hashlib.file_digest(f, "sha256").hexdigest()
|
|
|
|
|
|
def tree(root):
|
|
return {
|
|
str(p.relative_to(root)): (
|
|
{"link": os.readlink(p)}
|
|
if p.is_symlink()
|
|
else {"sha256": digest(p), "mode": p.stat().st_mode & 0o777}
|
|
)
|
|
for p in sorted(root.rglob("*"))
|
|
if p.is_symlink() or p.is_file()
|
|
}
|
|
|
|
|
|
def output(*args):
|
|
return subprocess.check_output(args, text=True).strip()
|
|
|
|
|
|
def cache_setup_command(url):
|
|
if not re.fullmatch(r"http://127\.0\.0\.1:[1-9][0-9]{0,4}", url):
|
|
raise ValueError("Run builds through the apt-cache wrapper.")
|
|
rewrite = "s#https://#" + url + "/HTTPS///#g"
|
|
return shlex.join(
|
|
[
|
|
"find",
|
|
"/etc/apt",
|
|
"-maxdepth",
|
|
"2",
|
|
"-type",
|
|
"f",
|
|
"(",
|
|
"-name",
|
|
"*.sources",
|
|
"-o",
|
|
"-name",
|
|
"*.list",
|
|
")",
|
|
"-exec",
|
|
"sed",
|
|
"-i",
|
|
rewrite,
|
|
"{}",
|
|
"+",
|
|
]
|
|
)
|
|
|
|
|
|
statepath = W / "state" / (module + ".json")
|
|
statepath.parent.mkdir(exist_ok=True)
|
|
identity = hashlib.sha256(json.dumps(plan, sort_keys=True).encode()).hexdigest()
|
|
old = json.loads(statepath.read_text()) if statepath.exists() else None
|
|
if old and old["inputs"] == identity:
|
|
for filename, sha in old["artifacts"].items():
|
|
assert digest(W / "results" / filename) == sha, filename
|
|
print(module + ": accepted build reused", flush=True)
|
|
raise SystemExit()
|
|
if old and old["version"] == version:
|
|
raise ValueError(
|
|
"Recipe/build inputs changed without a Debian revision bump: " + module
|
|
)
|
|
assert tree(source / "debian") == plan["prepared"]["prepared_debian"], (
|
|
"Prepared recipe changed"
|
|
)
|
|
subprocess.run(
|
|
["sha256sum", "--check", str(W / "records/build-root.sha256")],
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
logdir = W / "records/build-logs"
|
|
logdir.mkdir(exist_ok=True)
|
|
logpath = logdir / (module + ".log")
|
|
with logpath.open("a") as log:
|
|
subprocess.run(
|
|
["dpkg-source", "--build", str(source)],
|
|
cwd=W / "sources",
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
check=True,
|
|
)
|
|
# APT needs package indexes even before the repository has its first package.
|
|
subprocess.run(
|
|
["reprepro", "--basedir", str(W / "repository"), "export", suite],
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
check=True,
|
|
)
|
|
args = [
|
|
"sbuild",
|
|
"--dist=" + release["distribution"],
|
|
"--jobs=" + str(plan["jobs"]),
|
|
"--force-orig-source",
|
|
]
|
|
cache_url = os.environ.get("PLASMA_NEXT_APT_CACHE_URL")
|
|
if cache_url is not None:
|
|
args.append("--chroot-setup-commands=" + cache_setup_command(cache_url))
|
|
if plan["architecture_all"]:
|
|
args += ["--arch-all", "--no-arch-any"]
|
|
result = subprocess.run(
|
|
args + [str(W / "sources" / f"{name}_{version}.dsc")],
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
if result.returncode:
|
|
raise RuntimeError(
|
|
f"Build failed. Inspect {logpath} and results/{name}_{version}_{arch}.build"
|
|
)
|
|
buildlog = (W / "results" / f"{name}_{version}_{arch}.build").read_text()
|
|
successful = bool(re.search(r"^Status: successful$", buildlog, re.M))
|
|
clean = bool(re.search(r"^Lintian: pass$", buildlog, re.M))
|
|
if not (successful and clean):
|
|
allowpath = source / "debian/lintian-warning-allowlist.json"
|
|
allowed = (
|
|
json.loads(allowpath.read_text())["warnings"] if allowpath.exists() else []
|
|
)
|
|
lintian_section = buildlog.partition("\nRunning lintian...\n")[2].partition(
|
|
"\n+---"
|
|
)[0]
|
|
warnings = [
|
|
line
|
|
for line in lintian_section.splitlines()
|
|
if line.startswith(("E:", "W:"))
|
|
]
|
|
assert (
|
|
successful
|
|
and lintian_section
|
|
and allowed
|
|
and sorted(warnings) == sorted(allowed)
|
|
and all(
|
|
line.startswith("W:")
|
|
and any(
|
|
": " + tag in line
|
|
for tag in [
|
|
"package-has-long-file-name ",
|
|
"source-package-component-has-long-file-name ",
|
|
]
|
|
)
|
|
for line in warnings
|
|
)
|
|
), "Build/Lintian acceptance failed: " + str(logpath)
|
|
changes = list((W / "results").glob(f"{name}_{version}_*.changes"))
|
|
assert len(changes) == 1
|
|
record = email.parser.Parser().parsestr(changes[0].read_text())
|
|
artifacts = {}
|
|
for line in record["Checksums-Sha256"].splitlines():
|
|
if not line.strip():
|
|
continue
|
|
sha, size, filename = line.split()
|
|
assert Path(filename).name == filename
|
|
path = W / "results" / filename
|
|
assert path.stat().st_size == int(size) and digest(path) == sha, filename
|
|
artifacts[filename] = sha
|
|
for filename, sha in artifacts.items():
|
|
if ".orig" in filename and ".tar." in filename:
|
|
assert plan["prepared"]["inputs"]["source_files"][filename] == sha, (
|
|
"Original source archive changed"
|
|
)
|
|
artifacts[changes[0].name] = digest(changes[0])
|
|
owned = set()
|
|
for package in plan["packages"]:
|
|
debs = list((W / "results").glob(f"{package}_{version}_*.deb"))
|
|
assert len(debs) == 1, package
|
|
data = subprocess.check_output(["dpkg-deb", "--fsys-tarfile", str(debs[0])])
|
|
with tarfile.open(fileobj=io.BytesIO(data)) as tar:
|
|
paths = {m.name.removeprefix("./") for m in tar if not m.isdir()}
|
|
assert paths and not paths & owned, (module, "overlap")
|
|
owned |= paths
|
|
if module != "session-integration":
|
|
assert all(
|
|
path.startswith(
|
|
(
|
|
prefix + "/",
|
|
prefix + "-build/",
|
|
"usr/share/doc/",
|
|
"usr/share/lintian/overrides/",
|
|
)
|
|
)
|
|
for path in paths
|
|
), "Unexpected global package payload"
|
|
if not package.endswith("-dev"):
|
|
assert not any(
|
|
path.startswith(prefix + "/include/") for path in paths
|
|
), "Headers in runtime package"
|
|
dependencies = output("dpkg-deb", "--field", str(debs[0]), "Depends")
|
|
assert not re.search(r"\blibqt6", dependencies), (
|
|
module,
|
|
"native Qt dependency",
|
|
)
|
|
subprocess.run(
|
|
[
|
|
"reprepro",
|
|
"--basedir",
|
|
str(W / "repository"),
|
|
"--ignore=wrongdistribution",
|
|
"include",
|
|
suite,
|
|
str(changes[0]),
|
|
],
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
check=True,
|
|
)
|
|
for action in ["check", "checkpool"]:
|
|
subprocess.run(
|
|
["reprepro", "--basedir", str(W / "repository"), action],
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
check=True,
|
|
)
|
|
tmp = statepath.with_suffix(".json.tmp")
|
|
tmp.write_text(
|
|
json.dumps(
|
|
{"inputs": identity, "version": version, "artifacts": artifacts}, indent=2
|
|
)
|
|
+ "\n"
|
|
)
|
|
tmp.replace(statepath)
|
|
print(module + ": built, checked and imported", flush=True)
|