358 lines
13 KiB
Python
358 lines
13 KiB
Python
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Release definitions, dependency ordering and source preparation."""
|
|
|
|
import email.parser
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import urllib.request
|
|
from functools import lru_cache
|
|
from graphlib import TopologicalSorter
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
WORK = ROOT / "build"
|
|
|
|
|
|
def read(path):
|
|
return json.loads(Path(path).read_text())
|
|
|
|
|
|
def write(path, value):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
tmp.write_text(json.dumps(value, indent=2) + "\n")
|
|
tmp.replace(path)
|
|
|
|
|
|
def digest(path):
|
|
with Path(path).open("rb") as f:
|
|
return hashlib.file_digest(f, "sha256").hexdigest()
|
|
|
|
|
|
def tree(path):
|
|
result = {}
|
|
for p in sorted(Path(path).rglob("*")):
|
|
rel = str(p.relative_to(path))
|
|
if p.is_symlink():
|
|
result[rel] = {"link": os.readlink(p)}
|
|
elif p.is_file():
|
|
result[rel] = {"sha256": digest(p), "mode": p.stat().st_mode & 0o777}
|
|
return result
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def packages():
|
|
return read(ROOT / "config/sources.lock.json")["packages"]
|
|
|
|
|
|
def lock():
|
|
WORK.mkdir(exist_ok=True)
|
|
# Read-only access is enough for flock and still works when the sudo
|
|
# helpers created the file as root.
|
|
fd = os.open(WORK / "operation.lock", os.O_RDONLY | os.O_CREAT, 0o644)
|
|
f = os.fdopen(fd)
|
|
try:
|
|
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BaseException:
|
|
f.close()
|
|
raise
|
|
return f
|
|
|
|
|
|
def graph():
|
|
definitions = packages()
|
|
provider = {}
|
|
dependencies = {}
|
|
for module in definitions:
|
|
control = fields((ROOT / "packages" / module / "debian/control").read_text())
|
|
if control[0]["Source"] != definitions[module]["source"]:
|
|
raise ValueError("Source/control mismatch: " + module)
|
|
for paragraph in control[1:]:
|
|
binary = paragraph["Package"]
|
|
if binary in provider:
|
|
raise ValueError("Duplicate binary package: " + binary)
|
|
provider[binary] = module
|
|
dependencies[module] = " ".join(
|
|
control[0].get(k, "")
|
|
for k in ["Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"]
|
|
)
|
|
result = {}
|
|
for module, text in dependencies.items():
|
|
names = set(re.findall(r"\bplasma-next-[a-z0-9+.-]+", text))
|
|
missing = names - provider.keys()
|
|
if missing:
|
|
raise ValueError("Unknown private build dependencies: " + str(missing))
|
|
result[module] = sorted({provider[d] for d in names if provider[d] != module})
|
|
return result
|
|
|
|
|
|
def order(selected, dependencies=False):
|
|
definitions = packages()
|
|
selected = set(selected)
|
|
edges = graph()
|
|
if not selected <= definitions.keys():
|
|
raise ValueError(
|
|
"Unknown packages: " + ", ".join(sorted(selected - definitions.keys()))
|
|
)
|
|
if dependencies:
|
|
pending = list(selected)
|
|
while pending:
|
|
for dep in edges[pending.pop()]:
|
|
if dep not in selected:
|
|
selected.add(dep)
|
|
pending.append(dep)
|
|
return list(
|
|
TopologicalSorter(
|
|
{m: set(edges[m]) & selected for m in sorted(selected)}
|
|
).static_order()
|
|
)
|
|
|
|
|
|
def fetch(module, offline=False):
|
|
entry = packages()[module]
|
|
cache = WORK / "cache"
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
for f in entry["files"]:
|
|
path = cache / f["name"]
|
|
assert Path(f["name"]).name == f["name"]
|
|
if path.exists():
|
|
if digest(path) != f["sha256"] or path.stat().st_size != f["size"]:
|
|
raise ValueError("Cached input changed: " + f["name"])
|
|
continue
|
|
if offline:
|
|
raise FileNotFoundError(path)
|
|
fd, temp = tempfile.mkstemp(prefix=".download-", dir=cache)
|
|
try:
|
|
with (
|
|
os.fdopen(fd, "wb") as output,
|
|
urllib.request.urlopen(f["url"], timeout=60) as stream,
|
|
):
|
|
size = 0
|
|
while chunk := stream.read(1024 * 1024):
|
|
size += len(chunk)
|
|
if size > f["size"]:
|
|
raise ValueError("Download exceeds locked size: " + f["name"])
|
|
output.write(chunk)
|
|
tmp = Path(temp)
|
|
if size != f["size"] or digest(tmp) != f["sha256"]:
|
|
raise ValueError("Downloaded input identity mismatch: " + f["name"])
|
|
tmp.replace(path)
|
|
finally:
|
|
Path(temp).unlink(missing_ok=True)
|
|
|
|
|
|
def fields(text):
|
|
return [email.parser.Parser().parsestr(p) for p in text.split("\n\n") if p.strip()]
|
|
|
|
|
|
def recipe(module):
|
|
path = ROOT / "packages" / module / "debian"
|
|
first = (path / "changelog").read_text().splitlines()[0]
|
|
match = re.match(r"(\S+) \(([^)]+)\)", first)
|
|
if not match or match[1] != packages()[module]["source"]:
|
|
raise ValueError("Invalid source/changelog: " + module)
|
|
return path, match[2]
|
|
|
|
|
|
def data_filter(member, destination):
|
|
# This exact AppStream regression fixture is an intentionally dangling symlink.
|
|
if (
|
|
member.issym()
|
|
and member.name.endswith(
|
|
"/tests/samples/compose/usr/share/metainfo/org.example.nonexistent-badlink.metainfo.xml"
|
|
)
|
|
and member.linkname == "/nonexistent"
|
|
):
|
|
return member
|
|
filtered = tarfile.data_filter(member, destination)
|
|
if (
|
|
filtered is not None
|
|
and member.issym()
|
|
and member.linkname.startswith("./")
|
|
and filtered.linkname == (member.linkname[2:] or ".")
|
|
):
|
|
# Preserve source archive spelling after validating the link destination.
|
|
filtered = filtered.replace(linkname=member.linkname)
|
|
return filtered
|
|
|
|
|
|
def extract_root(archive, destination):
|
|
with tempfile.TemporaryDirectory(
|
|
prefix=".extract-", dir=destination.parent
|
|
) as temp:
|
|
with tarfile.open(archive) as t:
|
|
t.extractall(temp, filter=data_filter)
|
|
entries = list(Path(temp).iterdir())
|
|
if len(entries) != 1 or not entries[0].is_dir() or entries[0].is_symlink():
|
|
raise ValueError("Expected one source root: " + str(archive))
|
|
entries[0].rename(destination)
|
|
|
|
|
|
def source_materials(entry):
|
|
"""Return source input metadata used to identify a build."""
|
|
return [
|
|
{key: value for key, value in artifact.items() if key != "url"}
|
|
for artifact in entry["files"]
|
|
]
|
|
|
|
|
|
def prepare_crate(artifact, destination):
|
|
directory = artifact["directory"]
|
|
if Path(directory).name != directory or directory in {"", ".", ".."}:
|
|
raise ValueError("Invalid crate directory: " + directory)
|
|
target = destination / "debian/vendor" / directory
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
extract_root(WORK / "cache" / artifact["name"], target)
|
|
for path in target.rglob("*"):
|
|
if path.is_file() and not path.is_symlink():
|
|
path.chmod(0o755 if path.stat().st_mode & 0o111 else 0o644)
|
|
write(target / ".plasma-next-source.json", {"archive_sha256": artifact["sha256"]})
|
|
(target / ".plasma-next-source.json").chmod(0o644)
|
|
checksums = {
|
|
str(path.relative_to(target)): digest(path)
|
|
for path in sorted(target.rglob("*"))
|
|
if path.is_file() and path.name != ".cargo-checksum.json"
|
|
}
|
|
(target / ".cargo-checksum.json").write_text(
|
|
json.dumps(
|
|
{"files": checksums, "package": artifact["sha256"]},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
+ "\n"
|
|
)
|
|
(target / ".cargo-checksum.json").chmod(0o644)
|
|
|
|
|
|
def prepare(module, offline=False):
|
|
fetch(module, offline)
|
|
entry = packages()[module]
|
|
debian, version = recipe(module)
|
|
native = entry["format"] == "3.0 (native)"
|
|
if not native and version.rsplit("-", 1)[0] != entry["upstream_version"]:
|
|
raise ValueError(
|
|
"Update the locked upstream inputs before changing upstream version: "
|
|
+ module
|
|
)
|
|
inputs = {
|
|
"source_files": {
|
|
artifact["name"]: artifact["sha256"] for artifact in entry["files"]
|
|
},
|
|
"materials": source_materials(entry),
|
|
"format": entry["format"],
|
|
"debian": tree(debian),
|
|
"version": version,
|
|
"source": tree(ROOT / "packages" / module / "source"),
|
|
}
|
|
target = WORK / "sources" / (entry["source"] + "-" + version)
|
|
state = WORK / "prepared" / (module + ".json")
|
|
if state.exists() and target.exists() and read(state)["inputs"] == inputs:
|
|
if tree(target / "debian") != read(state)["prepared_debian"]:
|
|
raise ValueError("Prepared recipe modified outside packages/: " + module)
|
|
return target
|
|
if target.exists():
|
|
raise ValueError(
|
|
"Prepared source already exists with different inputs. Run clean for this project before preparing it again: "
|
|
+ module
|
|
)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with tempfile.TemporaryDirectory(prefix=".prepare-", dir=target.parent) as temp:
|
|
temporary = Path(temp) / "source"
|
|
if native:
|
|
temporary.mkdir()
|
|
else:
|
|
original = next(
|
|
artifact
|
|
for artifact in entry["files"]
|
|
if artifact["role"] == "original"
|
|
)
|
|
extract_root(WORK / "cache" / original["name"], temporary)
|
|
if (temporary / "debian").exists():
|
|
shutil.rmtree(temporary / "debian")
|
|
shutil.copytree(debian, temporary / "debian", symlinks=True)
|
|
payload = ROOT / "packages" / module / "source"
|
|
if native and payload.exists():
|
|
shutil.copytree(payload, temporary, dirs_exist_ok=True, symlinks=True)
|
|
for artifact in entry["files"]:
|
|
role = artifact["role"]
|
|
if role == "original-component":
|
|
component = artifact["component"]
|
|
if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9-]*", component):
|
|
raise ValueError("Invalid original source component: " + component)
|
|
extract_root(WORK / "cache" / artifact["name"], temporary / component)
|
|
elif role == "crate":
|
|
prepare_crate(artifact, temporary)
|
|
elif role == "bootstrap":
|
|
bootstrap = temporary / "bootstrap"
|
|
bootstrap.mkdir(exist_ok=True)
|
|
shutil.copyfile(
|
|
WORK / "cache" / artifact["name"], bootstrap / artifact["name"]
|
|
)
|
|
elif role != "original":
|
|
raise ValueError("Unknown source input role: " + role)
|
|
for artifact in entry["files"]:
|
|
if artifact["role"] in {"original", "original-component"}:
|
|
original = target.parent / artifact["name"]
|
|
if not original.exists() or digest(original) != artifact["sha256"]:
|
|
subprocess.run(
|
|
[
|
|
"cp",
|
|
"--reflink=auto",
|
|
"--force",
|
|
"--",
|
|
str(WORK / "cache" / artifact["name"]),
|
|
str(original),
|
|
],
|
|
check=True,
|
|
)
|
|
temporary.rename(target)
|
|
write(
|
|
state,
|
|
{
|
|
"inputs": inputs,
|
|
"prepared_debian": tree(target / "debian"),
|
|
"target": str(target.relative_to(WORK)),
|
|
},
|
|
)
|
|
return target
|
|
|
|
|
|
def check(modules, baseline=False):
|
|
definitions = packages()
|
|
references = read(ROOT / "config/recipes.lock.json")["packages"]
|
|
for module in modules:
|
|
path, version = recipe(module)
|
|
if version != definitions[module]["version"]:
|
|
raise ValueError("Version/lock mismatch: " + module)
|
|
if (path / "source/format").read_text().strip() != definitions[module][
|
|
"format"
|
|
]:
|
|
raise ValueError("Source format mismatch: " + module)
|
|
if baseline:
|
|
expected = {
|
|
p: v
|
|
for p, v in references[module]["debian"].items()
|
|
if not any(
|
|
p == d or p.startswith(d + "/")
|
|
for d in definitions[module]["external_recipe_dirs"]
|
|
)
|
|
}
|
|
if tree(path) != expected:
|
|
raise ValueError("Recipe differs from accepted baseline: " + module)
|
|
if (
|
|
tree(ROOT / "packages" / module / "source")
|
|
!= references[module]["native_source"]
|
|
):
|
|
raise ValueError(
|
|
"Native payload differs from accepted baseline: " + module
|
|
)
|
|
order(definitions)
|