From 6710ec715f12a6c76af5a09f6e3054b07e45bb3d Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Mon, 23 Dec 2024 00:34:11 +0100 Subject: abicheck: Add symbol-merge.py helper This allows merging symbol diffs from the various build logs --- abicheck/symbol-merge.py | 206 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 abicheck/symbol-merge.py diff --git a/abicheck/symbol-merge.py b/abicheck/symbol-merge.py new file mode 100644 index 000000000..1fa540c94 --- /dev/null +++ b/abicheck/symbol-merge.py @@ -0,0 +1,206 @@ +""" +Download packages from the archive and generate a symbols file for them. +""" + +import argparse +import glob +import os +import subprocess +import re +import sys +import urllib.request + +import distro_info + +import apt_pkg +import apt_inst + +import apt.progress.text + +IGNORE = ( + "^typeinfo for std::" + "|^vtable for std::" + "|^typeinfo name for std::" + "|^guard variable for std::" + "|^std::" + "|^[a-z]* std::" + "|^typeinfo for __gnu_cxx::" + "|^vtable for __gnu_cxx::" + "|^typeinfo name for __gnu_cxx::" + "|^guard variable for __gnu_cxx::" + "|^__gnu_cxx::" + "|^[a-z]* __gnu_cxx::" +) +IGNORE_RE = re.compile(IGNORE) + + +def get_archs(dist: str) -> set[str]: + if dist in distro_info.UbuntuDistroInfo().all: + url = f"https://snapshot.ubuntu.com/ubuntu/dists/{dist}/InRelease" + else: + url = f"https://deb.debian.org/debian/dists/{dist}/InRelease" + with urllib.request.urlopen(url) as rel: + for line in rel: + line = line.decode("utf-8") + if line.startswith("Architectures:"): + return set(line.split(":")[1].split()) - {"all"} + raise ValueError("Invalid Release file") + + +def download_debs(dist: str) -> list[str]: + """Download the debs and return the list of filenames.""" + runtime_dir = os.getenv("XDG_RUNTIME_DIR") + if runtime_dir is None: + raise ValueError("Need to set XDG_RUNTIME_DIR") + tmpdir = f"{runtime_dir}/apt/symbol-merger" + os.makedirs(f"{tmpdir}/etc/apt/sources.list.d", exist_ok=True) + os.makedirs(f"{tmpdir}/etc/apt/apt.conf.d", exist_ok=True) + os.makedirs(f"{tmpdir}/etc/apt/trusted.gpg.d", exist_ok=True) + os.makedirs(f"{tmpdir}/var/lib/apt/lists/partial", exist_ok=True) + os.makedirs(f"{tmpdir}/var/cache/apt/archives/partial", exist_ok=True) + + if dist in distro_info.UbuntuDistroInfo().all: + url = "https://snapshot.ubuntu.com/ubuntu/" + keyring = "/usr/share/keyrings/ubuntu-archive-keyring.gpg" + suites = f"{dist} {dist}-updates" + else: + url = "https://deb.debian.org/debian/" + keyring = "/usr/share/keyrings/debian-archive-keyring.gpg" + suites = f"{dist}" + + with open(f"{tmpdir}/etc/apt/sources.list.d/debian.sources", "w") as sources: + print("Types: deb", file=sources) + print(f"URIs: {url}", file=sources) + print(f"Suites: {suites}", file=sources) + print("Components: main", file=sources) + print(f"Signed-By: {keyring}", file=sources) + print(f"Architectures: {" ".join(archs)}", file=sources) + print("Targets: Packages", file=sources) + print("", file=sources) + + apt_pkg.config.set("Dir", tmpdir) + apt_pkg.config.set("Dir::State::status", "/dev/null") + apt_pkg.config.set("APT::List-Cleanup", "false") + apt_pkg.init() + cache = apt_pkg.Cache() + sl = apt_pkg.SourceList() + sl.read_main_list() + cache.update(apt.progress.text.AcquireProgress(), sl, 500000) + cache = apt_pkg.Cache() + depcache = apt_pkg.DepCache(cache) + + for arch in archs: + for _, _, ver in cache[("libapt-pkg", arch)].provides_list: + print("Mark", ver.parent_pkg.get_fullname(False), "for", arch, ver.ver_str) + depcache.mark_install(ver.parent_pkg, False) + + acq = apt_pkg.Acquire(apt.progress.text.AcquireProgress()) + pm = apt_pkg.PackageManager(depcache) + recs = apt_pkg.PackageRecords(cache) + pm.get_archives(acq, sl, recs) + acq.run() + + return [itm.destfile for itm in acq.items] + + +def read_main_symbols() -> dict[str, str]: + """Read the versions of already known symbols.""" + symbols = {} + with open(glob.glob("debian/libapt-pkg*.symbols")[0]) as sf: + for line in sf: + if not line.startswith(" "): + prelude.append(line.strip()) + continue + if line.startswith(" "): + symbol, version = line.strip().rsplit(None, 1) + if '"' in symbol: + print(symbol.strip().split('"')) + symbol = '(c++)"{}"'.format(symbol.strip().split('"')[1]) + symbols[symbol] = version + # Register the symbol so we keep the ordering later. + print("SYMBOL", symbol) + symbol_archs[symbol] = set() + return symbols + + +def read_symbol_file(debname: str) -> None: + deb = apt_inst.DebFile(debname) + arch = debname.split("_")[-1].split(".")[0] + symbols = deb.control.extractdata("symbols").decode("utf-8") + decoded_symbols = subprocess.Popen( + ["c++filt"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True + ).communicate(symbols)[0] + for line, decoded_line in zip(symbols.splitlines(), decoded_symbols.splitlines()): + if not line.startswith(" "): + continue + symbol, version = decoded_line.strip().rsplit(None, 1) + symbol = symbol.strip() + # Our version is higher than the registered one, so lower it again + if IGNORE_RE.search(symbol): + continue + if line != decoded_line: + symbol = '(c++)"{}"'.format(symbol) + if ( + symbol not in smallest_symbol_version + or apt_pkg.version_compare(version, smallest_symbol_version[symbol]) < 0 + ): + smallest_symbol_version[symbol] = version + + try: + symbol_archs[symbol].add(arch) + except KeyError: + symbol_archs[symbol] = {arch} + try: + latest_symbol_archs[symbol].add(arch) + except KeyError: + latest_symbol_archs[symbol] = {arch} + + +def print_merged() -> None: + for line in prelude: + print(line) + + print(f' (c++|regex)"{IGNORE}" 999:0misbuilt') + + for symbol, line_archs in symbol_archs.items(): + if archs == line_archs: + print("", symbol, smallest_symbol_version[symbol]) + + for symbol, line_archs in symbol_archs.items(): + if archs != line_archs and line_archs: + print_archs = " ".join(sorted(line_archs)) + if len(line_archs) > len(archs - line_archs): + print_archs = " ".join("!" + a for a in sorted(archs - line_archs)) + + if "(c++)" in symbol: + print( + "", + symbol.replace("(c++)", f"(arch={print_archs}|c++)"), + smallest_symbol_version[symbol], + ) + else: + print( + f" (arch={print_archs}){symbol} {smallest_symbol_version[symbol]}" + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("dist", nargs="+") + args = parser.parse_args() + prelude: list[str] = [] + symbol_archs: dict[str, set[str]] = {} + smallest_symbol_version: dict[str, str] = read_main_symbols() + for dist in args.dist: + latest_symbol_archs: dict[str, set[str]] = {} + archs = get_archs(dist) + for deb in download_debs(dist): + read_symbol_file(deb) + # Clean up removed symbols + for symbol in list(symbol_archs): + if symbol not in latest_symbol_archs: + del symbol_archs[symbol] + else: + symbol_archs[symbol] = latest_symbol_archs[symbol] + with open(glob.glob("debian/libapt-pkg*.symbols")[0], "w") as sys.stdout: + print_merged() -- cgit v1.2.3-70-g09d2