#!/usr/bin/env python3 """USM system-package-manager helper for Portage (Gentoo) systems. Implements the USM SPM contract (see usm/README.md, "System package manager integration") for Gentoo systems: usm-spm-emerge query ... -> contract JSON on STDOUT usm-spm-emerge install [--test] ... -> contract JSONL events on STDOUT API choice: a Gentoo stage3 ships Portage and python3, so this helper reads the installed-package database through the portage Python API (with a raw /var/db/pkg CONTENTS scan as fallback) and shells out to `emerge` for dependency resolution (pretend) and installation. There is no binary metadata to bind against beyond that: ebuild repositories carry no file lists, so unlike the dnf/apt/apk helpers this one cannot search "which package contains file X" for packages that are not installed yet. File-backed refs are therefore resolved in two documented stages: 1. INSTALLED files (exact): the ref is translated to candidate filesystem paths (gentoo layout: /usr/bin, /usr/sbin, /usr/lib64, /usr/share, ...) and matched against the portage vdb (/var/db/pkg/*/CONTENTS). A ref resolves to the installed package that owns the file. 2. NOT-installed files (exception table only): Gentoo has no offline file-provides (PFL, the portage file list, is an online service), so a small curated table maps the USM stack's build/platform refs to Gentoo package atoms (valac -> dev-lang/vala, glib-2.0.pc -> dev-libs/glib, ...). Refs outside the table land in not-found; anything else would be guesswork. dependency-count / installed-dependency-count are honest numbers parsed from `emerge --pretend --quiet=y --ask=n `: dependency-count is the number of packages in the merge list including the package itself, installed-dependency-count is the subset of the merge list already installed (action letters without N, e.g. [ebuild R]/[ebuild U] reinstalls and upgrades; a fresh stage3 typically shows an all-N list). If the pretend run fails (atom missing from the tree, masked, unresolvable) an exception-table candidate falls back to not-found. Install maps `emerge --ask=n --quiet=y ` onto the JSONL contract, linking the unversioned tool names Gentoo ships versioned (UNVERSIONED_TOOL_LINKS): `begin`/`total` come from a pretend run first, a `package` event is emitted per ">>> Emerging (n of N)" status line, `package-complete` per ">>> Installing (n of N)" line (the moment a merge lands in the vdb), `complete` carries the number of completed merges, and any failure emits a terminal `error` event before a non-zero exit (resolution failures map to exit 3, download failures to 4, build/merge failures to 5). Documented Gentoo limitations: * No offline file-provides for not-yet-installed packages (see stage 2 above); installed-file queries are exact. * USE flags are never touched: emerge runs with the system's configured defaults. The stack's platform libraries (glib, json-glib, libgee, ...) enable their introspection USE flag by default, but a system that disabled it (package.use/make.conf) will lack the matching .gir/ .typelib/.vapi artifacts after install; re-enable the flag and re-emerge if a build needs them. * Packages are compiled from source; even small installs take minutes. Progress granularity is per package; emerge exposes no useful intra-package progress, so `package` events carry progress 0.0 and the completion signal is the ">>> Installing" line. * The query subcommand never modifies system state (at most Portage regenerates its dependency cache); install must run as root and never prompts (--ask=n answers Portage's own prompt for it). """ import argparse import glob import json import os import re import subprocess import sys import threading EMERGE = "/usr/bin/emerge" EXIT_OK = 0 EXIT_FAILURE = 1 EXIT_RESOLVE = 3 EXIT_DOWNLOAD = 4 EXIT_TRANSACTION = 5 FILE_ROOTS = { "sbin": ["/usr/sbin", "/sbin"], "libexec": ["/usr/libexec"], "gir": ["/usr/share/gir-1.0"], "typelib": [ "/usr/lib64/girepository-1.0", "/usr/lib/girepository-1.0", "/lib64/girepository-1.0", "/lib/girepository-1.0", ], "gio": [ "/usr/lib64/gio/modules", "/usr/lib/gio/modules", ], "res": ["/usr/share"], "cfg": ["/etc"], "man": ["/usr/share/man"], "info": ["/usr/share/info"], "locale": ["/usr/share/locale"], "app": ["/usr/share/applications"], "opt": ["/opt"], "rootpath": [""], "tag": ["/usr/share/usm-tags"], } LIB_ROOTS = ["/usr/lib64", "/lib64", "/usr/lib", "/lib"] PC_ROOTS = ["/usr/lib64/pkgconfig", "/usr/share/pkgconfig", "/usr/lib/pkgconfig"] TRANSLATED_PREFIXES = set(FILE_ROOTS) | { "bin", "lib", "libres", "pc", "vapi", "inc"} USR_MERGE_ALIASES = [ ("/lib64/", "/usr/lib64/"), ("/lib/", "/usr/lib/"), ("/bin/", "/usr/bin/"), ("/sbin/", "/usr/sbin/"), ] EXCEPTIONS = { "bin:valac": "dev-lang/vala", "bin:vapigen": "dev-lang/vala", "bin:meson": "dev-build/meson", "bin:ninja": "dev-build/ninja", "bin:pkg-config": "dev-util/pkgconf", "bin:pkgconf": "dev-util/pkgconf", "bin:g-ir-scanner": "dev-libs/gobject-introspection", "bin:g-ir-compiler": "dev-libs/gobject-introspection", "bin:gcc": "sys-devel/gcc", "bin:g++": "sys-devel/gcc", "bin:cc": "sys-devel/gcc", "bin:c++": "sys-devel/gcc", "bin:ld": "sys-devel/binutils", "bin:make": "dev-build/make", "bin:python3": "dev-lang/python", "bin:bash": "app-shells/bash", "bin:git": "dev-vcs/git", "bin:curl": "net-misc/curl", "bin:tar": "app-arch/tar", "bin:xz": "app-arch/xz-utils", "bin:gzip": "app-arch/gzip", "bin:sed": "sys-apps/sed", "bin:awk": "sys-apps/gawk", "bin:grep": "sys-apps/grep", "bin:ldconfig": "sys-libs/glibc", "pc:glib-2.0.pc": "dev-libs/glib", "pc:gobject-2.0.pc": "dev-libs/glib", "pc:gio-2.0.pc": "dev-libs/glib", "pc:gio-unix-2.0.pc": "dev-libs/glib", "pc:gmodule-2.0.pc": "dev-libs/glib", "pc:gmodule-no-export-2.0.pc": "dev-libs/glib", "pc:gthread-2.0.pc": "dev-libs/glib", "pc:json-glib-1.0.pc": "dev-libs/json-glib", "pc:gee-0.8.pc": "dev-libs/libgee", "pc:libsodium.pc": "dev-libs/libsodium", "pc:libarchive.pc": "app-arch/libarchive", "pc:sqlite3.pc": "dev-db/sqlite", "pc:libxml-2.0.pc": "dev-libs/libxml2", "pc:libsoup-3.0.pc": "net-libs/libsoup", "pc:libmicrohttpd.pc": "net-libs/libmicrohttpd", "pc:gobject-introspection-1.0.pc": "dev-libs/gobject-introspection", "pc:zlib.pc": "sys-libs/zlib", "pc:libzstd.pc": "app-arch/zstd", "pc:libbrotlienc.pc": "app-arch/brotli", "pc:libbrotlidec.pc": "app-arch/brotli", "pc:libbrotlicommon.pc": "app-arch/brotli", "pc:libffi.pc": "dev-libs/libffi", "pc:pcre2.pc": "dev-libs/libpcre2", "vapi:glib-2.0.vapi": "dev-lang/vala", "vapi:gobject-2.0.vapi": "dev-lang/vala", "vapi:gio-2.0.vapi": "dev-lang/vala", "vapi:gio-unix-2.0.vapi": "dev-lang/vala", "vapi:posix.vapi": "dev-lang/vala", "vapi:json-glib-1.0.vapi": "dev-lang/vala", "vapi:libxml-2.0.vapi": "dev-lang/vala", "vapi:sqlite3.vapi": "dev-lang/vala", "vapi:gee-0.8.vapi": "dev-libs/libgee", "gir:GLib-2.0.gir": "dev-libs/gobject-introspection", "gir:GObject-2.0.gir": "dev-libs/gobject-introspection", "gir:Gio-2.0.gir": "dev-libs/gobject-introspection", "gir:GioUnix-2.0.gir": "dev-libs/gobject-introspection", "typelib:GLib-2.0.typelib": "dev-libs/gobject-introspection", "typelib:GObject-2.0.typelib": "dev-libs/gobject-introspection", "typelib:Gio-2.0.typelib": "dev-libs/gobject-introspection", "typelib:GioUnix-2.0.typelib": "dev-libs/gobject-introspection", "lib:libglib-2.0.so.0": "dev-libs/glib", "lib:libgobject-2.0.so.0": "dev-libs/glib", "lib:libgio-2.0.so.0": "dev-libs/glib", "lib:libgmodule-2.0.so.0": "dev-libs/glib", "lib:libgthread-2.0.so.0": "dev-libs/glib", "lib:libjson-glib-1.0.so.0": "dev-libs/json-glib", "lib:libgee-0.8.so.2": "dev-libs/libgee", "lib:libsodium.so.26": "dev-libs/libsodium", "lib:libarchive.so.13": "app-arch/libarchive", "lib:libsqlite3.so.0": "dev-db/sqlite", "lib:libxml2.so.2": "dev-libs/libxml2", "lib:libsoup-3.0.so.0": "net-libs/libsoup", "lib:libmicrohttpd.so.12": "net-libs/libmicrohttpd", "lib:libz.so.1": "sys-libs/zlib", "lib:libzstd.so.1": "app-arch/zstd", "lib:libbrotlienc.so.1": "app-arch/brotli", "lib:libbrotlidec.so.1": "app-arch/brotli", "lib:libbrotlicommon.so.1": "app-arch/brotli", "lib:libffi.so.8": "dev-libs/libffi", "lib:libpcre2-8.so.0": "dev-libs/libpcre2", "gio:libgiognutls.so": "net-libs/glib-networking", "lib:libc.so.6": "sys-libs/glibc", "lib:libm.so.6": "sys-libs/glibc", "lib:libblkid.so.1": "sys-apps/util-linux", "lib:libmount.so.1": "sys-apps/util-linux", "lib:libuuid.so.1": "sys-apps/util-linux", "inc:glib-2.0": "dev-libs/glib", "inc:json-glib-1.0": "dev-libs/json-glib", "inc:gee-0.8": "dev-libs/libgee", "inc:libsodium.h": "dev-libs/libsodium", "inc:archive.h": "app-arch/libarchive", "inc:sqlite3.h": "dev-db/sqlite", } class HelperError(Exception): """Fatal helper failure; message is emitted as a contract error event.""" def __init__(self, message, exit_code=EXIT_FAILURE): super(HelperError, self).__init__(message) self.exit_code = exit_code def emit_event(event): """Write one contract JSONL event to STDOUT and flush it.""" sys.stdout.write(json.dumps(event) + "\n") sys.stdout.flush() def fail(message, exit_code=EXIT_FAILURE): """Emit the terminal error event and exit non-zero.""" emit_event({"type": "error", "message": message}) sys.exit(exit_code) def fail_plain(message, exit_code=EXIT_FAILURE): """Report a query failure on STDERR only, keeping STDOUT contract-clean.""" sys.stderr.write("usm-spm-emerge: %s\n" % message) sys.exit(exit_code) def category_package(cpv): """Strip version and ::repository suffix from a cpv -> category/package.""" atom = cpv.split("::", 1)[0] try: import portage key = portage.cpv_getkey(atom) if key: return key except Exception: pass category, sep, package_version = atom.partition("/") if not sep: return atom return re.sub(r"-\d.*$", "", package_version) or package_version def parse_contents_line(line): """Path recorded on one CONTENTS line, or None for junk lines.""" fields = line.split() if len(fields) >= 2 and fields[0] in ("obj", "dir", "sym", "dev", "fif"): return fields[1] return None def build_vdb_index(): """Map every installed file path -> cpv via the portage vdb. Prefers the portage API (aux_get CONTENTS, present on any portage system); falls back to a raw /var/db/pkg///CONTENTS scan. """ index = {} try: import portage root = portage.settings.get("ROOT", "/") vdb = portage.db[root]["vartree"].dbapi for cpv in vdb.cpv_all(): raw = vdb.aux_get(cpv, ["CONTENTS"])[0] or "" for line in raw.splitlines(): path = parse_contents_line(line) if path is not None: index.setdefault(path, cpv) if index: return index except Exception as e: sys.stderr.write("usm-spm-emerge: portage API vdb read failed (%s), " "falling back to /var/db/pkg scan\n" % e) for contents_path in glob.glob("/var/db/pkg/*/*/CONTENTS"): category = os.path.basename(os.path.dirname(os.path.dirname(contents_path))) package_version = os.path.basename(os.path.dirname(contents_path)) cpv = category + "/" + package_version try: with open(contents_path, "r", errors="replace") as handle: for line in handle: path = parse_contents_line(line) if path is not None: index.setdefault(path, cpv) except OSError: continue return index def path_aliases(path): """path plus its /usr-merge variant(s) (/lib64/x -> /usr/lib64/x...).""" aliases = [path] for merged, unmerged in USR_MERGE_ALIASES: if path.startswith(merged): variant = unmerged + path[len(merged):] elif path.startswith(unmerged): variant = merged + path[len(unmerged):] else: continue if variant not in aliases: aliases.append(variant) return aliases def bin_candidates(resource): """PATH directories in order (plus /usr/bin) as candidate directories.""" directories = [d for d in os.environ.get("PATH", "").split(os.pathsep) if d] if "/usr/bin" not in directories: directories.append("/usr/bin") return [os.path.join(d, resource) for d in directories] def translate_paths(prefix, resource): """Candidate filesystem paths for a file-backed ref prefix.""" if prefix == "bin": return bin_candidates(resource) if prefix == "tag": return ["/usr/share/usm-tags/" + resource.replace(".", "/") + ".tag"] if prefix in FILE_ROOTS: return ["/".join(FILE_ROOTS[prefix] + [resource])] if prefix in ("lib", "libres"): return [os.path.join(d, resource) for d in LIB_ROOTS] if prefix == "pc": return [os.path.join(d, resource) for d in PC_ROOTS] return None def resolve_installed(ref, index): """cpv of the installed package owning the ref's file, or None. Candidate paths (PATH order for bin:, gentoo root table otherwise) are matched exactly against the vdb index, with /usr-merge aliases tried on both sides; files present on disk but owned by no package do not count as provided by the system package manager. """ prefix, sep, resource = ref.partition(":") if not sep or not resource: return None resource = resource.strip("/") if prefix == "vapi": pattern = re.compile( r"^/usr/share/vala[^/]*/vapi/" + re.escape(resource) + "$") for path, cpv in index.items(): if pattern.match(path): return cpv return None if prefix == "inc": directory = "/usr/include/" + resource.strip("/") for path, cpv in index.items(): if path == directory or path.startswith(directory + "/"): return cpv return None candidates = translate_paths(prefix, resource) if candidates is None: return None for path in candidates: for alias in path_aliases(path): if alias in index: return index[alias] return None def emerge_pretend(atoms): """Run emerge --pretend; returns (returncode, stdout, stderr).""" command = [EMERGE, "--pretend", "--quiet=y", "--ask=n"] + list(atoms) try: proc = subprocess.run( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, errors="replace") except OSError as e: return EXIT_FAILURE, "", str(e) return proc.returncode, proc.stdout, proc.stderr MERGE_LINE = re.compile(r"^\[(ebuild|binary)\s+([A-Za-z*]+)\s*\]\s+(\S+)") def parse_merge_list(output): """(total, already_installed, atoms) from pretend output merge lines.""" total = 0 installed = 0 atoms = [] for line in output.splitlines(): match = MERGE_LINE.match(line) if not match: continue total += 1 if "N" not in match.group(2): installed += 1 atoms.append(match.group(3)) return total, installed, atoms def pretend_counts(atom, cache): """(dependency-count, installed-dependency-count) for a solo emerge. None means the atom cannot be resolved against the tree at all (no ebuilds, masked, conflicting): exception-table candidates then fall back to not-found; installed candidates degrade to (1, 1). """ if atom in cache: return cache[atom] returncode, stdout, stderr = emerge_pretend([atom]) if returncode != 0: cache[atom] = None return None total, installed, _ = parse_merge_list(stdout) counts = (total, installed) if total else (1, 1) cache[atom] = counts return counts def cmd_query(args): try: return run_query(args) except SystemExit: raise except Exception as e: fail_plain("query failed: %s" % e, EXIT_RESOLVE) def run_query(args): index = build_vdb_index() refs = list(dict.fromkeys(args.refs)) candidates = {} resolved_ref = {} for ref in refs: cpv = resolve_installed(ref, index) if cpv is not None: name = category_package(cpv) else: name = EXCEPTIONS.get(ref) if name is None: if ref.partition(":")[0] not in TRANSLATED_PREFIXES: sys.stderr.write( "usm-spm-emerge: resource type of \"%s\" has no " "system-package-manager translation\n" % ref) continue resolved_ref[ref] = name entry = candidates.setdefault(name, {"resources": [], "installed": False}) entry["resources"].append(ref) if cpv is not None: entry["installed"] = True counts_cache = {} package_counts = {} for name, entry in candidates.items(): counts = pretend_counts(name, counts_cache) if counts is None and entry["installed"]: sys.stderr.write( "usm-spm-emerge: %s is installed but emerge cannot resolve " "its atom; estimating dependency counts as installed-only\n" % name) counts = (1, 1) package_counts[name] = counts not_found = [] for ref in refs: name = resolved_ref.get(ref) if name is None or package_counts[name] is None: not_found.append(ref) packages = [] for name in sorted(candidates): if package_counts[name] is None: continue dependency_count, installed_dependency_count = package_counts[name] packages.append({ "name": name, "resources": candidates[name]["resources"], "dependency-count": dependency_count, "installed-dependency-count": installed_dependency_count, }) sys.stdout.write(json.dumps({ "not-found": not_found, "packages": packages, }) + "\n") sys.stdout.flush() return EXIT_OK EMERGING_LINE = re.compile(r">>> Emerging \((\d+) of (\d+)\) (\S+)") INSTALLING_LINE = re.compile(r">>> Installing \((\d+) of (\d+)\) (\S+)") def classify_failure(output): """Map an emerge failure transcript onto a contract exit code.""" lowered = output.lower() if ("all ebuilds that could satisfy" in lowered or "dependency conflict" in lowered or "conflicting requests" in lowered or "blocked" in lowered or "no ebuilds to satisfy" in lowered): return EXIT_RESOLVE if ("couldn't download" in lowered or "fetch instructions" in lowered or "failed to fetch" in lowered): return EXIT_DOWNLOAD return EXIT_TRANSACTION NOISE_LINE = re.compile(r"^\s*\*\s+(IMPORTANT:|Use eselect news)") def failure_message(lines, limit=15): """Last few meaningful output lines as a terminal error message.""" meaningful = [line.strip() for line in lines if line.strip() and not NOISE_LINE.match(line)] tail = meaningful[-limit:] message = " | ".join(tail) return message[:800] if message else "emerge exited non-zero" # Gentoo installs some tool binaries under versioned names only (vala's # ebuild ships valac-0.56 without an unversioned valac), while USM # manifests reference the unversioned bin:. After a transaction that # installed one of these atoms, link each unversioned name to the newest # versioned binary so bin: refs resolve on the filesystem. UNVERSIONED_TOOL_LINKS = { "dev-lang/vala": ["valac", "vala-gen-introspect", "vapigen"], } def link_unversioned_tools(atoms): """Ensure unversioned tool symlinks exist for versioned ebuild tools.""" wanted = set() for atom in atoms: for prefix, tools in UNVERSIONED_TOOL_LINKS.items(): if atom == prefix or atom.startswith(prefix + ":"): wanted.update(tools) for tool in sorted(wanted): target = os.path.join("/usr/bin", tool) if os.path.exists(target): continue candidates = sorted(glob.glob("/usr/bin/%s-[0-9]*" % tool)) if candidates: try: os.symlink(os.path.basename(candidates[-1]), target) except OSError: pass def cmd_install(args): returncode, stdout, stderr = emerge_pretend(args.names) if returncode != 0: fail("dependency resolution failed: %s" % failure_message((stdout + "\n" + stderr).splitlines()), EXIT_RESOLVE) total, _, atoms = parse_merge_list(stdout) emit_event({"type": "begin", "total": total}) if args.test: emit_event({"type": "complete", "status": "ok", "installed": 0}) return EXIT_OK command = [EMERGE, "--ask=n", "--quiet=y"] + list(args.names) try: proc = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, errors="replace", bufsize=1) except OSError as e: fail("could not run emerge: %s" % e, EXIT_FAILURE) transcript = [] def drain_stderr(): for line in proc.stderr: transcript.append(line) sys.stderr.write(line) sys.stderr.flush() reader = threading.Thread(target=drain_stderr) reader.daemon = True reader.start() completed = [] for line in proc.stdout: transcript.append(line) sys.stderr.write(line) sys.stderr.flush() emerging = EMERGING_LINE.match(line) if emerging: emit_event({ "type": "package", "name": category_package(emerging.group(3)), "current": int(emerging.group(1)), "total": int(emerging.group(2)), "progress": 0.0, }) continue installing = INSTALLING_LINE.match(line) if installing: name = category_package(installing.group(3)) completed.append(name) emit_event({"type": "package-complete", "name": name}) returncode = proc.wait() reader.join(timeout=5) if returncode != 0: fail("emerge failed (%s): %s" % (returncode, failure_message(transcript)), classify_failure("".join(transcript))) link_unversioned_tools(args.names) emit_event({ "type": "complete", "status": "ok", "installed": len(completed), }) return EXIT_OK def main(): parser = argparse.ArgumentParser( prog="usm-spm-emerge", description="USM system-package-manager helper for Portage (Gentoo)") subparsers = parser.add_subparsers(dest="command", required=True) query_parser = subparsers.add_parser( "query", help="resolve USM resource refs to system packages") query_parser.add_argument( "refs", nargs="+", metavar="USM-REF", help="resource ref, e.g. bin:valac or pc:glib-2.0.pc") query_parser.set_defaults(handler=cmd_query) install_parser = subparsers.add_parser( "install", help="install system packages, streaming progress events") install_parser.add_argument( "--test", action="store_true", help="resolve only (emerge --pretend); no packages are built") install_parser.add_argument( "names", nargs="+", metavar="ATOM", help="native gentoo package atom, e.g. dev-build/meson") install_parser.set_defaults(handler=cmd_install) args = parser.parse_args() try: return args.handler(args) except HelperError as e: fail(str(e), e.exit_code) except SystemExit: raise except Exception as e: message = "unexpected failure: %s" % e if args.command == "install": fail(message, EXIT_FAILURE) fail_plain(message, EXIT_FAILURE) if __name__ == "__main__": sys.exit(main())