usm-spm-emerge 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701
  1. #!/usr/bin/env python3
  2. """USM system-package-manager helper for Portage (Gentoo) systems.
  3. Implements the USM SPM contract (see usm/README.md, "System package manager
  4. integration") for Gentoo systems:
  5. usm-spm-emerge query <usm-ref>... -> contract JSON on STDOUT
  6. usm-spm-emerge plan <atom>... -> contract JSON on STDOUT
  7. usm-spm-emerge install [--test] <atom>... -> contract JSONL events on STDOUT
  8. API choice: a Gentoo stage3 ships Portage and python3, so this helper reads
  9. the installed-package database through the portage Python API (with a raw
  10. /var/db/pkg CONTENTS scan as fallback) and shells out to `emerge` for
  11. dependency resolution (pretend) and installation. There is no binary
  12. metadata to bind against beyond that: ebuild repositories carry no file
  13. lists, so unlike the dnf/apt/apk helpers this one cannot search "which
  14. package contains file X" for packages that are not installed yet.
  15. File-backed refs are therefore resolved in two documented stages:
  16. 1. INSTALLED files (exact): the ref is translated to candidate filesystem
  17. paths (gentoo layout: /usr/bin, /usr/sbin, /usr/lib64, /usr/share, ...)
  18. and matched against the portage vdb (/var/db/pkg/*/CONTENTS). A ref
  19. resolves to the installed package that owns the file.
  20. 2. NOT-installed files (exception table only): Gentoo has no offline
  21. file-provides (PFL, the portage file list, is an online service), so a
  22. small curated table maps the USM stack's build/platform refs to Gentoo
  23. package atoms (valac -> dev-lang/vala, glib-2.0.pc -> dev-libs/glib,
  24. ...). Refs outside the table land in not-found; anything else would be
  25. guesswork.
  26. dependency-count / installed-dependency-count are honest numbers parsed
  27. from `emerge --pretend --quiet=y --ask=n <atom>`: dependency-count is the
  28. number of packages in the merge list including the package itself,
  29. installed-dependency-count is the subset of the merge list already
  30. installed (action letters without N, e.g. [ebuild R]/[ebuild U] reinstalls
  31. and upgrades; a fresh stage3 typically shows an all-N list). If the pretend
  32. run fails (atom missing from the tree, masked, unresolvable) an
  33. exception-table candidate falls back to not-found.
  34. Install maps `emerge --ask=n --quiet=y <atoms>` onto the JSONL contract,
  35. linking the unversioned tool names Gentoo ships versioned (UNVERSIONED_TOOL_LINKS):
  36. `begin`/`total` come from a pretend run first, a `package` event is emitted
  37. per ">>> Emerging (n of N)" status line, `package-complete` per
  38. ">>> Installing (n of N)" line (the moment a merge lands in the vdb),
  39. `complete` carries the number of completed merges, and any failure emits a
  40. terminal `error` event before a non-zero exit (resolution failures map to
  41. exit 3, download failures to 4, build/merge failures to 5).
  42. Documented Gentoo limitations:
  43. * No offline file-provides for not-yet-installed packages (see stage 2
  44. above); installed-file queries are exact.
  45. * USE flags are never touched: emerge runs with the system's configured
  46. defaults. The stack's platform libraries (glib, json-glib, libgee, ...)
  47. enable their introspection USE flag by default, but a system that
  48. disabled it (package.use/make.conf) will lack the matching .gir/
  49. .typelib/.vapi artifacts after install; re-enable the flag and re-emerge
  50. if a build needs them.
  51. * Packages are compiled from source; even small installs take minutes.
  52. Progress granularity is per package; emerge exposes no useful
  53. intra-package progress, so `package` events carry progress 0.0 and the
  54. completion signal is the ">>> Installing" line.
  55. * The query subcommand never modifies system state (at most Portage
  56. regenerates its dependency cache); install must run as root and never
  57. prompts (--ask=n answers Portage's own prompt for it).
  58. """
  59. import argparse
  60. import glob
  61. import json
  62. import os
  63. import re
  64. import subprocess
  65. import sys
  66. import threading
  67. EMERGE = "/usr/bin/emerge"
  68. EXIT_OK = 0
  69. EXIT_FAILURE = 1
  70. EXIT_RESOLVE = 3
  71. EXIT_DOWNLOAD = 4
  72. EXIT_TRANSACTION = 5
  73. FILE_ROOTS = {
  74. "sbin": ["/usr/sbin", "/sbin"],
  75. "libexec": ["/usr/libexec"],
  76. "gir": ["/usr/share/gir-1.0"],
  77. "typelib": [
  78. "/usr/lib64/girepository-1.0",
  79. "/usr/lib/girepository-1.0",
  80. "/lib64/girepository-1.0",
  81. "/lib/girepository-1.0",
  82. ],
  83. "gio": [
  84. "/usr/lib64/gio/modules",
  85. "/usr/lib/gio/modules",
  86. ],
  87. "res": ["/usr/share"],
  88. "cfg": ["/etc"],
  89. "man": ["/usr/share/man"],
  90. "info": ["/usr/share/info"],
  91. "locale": ["/usr/share/locale"],
  92. "app": ["/usr/share/applications"],
  93. "opt": ["/opt"],
  94. "rootpath": [""],
  95. "tag": ["/usr/share/usm-tags"],
  96. }
  97. LIB_ROOTS = ["/usr/lib64", "/lib64", "/usr/lib", "/lib"]
  98. PC_ROOTS = ["/usr/lib64/pkgconfig", "/usr/share/pkgconfig", "/usr/lib/pkgconfig"]
  99. TRANSLATED_PREFIXES = set(FILE_ROOTS) | {
  100. "bin", "lib", "libres", "pc", "vapi", "inc"}
  101. USR_MERGE_ALIASES = [
  102. ("/lib64/", "/usr/lib64/"),
  103. ("/lib/", "/usr/lib/"),
  104. ("/bin/", "/usr/bin/"),
  105. ("/sbin/", "/usr/sbin/"),
  106. ]
  107. EXCEPTIONS = {
  108. "bin:valac": "dev-lang/vala",
  109. "bin:vapigen": "dev-lang/vala",
  110. "bin:meson": "dev-build/meson",
  111. "bin:ninja": "dev-build/ninja",
  112. "bin:pkg-config": "dev-util/pkgconf",
  113. "bin:pkgconf": "dev-util/pkgconf",
  114. "bin:g-ir-scanner": "dev-libs/gobject-introspection",
  115. "bin:g-ir-compiler": "dev-libs/gobject-introspection",
  116. "bin:gcc": "sys-devel/gcc",
  117. "bin:g++": "sys-devel/gcc",
  118. "bin:cc": "sys-devel/gcc",
  119. "bin:c++": "sys-devel/gcc",
  120. "bin:ld": "sys-devel/binutils",
  121. "bin:make": "dev-build/make",
  122. "bin:python3": "dev-lang/python",
  123. "bin:bash": "app-shells/bash",
  124. "bin:git": "dev-vcs/git",
  125. "bin:curl": "net-misc/curl",
  126. "bin:tar": "app-arch/tar",
  127. "bin:xz": "app-arch/xz-utils",
  128. "bin:gzip": "app-arch/gzip",
  129. "bin:sed": "sys-apps/sed",
  130. "bin:awk": "sys-apps/gawk",
  131. "bin:grep": "sys-apps/grep",
  132. "bin:ldconfig": "sys-libs/glibc",
  133. "pc:glib-2.0.pc": "dev-libs/glib",
  134. "pc:gobject-2.0.pc": "dev-libs/glib",
  135. "pc:gio-2.0.pc": "dev-libs/glib",
  136. "pc:gio-unix-2.0.pc": "dev-libs/glib",
  137. "pc:gmodule-2.0.pc": "dev-libs/glib",
  138. "pc:gmodule-no-export-2.0.pc": "dev-libs/glib",
  139. "pc:gthread-2.0.pc": "dev-libs/glib",
  140. "pc:json-glib-1.0.pc": "dev-libs/json-glib",
  141. "pc:gee-0.8.pc": "dev-libs/libgee",
  142. "pc:libsodium.pc": "dev-libs/libsodium",
  143. "pc:libarchive.pc": "app-arch/libarchive",
  144. "pc:sqlite3.pc": "dev-db/sqlite",
  145. "pc:libxml-2.0.pc": "dev-libs/libxml2",
  146. "pc:libsoup-3.0.pc": "net-libs/libsoup",
  147. "pc:libmicrohttpd.pc": "net-libs/libmicrohttpd",
  148. "pc:gobject-introspection-1.0.pc": "dev-libs/gobject-introspection",
  149. "pc:zlib.pc": "sys-libs/zlib",
  150. "pc:libzstd.pc": "app-arch/zstd",
  151. "pc:libbrotlienc.pc": "app-arch/brotli",
  152. "pc:libbrotlidec.pc": "app-arch/brotli",
  153. "pc:libbrotlicommon.pc": "app-arch/brotli",
  154. "pc:libffi.pc": "dev-libs/libffi",
  155. "pc:pcre2.pc": "dev-libs/libpcre2",
  156. "vapi:glib-2.0.vapi": "dev-lang/vala",
  157. "vapi:gobject-2.0.vapi": "dev-lang/vala",
  158. "vapi:gio-2.0.vapi": "dev-lang/vala",
  159. "vapi:gio-unix-2.0.vapi": "dev-lang/vala",
  160. "vapi:posix.vapi": "dev-lang/vala",
  161. "vapi:json-glib-1.0.vapi": "dev-lang/vala",
  162. "vapi:libxml-2.0.vapi": "dev-lang/vala",
  163. "vapi:sqlite3.vapi": "dev-lang/vala",
  164. "vapi:gee-0.8.vapi": "dev-libs/libgee",
  165. "gir:GLib-2.0.gir": "dev-libs/gobject-introspection",
  166. "gir:GObject-2.0.gir": "dev-libs/gobject-introspection",
  167. "gir:Gio-2.0.gir": "dev-libs/gobject-introspection",
  168. "gir:GioUnix-2.0.gir": "dev-libs/gobject-introspection",
  169. "typelib:GLib-2.0.typelib": "dev-libs/gobject-introspection",
  170. "typelib:GObject-2.0.typelib": "dev-libs/gobject-introspection",
  171. "typelib:Gio-2.0.typelib": "dev-libs/gobject-introspection",
  172. "typelib:GioUnix-2.0.typelib": "dev-libs/gobject-introspection",
  173. "lib:libglib-2.0.so.0": "dev-libs/glib",
  174. "lib:libgobject-2.0.so.0": "dev-libs/glib",
  175. "lib:libgio-2.0.so.0": "dev-libs/glib",
  176. "lib:libgmodule-2.0.so.0": "dev-libs/glib",
  177. "lib:libgthread-2.0.so.0": "dev-libs/glib",
  178. "lib:libjson-glib-1.0.so.0": "dev-libs/json-glib",
  179. "lib:libgee-0.8.so.2": "dev-libs/libgee",
  180. "lib:libsodium.so.26": "dev-libs/libsodium",
  181. "lib:libarchive.so.13": "app-arch/libarchive",
  182. "lib:libsqlite3.so.0": "dev-db/sqlite",
  183. "lib:libxml2.so.2": "dev-libs/libxml2",
  184. "lib:libsoup-3.0.so.0": "net-libs/libsoup",
  185. "lib:libmicrohttpd.so.12": "net-libs/libmicrohttpd",
  186. "lib:libz.so.1": "sys-libs/zlib",
  187. "lib:libzstd.so.1": "app-arch/zstd",
  188. "lib:libbrotlienc.so.1": "app-arch/brotli",
  189. "lib:libbrotlidec.so.1": "app-arch/brotli",
  190. "lib:libbrotlicommon.so.1": "app-arch/brotli",
  191. "lib:libffi.so.8": "dev-libs/libffi",
  192. "lib:libpcre2-8.so.0": "dev-libs/libpcre2",
  193. "gio:libgiognutls.so": "net-libs/glib-networking",
  194. "lib:libc.so.6": "sys-libs/glibc",
  195. "lib:libm.so.6": "sys-libs/glibc",
  196. "lib:libblkid.so.1": "sys-apps/util-linux",
  197. "lib:libmount.so.1": "sys-apps/util-linux",
  198. "lib:libuuid.so.1": "sys-apps/util-linux",
  199. "inc:glib-2.0": "dev-libs/glib",
  200. "inc:json-glib-1.0": "dev-libs/json-glib",
  201. "inc:gee-0.8": "dev-libs/libgee",
  202. "inc:libsodium.h": "dev-libs/libsodium",
  203. "inc:archive.h": "app-arch/libarchive",
  204. "inc:sqlite3.h": "dev-db/sqlite",
  205. }
  206. class HelperError(Exception):
  207. """Fatal helper failure; message is emitted as a contract error event."""
  208. def __init__(self, message, exit_code=EXIT_FAILURE):
  209. super(HelperError, self).__init__(message)
  210. self.exit_code = exit_code
  211. def emit_event(event):
  212. """Write one contract JSONL event to STDOUT and flush it."""
  213. sys.stdout.write(json.dumps(event) + "\n")
  214. sys.stdout.flush()
  215. def fail(message, exit_code=EXIT_FAILURE):
  216. """Emit the terminal error event and exit non-zero."""
  217. emit_event({"type": "error", "message": message})
  218. sys.exit(exit_code)
  219. def fail_plain(message, exit_code=EXIT_FAILURE):
  220. """Report a query failure on STDERR only, keeping STDOUT contract-clean."""
  221. sys.stderr.write("usm-spm-emerge: %s\n" % message)
  222. sys.exit(exit_code)
  223. def category_package(cpv):
  224. """Strip version and ::repository suffix from a cpv -> category/package."""
  225. atom = cpv.split("::", 1)[0]
  226. try:
  227. import portage
  228. key = portage.cpv_getkey(atom)
  229. if key:
  230. return key
  231. except Exception:
  232. pass
  233. category, sep, package_version = atom.partition("/")
  234. if not sep:
  235. return atom
  236. return re.sub(r"-\d.*$", "", package_version) or package_version
  237. def parse_contents_line(line):
  238. """Path recorded on one CONTENTS line, or None for junk lines."""
  239. fields = line.split()
  240. if len(fields) >= 2 and fields[0] in ("obj", "dir", "sym", "dev", "fif"):
  241. return fields[1]
  242. return None
  243. def build_vdb_index():
  244. """Map every installed file path -> cpv via the portage vdb.
  245. Prefers the portage API (aux_get CONTENTS, present on any portage
  246. system); falls back to a raw /var/db/pkg/<cat>/<pkg>/CONTENTS scan.
  247. """
  248. index = {}
  249. try:
  250. import portage
  251. root = portage.settings.get("ROOT", "/")
  252. vdb = portage.db[root]["vartree"].dbapi
  253. for cpv in vdb.cpv_all():
  254. raw = vdb.aux_get(cpv, ["CONTENTS"])[0] or ""
  255. for line in raw.splitlines():
  256. path = parse_contents_line(line)
  257. if path is not None:
  258. index.setdefault(path, cpv)
  259. if index:
  260. return index
  261. except Exception as e:
  262. sys.stderr.write("usm-spm-emerge: portage API vdb read failed (%s), "
  263. "falling back to /var/db/pkg scan\n" % e)
  264. for contents_path in glob.glob("/var/db/pkg/*/*/CONTENTS"):
  265. category = os.path.basename(os.path.dirname(os.path.dirname(contents_path)))
  266. package_version = os.path.basename(os.path.dirname(contents_path))
  267. cpv = category + "/" + package_version
  268. try:
  269. with open(contents_path, "r", errors="replace") as handle:
  270. for line in handle:
  271. path = parse_contents_line(line)
  272. if path is not None:
  273. index.setdefault(path, cpv)
  274. except OSError:
  275. continue
  276. return index
  277. def path_aliases(path):
  278. """path plus its /usr-merge variant(s) (/lib64/x -> /usr/lib64/x...)."""
  279. aliases = [path]
  280. for merged, unmerged in USR_MERGE_ALIASES:
  281. if path.startswith(merged):
  282. variant = unmerged + path[len(merged):]
  283. elif path.startswith(unmerged):
  284. variant = merged + path[len(unmerged):]
  285. else:
  286. continue
  287. if variant not in aliases:
  288. aliases.append(variant)
  289. return aliases
  290. def bin_candidates(resource):
  291. """PATH directories in order (plus /usr/bin) as candidate directories."""
  292. directories = [d for d in os.environ.get("PATH", "").split(os.pathsep) if d]
  293. if "/usr/bin" not in directories:
  294. directories.append("/usr/bin")
  295. return [os.path.join(d, resource) for d in directories]
  296. def translate_paths(prefix, resource):
  297. """Candidate filesystem paths for a file-backed ref prefix."""
  298. if prefix == "bin":
  299. return bin_candidates(resource)
  300. if prefix == "tag":
  301. return ["/usr/share/usm-tags/" + resource.replace(".", "/") + ".tag"]
  302. if prefix in FILE_ROOTS:
  303. return ["/".join(FILE_ROOTS[prefix] + [resource])]
  304. if prefix in ("lib", "libres"):
  305. return [os.path.join(d, resource) for d in LIB_ROOTS]
  306. if prefix == "pc":
  307. return [os.path.join(d, resource) for d in PC_ROOTS]
  308. return None
  309. def resolve_installed(ref, index):
  310. """cpv of the installed package owning the ref's file, or None.
  311. Candidate paths (PATH order for bin:, gentoo root table otherwise) are
  312. matched exactly against the vdb index, with /usr-merge aliases tried on
  313. both sides; files present on disk but owned by no package do not count
  314. as provided by the system package manager.
  315. """
  316. prefix, sep, resource = ref.partition(":")
  317. if not sep or not resource:
  318. return None
  319. resource = resource.strip("/")
  320. if prefix == "vapi":
  321. pattern = re.compile(
  322. r"^/usr/share/vala[^/]*/vapi/" + re.escape(resource) + "$")
  323. for path, cpv in index.items():
  324. if pattern.match(path):
  325. return cpv
  326. return None
  327. if prefix == "inc":
  328. directory = "/usr/include/" + resource.strip("/")
  329. for path, cpv in index.items():
  330. if path == directory or path.startswith(directory + "/"):
  331. return cpv
  332. return None
  333. candidates = translate_paths(prefix, resource)
  334. if candidates is None:
  335. return None
  336. for path in candidates:
  337. for alias in path_aliases(path):
  338. if alias in index:
  339. return index[alias]
  340. return None
  341. def emerge_pretend(atoms):
  342. """Run emerge --pretend; returns (returncode, stdout, stderr)."""
  343. command = [EMERGE, "--pretend", "--quiet=y", "--ask=n"] + list(atoms)
  344. try:
  345. proc = subprocess.run(
  346. command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  347. text=True, errors="replace")
  348. except OSError as e:
  349. return EXIT_FAILURE, "", str(e)
  350. return proc.returncode, proc.stdout, proc.stderr
  351. MERGE_LINE = re.compile(r"^\[(ebuild|binary)\s+([A-Za-z*]+)\s*\]\s+(\S+)")
  352. def parse_merge_list(output):
  353. """(total, already_installed, atoms) from pretend output merge lines."""
  354. total = 0
  355. installed = 0
  356. atoms = []
  357. for line in output.splitlines():
  358. match = MERGE_LINE.match(line)
  359. if not match:
  360. continue
  361. total += 1
  362. if "N" not in match.group(2):
  363. installed += 1
  364. atoms.append(match.group(3))
  365. return total, installed, atoms
  366. def pretend_counts(atom, cache):
  367. """(dependency-count, installed-dependency-count) for a solo emerge.
  368. None means the atom cannot be resolved against the tree at all (no
  369. ebuilds, masked, conflicting): exception-table candidates then fall
  370. back to not-found; installed candidates degrade to (1, 1).
  371. """
  372. if atom in cache:
  373. return cache[atom]
  374. returncode, stdout, stderr = emerge_pretend([atom])
  375. if returncode != 0:
  376. cache[atom] = None
  377. return None
  378. total, installed, _ = parse_merge_list(stdout)
  379. counts = (total, installed) if total else (1, 1)
  380. cache[atom] = counts
  381. return counts
  382. def cmd_query(args):
  383. try:
  384. return run_query(args)
  385. except SystemExit:
  386. raise
  387. except Exception as e:
  388. fail_plain("query failed: %s" % e, EXIT_RESOLVE)
  389. def run_query(args):
  390. index = build_vdb_index()
  391. refs = list(dict.fromkeys(args.refs))
  392. candidates = {}
  393. resolved_ref = {}
  394. for ref in refs:
  395. cpv = resolve_installed(ref, index)
  396. if cpv is not None:
  397. name = category_package(cpv)
  398. else:
  399. name = EXCEPTIONS.get(ref)
  400. if name is None:
  401. if ref.partition(":")[0] not in TRANSLATED_PREFIXES:
  402. sys.stderr.write(
  403. "usm-spm-emerge: resource type of \"%s\" has no "
  404. "system-package-manager translation\n" % ref)
  405. continue
  406. resolved_ref[ref] = name
  407. entry = candidates.setdefault(name, {"resources": [], "installed": False})
  408. entry["resources"].append(ref)
  409. if cpv is not None:
  410. entry["installed"] = True
  411. counts_cache = {}
  412. package_counts = {}
  413. for name, entry in candidates.items():
  414. counts = pretend_counts(name, counts_cache)
  415. if counts is None and entry["installed"]:
  416. sys.stderr.write(
  417. "usm-spm-emerge: %s is installed but emerge cannot resolve "
  418. "its atom; estimating dependency counts as installed-only\n"
  419. % name)
  420. counts = (1, 1)
  421. package_counts[name] = counts
  422. not_found = []
  423. for ref in refs:
  424. name = resolved_ref.get(ref)
  425. if name is None or package_counts[name] is None:
  426. not_found.append(ref)
  427. packages = []
  428. for name in sorted(candidates):
  429. if package_counts[name] is None:
  430. continue
  431. dependency_count, installed_dependency_count = package_counts[name]
  432. packages.append({
  433. "name": name,
  434. "resources": candidates[name]["resources"],
  435. "dependency-count": dependency_count,
  436. "installed-dependency-count": installed_dependency_count,
  437. })
  438. sys.stdout.write(json.dumps({
  439. "not-found": not_found,
  440. "packages": packages,
  441. }) + "\n")
  442. sys.stdout.flush()
  443. return EXIT_OK
  444. EMERGING_LINE = re.compile(r">>> Emerging \((\d+) of (\d+)\) (\S+)")
  445. INSTALLING_LINE = re.compile(r">>> Installing \((\d+) of (\d+)\) (\S+)")
  446. def classify_failure(output):
  447. """Map an emerge failure transcript onto a contract exit code."""
  448. lowered = output.lower()
  449. if ("all ebuilds that could satisfy" in lowered
  450. or "dependency conflict" in lowered
  451. or "conflicting requests" in lowered
  452. or "blocked" in lowered
  453. or "no ebuilds to satisfy" in lowered):
  454. return EXIT_RESOLVE
  455. if ("couldn't download" in lowered
  456. or "fetch instructions" in lowered
  457. or "failed to fetch" in lowered):
  458. return EXIT_DOWNLOAD
  459. return EXIT_TRANSACTION
  460. NOISE_LINE = re.compile(r"^\s*\*\s+(IMPORTANT:|Use eselect news)")
  461. def failure_message(lines, limit=15):
  462. """Last few meaningful output lines as a terminal error message."""
  463. meaningful = [line.strip() for line in lines
  464. if line.strip() and not NOISE_LINE.match(line)]
  465. tail = meaningful[-limit:]
  466. message = " | ".join(tail)
  467. return message[:800] if message else "emerge exited non-zero"
  468. # Gentoo installs some tool binaries under versioned names only (vala's
  469. # ebuild ships valac-0.56 without an unversioned valac), while USM
  470. # manifests reference the unversioned bin:. After a transaction that
  471. # installed one of these atoms, link each unversioned name to the newest
  472. # versioned binary so bin: refs resolve on the filesystem.
  473. UNVERSIONED_TOOL_LINKS = {
  474. "dev-lang/vala": ["valac", "vala-gen-introspect", "vapigen"],
  475. }
  476. def link_unversioned_tools(atoms):
  477. """Ensure unversioned tool symlinks exist for versioned ebuild tools."""
  478. wanted = set()
  479. for atom in atoms:
  480. for prefix, tools in UNVERSIONED_TOOL_LINKS.items():
  481. if atom == prefix or atom.startswith(prefix + ":"):
  482. wanted.update(tools)
  483. for tool in sorted(wanted):
  484. target = os.path.join("/usr/bin", tool)
  485. if os.path.exists(target):
  486. continue
  487. candidates = sorted(glob.glob("/usr/bin/%s-[0-9]*" % tool))
  488. if candidates:
  489. try:
  490. os.symlink(os.path.basename(candidates[-1]), target)
  491. except OSError:
  492. pass
  493. def cmd_plan(args):
  494. """Resolve the material install set without touching the system.
  495. One emerge --pretend for every atom; the merge list's names (in
  496. emerge's own transaction order) print as one contract JSON object,
  497. matching the names the query contract reports.
  498. """
  499. returncode, stdout, stderr = emerge_pretend(args.names)
  500. if returncode != 0:
  501. fail_plain("dependency resolution failed: %s"
  502. % failure_message((stdout + "\n" + stderr).splitlines()),
  503. EXIT_RESOLVE)
  504. _, _, atoms = parse_merge_list(stdout)
  505. sys.stdout.write(json.dumps(
  506. {"packages": [category_package(atom) for atom in atoms]}) + "\n")
  507. sys.stdout.flush()
  508. return EXIT_OK
  509. def cmd_install(args):
  510. returncode, stdout, stderr = emerge_pretend(args.names)
  511. if returncode != 0:
  512. fail("dependency resolution failed: %s"
  513. % failure_message((stdout + "\n" + stderr).splitlines()),
  514. EXIT_RESOLVE)
  515. total, _, atoms = parse_merge_list(stdout)
  516. emit_event({"type": "begin", "total": total})
  517. if args.test:
  518. emit_event({"type": "complete", "status": "ok", "installed": 0})
  519. return EXIT_OK
  520. command = [EMERGE, "--ask=n", "--quiet=y"] + list(args.names)
  521. try:
  522. proc = subprocess.Popen(
  523. command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  524. text=True, errors="replace", bufsize=1)
  525. except OSError as e:
  526. fail("could not run emerge: %s" % e, EXIT_FAILURE)
  527. transcript = []
  528. def drain_stderr():
  529. for line in proc.stderr:
  530. transcript.append(line)
  531. sys.stderr.write(line)
  532. sys.stderr.flush()
  533. reader = threading.Thread(target=drain_stderr)
  534. reader.daemon = True
  535. reader.start()
  536. completed = []
  537. for line in proc.stdout:
  538. transcript.append(line)
  539. sys.stderr.write(line)
  540. sys.stderr.flush()
  541. emerging = EMERGING_LINE.match(line)
  542. if emerging:
  543. emit_event({
  544. "type": "package",
  545. "name": category_package(emerging.group(3)),
  546. "current": int(emerging.group(1)),
  547. "total": int(emerging.group(2)),
  548. "progress": 0.0,
  549. })
  550. continue
  551. installing = INSTALLING_LINE.match(line)
  552. if installing:
  553. name = category_package(installing.group(3))
  554. completed.append(name)
  555. emit_event({"type": "package-complete", "name": name})
  556. returncode = proc.wait()
  557. reader.join(timeout=5)
  558. if returncode != 0:
  559. fail("emerge failed (%s): %s" % (returncode, failure_message(transcript)),
  560. classify_failure("".join(transcript)))
  561. link_unversioned_tools(args.names)
  562. emit_event({
  563. "type": "complete",
  564. "status": "ok",
  565. "installed": len(completed),
  566. })
  567. return EXIT_OK
  568. def main():
  569. parser = argparse.ArgumentParser(
  570. prog="usm-spm-emerge",
  571. description="USM system-package-manager helper for Portage (Gentoo)")
  572. subparsers = parser.add_subparsers(dest="command", required=True)
  573. query_parser = subparsers.add_parser(
  574. "query", help="resolve USM resource refs to system packages")
  575. query_parser.add_argument(
  576. "refs", nargs="+", metavar="USM-REF",
  577. help="resource ref, e.g. bin:valac or pc:glib-2.0.pc")
  578. query_parser.set_defaults(handler=cmd_query)
  579. plan_parser = subparsers.add_parser(
  580. "plan", help="resolve the material install set for atoms")
  581. plan_parser.add_argument(
  582. "names", nargs="+", metavar="ATOM",
  583. help="native gentoo package atom, e.g. dev-build/meson")
  584. plan_parser.set_defaults(handler=cmd_plan)
  585. install_parser = subparsers.add_parser(
  586. "install", help="install system packages, streaming progress events")
  587. install_parser.add_argument(
  588. "--test", action="store_true",
  589. help="resolve only (emerge --pretend); no packages are built")
  590. install_parser.add_argument(
  591. "names", nargs="+", metavar="ATOM",
  592. help="native gentoo package atom, e.g. dev-build/meson")
  593. install_parser.set_defaults(handler=cmd_install)
  594. args = parser.parse_args()
  595. try:
  596. return args.handler(args)
  597. except HelperError as e:
  598. fail(str(e), e.exit_code)
  599. except SystemExit:
  600. raise
  601. except Exception as e:
  602. message = "unexpected failure: %s" % e
  603. if args.command == "install":
  604. fail(message, EXIT_FAILURE)
  605. fail_plain(message, EXIT_FAILURE)
  606. if __name__ == "__main__":
  607. sys.exit(main())