usm-spm-emerge 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674
  1. #!/usr/bin/env python3
  2. """USM system-package-manager helper for Portage (Gentoo) systems.
  3. Implements the USM SPM contract (see usm/README.md, "System package manager
  4. integration") for Gentoo systems:
  5. usm-spm-emerge query <usm-ref>... -> contract JSON on STDOUT
  6. usm-spm-emerge install [--test] <atom>... -> contract JSONL events on STDOUT
  7. API choice: a Gentoo stage3 ships Portage and python3, so this helper reads
  8. the installed-package database through the portage Python API (with a raw
  9. /var/db/pkg CONTENTS scan as fallback) and shells out to `emerge` for
  10. dependency resolution (pretend) and installation. There is no binary
  11. metadata to bind against beyond that: ebuild repositories carry no file
  12. lists, so unlike the dnf/apt/apk helpers this one cannot search "which
  13. package contains file X" for packages that are not installed yet.
  14. File-backed refs are therefore resolved in two documented stages:
  15. 1. INSTALLED files (exact): the ref is translated to candidate filesystem
  16. paths (gentoo layout: /usr/bin, /usr/sbin, /usr/lib64, /usr/share, ...)
  17. and matched against the portage vdb (/var/db/pkg/*/CONTENTS). A ref
  18. resolves to the installed package that owns the file.
  19. 2. NOT-installed files (exception table only): Gentoo has no offline
  20. file-provides (PFL, the portage file list, is an online service), so a
  21. small curated table maps the USM stack's build/platform refs to Gentoo
  22. package atoms (valac -> dev-lang/vala, glib-2.0.pc -> dev-libs/glib,
  23. ...). Refs outside the table land in not-found; anything else would be
  24. guesswork.
  25. dependency-count / installed-dependency-count are honest numbers parsed
  26. from `emerge --pretend --quiet=y --ask=n <atom>`: dependency-count is the
  27. number of packages in the merge list including the package itself,
  28. installed-dependency-count is the subset of the merge list already
  29. installed (action letters without N, e.g. [ebuild R]/[ebuild U] reinstalls
  30. and upgrades; a fresh stage3 typically shows an all-N list). If the pretend
  31. run fails (atom missing from the tree, masked, unresolvable) an
  32. exception-table candidate falls back to not-found.
  33. Install maps `emerge --ask=n --quiet=y <atoms>` onto the JSONL contract,
  34. linking the unversioned tool names Gentoo ships versioned (UNVERSIONED_TOOL_LINKS):
  35. `begin`/`total` come from a pretend run first, a `package` event is emitted
  36. per ">>> Emerging (n of N)" status line, `package-complete` per
  37. ">>> Installing (n of N)" line (the moment a merge lands in the vdb),
  38. `complete` carries the number of completed merges, and any failure emits a
  39. terminal `error` event before a non-zero exit (resolution failures map to
  40. exit 3, download failures to 4, build/merge failures to 5).
  41. Documented Gentoo limitations:
  42. * No offline file-provides for not-yet-installed packages (see stage 2
  43. above); installed-file queries are exact.
  44. * USE flags are never touched: emerge runs with the system's configured
  45. defaults. The stack's platform libraries (glib, json-glib, libgee, ...)
  46. enable their introspection USE flag by default, but a system that
  47. disabled it (package.use/make.conf) will lack the matching .gir/
  48. .typelib/.vapi artifacts after install; re-enable the flag and re-emerge
  49. if a build needs them.
  50. * Packages are compiled from source; even small installs take minutes.
  51. Progress granularity is per package; emerge exposes no useful
  52. intra-package progress, so `package` events carry progress 0.0 and the
  53. completion signal is the ">>> Installing" line.
  54. * The query subcommand never modifies system state (at most Portage
  55. regenerates its dependency cache); install must run as root and never
  56. prompts (--ask=n answers Portage's own prompt for it).
  57. """
  58. import argparse
  59. import glob
  60. import json
  61. import os
  62. import re
  63. import subprocess
  64. import sys
  65. import threading
  66. EMERGE = "/usr/bin/emerge"
  67. EXIT_OK = 0
  68. EXIT_FAILURE = 1
  69. EXIT_RESOLVE = 3
  70. EXIT_DOWNLOAD = 4
  71. EXIT_TRANSACTION = 5
  72. FILE_ROOTS = {
  73. "sbin": ["/usr/sbin", "/sbin"],
  74. "libexec": ["/usr/libexec"],
  75. "gir": ["/usr/share/gir-1.0"],
  76. "typelib": [
  77. "/usr/lib64/girepository-1.0",
  78. "/usr/lib/girepository-1.0",
  79. "/lib64/girepository-1.0",
  80. "/lib/girepository-1.0",
  81. ],
  82. "gio": [
  83. "/usr/lib64/gio/modules",
  84. "/usr/lib/gio/modules",
  85. ],
  86. "res": ["/usr/share"],
  87. "cfg": ["/etc"],
  88. "man": ["/usr/share/man"],
  89. "info": ["/usr/share/info"],
  90. "locale": ["/usr/share/locale"],
  91. "app": ["/usr/share/applications"],
  92. "opt": ["/opt"],
  93. "rootpath": [""],
  94. "tag": ["/usr/share/usm-tags"],
  95. }
  96. LIB_ROOTS = ["/usr/lib64", "/lib64", "/usr/lib", "/lib"]
  97. PC_ROOTS = ["/usr/lib64/pkgconfig", "/usr/share/pkgconfig", "/usr/lib/pkgconfig"]
  98. TRANSLATED_PREFIXES = set(FILE_ROOTS) | {
  99. "bin", "lib", "libres", "pc", "vapi", "inc"}
  100. USR_MERGE_ALIASES = [
  101. ("/lib64/", "/usr/lib64/"),
  102. ("/lib/", "/usr/lib/"),
  103. ("/bin/", "/usr/bin/"),
  104. ("/sbin/", "/usr/sbin/"),
  105. ]
  106. EXCEPTIONS = {
  107. "bin:valac": "dev-lang/vala",
  108. "bin:vapigen": "dev-lang/vala",
  109. "bin:meson": "dev-build/meson",
  110. "bin:ninja": "dev-build/ninja",
  111. "bin:pkg-config": "dev-util/pkgconf",
  112. "bin:pkgconf": "dev-util/pkgconf",
  113. "bin:g-ir-scanner": "dev-libs/gobject-introspection",
  114. "bin:g-ir-compiler": "dev-libs/gobject-introspection",
  115. "bin:gcc": "sys-devel/gcc",
  116. "bin:g++": "sys-devel/gcc",
  117. "bin:cc": "sys-devel/gcc",
  118. "bin:c++": "sys-devel/gcc",
  119. "bin:ld": "sys-devel/binutils",
  120. "bin:make": "dev-build/make",
  121. "bin:python3": "dev-lang/python",
  122. "bin:bash": "app-shells/bash",
  123. "bin:git": "dev-vcs/git",
  124. "bin:curl": "net-misc/curl",
  125. "bin:tar": "app-arch/tar",
  126. "bin:xz": "app-arch/xz-utils",
  127. "bin:gzip": "app-arch/gzip",
  128. "bin:sed": "sys-apps/sed",
  129. "bin:awk": "sys-apps/gawk",
  130. "bin:grep": "sys-apps/grep",
  131. "bin:ldconfig": "sys-libs/glibc",
  132. "pc:glib-2.0.pc": "dev-libs/glib",
  133. "pc:gobject-2.0.pc": "dev-libs/glib",
  134. "pc:gio-2.0.pc": "dev-libs/glib",
  135. "pc:gio-unix-2.0.pc": "dev-libs/glib",
  136. "pc:gmodule-2.0.pc": "dev-libs/glib",
  137. "pc:gmodule-no-export-2.0.pc": "dev-libs/glib",
  138. "pc:gthread-2.0.pc": "dev-libs/glib",
  139. "pc:json-glib-1.0.pc": "dev-libs/json-glib",
  140. "pc:gee-0.8.pc": "dev-libs/libgee",
  141. "pc:libsodium.pc": "dev-libs/libsodium",
  142. "pc:libarchive.pc": "app-arch/libarchive",
  143. "pc:sqlite3.pc": "dev-db/sqlite",
  144. "pc:libxml-2.0.pc": "dev-libs/libxml2",
  145. "pc:libsoup-3.0.pc": "net-libs/libsoup",
  146. "pc:libmicrohttpd.pc": "net-libs/libmicrohttpd",
  147. "pc:gobject-introspection-1.0.pc": "dev-libs/gobject-introspection",
  148. "pc:zlib.pc": "sys-libs/zlib",
  149. "pc:libzstd.pc": "app-arch/zstd",
  150. "pc:libbrotlienc.pc": "app-arch/brotli",
  151. "pc:libbrotlidec.pc": "app-arch/brotli",
  152. "pc:libbrotlicommon.pc": "app-arch/brotli",
  153. "pc:libffi.pc": "dev-libs/libffi",
  154. "pc:pcre2.pc": "dev-libs/libpcre2",
  155. "vapi:glib-2.0.vapi": "dev-lang/vala",
  156. "vapi:gobject-2.0.vapi": "dev-lang/vala",
  157. "vapi:gio-2.0.vapi": "dev-lang/vala",
  158. "vapi:gio-unix-2.0.vapi": "dev-lang/vala",
  159. "vapi:posix.vapi": "dev-lang/vala",
  160. "vapi:json-glib-1.0.vapi": "dev-lang/vala",
  161. "vapi:libxml-2.0.vapi": "dev-lang/vala",
  162. "vapi:sqlite3.vapi": "dev-lang/vala",
  163. "vapi:gee-0.8.vapi": "dev-libs/libgee",
  164. "gir:GLib-2.0.gir": "dev-libs/gobject-introspection",
  165. "gir:GObject-2.0.gir": "dev-libs/gobject-introspection",
  166. "gir:Gio-2.0.gir": "dev-libs/gobject-introspection",
  167. "gir:GioUnix-2.0.gir": "dev-libs/gobject-introspection",
  168. "typelib:GLib-2.0.typelib": "dev-libs/gobject-introspection",
  169. "typelib:GObject-2.0.typelib": "dev-libs/gobject-introspection",
  170. "typelib:Gio-2.0.typelib": "dev-libs/gobject-introspection",
  171. "typelib:GioUnix-2.0.typelib": "dev-libs/gobject-introspection",
  172. "lib:libglib-2.0.so.0": "dev-libs/glib",
  173. "lib:libgobject-2.0.so.0": "dev-libs/glib",
  174. "lib:libgio-2.0.so.0": "dev-libs/glib",
  175. "lib:libgmodule-2.0.so.0": "dev-libs/glib",
  176. "lib:libgthread-2.0.so.0": "dev-libs/glib",
  177. "lib:libjson-glib-1.0.so.0": "dev-libs/json-glib",
  178. "lib:libgee-0.8.so.2": "dev-libs/libgee",
  179. "lib:libsodium.so.26": "dev-libs/libsodium",
  180. "lib:libarchive.so.13": "app-arch/libarchive",
  181. "lib:libsqlite3.so.0": "dev-db/sqlite",
  182. "lib:libxml2.so.2": "dev-libs/libxml2",
  183. "lib:libsoup-3.0.so.0": "net-libs/libsoup",
  184. "lib:libmicrohttpd.so.12": "net-libs/libmicrohttpd",
  185. "lib:libz.so.1": "sys-libs/zlib",
  186. "lib:libzstd.so.1": "app-arch/zstd",
  187. "lib:libbrotlienc.so.1": "app-arch/brotli",
  188. "lib:libbrotlidec.so.1": "app-arch/brotli",
  189. "lib:libbrotlicommon.so.1": "app-arch/brotli",
  190. "lib:libffi.so.8": "dev-libs/libffi",
  191. "lib:libpcre2-8.so.0": "dev-libs/libpcre2",
  192. "gio:libgiognutls.so": "net-libs/glib-networking",
  193. "lib:libc.so.6": "sys-libs/glibc",
  194. "lib:libm.so.6": "sys-libs/glibc",
  195. "lib:libblkid.so.1": "sys-apps/util-linux",
  196. "lib:libmount.so.1": "sys-apps/util-linux",
  197. "lib:libuuid.so.1": "sys-apps/util-linux",
  198. "inc:glib-2.0": "dev-libs/glib",
  199. "inc:json-glib-1.0": "dev-libs/json-glib",
  200. "inc:gee-0.8": "dev-libs/libgee",
  201. "inc:libsodium.h": "dev-libs/libsodium",
  202. "inc:archive.h": "app-arch/libarchive",
  203. "inc:sqlite3.h": "dev-db/sqlite",
  204. }
  205. class HelperError(Exception):
  206. """Fatal helper failure; message is emitted as a contract error event."""
  207. def __init__(self, message, exit_code=EXIT_FAILURE):
  208. super(HelperError, self).__init__(message)
  209. self.exit_code = exit_code
  210. def emit_event(event):
  211. """Write one contract JSONL event to STDOUT and flush it."""
  212. sys.stdout.write(json.dumps(event) + "\n")
  213. sys.stdout.flush()
  214. def fail(message, exit_code=EXIT_FAILURE):
  215. """Emit the terminal error event and exit non-zero."""
  216. emit_event({"type": "error", "message": message})
  217. sys.exit(exit_code)
  218. def fail_plain(message, exit_code=EXIT_FAILURE):
  219. """Report a query failure on STDERR only, keeping STDOUT contract-clean."""
  220. sys.stderr.write("usm-spm-emerge: %s\n" % message)
  221. sys.exit(exit_code)
  222. def category_package(cpv):
  223. """Strip version and ::repository suffix from a cpv -> category/package."""
  224. atom = cpv.split("::", 1)[0]
  225. try:
  226. import portage
  227. key = portage.cpv_getkey(atom)
  228. if key:
  229. return key
  230. except Exception:
  231. pass
  232. category, sep, package_version = atom.partition("/")
  233. if not sep:
  234. return atom
  235. return re.sub(r"-\d.*$", "", package_version) or package_version
  236. def parse_contents_line(line):
  237. """Path recorded on one CONTENTS line, or None for junk lines."""
  238. fields = line.split()
  239. if len(fields) >= 2 and fields[0] in ("obj", "dir", "sym", "dev", "fif"):
  240. return fields[1]
  241. return None
  242. def build_vdb_index():
  243. """Map every installed file path -> cpv via the portage vdb.
  244. Prefers the portage API (aux_get CONTENTS, present on any portage
  245. system); falls back to a raw /var/db/pkg/<cat>/<pkg>/CONTENTS scan.
  246. """
  247. index = {}
  248. try:
  249. import portage
  250. root = portage.settings.get("ROOT", "/")
  251. vdb = portage.db[root]["vartree"].dbapi
  252. for cpv in vdb.cpv_all():
  253. raw = vdb.aux_get(cpv, ["CONTENTS"])[0] or ""
  254. for line in raw.splitlines():
  255. path = parse_contents_line(line)
  256. if path is not None:
  257. index.setdefault(path, cpv)
  258. if index:
  259. return index
  260. except Exception as e:
  261. sys.stderr.write("usm-spm-emerge: portage API vdb read failed (%s), "
  262. "falling back to /var/db/pkg scan\n" % e)
  263. for contents_path in glob.glob("/var/db/pkg/*/*/CONTENTS"):
  264. category = os.path.basename(os.path.dirname(os.path.dirname(contents_path)))
  265. package_version = os.path.basename(os.path.dirname(contents_path))
  266. cpv = category + "/" + package_version
  267. try:
  268. with open(contents_path, "r", errors="replace") as handle:
  269. for line in handle:
  270. path = parse_contents_line(line)
  271. if path is not None:
  272. index.setdefault(path, cpv)
  273. except OSError:
  274. continue
  275. return index
  276. def path_aliases(path):
  277. """path plus its /usr-merge variant(s) (/lib64/x -> /usr/lib64/x...)."""
  278. aliases = [path]
  279. for merged, unmerged in USR_MERGE_ALIASES:
  280. if path.startswith(merged):
  281. variant = unmerged + path[len(merged):]
  282. elif path.startswith(unmerged):
  283. variant = merged + path[len(unmerged):]
  284. else:
  285. continue
  286. if variant not in aliases:
  287. aliases.append(variant)
  288. return aliases
  289. def bin_candidates(resource):
  290. """PATH directories in order (plus /usr/bin) as candidate directories."""
  291. directories = [d for d in os.environ.get("PATH", "").split(os.pathsep) if d]
  292. if "/usr/bin" not in directories:
  293. directories.append("/usr/bin")
  294. return [os.path.join(d, resource) for d in directories]
  295. def translate_paths(prefix, resource):
  296. """Candidate filesystem paths for a file-backed ref prefix."""
  297. if prefix == "bin":
  298. return bin_candidates(resource)
  299. if prefix == "tag":
  300. return ["/usr/share/usm-tags/" + resource.replace(".", "/") + ".tag"]
  301. if prefix in FILE_ROOTS:
  302. return ["/".join(FILE_ROOTS[prefix] + [resource])]
  303. if prefix in ("lib", "libres"):
  304. return [os.path.join(d, resource) for d in LIB_ROOTS]
  305. if prefix == "pc":
  306. return [os.path.join(d, resource) for d in PC_ROOTS]
  307. return None
  308. def resolve_installed(ref, index):
  309. """cpv of the installed package owning the ref's file, or None.
  310. Candidate paths (PATH order for bin:, gentoo root table otherwise) are
  311. matched exactly against the vdb index, with /usr-merge aliases tried on
  312. both sides; files present on disk but owned by no package do not count
  313. as provided by the system package manager.
  314. """
  315. prefix, sep, resource = ref.partition(":")
  316. if not sep or not resource:
  317. return None
  318. resource = resource.strip("/")
  319. if prefix == "vapi":
  320. pattern = re.compile(
  321. r"^/usr/share/vala[^/]*/vapi/" + re.escape(resource) + "$")
  322. for path, cpv in index.items():
  323. if pattern.match(path):
  324. return cpv
  325. return None
  326. if prefix == "inc":
  327. directory = "/usr/include/" + resource.strip("/")
  328. for path, cpv in index.items():
  329. if path == directory or path.startswith(directory + "/"):
  330. return cpv
  331. return None
  332. candidates = translate_paths(prefix, resource)
  333. if candidates is None:
  334. return None
  335. for path in candidates:
  336. for alias in path_aliases(path):
  337. if alias in index:
  338. return index[alias]
  339. return None
  340. def emerge_pretend(atoms):
  341. """Run emerge --pretend; returns (returncode, stdout, stderr)."""
  342. command = [EMERGE, "--pretend", "--quiet=y", "--ask=n"] + list(atoms)
  343. try:
  344. proc = subprocess.run(
  345. command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  346. text=True, errors="replace")
  347. except OSError as e:
  348. return EXIT_FAILURE, "", str(e)
  349. return proc.returncode, proc.stdout, proc.stderr
  350. MERGE_LINE = re.compile(r"^\[(ebuild|binary)\s+([A-Za-z*]+)\s*\]\s+(\S+)")
  351. def parse_merge_list(output):
  352. """(total, already_installed, atoms) from pretend output merge lines."""
  353. total = 0
  354. installed = 0
  355. atoms = []
  356. for line in output.splitlines():
  357. match = MERGE_LINE.match(line)
  358. if not match:
  359. continue
  360. total += 1
  361. if "N" not in match.group(2):
  362. installed += 1
  363. atoms.append(match.group(3))
  364. return total, installed, atoms
  365. def pretend_counts(atom, cache):
  366. """(dependency-count, installed-dependency-count) for a solo emerge.
  367. None means the atom cannot be resolved against the tree at all (no
  368. ebuilds, masked, conflicting): exception-table candidates then fall
  369. back to not-found; installed candidates degrade to (1, 1).
  370. """
  371. if atom in cache:
  372. return cache[atom]
  373. returncode, stdout, stderr = emerge_pretend([atom])
  374. if returncode != 0:
  375. cache[atom] = None
  376. return None
  377. total, installed, _ = parse_merge_list(stdout)
  378. counts = (total, installed) if total else (1, 1)
  379. cache[atom] = counts
  380. return counts
  381. def cmd_query(args):
  382. try:
  383. return run_query(args)
  384. except SystemExit:
  385. raise
  386. except Exception as e:
  387. fail_plain("query failed: %s" % e, EXIT_RESOLVE)
  388. def run_query(args):
  389. index = build_vdb_index()
  390. refs = list(dict.fromkeys(args.refs))
  391. candidates = {}
  392. resolved_ref = {}
  393. for ref in refs:
  394. cpv = resolve_installed(ref, index)
  395. if cpv is not None:
  396. name = category_package(cpv)
  397. else:
  398. name = EXCEPTIONS.get(ref)
  399. if name is None:
  400. if ref.partition(":")[0] not in TRANSLATED_PREFIXES:
  401. sys.stderr.write(
  402. "usm-spm-emerge: resource type of \"%s\" has no "
  403. "system-package-manager translation\n" % ref)
  404. continue
  405. resolved_ref[ref] = name
  406. entry = candidates.setdefault(name, {"resources": [], "installed": False})
  407. entry["resources"].append(ref)
  408. if cpv is not None:
  409. entry["installed"] = True
  410. counts_cache = {}
  411. package_counts = {}
  412. for name, entry in candidates.items():
  413. counts = pretend_counts(name, counts_cache)
  414. if counts is None and entry["installed"]:
  415. sys.stderr.write(
  416. "usm-spm-emerge: %s is installed but emerge cannot resolve "
  417. "its atom; estimating dependency counts as installed-only\n"
  418. % name)
  419. counts = (1, 1)
  420. package_counts[name] = counts
  421. not_found = []
  422. for ref in refs:
  423. name = resolved_ref.get(ref)
  424. if name is None or package_counts[name] is None:
  425. not_found.append(ref)
  426. packages = []
  427. for name in sorted(candidates):
  428. if package_counts[name] is None:
  429. continue
  430. dependency_count, installed_dependency_count = package_counts[name]
  431. packages.append({
  432. "name": name,
  433. "resources": candidates[name]["resources"],
  434. "dependency-count": dependency_count,
  435. "installed-dependency-count": installed_dependency_count,
  436. })
  437. sys.stdout.write(json.dumps({
  438. "not-found": not_found,
  439. "packages": packages,
  440. }) + "\n")
  441. sys.stdout.flush()
  442. return EXIT_OK
  443. EMERGING_LINE = re.compile(r">>> Emerging \((\d+) of (\d+)\) (\S+)")
  444. INSTALLING_LINE = re.compile(r">>> Installing \((\d+) of (\d+)\) (\S+)")
  445. def classify_failure(output):
  446. """Map an emerge failure transcript onto a contract exit code."""
  447. lowered = output.lower()
  448. if ("all ebuilds that could satisfy" in lowered
  449. or "dependency conflict" in lowered
  450. or "conflicting requests" in lowered
  451. or "blocked" in lowered
  452. or "no ebuilds to satisfy" in lowered):
  453. return EXIT_RESOLVE
  454. if ("couldn't download" in lowered
  455. or "fetch instructions" in lowered
  456. or "failed to fetch" in lowered):
  457. return EXIT_DOWNLOAD
  458. return EXIT_TRANSACTION
  459. NOISE_LINE = re.compile(r"^\s*\*\s+(IMPORTANT:|Use eselect news)")
  460. def failure_message(lines, limit=15):
  461. """Last few meaningful output lines as a terminal error message."""
  462. meaningful = [line.strip() for line in lines
  463. if line.strip() and not NOISE_LINE.match(line)]
  464. tail = meaningful[-limit:]
  465. message = " | ".join(tail)
  466. return message[:800] if message else "emerge exited non-zero"
  467. # Gentoo installs some tool binaries under versioned names only (vala's
  468. # ebuild ships valac-0.56 without an unversioned valac), while USM
  469. # manifests reference the unversioned bin:. After a transaction that
  470. # installed one of these atoms, link each unversioned name to the newest
  471. # versioned binary so bin: refs resolve on the filesystem.
  472. UNVERSIONED_TOOL_LINKS = {
  473. "dev-lang/vala": ["valac", "vala-gen-introspect", "vapigen"],
  474. }
  475. def link_unversioned_tools(atoms):
  476. """Ensure unversioned tool symlinks exist for versioned ebuild tools."""
  477. wanted = set()
  478. for atom in atoms:
  479. for prefix, tools in UNVERSIONED_TOOL_LINKS.items():
  480. if atom == prefix or atom.startswith(prefix + ":"):
  481. wanted.update(tools)
  482. for tool in sorted(wanted):
  483. target = os.path.join("/usr/bin", tool)
  484. if os.path.exists(target):
  485. continue
  486. candidates = sorted(glob.glob("/usr/bin/%s-[0-9]*" % tool))
  487. if candidates:
  488. try:
  489. os.symlink(os.path.basename(candidates[-1]), target)
  490. except OSError:
  491. pass
  492. def cmd_install(args):
  493. returncode, stdout, stderr = emerge_pretend(args.names)
  494. if returncode != 0:
  495. fail("dependency resolution failed: %s"
  496. % failure_message((stdout + "\n" + stderr).splitlines()),
  497. EXIT_RESOLVE)
  498. total, _, atoms = parse_merge_list(stdout)
  499. emit_event({"type": "begin", "total": total})
  500. if args.test:
  501. emit_event({"type": "complete", "status": "ok", "installed": 0})
  502. return EXIT_OK
  503. command = [EMERGE, "--ask=n", "--quiet=y"] + list(args.names)
  504. try:
  505. proc = subprocess.Popen(
  506. command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  507. text=True, errors="replace", bufsize=1)
  508. except OSError as e:
  509. fail("could not run emerge: %s" % e, EXIT_FAILURE)
  510. transcript = []
  511. def drain_stderr():
  512. for line in proc.stderr:
  513. transcript.append(line)
  514. sys.stderr.write(line)
  515. sys.stderr.flush()
  516. reader = threading.Thread(target=drain_stderr)
  517. reader.daemon = True
  518. reader.start()
  519. completed = []
  520. for line in proc.stdout:
  521. transcript.append(line)
  522. sys.stderr.write(line)
  523. sys.stderr.flush()
  524. emerging = EMERGING_LINE.match(line)
  525. if emerging:
  526. emit_event({
  527. "type": "package",
  528. "name": category_package(emerging.group(3)),
  529. "current": int(emerging.group(1)),
  530. "total": int(emerging.group(2)),
  531. "progress": 0.0,
  532. })
  533. continue
  534. installing = INSTALLING_LINE.match(line)
  535. if installing:
  536. name = category_package(installing.group(3))
  537. completed.append(name)
  538. emit_event({"type": "package-complete", "name": name})
  539. returncode = proc.wait()
  540. reader.join(timeout=5)
  541. if returncode != 0:
  542. fail("emerge failed (%s): %s" % (returncode, failure_message(transcript)),
  543. classify_failure("".join(transcript)))
  544. link_unversioned_tools(args.names)
  545. emit_event({
  546. "type": "complete",
  547. "status": "ok",
  548. "installed": len(completed),
  549. })
  550. return EXIT_OK
  551. def main():
  552. parser = argparse.ArgumentParser(
  553. prog="usm-spm-emerge",
  554. description="USM system-package-manager helper for Portage (Gentoo)")
  555. subparsers = parser.add_subparsers(dest="command", required=True)
  556. query_parser = subparsers.add_parser(
  557. "query", help="resolve USM resource refs to system packages")
  558. query_parser.add_argument(
  559. "refs", nargs="+", metavar="USM-REF",
  560. help="resource ref, e.g. bin:valac or pc:glib-2.0.pc")
  561. query_parser.set_defaults(handler=cmd_query)
  562. install_parser = subparsers.add_parser(
  563. "install", help="install system packages, streaming progress events")
  564. install_parser.add_argument(
  565. "--test", action="store_true",
  566. help="resolve only (emerge --pretend); no packages are built")
  567. install_parser.add_argument(
  568. "names", nargs="+", metavar="ATOM",
  569. help="native gentoo package atom, e.g. dev-build/meson")
  570. install_parser.set_defaults(handler=cmd_install)
  571. args = parser.parse_args()
  572. try:
  573. return args.handler(args)
  574. except HelperError as e:
  575. fail(str(e), e.exit_code)
  576. except SystemExit:
  577. raise
  578. except Exception as e:
  579. message = "unexpected failure: %s" % e
  580. if args.command == "install":
  581. fail(message, EXIT_FAILURE)
  582. fail_plain(message, EXIT_FAILURE)
  583. if __name__ == "__main__":
  584. sys.exit(main())