using Invercargill; using Invercargill.DataStructures; namespace Usm { public class Transaction { public Paths paths { get; set; } public ResourceFinder resource_finder { get; set; } public SystemState state { get; set; } public Set to_install { get; set; default = new HashSet(); } public Set to_remove { get; set; default = new HashSet(); } /** * Optional install order (package names) taken from a * {@link ResolutionResult}'s {@link ResolutionResult.install_order}; * when set, packages build and install in exactly this order. */ public Vector? install_order { get; set; } /** * Optional removal order (package names) — the reverse of a * {@link ResolutionResult}'s install order; when set, removals follow * it exactly (dependents before their providers). */ public Vector? remove_order { get; set; } /** * Manifest names of the packages the user asked for by name (the * `usm install` arguments): each install records * {@link OriginInformation.explicitly_installed} true for these and * false for everything the resolver pulled in as a dependency. * Null — every command but `usm install` — marks the whole install * set implicit. */ public Vector? explicit_packages { get; set; } /** * Version downgrades executed by this transaction: each entry * removes {@link DowngradeEntry.current}'s installed resources — * keeping its state directory for a later downgrade back — and * installs {@link DowngradeEntry.target} in its place. */ public Vector to_downgrade { get; set; default = new Vector(); } public signal void progress_updated(TransactionTask task_type, string subject, uint current_task, uint total_tasks, float task_progress); private uint task_count = 0; private uint current_task = 0; private string current_subject = "transaction"; private TransactionTask current_task_type = TransactionTask.STRATEGISING; /** * Active rollback journal, non-null only for the duration of * {@link run} and {@link rebuild_package}: every install and * remove records its changes before executing them so a failure * can be undone in reverse (see {@link Journal}). */ private Journal? journal = null; /** * Working sets computed by {@link strategise}: {@link to_install} * plus downgrade targets, and {@link to_remove} plus downgrade * currents — the whole transaction runs against these. */ private Set planned_install = new HashSet(); private Set planned_removal = new HashSet(); /** Downgrade targets (installed with {@link InstallType.DOWNGRADE}) and currents (removed with {@link RemoveType.DOWNGRADE}). */ private Set downgrade_targets = new HashSet(); private Set downgrade_currents = new HashSet(); /** Removals whose package name this transaction reinstalls — update removals ({@link RemoveType.UPGRADE}). */ private Set upgrade_removals = new HashSet(); /** * Lots computed by {@link strategise}: each lot builds, tests and * installs together, and lots run in dependency order. A package * joins a lot only when every BUILD-phase ref is already present or * provided by an earlier lot, so downstream builds always see the * upstream package's installed `pc:`/`vapi:` artifacts. */ public Vector> install_lots { get; private set; } /** Removal order computed by {@link strategise}: dependents before their providers. */ public Vector removal_order { get; private set; } /** * Rebuild entries planned by {@link strategise} for packages in * {@link to_install} flagged * {@link ManifestFlag.REBUILD_DEPENDANTS}: each entry rebuilds its * installed {@link RebuildEntry.package} — same CachedPackage, * same version, same state path — through the standard pipeline, * tagged {@link TransactionTask.REBUILDING} in progress reports. * Empty when nothing is flagged or {@link state} is unset. */ public Vector rebuilds { get; private set; default = new Vector(); } public void run() throws TransactionError { // 1. Verify the transaction is valid strategise(); journal = new Journal(journal_backup_root()); try { journal.begin(); var all_packages = planned_removal.concat(planned_install); var rebuild_packages = rebuilds.select(r => r.package).to_vector(); // 2. Unpack packages do_for(all_packages, unpack_package, TransactionTask.UNPACKING); do_for(rebuild_packages, unpack_package, TransactionTask.REBUILDING); // 3. Remove packages do_for(removal_order, remove_package, TransactionTask.REMOVING); foreach (var lot in install_lots) { // 3. Build packages do_for(lot, build_package, TransactionTask.BUILDING); // 4. Test packages do_for(lot, test_package, TransactionTask.TESTING); // 5. Install packages do_for(lot, install_package, TransactionTask.INSTALLING); } // Rebuilds run the same pipeline once every flagged package is // installed, so dependants build against the fresh artifacts do_for(rebuild_packages, unpack_package, TransactionTask.UNPACKING); do_for(rebuild_packages, build_package, TransactionTask.REBUILDING); do_for(rebuild_packages, test_package, TransactionTask.TESTING); do_for(rebuild_packages, install_package, TransactionTask.INSTALLING); // 6. Clean up do_for(all_packages, cleanup_package, TransactionTask.CLEANING_UP); do_for(rebuild_packages, cleanup_package, TransactionTask.CLEANING_UP); journal.finish(); } catch(TransactionError e) { rollback_and_report(); throw e; } catch(Error e) { rollback_and_report(); throw new TransactionError.UNKNOWN_ERROR(@"Error running transaction: $(e.message)"); } finally { journal = null; } } /** * Rebuilds one already-installed package through the standard * pipeline — build-cache restore, clean-retry on failure, * reinstall of its resources — tagged * {@link TransactionTask.REBUILDING} throughout; the `usm * rebuild` entry point. No confirmation is needed: nothing * installed changes version. */ public void rebuild_package(CachedPackage package) throws TransactionError { var packages = new Vector(); packages.add(package); task_count = 5; current_task = 0; current_subject = package.package_name; current_task_type = TransactionTask.REBUILDING; journal = new Journal(journal_backup_root()); try { journal.begin(); do_for(packages, unpack_package, TransactionTask.UNPACKING); do_for(packages, build_package, TransactionTask.REBUILDING); do_for(packages, test_package, TransactionTask.TESTING); do_for(packages, install_package, TransactionTask.INSTALLING); do_for(packages, cleanup_package, TransactionTask.CLEANING_UP); journal.finish(); } catch(TransactionError e) { rollback_and_report(); throw e; } catch(Error e) { rollback_and_report(); throw new TransactionError.UNKNOWN_ERROR(@"Error rebuilding package $(package.package_name): $(e.message)"); } finally { journal = null; } } /** * Rolls the active journal back after a failure and prints what * was undone; a rollback failure is reported by the journal * itself and never masks the original error. */ private void rollback_and_report() { if(journal == null) { return; } journal.rollback(); printerr(journal.describe_rollback() + "\n"); } /** * `//usm/backup` — the journal's home, * rooted like every other destination this transaction writes * (`/var/usm/backup` on an unredirected system). */ private string journal_backup_root() { return Path.build_filename(paths.destination, paths.local_state, "usm", "backup"); } private string previous_key = ""; public void print_progress(TransactionTask task_type, string subject, uint current_task, uint total_tasks, float task_progress) { var verb = task_type.get_verb(); verb = verb[0].toupper().to_string() + verb.substring(1); var percent = (int)(task_progress * 100.0f); var key = @"$(task_type.get_verb())_$(subject)_$current_task"; var prefix = previous_key == key ? "\x1b[1F\x1b[2K" : ""; previous_key = key; printerr(@"$prefix[$(current_task+1)/$total_tasks] $verb $subject ($percent%)\n"); } public void print_progress_simple(TransactionTask task_type, string subject, uint current_task, uint total_tasks, float task_progress) { var verb = task_type.get_verb(); verb = verb[0].toupper().to_string() + verb.substring(1); var percent = (int)(task_progress * 100.0f); printerr(@"[$(current_task+1)/$total_tasks] $verb $subject ($percent%)\n"); } public void strategise() throws TransactionError { // Reset the task counters — strategise may be called twice // (once for the summary/confirm, once inside run()) current_task = 0; task_count = 0; planned_install = new HashSet(); planned_install.union_with(to_install); planned_removal = new HashSet(); planned_removal.union_with(to_remove); downgrade_targets = new HashSet(); downgrade_currents = new HashSet(); foreach(var entry in to_downgrade) { planned_install.add(entry.target); downgrade_targets.add(entry.target); planned_removal.add(entry.current); downgrade_currents.add(entry.current); } // A removal whose package name this transaction reinstalls is // an update removal, not a final one upgrade_removals = new HashSet(); try { var install_names = new HashSet(); foreach(var package in planned_install) { install_names.add(package.get_manifest().name); } foreach(var package in planned_removal) { if(install_names.contains(package.get_manifest().name)) { upgrade_removals.add(package); } } } catch(Error e) { throw new TransactionError.UNKNOWN_ERROR(@"Failed to read manifest while planning removals: $(e.message)"); } task_count = (planned_install.count() * 5) + (planned_removal.count() * 3) + 1; uint strategise_worst_case_task_count = (planned_removal.count() * planned_removal.count()) + (planned_install.count() * planned_install.count()); uint strategise_current_task = 0; report_progress(TransactionTask.STRATEGISING, 0.0f); // Installation strategy install_lots = new Vector>(); var touched = new HashSet(); var installed_by_earlier_lots = new HashSet(); var ordered_install = ordered_by_names(planned_install, install_order); var round = 0; while(true) { strategise_current_task = round * planned_install.count(); var lot = new Vector(); var installed_by_this_lot = new HashSet(); var remaining = ordered_install.exclude(touched); if(remaining.count() == 0) { break; } foreach (var package in ordered_install.exclude(touched)) { report_progress(TransactionTask.STRATEGISING, (float)strategise_current_task / (float)strategise_worst_case_task_count); try { var manifest = package.get_manifest(); // Build-phase refs must be satisfied by resources // already present or installed by an EARLIER lot: a // same-lot provide is not installed yet when this // package builds, so a build dependency on it defers // the package to the next lot PredicateDelegate buildtime_satisfied = d => resource_finder.has_resource(d) || installed_by_earlier_lots.any(r => d.satisfied_by(r)); // Manage-phase executables run at install time, when // everything this lot has already admitted (in // topological order) is installed PredicateDelegate installtime_satisfied = d => resource_finder.has_resource(d) || installed_by_earlier_lots.any(r => d.satisfied_by(r)) || installed_by_this_lot.any(r => d.satisfied_by(r)); if(manifest.dependencies.manage.is_satisfied(installtime_satisfied) && manifest.dependencies.build.is_satisfied(buildtime_satisfied)) { lot.add(package); touched.add(package); installed_by_this_lot.union_with(manifest.provides.select(p => p.key)); } strategise_current_task++; } catch(Error e) { throw new TransactionError.UNKNOWN_ERROR(@"Failed to read manifest for package \"$(package.package_name)\": $(e.message)"); } } if(lot.count() == 0) { var packages = ordered_install.exclude(touched).to_string(p => p.package_name, ", "); throw new TransactionError.INVALID_TRANSACTION(@"Could not build a transaction strategy, packages $(packages) have unmet or cyclical dependencies"); } installed_by_earlier_lots.union_with(installed_by_this_lot); install_lots.add(lot); round++; } var current_task_baseline = (planned_install.count() * planned_install.count()); strategise_current_task = current_task_baseline; report_progress(TransactionTask.STRATEGISING, (float)strategise_current_task / (float)strategise_worst_case_task_count); // Removal strategy removal_order = new Vector(); if(remove_order != null) { removal_order = ordered_by_names(planned_removal, remove_order); } else { Set remaining_to_remove; try { remaining_to_remove = planned_removal .attempt_select(p => new CachedPackageManifest(p)) .to_set(); } catch(Error e) { throw new TransactionError.UNKNOWN_ERROR(@"Failed to read manifest: $(e.message)"); } round = 0; while(true) { strategise_current_task = current_task_baseline + (round * planned_removal.count()); if(remaining_to_remove.count() == 0) { break; } var removed_this_round = false; foreach (var package in remaining_to_remove.sort((a, b) => a.package.package_name.collate(b.package.package_name))) { report_progress(TransactionTask.STRATEGISING, (float)strategise_current_task / (float)strategise_worst_case_task_count); if(remaining_to_remove.no(p => p.manifest.dependencies.manage.all_refs().any(d => package.manifest.provides.any(r => d.satisfied_by(r.key))))) { removal_order.add(package.package); remaining_to_remove.remove(package); strategise_current_task++; round++; removed_this_round = true; break; } strategise_current_task++; } if(!removed_this_round) { var packages = remaining_to_remove.to_string(p => p.package.package_name, ", "); throw new TransactionError.INVALID_TRANSACTION(@"Could not build a transaction strategy, packages $(packages) have unmet or cyclical dependencies"); } } } plan_rebuilds(); task_count += rebuilds.length * 5; report_progress(TransactionTask.STRATEGISING, 1.0f); current_task++; } /** * Plans rebuildDependants rebuilds: every package in * {@link to_install} flagged * {@link ManifestFlag.REBUILD_DEPENDANTS} names its installed * dependants via {@link SystemState.find_dependant_names}, and each * one neither installed nor removed by this transaction rebuilds * exactly once — dependants the transaction already handles are * skipped, and two flagged packages sharing a dependant rebuild it * once only. Without a {@link state} there is nothing to scan. */ private void plan_rebuilds() throws TransactionError { rebuilds = new Vector(); if(state == null) { return; } try { var installed_by_name = new Dictionary(); foreach(var installed in state.get_installed_packages()) { // Resolve through the installed symlink so a rebuild // carries the cache path: reinstalling through the // installed path itself would make mark_installed // relink the symlink onto itself var info = File.new_for_path(installed.state_path).query_info(FileAttribute.STANDARD_SYMLINK_TARGET, FileQueryInfoFlags.NOFOLLOW_SYMLINKS); var target = info.get_symlink_target(); installed_by_name.set(installed.package_name, target != null ? new CachedPackage(target) : installed); } var planned = new HashSet(); foreach(var package in planned_install) { planned.add(package.get_manifest().name); } foreach(var package in planned_removal) { planned.add(package.get_manifest().name); } foreach(var package in planned_install) { var manifest = package.get_manifest(); if(manifest.flags == null || !manifest.flags.has(ManifestFlag.REBUILD_DEPENDANTS)) { continue; } foreach(var name in state.find_dependant_names(package)) { CachedPackage dependant; if(!installed_by_name.try_get(name, out dependant)) { continue; } var dependant_name = dependant.get_manifest().name; if(planned.has(dependant_name)) { continue; } planned.add(dependant_name); rebuilds.add(new RebuildEntry() { package = dependant, trigger = package }); } } } catch(Error e) { throw new TransactionError.UNKNOWN_ERROR(@"Failed to plan dependant rebuilds: $(e.message)"); } } /** * Orders a transaction package set by the given manifest-name order * (from a {@link ResolutionResult}); names with no package in the set * are skipped (for example supplied cache packages filtered out by * the caller). Without an order the set is sorted by name for * determinism. Throws when an order covers none of a package's set * entries — every package must be covered. */ private Vector ordered_by_names(Set packages, Vector? names) throws TransactionError { if(names == null) { return packages.sort((a, b) => a.package_name.collate(b.package_name)).to_vector(); } var by_name = new Dictionary(); foreach(var package in packages) { try { by_name.set(package.get_manifest().name, package); } catch(Error e) { throw new TransactionError.UNKNOWN_ERROR(@"Failed to read manifest for package \"$(package.package_name)\": $(e.message)"); } } var ordered = new Vector(); var covered = new HashSet(); foreach(var name in names) { CachedPackage package; if(by_name.try_get(name, out package)) { ordered.add(package); covered.add(package); } } var uncovered_names = packages.exclude(covered).to_string(p => p.package_name, ", "); if(uncovered_names.length > 0) { throw new TransactionError.INVALID_TRANSACTION(@"The supplied package order does not cover: $(uncovered_names)"); } return ordered; } private delegate void PackageDelegate(CachedPackage package) throws Error; private void do_for(Enumerable packages, PackageDelegate func, TransactionTask task_type) throws TransactionError { foreach (var package in packages) { try { current_subject = package.package_name; current_task_type = task_type; report_progress(task_type, 0.0f); func(package); current_task++; } catch (TransactionError e) { throw e; } catch(Error e) { throw new TransactionError.UNKNOWN_ERROR(@"Error $(task_type.get_verb()) package $(package.package_name): $(e.message)"); } } } private void report_progress(TransactionTask task, float progress) { progress_updated(task, current_subject, current_task, task_count, progress); } public void unpack_package(CachedPackage package) throws Error { // A clean copy of the sources is all any phase needs — builds, // tests, installs and remove scripts all run from the source // directory — so removals must not require a build artifact // to exist (see {@link cleanup_package}) package.clean_source(); report_progress(current_task_type, 0.5f); package.get_source_directory((fraction) => { report_progress(current_task_type, 0.5f + (fraction * 0.5f)); }); report_progress(current_task_type, 1.0f); } /** * Builds a package through the unified build-cache path: an * existing build directory or a build archive (restored by * {@link CachedPackage.get_build_directory}) makes the build * incremental, and a failure atop either is retried once from * extracted sources — a stale cache is the likeliest culprit, and * the clean-retry error propagates when that also fails. A failure * on an already-clean build propagates directly. */ private void build_package(CachedPackage package) throws Error { var reused = package.has_build_directory() || package.has_build_archive(); try { attempt_build(package); } catch(Error e) { if(!reused) { throw e; } package.clean_build_directory(); package.clean_source(); package.get_source_directory(); package.create_build_directory(); attempt_build(package); } report_progress(current_task_type, 1.0f); } /** * Runs the package's build script against the unified * build-directory path: an existing build tree is reused and a * build archive is restored in place (both incremental), and only * with neither does a fresh build directory appear — a missing * cache is fine, the user may have cleaned it up. The caller owns * the clean-retry decision (see {@link build_package}). */ private void attempt_build(CachedPackage package) throws Error { // Get source directory, and reuse, restore or create the build directory string build_dir; if(package.has_build_directory() || package.has_build_archive()) { build_dir = package.get_build_directory(); } else { build_dir = package.create_build_directory(); } var source_dir = package.get_source_directory(); // Change directory to sources Environment.set_current_dir(source_dir); var manifest = new Usm.Manifest.from_file("MANIFEST.usm"); // Build package var build_proc = manifest.run_build(build_dir, paths, SubprocessFlags.STDOUT_SILENCE, (progress) => { report_progress(current_task_type, progress); }); build_proc.wait_check(); } private void test_package(CachedPackage package) throws Error { var source_dir = package.get_source_directory(); var build_dir = package.get_build_directory(); // "cd" into the source directory and read the manifest Environment.set_current_dir(source_dir); var manifest = new Usm.Manifest.from_file("MANIFEST.usm"); // Run test process if present var test_proc = manifest.run_test(build_dir, SubprocessFlags.STDOUT_SILENCE); if(test_proc != null) { test_proc.wait_check(); } // A package with no test executable still transitions through // the testing task, so completion is reported either way report_progress(current_task_type, 1.0f); } private void remove_package(CachedPackage package) throws Error { // Get source and build directories var source_dir = package.get_source_directory(); // "cd" into the source directory and read the manifest Environment.set_current_dir(source_dir); var manifest = new Usm.Manifest.from_file("MANIFEST.usm"); // A downgrade removal keeps the state directory for the way // back; an update removal is followed by a reinstall; anything // else is final var removal_type = downgrade_currents.has(package) ? RemoveType.DOWNGRADE : upgrade_removals.has(package) ? RemoveType.UPGRADE : RemoveType.FINAL; // Journal before anything is deleted: every existing resource // file is backed up and the installed mark's target recorded, // so a later failure puts the package back exactly as it was var files_removed = manifest.report_remove_resources(paths); journal.entry_remove(package, files_removed.to_array(), state.installed_mark_path(package), state.read_installed_target(package)); foreach(var path in files_removed) { journal.backup_file(path, package.package_name); } // Run remove process if present var build_proc = manifest.run_remove(removal_type, SubprocessFlags.STDOUT_SILENCE); if(build_proc != null) build_proc.wait_check(); manifest.remove_resources( paths, (r, cr, tr, f) => report_progress(current_task_type, ((float)cr + (float)f) / (float)tr)); state.unmark_installed(package); journal.complete(package.package_name); } private void install_package(CachedPackage package) throws Error { var source_dir = package.get_source_directory(); var build_dir = package.get_build_directory(); // "cd" into the source directory and read the manifest Environment.set_current_dir(source_dir); var manifest = new Usm.Manifest.from_file("MANIFEST.usm"); report_progress(current_task_type, 0.0f); string? install_dir = null; // Downgrade targets install over their current version rather // than fresh var install_type = downgrade_targets.has(package) ? InstallType.DOWNGRADE : InstallType.FRESH; // Journal before anything changes on disk: the destination // set comes from the manifest's provides, every file about to // be overwritten is backed up, and the installed mark's // previous target is recorded so the rollback restores it Vector files_created; Vector files_overwritten; manifest.report_install_resources(paths, out files_created, out files_overwritten); journal.entry_install(package, files_created.to_array(), files_overwritten.to_array(), state.installed_mark_path(package), state.read_installed_target(package)); foreach(var path in files_overwritten) { journal.backup_file(path, package.package_name); } // Run install process if present if(manifest.executables.install != null) { install_dir = package.create_install_directory(); // Only a ninjaStyleProgress install script streams // parseable Installing lines; everything else stays // silenced (see {@link Manifest.run_install}) ProgressDelegate? install_progress = null; if(manifest.flags != null && manifest.flags.has(ManifestFlag.NINJA_STYLE_PROGRESS)) { install_progress = (fraction) => { report_progress(current_task_type, fraction); }; } var install_proc = manifest.run_install(build_dir, install_dir, paths, install_type, SubprocessFlags.STDOUT_SILENCE, install_progress); install_proc.wait_check(); } // Install the package's resources manifest.install_resources(source_dir, build_dir, install_dir, paths, (r, cr, tr, f) => report_progress(current_task_type, ((float)cr + (float)f) / (float)tr)); // Run post install process if present report_progress(current_task_type, 1.0f); var post_install_proc = manifest.run_post_install(build_dir, install_type, SubprocessFlags.STDOUT_SILENCE); if(post_install_proc != null) post_install_proc.wait_check(); // Update the system state, and cleanup state.mark_installed(package); record_origin(package, manifest); journal.complete(package.package_name); } /** * Writes the package's origin record after install: a record * already in this cache directory (a rebuild reinstalls the same * one) keeps its provenance and its explicit mark — a rebuild * must not demote an explicitly installed package to * orphan-removable — while a fresh directory starts a record * whose only known field is the mark, set from whether the user * named the package in this transaction. */ private void record_origin(CachedPackage package, Manifest manifest) throws Error { OriginInformation origin; try { origin = package.get_origin_information(); } catch(Error e) { origin = new OriginInformation(); } var named = explicit_packages != null && explicit_packages.contains(manifest.name); origin.explicitly_installed = named || origin.explicitly_installed; package.update_origin_information(origin); } private void cleanup_package(CachedPackage package) throws Error { // A package with no build artifact (for example after `usm // clean builds`) has nothing to archive; its cache stays as-is if(package.has_build_directory() || package.has_build_archive()) { package.archive_build((compression) => { report_progress(TransactionTask.CLEANING_UP, compression * 0.25f); }); } report_progress(TransactionTask.CLEANING_UP, 0.25f); package.clean_source(); report_progress(TransactionTask.CLEANING_UP, 0.5f); package.clean_install(); report_progress(TransactionTask.CLEANING_UP, 0.75f); report_progress(TransactionTask.CLEANING_UP, 1.0f); } } /** * One planned rebuildDependants rebuild: {@link package} is the * installed {@link CachedPackage} — same version, same state path — * that rebuilds through the standard pipeline tagged * {@link TransactionTask.REBUILDING}, and {@link trigger} is the * flagged package whose install/update pulled it in. */ public class RebuildEntry { public CachedPackage package { get; set; } public CachedPackage trigger { get; set; } } /** * One planned version downgrade: {@link current} is the installed * {@link CachedPackage} whose resources are removed — its state * directory is kept, enabling the downgrade back — and {@link target} * is the older version installed in its place. */ public class DowngradeEntry { public CachedPackage target { get; set; } public CachedPackage current { get; set; } } public enum TransactionTask { STRATEGISING, UNPACKING, BUILDING, TESTING, REMOVING, INSTALLING, CLEANING_UP, /** The rebuildDependants pipeline for an already-installed package (see {@link RebuildEntry}). */ REBUILDING; public string get_verb() { switch (this) { case STRATEGISING: return "preparing a strategy for"; case UNPACKING: return "unpacking"; case BUILDING: return "building"; case TESTING: return "testing"; case REMOVING: return "removing"; case INSTALLING: return "installing"; case CLEANING_UP: return "cleaning up"; case REBUILDING: return "rebuilding"; default: assert_not_reached(); } } } public errordomain TransactionError { INVALID_TRANSACTION, UNKNOWN_ERROR, BUILD_ERROR, INSTALL_ERROR } }