Moved some logic around

README.md

@@ -147,25 +147,44 @@ Complete package is simply a tar.xz file containing a MANIFEST.usm file and all
 }
 ```
 
-### Transaction format
+
+## Resource Ref Additions
+
+- Add version expressions (can copy from MPK), i.e. == >= <= <>
+- Add support for version expressions to *some* resource types, namely pkg and pc
+
+
+Below here are draft ideas that may or may not get implemented
+
+## Transactions
+
+- Create backup BTRFS snapshot
+- Create transaction BTRFS snapshot from backup snapshot
+- Apply difference between backup and transaction snapshots to live filesystem
+- Delete transaction snapshot
+
+JSONL transaction journal format
+
 ```json
 {
-    "complete": true, // False when in progress
-    "snapshot": "snapshot path", // For later when BTRFS snapshotting implemented, for rollbacks
-    "operations": [
-        {
-            "operation": "install", // or "remove"
-            "package": {
-                "name": "packagename",
-                "version": "0.1.1"
-            },
-            "origin" {
-                "repository": "repositoryname",
-                "listfile": "2020-03-20T14:34:42.382748.usml",
-                "original_path": "packagename-0.1.1.usmc",
-                "signature_verified": true
-            }
-        }
-    ]
+  "operation": "begin-transaction",
+  "timestamp": "2020-03-20T14:34:42.382748",
+  "snapshot": "path/to/backup/snapshot",
+  "subvolume": "path/to/transaction/subvolume"
 }
-```
+```
+
+```json
+{
+  "operation": "begin-transaction",
+  "timestamp": "2020-03-20T14:34:42.382748",
+}
+```
+
+
+## Scriptable installation
+
+- USM will not manage the installation if an install exec is present
+- Compare backup and transaction snapshots for validation
+- Scripts can optionally handle installation but it must be verified that the expected files (and only the expected files) are installed
+- Add post-install and maybe post-transaction execs that are not subject to validations

src/cli/Install.vala

@@ -27,29 +27,20 @@ private int install_main(string[] args) {
     var path = state.generate_cache_path(target.manifest);
     File.new_for_path(path).make_directory();
     var package_path = Path.build_filename(path, "package.usmc");
-    var extract_path = Path.build_filename(path, "package");
-    File.new_for_path(extract_path).make_directory();
     client.download_package(package_path, target.repository_entry, (f, c, t) => printerr(@"Downloading $f $c/$t bytes\r"));
     client.verify_package(package_path, target.repository_entry, (f, c, t) => printerr(@"Verifying $f $c/$t bytes\r"));
     var cached_package = new Usm.CachedPackage(path);
 
-    var proc = new Subprocess.newv(new string[] { "tar", "-xf", package_path, "-C", extract_path }, SubprocessFlags.INHERIT_FDS);
-    if(!proc.wait_check()) {
-        printerr("Failed to extract package\n");
-        return 20;
-    }
-
-    Environment.set_current_dir(extract_path);
-    manifest = new Usm.Manifest.from_file("MANIFEST.usm");
-    build_path = cached_package.create_build_directory();
+    var transaction = new Usm.Transaction() {
+        paths = paths,
+        to_remove = new Set<Usm.CachedPackage>(),
+        to_install = single(cached_package).to_set(),
+        state = state
+    };
 
-    var result = install();
-    if(result != 0) {
-        return result;
-    }
+    transaction.progress_updated.connect(transaction.print_progress);
+    transaction.run();
 
-    state.mark_installed(cached_package);
-    cached_package.get_build_archive();
 
     // todo cleanup
 

src/lib/Manifest.vala

@@ -157,8 +157,6 @@ namespace Usm {
             return proc;
         }
 
-        
-
     }
 
 

src/lib/Resolver.vala

@@ -51,6 +51,7 @@ namespace Usm {
         public Repository? repository { get; set; }
         public RepositoryListingEntry? repository_entry { get; set; }
         public string? package_path { get; set; }
+        public CachedPackage? cached_package { get; set; }
 
         public AbstractPackage.from_package(string path) throws Error {
             package_path = path;

src/lib/State/CachedPackage.vala

@@ -47,10 +47,7 @@ namespace Usm {
                 throw new StateError.NO_BUILD_ARTIFACT(@"The package \"$package_name\" has no build artifact");
             }
 
-            var proc = new Subprocess.newv(new string[] { "tar", "-xf", archive_path, "-C", path }, SubprocessFlags.INHERIT_FDS);
-            if(!proc.wait_check()) {
-                throw new StateError.EXTRACTION_FAILED(@"Could not extract \"$archive_path\": tar returned non-zero exit status $(proc.get_status())");
-            }
+            Util.unarchive(archive_path, path);
             return path;
         }
 
@@ -65,13 +62,53 @@ namespace Usm {
                 throw new StateError.NO_BUILD_ARTIFACT(@"The package \"$package_name\" has no build artifact");
             }
 
-            var proc = new Subprocess.newv(new string[] { "tar", "-cJf", path, "--directory", dir_path, "." }, SubprocessFlags.INHERIT_FDS);
-            if(!proc.wait_check()) {
-                throw new StateError.ARCHIVAL_FAILED(@"Could not archive \"$dir_path\": tar returned non-zero exit status $(proc.get_status())");
+            Util.archive(dir_path, path);
+            return path;
+        }
+
+        public void archive_build() throws Error {
+            var build_dir = get_build_directory();
+
+            // Delete existing archive, if it exists
+            var archive_file = File.new_for_path(build_archive_path());
+            if(archive_file.query_exists()) {
+                archive_file.delete();
+            }
+
+            Util.archive(build_dir, archive_file.get_path());
+            clean_build_directory();
+        }
+
+        public void unarchive_build() throws Error {
+            clean_build_directory();
+            get_build_directory();
+        }
+
+        public void clean_build_directory() throws Error {
+            var path = build_directory_path();
+            if(File.new_for_path(path).query_exists()) {
+                Util.delete_tree(path);
             }
+        }
+
+        public string get_source_directory() throws Error {
+            var path = source_directory_path();
+            if(File.new_for_path(path).query_exists()) {
+                return path;
+            }
+
+            print(@"Extracting $(package_path) to $(path)\n");
+            Util.unarchive(package_path, path);
             return path;
         }
 
+        public void clean_source() throws Error {
+            var path = source_directory_path();
+            if(File.new_for_path(path).query_exists()) {
+                Util.delete_tree(path);
+            }
+        }
+
         public void update_origin_information(OriginInformation info) throws Error {
             var properties = OriginInformation.get_mapper().map_from(info);
             var json = new InvercargillJson.JsonElement.from_properties(properties);
@@ -86,6 +123,10 @@ namespace Usm {
             return Path.build_filename(state_path, "build.tar.xz");
         }
 
+        private string source_directory_path() {
+            return Path.build_filename(state_path, "package");
+        }
+
     }
 
 }

src/lib/State/State.vala

@@ -77,8 +77,6 @@ namespace Usm {
 
     public errordomain StateError {
         NO_BUILD_ARTIFACT,
-        EXTRACTION_FAILED,
-        ARCHIVAL_FAILED,
     }
 
 }

src/lib/Transaction.vala

@@ -0,0 +1,254 @@
+using Invercargill;
+
+namespace Usm {
+
+    public class Transaction {
+
+        public Paths paths { get; set; }
+        public SystemState state { get; set; }
+        public Set<CachedPackage> to_install { get; set; }
+        public Set<CachedPackage> to_remove { get; set; }
+
+        public signal void progress_updated(TransactionTask task_type, string subject, int current_task, int total_tasks, float task_progress);
+
+        private int task_count = 0;
+        private int current_task = 0;
+        private string current_subject = "transaction";
+        private Vector<Vector<CachedPackage>> install_lots;
+        private Vector<CachedPackage> remove_order;
+
+        public void run() throws TransactionError {
+
+            // 1. Verify the transaction is valid
+            strategise();
+
+            var all_packages = to_remove.concat(to_install);
+
+            // 2. Unpack packages
+            do_for(all_packages, unpack_package, TransactionTask.UNPACKING);
+
+            // 3. Remove packages
+            do_for(remove_order, remove_package, TransactionTask.REMOVING);
+
+            foreach (var lot in install_lots) {
+                // 3. Build packages
+                do_for(lot, build_package, TransactionTask.BUILDING);
+    
+                // 5. Install packages
+                do_for(lot, install_package, TransactionTask.INSTALLING);
+            }
+
+            // 6. Clean up
+            do_for(all_packages, cleanup_package, TransactionTask.CLEANING_UP);
+
+        }
+
+        private string previous_key = "";
+        public void print_progress(TransactionTask task_type, string subject, int current_task, int total_tasks, float task_progress) {
+            var verb = task_type.get_verb();
+            verb = verb[0].toupper().to_string() + verb.substring(1);
+            var percent = (int)(task_progress * 100.0f);
+
+            var key = @"$(task_type.get_verb())_$(subject)_$current_task";
+            var prefix = previous_key == key ? "\x1b[1F\x1b[2K" : "";
+            previous_key = key;
+
+            printerr(@"$prefix[$current_task/$total_tasks] $verb $subject ($percent%)\n");
+        }
+
+        public void strategise() throws TransactionError {
+            report_progress(TransactionTask.STRATEGISING, 0.0f);
+            install_lots = new Vector<Vector<CachedPackage>>();
+            install_lots.add(to_install.to_vector());
+
+
+            report_progress(TransactionTask.STRATEGISING, 1.0f);
+            task_count = (to_install.count() * 4) + (to_remove.count() * 3);
+            current_task++;
+        }
+
+        private delegate void PackageDelegate(CachedPackage package) throws Error;
+        private void do_for(Enumerable<CachedPackage> packages, PackageDelegate func, TransactionTask task_type) throws TransactionError {
+            foreach (var package in packages) {
+                try {
+                    current_subject = package.package_name;
+                    report_progress(task_type, 0.0f);
+                    func(package);
+                    current_task++;
+                }
+                catch (TransactionError e) {
+                    throw e;
+                }
+                catch(Error e) {
+                    throw new TransactionError.UNKNOWN_ERROR(@"Error $(task_type.get_verb()) package $(package.package_name): $(e.message)");
+                }
+            }
+        }
+
+        private void report_progress(TransactionTask task, float progress) {
+            progress_updated(task, current_subject, current_task, task_count, progress);
+        }
+
+        public void unpack_package(CachedPackage package) throws Error {
+            var will_remove = to_remove.has(package);
+
+            // Get a clean copy of the sources
+            package.clean_source();
+            report_progress(TransactionTask.UNPACKING, will_remove ? 0.25f : 0.5f);
+            package.get_source_directory();
+            report_progress(TransactionTask.UNPACKING, will_remove ? 0.5f : 1.0f);
+
+            if(will_remove) {
+                // Get a clean copy of the build artifact
+                package.clean_build_directory();
+                report_progress(TransactionTask.UNPACKING, 0.75f);
+                package.get_build_directory();
+                report_progress(TransactionTask.UNPACKING, 1.0f);
+            }
+        }
+
+        private void build_package(CachedPackage package) throws Error {
+            // Get source directory, and create build directory
+            var source_dir = package.get_source_directory();
+            var build_dir = package.create_build_directory();
+                    
+            // Change directory to sources
+            Environment.set_current_dir(source_dir);
+            var manifest = new Usm.Manifest.from_file("MANIFEST.usm");
+
+            // Build package
+            var build_proc = manifest.run_build(build_dir, paths, SubprocessFlags.STDOUT_SILENCE);
+            build_proc.wait_check();   
+            report_progress(TransactionTask.BUILDING, 1.0f);
+        }
+
+        private void remove_package(CachedPackage package) throws Error {
+            // Get source and build directories
+            var source_dir = package.get_source_directory();
+            var build_dir = package.get_build_directory();
+
+            // "cd" into the source directory and read the manifest
+            Environment.set_current_dir(source_dir);
+            var manifest = new Usm.Manifest.from_file("MANIFEST.usm");
+
+            // Run remove process if present
+            var build_proc = manifest.run_remove(RemoveType.FINAL, SubprocessFlags.STDOUT_SILENCE);
+            if(build_proc != null)
+                build_proc.wait_check();
+
+            // Delete files and symlinks first
+            foreach (var resource in manifest.provides.where(r => r.value.file_type != Usm.ManifestFileType.DIRECTORY)) {
+                var path = paths.get_suggested_path(resource.key);
+                var file = File.new_for_path(path);
+                if(file.query_exists()) {
+                    file.delete();
+                }
+            }
+
+            // Delete directories last
+            foreach (var resource in manifest.provides.where(r => r.value.file_type == Usm.ManifestFileType.DIRECTORY)) {
+                var path = paths.get_suggested_path(resource.key);
+                try {
+                    var file = File.new_for_path(path);
+                    if(file.query_exists()) {
+                        file.delete();
+                    }
+                }
+                catch(IOError.NOT_EMPTY e) {
+                    warning(@"Did not remove resource \"$path\": directory is not empty\n");
+                }
+            }
+
+            state.unmark_installed(package);
+        }
+
+        private void install_package(CachedPackage package) throws Error {
+            var source_dir = package.get_source_directory();
+            var build_dir = package.get_build_directory();
+
+            // "cd" into the source directory and read the manifest
+            Environment.set_current_dir(source_dir);
+            var manifest = new Usm.Manifest.from_file("MANIFEST.usm");
+            report_progress(TransactionTask.INSTALLING, 0.0f);
+
+            // Install each resource speficied by the manifest
+            var resource_count = manifest.provides.count();
+            var resources_installed = 0;
+            foreach (var resource in manifest.provides) {
+                var path = paths.get_suggested_path(resource.key);
+                if(resource.value.file_type == Usm.ManifestFileType.REGULAR) {
+                    var src = File.new_build_filename(build_dir, resource.value.path);
+                    var dest = File.new_for_path(path);
+                    src.copy(dest, FileCopyFlags.OVERWRITE);
+                }
+                else if(resource.value.file_type == Usm.ManifestFileType.DIRECTORY) {
+                    var dest = File.new_for_path(path);
+                    dest.make_directory();
+                }
+                else if(resource.value.file_type == Usm.ManifestFileType.SYMBOLIC_LINK) {
+                    var dest = File.new_for_path(path);
+                    dest.make_symbolic_link(resource.value.path);
+                }
+                else {
+                    throw new TransactionError.INSTALL_ERROR(@"Could not understand resource key \"$(resource.key)\"");
+                }
+
+                resources_installed++;
+                report_progress(TransactionTask.INSTALLING, (float)resources_installed / (float)resource_count);
+            }
+
+            // Run install process if present
+            var build_proc = manifest.run_install(build_dir, InstallType.FRESH, SubprocessFlags.STDOUT_SILENCE);
+            if(build_proc != null)
+                build_proc.wait_check();
+
+            // Update the system state, and cleanup
+            state.mark_installed(package);
+        }
+
+        private void cleanup_package(CachedPackage package) throws Error {
+            package.archive_build();
+            report_progress(TransactionTask.CLEANING_UP, 0.5f);
+            package.clean_source();
+            report_progress(TransactionTask.CLEANING_UP, 1.0f);
+        }
+    }
+
+    public enum TransactionTask {
+        STRATEGISING,
+        UNPACKING,
+        BUILDING,
+        REMOVING,
+        INSTALLING,
+        CLEANING_UP;
+
+        public string get_verb() {
+            switch (this) {
+                case STRATEGISING:
+                    return "preparing a strategy for";
+                case UNPACKING:
+                    return "unpacking";
+                case BUILDING:
+                    return "building";
+                case REMOVING:
+                    return "removing";
+                case INSTALLING:
+                    return "installing";
+                case CLEANING_UP:
+                    return "cleaning up";
+                default:
+                    assert_not_reached();
+            }
+        }
+    }
+
+    public errordomain TransactionError {
+
+        INVALID_TRANSACTION,
+        UNKNOWN_ERROR,
+        BUILD_ERROR,
+        INSTALL_ERROR
+    }
+
+
+}

src/lib/Util.vala

@@ -0,0 +1,54 @@
+
+namespace Usm.Util {
+
+    public static void delete_tree(string path) throws Error {
+        // Creating a new Directory object for the given folder path
+        var folder = File.new_for_path(path);
+
+        // Getting a list of all files and folders inside the directory
+        var enumerator = folder.enumerate_children("*", FileQueryInfoFlags.NONE);
+
+        // Looping through each file/folder and removing them
+        while (true) {
+            FileInfo? info = enumerator.next_file();
+            if (info == null) {
+                break;
+            }
+
+            // Checking if the current item is a file or a folder
+            if (info.get_file_type() == FileType.REGULAR) {
+                // Removing the file
+                var file = folder.get_child(info.get_name());
+                    file.delete();
+
+            }
+            else if (info.get_file_type() == FileType.DIRECTORY) {
+                // Removing the folder recursively
+                var subfolder = folder.get_child(info.get_name());
+                delete_tree(subfolder.get_path());
+            }
+        }
+        
+        folder.delete();
+    }
+    
+
+    public static void unarchive(string archive, string destination) throws Error {
+        var dest = File.new_for_path(destination);
+        if(!dest.query_exists()) {
+            dest.make_directory();
+        }
+        var proc = new Subprocess.newv(new string[] { "tar", "-xf", archive, "-C", destination }, SubprocessFlags.INHERIT_FDS);
+        if(!proc.wait_check()) {
+            throw new IOError.FAILED(@"Could not extract \"$archive\": tar returned non-zero exit status $(proc.get_status())");
+        }
+    }
+
+    public static void archive(string source, string archive) throws Error {
+        var proc = new Subprocess.newv(new string[] { "tar", "-cJf", archive, "--directory", source, "." }, SubprocessFlags.INHERIT_FDS);
+        if(!proc.wait_check()) {
+            throw new IOError.FAILED(@"Could not archive \"$source\": tar returned non-zero exit status $(proc.get_status())");
+        }
+    }
+
+}

src/lib/meson.build

@@ -11,6 +11,8 @@ sources += files('Paths.vala')
 sources += files('ResourceRef.vala')
 sources += files('Version.vala')
 sources += files('Resolver.vala')
+sources += files('Util.vala')
+sources += files('Transaction.vala')
 sources += files('Repository/Repository.vala')
 sources += files('Repository/RepositoryListing.vala')
 sources += files('Repository/RepositoryClient.vala')