Implement applicator

src/lib/Applicator.vala

@@ -7,6 +7,8 @@ namespace Binman {
         public ApplicationType application_type { get; private set; }
 
         private Enumerable<ManifestFile> manifests;
+        private Dictionary<ManifestFile, ManifestHeader> headers;
+        private Dictionary<ManifestFile, BlobStore> blob_stores;
         private Dictionary<string, ManifestFile> path_ownership;
 
         public Applicator(ApplicationType application_type, Enumerable<CompositionComponent>? composition = null) throws Error {
@@ -22,28 +24,146 @@ namespace Binman {
                 .try_select<ManifestFile>(c => new ManifestFile.from_name(c.name, application_type, c.key))
                 .to_series();
 
+            headers = new Dictionary<ManifestFile, ManifestHeader>();
+            blob_stores = new Dictionary<ManifestFile, BlobStore>();
             path_ownership = new Dictionary<string, ManifestFile>();
             foreach (var manifest in manifests) {
+                var header = manifest.read_header();
+                headers[manifest] = header;
+                blob_stores[manifest] = new BlobStore(manifest, header);
                 foreach (var entry in manifest.get_entries()) {
                     path_ownership[entry.unwrap().path] = manifest;
                 }
             }
         }
 
-        public void apply() {
-            
+        public void apply() throws Error {
+            // Figure out what files we need
+            var details = get_file_details().to_series();
+            var need_applied = details
+                .where(f => f.installation_status != FileStatus.GOOD)
+                .cache();
+
+            var need_downloaded = need_applied
+                .where(f => f.file_description.binary != null)
+                .where(f => !blob_stores.get_or_default(f.originating_manifest).has_descriptor(f.file_description.binary, true))
+                .cache();
+
+            // Download and verify the files we need
+            foreach (var file in need_downloaded) {
+                var blob_file = blob_stores[file.originating_manifest].get_file_for_descriptor(file.file_description.binary);
+                download_file(file.file_description.binary, file.originating_manifest, blob_file);
+                posix_to_error(Posix.chmod(blob_file.get_path(), 0600));
+                if(!verify_file(file.file_description.binary, blob_file)) {
+                    throw new IOError.INVALID_DATA(@"Checksum mismatch on file \"$(file.file_description.path)\" from manifest \"$(headers[file.originating_manifest].name)\"");
+                }
+            }
+
+            // Apply each file
+            foreach (var file in need_applied) {
+                apply_file(file);
+            }
+        }
+
+        public void apply_file(FileDetails file) throws Error {
+            // Case where file is already applied properly
+            if(file.installation_status == FileStatus.GOOD){
+                return;
+            }
+
+            // Delete the system file if the type or content is bad
+            if(file.installation_status.any_set(FileStatus.BAD_TYPE | FileStatus.BAD_CONTENT)) {
+                posix_to_error(Posix.unlink(file.file_description.path));
+
+                // If mode is zeroed out, it is a blanking entry, don't continue
+                if(file.file_description.mode == 0) {
+                    return;
+                }
+            }
+
+            // Cases where we need to create the file
+            var is_newly_create_file = file.installation_status.any_set(FileStatus.MISSING | FileStatus.BAD_TYPE | FileStatus.BAD_CONTENT);
+            if(is_newly_create_file) {
+                if(Posix.S_ISDIR(file.file_description.mode)) {
+                    posix_to_error(Posix.mkdir(file.file_description.path, file.file_description.mode));
+                }
+                else if(Posix.S_ISFIFO(file.file_description.mode)) {
+                    posix_to_error(Posix.mkfifo(file.file_description.path, file.file_description.mode));
+                }
+                else if(Posix.S_ISLNK(file.file_description.mode)) {
+                    posix_to_error(Posix.symlink(file.file_description.target, file.file_description.path));
+                }
+                else if(Posix.S_ISREG(file.file_description.mode) && file.file_description.binary == null) {
+                    posix_to_error(Posix.creat(file.file_description.path, file.file_description.mode));
+                }
+                else if(Posix.S_ISREG(file.file_description.mode) && file.file_description.binary != null) {
+                    reflink_copy(blob_stores[file.originating_manifest].get_path_for_discriptor(file.file_description.binary), file.file_description.path);
+                }
+                else {
+                    throw new IOError.NOT_SUPPORTED(@"No implemented way to create entry with specifed mode");
+                }
+            }
+
+            // Cases where we need to set the owner or group
+            if(is_newly_create_file || file.installation_status.any_set(FileStatus.BAD_OWNER | FileStatus.BAD_GROUP)) {
+                if(Posix.S_ISLNK(file.file_description.mode)) {
+                    posix_to_error(Posix.lchown(file.file_description.path, file.file_description.user, file.file_description.group));
+                }
+                else {
+                    posix_to_error(Posix.chown(file.file_description.path, file.file_description.user, file.file_description.group));
+                }
+            }
+
+            // Cases where we need to set the permissions
+            if(is_newly_create_file || file.installation_status.any_set(FileStatus.BAD_PERMISSIONS)) {
+                if(Posix.S_ISLNK(file.file_description.mode) && (file.file_description.mode & 0777) != 0777) {
+                    throw new IOError.NOT_SUPPORTED("Symlink with permissions other than 0777 is not supported");
+                }
+                posix_to_error(Posix.chmod(file.file_description.path, file.file_description.mode));
+            }      
         }
 
-        public Dictionary<string, FileStatus> check_files() throws Error {
+        public void download_file(ManifestFileDescriptor descriptor, ManifestFile manifest, File destination, FileProgressCallback? progress_callback = null) throws Error {
+            var header = headers.get_or_default(manifest) ?? manifest.read_header();
+            var downloaded = false;
+            foreach (var remote in header.remotes) {
+                try {
+                    var remote_file = File.new_for_uri(Uri.resolve_relative(remote, descriptor.remote_path, UriFlags.NONE));
+                    remote_file.copy(destination, FileCopyFlags.OVERWRITE | FileCopyFlags.TARGET_DEFAULT_PERMS | FileCopyFlags.TARGET_DEFAULT_MODIFIED_TIME, null, progress_callback);
+                    downloaded = true;
+                    break;
+                }
+                catch(Error e) {
+                    warning(@"Download \"$(descriptor.remote_path)\" from remote \"$remote\" failed: $(e.message)");
+                }
+            }
+
+            if(!downloaded) {
+                throw new IOError.FAILED("Could not download file, all remotes were tried.");
+            }
+        }
+
+        public bool verify_file(ManifestFileDescriptor descriptor, File file, FileProgressCallback? progress_callback = null) throws Error {
+            return !calculate_file_checksum(file.get_path(), progress_callback).equals(descriptor.checksum);
+        }
+
+        public Attempts<FileDetails> get_file_details() throws Error {
             return manifests
-                .try_select_nested<ManifestEntry>(m => m.get_entries().where(e => path_ownership.get_or_default(e.path) == m))
-                .select_to_dictionary<string, FileStatus>(e => e.path, e => check_installed_file(e));
+                .try_select_nested<FileDetails>(m => m.get_entries()
+                    .where(e => path_ownership.get_or_default(e.path) == m)
+                    .select<FileDetails>(e => new FileDetails() {
+                        originating_manifest = m,
+                        file_description = e,
+                        installation_status = check_installed_file(e)
+                    })
+                );
         }
 
         public FileStatus check_installed_file(ManifestEntry entry) {
             Posix.Stat stat_info;
             if(Posix.stat(entry.path, out stat_info) != 0) {
-                return FileStatus.MISSING;
+                // If mode is zero, file is supposed to be deleted
+                return entry.mode == 0 ? FileStatus.GOOD : FileStatus.MISSING;
             }
 
             var status = FileStatus.GOOD;
@@ -54,9 +174,14 @@ namespace Binman {
                 status |= FileStatus.BAD_OWNER;
             }
 
-            if(stat_info.st_mode != entry.mode){
-                status |= FileStatus.BAD_MODE;
+            if((stat_info.st_mode | 0777) != (entry.mode | 0777)){
+                status |= FileStatus.BAD_TYPE;
             }
+
+            if((stat_info.st_mode | ~0777) != (entry.mode | ~0777)){
+                status |= FileStatus.BAD_PERMISSIONS;
+            }
+
             else if(Posix.S_ISLNK(entry.mode)) {
                 var buffer = new char[uint16.MAX];
                 buffer.length = (int)Posix.readlink(entry.path, buffer);
@@ -83,18 +208,20 @@ namespace Binman {
             return status;
         }
 
-        public BinaryData calculate_file_checksum(string path) throws Error {
+        public static BinaryData calculate_file_checksum(string path, FileProgressCallback? progress_callback = null) throws Error {
             var stream = File.new_for_path(path).read();
             var checksum = new Checksum(ChecksumType.SHA512);
 
             const int increment = 1024 * 1024 * 128;
             uint64 read = 0;
-            var to_read = stream.query_info("*").get_size();
+            var file_size = stream.query_info("*").get_size();
+            var to_read = file_size;
             while(read < to_read) {
                 var data = new uint8[uint64.min(increment, to_read - read)];
                 data.length = (int)stream.read(data);
                 checksum.update(data, data.length);
                 read += data.length;
+                file_callback(progress_callback, (int64)read, file_size);
             }
 
             var checksum_bytes = new uint8[ChecksumType.SHA512.get_length()];
@@ -104,6 +231,36 @@ namespace Binman {
             return new Invercargill.BinaryData.from_byte_array(checksum_bytes);
         }
 
+        private static void file_callback(FileProgressCallback? cb, int64 current, int64 total) {
+            if(cb != null) {
+                cb(current, total);
+            }
+        }
+
+        internal static void posix_to_error(int result) throws Error {
+            if(result == 0) {
+                return;
+            }
+            throw new Error(GLib.Quark.from_string("posix"), errno, Posix.strerror(errno));
+        }
+
+        internal static void reflink_copy(string src, string dest) throws Error {
+            int pid = Posix.fork();
+            if (pid == 0) {
+                posix_to_error(Posix.execv("/bin/cp", new string[] {"--reflink=always", src, dest}));
+                Posix.exit(Posix.ENOENT);
+            }
+            int status;
+            Posix.waitpid(pid, out status, 0);
+            posix_to_error(status);
+        }
+
+    }
+
+    public class FileDetails {
+        public ManifestFile originating_manifest { get; set; }
+        public ManifestEntry file_description { get; set; }
+        public FileStatus installation_status { get; set; }
     }
 
     public enum ApplicationType {
@@ -126,7 +283,15 @@ namespace Binman {
         MISSING = (1 << 1),
         BAD_OWNER = (1 << 2),
         BAD_GROUP = (1 << 3),
-        BAD_MODE = (1 << 4),
-        BAD_CONTENT = (1 << 5),
+        BAD_PERMISSIONS = (1 << 4),
+        BAD_TYPE = (1 << 5),
+        BAD_CONTENT = (1 << 6);
+
+        public bool any_set(FileStatus flag) {
+            if(flag == GOOD) {
+                return this == GOOD;
+            }
+            return (this | flag) != 0;
+        }
     }
 }

src/lib/BlobStorage.vala

@@ -0,0 +1,45 @@
+
+namespace Binman {
+
+    public class BlobStore {
+
+        public ManifestFile manifest { get; private set; }
+        private ManifestHeader manifest_header;
+
+        public BlobStore(ManifestFile manifest, ManifestHeader? header = null) throws Error {
+            this.manifest = manifest;
+            this.manifest_header = header ?? manifest.read_header();
+        }
+
+        public string get_path_for_discriptor(ManifestFileDescriptor descriptor) {
+            var filename = descriptor.checksum.to_base64().replace("/", "_") + @"-$(descriptor.size)";
+            return Path.build_filename("/var/binman/", manifest_header.name, "blob", filename);
+        }
+
+        public File get_file_for_descriptor(ManifestFileDescriptor descriptor) {
+            var file = File.new_for_path(get_path_for_discriptor(descriptor));
+            return file;
+        }
+
+        public bool has_descriptor(ManifestFileDescriptor descriptor, bool verify_checksum = false) {
+            var file = get_file_for_descriptor(descriptor);
+            if(!file.query_exists()) {
+                return false;
+            }
+
+            if(!verify_checksum) {
+                return true;
+            }
+
+            try {
+                return Applicator.calculate_file_checksum(file.get_path()).equals(descriptor.checksum);
+            }
+            catch (Error e) {
+                warning("Failed to verify file in blob storage: " + e.message);
+                return false;
+            }
+        }
+
+    }
+
+}