usm/src/lib/Manifest.vala

using Invercargill;
using Invercargill.Mapping;
using Invercargill.DataStructures;

namespace Usm {

    public errordomain ManifestError {
        MISSING_FIELD,
        INVALID_VERSION,
        INVALID_LICENCE_CATEGORY,
        INVALID_RESOURCE_TYPE,
        INVALID_FILE_TYPE,
        INVALID_REMOVE_TYPE,
        INVALID_INSTALL_TYPE,
        INVALID_PACKAGE,
        INVALID_PATH_BASE,
        INVALID_FILE_PATH,
        INVALID_FLAG
    }

    public class Manifest {
        public string name { get; set; }
        public string summary { get; set; }
        public Version version { get; set; }
        public Vector<Licence> licences { get; set; }
        public Dictionary<ResourceRef, ManifestFile> provides { get; set; }
        public Dependencies dependencies { get; set; }
        public Executables executables { get; set; }
        public Set<ManifestFlag> flags { get; set; }

        public string? markdown_path { get; set; }
        public string? url { get; set; }
        public Vector<string>? screenshot_paths { get; set; }
        public string? icon_path { get; set; }
        public string? metainfo_path { get; set; }
        public Git? git { get; set; }
        public Properties? extra_properties { get; set; }

        public static PropertyMapper<Manifest> get_mapper() {
            return PropertyMapper.build_for<Manifest>(cfg => {
                cfg.map<string>("name", o => o.name, (o, v) => o.name = v);
                cfg.map<string>("version", o => o.version.to_string(), (o, v) => o.version = new Version.from_string(v));
                cfg.map<string>("summary", o => o.summary, (o, v) => o.summary = v);
                cfg.map_property_groups_with<Licence>("licences", o => o.licences, (o, v) => o.licences = v.to_vector(), Licence.get_mapper());
                cfg.map<Properties>("provides", o => o.map_from_provides_dict(), (o, v) => o.build_provides_dict(v));
                cfg.map_properties_with<Dependencies>("depends", o => o.dependencies, (o, v) => o.dependencies = v, Dependencies.get_mapper());
                cfg.map_properties_with<Executables>("execs", o => o.executables, (o, v) => o.executables = v, Executables.get_mapper());
                cfg.map_many<string>("flags", o => o.flags.select<string>(f => f.to_string()), (o, v) => o.flags = v.attempt_select<ManifestFlag>(f => ManifestFlag.from_string(f)).to_set());
                cfg.map<string>("md", o => o.markdown_path, (o, v) => o.markdown_path = v)
                    .undefined_when(o => o.markdown_path == null)
                    .when_undefined(o => o.markdown_path = null);
                cfg.map<string>("url", o => o.url, (o, v) => o.url = v)
                    .undefined_when(o => o.url == null)
                    .when_undefined(o => o.url = null);
                cfg.map_many<string>("screenshots", o => o.screenshot_paths, (o, v) => o.screenshot_paths = v.to_vector())
                    .undefined_when(o => o.screenshot_paths == null)
                    .when_undefined(o => o.screenshot_paths = null);
                cfg.map<string>("icon", o => o.icon_path, (o, v) => o.icon_path = v)
                    .undefined_when(o => o.icon_path == null)
                    .when_undefined(o => o.icon_path = null);
                cfg.map<string>("metainfo", o => o.metainfo_path, (o, v) => o.metainfo_path = v)
                    .undefined_when(o => o.metainfo_path == null)
                    .when_undefined(o => o.metainfo_path = null);
                //  cfg.map_with<Git>("git", o => o.git, (o, v) => o.git = v, Git.get_mapper(), false);
                cfg.map<Properties>("extras", o => o.extra_properties, (o, v) => o.extra_properties = v)
                    .undefined_when(o => o.extra_properties == null)
                    .when_undefined(o => o.extra_properties = null);
                cfg.set_constructor(() => new Manifest());
            });
        }

        public Manifest.from_file(string path) throws Error {
            var element = new InvercargillJson.JsonElement.from_file(path);
            Manifest.get_mapper().map_into(this, element.as<Invercargill.Properties>());
        }

        public Manifest.from_package(string path) throws Error {
            var archive = new Archive.Read();
            archive.support_format_tar();
            archive.support_filter_xz();
            var result = archive.open_filename(path, 10240);
            if(result != Archive.Result.OK) {
                throw new ManifestError.INVALID_PACKAGE("Could not read archive");
            }

            unowned Archive.Entry entry;
            while(archive.next_header(out entry) == Archive.Result.OK) {
                var path_name = entry.pathname();
                if(path_name != "./MANIFEST.usm") {
                    continue;
                }

                var manifest_blob = new ByteComposition();
				uint8[] buffer;
				Posix.off_t offset;
				while (archive.read_data_block (out buffer, out offset) == Archive.Result.OK) {
					manifest_blob.append_byte_array(buffer[offset:]);
				}

                var element = new InvercargillJson.JsonElement.from_string(manifest_blob.to_raw_string());
                Manifest.get_mapper().map_into(this, element.as<Invercargill.Properties>());
                return;
            }

            throw new ManifestError.INVALID_PACKAGE("MANIFEST.usm not found within archive");
        }

        private void build_provides_dict(Properties obj) throws Error {
            provides = new Dictionary<ResourceRef, ManifestFile>();
            var mapper = ManifestFile.get_mapper();
            foreach (var pair in obj) {
                ManifestFile file;
                if(pair.value.assignable_to<string>()) {
                    file = new ManifestFile.from_string(pair.value.as<string>());
                }
                else {
                    file = mapper.materialise(pair.value.as<Properties>());
                }
                // Validate the ManifestFile after creation
                file.validate();
                provides[new ResourceRef(pair.key)] = file;
            }
        }

        private Properties map_from_provides_dict() {
            var dict = new PropertyDictionary();
            var mapper = ManifestFile.get_mapper();
            foreach (var pair in provides) {
                try {
                    // Handle special case: when pathBase is as-expected, path is empty, and type is reg
                    // output string "as-expected" instead of full JSON object
                    if(pair.value.path_base == ManifestFilePathBase.AS_EXPECTED &&
                       pair.value.path == "" &&
                       pair.value.file_type == ManifestFileType.REGULAR) {
                        dict.set_native<string>(pair.key.to_string(), "as-expected");
                    } else {
                        dict.set_native<Properties>(pair.key.to_string(), mapper.map_from(pair.value));
                    }
                }
                catch(Error e) {
                    assert_not_reached();
                }
            }
            return dict;
        }

        
        public Subprocess run_build(string build_path, Paths paths, SubprocessFlags flags, ProgressDelegate? progress_delegate = null) throws Error {
            // Handle SIMPLE_BUILD_ENVIRONMENT flag
            string original_working_dir = Environment.get_current_dir();
            string working_dir = original_working_dir;
            string effective_build_path = build_path;
            
            if (this.flags.contains(ManifestFlag.SIMPLE_BUILD_ENVIRONMENT)) {
                // Copy the full source tree to the build directory
                var source_dir = File.new_for_path(working_dir);
                var build_dir = File.new_for_path(build_path);
                
                // Copy all files from source to build directory
                copy_directory(source_dir, build_dir);
                
                // Use build directory as working directory
                working_dir = build_path;
            }
            
            var path = Path.build_filename(working_dir, executables.build);
            
            // Change to the working directory for subprocess execution
            Environment.set_current_dir(working_dir);
            paths.set_envs();
            
            // Check if NINJA_STYLE_PROGRESS flag is set and progress delegate is provided
            if (this.flags.contains(ManifestFlag.NINJA_STYLE_PROGRESS) && progress_delegate != null) {
                // Set up subprocess to capture STDOUT for progress parsing
                var modified_flags = flags;
                // Ensure STDOUT is not silenced when we need to parse progress
                if ((modified_flags & SubprocessFlags.STDOUT_SILENCE) != 0) {
                    modified_flags = modified_flags & ~SubprocessFlags.STDOUT_SILENCE;
                }
                modified_flags = modified_flags | SubprocessFlags.STDOUT_PIPE;
                
                var proc = new Subprocess.newv(new string[] { path, effective_build_path }, modified_flags);
                
                // Start a new thread to monitor STDOUT for progress information
                ThreadFunc<void> progress_thread_func = () => {
                    try {
                        var stdout_pipe = proc.get_stdout_pipe();
                        if (stdout_pipe != null) {
                            var dis = new DataInputStream(stdout_pipe);
                            string line;
                            
                            // Read lines from STDOUT until the process ends
                            while ((line = dis.read_line(null)) != null) {
                                // Look for Ninja progress pattern: "[x/x] "
                                if (line.has_prefix("[")) {
                                    var end_bracket = line.index_of("]");
                                    if (end_bracket > 1) {
                                        var progress_str = line.substring(1, end_bracket - 1);
                                        var parts = progress_str.split("/");
                                        
                                        if (parts.length == 2) {
                                            int current_task = 0;
                                            int total_tasks = 0;
                                            
                                            // Parse the current and total task numbers
                                            if (int.try_parse(parts[0], out current_task) &&
                                                int.try_parse(parts[1], out total_tasks) &&
                                                total_tasks > 0) {
                                                
                                                // Calculate progress as a float between 0.0 and 1.0
                                                float progress = (float)current_task / (float)total_tasks;
                                                
                                                // Ensure progress is within valid bounds
                                                progress = float.max(0.0f, float.min(1.0f, progress));
                                                
                                                // Call the progress delegate with the calculated progress
                                                progress_delegate(progress);
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    } catch (Error e) {
                        // Log any errors during progress monitoring but don't fail the build
                        warning(@"Error monitoring build progress: $(e.message)");
                    }
                };
                
                try {
                    // Start the progress monitoring thread
                    new Thread<void>(null, progress_thread_func);
                } catch (Error e) {
                    // If we can't start the thread, log a warning but continue with the build
                    warning(@"Failed to start progress monitoring thread: $(e.message)");
                }
                
                // Restore original working directory after subprocess completes
                Environment.set_current_dir(original_working_dir);
                
                return proc;
            } else {
                var proc = new Subprocess.newv(new string[] { path, effective_build_path }, flags);
                
                // Restore original working directory after subprocess completes
                Environment.set_current_dir(original_working_dir);
                
                return proc;
            }
        }

        public Subprocess? run_rebuild(string build_path, SubprocessFlags flags) throws Error {
            if(executables.rebuild == null) {
                return null;
            }
            string original_working_dir = Environment.get_current_dir();
            string working_dir = original_working_dir;
            if (this.flags.contains(ManifestFlag.SIMPLE_BUILD_ENVIRONMENT)) {
                working_dir = build_path;
            }
            var path = Path.build_filename(working_dir, executables.rebuild);
            
            // Change to the working directory for subprocess execution
            Environment.set_current_dir(working_dir);
            var proc = new Subprocess.newv(new string[] { path, build_path }, flags);
            
            // Restore original working directory after subprocess completes
            Environment.set_current_dir(original_working_dir);
            
            return proc;
        }

        public Subprocess? run_acquire(SubprocessFlags flags) throws Error {
            if(executables.acquire == null) {
                return null;
            }
            string working_dir = Environment.get_current_dir();
            // Note: acquire doesn't have a build_path parameter, so it can't use SIMPLE_BUILD_ENVIRONMENT
            var path = Path.build_filename(working_dir, executables.acquire);
            var proc = new Subprocess.newv(new string[] { path }, flags);
            return proc;
        }

        public Subprocess? run_install(string build_path, string install_path, Paths paths, InstallType type, SubprocessFlags flags) throws Error {
            if(executables.install == null) {
                return null;
            }
            string original_working_dir = Environment.get_current_dir();
            string working_dir = original_working_dir;
            if (this.flags.contains(ManifestFlag.SIMPLE_BUILD_ENVIRONMENT)) {
                working_dir = build_path;
            }
            var path = Path.build_filename(working_dir, executables.install);

            // Override destination environment variable
            var new_paths = paths.clone();
            new_paths.destination = install_path;
            
            // Change to the working directory for subprocess execution
            Environment.set_current_dir(working_dir);
            new_paths.set_envs();
            var proc = new Subprocess.newv(new string[] { path, build_path, install_path, type.to_string() }, flags);
            
            // Restore original working directory after subprocess completes
            Environment.set_current_dir(original_working_dir);
            
            return proc;
        }

        public Subprocess? run_post_install(string build_path, InstallType type, SubprocessFlags flags) throws Error {
            if(executables.post_install == null) {
                return null;
            }
            string original_working_dir = Environment.get_current_dir();
            string working_dir = original_working_dir;
            if (this.flags.contains(ManifestFlag.SIMPLE_BUILD_ENVIRONMENT)) {
                working_dir = build_path;
            }
            var path = Path.build_filename(working_dir, executables.post_install);
            
            // Change to the working directory for subprocess execution
            Environment.set_current_dir(working_dir);
            var proc = new Subprocess.newv(new string[] { path, build_path, type.to_string() }, flags);
            
            // Restore original working directory after subprocess completes
            Environment.set_current_dir(original_working_dir);
            
            return proc;
        }

        public Subprocess? run_remove(RemoveType type, SubprocessFlags flags) throws Error {
            if(executables.remove == null) {
                return null;
            }
            string working_dir = Environment.get_current_dir();
            // Note: remove doesn't have a build_path parameter, so it can't use SIMPLE_BUILD_ENVIRONMENT
            var path = Path.build_filename(working_dir, executables.remove);
            var proc = new Subprocess.newv(new string[] { path, type.to_string() }, flags);
            return proc;
        }

        public Subprocess? run_test(string build_path, SubprocessFlags flags) throws Error {
            if(executables.test == null) {
                return null;
            }
            string original_working_dir = Environment.get_current_dir();
            string working_dir = original_working_dir;
            if (this.flags.contains(ManifestFlag.SIMPLE_BUILD_ENVIRONMENT)) {
                working_dir = build_path;
            }
            var path = Path.build_filename(working_dir, executables.test);
            
            // Change to the working directory for subprocess execution
            Environment.set_current_dir(working_dir);
            var proc = new Subprocess.newv(new string[] { path, build_path }, flags);
            
            // Restore original working directory after subprocess completes
            Environment.set_current_dir(original_working_dir);
            
            return proc;
        }

        public delegate void ResourceProgressCallback(ResourceRef resource, uint current_resource, uint total_resources, float resource_frac);
        public void install_resources(string source_path, string build_path, string? install_path, Paths paths, ResourceProgressCallback callback, bool dry_run = false) throws Error {
            // Install each resource speficied by the manifest
            var resource_count = provides.count();
            var resources_installed = 0;

            // Install from shortest path to longest path, to ensure directories are created before children
            var install_order = provides.sort((a, b) => paths.get_suggested_path_for_resource(a.key).length - paths.get_suggested_path_for_resource(b.key).length);
            foreach (var resource in install_order) {
                callback(resource.key, resources_installed, resource_count, 0.0f);
                var path = paths.get_suggested_path_for_resource(resource.key);
                
                if(resource.key.resource_type == ResourceType.TAG) {
                    // Ensure parent directories are created first
                    var parent_dir = File.new_for_path(Path.get_basename(path));
                    if(!parent_dir.query_exists() && !dry_run) {
                        parent_dir.make_directory_with_parents();
                    }
                }

                if(resource.value.file_type == Usm.ManifestFileType.REGULAR) {
                    var base_path = "";
                    switch (resource.value.path_base) {
                        case ManifestFilePathBase.BUILD:
                            base_path = build_path;
                            break;
                        case ManifestFilePathBase.SOURCE:
                            base_path = source_path;
                            break;
                        case ManifestFilePathBase.INSTALL:
                            if(install_path == null) {
                                throw new ManifestError.INVALID_FILE_PATH("Install path was not provided");
                            }
                            base_path = install_path;
                            break;
                        case ManifestFilePathBase.AS_EXPECTED:
                            if(install_path == null) {
                                throw new ManifestError.INVALID_FILE_PATH("Install path was not provided");
                            }
                            base_path = Path.build_filename(install_path, paths.get_suggested_path_for_resource(resource.key));
                            break;
                        default:
                            assert_not_reached();
                    }
                    var src = File.new_build_filename(base_path, resource.value.path ?? "");
                    var dest = File.new_for_path(path);

                    if(!src.query_exists()) {
                        throw new ManifestError.INVALID_FILE_PATH(@"Expected to find file listed in manifest at \"$(src.get_path())\", but no such file was found.");
                    }

                    if(!dry_run) {
                        src.copy(dest, FileCopyFlags.OVERWRITE, null, (c, t) => callback(resource.key, resources_installed, resource_count, (float)c / (float)t));
                    }
                }
                else if(resource.value.file_type == Usm.ManifestFileType.DIRECTORY) {
                    var dest = File.new_for_path(path);
                    if(!dry_run) {
                        dest.make_directory();
                    }
                }
                else if(resource.value.file_type == Usm.ManifestFileType.SYMBOLIC_LINK) {
                    var dest = File.new_for_path(path);
                    if(!dry_run){
                        dest.make_symbolic_link(resource.value.path ?? "");
                    }
                }
                else {
                    throw new TransactionError.INSTALL_ERROR(@"Could not understand resource key \"$(resource.key)\"");
                }
                
                callback(resource.key, resources_installed, resource_count, 1.0f);
                resources_installed++;
            }
        }

        public void remove_resources(Paths paths, ResourceProgressCallback callback) throws Error {
            var non_directories = provides.where(r => r.value.file_type != Usm.ManifestFileType.DIRECTORY).cache();
            var directories = provides.where(r => r.value.file_type == Usm.ManifestFileType.DIRECTORY).cache();
            var total_operations = non_directories.count() + directories.count();
            var current_operation = 0;

            // Delete files and symlinks first
            foreach (var resource in non_directories) {
                callback(resource.key, current_operation, total_operations, 0.0f);
                var path = paths.get_suggested_path_for_resource(resource.key);
                var file = File.new_for_path(path);
                if(file.query_exists()) {
                    file.delete();
                }
                callback(resource.key, current_operation, total_operations, 1.0f);
                current_operation++;
            }

            // Delete directories last
            foreach (var resource in directories) {
                callback(resource.key, current_operation, total_operations, 0.0f);
                var path = paths.get_suggested_path_for_resource(resource.key);
                try {
                    var file = File.new_for_path(path);
                    if(file.query_exists()) {
                        file.delete();
                    }
                }
                catch(IOError.NOT_EMPTY e) {
                    warning(@"Did not remove resource \"$path\": directory is not empty\n");
                }
                callback(resource.key, current_operation, total_operations, 1.0f);
                current_operation++;
            }
        }

        private void copy_directory(File source, File destination) throws Error {
            // Ensure destination directory exists
            if (!destination.query_exists()) {
                destination.make_directory_with_parents();
            }

            var enumerator = source.enumerate_children(FileAttribute.STANDARD_NAME + "," + FileAttribute.STANDARD_TYPE, FileQueryInfoFlags.NONE);
            FileInfo file_info;

            while ((file_info = enumerator.next_file()) != null) {
                var source_child = source.get_child(file_info.get_name());
                var destination_child = destination.get_child(file_info.get_name());

                if (file_info.get_file_type() == FileType.DIRECTORY) {
                    // Recursively copy subdirectories
                    copy_directory(source_child, destination_child);
                } else {
                    // Copy files
                    source_child.copy(destination_child, FileCopyFlags.OVERWRITE | FileCopyFlags.ALL_METADATA);
                }
            }
        }

    }


    public enum InstallType {
        FRESH,
        UPGRADE,
        DOWNGRADE;

        public string to_string() {
            switch (this) {
                case InstallType.FRESH:
                    return "fresh";
                case InstallType.UPGRADE:
                    return "upgrade";
                case InstallType.DOWNGRADE:
                    return "downgrade";
                default:
                    assert_not_reached();
            }
        }

        public static InstallType from_string(string str) throws ManifestError {
            switch (str) {
                case "fresh":
                    return InstallType.FRESH;
                case "upgrade":
                    return InstallType.UPGRADE;
                case "downgrade":
                    return InstallType.DOWNGRADE;
                default:
                    throw new ManifestError.INVALID_REMOVE_TYPE(@"Unknown install type \"$str\".");
            }
        }
    }

    public enum RemoveType {
        FINAL,
        UPGRADE,
        DOWNGRADE;

        public string to_string() {
            switch (this) {
                case RemoveType.FINAL:
                    return "final";
                case RemoveType.UPGRADE:
                    return "upgrade";
                case RemoveType.DOWNGRADE:
                    return "downgrade";
                default:
                    assert_not_reached();
            }
        }

        public static RemoveType from_string(string str) throws ManifestError {
            switch (str) {
                case "final":
                    return RemoveType.FINAL;
                case "upgrade":
                    return RemoveType.UPGRADE;
                case "downgrade":
                    return RemoveType.DOWNGRADE;
                default:
                    throw new ManifestError.INVALID_REMOVE_TYPE(@"Unknown remove type \"$str\".");
            }
        }
    }

    public delegate void ProgressDelegate(float progress);

    public enum ManifestFlag {
        BUILD_IN_SOURCE_TREE,
        SET_MANIFEST_PROPERTY_ENVS,
        NINJA_STYLE_PROGRESS,
        SIMPLE_BUILD_ENVIRONMENT;

        public string to_string() {
            switch (this) {
                case ManifestFlag.BUILD_IN_SOURCE_TREE:
                    return "buildInSourceTree";
                case ManifestFlag.SET_MANIFEST_PROPERTY_ENVS:
                    return "setManifestPropertyEnvs";
                case ManifestFlag.NINJA_STYLE_PROGRESS:
                    return "ninjaStyleProgress";
                case ManifestFlag.SIMPLE_BUILD_ENVIRONMENT:
                    return "simpleBuildEnvironment";
                default:
                    assert_not_reached();
            }
        }

        public static ManifestFlag from_string(string str) throws ManifestError {
            switch (str) {
                case "buildInSourceTree":
                    return ManifestFlag.BUILD_IN_SOURCE_TREE;
                case "setManifestPropertyEnvs":
                    return ManifestFlag.SET_MANIFEST_PROPERTY_ENVS;
                case "ninjaStyleProgress":
                    return ManifestFlag.NINJA_STYLE_PROGRESS;
                case "simpleBuildEnvironment":
                    return ManifestFlag.SIMPLE_BUILD_ENVIRONMENT;
                default:
                    throw new ManifestError.INVALID_FLAG(@"Unknown flag \"$str\".");
            }
        }
    }
}