usm/src/cli/Dcba.vala

using Invercargill;
using Invercargill.DataStructures;

private int dcba_main(string[] args) {
    if(args.length < 3) {
        return dcba_usage();
    }

    if(args[2] == "repository") {
        return dcba_repository_main(args);
    }
    if(args[2] == "add-repo") {
        return Usm.Cli.DcbaAddRepo.run(args);
    }

    return dcba_usage();
}

private int dcba_usage() {
    printerr("USAGE:\n\tusm dcba repository <init|add|remove|list|verify|publish|build-list>\n\tusm dcba add-repo <url-or-path>\n");
    return 255;
}

private int dcba_repository_main(string[] args) {
    if(args.length < 4) {
        return dcba_repository_usage();
    }

    if(args[2] != "repository") {
        return dcba_repository_usage();
    }

    if(args[3] == "init") {
        if(args.length != 5) {
            return dcba_repository_usage();
        }
        return dcba_repository_init(args[4]);
    }
    if(args[3] == "add") {
        if(args.length != 5) {
            return dcba_repository_usage();
        }
        return dcba_repository_add(args[4]);
    }
    if(args[3] == "remove") {
        if(args.length != 5) {
            return dcba_repository_usage();
        }
        return dcba_repository_remove(args[4]);
    }
    if(args[3] == "list") {
        if(args.length > 5) {
            return dcba_repository_usage();
        }
        return dcba_repository_list(args.length == 5 ? args[4] : null);
    }
    if(args[3] == "verify") {
        if(args.length != 5) {
            return dcba_repository_usage();
        }
        return dcba_repository_verify(args[4]);
    }
    if(args[3] == "publish") {
        if(args.length != 5) {
            return dcba_repository_usage();
        }
        return dcba_repository_publish(args[4]);
    }
    if(args[3] == "build-list") {
        if(args.length != 4) {
            return dcba_repository_usage();
        }
        return dcba_repository_build_list();
    }

    return dcba_repository_usage();
}

private int dcba_repository_usage() {
    printerr("USAGE:\n\tusm dcba repository init <repository name>\n\tusm dcba repository add <artifact.dcba>\n\tusm dcba repository remove <name|name-version|artifact-filename>\n\tusm dcba repository list [repo.dcbar]\n\tusm dcba repository verify repo.dcbar\n\tusm dcba repository publish <path>\n\tusm dcba repository build-list\n");
    return 255;
}

private int dcba_repository_init(string name) {
    if(dcba_find_repo_file(Environment.get_current_dir()) != null
        || FileUtils.test(Path.build_filename("keys", "private-key"), FileTest.EXISTS)) {
        printerr("A DCBA repository has already been initialised in this directory\n");
        return 1;
    }

    uint8[] public_key;
    uint8[] private_key;
    dcba_generate_keypair(out public_key, out private_key);

    DirUtils.create("keys", 0700);
    DirUtils.create("public", 0755);

    try {
        FileUtils.set_data(Path.build_filename("keys", "public-key"), public_key);
        FileUtils.set_data(Path.build_filename("keys", "private-key"), private_key);
        FileUtils.chmod(Path.build_filename("keys", "private-key"), 0600);

        var repository = new Usm.Repository();
        repository.key = Wrap.byte_array(public_key);
        repository.name = name;
        repository.url = @"file://$(Environment.get_current_dir())/public";
        repository.summary = @"My $name DCBA repository";

        var properties = Usm.Repository.get_mapper().map_from(repository);
        var json = new InvercargillJson.JsonElement.from_properties(properties);
        json.write_to_file(@"$name.dcbar");

        var gitignore = "keys/private*\n";
        if(FileUtils.test(".gitignore", FileTest.EXISTS)) {
            string existing;
            FileUtils.get_contents(".gitignore", out existing);
            if(!existing.has_suffix("\n")) {
                existing += "\n";
            }
            if(existing.index_of("keys/private*") == -1) {
                FileUtils.set_contents(".gitignore", existing + gitignore);
            }
        }
        else {
            FileUtils.set_contents(".gitignore", gitignore);
        }
    }
    catch(Error e) {
        printerr(@"Error: $(e.message)\n");
        return e.code;
    }

    printerr("DCBA repository initialised, add artifacts with `usm dcba repository add <name-version.dcba>` then publish them with `usm dcba repository publish <path>`\n");
    return 0;
}

/**
 * Locates the nearest DCBA repository root: the first directory at or
 * above the current directory holding a `*.dcbar` file, a `keys/`
 * directory and a `public/` directory — the `.dcbar`-suffix counterpart
 * of `repository_find_root`, deliberately separate so a USM source
 * repository and a DCBA repository can never be mistaken for one
 * another even when nested in the same tree.
 */
private string? dcba_find_root() {
    var directory = Environment.get_current_dir();
    while(true) {
        if(dcba_find_repo_file(directory) != null
            && FileUtils.test(Path.build_filename(directory, "keys"), FileTest.IS_DIR)
            && FileUtils.test(Path.build_filename(directory, "public"), FileTest.IS_DIR)) {
            return directory;
        }

        var parent = Path.get_dirname(directory);
        if(parent == directory) {
            return null;
        }
        directory = parent;
    }
}

/**
 * Finds the first `*.dcbar` file directly inside `root`, or null when
 * there is none.
 */
private string? dcba_find_repo_file(string root) {
    try {
        foreach(var file in Iterate.directory(root)) {
            if(file.has_suffix(".dcbar")) {
                return Path.build_filename(root, file);
            }
        }
    }
    catch(Error e) {
        return null;
    }
    return null;
}

/**
 * Loads the signing keys from `keys/public-key` and `keys/private-key` in
 * the DCBA repository root, falling back to the legacy loose
 * `public-key` and `private-key` files there.
 */
private bool dcba_load_keys(string root, out uint8[] public_key, out uint8[] private_key) {
    public_key = new uint8[0];
    private_key = new uint8[0];

    var key_dir = Path.build_filename(root, "keys");
    if(dcba_read_key(Path.build_filename(key_dir, "public-key"), out public_key)
        && dcba_read_key(Path.build_filename(key_dir, "private-key"), out private_key)) {
        return true;
    }

    if(dcba_read_key(Path.build_filename(root, "public-key"), out public_key)
        && dcba_read_key(Path.build_filename(root, "private-key"), out private_key)) {
        return true;
    }

    printerr("No signing keys found in \"keys/\" (or as legacy \"public-key\"/\"private-key\") within the DCBA repository root\n");
    return false;
}

private bool dcba_read_key(string path, out uint8[] key) {
    key = new uint8[0];
    if(!FileUtils.test(path, FileTest.EXISTS)) {
        return false;
    }
    try {
        FileUtils.get_data(path, out key);
        return true;
    }
    catch(Error e) {
        printerr(@"Error: $(e.message)\n");
        return false;
    }
}

private void dcba_generate_keypair(out uint8[] public_key, out uint8[] private_key) {
    public_key = new uint8[Sodium.Asymmetric.Signing.PUBLIC_KEY_BYTES];
    private_key = new uint8[Sodium.Asymmetric.Signing.SECRET_KEY_BYTES];
    Sodium.Asymmetric.Signing.generate_keypair(public_key, private_key);
}


/**
 * Validates that the file at `path` is a readable tar.xz archive by
 * listing it through `tar -tf` — the same archive family USM's build
 * artifacts (`build.tar.xz`) and `.dcba` files use.
 */
private void dcba_validate_tar_xz(string path) throws Error {
    if(!FileUtils.test(path, FileTest.EXISTS)) {
        throw new IOError.NOT_FOUND(@"\"$path\" does not exist");
    }
    var process = new Subprocess.newv(
        new string[] { "tar", "-tf", path },
        SubprocessFlags.STDOUT_PIPE | SubprocessFlags.STDERR_PIPE);
    // Drain both pipes so a large archive listing cannot deadlock on a full buffer
    var buffer = new uint8[64 * 1024];
    while(process.get_stdout_pipe().read(buffer) > 0) {
    }
    while(process.get_stderr_pipe().read(buffer) > 0) {
    }
    try {
        process.wait_check();
    }
    catch(Error e) {
        throw new IOError.INVALID_ARGUMENT(@"\"$(Path.get_basename(path))\" is not a readable tar.xz archive\n");
    }
}

private int dcba_repository_add(string artifact_path) {
    var root = dcba_find_root();
    if(root == null) {
        printerr(@"No DCBA repository root found (a directory containing a *.dcbar file, \"keys/\" and \"public/\" at or above the current directory)\n");
        return 1;
    }

    uint8[] public_key;
    uint8[] private_key;
    if(!dcba_load_keys(root, out public_key, out private_key)) {
        return 1;
    }

    var filename = Path.get_basename(artifact_path);
    if(!filename.has_suffix(".dcba")) {
        printerr(@"\"$filename\" is not a .dcba artifact\n");
        return 1;
    }
    var identity = Usm.DcbaArtifact.parse_filename(filename);
    if(identity == null) {
        printerr(@"\"$filename\" does not carry a parseable version, rename it to <name>-<version>.dcba (e.g. app-1.0.0.dcba)\n");
        return 1;
    }

    Invercargill.BinaryData checksum;
    try {
        dcba_validate_tar_xz(artifact_path);
        checksum = calculate_file_checksum(artifact_path);
    }
    catch(Error e) {
        printerr(@"Invalid artifact \"$artifact_path\": $(e.message)\n");
        return 1;
    }

    var public_dir = Path.build_filename(root, "public");
    try {
        foreach(var file in Iterate.directory(public_dir)) {
            if(!file.has_suffix(".dcba")) {
                continue;
            }

            var existing_path = Path.build_filename(public_dir, file);
            var existing_identity = Usm.DcbaArtifact.parse_filename(file);
            var same_artifact = existing_identity != null && ((!)existing_identity).same_artifact((!)identity);
            if(!same_artifact && file != filename) {
                continue;
            }

            if(calculate_file_checksum(existing_path).equals(checksum)) {
                printerr(@"\"$(((!)identity).name)-$(((!)identity).version)\" is already present as \"$file\", nothing to add\n");
                return dcba_rebuild_listing(root, public_key, private_key);
            }

            printerr(@"\"$file\" already exists with different content, remove it first with \"usm dcba repository remove\"\n");
            return 1;
        }

        File.new_for_path(artifact_path).copy(File.new_for_path(Path.build_filename(public_dir, filename)), FileCopyFlags.OVERWRITE);
    }
    catch(Error e) {
        printerr(@"Error: $(e.message)\n");
        return e.code;
    }

    printerr(@"Added \"$(((!)identity).name)-$(((!)identity).version)\" as \"$filename\"\n");
    return dcba_rebuild_listing(root, public_key, private_key);
}

private int dcba_repository_remove(string selector) {
    var root = dcba_find_root();
    if(root == null) {
        printerr("No DCBA repository root found (a directory containing a *.dcbar file, \"keys/\" and \"public/\" at or above the current directory)\n");
        return 1;
    }

    uint8[] public_key;
    uint8[] private_key;
    if(!dcba_load_keys(root, out public_key, out private_key)) {
        return 1;
    }

    var public_dir = Path.build_filename(root, "public");
    var matches = new Vector<string>();

    try {
        var filename = selector.has_suffix(".dcba") ? selector : @"$selector.dcba";
        if(FileUtils.test(Path.build_filename(public_dir, filename), FileTest.EXISTS)) {
            matches.add(filename);
        }
        else {
            foreach(var file in Iterate.directory(public_dir)) {
                if(file.has_suffix(".dcba") && file.has_prefix(@"$selector-")) {
                    matches.add(file);
                }
            }
        }

        if(matches.length == 0) {
            printerr(@"No artifact matching \"$selector\" found in \"public/\"\n");
            return 1;
        }
        if(matches.length > 1) {
            printerr(@"\"$selector\" matches multiple artifacts, specify an exact filename:\n");
            foreach(var file in matches) {
                printerr(@"\t$file\n");
            }
            return 1;
        }

        var match = matches.first_or_default();
        File.new_for_path(Path.build_filename(public_dir, match)).delete();
    }
    catch(Error e) {
        printerr(@"Error: $(e.message)\n");
        return e.code;
    }

    printerr(@"Removed \"$(matches.first_or_default())\"\n");
    return dcba_rebuild_listing(root, public_key, private_key);
}

private int dcba_repository_list(string? repository_file) {
    try {
        Usm.Repository repository;
        Usm.DcbaListing listing;

        if(repository_file != null) {
            repository = new Usm.Repository.from_file(repository_file);
            var stream = File.new_for_uri(repository.url + "/" + Usm.RepositoryClient.DCBA_LISTING_NAME).read();
            listing = new Usm.DcbaListing.from_stream(new DataInputStream(stream));
        }
        else {
            var root = dcba_find_root();
            if(root == null) {
                printerr("No DCBA repository root found (a directory containing a *.dcbar file, \"keys/\" and \"public/\" at or above the current directory)\n");
                return 1;
            }

            var repo_file = dcba_find_repo_file(root);
            if(repo_file == null) {
                printerr("No *.dcbar file found in the DCBA repository root\n");
                return 1;
            }
            repository = new Usm.Repository.from_file(repo_file);

            var listing_path = Path.build_filename(root, "public", Usm.RepositoryClient.DCBA_LISTING_NAME);
            if(!FileUtils.test(listing_path, FileTest.EXISTS)) {
                printerr("No ARTIFACTS.usml in \"public/\", add an artifact with `usm dcba repository add` or run `usm dcba repository build-list` first\n");
                return 1;
            }
            listing = new Usm.DcbaListing.from_stream(new DataInputStream(File.new_for_path(listing_path).read()));
        }

        if(!listing.valid_signature(repository.key)) {
            printerr(@"Artifact listing signature verification failed for \"$(repository.name)\"\n");
            return 1;
        }

        foreach(var entry in listing.entries) {
            print(@"$(entry.name) $(entry.version) $(dcba_format_size(entry.size))\n");
        }
    }
    catch(Error e) {
        printerr(@"Error: $(e.message)\n");
        return e.code;
    }

    return 0;
}

/** Human-readable byte count (`12.0 MiB`, `512 B`); size 0 (listings predating the field) prints `?`. */
private string dcba_format_size(uint64 size) {
    if(size == 0) {
        return "?";
    }
    double value = size;
    foreach(var unit in new string[] { "B", "KiB", "MiB", "GiB", "TiB" }) {
        if(value < 1024.0 || unit == "TiB") {
            return unit == "B" ? @"$size B" : "%.1f %s".printf(value, unit);
        }
        value /= 1024.0;
    }
    assert_not_reached();
}

private int dcba_repository_verify(string repository_file) {
    try {
        var json = new InvercargillJson.JsonElement.from_file(repository_file);
        var repository = Usm.Repository.get_mapper().materialise(json.as<InvercargillJson.JsonObject>());

        var stream = File.new_for_uri(repository.url + "/" + Usm.RepositoryClient.DCBA_LISTING_NAME).read();
        var listing = new Usm.DcbaListing.from_stream(new DataInputStream(stream));

        if(!listing.valid_signature(repository.key)) {
            printerr("Invalid signature.\n");
            return 1;
        }
        printerr(@"Valid signature for \"$(repository.name)\" artifact listing.\n");

        var verified = 0;
        var failures = 0;
        foreach(var entry in listing.entries) {
            var artifact_uri = repository.url.has_suffix("/") ? repository.url + entry.path : repository.url + "/" + entry.path;
            try {
                var artifact_stream = File.new_for_uri(artifact_uri).read();
                if(repository_stream_checksum(artifact_stream).equals(entry.sha512sum)) {
                    printerr(@"Verified \"$(entry.path)\"\n");
                    verified++;
                }
                else {
                    printerr(@"Checksum mismatch for \"$(entry.path)\"\n");
                    failures++;
                }
            }
            catch(Error e) {
                printerr(@"Could not verify \"$(entry.path)\": $(e.message)\n");
                failures++;
            }
        }

        if(failures > 0) {
            printerr(@"$failures of $(verified + failures) artifact(s) failed verification\n");
            return 1;
        }
        printerr(@"Deep-verified $verified artifact(s) in \"$(repository.name)\"\n");
        return 0;
    }
    catch(Error e) {
        printerr(@"Error: $(e.message)\n");
        return e.code;
    }
}

private uint64 dcba_modification_time(string path) throws Error {
    return File.new_for_path(path).query_info(FileAttribute.TIME_MODIFIED, FileQueryInfoFlags.NONE).get_attribute_uint64(FileAttribute.TIME_MODIFIED);
}

private int dcba_repository_publish(string destination_path) {
    var root = dcba_find_root();
    if(root == null) {
        printerr("No DCBA repository root found (a directory containing a *.dcbar file, \"keys/\" and \"public/\" at or above the current directory)\n");
        return 1;
    }

    var repo_file = dcba_find_repo_file(root);
    if(repo_file == null) {
        printerr("No *.dcbar file found in the DCBA repository root\n");
        return 1;
    }

    var public_dir = Path.build_filename(root, "public");
    var listing_path = Path.build_filename(public_dir, Usm.RepositoryClient.DCBA_LISTING_NAME);
    if(!FileUtils.test(listing_path, FileTest.EXISTS)) {
        printerr("No ARTIFACTS.usml in \"public/\", add an artifact with `usm dcba repository add` or run `usm dcba repository build-list` first\n");
        return 1;
    }

    try {
        var destination = File.new_for_path(destination_path);
        if(!destination.query_exists()) {
            destination.make_directory_with_parents();
        }

        File.new_for_path(repo_file).copy(destination.get_child(Path.get_basename(repo_file)), FileCopyFlags.OVERWRITE);
        printerr(@"Published \"$(Path.get_basename(repo_file))\"\n");

        File.new_for_path(listing_path).copy(destination.get_child(Usm.RepositoryClient.DCBA_LISTING_NAME), FileCopyFlags.OVERWRITE);
        printerr(@"Published \"ARTIFACTS.usml\"\n");

        foreach(var file in Iterate.directory(public_dir)) {
            if(!file.has_suffix(".dcba")) {
                continue;
            }

            var source_path = Path.build_filename(public_dir, file);
            var destination_file = destination.get_child(file);
            if(destination_file.query_exists()
                && dcba_modification_time(destination_file.get_path()) >= dcba_modification_time(source_path)) {
                printerr(@"Skipped \"$file\" (up to date)\n");
                continue;
            }

            File.new_for_path(source_path).copy(destination_file, FileCopyFlags.OVERWRITE);
            printerr(@"Published \"$file\"\n");
        }
    }
    catch(Error e) {
        printerr(@"Error: $(e.message)\n");
        return e.code;
    }

    return 0;
}

private int dcba_repository_build_list() {
    var root = dcba_find_root();
    if(root == null) {
        root = Environment.get_current_dir();
    }

    uint8[] public_key;
    uint8[] private_key;
    if(!dcba_load_keys(root, out public_key, out private_key)) {
        return 1;
    }

    return dcba_rebuild_listing(root, public_key, private_key);
}

/**
 * Rebuilds and re-signs `public/ARTIFACTS.usml` for the DCBA repository
 * rooted at `root`, the shared implementation behind `add`, `remove`
 * and `build-list` — a thin CLI shell over
 * {@link Usm.DcbaListingWriter.write}.
 */
private int dcba_rebuild_listing(string root, uint8[] public_key, uint8[] private_key) {
    try {
        var public_dir = Path.build_filename(root, "public");
        var entries = new Vector<Usm.DcbaListingEntry>();
        foreach(var file in Iterate.directory(public_dir)) {
            var path = Path.build_filename(public_dir, file);
            if(!path.has_suffix(".dcba")) {
                continue;
            }

            var identity = Usm.DcbaArtifact.parse_filename(file);
            if(identity == null) {
                throw new IOError.INVALID_ARGUMENT(@"\"$file\" does not carry a parseable version, rename it to <name>-<version>.dcba\n");
            }

            entries.add(new Usm.DcbaListingEntry() {
                name = ((!)identity).name,
                version = ((!)identity).version,
                path = file,
                sha512sum = calculate_file_checksum(path),
                size = (uint64)File.new_for_path(path).query_info(FileAttribute.STANDARD_SIZE, FileQueryInfoFlags.NONE).get_size()
            });
        }

        Usm.DcbaListingWriter.write(Path.build_filename(public_dir, Usm.RepositoryClient.DCBA_LISTING_NAME), entries, public_key, private_key);
    }
    catch(Error e) {
        printerr(@"Error: $(e.message)\n");
        return e.code;
    }
    return 0;
}
namespace Usm.Cli {

    /**
     * `usm dcba add-repo <url-or-path>` — adds a **DCBA** build-artifact
     * repository (`.dcbar` descriptor) to the configuration; the
     * {@link AddRepoRunner} shell around the `.dcbar`
     * {@link AddRepoDefinition}. Mirrors `usm add-repo` verb for verb,
     * refreshing `ARTIFACTS.usml` instead of `PACKAGES.usml`.
     */
    public class DcbaAddRepo {

        public static int run(string[] args) {
            var definition = new AddRepoDefinition() {
                extension = ".dcbar",
                refresh_noun = "artifacts",
                usage_line = "usm dcba add-repo [-y|--yes] <url-or-path>",
                followup_line = "Pre-built artifacts are now used automatically by usm install and usm downgrade\n",
                refresh = destination => {
                    var state = new Usm.SystemState(paths);
                    var added = new Usm.Repository.from_file(destination);
                    state.refresh_dcba_list(added);
                    var listing = state.get_latest_dcba_list(added);
                    return listing == null ? 0 : (int)listing.entries.count();
                }
            };
            return AddRepoRunner.run(args, definition, 3);
        }
    }
}