using Invercargill; using Invercargill.DataStructures; private int dcba_main(string[] args) { if(args.length < 3) { return dcba_usage(); } if(args[2] == "repository") { return dcba_repository_main(args); } if(args[2] == "add-repo") { return Usm.Cli.DcbaAddRepo.run(args); } return dcba_usage(); } private int dcba_usage() { printerr("USAGE:\n\tusm dcba repository \n\tusm dcba add-repo \n"); return 255; } private int dcba_repository_main(string[] args) { if(args.length < 4) { return dcba_repository_usage(); } if(args[2] != "repository") { return dcba_repository_usage(); } if(args[3] == "init") { if(args.length != 5) { return dcba_repository_usage(); } return dcba_repository_init(args[4]); } if(args[3] == "add") { if(args.length != 5) { return dcba_repository_usage(); } return dcba_repository_add(args[4]); } if(args[3] == "remove") { if(args.length != 5) { return dcba_repository_usage(); } return dcba_repository_remove(args[4]); } if(args[3] == "list") { if(args.length > 5) { return dcba_repository_usage(); } return dcba_repository_list(args.length == 5 ? args[4] : null); } if(args[3] == "verify") { if(args.length != 5) { return dcba_repository_usage(); } return dcba_repository_verify(args[4]); } if(args[3] == "publish") { if(args.length != 5) { return dcba_repository_usage(); } return dcba_repository_publish(args[4]); } if(args[3] == "build-list") { if(args.length != 4) { return dcba_repository_usage(); } return dcba_repository_build_list(); } return dcba_repository_usage(); } private int dcba_repository_usage() { printerr("USAGE:\n\tusm dcba repository init \n\tusm dcba repository add \n\tusm dcba repository remove \n\tusm dcba repository list [repo.dcbar]\n\tusm dcba repository verify repo.dcbar\n\tusm dcba repository publish \n\tusm dcba repository build-list\n"); return 255; } private int dcba_repository_init(string name) { if(dcba_find_repo_file(Environment.get_current_dir()) != null || FileUtils.test(Path.build_filename("keys", "private-key"), FileTest.EXISTS)) { printerr("A DCBA repository has already been initialised in this directory\n"); return 1; } uint8[] public_key; uint8[] private_key; dcba_generate_keypair(out public_key, out private_key); DirUtils.create("keys", 0700); DirUtils.create("public", 0755); try { FileUtils.set_data(Path.build_filename("keys", "public-key"), public_key); FileUtils.set_data(Path.build_filename("keys", "private-key"), private_key); FileUtils.chmod(Path.build_filename("keys", "private-key"), 0600); var repository = new Usm.Repository(); repository.key = Wrap.byte_array(public_key); repository.name = name; repository.url = @"file://$(Environment.get_current_dir())/public"; repository.summary = @"My $name DCBA repository"; var properties = Usm.Repository.get_mapper().map_from(repository); var json = new InvercargillJson.JsonElement.from_properties(properties); json.write_to_file(@"$name.dcbar"); var gitignore = "keys/private*\n"; if(FileUtils.test(".gitignore", FileTest.EXISTS)) { string existing; FileUtils.get_contents(".gitignore", out existing); if(!existing.has_suffix("\n")) { existing += "\n"; } if(existing.index_of("keys/private*") == -1) { FileUtils.set_contents(".gitignore", existing + gitignore); } } else { FileUtils.set_contents(".gitignore", gitignore); } } catch(Error e) { printerr(@"Error: $(e.message)\n"); return e.code; } printerr("DCBA repository initialised, add artifacts with `usm dcba repository add ` then publish them with `usm dcba repository publish `\n"); return 0; } /** * Locates the nearest DCBA repository root: the first directory at or * above the current directory holding a `*.dcbar` file, a `keys/` * directory and a `public/` directory — the `.dcbar`-suffix counterpart * of `repository_find_root`, deliberately separate so a USM source * repository and a DCBA repository can never be mistaken for one * another even when nested in the same tree. */ private string? dcba_find_root() { var directory = Environment.get_current_dir(); while(true) { if(dcba_find_repo_file(directory) != null && FileUtils.test(Path.build_filename(directory, "keys"), FileTest.IS_DIR) && FileUtils.test(Path.build_filename(directory, "public"), FileTest.IS_DIR)) { return directory; } var parent = Path.get_dirname(directory); if(parent == directory) { return null; } directory = parent; } } /** * Finds the first `*.dcbar` file directly inside `root`, or null when * there is none. */ private string? dcba_find_repo_file(string root) { try { foreach(var file in Iterate.directory(root)) { if(file.has_suffix(".dcbar")) { return Path.build_filename(root, file); } } } catch(Error e) { return null; } return null; } /** * Loads the signing keys from `keys/public-key` and `keys/private-key` in * the DCBA repository root, falling back to the legacy loose * `public-key` and `private-key` files there. */ private bool dcba_load_keys(string root, out uint8[] public_key, out uint8[] private_key) { public_key = new uint8[0]; private_key = new uint8[0]; var key_dir = Path.build_filename(root, "keys"); if(dcba_read_key(Path.build_filename(key_dir, "public-key"), out public_key) && dcba_read_key(Path.build_filename(key_dir, "private-key"), out private_key)) { return true; } if(dcba_read_key(Path.build_filename(root, "public-key"), out public_key) && dcba_read_key(Path.build_filename(root, "private-key"), out private_key)) { return true; } printerr("No signing keys found in \"keys/\" (or as legacy \"public-key\"/\"private-key\") within the DCBA repository root\n"); return false; } private bool dcba_read_key(string path, out uint8[] key) { key = new uint8[0]; if(!FileUtils.test(path, FileTest.EXISTS)) { return false; } try { FileUtils.get_data(path, out key); return true; } catch(Error e) { printerr(@"Error: $(e.message)\n"); return false; } } private void dcba_generate_keypair(out uint8[] public_key, out uint8[] private_key) { public_key = new uint8[Sodium.Asymmetric.Signing.PUBLIC_KEY_BYTES]; private_key = new uint8[Sodium.Asymmetric.Signing.SECRET_KEY_BYTES]; Sodium.Asymmetric.Signing.generate_keypair(public_key, private_key); } /** * Validates that the file at `path` is a readable tar.xz archive by * listing it through `tar -tf` — the same archive family USM's build * artifacts (`build.tar.xz`) and `.dcba` files use. */ private void dcba_validate_tar_xz(string path) throws Error { if(!FileUtils.test(path, FileTest.EXISTS)) { throw new IOError.NOT_FOUND(@"\"$path\" does not exist"); } var process = new Subprocess.newv( new string[] { "tar", "-tf", path }, SubprocessFlags.STDOUT_PIPE | SubprocessFlags.STDERR_PIPE); // Drain both pipes so a large archive listing cannot deadlock on a full buffer var buffer = new uint8[64 * 1024]; while(process.get_stdout_pipe().read(buffer) > 0) { } while(process.get_stderr_pipe().read(buffer) > 0) { } try { process.wait_check(); } catch(Error e) { throw new IOError.INVALID_ARGUMENT(@"\"$(Path.get_basename(path))\" is not a readable tar.xz archive\n"); } } private int dcba_repository_add(string artifact_path) { var root = dcba_find_root(); if(root == null) { printerr(@"No DCBA repository root found (a directory containing a *.dcbar file, \"keys/\" and \"public/\" at or above the current directory)\n"); return 1; } uint8[] public_key; uint8[] private_key; if(!dcba_load_keys(root, out public_key, out private_key)) { return 1; } var filename = Path.get_basename(artifact_path); if(!filename.has_suffix(".dcba")) { printerr(@"\"$filename\" is not a .dcba artifact\n"); return 1; } var identity = Usm.DcbaArtifact.parse_filename(filename); if(identity == null) { printerr(@"\"$filename\" does not carry a parseable version, rename it to -.dcba (e.g. app-1.0.0.dcba)\n"); return 1; } Invercargill.BinaryData checksum; try { dcba_validate_tar_xz(artifact_path); checksum = calculate_file_checksum(artifact_path); } catch(Error e) { printerr(@"Invalid artifact \"$artifact_path\": $(e.message)\n"); return 1; } var public_dir = Path.build_filename(root, "public"); try { foreach(var file in Iterate.directory(public_dir)) { if(!file.has_suffix(".dcba")) { continue; } var existing_path = Path.build_filename(public_dir, file); var existing_identity = Usm.DcbaArtifact.parse_filename(file); var same_artifact = existing_identity != null && ((!)existing_identity).same_artifact((!)identity); if(!same_artifact && file != filename) { continue; } if(calculate_file_checksum(existing_path).equals(checksum)) { printerr(@"\"$(((!)identity).name)-$(((!)identity).version)\" is already present as \"$file\", nothing to add\n"); return dcba_rebuild_listing(root, public_key, private_key); } printerr(@"\"$file\" already exists with different content, remove it first with \"usm dcba repository remove\"\n"); return 1; } File.new_for_path(artifact_path).copy(File.new_for_path(Path.build_filename(public_dir, filename)), FileCopyFlags.OVERWRITE); } catch(Error e) { printerr(@"Error: $(e.message)\n"); return e.code; } printerr(@"Added \"$(((!)identity).name)-$(((!)identity).version)\" as \"$filename\"\n"); return dcba_rebuild_listing(root, public_key, private_key); } private int dcba_repository_remove(string selector) { var root = dcba_find_root(); if(root == null) { printerr("No DCBA repository root found (a directory containing a *.dcbar file, \"keys/\" and \"public/\" at or above the current directory)\n"); return 1; } uint8[] public_key; uint8[] private_key; if(!dcba_load_keys(root, out public_key, out private_key)) { return 1; } var public_dir = Path.build_filename(root, "public"); var matches = new Vector(); try { var filename = selector.has_suffix(".dcba") ? selector : @"$selector.dcba"; if(FileUtils.test(Path.build_filename(public_dir, filename), FileTest.EXISTS)) { matches.add(filename); } else { foreach(var file in Iterate.directory(public_dir)) { if(file.has_suffix(".dcba") && file.has_prefix(@"$selector-")) { matches.add(file); } } } if(matches.length == 0) { printerr(@"No artifact matching \"$selector\" found in \"public/\"\n"); return 1; } if(matches.length > 1) { printerr(@"\"$selector\" matches multiple artifacts, specify an exact filename:\n"); foreach(var file in matches) { printerr(@"\t$file\n"); } return 1; } var match = matches.first_or_default(); File.new_for_path(Path.build_filename(public_dir, match)).delete(); } catch(Error e) { printerr(@"Error: $(e.message)\n"); return e.code; } printerr(@"Removed \"$(matches.first_or_default())\"\n"); return dcba_rebuild_listing(root, public_key, private_key); } private int dcba_repository_list(string? repository_file) { try { Usm.Repository repository; Usm.DcbaListing listing; if(repository_file != null) { repository = new Usm.Repository.from_file(repository_file); var stream = File.new_for_uri(repository.url + "/" + Usm.RepositoryClient.DCBA_LISTING_NAME).read(); listing = new Usm.DcbaListing.from_stream(new DataInputStream(stream)); } else { var root = dcba_find_root(); if(root == null) { printerr("No DCBA repository root found (a directory containing a *.dcbar file, \"keys/\" and \"public/\" at or above the current directory)\n"); return 1; } var repo_file = dcba_find_repo_file(root); if(repo_file == null) { printerr("No *.dcbar file found in the DCBA repository root\n"); return 1; } repository = new Usm.Repository.from_file(repo_file); var listing_path = Path.build_filename(root, "public", Usm.RepositoryClient.DCBA_LISTING_NAME); if(!FileUtils.test(listing_path, FileTest.EXISTS)) { printerr("No ARTIFACTS.usml in \"public/\", add an artifact with `usm dcba repository add` or run `usm dcba repository build-list` first\n"); return 1; } listing = new Usm.DcbaListing.from_stream(new DataInputStream(File.new_for_path(listing_path).read())); } if(!listing.valid_signature(repository.key)) { printerr(@"Artifact listing signature verification failed for \"$(repository.name)\"\n"); return 1; } foreach(var entry in listing.entries) { print(@"$(entry.name) $(entry.version) $(dcba_format_size(entry.size))\n"); } } catch(Error e) { printerr(@"Error: $(e.message)\n"); return e.code; } return 0; } /** Human-readable byte count (`12.0 MiB`, `512 B`); size 0 (listings predating the field) prints `?`. */ private string dcba_format_size(uint64 size) { if(size == 0) { return "?"; } double value = size; foreach(var unit in new string[] { "B", "KiB", "MiB", "GiB", "TiB" }) { if(value < 1024.0 || unit == "TiB") { return unit == "B" ? @"$size B" : "%.1f %s".printf(value, unit); } value /= 1024.0; } assert_not_reached(); } private int dcba_repository_verify(string repository_file) { try { var json = new InvercargillJson.JsonElement.from_file(repository_file); var repository = Usm.Repository.get_mapper().materialise(json.as()); var stream = File.new_for_uri(repository.url + "/" + Usm.RepositoryClient.DCBA_LISTING_NAME).read(); var listing = new Usm.DcbaListing.from_stream(new DataInputStream(stream)); if(!listing.valid_signature(repository.key)) { printerr("Invalid signature.\n"); return 1; } printerr(@"Valid signature for \"$(repository.name)\" artifact listing.\n"); var verified = 0; var failures = 0; foreach(var entry in listing.entries) { var artifact_uri = repository.url.has_suffix("/") ? repository.url + entry.path : repository.url + "/" + entry.path; try { var artifact_stream = File.new_for_uri(artifact_uri).read(); if(repository_stream_checksum(artifact_stream).equals(entry.sha512sum)) { printerr(@"Verified \"$(entry.path)\"\n"); verified++; } else { printerr(@"Checksum mismatch for \"$(entry.path)\"\n"); failures++; } } catch(Error e) { printerr(@"Could not verify \"$(entry.path)\": $(e.message)\n"); failures++; } } if(failures > 0) { printerr(@"$failures of $(verified + failures) artifact(s) failed verification\n"); return 1; } printerr(@"Deep-verified $verified artifact(s) in \"$(repository.name)\"\n"); return 0; } catch(Error e) { printerr(@"Error: $(e.message)\n"); return e.code; } } private uint64 dcba_modification_time(string path) throws Error { return File.new_for_path(path).query_info(FileAttribute.TIME_MODIFIED, FileQueryInfoFlags.NONE).get_attribute_uint64(FileAttribute.TIME_MODIFIED); } private int dcba_repository_publish(string destination_path) { var root = dcba_find_root(); if(root == null) { printerr("No DCBA repository root found (a directory containing a *.dcbar file, \"keys/\" and \"public/\" at or above the current directory)\n"); return 1; } var repo_file = dcba_find_repo_file(root); if(repo_file == null) { printerr("No *.dcbar file found in the DCBA repository root\n"); return 1; } var public_dir = Path.build_filename(root, "public"); var listing_path = Path.build_filename(public_dir, Usm.RepositoryClient.DCBA_LISTING_NAME); if(!FileUtils.test(listing_path, FileTest.EXISTS)) { printerr("No ARTIFACTS.usml in \"public/\", add an artifact with `usm dcba repository add` or run `usm dcba repository build-list` first\n"); return 1; } try { var destination = File.new_for_path(destination_path); if(!destination.query_exists()) { destination.make_directory_with_parents(); } File.new_for_path(repo_file).copy(destination.get_child(Path.get_basename(repo_file)), FileCopyFlags.OVERWRITE); printerr(@"Published \"$(Path.get_basename(repo_file))\"\n"); File.new_for_path(listing_path).copy(destination.get_child(Usm.RepositoryClient.DCBA_LISTING_NAME), FileCopyFlags.OVERWRITE); printerr(@"Published \"ARTIFACTS.usml\"\n"); foreach(var file in Iterate.directory(public_dir)) { if(!file.has_suffix(".dcba")) { continue; } var source_path = Path.build_filename(public_dir, file); var destination_file = destination.get_child(file); if(destination_file.query_exists() && dcba_modification_time(destination_file.get_path()) >= dcba_modification_time(source_path)) { printerr(@"Skipped \"$file\" (up to date)\n"); continue; } File.new_for_path(source_path).copy(destination_file, FileCopyFlags.OVERWRITE); printerr(@"Published \"$file\"\n"); } } catch(Error e) { printerr(@"Error: $(e.message)\n"); return e.code; } return 0; } private int dcba_repository_build_list() { var root = dcba_find_root(); if(root == null) { root = Environment.get_current_dir(); } uint8[] public_key; uint8[] private_key; if(!dcba_load_keys(root, out public_key, out private_key)) { return 1; } return dcba_rebuild_listing(root, public_key, private_key); } /** * Rebuilds and re-signs `public/ARTIFACTS.usml` for the DCBA repository * rooted at `root`, the shared implementation behind `add`, `remove` * and `build-list` — a thin CLI shell over * {@link Usm.DcbaListingWriter.write}. */ private int dcba_rebuild_listing(string root, uint8[] public_key, uint8[] private_key) { try { var public_dir = Path.build_filename(root, "public"); var entries = new Vector(); foreach(var file in Iterate.directory(public_dir)) { var path = Path.build_filename(public_dir, file); if(!path.has_suffix(".dcba")) { continue; } var identity = Usm.DcbaArtifact.parse_filename(file); if(identity == null) { throw new IOError.INVALID_ARGUMENT(@"\"$file\" does not carry a parseable version, rename it to -.dcba\n"); } entries.add(new Usm.DcbaListingEntry() { name = ((!)identity).name, version = ((!)identity).version, path = file, sha512sum = calculate_file_checksum(path), size = (uint64)File.new_for_path(path).query_info(FileAttribute.STANDARD_SIZE, FileQueryInfoFlags.NONE).get_size() }); } Usm.DcbaListingWriter.write(Path.build_filename(public_dir, Usm.RepositoryClient.DCBA_LISTING_NAME), entries, public_key, private_key); } catch(Error e) { printerr(@"Error: $(e.message)\n"); return e.code; } return 0; } namespace Usm.Cli { /** * `usm dcba add-repo ` — adds a **DCBA** build-artifact * repository (`.dcbar` descriptor) to the configuration; the * {@link AddRepoRunner} shell around the `.dcbar` * {@link AddRepoDefinition}. Mirrors `usm add-repo` verb for verb, * refreshing `ARTIFACTS.usml` instead of `PACKAGES.usml`. */ public class DcbaAddRepo { public static int run(string[] args) { var definition = new AddRepoDefinition() { extension = ".dcbar", refresh_noun = "artifacts", usage_line = "usm dcba add-repo [-y|--yes] ", followup_line = "Pre-built artifacts are now used automatically by usm install and usm downgrade\n", refresh = destination => { var state = new Usm.SystemState(paths); var added = new Usm.Repository.from_file(destination); state.refresh_dcba_list(added); var listing = state.get_latest_dcba_list(added); return listing == null ? 0 : (int)listing.entries.count(); } }; return AddRepoRunner.run(args, definition, 3); } } }