Implement create tool

.gitignore

@@ -3,4 +3,5 @@ meson-*
 /tools
 *.ninja
 .ninja_*
-compile_commands.json
+compile_commands.json
+/extract

src/lib/Builder.vala

@@ -5,27 +5,36 @@ namespace Ppub {
 
     public class Builder {
 
-        private Invercargill.Sequence<BuilderAsset> assets {get; set;}
+        private Invercargill.Sequence<BuilderAsset> assets = new Invercargill.Sequence<BuilderAsset>();
 
         private uint64 position = 0;
 
         public void add_metadata(Metadata metadata) {
             var data = (uint8[])metadata.to_string().to_utf8();
             var mis = new MemoryInputStream.from_data(data);
-            add_asset("metadata", "application/x-ppub-metadata", mis, data.length);
+            var compress = new CompressionInfo(mis);
+            mis.seek(0, SeekType.SET);
+            add_asset("metadata", "application/x-ppub-metadata", mis, empty<string>(), compress);
         }
 
-        public void add_asset(string name, string mimetype, InputStream stream, uint64 size) {
+        public void add_asset(string name, string mimetype, InputStream stream, Enumerable<string> flags, CompressionInfo compression) {
 
             lock(assets) {
                 var asset = new BuilderAsset();
                 asset.name = name;
                 asset.start_location = position;
-                asset.end_location = position + size;
+                asset.end_location = position + compression.size_in_ppub;
                 asset.mimetype = mimetype;
                 asset.stream = stream;
+
+                // Exclude any gzip flags, as we will add that later if we are compressing the content
+                asset.flags = flags.where(f => f != "gzip").to_collection();
+
+                if(compression.compress) {
+                    asset.flags.add("gzip");
+                }
     
-                position += size;
+                position += compression.size_in_ppub;
                 assets.add(asset);
             }
 
@@ -33,21 +42,28 @@ namespace Ppub {
 
         public void write(OutputStream stream) throws IOError {
 
-            var index_entries = assets.select<string>(a => a.to_string() + "\n").to_array();
-            var index_data = (uint8[])string.join("", index_entries).to_utf8();
-            var header_data = (uint8[])(@"$(index_data.length)\n".to_utf8());
+            var index_entries = assets.to_string(a => a.to_string());
+
+            print(index_entries);
+            var index_data = (uint8[])index_entries.to_utf8();
+            var header_data = (uint8[])(@"$(Publication.MAGIC)\n$(index_data.length)\n".to_utf8());
 
             stream.write(header_data);
             stream.write(index_data);
 
             foreach (var asset in assets.as_iterable()) {
-                stream.splice(asset.stream, OutputStreamSpliceFlags.CLOSE_SOURCE);
+                if(asset.flags.contains("gzip")) {
+                    var compressor = new ZlibCompressor(ZlibCompressorFormat.GZIP, 9);
+                    var conv_stream = new ConverterInputStream(asset.stream, compressor);
+                    stream.splice(conv_stream, OutputStreamSpliceFlags.CLOSE_SOURCE);
+                }
+                else {
+                    stream.splice(asset.stream, OutputStreamSpliceFlags.CLOSE_SOURCE);
+                }
             }
 
         }
 
-
-
     }
 
     public class BuilderAsset : Asset {
@@ -56,4 +72,52 @@ namespace Ppub {
 
     }
 
+    public class CompressionInfo {
+
+        public bool compress {get; private set;}
+        public uint64 uncompressed_size {get; private set;}
+        public uint64 compressed_size {get; private set;}
+
+        public CompressionInfo.uncompressed(uint64 size) {
+            compress = false;
+            uncompressed_size = size;
+        }
+
+        public CompressionInfo(InputStream stream, bool force = false) throws IOError {
+            var uncompressed_counter = new InputStreamMonitor(stream);
+            var compressor = new ZlibCompressor(ZlibCompressorFormat.GZIP, 9);
+            var conv_stream = new ConverterInputStream(uncompressed_counter, compressor);
+            var compressed_counter = new InputStreamMonitor(conv_stream);
+            
+            compressed_counter.count_to_end();
+
+            uncompressed_size = uncompressed_counter.bytes_read;
+            compressed_size = compressed_counter.bytes_read;
+
+            compress = force || uncompressed_size > compressed_size;
+        }
+
+        public uint64 size_in_ppub {
+            get {
+                return compress ? compressed_size : uncompressed_size;
+            }
+        }
+
+    }
+
+    public static string guess_mimetype(string name, uint8[] sample) {
+        if(name == "metadata") {
+            return "application/x-ppub-metadata";
+        }
+        if(name.has_suffix(".ppvm")) {
+            return "application/x-ppvm";
+        }
+
+        bool result_uncertain;
+        var content_type = ContentType.guess(name, sample, out result_uncertain);
+        var mime = ContentType.get_mime_type(content_type);
+
+        return mime ?? "application/octet-stream";
+    }
+
 }

src/lib/Metadata.vala

@@ -1,3 +1,4 @@
+using Invercargill;
 
 namespace Ppub {
 
@@ -76,7 +77,7 @@ namespace Ppub {
                 return get_value(TAGS).split(" ");
             }
             set {
-                set_value(TAGS, string.join(" ", value));
+                set_value(TAGS, ate(value).to_string(s => s, " "));
             }
         }
 

src/lib/StreamMonitor.vala

@@ -0,0 +1,38 @@
+namespace Ppub {
+
+    internal class InputStreamMonitor : InputStream {
+
+        private InputStream base_stream;
+
+        public uint64 bytes_read {get; private set;}
+
+        public InputStreamMonitor(InputStream base_stream) {
+            this.base_stream = base_stream;
+            bytes_read = 0;
+        }
+
+        public override bool close (GLib.Cancellable? cancellable) throws IOError {
+            return true;
+        }
+
+        public override ssize_t read (uint8[] buffer, GLib.Cancellable? cancellable) throws IOError {
+            var size = base_stream.read (buffer, cancellable);
+            bytes_read += (uint64)size;
+            return size;
+        }
+
+        public uint64 count_to_end() throws IOError {
+
+            var buffer = new uint8[1048576];
+            while(true) {
+                var size = read(buffer);
+                if(size == 0) {
+                    break;
+                }
+            }
+            return bytes_read;
+        }
+
+    }
+
+}

src/lib/meson.build

@@ -11,6 +11,7 @@ sources += files('Asset.vala')
 sources += files('Metadata.vala')
 sources += files('AssetStream.vala')
 sources += files('Builder.vala')
+sources += files('StreamMonitor.vala')
 
 ppub = shared_library('libppub', sources,
     name_prefix: '',

src/tools/create/Create.vala

@@ -0,0 +1,45 @@
+
+
+static bool verbose = true;
+
+private static void print_verbose(string text) {
+    if (verbose) {
+        print(text);
+    }
+}
+
+public static int main(string[] args) {
+
+    var ppub_file = args[1];
+
+    var builder = new Ppub.Builder();
+
+    foreach (var path in args[2:]) {
+
+        print_verbose(@"Determing '$path' mimetype... ");
+        var file = File.new_for_path(path);
+        var stream = file.read();
+
+        var file_sample = new uint8[1024];
+        stream.read(file_sample);
+        stream.seek(0, SeekType.SET);
+
+        var mimetype = Ppub.guess_mimetype(path, file_sample);
+        print_verbose(@"$mimetype\n");
+
+        print_verbose(@"Determing if '$path' should be compressed... ");
+        var compression_info = new Ppub.CompressionInfo(stream);
+        stream.seek(0, SeekType.SET);
+        print_verbose(compression_info.compress ? @"yes! ($(compression_info.uncompressed_size)b uncompressed > $(compression_info.compressed_size)b compressed)\n" : @"no! ($(compression_info.compressed_size)b compressed > $(compression_info.uncompressed_size)b uncompressed)\n");
+
+        builder.add_asset(file.get_basename(), mimetype, stream, Invercargill.empty<string>(), compression_info);
+
+    }
+
+    print_verbose("Writing ppub...\n");
+
+    var stream = File.new_for_path(ppub_file).create(FileCreateFlags.NONE);
+    builder.write(stream);
+
+    return 0;
+}

src/tools/create/meson.build

@@ -0,0 +1,7 @@
+
+sources = files('Create.vala')
+
+deps = dependencies
+deps += ppub_dep
+
+executable('ppub-create', sources, dependencies: deps, install: true)

src/tools/meson.build

@@ -1,2 +1,3 @@
 subdir('extract')
-subdir('viewer')
+subdir('viewer')
+subdir('create')