Bug fixes

src/lib/Metadata.vala

@@ -27,8 +27,13 @@ namespace Ppub {
             return null;
         }
 
-        public void set_value(string name, string value) {
-            entries.set(name, value);
+        public void set_value(string name, string? value) {
+            if(value == null) {
+                entries.unset(name);
+            }
+            else {
+                entries.set(name, value);
+            }
         }
 
 

src/lib/Ppcl.vala

@@ -577,7 +577,7 @@ namespace Ppub {
 
         public string to_string() {
             var urlencoded = Base64.encode(collection_id);
-            urlencoded = urlencoded[0:urlencoded.length-2];
+            urlencoded = urlencoded[0:urlencoded.length-1];
             urlencoded = urlencoded.replace("/", "-");
             var url = @"ppcl://$urlencoded";
             if(via_server_record != null) {
@@ -613,6 +613,9 @@ namespace Ppub {
             }
 
             var path = uri.get_path();
+            if(path.has_prefix("/")) {
+                path = path.substring(1);
+            }
             var parts = path.split("/", 2);
             if(parts[0] != "") {
                 publication = parts[0];

src/lib/Ppub.vala

@@ -3,32 +3,20 @@ using Gee;
 
 namespace Ppub {
 
-    public class Publication {
-
+    public abstract class Publication {
         public const string MAGIC = "ppub";
 
-        public Enumerable<Asset> assets {get; private set;}
-
-        public Metadata metadata {get; private set;}
-
-        private uint64 asset_offset;
-        private File ppub_file;
-
-        public Publication(string path) throws Error {
-            ppub_file = File.new_for_path(path);
-            from_stream(ppub_file.read());
-            metadata = new Metadata.from_stream(read_asset("metadata"));
-        }
+        public Enumerable<Asset> assets {get; protected set;}
+        public Metadata metadata {get; protected set;}        
 
-        private void from_stream(InputStream stream) throws Error {
-            var dis = new DataInputStream(stream);
-            var magic = dis.read_line();
+        protected void read_header(DataInputStream stream) throws Error {
+            var magic = stream.read_line();
             if(magic != MAGIC) {
                 throw new IOError.INVALID_DATA("Invalid magic number");
             }
 
-            var asset_index_size = int.parse(dis.read_line());
-            var asset_index_data = ((string)dis.read_bytes(asset_index_size).get_data()).split("\n");
+            var asset_index_size = int.parse(stream.read_line());
+            var asset_index_data = ((string)stream.read_bytes(asset_index_size).get_data()).split("\n");
             
             var asset_seq = new Series<Asset>();
             foreach (var index_data in asset_index_data) {
@@ -38,15 +26,39 @@ namespace Ppub {
             }
             
             assets = asset_seq;
-            asset_offset = dis.tell();
-            dis.close();
         }
 
-        public Asset? get_asset(string name) {
+        public virtual Asset? get_asset(string name) {
             return assets.where(a => a.name == name).first_or_default();
         }
 
-        public InputStream? read_asset(string name) throws Error {
+        public virtual Asset get_default_asset() {
+            return assets.skip(1).first_or_default();
+        }
+
+        public abstract InputStream? read_asset(string name) throws Error;
+        public abstract async InputStream? read_asset_async(string name) throws Error;
+    }
+
+    public class FilePublication : Publication {
+
+        private uint64 asset_offset;
+        private File ppub_file;
+
+        public FilePublication(string path) throws Error {
+            ppub_file = File.new_for_path(path);
+            from_stream(ppub_file.read());
+            metadata = new Metadata.from_stream(read_asset("metadata"));
+        }
+
+        private void from_stream(InputStream stream) throws Error {
+            var dis = new DataInputStream(stream);
+            read_header(dis);
+            asset_offset = dis.tell();
+            dis.close();
+        }
+
+        public override InputStream? read_asset(string name) throws Error {
             var asset = get_asset(name);
             if(asset == null) {
                 return null;
@@ -63,8 +75,21 @@ namespace Ppub {
             return asset_stream;
         }
 
-        public Asset get_default_asset() {
-            return assets.skip(1).first_or_default();
+        public override async InputStream? read_asset_async(string name) throws Error {
+            var asset = get_asset(name);
+            if(asset == null) {
+                return null;
+            }
+
+            var stream = yield ppub_file.read_async(Priority.DEFAULT);
+            var asset_stream = new AssetStream(asset, stream, asset_offset + asset.start_location);
+
+            if(asset.flags.any(f => f == "gzip")) {
+                var converter = new ZlibDecompressor(GLib.ZlibCompressorFormat.GZIP);
+                return new ConverterInputStream(asset_stream, converter);
+            }
+
+            return asset_stream;
         }
 
     }

src/pprf/Client.vala

@@ -96,27 +96,41 @@ namespace Pprf {
             return (Messages.CollectionListing)response;
         }
 
-        public Messages.Asset get_asset(BinaryData identifier, string publication, string asset) throws Error {
+        public Messages.Asset get_asset(BinaryData identifier, string publication, string asset, uint16 flags = 0) throws Error {
             var message = new Messages.GetAsset();
             message.collection_id = identifier;
             message.publication_name = publication;
             message.asset_name = asset;
+            message.flags = flags;
 
             var response = send_message(message);
             assert_expected_type(response, typeof(Messages.Asset));
             return ((Messages.Asset)response);
         }
 
-        public Messages.Publication get_publication(BinaryData identifier, string publication) throws Error {
+        public Messages.Publication get_publication(BinaryData identifier, string publication, uint16 flags = 0) throws Error {
             var message = new Messages.GetPublication();
             message.collection_id = identifier;
             message.publication_name = publication;
+            message.flags = flags;
 
             var response = send_message(message);
             assert_expected_type(response, typeof(Messages.Publication));
             return ((Messages.Publication)response);
         }
 
+        public RemotePublication open_publication(BinaryData identifier, string publication, bool autofetch_default = false) throws Error {
+            var truncate_level = autofetch_default ? Messages.PublicationTruncationFlags.TRUNCATE_AFTER_DEFAULT_ASSET : Messages.PublicationTruncationFlags.TRUNCATE_AFTER_METADATA;
+
+            var result = get_publication(identifier, publication, truncate_level);
+            var stream = result.ppub_data.as_stream();
+            var data = new BinaryData();
+            var size = (size_t)result.ppub_data.body_size;
+            while(data.read_in(stream.read_bytes(size), ref size));
+            print("Got publication\n");
+            return new RemotePublication(this, identifier, publication, data);
+        }
+
         public void publish(BinaryData collection_id, Ppub.CollectionPublication publication, MemberIdentity identity) throws Error {
             var message = new Messages.Publish();
             message.collection_id = collection_id;

src/pprf/Messages/GetAsset.vala

@@ -2,11 +2,16 @@ using Invercargill;
 
 namespace Pprf.Messages {
 
+    public enum GetAssetFlags {
+        RAW = (1 << 0),
+    }
+
     public class GetAsset : CollectionMessage {
 
         public string publication_name { get; set; }
         public string asset_name { get; set; }
-
+        
+        public uint16 flags { get; set; }
         public uint64 skip { get; set; }
         public uint64 take { get; set; }
 
@@ -18,7 +23,7 @@ namespace Pprf.Messages {
             base.deserialise (stream);
             
             // Reserved flags
-            stream.read_uint16();
+            flags = stream.read_uint16();
 
             var pub_name_len = stream.read_byte();
             if(pub_name_len > 0) {
@@ -52,7 +57,7 @@ namespace Pprf.Messages {
         public override void serialise(DataOutputStream stream) throws Error {
             base.serialise(stream);
 
-            stream.put_uint16(0);
+            stream.put_uint16(flags);
             var str_size = publication_name == null ? 0 : publication_name.data.length;
             stream.put_byte(str_size);
             if(str_size > 0) {
@@ -74,6 +79,7 @@ namespace Pprf.Messages {
     
         public string mime_type { get; set; }
         public MessageBody asset_data { get; set; }
+        public uint16 flags { get; set; }
     
         public Asset() {
             message_type = MessageType.ASSET;
@@ -83,7 +89,7 @@ namespace Pprf.Messages {
             base.deserialise (stream);
             
             // Reserved flags
-            stream.read_uint16();
+            flags = stream.read_uint16();
 
             var mime_len = stream.read_byte();
             if(mime_len > 0) {
@@ -109,7 +115,7 @@ namespace Pprf.Messages {
             base.serialise(stream);
 
             // Reserved flags
-            stream.put_uint16(0);
+            stream.put_uint16(flags);
             var str_size = mime_type == null ? 0 : mime_type.data.length;
             stream.put_byte(str_size);
             if(str_size > 0) {

src/pprf/Messages/GetListing.vala

@@ -17,9 +17,9 @@ namespace Pprf.Messages {
 
     public enum ListingRequestFlags {
         TAG = (1 << 0),
-        SEARCH = (2 << 0),
-        SINCE = (3 << 0),
-        INCLUDE_HIDDEN = (4 << 0)
+        SEARCH = (1 << 1),
+        SINCE = (1 << 2),
+        INCLUDE_HIDDEN = (1 << 3)
     }
 
     public class GetListing : CollectionMessage {
@@ -184,7 +184,7 @@ namespace Pprf.Messages {
             }
             if((cols & ListingColumn.DESCRIPTION) != 0) {
                 var size = stream.read_uint16();
-                metadata.author = read_string(stream, size);
+                metadata.description = read_string(stream, size);
             }
             if((cols & ListingColumn.TIMESTAMP) != 0) {
                 var size = stream.read_byte();
@@ -287,7 +287,10 @@ namespace Pprf.Messages {
             }
         }
 
-        private string read_string(DataInputStream stream, int length) throws Error {
+        private string? read_string(DataInputStream stream, int length) throws Error {
+            if(length == 0) {
+                return null;
+            }
             var data = new BinaryData();
             var remaining = (size_t)length;
             while(data.read_in(stream.read_bytes(remaining), ref remaining));

src/pprf/Messages/GetPublication.vala

@@ -2,9 +2,17 @@ using Invercargill;
 
 namespace Pprf.Messages {
 
+    public enum PublicationTruncationFlags {
+        NO_TRUNCATION = 0,
+        TRUNCATE_AFTER_DEFAULT_ASSET = 1,
+        TRUNCATE_AFTER_METADATA = 2,
+        TRUNCATE_AFTER_HEADER = 3
+    }
+
     public class GetPublication : CollectionMessage {
 
         public string publication_name { get; set; }
+        public uint16 flags { get; set; }
 
         public GetPublication() {
             message_type = MessageType.GET_PUBLICATION;
@@ -14,7 +22,7 @@ namespace Pprf.Messages {
             base.deserialise (stream);
             
             // Reserved flags
-            stream.read_uint16();
+            flags = stream.read_uint16();
 
             var pub_name_len = stream.read_byte();
             if(pub_name_len > 0) {
@@ -34,7 +42,7 @@ namespace Pprf.Messages {
         public override void serialise(DataOutputStream stream) throws Error {
             base.serialise(stream);
 
-            stream.put_uint16(0);
+            stream.put_uint16(flags);
             var str_size = publication_name == null ? 0 : publication_name.data.length;
             stream.put_byte(str_size);
             if(str_size > 0) {

src/pprf/RemotePublication.vala

@@ -0,0 +1,94 @@
+using Invercargill;
+
+namespace Pprf {
+
+    public class RemotePublication : Ppub.Publication {
+
+        private uint64 asset_offset;
+        private Bytes initial_cache;
+        private Client client;
+        private BinaryData collection_id;
+        private string publication_name;
+
+        public RemotePublication(Client client, BinaryData collection_id, string name, BinaryData initial_cache) throws Error {
+
+            this.initial_cache = initial_cache.to_bytes();
+            this.collection_id = collection_id;
+            this.publication_name = name;
+            this.client = client;
+
+            var memory_stream = new DataInputStream (new MemoryInputStream.from_bytes(this.initial_cache));
+            read_header(memory_stream);
+            asset_offset = memory_stream.tell();
+
+            metadata = new Ppub.Metadata.from_stream(read_asset("metadata"));
+        }
+
+        public override GLib.InputStream? read_asset (string name) throws Error {
+            var asset = get_asset(name);
+            if(asset == null) {
+                return null;
+            }
+
+            InputStream? stream = null;
+            bool originally_gzipped = asset.flags.any(f => f == "gzip");
+            bool ungzip_required = false;
+
+            // Use cache if possible
+            if(asset.end_location + asset_offset <= initial_cache.length) {
+                stream = new MemoryInputStream.from_bytes (initial_cache.slice ((size_t)(asset.start_location+asset_offset), (size_t)(asset.end_location+asset_offset)));
+                ungzip_required = originally_gzipped;
+            }
+            else {
+                var request_flags = 0;
+
+                // Prefer raw content if already gzipped, we will decode client side to bandwidth
+                if(originally_gzipped) {
+                    request_flags = Messages.GetAssetFlags.RAW;
+                }
+
+                var res = client.get_asset (collection_id, publication_name, name, request_flags);
+                stream = res.asset_data.as_stream();
+
+                if(originally_gzipped) {
+                    // We will need to ungzip if the server returned the data raw
+                    ungzip_required = (res.flags & Messages.GetAssetFlags.RAW) != 0;
+                }
+            }
+
+            if(ungzip_required) {
+                var converter = new ZlibDecompressor(GLib.ZlibCompressorFormat.GZIP);
+                return new ConverterInputStream(stream, converter);
+            }
+
+            return stream;
+        }
+
+        public override async InputStream? read_asset_async (string name) throws Error {
+            SourceFunc callback = read_asset_async.callback;
+            InputStream? result = null;
+            Error? error = null;
+    
+            owned ThreadFunc<bool> run = () => {
+                try {
+                    result = read_asset (name);
+                }
+                catch(Error e) {
+                    error = e;
+                }
+                Idle.add((owned)callback);
+                return true;
+            };
+            new Thread<bool>(null, run);
+    
+            yield;
+            if(error != null) {
+                throw error;
+            }
+            return result;
+        }
+    }
+
+
+    
+}

src/pprf/meson.build

@@ -16,6 +16,7 @@ sources = files('Client.vala')
 sources += files('HttpClient.vala')
 sources += files('Util.vala')
 sources += files('Identity.vala')
+sources += files('RemotePublication.vala')
 sources += files('Messages/Message.vala')
 sources += files('Messages/CollectionMessage.vala')
 sources += files('Messages/UploadSessionMessage.vala')

src/tools/extract/Extract.vala

@@ -14,7 +14,7 @@ public static int main(string[] args) {
 
     print_verbose(@"Opening ppub file at $ppub_file.");
 
-    var ppub = new Ppub.Publication(ppub_file);
+    var ppub = new Ppub.FilePublication(ppub_file);
 
     print_verbose("Reading assets...");
     

src/tools/viewer/Main.vala

@@ -13,7 +13,7 @@ int main (string[] argv) {
             window.width_request = 500;
             window.height_request = 900;
 
-            var ppub = new Ppub.Publication(file.get_path());
+            var ppub = new Ppub.FilePublication(file.get_path());
 
             print(ppub.metadata.title);
             window.title = @"$(ppub.metadata.title) - PPUB Viewer";

src/tools/viewer/meson.build

@@ -10,7 +10,7 @@ dependencies = [
     dependency('invercargill'),
     dependency('gtk4'),
     dependency('gtkcommonmark'),
-    ppub_dep
+    ppub_dep,
 ]
 
 executable('ppub-viewer-simple', sources, dependencies: dependencies)