PPCL and PPRF refactors

src/lib/Ppcl.vala

@@ -8,32 +8,33 @@ namespace Ppub {
         INVALID_MEMBER,
         INVALID_MEMBER_SIGNATURE,
         INVALID_ID,
-        INVALID_FORMAT
+        INVALID_FORMAT,
+        INVALID_SHARED_SIGNATURE,
+        INVALID_CREDENTIALS
     }
     
     public class Collection {
         public uint8[] id { get; private set; }
         public int serial { get; private set; }
+        public uint8[] shared_signature_key { get; private set; }
         public Vector<string> domains { get; private set; }
         public Vector<CollectionMember> members { get; private set; }
-        public uint8[] auth { get; private set; }
+        public Vector<CollectionAgent> agents { get; private set; }
+        public uint8[] collection_signature { get; private set; }
+        public uint8[] shared_signature { get; private set; }
         public Vector<CollectionPublication> publications { get; private set; }
         public DateTime modified { get; private set; }
 
+        private string authoritative_section;
+        private string shared_section;
+
         private void initialise() {
             domains = new Vector<string>();
             members = new Vector<CollectionMember>();
+            agents = new Vector<CollectionAgent>();
             publications = new Vector<CollectionPublication>();
         }
 
-        public void increment_serial() {
-            serial++;
-        }
-
-        public void touch() {
-            modified = new DateTime.now_local();
-        }
-
         public Collection(out uint8[] private_signing_key) {
             id = new uint8[Sodium.Asymmetric.Signing.PUBLIC_KEY_BYTES];
             private_signing_key = new uint8[Sodium.Asymmetric.Signing.SECRET_KEY_BYTES];
@@ -43,6 +44,9 @@ namespace Ppub {
         }
 
         public Collection.from_stream(DataInputStream stream) throws Error {
+            authoritative_section = "";
+            shared_section = "";
+
             var line = stream.read_line();
             var signed_portion = line;
             var header = line.split(" ");
@@ -74,14 +78,20 @@ namespace Ppub {
                     throw new CollectionError.INVALID_FORMAT(@"Malformed collection entry \"$(entry[0])\"");
                 }
 
+                if(entry[0] == "SSK") {
+                    shared_signature_key = Base64.decode(entry[1]);
+                }
                 if(entry[0] == "MEM") {
                     members.add(new CollectionMember.from_string(entry[1]));
                 }
                 else if(entry[0] == "DOM") {
                     domains.add(entry[1]);
                 }
+                else if(entry[0] == "AGT") {
+                    agents.add(new CollectionAgent.from_string(entry[1]));
+                }
                 else if(entry[0] == "SIG") {
-                    auth = Base64.decode(entry[1]);
+                    collection_signature = Base64.decode(entry[1]);
                     break;
                 }
                 else {
@@ -91,7 +101,10 @@ namespace Ppub {
                 signed_portion += @"\n$line";
             }
 
+            authoritative_section = @"$signed_portion\n$line";
+
             line = stream.read_line();
+            shared_section += line;
             var modified_entry = line.split(" ", 2);
             if(modified_entry.length != 2 || modified_entry[0] != "MOD") {
                 throw new CollectionError.INVALID_FORMAT(@"Malformed or missing last modified date");
@@ -100,18 +113,19 @@ namespace Ppub {
             modified = new DateTime.from_iso8601(modified_entry[1], null);
 
             line = stream.read_line();
+            shared_section += @"\n$line";
             if(line != "" && line != null) {
                 throw new CollectionError.INVALID_FORMAT(@"Line following MOD entry not blank");
             }
 
-            // Verify auth
+            // Verify authoratative section
             var checksum = new BinaryData.from_byte_array(get_string_checksum(signed_portion));
-            var signed_checksum = Sodium.Asymmetric.Signing.verify(auth, id);
+            var signed_checksum = Sodium.Asymmetric.Signing.verify(collection_signature, id);
             if(signed_checksum == null) {
-                throw new CollectionError.INVALID_COLLECTION_SIGNATURE("Could not verify signature portion of collection");
+                throw new CollectionError.INVALID_COLLECTION_SIGNATURE("Could not verify authoratative signature");
             }
             if(!checksum.equals(ate(signed_checksum))) {
-                throw new CollectionError.INVALID_COLLECTION_SIGNATURE("Signature checksum does not match calculated checksum");
+                throw new CollectionError.INVALID_COLLECTION_SIGNATURE("Authoratative signature checksum does not match calculated checksum");
             }
 
             if(line == null) {
@@ -124,35 +138,85 @@ namespace Ppub {
                     break;
                 }
                 var entry = line.split(" ", 2);
-
+                
                 if(entry[0] == "PUB") {
                     publications.add(new CollectionPublication.from_string(entry[1], members));
                 }
+                else if(entry[0] == "SSG") {
+                    shared_signature = Base64.decode(entry[1]);
+                    break;
+                }
                 else {
                     warning(@"Unrecognised entry type $(entry[0]), ignoring");
                 }
+                shared_section += @"\n$line";
+            }
+            
+            if(shared_signature == null) {
+                throw new CollectionError.INVALID_SHARED_SIGNATURE("Shared signature entry missing");
+            }
+
+            // Verify shared signature
+            checksum = get_checksum();
+            signed_checksum = Sodium.Asymmetric.Signing.verify(shared_signature, shared_signature_key);
+            if(signed_checksum == null) {
+                throw new CollectionError.INVALID_SHARED_SIGNATURE("Could not verify shared signature");
+            }
+            if(!checksum.equals(ate(signed_checksum))) {
+                throw new CollectionError.INVALID_SHARED_SIGNATURE("Shared signature checksum does not match calculated checksum");
             }
         }
 
-        public string to_string(uint8[] signing_key) {
-            var strings = domains
-                .select<string>(d => @"DOM $d")
-                .concat(members.select<string>(m => m.to_string()));
-            var data = strings.to_string(s => s, "\n");
-            var pub_str = publications.to_string(p => p.to_string(), "\n");
+        public string authoratative_update(uint8[] authoratative_signing_key) {
+            serial++;
+            modified = new DateTime.now_local();
+
+            // Generate new collection signature/secret
+            var collection_secret = new uint8[Sodium.Asymmetric.Signing.SECRET_KEY_BYTES];
+            shared_signature_key = new uint8[Sodium.Asymmetric.Signing.PUBLIC_KEY_BYTES];
+            Sodium.Asymmetric.Signing.generate_keypair(shared_signature_key, collection_secret);
+
+            var strings = domains.select<string>(d => @"DOM $d").to_vector();
+
+            foreach (var agent in agents) {
+                agent.set_secret(collection_secret);
+                strings.add(agent.to_string());
+            }
 
-            var signed_portion = @"PPCL $(Base64.encode(id)) $serial";
+            foreach (var member in members) {
+                member.set_secret(collection_secret);
+                strings.add(member.to_string());
+            }
+
+            var data = strings.to_string(s => s, "\n");
+            var signed_portion = @"PPCL $(Base64.encode(id)) $serial\nSSK $(Base64.encode(shared_signature_key))";
             if(data != "") {
                 signed_portion += @"\n$data";
             }
 
-            auth = Sodium.Asymmetric.Signing.sign(get_string_checksum(signed_portion), signing_key);
-            
-            var full_str = @"$signed_portion\nSIG $(Base64.encode(auth))\nMOD $(modified.format_iso8601())\n";
+            collection_signature = Sodium.Asymmetric.Signing.sign(get_string_checksum(signed_portion), authoratative_signing_key);
+            authoritative_section = @"$signed_portion\nSIG $(Base64.encode(collection_signature))";
+            return member_update(collection_secret);
+        }
+
+        public string member_update(uint8[] collection_secret) {
+            modified = new DateTime.now_local();
+            var full_str = @"MOD $(modified.format_iso8601())\n";
+            var pub_str = publications.to_string(p => p.to_string(), "\n");
             if(pub_str != "") {
-                full_str += @"\n$pub_str\n";
+                full_str += @"\n$pub_str";
             }
-            return full_str;
+            shared_section = full_str;
+            shared_signature = Sodium.Asymmetric.Signing.sign(get_checksum().to_array(), collection_secret);
+            return to_string();
+        }
+
+        public string to_string() {
+            return @"$authoritative_section\n$shared_section\nSSG $(Base64.encode(shared_signature))";
+        }
+
+        private BinaryData get_checksum() {
+            return new BinaryData.from_byte_array(get_string_checksum(@"$authoritative_section\n$shared_section"));
         }
 
         internal static uint8[] get_string_checksum(string data) {
@@ -162,7 +226,8 @@ namespace Ppub {
             var arr = signed_data.to_array();
             print(@"ARR.length $(arr.length), str $data\n");
             checksum_calculator.update(arr, arr.length);
-            var checksum = new uint8[64];
+
+            var checksum = new uint8[ChecksumType.SHA512.get_length()];
             size_t chk_len = checksum.length;
             checksum_calculator.get_digest(checksum, ref chk_len);
             checksum.length = (int)chk_len;
@@ -173,30 +238,109 @@ namespace Ppub {
 
     public class CollectionMember {
         public string name { get; private set; }
-        public uint8[] public_key { get; private set; }
+        public uint8[] public_signing_key { get; private set; }
+        public uint8[] public_sealing_key { get; private set; }
+        public uint8[] collection_secret { get; private set; }
 
         public string to_string() {
-            return @"MEM $name $(Base64.encode(public_key))";
+            return @"MEM $name $(Base64.encode(public_signing_key)) $(Base64.encode(public_sealing_key)) $(Base64.encode(collection_secret))";
         }
 
-        public CollectionMember(string name, uint8[] public_key) {
+        public CollectionMember(string name, uint8[] public_signing_key, uint8[] public_sealing_key) {
             this.name = name;
-            this.public_key = public_key;
+            this.public_signing_key = public_signing_key;
+            this.public_sealing_key = public_sealing_key;
         }
 
         public CollectionMember.from_string(string line) throws CollectionError {
             var parts = line.split(" ");
-            if(parts.length < 2) {
-                throw new CollectionError.INVALID_FORMAT("Member line must contain at least two fields, name and public key");
+            if(parts.length < 4) {
+                throw new CollectionError.INVALID_FORMAT("Member line must contain at least four fields, name, signing public key, sealing public key, and collection secret");
             }
             name = parts[0];
-            public_key = Base64.decode(parts[1]);
+            public_signing_key = Base64.decode(parts[1]);
+            public_sealing_key = Base64.decode(parts[2]);
+            collection_secret = Base64.decode(parts[3]);
+        }
+
+        public void set_secret(uint8[] secret) {
+            collection_secret = Sodium.Asymmetric.Sealing.seal(secret, public_sealing_key);
+        }
+
+        public uint8[]? decrypt_collection_secret(CollectionMemberCredentials credentials) {
+            return Sodium.Asymmetric.Sealing.unseal(collection_secret, public_sealing_key, credentials.secret_sealing_key);
+        }
+    }
+
+    public class CollectionAgent {
+        public string name { get; private set; }
+        public uint8[] public_sealing_key { get; private set; }
+        public uint8[] collection_secret { get; private set; }
+
+        public string to_string() {
+            return @"AGT $name $(Base64.encode(public_sealing_key)) $(Base64.encode(collection_secret))";
+        }
+
+        public CollectionAgent(string name, uint8[] public_key) {
+            this.name = name;
+            this.public_sealing_key = public_key;
+        }
+
+        public CollectionAgent.from_string(string line) throws CollectionError {
+            var parts = line.split(" ");
+            if(parts.length < 3) {
+                throw new CollectionError.INVALID_FORMAT("Member line must contain at least two fields, name, public key, and collection secret");
+            }
+            name = parts[0];
+            public_sealing_key = Base64.decode(parts[1]);
+            collection_secret = Base64.decode(parts[2]);
+        }
+
+        public uint8[]? decrypt_collection_secret(uint8[] private_sealing_key) {
+            return Sodium.Asymmetric.Sealing.unseal(collection_secret, public_sealing_key, private_sealing_key);
+        }
+
+        public void set_secret(uint8[] secret) {
+            collection_secret = Sodium.Asymmetric.Sealing.seal(secret, public_sealing_key);
         }
 
         public static void new_keypair(out uint8[] private_key, out uint8[] public_key) {
-            private_key = new uint8[Sodium.Asymmetric.Signing.SECRET_KEY_BYTES];
-            public_key = new uint8[Sodium.Asymmetric.Signing.PUBLIC_KEY_BYTES];
-            Sodium.Asymmetric.Signing.generate_keypair(public_key, private_key);
+            private_key = new uint8[Sodium.Asymmetric.Sealing.SECRET_KEY_BYTES];
+            public_key = new uint8[Sodium.Asymmetric.Sealing.PUBLIC_KEY_BYTES];
+            Sodium.Asymmetric.Sealing.generate_keypair(public_key, private_key);
+        }
+    }
+
+    public class CollectionMemberCredentials {
+        public uint8[] public_signing_key { get; private set; }
+        public uint8[] secret_signing_key { get; private set; }
+        public uint8[] public_sealing_key { get; private set; }
+        public uint8[] secret_sealing_key { get; private set; }
+
+        public CollectionMemberCredentials() {
+            public_signing_key = new uint8[Sodium.Asymmetric.Signing.PUBLIC_KEY_BYTES];
+            secret_signing_key = new uint8[Sodium.Asymmetric.Signing.SECRET_KEY_BYTES];
+            Sodium.Asymmetric.Signing.generate_keypair(public_signing_key, secret_signing_key);
+            
+            public_sealing_key = new uint8[Sodium.Asymmetric.Sealing.PUBLIC_KEY_BYTES];
+            secret_sealing_key = new uint8[Sodium.Asymmetric.Sealing.SECRET_KEY_BYTES];
+            Sodium.Asymmetric.Sealing.generate_keypair(public_sealing_key, secret_sealing_key);
+        }
+
+        public string to_string() {
+            return @"$(Base64.encode(public_signing_key)):$(Base64.encode(public_sealing_key)):$(Base64.encode(secret_signing_key)):$(Base64.encode(secret_sealing_key))";
+        }
+
+        public CollectionMemberCredentials.from_string(string str) throws CollectionError {
+            var clean = str.chug().chomp();
+            var parts = clean.split(":");
+            if(parts.length < 4) {
+                throw new CollectionError.INVALID_CREDENTIALS("Invalid member credential format");
+            }
+            public_signing_key = Base64.decode(parts[0]);
+            public_sealing_key = Base64.decode(parts[1]);
+            secret_signing_key = Base64.decode(parts[2]);
+            secret_sealing_key = Base64.decode(parts[3]);
         }
     }
 
@@ -235,21 +379,21 @@ namespace Ppub {
             return digest;
         }
 
-        public CollectionPublication(string file_name, DateTime publication_time, string member, uint8[] ppub_checksum, uint8[] signing_key) {
+        public CollectionPublication(string file_name, DateTime publication_time, string member, CollectionMemberCredentials member_credentials, uint8[] ppub_checksum) {
             this.file_name = file_name;
             this.publication_time = publication_time;
             this.member = member;
             this.publication_checksum = ppub_checksum;
-            sign(signing_key);
+            sign(member_credentials);
         }
 
-        private void sign(uint8[] signing_key) {
+        private void sign(CollectionMemberCredentials member_credentials) {
             var signed_portion = get_signed_portion();
             var line_checksum = Collection.get_string_checksum(signed_portion);
             var to_sign = new BinaryData.from_byte_array(line_checksum);
             to_sign.append_byte_array(publication_checksum);
 
-            signature = Sodium.Asymmetric.Signing.sign(to_sign.to_array(), signing_key);
+            signature = Sodium.Asymmetric.Signing.sign(to_sign.to_array(), member_credentials.secret_signing_key);
 
             raw_line = @"PUB $signed_portion $(Base64.encode(signature))";
         }
@@ -277,7 +421,7 @@ namespace Ppub {
 
             var checksum = new BinaryData.from_byte_array(Collection.get_string_checksum(get_signed_portion()));
             print(@"\t\t$(get_signed_portion()) == $(Base64.encode(checksum.to_array()))\n");
-            var signature_data = Sodium.Asymmetric.Signing.verify(signature, collection_member.public_key);
+            var signature_data = Sodium.Asymmetric.Signing.verify(signature, collection_member.public_signing_key);
             if(signature_data == null) {
                 throw new CollectionError.INVALID_COLLECTION_SIGNATURE("Invalid publication signature");
             }

src/lib/Pprf/GetListing.vala

@@ -1,86 +0,0 @@
-using Invercargill;
-
-namespace Ppub.Pprf {
-
-    public enum GetListingColumn {
-        TITLE = (uint16)(1 << 0),
-        AUTHOR = (uint16)(1 << 1),
-        DESCRIPTION = (uint16)(1 << 2),
-        TIMESTAMP = (uint16)(1 << 3),
-        TAGS = (uint16)(1 << 4),
-        POSTER = (uint16)(1 << 5),
-        COPYRIGHT = (uint16)(1 << 6),
-        CHECKSUM = (uint16)(1 << 7),
-    }
-
-    public class GetListing : CollectionMessage {
-    
-        public uint16 columns { get; set; }
-        public uint32 skip { get; set; }
-        public uint8 take { get; set; }
-        public string? tag { get; set; }
-        public string? query { get; set; }
-    
-        public GetListing() {
-            message_type = MessageType.GET_LISTING;
-        }
-    
-        public override void deserialise (GLib.DataInputStream stream) throws Error {
-            base.deserialise (stream);
-            columns = stream.read_uint16();
-            skip = stream.read_uint32();
-            take = stream.read_byte();
-
-            var tag_len = stream.read_uint16();
-            if(tag_len > 0) {
-                var tag_data = new uint8[tag_len];
-                stream.read(tag_data);
-                tag = new BinaryData.from_byte_array(tag_data).to_raw_string();
-            }
-
-            var query_len = stream.read_uint16();
-            if(query_len > 0) {
-                var query_data = new uint8[query_len];
-                stream.read(query_data);
-                query = new BinaryData.from_byte_array(query_data).to_raw_string();
-            }
-        }
-    
-        public override uint64 calculate_size() {
-            var text_size = 0;
-            if(query != null) {
-                text_size += query.data.length;
-            }
-            if(tag != null) {
-                text_size += tag.data.length;
-            }
-            return base.calculate_size() + 
-                2 + // Columns
-                4 + // Skip
-                1 + // Take
-                2 + // Tag size
-                2 + // Query size
-                text_size; // Query + tag
-        }
-        
-        public override void serialise(DataOutputStream stream) throws Error {
-            base.serialise(stream);
-            stream.put_uint16(columns);
-            stream.put_uint32(skip);
-            stream.put_byte(take);
-
-            var tag_size = tag == null ? 0 : tag.data.length-1;
-            stream.put_uint16(tag_size);
-            if(tag_size > 0) {
-                stream.put_string(tag);
-            }
-
-            var query_size = query == null ? 0 : query.data.length-1;
-            stream.put_uint16(query_size);
-            if(query_size > 0) {
-                stream.put_string(query);
-            }
-
-        }
-    }
-}

src/lib/meson.build

@@ -18,23 +18,6 @@ sources += files('Ppvm.vala')
 sources += files('Ppix.vala')
 sources += files('Ppcl.vala')
 
-sources += files('Pprf/Message.vala')
-sources += files('Pprf/MessageFactory.vala')
-sources += files('Pprf/CollectionMessage.vala')
-sources += files('Pprf/EncryptedMessage.vala')
-sources += files('Pprf/SignedMessage.vala')
-sources += files('Pprf/BeginUpload.vala')
-sources += files('Pprf/Failure.vala')
-sources += files('Pprf/GetAsset.vala')
-sources += files('Pprf/GetListing.vala')
-sources += files('Pprf/GetPublicationInformation.vala')
-sources += files('Pprf/GetPublication.vala')
-sources += files('Pprf/IdentifyServer.vala')
-sources += files('Pprf/Publish.vala')
-sources += files('Pprf/RegisterName.vala')
-sources += files('Pprf/UpdateCollection.vala')
-sources += files('Pprf/Upload.vala')
-
 ppub = shared_library('libppub', sources,
     name_prefix: '',
     dependencies: dependencies,

src/meson.build

@@ -5,6 +5,7 @@ add_project_arguments(['--disable-warnings', '--enable-checking','--vapidir', va
 
 
 subdir('lib')
+subdir('pprf')
 subdir('tools')
 
 dependencies += ppub_dep

src/pprf/HttpClient.vala

@@ -0,0 +1,133 @@
+
+using Pprf.Messages;
+using Invercargill;
+
+namespace Pprf {
+
+    public class HttpPprfClient {
+
+        public string endpoint { get; set; }
+        public const string HTTP_CONTENT_TYPE = "application/pprf";
+        
+        private Soup.Session session;
+
+        public HttpPprfClient(string endpoint) {
+            this.endpoint = endpoint;
+            session = new Soup.Session();
+        }
+
+        public Message send_message(Message message) throws Error {
+            var soup_message = new Soup.Message("POST", endpoint);
+
+            var streamer = new MessageStreamer(message);
+
+            soup_message.set_request_body(HTTP_CONTENT_TYPE, streamer, (ssize_t)message.calculate_true_size());
+
+            var stream = session.send(soup_message);
+            return MessageFactory.from_stream(stream);
+        }
+
+    }
+
+    private class MessageStreamer : InputStream {
+
+        private Message message;
+        private Receipticle receipticle;
+        private int total = 0;
+
+        private bool thread_started = false;
+
+        public MessageStreamer(Message message) {
+            this.message = message;
+            this.receipticle = new Receipticle();
+        }
+
+        public override bool close(GLib.Cancellable? cancellable) {
+            return true;
+        }
+        public override ssize_t read(uint8[] buffer, GLib.Cancellable? cancellable) throws IOError {
+            lock(thread_started) {
+                if(!thread_started) {
+                    thread_started = true;
+                    new Thread<bool>("serialisation thread", serialise);
+                }
+            }
+            var s = (ssize_t)receipticle.read(buffer);
+            total += (int)s;
+            return s;
+        }
+
+        private bool serialise() {
+            var stream = new DataOutputStream(receipticle);
+            try {
+                message.serialise(stream);
+                stream.flush();
+                stream.close();
+            }
+            catch(IOError e) {
+                receipticle.set_error(e);
+            }
+            catch(Error e) {
+                receipticle.set_error(new IOError.FAILED("Serialisation error: " + e.message));
+            }
+            return true;
+        }
+
+        public class Receipticle : OutputStream {
+
+            private BinaryData output = new BinaryData();
+            private const int max_size = 8192;
+
+            private Cond cond = Cond ();
+            private Mutex mutex = Mutex ();
+            private bool closed = false;
+            private IOError? error = null;
+            private int written = 0;
+
+            public void set_error(IOError e) {
+                mutex.lock();
+                error = e;
+                cond.broadcast();
+                mutex.unlock();
+            }
+
+            public override bool close(GLib.Cancellable? cancellable) {
+                mutex.lock();
+                closed = true;
+                cond.broadcast();
+                mutex.unlock();
+                return true;
+            }
+            public override ssize_t write(uint8[] buffer, GLib.Cancellable? cancellable) {
+                mutex.lock();
+                while(output.count() > max_size) {
+                    cond.wait(mutex);
+                }
+                output.append_byte_array(buffer);
+                written += buffer.length;
+                cond.broadcast();
+                mutex.unlock();
+                return buffer.length;
+            }
+
+            public size_t read(uint8[] buffer) throws IOError {
+                mutex.lock();
+                while(output.count() == 0 && !closed && error == null) {
+                    cond.wait(mutex);
+                }
+                if(error != null) {
+                    throw error;
+                }
+                var written = output.write_to(buffer, buffer.length);
+                output = output.slice((int)written, output.count());
+                cond.broadcast();
+                mutex.unlock();
+                return written;
+            }
+
+        }
+        
+
+    }
+
+}

src/pprf/Messages/BeginUpload.vala

@@ -0,0 +1,35 @@
+using Invercargill;
+
+namespace Pprf.Messages {
+
+    public class BeginUpload : CollectionMessage {
+    
+        public uint64 file_size { get; set; }
+        public string member_name { get; set; }
+    
+        public BeginUpload() {
+            message_type = MessageType.BEGIN_UPLOAD;
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+            file_size = stream.read_uint64();
+            var name_size = stream.read_byte();
+            member_name = new BinaryData.from_bytes(stream.read_bytes(name_size)).to_raw_string();
+        }
+    
+        public override uint64 calculate_size() {
+            return base.calculate_size() + 
+                8 + // File size field
+                1 + // Member name size field
+                member_name.data.length;
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+            stream.put_uint64(file_size);
+            stream.put_byte((uint8)member_name.data.length);
+            stream.put_string(member_name);
+        }
+    }
+}

src/lib/Pprf/CollectionMessage.vala → src/pprf/Messages/CollectionMessage.vala

@@ -1,6 +1,6 @@
 using Invercargill;
 
-namespace Ppub.Pprf {
+namespace Pprf.Messages {
 
     public abstract class CollectionMessage : Message {
 
@@ -19,6 +19,9 @@ namespace Ppub.Pprf {
         
         public override void serialise(DataOutputStream stream) throws Error {
             base.serialise(stream);
+            if(collection_id.count() != Sodium.Asymmetric.Signing.PUBLIC_KEY_BYTES) {
+                error("Collection ID is not the correct size");
+            }
             stream.write(collection_id.to_array());
         }
 

src/pprf/Messages/Confirmation.vala

@@ -0,0 +1,9 @@
+
+namespace Pprf.Messages {
+
+    public class Confirmation : Message {
+        public Confirmation() {
+            message_type = MessageType.CONFIRMATION;
+        }
+    }
+}

src/pprf/Messages/Failure.vala

@@ -0,0 +1,64 @@
+using Invercargill;
+
+namespace Pprf.Messages {
+
+    public class Failure : Message {
+    
+        public uint16 code { get; set; }
+        public string? message { get; set; }
+    
+        public Failure() {
+            message_type = MessageType.FAILURE;
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+            code = stream.read_uint16();
+
+            var msg_len = stream.read_uint16();
+            if(msg_len > 0) {
+                var name_data = new uint8[msg_len];
+                stream.read(name_data);
+                message = new BinaryData.from_byte_array(name_data).to_raw_string();
+            }
+        }
+    
+        public override uint64 calculate_size() {
+            var message_size = message == null ? 0 : message.data.length;
+            return base.calculate_size() + 
+                2 + // Error code field
+                2 + // Name size field
+                message_size;
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+            stream.put_uint16(code);
+
+            var msg_size = message == null ? 0 : message.data.length;
+            stream.put_uint16((uint16)msg_size);
+            if(msg_size > 0) {
+                stream.put_string(message);
+            }
+
+        }
+    }
+
+    public enum FailureCode {
+
+        UNINTELLIGIBLE = 0,
+        UNSUPPORTED_MESSAGE = 1,
+        UNKNOWN_COLLECTION = 2,
+        MEMBER_NOT_IN_COLLECTION = 3,
+        INVALID_UPLOAD_SESSION = 4,
+        INVALID_UPLOAD_AUTH = 5,
+        INVALID_UPLOAD_OFFSET = 6,
+        INVALID_NAME = 7,
+        NAME_EXISTS = 8,
+        DESTINATION_NOT_REGISTERED = 9,
+        DESTINATION_NOT_EMPTY = 10,
+        INVALID_MEMBER_SIGNATURE = 11,
+        INVALID_BSPATCH_CHECKSUM = 12,
+        BSPATCH_ERROR = 13,
+    }
+}

src/pprf/Messages/FinaliseUpload.vala

@@ -0,0 +1,66 @@
+using Invercargill;
+
+namespace Pprf.Messages {
+
+    public enum FinaliseUploadFlags {
+        CANCEL = (1 << 0),
+        OVERWRITE_DESTINATION = (1 << 1),
+        BSPATCH = (1 << 2)
+    }
+
+    public class FinaliseUpload : UploadSessionMessage {
+    
+        public uint8 flags { get; set; }
+        public string destination { get; set; }
+        public uint8[] bspatch_old_checksum { get; set; }
+    
+        public FinaliseUpload() {
+            message_type = MessageType.FINALISE_UPLOAD;
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+
+            flags = stream.read_byte();
+
+            var name_len = stream.read_uint16();
+            destination = "";
+            if(name_len > 0) {
+                var name_data = new uint8[name_len];
+                stream.read(name_data);
+                destination = new BinaryData.from_byte_array(name_data).to_raw_string();
+            }
+            if((flags & FinaliseUploadFlags.BSPATCH) != 0) {
+                bspatch_old_checksum = new uint8[64];
+                stream.read(bspatch_old_checksum);
+            }
+        }
+    
+        public override uint64 calculate_size() {
+            return base.calculate_size() + 
+                1 + // Flags
+                2 + // Destination name length
+                ((flags & FinaliseUploadFlags.BSPATCH) == 0 ? 0 : 64) + // bspatch old file checksum (only if flag set)
+                destination.data.length;
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+
+            stream.put_byte(flags);
+
+            var name_size = destination.data.length;
+            stream.put_uint16((uint16)name_size);
+            if(name_size > 0) {
+                stream.put_string(destination);
+            }
+
+            if((flags & FinaliseUploadFlags.BSPATCH) != 0) {
+                if(bspatch_old_checksum.length != 64) {
+                    error("BSPATCH flag set but old checksum property is not correct length");
+                }
+                stream.write(bspatch_old_checksum);
+            }
+        }
+    }
+}

src/pprf/Messages/GetListing.vala

@@ -0,0 +1,267 @@
+using Invercargill;
+using Invercargill.Convert;
+using Ppub;
+
+namespace Pprf.Messages {
+
+    public enum ListingColumn {
+        TITLE = (1 << 0),
+        AUTHOR = (1 << 1),
+        DESCRIPTION = (1 << 2),
+        TIMESTAMP = (1 << 3),
+        TAGS = (1 << 4),
+        POSTER = (1 << 5),
+        COPYRIGHT = (1 << 6),
+        CHECKSUM = (1 << 7),
+    }
+
+    public class GetListing : CollectionMessage {
+    
+        public uint16 columns { get; set; }
+        public uint32 skip { get; set; }
+        public uint8 take { get; set; }
+        public string? tag { get; set; }
+        public string? query { get; set; }
+    
+        public GetListing() {
+            message_type = MessageType.GET_LISTING;
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+            columns = stream.read_uint16();
+            skip = stream.read_uint32();
+            take = stream.read_byte();
+
+            var tag_len = stream.read_uint16();
+            if(tag_len > 0) {
+                var tag_data = new uint8[tag_len];
+                stream.read(tag_data);
+                tag = new BinaryData.from_byte_array(tag_data).to_raw_string();
+            }
+
+            var query_len = stream.read_uint16();
+            if(query_len > 0) {
+                var query_data = new uint8[query_len];
+                stream.read(query_data);
+                query = new BinaryData.from_byte_array(query_data).to_raw_string();
+            }
+        }
+    
+        public override uint64 calculate_size() {
+            var text_size = 0;
+            if(query != null) {
+                text_size += query.data.length;
+            }
+            if(tag != null) {
+                text_size += tag.data.length;
+            }
+            return base.calculate_size() + 
+                2 + // Columns
+                4 + // Skip
+                1 + // Take
+                2 + // Tag size
+                2 + // Query size
+                text_size; // Query + tag
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+            stream.put_uint16(columns);
+            stream.put_uint32(skip);
+            stream.put_byte(take);
+
+            var tag_size = tag == null ? 0 : tag.data.length;
+            stream.put_uint16(tag_size);
+            if(tag_size > 0) {
+                stream.put_string(tag);
+            }
+
+            var query_size = query == null ? 0 : query.data.length;
+            stream.put_uint16(query_size);
+            if(query_size > 0) {
+                stream.put_string(query);
+            }
+
+        }
+    }
+
+    public class CollectionListing : CollectionMessage {
+
+        public uint16 columns { get; set; }
+        public Vector<CollectionListingItem> results { get; set; }
+
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+            columns = stream.read_uint16();
+            var result_count = stream.read_byte();
+
+            for(int i = 0; i < result_count; i++) {
+                results.add(new CollectionListingItem.from_stream(stream, columns));
+            }
+        }
+
+        public override uint64 calculate_size() {
+            var base_size = base.calculate_size();
+            var header_size = 2 + 1; // Columns + result_count
+            var rows_size = results
+                .select<int>(r => r.get_size(columns))
+                .promote_to<SignedNativeIntegers>()
+                .sum();
+
+            return base_size + header_size + rows_size;
+        }
+        
+
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+            stream.put_uint16(columns);
+            stream.put_byte((uint8)results.count());
+
+            foreach (var result in results) {
+                result.serialise(stream, columns);
+            }
+        }
+
+    }
+
+    public class CollectionListingItem {
+
+        public string name { get; set; }
+        public Metadata metadata { get; set; }
+        public uint8[]? checksum { get; set; }
+        public uint8[]? poster { get; set; }
+
+        public CollectionListingItem.from_stream(DataInputStream stream, uint16 cols) throws Error {
+            metadata = new Metadata();
+
+            var name_len = stream.read_uint16();
+            name = read_string(stream, name_len);
+
+            if((cols & ListingColumn.TITLE) != 0) {
+                var size = stream.read_uint16();
+                metadata.title = read_string(stream, size);
+            }
+            if((cols & ListingColumn.AUTHOR) != 0) {
+                var size = stream.read_uint16();
+                metadata.author = read_string(stream, size);
+            }
+            if((cols & ListingColumn.DESCRIPTION) != 0) {
+                var size = stream.read_uint16();
+                metadata.author = read_string(stream, size);
+            }
+            if((cols & ListingColumn.TIMESTAMP) != 0) {
+                var size = stream.read_byte();
+                var date = read_string(stream, size);
+                metadata.date = new DateTime.from_iso8601(date, null);
+            }
+            if((cols & ListingColumn.TAGS) != 0) {
+                var size = stream.read_uint16();
+                metadata.tags = read_string(stream, size).split(" ");
+            }
+            if((cols & ListingColumn.POSTER) != 0) {
+                var size = stream.read_uint32();
+                var data = new BinaryData();
+                var remaining = (size_t)size;
+                while(data.read_in(stream.read_bytes(remaining), ref remaining));
+                poster = data.to_array();
+            }
+            if((cols & ListingColumn.COPYRIGHT) != 0) {
+                var size = stream.read_uint16();
+                metadata.copyright = read_string(stream, size);
+            }
+            if((cols & ListingColumn.CHECKSUM) != 0) {
+                checksum = new uint8[64];
+                stream.read(checksum);
+            }
+        }
+
+        public int get_size(uint16 cols) {
+            var s = 2 + name.data.length;
+
+            if((cols & ListingColumn.TITLE) != 0) {
+                s += 2 + metadata.title.data.length;
+            }
+            if((cols & ListingColumn.AUTHOR) != 0) {
+                s += 2 + metadata.author.data.length;
+            }
+            if((cols & ListingColumn.DESCRIPTION) != 0) {
+                s += 2 + metadata.description.data.length;
+            }
+            if((cols & ListingColumn.TIMESTAMP) != 0) {
+                var date_str = metadata.date.format_iso8601();
+                s += 1 + date_str.data.length;
+            }
+            if((cols & ListingColumn.TAGS) != 0) {
+                var tag_str = ate(metadata.tags).to_string(s => s, " ");
+                s += 2 + tag_str.data.length;
+            }
+            if((cols & ListingColumn.POSTER) != 0) {
+                s += 4 + poster.length;
+            }
+            if((cols & ListingColumn.COPYRIGHT) != 0) {
+                s += 2 + metadata.copyright.data.length;
+            }
+            if((cols & ListingColumn.CHECKSUM) != 0) {
+                s += 64;
+            }
+            
+            return s;
+        }
+
+        public void serialise(DataOutputStream stream, uint16 cols) throws Error {
+
+            stream.put_uint16((uint16)name.data.length);
+            stream.put_string(name);
+
+            if((cols & ListingColumn.TITLE) != 0) {
+                stream.put_uint16((uint16)metadata.title.data.length);
+                stream.put_string(metadata.title);
+            }
+            if((cols & ListingColumn.AUTHOR) != 0) {
+                stream.put_uint16((uint16)metadata.author.data.length);
+                stream.put_string(metadata.author);
+            }
+            if((cols & ListingColumn.DESCRIPTION) != 0) {
+                stream.put_uint16((uint16)metadata.description.data.length);
+                stream.put_string(metadata.description);
+            }
+            if((cols & ListingColumn.TIMESTAMP) != 0) {
+                var date = metadata.date.format_iso8601();
+                stream.put_byte((uint8)date.data.length);
+                stream.put_string(date);
+            }
+            if((cols & ListingColumn.TAGS) != 0) {
+                var tags = ate(metadata.tags).to_string(s => s, " ");
+                stream.put_byte((uint8)tags.data.length);
+                stream.put_string(tags);
+            }
+            if((cols & ListingColumn.POSTER) != 0) {
+                stream.put_uint32(poster.length);
+                if(poster.length > 0) {
+                    stream.write(poster);
+                }
+            }
+            if((cols & ListingColumn.COPYRIGHT) != 0) {
+                stream.put_uint16((uint16)metadata.copyright.data.length);
+                stream.put_string(metadata.copyright);
+            }
+            if((cols & ListingColumn.CHECKSUM) != 0) {
+                if(checksum == null) {
+                    stream.write(new uint8[64]);
+                }
+                else{
+                    stream.write(checksum);
+                }
+            }
+        }
+
+        private string read_string(DataInputStream stream, int length) throws Error {
+            var data = new BinaryData();
+            var remaining = (size_t)length;
+            while(data.read_in(stream.read_bytes(remaining), ref remaining));
+            return data.to_raw_string();
+        }
+
+    }
+}

src/lib/Pprf/IdentifyServer.vala → src/pprf/Messages/IdentifyServer.vala

@@ -1,6 +1,6 @@
 using Invercargill;
 
-namespace Ppub.Pprf {
+namespace Pprf.Messages {
 
     public class IdentifyServer : Message {
         public IdentifyServer() {

src/lib/Pprf/Message.vala → src/pprf/Messages/Message.vala

@@ -1,6 +1,6 @@
 using Invercargill;
 
-namespace Ppub.Pprf {
+namespace Pprf.Messages {
 
     public errordomain PprfMessageError {
         INVALID_HEADER
@@ -14,7 +14,14 @@ namespace Ppub.Pprf {
             return 1;
         }
 
+        public uint64 calculate_true_size() {
+            return calculate_size() +
+                5 + // Magic
+                8; // Size field
+        }
+
         public virtual void deserialise(DataInputStream stream) throws Error {
+            stream.byte_order = DataStreamByteOrder.LITTLE_ENDIAN;
             var magic = new uint8[5];
             stream.read(magic);
             if(magic[0] != 'P' || magic[1] != 'P' || magic[2] != 'R' || magic[3] != 'F' || magic[4] != 0x00) {
@@ -26,6 +33,7 @@ namespace Ppub.Pprf {
         }
 
         public virtual void serialise(DataOutputStream stream) throws Error {
+            stream.byte_order = DataStreamByteOrder.LITTLE_ENDIAN;
             stream.write(new uint8[] { 'P', 'P', 'R', 'F', 0x00 });
             size = calculate_size();
             stream.put_uint64(size);
@@ -33,53 +41,49 @@ namespace Ppub.Pprf {
         }
     }
 
-    public abstract class MessageBody : InputStream {
+    public abstract class MessageBody {
         public uint64 body_size { get; set; }
-        public abstract BinaryData? get_next_chunk(GLib.Cancellable? cancellable) throws Error;
-        protected virtual void cleanup() {}
+        public abstract void write_to(OutputStream out_stream) throws Error;
+    }
 
-        private BinaryData buffered = null;
+    public class StreamMessageBody : MessageBody {
 
-        public virtual DataInputStream as_stream() {
-            return new DataInputStream(this);
-        }
+        private DataInputStream stream;
+        private uint64 data_read = 0;
 
-        public override bool close(GLib.Cancellable? cancellable) {
-            cleanup();
-            return true;            
+        public StreamMessageBody(int size, DataInputStream stream) {
+            body_size = size;
+            this.stream = stream;
         }
 
-        public override ssize_t read(uint8[] buffer, GLib.Cancellable? cancellable) throws IOError {
-            if(buffered != null && buffered.count() != 0) {
-                lock(buffered) {
-                    var size = buffered.write_to(buffer, buffer.length);
-                    buffered = buffered.slice((int)size, buffered.count());
-                    return (ssize_t)size;
+        public override void write_to(GLib.OutputStream out_stream) throws Error {
+            while(data_read < body_size) {
+                uint64 chunk_size = 10240;
+                if(body_size - data_read < chunk_size) {
+                    chunk_size = body_size - data_read;
                 }
+                var data = new uint8[chunk_size];
+                var read = stream.read(data);
+                data_read += (uint64)read;
+                data.length = (int)read;
+                out_stream.write(data);
             }
+        }
 
-            try {
-                var data = get_next_chunk(cancellable);
-                if(data == null) {
-                    return 0;
-                }
+    }
 
-                var size = data.write_to(buffer, buffer.length);
-                if(data.count() != size) {
-                    buffered = data.slice((int)size, data.count());
-                }
-                return (ssize_t)size;
-            }
-            catch(PprfMessageError e) {
-                throw new IOError.INVALID_DATA(@"PPRF Message Error: $(e.message) ($(e.domain))");
-            }
-            catch(IOError e) {
-                throw e;
-            }
-            catch(Error e) {
-                throw new IOError.INVALID_DATA(@"Unhandeled error: $(e.message) ($(e.domain))");
-            }
+    public class BytesMessageBody : MessageBody {
+        private Bytes data;
+
+        public BytesMessageBody(Bytes data) {
+            body_size = data.length;
+            this.data = data;
+        }
+
+        public override void write_to(OutputStream out_stream) throws Error {
+            out_stream.write(data.get_data());
         }
+
     }
 
     public enum MessageType {
@@ -90,6 +94,7 @@ namespace Ppub.Pprf {
         GET_PUBLICATION_INFORMATION = 2,
         GET_ASSET = 3,
         GET_PUBLICATION = 4,
+        GET_INDEX = 5,
 
         // 32 - 63: Administration requests
         REGISTER_NAME = 32,
@@ -99,6 +104,7 @@ namespace Ppub.Pprf {
         PUBLISH = 36,
         UNPUBLISH = 37,
         UPDATE_COLLECTION = 38,
+        REBUILD_INDEX = 39,
 
         // 64 - 95: Encapsulated messages
         ENCRYPTED_MESSAGE = 64,
@@ -107,8 +113,8 @@ namespace Ppub.Pprf {
         // 96 - 127: Server initiated requests
         PROPOGATE_PUBLICATION = 96,
         PROPOGATE_COLLECTION = 97,
-        SYNCHRONISE_COLLECTION = 98,
-        GET_PUBLICATIONS = 99,
+        SYNCHRONISE_COLLECTION = 99,
+        GET_PUBLICATIONS = 100,
 
 
         // 128 - 159: Server responses to public user requests
@@ -120,8 +126,7 @@ namespace Ppub.Pprf {
         TRY_FIRST = 134,
 
         // 160 - 191: Server responses to administration requests
-        NAME_REGISTRATION = 160,
-        UPLOAD_SESSION = 161,
+        UPLOAD_SESSION = 160,
 
         // 192 - 223: Generic server responses
         CONFIRMATION = 192,

src/pprf/Messages/MessageFactory.vala

@@ -0,0 +1,44 @@
+
+namespace Pprf.Messages {
+
+    public class MessageFactory {
+
+        public static Message construct_message(MessageType mtype) {
+            switch (mtype) {
+                case MessageType.REGISTER_NAME:
+                    return new RegisterName();
+                case MessageType.BEGIN_UPLOAD:
+                    return new BeginUpload();
+                case MessageType.UPLOAD_SESSION:
+                    return new UploadSession();
+                case MessageType.UPLOAD:
+                    return new Upload();
+                case MessageType.FINALISE_UPLOAD:
+                    return new FinaliseUpload();
+                case MessageType.PUBLISH:
+                    return new RegisterName();
+                case MessageType.CONFIRMATION:
+                    return new Confirmation();
+                case MessageType.FAILURE:
+                    return new Failure();
+
+                default:
+                    assert_not_reached();
+            }
+        }
+
+        public static Message from_stream(InputStream stream) throws Error {
+            var buffered = new BufferedInputStream(stream);
+            buffered.fill(14);
+            var peek = new uint8[14];
+            buffered.peek(peek);
+
+            var type = (MessageType)peek[13];
+            var message = construct_message(type);
+            message.deserialise(new DataInputStream(buffered));
+            return message;
+        }
+
+    }
+
+}

src/pprf/Messages/Publish.vala

@@ -0,0 +1,38 @@
+using Invercargill;
+
+namespace Pprf.Messages {
+
+    public class Publish : CollectionMessage {
+    
+        public string publication_string { get; set; }
+    
+        public Publish() {
+            message_type = MessageType.PUBLISH;
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+
+            var data_len = stream.read_uint16();
+            if(data_len > 0) {
+                publication_string = new BinaryData.from_bytes(stream.read_bytes (data_len)).to_raw_string();
+            }
+        }
+    
+        public override uint64 calculate_size() {
+            return base.calculate_size() + 
+                2 + // Data length
+                publication_string.data.length;
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+
+            var data_len = publication_string.data.length;
+            stream.put_uint16((uint16)data_len);
+            if(data_len > 0) {
+                stream.put_string(publication_string);
+            }
+        }
+    }
+}

src/pprf/Messages/RegisterName.vala

@@ -0,0 +1,55 @@
+using Invercargill;
+
+namespace Pprf.Messages {
+
+    public class RegisterName : CollectionMessage {
+    
+        public string member_name { get; set; }
+        public uint8[] signed_name { get; set; }
+    
+        public RegisterName() {
+            message_type = MessageType.REGISTER_NAME;
+        }
+
+        public void sign_name(string name, Ppub.CollectionMemberCredentials credentials) {
+            signed_name = Sodium.Asymmetric.Signing.sign(name.data, credentials.secret_signing_key);
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+
+            var member_len = stream.read_byte();
+            member_name = "";
+            if(member_len > 0) {
+                member_name = new BinaryData.from_bytes(stream.read_bytes (member_len)).to_raw_string();
+            }
+
+            var name_len = stream.read_uint16();
+            if(name_len > 0) {
+                signed_name = stream.read_bytes (name_len).get_data().copy();
+            }
+        }
+    
+        public override uint64 calculate_size() {
+            return base.calculate_size() + 
+                1 + // Member name length field
+                member_name.data.length +
+                2 + // Signed name length field
+                signed_name.length;
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+            
+            stream.put_byte((uint8)member_name.data.length);
+            if(member_name.data.length > 0) {
+                stream.put_string(member_name);
+            }
+
+            stream.put_uint16((uint8)signed_name);
+            if(signed_name.length > 0) {
+                stream.write(signed_name);
+            }
+        }
+    }
+}

src/pprf/Messages/Upload.vala

@@ -0,0 +1,38 @@
+using Invercargill;
+
+namespace Pprf.Messages {
+    
+    public class Upload : UploadSessionMessage {
+    
+        public uint64 offset { get; set; }
+        public MessageBody upload_chunk { get; set; }
+    
+        public Upload() {
+            message_type = MessageType.UPLOAD;
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+
+            offset = stream.read_uint64();
+
+            // Size of message minus headers
+            var chunk_size = size - (base.calculate_size() + 8);
+            upload_chunk = new StreamMessageBody((int)chunk_size, stream);
+        }
+    
+        public override uint64 calculate_size() {
+            return base.calculate_size() + 
+                8 + // Offset field
+                upload_chunk.body_size;
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+            
+            stream.put_uint64(offset);
+            upload_chunk.write_to(stream);
+            stream.flush();
+        }
+    }
+}

src/pprf/Messages/UploadSession.vala

@@ -0,0 +1,39 @@
+using Invercargill;
+
+namespace Pprf.Messages {
+
+    public class UploadSession : Message {
+    
+        public uint8[] session_authentication { get; set; }
+        public uint32 max_chunk_size { get; set; }
+    
+        public UploadSession() {
+            message_type = MessageType.UPLOAD_SESSION;
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+            
+            var auth_size = stream.read_uint16();
+            session_authentication = new uint8[auth_size];
+            stream.read(session_authentication);
+
+            max_chunk_size = stream.read_uint32();
+        }
+    
+        public override uint64 calculate_size() {
+            return base.calculate_size() + 
+                2 + // Auth size field
+                session_authentication.length +
+                4; // Chunk size field
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+            
+            stream.put_uint16((uint16)session_authentication.length);
+            stream.write(session_authentication);
+            stream.put_uint32(max_chunk_size);
+        }
+    }
+}

src/pprf/Messages/UploadSessionMessage.vala

@@ -0,0 +1,48 @@
+using Invercargill;
+
+namespace Pprf.Messages {
+
+    public class UploadSessionMessage : CollectionMessage {
+    
+        public uint8[] token { get; set; }
+        public uint8[] upload_auth { get; set; }
+
+        public void authenticate(uint8[] session_auth, uint8[] chunk_checksum, Ppub.CollectionMemberCredentials credentials) {
+            var session = Sodium.Asymmetric.Sealing.unseal(session_auth, credentials.public_sealing_key, credentials.secret_sealing_key);
+            var auth_key = session[0:Sodium.Asymmetric.Sealing.PUBLIC_KEY_BYTES];
+            token = session[Sodium.Asymmetric.Sealing.PUBLIC_KEY_BYTES:session.length];
+
+            var signed = Sodium.Asymmetric.Signing.sign(chunk_checksum, credentials.secret_signing_key);
+            upload_auth = Sodium.Asymmetric.Sealing.seal(signed, auth_key);
+        }
+    
+        public override void deserialise (GLib.DataInputStream stream) throws Error {
+            base.deserialise (stream);
+            
+            var token_size = stream.read_uint16();
+            token = new uint8[token_size];
+            stream.read(token);
+
+            var auth_size = stream.read_uint16();
+            upload_auth = new uint8[auth_size];
+            stream.read(upload_auth);
+        }
+    
+        public override uint64 calculate_size() {
+            return base.calculate_size() + 
+                2 + // Token size field
+                token.length +
+                2 + // Upload auth size field
+                upload_auth.length;
+        }
+        
+        public override void serialise(DataOutputStream stream) throws Error {
+            base.serialise(stream);
+            stream.put_uint16((uint16)token.length);
+            stream.write(token);
+            stream.put_uint16((uint16)upload_auth.length);
+            stream.write(upload_auth);
+        }
+    }
+    
+}

src/pprf/meson.build

@@ -0,0 +1,55 @@
+
+dependencies = [
+    dependency('glib-2.0'),
+    dependency('gobject-2.0'),
+    dependency('gio-2.0'),
+    dependency('gee-0.8'),
+    dependency('invercargill'),
+    dependency('libriddle'),
+    dependency('libsoup-3.0'),
+    meson.get_compiler('vala').find_library('libsodium', dirs: vapi_dir),
+    meson.get_compiler('c').find_library('sodium'),
+    ppub_dep,
+]
+
+sources = files('HttpClient.vala')
+sources += files('Messages/Message.vala')
+sources += files('Messages/CollectionMessage.vala')
+sources += files('Messages/UploadSessionMessage.vala')
+sources += files('Messages/MessageFactory.vala')
+sources += files('Messages/GetAsset.vala')
+sources += files('Messages/BeginUpload.vala')
+sources += files('Messages/Upload.vala')
+sources += files('Messages/FinaliseUpload.vala')
+sources += files('Messages/Publish.vala')
+sources += files('Messages/Failure.vala')
+sources += files('Messages/Confirmation.vala')
+sources += files('Messages/UploadSession.vala')
+
+sources += files('Messages/GetListing.vala')
+sources += files('Messages/GetPublicationInformation.vala')
+sources += files('Messages/GetPublication.vala')
+sources += files('Messages/IdentifyServer.vala')
+sources += files('Messages/RegisterName.vala')
+sources += files('Messages/UpdateCollection.vala')
+
+pprf = shared_library('libpprf', sources,
+    name_prefix: '',
+    dependencies: dependencies,
+    install: true,
+    vala_gir: 'Pprf-1.0.gir',
+    install_dir: [true, true, true, true]
+)
+pprf_dep = declare_dependency(link_with: pprf, include_directories: include_directories('.'))
+
+pkg = import('pkgconfig')
+pkg.generate(pprf,
+    version : '0.1',
+    name : 'libpprf',)
+    
+g_ir_compiler = find_program('g-ir-compiler')
+custom_target('pprf typelib', command: [g_ir_compiler, '--shared-library=libpprf.so', '--output', '@OUTPUT@', meson.current_build_dir() / 'Pprf-1.0.gir'],
+              output: 'Pprf-1.0.typelib',
+              depends: pprf,
+              install: true,
+              install_dir: get_option('libdir') / 'girepository-1.0')

src/tools/meson.build

@@ -1,4 +1,5 @@
 subdir('extract')
 subdir('viewer')
 subdir('create')
-subdir('ppcl')
+subdir('ppcl')
+subdir('pprf')

src/tools/ppcl/Ppcl.vala

@@ -11,7 +11,7 @@ public static int main(string[] args) {
         uint8[] signing_key;
         var collection = new Ppub.Collection(out signing_key);
         
-        var col_str = collection.to_string(signing_key);
+        var col_str = collection.authoratative_update(signing_key);
         FileUtils.set_contents(args[2], col_str, col_str.length);
 
         var key_str = Base64.encode(signing_key);
@@ -20,15 +20,10 @@ public static int main(string[] args) {
     }
 
     if(args[1] == "new-member") {
-        uint8[] sk;
-        uint8[] pk;
-        Ppub.CollectionMember.new_keypair(out sk, out pk);
-        
-        var sk_str = Base64.encode(sk);
-        FileUtils.set_contents(args[2] + ".private.key", sk_str, sk_str.length);
+        var member_cred = new Ppub.CollectionMemberCredentials();
+        var cred_str = member_cred.to_string();
+        FileUtils.set_contents(args[2] + ".credentials", cred_str, cred_str.length);
 
-        var pk_str = Base64.encode(pk);
-        FileUtils.set_contents(args[2] + ".public.key", pk_str, pk_str.length);
         return 0;
     }
 
@@ -41,32 +36,36 @@ public static int main(string[] args) {
     var collection = new Ppub.Collection.from_stream(collection_stream);
 
     if(args[1] == "add-member") {
-        collection.increment_serial();
-        collection.members.add(new Ppub.CollectionMember(args[3], Base64.decode(args[4])));
+        var keys = args[4].split(":", 2);
+        collection.members.add(new Ppub.CollectionMember(args[3], Base64.decode(keys[0]), Base64.decode(keys[1])));
+        collection.authoratative_update(signing_key);
     }
 
     else if(args[1] == "add-domain") {
-        collection.increment_serial();
         collection.domains.add(args[3]);
+        collection.authoratative_update(signing_key);
     }
 
     else if(args[1] == "publish") {
+        var member = collection.members.single(m => m.name == args[4]);
+
         var checksum = Ppub.CollectionPublication.generate_ppub_checksum(File.new_for_path(args[3]));
         
-        string mem_key_str;
-        FileUtils.get_contents(args[4] + ".private.key", out mem_key_str, null);
-        var mem_signing_key = Base64.decode(mem_key_str);
+        string cred_str;
+        FileUtils.get_contents(args[4] + ".credentials", out cred_str, null);
+        var credentials = new Ppub.CollectionMemberCredentials.from_string(cred_str);
 
-        var publication = new Ppub.CollectionPublication(args[3], new DateTime.now_local(), args[4], checksum, mem_signing_key);
+        var publication = new Ppub.CollectionPublication(args[3], new DateTime.now_local(), args[4], credentials, checksum);
         collection.publications.add(publication);
+        var secret = member.decrypt_collection_secret(credentials);
+        collection.member_update(secret);
     }
     else {
         print(@"Unrecognised command $(args[1])\n");
         return -2;
     }
 
-    collection.touch();
-    var col_str = collection.to_string(signing_key);
+    var col_str = collection.to_string();
     FileUtils.set_contents(args[2], col_str, col_str.length);
 
     return 0;

src/tools/pprf/Pprf.vala

@@ -0,0 +1,89 @@
+
+
+public static int main(string[] args) {
+
+    var creds = new Ppub.CollectionMemberCredentials.from_string("PYuKgL7SdQYc2Kf6UGG9pCE58m27qrYnCaM45cnxs64=:JbJ6OoNn2KcGX+Tk5C/hotGZCoHOkTNbadUrlk6aCRs=:tL+557eP7kE6ObAW0b5RjvYyU8Dl3oVTOvYA7LAwSdI9i4qAvtJ1BhzYp/pQYb2kITnybbuqticJozjlyfGzrg==:AJbFO6n/cOuD7kk+wu7DmQ58w6z0G3HsukVmIzxGaUM=");
+
+    var upload_file = File.new_for_path(args[1]);
+    var file_size = upload_file.query_info("*", FileQueryInfoFlags.NONE).get_size();
+    var client = new Pprf.HttpPprfClient("http://localhost:8080/test.php");
+    var file_name = upload_file.get_basename();
+
+    //  print("Regisering name\n");
+    //  var rmessage = new Pprf.Messages.RegisterName();
+    //  rmessage.collection_id = new Invercargill.BinaryData.from_base64("ctA1tlLAk1hZgEvHcstCJWM+0OkdT0tYdIUeNmRvC5o=");
+    //  rmessage.sign_name (file_name, creds);
+    //  rmessage.member_name = "billy";
+
+    //  var response = client.send_message(rmessage);
+    //  if(response is Pprf.Messages.Failure) {
+    //      print(@"Failure $(response.code): $(response.message)\n");
+    //      return -1;
+    //  }
+
+    print("Beginning upload\n");
+    var message = new Pprf.Messages.BeginUpload();
+    message.collection_id = new Invercargill.BinaryData.from_base64("ctA1tlLAk1hZgEvHcstCJWM+0OkdT0tYdIUeNmRvC5o=");
+    message.file_size = file_size;
+    message.member_name = "billy";
+
+    var response = client.send_message(message);
+    if(response is Pprf.Messages.Failure) {
+        print(@"Failure $(response.code): $(response.message)\n");
+        return -1;
+    }
+    
+    var session = (Pprf.Messages.UploadSession)response;
+    print(@"Got session! Max chunk size: $(session.max_chunk_size)\n");
+
+    uint64 offset = 0;
+    var full_checksum = new Checksum (ChecksumType.SHA512);
+    var file_stream = upload_file.read();
+    while(offset < file_size) {
+        var chk = new Checksum(ChecksumType.SHA512);
+        var data = file_stream.read_bytes(session.max_chunk_size);
+        chk.update(data.get_data(), data.length);
+        full_checksum.update(data.get_data(), data.length);
+
+        size_t dig_len = 64;
+        var digest = new uint8[64];
+        chk.get_digest(digest, ref dig_len);
+        digest.length = (int)dig_len;
+
+        var umessage = new Pprf.Messages.Upload();
+        umessage.collection_id = new Invercargill.BinaryData.from_base64("ctA1tlLAk1hZgEvHcstCJWM+0OkdT0tYdIUeNmRvC5o=");
+        umessage.offset = offset;
+        umessage.upload_chunk = new Pprf.Messages.BytesMessageBody(data);
+        umessage.authenticate(session.session_authentication, digest, creds);
+
+        response = client.send_message(umessage);
+        if(response is Pprf.Messages.Failure) {
+            print(@"Failure $(response.code): $(response.message)\n");
+            return -1;
+        }
+        offset += data.length;
+        print(@"Sent $offset bytes\n");
+    }
+
+    print("Finalising upload\n");
+    size_t dig_len = 64;
+    var digest = new uint8[64];
+    full_checksum.get_digest(digest, ref dig_len);
+    digest.length = (int)dig_len;
+
+    var fmessage = new Pprf.Messages.FinaliseUpload();
+    fmessage.collection_id = new Invercargill.BinaryData.from_base64("ctA1tlLAk1hZgEvHcstCJWM+0OkdT0tYdIUeNmRvC5o=");
+    fmessage.destination = file_name;
+    fmessage.authenticate(session.session_authentication, digest, creds);
+    fmessage.bspatch_old_checksum = new Invercargill.BinaryData.from_base64("2+1VxEx6bMUfJ0g0b3vnJwDFVf/acGG5IXENvb3ElCFU9dFXi+6muFb2GXUVA8B5+PV2AWiOEKhD3RqNTYwBpw==").to_array();
+    fmessage.flags = Pprf.Messages.FinaliseUploadFlags.BSPATCH | Pprf.Messages.FinaliseUploadFlags.OVERWRITE_DESTINATION;
+
+    response = client.send_message(fmessage);
+    if(response is Pprf.Messages.Failure) {
+        print(@"Failure $(response.code): $(response.message)\n");
+        return -1;
+    }
+    
+    print("Done\n");
+    return 0;
+}

src/tools/pprf/meson.build

@@ -0,0 +1,8 @@
+
+sources = files('Pprf.vala')
+
+deps = dependencies
+deps += ppub_dep
+deps += pprf_dep
+
+executable('pprf', sources, dependencies: deps, install: true)