using Xml; using Html; using Astralis; using Invercargill; using Invercargill.DataStructures; namespace Statum.Tools { /** * `statum-mkpstm` — the build-time Statum pre-rendering pipeline. * * Turns an app HTML page (plus its templates/fragments) into a Vala * {@link Statum.StatumPage} subclass carrying precompressed variant byte * arrays and a compiled {@link Statum.StatumPage.select_variant} that * evaluates `pstm-guard`s and the `pstm-if` matrix against the request's held * slots using the Invercargill runtime expression evaluator. * * See `phase-1-plan.md` Subsystem B for the supported `pstm-*` directives. */ public class Mkpstm : Object { private static string? output_file = null; private static string? namespace_name = null; private static string? class_name_override = null; private static string? route_override = null; private static List template_dirs = new List(); private Dictionary templates_by_name = new Dictionary(); private Dictionary fragments_by_name = new Dictionary(); private const OptionEntry[] options = { { "output", 'o', 0, OptionArg.FILENAME, ref output_file, "Output Vala file name", "FILE" }, { "ns", '\0', 0, OptionArg.STRING, ref namespace_name, "Namespace for the generated class", "NAMESPACE" }, { "name", 'n', 0, OptionArg.STRING, ref class_name_override, "Generated class name (default: Page)", "NAME" }, { "route", 'r', 0, OptionArg.STRING, ref route_override, "Page route (default: from )", "ROUTE" }, { "template-dir", 't', 0, OptionArg.FILENAME_ARRAY, ref template_dirs, "Directory to search for templates/fragments (repeatable)", "DIR" }, { null } }; public static int main(string[] args) { try { var opt = new OptionContext("PAGE_FILE - Generate a Statum StatumPage"); opt.add_main_entries(options, null); opt.parse(ref args); } catch (OptionError e) { stderr.printf("Error: %s\n", e.message); return 1; } string? page_file = null; if (args.length > 1) { page_file = args[1]; } if (page_file == null) { stderr.printf("Error: No page file specified.\n"); return 1; } try { var tool = new Mkpstm(); var generated = tool.run((!)page_file); var out_path = tool.output_file ?? (make_pascal_case((!)page_file) + "Page.vala"); write_file(out_path, generated); stdout.printf("Generated: %s\n", out_path); return 0; } catch (GLib.Error e) { stderr.printf("Error: %s\n", e.message); return 1; } } public string run(string page_file) throws GLib.Error { if (template_dirs.length() == 0) { var parent = File.new_for_path(page_file).get_parent(); template_dirs.append(parent != null ? ((!)parent).get_path() : "."); } load_templates_and_fragments(page_file); var doc = parse_html_file(page_file); validate_document(doc, page_file); validate_head_directives(doc, page_file); // Capture the page's routes before composition (the // directive is dropped while merging templates). var routes = read_routes(doc); doc = compose_templates(doc); inline_fragments(doc); rewrite_resources(doc); validate_no_conditionals(doc, page_file); var variant = build_variant(doc); var class_name = class_name_override ?? (make_pascal_case(page_file) + "Page"); var route = route_override ?? (routes.length > 0 ? routes[0] : "/"); return emit(class_name, route, variant); } // ------------------------------------------------------------------ // Template/fragment loading // ------------------------------------------------------------------ private void load_templates_and_fragments(string page_file) throws GLib.Error { foreach (var dir in template_dirs) { var dir_file = File.new_for_path(dir); if (!dir_file.query_exists()) { continue; } var enumerator = dir_file.enumerate_children("standard::name", 0); FileInfo info; while ((info = enumerator.next_file()) != null) { var name = info.get_name(); if (!name.has_suffix(".html") || name.has_prefix(".")) { continue; } var path = Path.build_filename(dir, name); if (path == page_file) { continue; } var tdoc = parse_html_file(path); var source = Path.build_filename(dir, name); validate_document(tdoc, source); validate_head_directives(tdoc, source); var tname = read_pstm_name(tdoc); if (tname == null) { continue; } if (find_first_element(tdoc->get_root_element(), "pstm-content") != null) { templates_by_name.set((!)tname, tdoc); } else { fragments_by_name.set((!)tname, tdoc); } } } } // ------------------------------------------------------------------ // Composition // ------------------------------------------------------------------ private Xml.Doc* compose_templates(Xml.Doc* doc) throws GLib.Error { Xml.Doc* current_doc = doc; string merged_title = read_directive_in_doc(current_doc, "pstm-title") ?? ""; // Guard against cyclic template chains (A templates B templates A). var visited = new HashSet(); while (true) { var parent_name = read_directive_in_doc(current_doc, "pstm-template"); if (parent_name == null || parent_name.length == 0) { break; } if (visited.contains((!)parent_name)) { break; } visited.add((!)parent_name); Xml.Doc* parent_doc; if (!templates_by_name.try_get((!)parent_name, out parent_doc)) { throw new IOError.FAILED(@"Unknown template \"$((!)parent_name)\""); } // Merge this document's into the parent's (so // pages, templates and nested components can all declare head // tags), then splice its into the parent's content outlet. merge_head(current_doc, parent_doc); splice_body(current_doc, parent_doc); var parent_title = read_directive_in_doc(parent_doc, "pstm-title"); if (parent_title != null && parent_title.length > 0) { merged_title = merged_title + (!)parent_title; } current_doc = parent_doc; } apply_title(current_doc, merged_title); return current_doc; } /** * Copies the children of `child_doc`'s into `parent_doc`'s , * skipping the consumed structural directives (pstm-template/name/title/ * uri) which are page-level metadata and must not be carried up. */ private void merge_head(Xml.Doc* child_doc, Xml.Doc* parent_doc) { var child_head = find_head(child_doc); var parent_head = find_head(parent_doc); if (child_head == null || parent_head == null) { return; } for (var child = child_head->children; child != null; child = child->next) { if (!is_element(child)) { continue; } var name = (string) child->name; if (name == "pstm-template" || name == "pstm-name" || name == "pstm-title" || name == "pstm-uri") { continue; } var copied = child->doc_copy(parent_doc, 1); if (copied != null) { parent_head->add_child(copied); } } } /** * Splices `child_doc`'s into `parent_doc`'s outlet * (or appends to the parent's when there is no outlet). When the * child carries `pstm-materialise-as`, it is wrapped in an element * of that tag (carrying the body's non-pstm attributes); otherwise its * children are spliced directly. */ private void splice_body(Xml.Doc* child_doc, Xml.Doc* parent_doc) { var child_body = find_body(child_doc); var parent_body = find_body(parent_doc); if (child_body == null || parent_body == null) { return; } var outlet = find_first_element(parent_body, "pstm-content"); var materialise_as = child_body->get_prop("pstm-materialise-as"); if (materialise_as != null) { var wrapper = parent_body->new_child(null, (!)materialise_as, null); wrapper->unlink(); copy_non_pstm_attrs(child_body, wrapper); for (var child = child_body->children; child != null; child = child->next) { if (is_structural_directive(child)) { continue; } var copied = child->doc_copy(parent_doc, 1); if (copied != null) { wrapper->add_child(copied); } } if (outlet != null) { outlet->add_prev_sibling(wrapper); } else { parent_body->add_child(wrapper); } } else { for (var child = child_body->children; child != null; child = child->next) { if (is_structural_directive(child)) { continue; } var copied = child->doc_copy(parent_doc, 1); if (copied == null) { continue; } if (outlet != null) { outlet->add_prev_sibling(copied); } else { parent_body->add_child(copied); } } } if (outlet != null) { outlet->unlink(); } } /** Copies an element's non-`pstm-*` attributes onto another element. */ private static void copy_non_pstm_attrs(Xml.Node* src, Xml.Node* dst) { for (var attr = src->properties; attr != null; attr = attr->next) { var attr_name = (string) attr->name; if (attr_name.has_prefix("pstm-")) { continue; } var value = src->get_prop(attr_name); if (value != null) { dst->set_prop(attr_name, (!)value); } } } private static string? read_directive_in_doc(Xml.Doc* doc, string name) { var root = doc->get_root_element(); if (root == null) { return null; } return read_directive_text(root, name); } private static bool is_structural_directive(Xml.Node* node) { if (!is_element(node)) { return false; } var name = (string) node->name; return name == "pstm-template" || name == "pstm-name" || name == "pstm-title" || name == "pstm-uri" || name == "pstm-content"; } /** Fragment-body metadata elements skipped when inlining (e.g. pstm-input). */ private static bool is_fragment_metadata(Xml.Node* node) { if (!is_element(node)) { return false; } var name = (string) node->name; return name == "pstm-input" || name.has_prefix("pstm-"); } private void inline_fragments(Xml.Doc* doc) { bool changed = true; while (changed) { changed = false; var includes = find_all_elements(doc, "pstm-fragment"); foreach (var include in includes) { if (inline_one_fragment(doc, include)) { changed = true; } } } } private bool inline_one_fragment(Xml.Doc* doc, Xml.Node* include) { var name = include->get_prop("name"); if (name == null) { return false; } Xml.Doc* fragment_doc; if (!fragments_by_name.try_get((!)name, out fragment_doc)) { return false; } var fragment_body = find_body(fragment_doc); if (fragment_body == null) { return false; } // Merge the fragment's into the document's . merge_head(fragment_doc, doc); var inputs = collect_input_names(fragment_body); var include_materialise = include->get_prop("pstm-materialise-as"); var fragment_materialise = fragment_body->get_prop("pstm-materialise-as"); var materialise_as = include_materialise ?? fragment_materialise; bool wrap = (materialise_as != null) || inputs.length > 0; string tag = materialise_as ?? "div"; if (wrap) { var parent = include->parent; if (parent == null) { return false; } var wrapper = parent->new_child(null, tag, null); wrapper->unlink(); include->add_prev_sibling(wrapper); // Carry the fragment 's attributes when it supplied the tag. if (include_materialise == null && fragment_materialise != null) { copy_non_pstm_attrs(fragment_body, wrapper); } for (var child = fragment_body->children; child != null; child = child->next) { if (is_fragment_metadata(child)) { continue; } var copied = child->doc_copy(doc, 1); if (copied != null) { wrapper->add_child(copied); } } foreach (var input_name in inputs) { var expr = include->get_prop(input_name); if (expr != null) { wrapper->set_prop(@"stm-def-$input_name-as", (!)expr); } } } else { for (var child = fragment_body->children; child != null; child = child->next) { if (is_fragment_metadata(child)) { continue; } var copied = child->doc_copy(doc, 1); if (copied != null) { include->add_prev_sibling(copied); } } } include->unlink(); return true; } private string[] collect_input_names(Xml.Node* fragment_body) { var names = new string[0]; for (var n = fragment_body->children; n != null; n = n->next) { if (is_element(n) && (string) n->name == "pstm-input") { var input_name = n->get_prop("name"); if (input_name != null) { names += (!)input_name; } } } return names; } private void rewrite_resources(Xml.Doc* doc) { var root = doc->get_root_element(); if (root != null) { rewrite_resources_in(root); } } private void rewrite_resources_in(Xml.Node* node) { if (!is_element(node)) { return; } string[] to_add_attr = new string[0]; string[] to_add_val = new string[0]; string[] to_remove = new string[0]; for (var attr = node->properties; attr != null; attr = attr->next) { var attr_name = (string) attr->name; if (attr_name.has_prefix("pstm-res-")) { var target_attr = attr_name.substring("pstm-res-".length); var value = node->get_prop(attr_name); if (value != null) { to_add_attr += target_attr; to_add_val += @"/_statum/resource/$((!)value)"; } to_remove += attr_name; } } for (int i = 0; i < to_add_attr.length; i++) { node->set_prop(to_add_attr[i], to_add_val[i]); } foreach (var attr_name in to_remove) { node->unset_prop(attr_name); } for (var child = node->children; child != null; child = child->next) { rewrite_resources_in(child); } } // ------------------------------------------------------------------ // Variant rendering // ------------------------------------------------------------------ /** * Renders the composed document to its single HTML form (stripping every * `pstm-*` directive) and precompresses it. Pages no longer have * variants: a page is a static, precompressed document, so there is * exactly one. */ private VariantData build_variant(Xml.Doc* doc) throws GLib.Error { strip_pstm(doc); var html = serialize(doc); return compress_variant(html); } private VariantData compress_variant(string html) throws GLib.Error { var data = new VariantData(); data.identity = html.data; var buffer = new ByteBuffer.from_byte_array(html.data); var gzip = new GzipCompressor(9).compress_buffer(buffer, null).to_array(); var zstd = new ZstdCompressor(19).compress_buffer(buffer, null).to_array(); var br = new BrotliCompressor(11).compress_buffer(buffer, null).to_array(); data.has_gzip = gzip.length < html.length; data.has_zstd = zstd.length < html.length; data.has_br = br.length < html.length; data.gzip = data.has_gzip ? gzip : null; data.zstd = data.has_zstd ? zstd : null; data.br = data.has_br ? br : null; return data; } /** Remove every `pstm-*` attribute and `pstm-*` element from the tree. */ private void strip_pstm(Xml.Doc* doc) { var root = doc->get_root_element(); if (root != null) { strip_pstm_from(root); } } private void strip_pstm_from(Xml.Node* node) { string[] to_remove = new string[0]; for (var attr = node->properties; attr != null; attr = attr->next) { var attr_name = (string) attr->name; if (attr_name.has_prefix("pstm-")) { to_remove += attr_name; } } foreach (var attr_name in to_remove) { node->unset_prop(attr_name); } var child = node->children; while (child != null) { var next = child->next; if (is_element(child) && ((string) child->name).has_prefix("pstm-")) { child->unlink(); } else if (is_element(child)) { strip_pstm_from(child); } child = next; } } /** * Errors if any removed server-side conditional (`pstm-if`/`pstm-else-if`/ * `pstm-else`) is still present; pages are static now, so conditionals * are client-side `stm-if` only. */ private void validate_no_conditionals(Xml.Doc* doc, string source) throws GLib.Error { var root = doc->get_root_element(); if (root != null) { validate_no_conditionals_from(root, source); } } private void validate_no_conditionals_from(Xml.Node* node, string source) throws GLib.Error { if (!is_element(node)) { return; } string[] names = { "pstm-if", "pstm-else-if", "pstm-else" }; foreach (var name in names) { if (node->get_prop(name) != null) { throw new IOError.FAILED(@"\"$source\": the \"$name\" directive is no longer supported; use the client-side stm-if instead"); } } for (var child = node->children; child != null; child = child->next) { validate_no_conditionals_from(child, source); } } // ------------------------------------------------------------------ // Code generation // ------------------------------------------------------------------ private string emit(string class_name, string route, VariantData variant) throws GLib.Error { var page_hash = compute_hash(variant.identity); var b = new StringBuilder(); string i1 = namespace_name != null ? " " : ""; string i2 = namespace_name != null ? " " : " "; string i3 = namespace_name != null ? " " : " "; b.append("using Statum;\n"); b.append("using Invercargill;\n"); b.append("using Invercargill.DataStructures;\n"); b.append("using Astralis;\n\n"); if (namespace_name != null) { b.append_printf("namespace %s {\n\n", namespace_name); } b.append_printf("%s// Generated by statum-mkpstm\n", i1); b.append_printf("%spublic class %s : StatumPage {\n\n", i1, class_name); b.append_printf("%spublic const string ROUTE = \"%s\";\n", i2, escape(route)); b.append_printf("%spublic override string content_type { get { return \"text/html; charset=utf-8\"; } }\n\n", i2); // get_best_encoding: smallest available supported encoding. b.append_printf("%spublic override string get_best_encoding(Set supported) {\n", i2); if (variant.has_br) { b.append_printf("%sif (supported.has(\"br\")) return \"br\";\n", i3); } if (variant.has_zstd) { b.append_printf("%sif (supported.has(\"zstd\")) return \"zstd\";\n", i3); } if (variant.has_gzip) { b.append_printf("%sif (supported.has(\"gzip\")) return \"gzip\";\n", i3); } b.append_printf("%sreturn \"identity\";\n", i3); b.append_printf("%s}\n\n", i2); // get_encoding b.append_printf("%spublic override unowned uint8[] get_encoding(string encoding) {\n", i2); b.append_printf("%sswitch (encoding) {\n", i3); b.append_printf("%scase \"identity\": return IDENTITY;\n", i3); if (variant.has_gzip && variant.gzip != null) { b.append_printf("%scase \"gzip\": return GZIP;\n", i3); } if (variant.has_zstd && variant.zstd != null) { b.append_printf("%scase \"zstd\": return ZSTD;\n", i3); } if (variant.has_br && variant.br != null) { b.append_printf("%scase \"br\": return BR;\n", i3); } b.append_printf("%sdefault: return IDENTITY;\n", i3); b.append_printf("%s}\n", i3); b.append_printf("%s}\n\n", i2); // get_etag_for (ETag = "-") b.append_printf("%spublic override string get_etag_for(string encoding) {\n", i2); b.append_printf("%sreturn \"\\\"%s-%%s\\\"\".printf(encoding);\n", i3, page_hash); b.append_printf("%s}\n\n", i2); // Byte arrays. emit_one_array(b, i2, i3, "IDENTITY", variant.identity); if (variant.has_gzip && variant.gzip != null) { emit_one_array(b, i2, i3, "GZIP", (!)variant.gzip); } if (variant.has_zstd && variant.zstd != null) { emit_one_array(b, i2, i3, "ZSTD", (!)variant.zstd); } if (variant.has_br && variant.br != null) { emit_one_array(b, i2, i3, "BR", (!)variant.br); } b.append_printf("%s}\n", i1); if (namespace_name != null) { b.append("}\n"); } return b.str; } private void emit_one_array(StringBuilder b, string i2, string i3, string name, uint8[] data) { b.append_printf("%sprivate const uint8[] %s = {\n", i2, name); for (int i = 0; i < data.length; i++) { if (i == 0) { b.append(i3); } else if (i % 16 == 0) { b.append(",\n").append(i3); } else { b.append(", "); } b.append_printf("0x%02x", data[i]); } b.append_printf("\n%s};\n\n", i2); } // ------------------------------------------------------------------ // DOM helpers // ------------------------------------------------------------------ private Xml.Doc* parse_html_file(string path) throws GLib.Error { var file = File.new_for_path(path); if (!file.query_exists()) { throw new IOError.NOT_FOUND(@"Page file '$path' does not exist"); } uint8[] contents; string etag_out; file.load_contents(null, out contents, out etag_out); // Parse with libxml2's HTML parser (mirrors Astralis' MarkupDocument) // so the tree has HTML semantics and serializes as HTML, not XML. int opts = (int)(Html.ParserOption.RECOVER | Html.ParserOption.NOERROR | Html.ParserOption.NOWARNING | Html.ParserOption.NOBLANKS | Html.ParserOption.NONET); char[] buffer = ((string) contents).to_utf8(); var doc = Html.Doc.read_memory(buffer, buffer.length, path, "utf-8", opts); if (doc == null) { throw new IOError.FAILED(@"Failed to parse HTML file '$path'"); } return doc; } private static Xml.Node* find_body(Xml.Doc* doc) { var root = doc->get_root_element(); if (root == null) { return null; } return find_first_element(root, "body"); } private static Xml.Node* find_head(Xml.Doc* doc) { var root = doc->get_root_element(); if (root == null) { return null; } return find_first_element(root, "head"); } /** * Validates that `doc` has both a and a element, which every * Statum HTML input (pages, templates and fragments) must declare. */ private void validate_document(Xml.Doc* doc, string source) throws GLib.Error { if (find_head(doc) == null) { throw new IOError.FAILED(@"\"$source\" has no element (a and are required)"); } if (find_body(doc) == null) { throw new IOError.FAILED(@"\"$source\" has no element (a and are required)"); } } /** * Validates that the structural directives , * and appear only inside . */ private void validate_head_directives(Xml.Doc* doc, string source) throws GLib.Error { var root = doc->get_root_element(); if (root == null) { return; } string[] names = { "pstm-title", "pstm-template", "pstm-name", "pstm-uri" }; foreach (var name in names) { var nodes = new Series(); find_all_from(root, name, nodes); foreach (var node in nodes) { if (!is_descendant_of_head(node)) { throw new IOError.FAILED(@"\"$source\": <$name> must appear inside "); } } } } /** Walks the ancestor chain to determine whether `node` is within . */ private static bool is_descendant_of_head(Xml.Node* node) { for (var p = node->parent; p != null; p = p->parent) { if (is_element(p) && ((string) p->name).down() == "head") { return true; } } return false; } private static Xml.Node* find_first_element(Xml.Node* root, string name) { if (!is_element(root)) { return null; } if (((string) root->name).down() == name.down()) { return root; } for (var child = root->children; child != null; child = child->next) { var found = find_first_element(child, name); if (found != null) { return found; } } return null; } private static Xml.Node*[] find_all_elements(Xml.Doc* doc, string name) { var collected = new Series(); var root = doc->get_root_element(); if (root != null) { find_all_from(root, name, collected); } return collected.to_array(); } private static void find_all_from(Xml.Node* node, string name, Series result) { if (is_element(node) && ((string) node->name).down() == name.down()) { result.add(node); } if (is_element(node)) { for (var child = node->children; child != null; child = child->next) { find_all_from(child, name, result); } } } private static string? read_directive_text(Xml.Node* body, string directive) { var node = find_first_element(body, directive); if (node == null) { return null; } var text = element_text(node); return text.strip().length > 0 ? text.strip() : null; } private static string? read_pstm_name(Xml.Doc* doc) { var node = find_first_element(doc->get_root_element(), "pstm-name"); if (node == null) { return null; } return element_text(node).strip(); } private static string[] read_routes(Xml.Doc* doc) { string[] routes = new string[0]; var nodes = find_all_elements(doc, "pstm-uri"); foreach (var node in nodes) { var text = element_text(node).strip(); if (text.length > 0) { routes += text; } } return routes; } private static string element_text(Xml.Node* node) { var b = new StringBuilder(); for (var child = node->children; child != null; child = child->next) { if (child->type == ElementType.TEXT_NODE && child->content != null) { b.append((string) child->content); } else if (is_element(child)) { b.append(element_text(child)); } } return b.str; } private static void apply_title(Xml.Doc* doc, string? title) { if (title == null || title.length == 0) { return; } var head = find_first_element(doc->get_root_element(), "head"); if (head == null) { return; } var title_node = find_first_element(head, "title"); if (title_node == null) { title_node = head->new_child(null, "title", null); } for (var child = title_node->children; child != null; child = child->next) { child->unlink(); } title_node->add_content(title); } private static string serialize(Xml.Doc* doc) { // Serialize with libxml2's HTML serializer (htmlDocDumpMemory) so the // output is valid HTML: no prolog, no self-closing // `