using Xml;
using Html;
using Astralis;
using Invercargill;
using Invercargill.DataStructures;
namespace Statum.Tools {
/**
* `statum-mkpstm` — the build-time Statum pre-rendering pipeline.
*
* Turns an app HTML page (plus its templates/fragments) into a Vala
* {@link Statum.StatumPage} subclass: a static, precompressed document whose
* route comes from its ``. Composition (templates, fragments,
* `pstm-res` rewriting) is applied, then the result is precompressed with
* identity/gzip/zstd/br.
*/
public class Mkpstm : Object {
private static string? output_file = null;
private static string? namespace_name = null;
private static string? class_name_override = null;
private static string? route_override = null;
private static List template_dirs = new List();
private Dictionary templates_by_name = new Dictionary();
private Dictionary fragments_by_name = new Dictionary();
private const OptionEntry[] options = {
{ "output", 'o', 0, OptionArg.FILENAME, ref output_file, "Output Vala file name", "FILE" },
{ "ns", '\0', 0, OptionArg.STRING, ref namespace_name, "Namespace for the generated class", "NAMESPACE" },
{ "name", 'n', 0, OptionArg.STRING, ref class_name_override, "Generated class name (default: Page)", "NAME" },
{ "route", 'r', 0, OptionArg.STRING, ref route_override, "Page route (default: from )", "ROUTE" },
{ "template-dir", 't', 0, OptionArg.FILENAME_ARRAY, ref template_dirs, "Directory to search for templates/fragments (repeatable)", "DIR" },
{ null }
};
public static int main(string[] args) {
try {
var opt = new OptionContext("PAGE_FILE - Generate a Statum StatumPage");
opt.add_main_entries(options, null);
opt.parse(ref args);
} catch (OptionError e) {
stderr.printf("Error: %s\n", e.message);
return 1;
}
string? page_file = null;
if (args.length > 1) {
page_file = args[1];
}
if (page_file == null) {
stderr.printf("Error: No page file specified.\n");
return 1;
}
try {
var tool = new Mkpstm();
var generated = tool.run((!)page_file);
var out_path = tool.output_file ?? (make_pascal_case((!)page_file) + "Page.vala");
write_file(out_path, generated);
stdout.printf("Generated: %s\n", out_path);
return 0;
} catch (GLib.Error e) {
stderr.printf("Error: %s\n", e.message);
return 1;
}
}
public string run(string page_file) throws GLib.Error {
if (template_dirs.length() == 0) {
var parent = File.new_for_path(page_file).get_parent();
template_dirs.append(parent != null ? ((!)parent).get_path() : ".");
}
load_templates_and_fragments(page_file);
var doc = parse_html_file(page_file);
validate_document(doc, page_file);
validate_head_directives(doc, page_file);
// Capture the page's routes before composition (the
// directive is dropped while merging templates).
var routes = read_routes(doc);
doc = compose_templates(doc);
inline_fragments(doc);
rewrite_resources(doc);
var variant = build_variant(doc);
var class_name = class_name_override ?? (make_pascal_case(page_file) + "Page");
var route = route_override ?? (routes.length > 0 ? routes[0] : "");
return emit(class_name, route, variant);
}
// ------------------------------------------------------------------
// Template/fragment loading
// ------------------------------------------------------------------
private void load_templates_and_fragments(string page_file) throws GLib.Error {
foreach (var dir in template_dirs) {
var dir_file = File.new_for_path(dir);
if (!dir_file.query_exists()) {
continue;
}
var enumerator = dir_file.enumerate_children("standard::name", 0);
FileInfo info;
while ((info = enumerator.next_file()) != null) {
var name = info.get_name();
if (!name.has_suffix(".html") || name.has_prefix(".")) {
continue;
}
var path = Path.build_filename(dir, name);
if (path == page_file) {
continue;
}
var tdoc = parse_html_file(path);
var source = Path.build_filename(dir, name);
validate_document(tdoc, source);
validate_head_directives(tdoc, source);
var tname = read_pstm_name(tdoc);
if (tname == null) {
continue;
}
if (find_first_element(tdoc->get_root_element(), "pstm-content") != null) {
templates_by_name.set((!)tname, tdoc);
} else {
fragments_by_name.set((!)tname, tdoc);
}
}
}
}
// ------------------------------------------------------------------
// Composition
// ------------------------------------------------------------------
private Xml.Doc* compose_templates(Xml.Doc* doc) throws GLib.Error {
Xml.Doc* current_doc = doc;
string merged_title = read_directive_in_doc(current_doc, "pstm-title") ?? "";
// Guard against cyclic template chains (A templates B templates A).
var visited = new HashSet();
while (true) {
var parent_name = read_directive_in_doc(current_doc, "pstm-template");
if (parent_name == null || parent_name.length == 0) {
break;
}
if (visited.contains((!)parent_name)) {
break;
}
visited.add((!)parent_name);
Xml.Doc* parent_doc;
if (!templates_by_name.try_get((!)parent_name, out parent_doc)) {
throw new IOError.FAILED(@"Unknown template \"$((!)parent_name)\"");
}
// Merge this document's into the parent's (so
// pages, templates and nested components can all declare head
// tags), then splice its into the parent's content outlet.
merge_head(current_doc, parent_doc);
splice_body(current_doc, parent_doc);
var parent_title = read_directive_in_doc(parent_doc, "pstm-title");
if (parent_title != null && parent_title.length > 0) {
merged_title = merged_title + (!)parent_title;
}
current_doc = parent_doc;
}
apply_title(current_doc, merged_title);
return current_doc;
}
/**
* Copies the children of `child_doc`'s into `parent_doc`'s ,
* skipping the consumed structural directives (pstm-template/name/title/
* uri) which are page-level metadata and must not be carried up.
*/
private void merge_head(Xml.Doc* child_doc, Xml.Doc* parent_doc) {
var child_head = find_head(child_doc);
var parent_head = find_head(parent_doc);
if (child_head == null || parent_head == null) {
return;
}
for (var child = child_head->children; child != null; child = child->next) {
if (!is_element(child)) {
continue;
}
var name = (string) child->name;
if (name == "pstm-template" || name == "pstm-name" || name == "pstm-title" || name == "pstm-uri") {
continue;
}
var copied = child->doc_copy(parent_doc, 1);
if (copied != null) {
parent_head->add_child(copied);
}
}
}
/**
* Splices `child_doc`'s into `parent_doc`'s outlet
* (or appends to the parent's when there is no outlet). When the
* child carries `pstm-materialise-as`, it is wrapped in an element
* of that tag (carrying the body's non-pstm attributes); otherwise its
* children are spliced directly.
*/
private void splice_body(Xml.Doc* child_doc, Xml.Doc* parent_doc) {
var child_body = find_body(child_doc);
var parent_body = find_body(parent_doc);
if (child_body == null || parent_body == null) {
return;
}
var outlet = find_first_element(parent_body, "pstm-content");
var materialise_as = child_body->get_prop("pstm-materialise-as");
if (materialise_as != null) {
var wrapper = parent_body->new_child(null, (!)materialise_as, null);
wrapper->unlink();
copy_non_pstm_attrs(child_body, wrapper);
for (var child = child_body->children; child != null; child = child->next) {
if (is_structural_directive(child)) {
continue;
}
var copied = child->doc_copy(parent_doc, 1);
if (copied != null) {
wrapper->add_child(copied);
}
}
if (outlet != null) {
outlet->add_prev_sibling(wrapper);
} else {
parent_body->add_child(wrapper);
}
} else {
for (var child = child_body->children; child != null; child = child->next) {
if (is_structural_directive(child)) {
continue;
}
var copied = child->doc_copy(parent_doc, 1);
if (copied == null) {
continue;
}
if (outlet != null) {
outlet->add_prev_sibling(copied);
} else {
parent_body->add_child(copied);
}
}
}
if (outlet != null) {
outlet->unlink();
}
}
/** Copies an element's non-`pstm-*` attributes onto another element. */
private static void copy_non_pstm_attrs(Xml.Node* src, Xml.Node* dst) {
for (var attr = src->properties; attr != null; attr = attr->next) {
var attr_name = (string) attr->name;
if (attr_name.has_prefix("pstm-")) {
continue;
}
var value = src->get_prop(attr_name);
if (value != null) {
dst->set_prop(attr_name, (!)value);
}
}
}
private static string? read_directive_in_doc(Xml.Doc* doc, string name) {
var root = doc->get_root_element();
if (root == null) {
return null;
}
return read_directive_text(root, name);
}
private static bool is_structural_directive(Xml.Node* node) {
if (!is_element(node)) {
return false;
}
var name = (string) node->name;
return name == "pstm-template" || name == "pstm-name" || name == "pstm-title" || name == "pstm-uri" || name == "pstm-content";
}
/** Fragment-body metadata elements skipped when inlining (e.g. pstm-input). */
private static bool is_fragment_metadata(Xml.Node* node) {
if (!is_element(node)) {
return false;
}
var name = (string) node->name;
return name == "pstm-input" || name.has_prefix("pstm-");
}
private void inline_fragments(Xml.Doc* doc) {
bool changed = true;
while (changed) {
changed = false;
var includes = find_all_elements(doc, "pstm-fragment");
foreach (var include in includes) {
if (inline_one_fragment(doc, include)) {
changed = true;
}
}
}
}
private bool inline_one_fragment(Xml.Doc* doc, Xml.Node* include) {
var name = include->get_prop("name");
if (name == null) {
return false;
}
Xml.Doc* fragment_doc;
if (!fragments_by_name.try_get((!)name, out fragment_doc)) {
return false;
}
var fragment_body = find_body(fragment_doc);
if (fragment_body == null) {
return false;
}
// Merge the fragment's into the document's .
merge_head(fragment_doc, doc);
var inputs = collect_input_names(fragment_body);
var include_materialise = include->get_prop("pstm-materialise-as");
var fragment_materialise = fragment_body->get_prop("pstm-materialise-as");
var materialise_as = include_materialise ?? fragment_materialise;
bool wrap = (materialise_as != null) || inputs.length > 0;
string tag = materialise_as ?? "div";
if (wrap) {
var parent = include->parent;
if (parent == null) {
return false;
}
var wrapper = parent->new_child(null, tag, null);
wrapper->unlink();
include->add_prev_sibling(wrapper);
// Carry the fragment 's attributes when it supplied the tag.
if (include_materialise == null && fragment_materialise != null) {
copy_non_pstm_attrs(fragment_body, wrapper);
}
for (var child = fragment_body->children; child != null; child = child->next) {
if (is_fragment_metadata(child)) {
continue;
}
var copied = child->doc_copy(doc, 1);
if (copied != null) {
wrapper->add_child(copied);
}
}
foreach (var input_name in inputs) {
var expr = include->get_prop(input_name);
if (expr != null) {
wrapper->set_prop(@"stm-def-$input_name-as", (!)expr);
}
}
} else {
for (var child = fragment_body->children; child != null; child = child->next) {
if (is_fragment_metadata(child)) {
continue;
}
var copied = child->doc_copy(doc, 1);
if (copied != null) {
include->add_prev_sibling(copied);
}
}
}
include->unlink();
return true;
}
private string[] collect_input_names(Xml.Node* fragment_body) {
var names = new string[0];
for (var n = fragment_body->children; n != null; n = n->next) {
if (is_element(n) && (string) n->name == "pstm-input") {
var input_name = n->get_prop("name");
if (input_name != null) {
names += (!)input_name;
}
}
}
return names;
}
private void rewrite_resources(Xml.Doc* doc) {
var root = doc->get_root_element();
if (root != null) {
rewrite_resources_in(root);
}
}
private void rewrite_resources_in(Xml.Node* node) {
if (!is_element(node)) {
return;
}
string[] to_add_attr = new string[0];
string[] to_add_val = new string[0];
string[] to_remove = new string[0];
for (var attr = node->properties; attr != null; attr = attr->next) {
var attr_name = (string) attr->name;
if (attr_name.has_prefix("pstm-res-")) {
var target_attr = attr_name.substring("pstm-res-".length);
var value = node->get_prop(attr_name);
if (value != null) {
to_add_attr += target_attr;
to_add_val += @"/_statum/resource/$((!)value)";
}
to_remove += attr_name;
}
}
for (int i = 0; i < to_add_attr.length; i++) {
node->set_prop(to_add_attr[i], to_add_val[i]);
}
foreach (var attr_name in to_remove) {
node->unset_prop(attr_name);
}
for (var child = node->children; child != null; child = child->next) {
rewrite_resources_in(child);
}
}
// ------------------------------------------------------------------
// Variant rendering
// ------------------------------------------------------------------
/**
* Renders the composed document to its single HTML form (stripping every
* `pstm-*` directive) and precompresses it. Pages no longer have
* variants: a page is a static, precompressed document, so there is
* exactly one.
*/
private VariantData build_variant(Xml.Doc* doc) throws GLib.Error {
strip_pstm(doc);
var html = serialize(doc);
return compress_variant(html);
}
private VariantData compress_variant(string html) throws GLib.Error {
var data = new VariantData();
data.identity = html.data;
var buffer = new ByteBuffer.from_byte_array(html.data);
var gzip = new GzipCompressor(9).compress_buffer(buffer, null).to_array();
var zstd = new ZstdCompressor(19).compress_buffer(buffer, null).to_array();
var br = new BrotliCompressor(11).compress_buffer(buffer, null).to_array();
data.has_gzip = gzip.length < html.length;
data.has_zstd = zstd.length < html.length;
data.has_br = br.length < html.length;
data.gzip = data.has_gzip ? gzip : null;
data.zstd = data.has_zstd ? zstd : null;
data.br = data.has_br ? br : null;
return data;
}
/** Remove every `pstm-*` attribute and `pstm-*` element from the tree. */
private void strip_pstm(Xml.Doc* doc) {
var root = doc->get_root_element();
if (root != null) {
strip_pstm_from(root);
}
}
private void strip_pstm_from(Xml.Node* node) {
string[] to_remove = new string[0];
for (var attr = node->properties; attr != null; attr = attr->next) {
var attr_name = (string) attr->name;
if (attr_name.has_prefix("pstm-")) {
to_remove += attr_name;
}
}
foreach (var attr_name in to_remove) {
node->unset_prop(attr_name);
}
var child = node->children;
while (child != null) {
var next = child->next;
if (is_element(child) && ((string) child->name).has_prefix("pstm-")) {
child->unlink();
} else if (is_element(child)) {
strip_pstm_from(child);
}
child = next;
}
}
// ------------------------------------------------------------------
// Code generation
// ------------------------------------------------------------------
private string emit(string class_name, string route, VariantData variant) throws GLib.Error {
var page_hash = compute_hash(variant.identity);
var b = new StringBuilder();
string i1 = namespace_name != null ? " " : "";
string i2 = namespace_name != null ? " " : " ";
string i3 = namespace_name != null ? " " : " ";
b.append("using Statum;\n");
b.append("using Invercargill;\n");
b.append("using Invercargill.DataStructures;\n");
b.append("using Astralis;\n\n");
if (namespace_name != null) {
b.append_printf("namespace %s {\n\n", namespace_name);
}
b.append_printf("%s// Generated by statum-mkpstm\n", i1);
b.append_printf("%spublic class %s : StatumPage {\n\n", i1, class_name);
b.append_printf("%spublic override string route { get { return \"%s\"; } }\n", i2, escape(route));
b.append_printf("%spublic override string content_type { get { return \"text/html; charset=utf-8\"; } }\n\n", i2);
// get_best_encoding: smallest available supported encoding.
b.append_printf("%spublic override string get_best_encoding(Set supported) {\n", i2);
if (variant.has_br) {
b.append_printf("%sif (supported.has(\"br\")) return \"br\";\n", i3);
}
if (variant.has_zstd) {
b.append_printf("%sif (supported.has(\"zstd\")) return \"zstd\";\n", i3);
}
if (variant.has_gzip) {
b.append_printf("%sif (supported.has(\"gzip\")) return \"gzip\";\n", i3);
}
b.append_printf("%sreturn \"identity\";\n", i3);
b.append_printf("%s}\n\n", i2);
// get_encoding
b.append_printf("%spublic override unowned uint8[] get_encoding(string encoding) {\n", i2);
b.append_printf("%sswitch (encoding) {\n", i3);
b.append_printf("%scase \"identity\": return IDENTITY;\n", i3);
if (variant.has_gzip && variant.gzip != null) {
b.append_printf("%scase \"gzip\": return GZIP;\n", i3);
}
if (variant.has_zstd && variant.zstd != null) {
b.append_printf("%scase \"zstd\": return ZSTD;\n", i3);
}
if (variant.has_br && variant.br != null) {
b.append_printf("%scase \"br\": return BR;\n", i3);
}
b.append_printf("%sdefault: return IDENTITY;\n", i3);
b.append_printf("%s}\n", i3);
b.append_printf("%s}\n\n", i2);
// get_etag_for (ETag = "-")
b.append_printf("%spublic override string get_etag_for(string encoding) {\n", i2);
b.append_printf("%sreturn \"\\\"%s-%%s\\\"\".printf(encoding);\n", i3, page_hash);
b.append_printf("%s}\n\n", i2);
// Byte arrays.
emit_one_array(b, i2, i3, "IDENTITY", variant.identity);
if (variant.has_gzip && variant.gzip != null) {
emit_one_array(b, i2, i3, "GZIP", (!)variant.gzip);
}
if (variant.has_zstd && variant.zstd != null) {
emit_one_array(b, i2, i3, "ZSTD", (!)variant.zstd);
}
if (variant.has_br && variant.br != null) {
emit_one_array(b, i2, i3, "BR", (!)variant.br);
}
b.append_printf("%s}\n", i1);
if (namespace_name != null) {
b.append("}\n");
}
return b.str;
}
private void emit_one_array(StringBuilder b, string i2, string i3, string name, uint8[] data) {
b.append_printf("%sprivate const uint8[] %s = {\n", i2, name);
for (int i = 0; i < data.length; i++) {
if (i == 0) {
b.append(i3);
} else if (i % 16 == 0) {
b.append(",\n").append(i3);
} else {
b.append(", ");
}
b.append_printf("0x%02x", data[i]);
}
b.append_printf("\n%s};\n\n", i2);
}
// ------------------------------------------------------------------
// DOM helpers
// ------------------------------------------------------------------
private Xml.Doc* parse_html_file(string path) throws GLib.Error {
var file = File.new_for_path(path);
if (!file.query_exists()) {
throw new IOError.NOT_FOUND(@"Page file '$path' does not exist");
}
uint8[] contents;
string etag_out;
file.load_contents(null, out contents, out etag_out);
// Parse with libxml2's HTML parser (mirrors Astralis' MarkupDocument)
// so the tree has HTML semantics and serializes as HTML, not XML.
int opts = (int)(Html.ParserOption.RECOVER |
Html.ParserOption.NOERROR |
Html.ParserOption.NOWARNING |
Html.ParserOption.NOBLANKS |
Html.ParserOption.NONET);
char[] buffer = ((string) contents).to_utf8();
var doc = Html.Doc.read_memory(buffer, buffer.length, path, "utf-8", opts);
if (doc == null) {
throw new IOError.FAILED(@"Failed to parse HTML file '$path'");
}
return doc;
}
private static Xml.Node* find_body(Xml.Doc* doc) {
var root = doc->get_root_element();
if (root == null) {
return null;
}
return find_first_element(root, "body");
}
private static Xml.Node* find_head(Xml.Doc* doc) {
var root = doc->get_root_element();
if (root == null) {
return null;
}
return find_first_element(root, "head");
}
/**
* Validates that `doc` has both a and a element, which every
* Statum HTML input (pages, templates and fragments) must declare.
*/
private void validate_document(Xml.Doc* doc, string source) throws GLib.Error {
if (find_head(doc) == null) {
throw new IOError.FAILED(@"\"$source\" has no element (a and are required)");
}
if (find_body(doc) == null) {
throw new IOError.FAILED(@"\"$source\" has no element (a and are required)");
}
}
/**
* Validates that the structural directives ,
* and appear only inside .
*/
private void validate_head_directives(Xml.Doc* doc, string source) throws GLib.Error {
var root = doc->get_root_element();
if (root == null) {
return;
}
string[] names = { "pstm-title", "pstm-template", "pstm-name", "pstm-uri" };
foreach (var name in names) {
var nodes = new Series();
find_all_from(root, name, nodes);
foreach (var node in nodes) {
if (!is_descendant_of_head(node)) {
throw new IOError.FAILED(@"\"$source\": <$name> must appear inside ");
}
}
}
}
/** Walks the ancestor chain to determine whether `node` is within . */
private static bool is_descendant_of_head(Xml.Node* node) {
for (var p = node->parent; p != null; p = p->parent) {
if (is_element(p) && ((string) p->name).down() == "head") {
return true;
}
}
return false;
}
private static Xml.Node* find_first_element(Xml.Node* root, string name) {
if (!is_element(root)) {
return null;
}
if (((string) root->name).down() == name.down()) {
return root;
}
for (var child = root->children; child != null; child = child->next) {
var found = find_first_element(child, name);
if (found != null) {
return found;
}
}
return null;
}
private static Xml.Node*[] find_all_elements(Xml.Doc* doc, string name) {
var collected = new Series();
var root = doc->get_root_element();
if (root != null) {
find_all_from(root, name, collected);
}
return collected.to_array();
}
private static void find_all_from(Xml.Node* node, string name, Series result) {
if (is_element(node) && ((string) node->name).down() == name.down()) {
result.add(node);
}
if (is_element(node)) {
for (var child = node->children; child != null; child = child->next) {
find_all_from(child, name, result);
}
}
}
private static string? read_directive_text(Xml.Node* body, string directive) {
var node = find_first_element(body, directive);
if (node == null) {
return null;
}
var text = element_text(node);
return text.strip().length > 0 ? text.strip() : null;
}
private static string? read_pstm_name(Xml.Doc* doc) {
var node = find_first_element(doc->get_root_element(), "pstm-name");
if (node == null) {
return null;
}
return element_text(node).strip();
}
private static string[] read_routes(Xml.Doc* doc) {
string[] routes = new string[0];
var nodes = find_all_elements(doc, "pstm-uri");
foreach (var node in nodes) {
var text = element_text(node).strip();
if (text.length > 0) {
routes += text;
}
}
return routes;
}
private static string element_text(Xml.Node* node) {
var b = new StringBuilder();
for (var child = node->children; child != null; child = child->next) {
if (child->type == ElementType.TEXT_NODE && child->content != null) {
b.append((string) child->content);
} else if (is_element(child)) {
b.append(element_text(child));
}
}
return b.str;
}
private static void apply_title(Xml.Doc* doc, string? title) {
if (title == null || title.length == 0) {
return;
}
var head = find_first_element(doc->get_root_element(), "head");
if (head == null) {
return;
}
var title_node = find_first_element(head, "title");
if (title_node == null) {
title_node = head->new_child(null, "title", null);
}
for (var child = title_node->children; child != null; child = child->next) {
child->unlink();
}
title_node->add_content(title);
}
private static string serialize(Xml.Doc* doc) {
// Serialize with libxml2's HTML serializer (htmlDocDumpMemory) so the
// output is valid HTML: no prolog, no self-closing
// ``, and correct void/non-void element handling. The doc was
// produced by the HTML parser, so it carries HTML structure.
string mem;
int len;
((Html.Doc*) doc)->dump_memory(out mem, out len);
return len > 0 ? mem.substring(0, len) : mem;
}
private static bool is_element(Xml.Node* node) {
return node != null && node->type == ElementType.ELEMENT_NODE;
}
private static string compute_hash(uint8[] data) {
var checksum = new Checksum(ChecksumType.SHA512);
checksum.update(data, data.length);
return checksum.get_string();
}
private static string escape(string s) {
return s.replace("\\", "\\\\").replace("\"", "\\\"");
}
private static void write_file(string path, string contents) throws GLib.Error {
var file = File.new_for_path(path);
var stream = new DataOutputStream(file.replace(null, false, FileCreateFlags.NONE));
stream.put_string(contents);
stream.close();
}
private static string make_pascal_case(string input) {
var basename = Path.get_basename(input);
var dot = basename.last_index_of(".");
if (dot > 0) {
basename = basename.substring(0, dot);
}
var result = new StringBuilder();
var capitalize_next = true;
foreach (var c in basename.data) {
if (c == '-' || c == '_' || c == ' ' || c == '.') {
capitalize_next = true;
} else {
if (capitalize_next) {
result.append_printf("%c", c >= 'a' && c <= 'z' ? c - 32 : c);
capitalize_next = false;
} else {
result.append_printf("%c", c);
}
}
}
return result.str;
}
// ------------------------------------------------------------------
// Model
// ------------------------------------------------------------------
private class VariantData : Object {
public uint8[] identity;
public uint8[]? gzip;
public uint8[]? zstd;
public uint8[]? br;
public bool has_gzip;
public bool has_zstd;
public bool has_br;
}
}
}