⚠️ CRITICAL IMPLEMENTATION NOTES - READ FIRST ⚠️
Code Style Requirements
DO NOT use GLib.List, GLib.HashTable, or Libgee collections
- Use ONLY
Invercargill.DataStructuresfor all collectionsVector<T>instead ofGLib.List<T>or arraysDictionary<TKey, TValue>instead ofGLib.HashTableorGLib.MapHashSet<T>for set operationsReference the Invercargill Library
- Analyze
../Invercargill/src/lib/for DataStructures and Expressions patterns- Key directories:
DataStructures/,Expressions/,Mapping/- Use
Invercargill.Expressionsfor all expression handlingNo Raw SQL in High-Level APIs
- All expression parameters use Invercargill.Expressions syntax
- Raw SQL is only generated internally by dialect implementations
- Example:
.where("user_id > 100")NOT.where("user_id > 100 AND ...")Honor the Phase 4 Design Document
- The
phase-4-projections-and-joins.mddocument contains carefully designed APIs- Variable scoping and type-based translation are core concepts
- WHERE/HAVING split for aggregate detection is required
- Nested projections via
select<TProjection>()andselect_many<TProjection>()Key Invercargill.DataStructures Types
Type Usage Vector<T>Ordered collection, use instead of arrays or GLib.List Dictionary<TKey, TValue>Key-value mapping, use instead of GLib.HashTable HashSet<T>Unique items collection Enumerable<T>Base class for sequences, provides LINQ-style operations Key Invercargill.Expressions Types
Type Usage ExpressionBase interface for all expressions ExpressionVisitorVisitor pattern for expression traversal ExpressionParserParses expression strings to expression trees BinaryExpressionBinary operations (AND, OR, comparisons) GlobalFunctionCallExpressionSQL functions like COUNT, SUM, etc.
This document provides a detailed implementation plan for Phase 4: Projections and Joins. The implementation adds read-only, composable query shapes that can join multiple entities and support nested projections.
flowchart TB
subgraph Public API
OS[OrmSession]
PB[ProjectionBuilder T]
PQ[ProjectionQuery T]
end
subgraph Core Types
PD[ProjectionDefinition]
SD[SourceDefinition]
SELD[SelectionDefinition]
JD[JoinDefinition]
end
subgraph Expression Translation
ETSV[ExpressionToSqlVisitor]
AA[AggregateAnalyzer]
FNR[FriendlyNameResolver]
VT[VariableTranslator]
end
subgraph SQL Generation
SDI[SqlDialect Interface]
SQLD[SqliteDialect]
PSB[ProjectionSqlBuilder]
end
subgraph Materialization
PM[ProjectionMapper T]
end
OS --> PB
OS --> PQ
PB --> PD
PD --> SD
PD --> SELD
PD --> JD
PQ --> ETSV
PQ --> PSB
PSB --> SDI
SDI --> SQLD
ETSV --> AA
ETSV --> FNR
ETSV --> VT
PQ --> PM
src/orm/projections/projection-definition.valaResponsibility: Stores the complete definition of a projection including source, joins, selections, and group by clauses.
Key Classes:
ProjectionDefinition - Main container classSourceDefinition - Represents the primary entity source with variable name and typeJoinDefinition - Represents a joined entity with conditionSelectionDefinition - Base class for selectionsDependencies: None (pure data structures)
// Key structure
public class ProjectionDefinition : Object {
public Type result_type { get; set; }
public SourceDefinition source { get; set; }
public Vector<JoinDefinition> joins { get; set; }
public Vector<SelectionDefinition> selections { get; set; }
public Vector<string> group_by { get; set; }
}
public class SourceDefinition : Object {
public Type entity_type { get; set; }
public string variable_name { get; set; }
public string table_name { get; set; }
}
public class JoinDefinition : Object {
public Type entity_type { get; set; }
public string variable_name { get; set; }
public string table_name { get; set; }
public string join_condition { get; set; } // Invercargill expression
}
src/orm/projections/selection-types.valaResponsibility: Defines the different types of selections (scalar, nested projection, collection).
Key Classes:
ScalarSelection - Simple value selection with setterNestedProjectionSelection - 1:1 nested projectionCollectionProjectionSelection - 1:N nested projectionDependencies: ProjectionDefinition
public abstract class SelectionDefinition : Object {
public string friendly_name { get; set; }
public string expression { get; set; } // Invercargill expression
public Type value_type { get; set; }
}
public class ScalarSelection : SelectionDefinition {
public PropertySetter setter { get; set; }
}
public class NestedProjectionSelection : SelectionDefinition {
public Type projection_type { get; set; }
public string entry_point_expression { get; set; }
public PropertySetter setter { get; set; }
}
public class CollectionProjectionSelection : SelectionDefinition {
public Type projection_type { get; set; }
public string entry_point_expression { get; set; }
public PropertySetter setter { get; set; }
}
src/orm/projections/projection-builder.valaResponsibility: Fluent builder for creating projection definitions. Follows the same pattern as EntityMapperBuilder<T>.
Key Classes:
ProjectionBuilder<T> - Generic builder with fluent APIDependencies:
ProjectionDefinitionSelectionTypesOrmSession (for entity mapper lookup)Pattern Reference: See EntityMapperBuilder<T>
public class ProjectionBuilder<T> : Object {
private ProjectionDefinition _definition;
private OrmSession _session;
public ProjectionBuilder<T> source<TEntity>(string variable_name)
public ProjectionBuilder<T> join<TEntity>(string variable_name, string condition)
public ProjectionBuilder<T> select<TProp>(string friendly_name, string expr, owned PropertySetter<T, TProp> setter)
public ProjectionBuilder<T> select<TProjection>(string friendly_name, string entry_point, owned PropertySetter<T, TProjection> setter)
public ProjectionBuilder<T> select_many<TProjection>(string friendly_name, string entry_point, owned PropertySetter<T, Enumerable<TProjection>> setter)
public ProjectionBuilder<T> group_by(params string[] expressions)
public ProjectionDefinition build()
}
src/orm/projections/projection-query.valaResponsibility: Query builder for projections, similar to Query<T> but with projection-specific features.
Key Classes:
ProjectionQuery<T> - Fluent query builderDependencies:
ProjectionDefinitionOrmSessionExpressionToSqlVisitorPattern Reference: See Query<T>
public class ProjectionQuery<T> : Object {
private OrmSession _session;
private ProjectionDefinition _projection;
private Expression? _where_filter;
private Expression? _having_filter;
private Vector<OrderByClause> _orderings;
private int? _limit;
private int? _offset;
public ProjectionQuery<T> where(string expression)
public ProjectionQuery<T> or_where(string expression)
public ProjectionQuery<T> order_by(string expression)
public ProjectionQuery<T> order_by_desc(string expression)
public ProjectionQuery<T> limit(int count)
public ProjectionQuery<T> offset(int count)
public Enumerable<T> materialise() throws SqlError
public async Enumerable<T> materialise_async() throws SqlError
}
src/orm/projections/projection-mapper.valaResponsibility: Materializes projection results from database rows into result objects.
Key Classes:
ProjectionMapper<T> - Materialization logicDependencies:
ProjectionDefinitionSelectionTypesPattern Reference: See EntityMapper<T>
public class ProjectionMapper<T> : Object {
private ProjectionDefinition _definition;
public T materialise(Properties row) throws Error
public Enumerable<T> materialise_all(IEnumerable<Properties> rows) throws Error
}
src/orm/projections/aggregate-analyzer.valaResponsibility: Analyzes expressions to detect aggregate functions for WHERE/HAVING split.
Key Classes:
AggregateAnalyzer - Expression visitor that detects aggregatesDependencies: Invercargill.Expressions
public class AggregateAnalyzer : Object, ExpressionVisitor {
private bool _contains_aggregate;
public bool contains_aggregate(Expression expr)
// Visitor methods detect COUNT, SUM, AVG, MIN, MAX
public void visit_global_function_call(GlobalFunctionCallExpression expr)
}
src/orm/projections/friendly-name-resolver.valaResponsibility: Resolves friendly names to their underlying expressions.
Key Classes:
FriendlyNameResolver - Lookup serviceDependencies: ProjectionDefinition
public class FriendlyNameResolver : Object {
private ProjectionDefinition _projection;
private Map<string, string> _friendly_names; // name -> expression
public string? resolve(string friendly_name)
public Expression? resolve_as_expression(string friendly_name)
public bool has_nested_projection(string friendly_name)
public ProjectionDefinition? get_nested_projection(string friendly_name)
}
src/orm/projections/variable-translator.valaResponsibility: Translates user-defined variable names to SQL aliases with type tracking.
Key Classes:
VariableTranslator - Variable to alias mappingVariableScope - Scope for a single projectionDependencies: ProjectionDefinition
public class VariableTranslator : Object {
private Map<string, string> _user_to_sql; // user var -> sql alias
private Map<string, Type> _variable_types; // user var -> entity type
private int _alias_counter;
public void register_variable(string user_var, Type entity_type, string table_name)
public string translate(string user_var)
public Type get_variable_type(string user_var)
public string generate_alias(Type entity_type)
}
src/orm/projections/projection-sql-builder.valaResponsibility: Builds complete SELECT queries from ProjectionDefinition.
Key Classes:
ProjectionSqlBuilder - SQL generation coordinatorDependencies:
ProjectionDefinitionVariableTranslatorFriendlyNameResolverAggregateAnalyzerSqlDialect
public class ProjectionSqlBuilder : Object {
private ProjectionDefinition _definition;
private SqlDialect _dialect;
private VariableTranslator _translator;
private FriendlyNameResolver _resolver;
public string build_select(
Expression? where_clause,
Expression? having_clause,
Vector<OrderByClause> order_by,
int64? limit,
int64? offset
)
public string build_subquery_wrapper(string inner_sql, string combined_where)
}
src/orm/projections/projection-errors.valaResponsibility: Define projection-specific error types.
public errordomain ProjectionError {
ENTRY_TYPE_MISMATCH, // Entry point type doesn't match child's source
UNDEFINED_FRIENDLY_NAME, // Friendly name not found
DUPLICATE_VARIABLE, // Variable name already defined
MISSING_GROUP_BY, // Aggregate without group_by
NOT_REGISTERED, // Projection not registered
NESTED_PROJECTION_ERROR // Error in nested projection
}
src/orm/orm-session.valaChanges Required:
Add projection registry:
private Dictionary<Type, ProjectionDefinition> _projections;
Add registration method:
public void register_projection<T>(owned GLib.Func<ProjectionBuilder<T>> func)
Modify query<T>() to support both entities and projections:
public Query<T> query<T>() // Existing - for entities
public ProjectionQuery<T> projection_query<T>() // New - for projections
Add internal method for projection execution:
internal Enumerable<T> execute_projection_query<T>(ProjectionQuery<T> query) throws SqlError
internal async Enumerable<T> execute_projection_query_async<T>(ProjectionQuery<T> query) throws SqlError
Add helper method to check if type is registered:
public bool is_entity_registered<T>()
public bool is_projection_registered<T>()
Lines to modify: ~29-41 (constructor), ~110-112 (query method), add new methods after line 327
src/expressions/expression-to-sql-visitor.valaChanges Required:
Add constructor overload for projection context:
private ProjectionDefinition? _projection;
private Map<string, string>? _variable_aliases;
public ExpressionToSqlVisitor.with_projection(
SqlDialect dialect,
ProjectionDefinition? projection,
Map<string, string>? variable_aliases
)
Modify visit_variable() to handle variable translation:
public void visit_variable(VariableExpression expr) {
if (_variable_aliases != null && _variable_aliases.has_key(expr.variable_name)) {
_sql.append(_variable_aliases[expr.variable_name]);
return;
}
// Existing logic...
}
Add aggregate detection method:
public bool contains_aggregate(Expression expr)
Track aggregate functions during visit:
private bool _found_aggregate;
public void visit_global_function_call(GlobalFunctionCallExpression expr) {
string func_name = expr.function_name.up();
if (func_name == "COUNT" || func_name == "SUM" || ...) {
_found_aggregate = true;
}
// Existing logic...
}
Lines to modify: ~23-46 (fields and constructors), ~169-178 (visit_variable), ~256-293 (visit_global_function_call)
src/dialects/sql-dialect.valaChanges Required:
Add projection SELECT builder method:
/**
* Builds a SELECT statement with JOINs for a projection query.
*/
public abstract string build_projection_select(
Vector<SourceDefinition> sources,
Vector<SelectionDefinition> selections,
Vector<string> group_by,
string? where_clause,
string? having_clause,
Vector<OrderByClause> order_by,
int64? limit,
int64? offset
);
Add subquery wrapper method:
/**
* Builds a subquery wrapper for mixed aggregate/non-aggregate OR conditions.
*/
public abstract string wrap_subquery_for_mixed_or(
string inner_query,
string combined_where
);
Add alias generation method:
/**
* Generates a table alias with type information for debugging.
*/
public abstract string generate_table_alias(int index, string type_name);
Lines to add: After line 170 (end of interface)
src/dialects/sqlite-dialect.valaChanges Required:
Implement build_projection_select():
public string build_projection_select(
Vector<SourceDefinition> sources,
Vector<SelectionDefinition> selections,
Vector<string> group_by,
string? where_clause,
string? having_clause,
Vector<OrderByClause> order_by,
int64? limit,
int64? offset
) {
// Implementation as specified in phase-4-projections-and-joins.md
}
Implement wrap_subquery_for_mixed_or():
public string wrap_subquery_for_mixed_or(string inner_query, string combined_where) {
return @"SELECT * FROM ($inner_query) subq WHERE $combined_where";
}
Implement generate_table_alias():
public string generate_table_alias(int index, string type_name) {
return @"val_$(index)_$(type_name)";
}
Lines to add: After line 544 (end of class)
src/meson.buildChanges Required:
Add new projection source files:
# Projection sources
sources += files('orm/projections/projection-definition.vala')
sources += files('orm/projections/selection-types.vala')
sources += files('orm/projections/projection-builder.vala')
sources += files('orm/projections/projection-query.vala')
sources += files('orm/projections/projection-mapper.vala')
sources += files('orm/projections/aggregate-analyzer.vala')
sources += files('orm/projections/friendly-name-resolver.vala')
sources += files('orm/projections/variable-translator.vala')
sources += files('orm/projections/projection-sql-builder.vala')
sources += files('orm/projections/projection-errors.vala')
Add projection test executable:
# Projection test executable
projection_test_exe = executable('projection-test', 'tests/projection-test.vala',
dependencies: [dependencies, invercargill_sql_dep],
link_with: invercargill_sql
)
test('Projection tests', projection_test_exe)
Lines to modify: After line 41 (ORM sources section), after line 119 (test section)
1. src/orm/projections/projection-errors.vala
└── Pure error definitions
2. src/orm/projections/projection-definition.vala
└── Core data structures
3. src/orm/projections/selection-types.vala
└── Selection type hierarchy
4. src/orm/projections/aggregate-analyzer.vala
└── Depends: Invercargill.Expressions
5. src/orm/projections/variable-translator.vala
└── Depends: ProjectionDefinition
6. src/orm/projections/friendly-name-resolver.vala
└── Depends: ProjectionDefinition, SelectionTypes
7. src/orm/projections/projection-builder.vala
└── Depends: ProjectionDefinition, SelectionTypes, OrmSession
8. Modify: src/dialects/sql-dialect.vala
└── Add abstract methods
9. Modify: src/dialects/sqlite-dialect.vala
└── Implement abstract methods
10. Modify: src/expressions/expression-to-sql-visitor.vala
└── Add projection context support
11. src/orm/projections/projection-sql-builder.vala
└── Depends: All above components
12. src/orm/projections/projection-mapper.vala
└── Depends: ProjectionDefinition
13. src/orm/projections/projection-query.vala
└── Depends: All above components
14. Modify: src/orm/orm-session.vala
└── Add projection registration and query methods
15. Modify: src/meson.build
└── Add new files and test target
16. Create: src/tests/projection-test.vala
└── Comprehensive tests
flowchart TD
subgraph Foundation
PE[projection-errors.vala]
PD[projection-definition.vala]
ST[selection-types.vala]
end
subgraph Analysis
AA[aggregate-analyzer.vala]
VT[variable-translator.vala]
FNR[friendly-name-resolver.vala]
end
subgraph Dialect
SD[sql-dialect.vala - modify]
SQLD[sqlite-dialect.vala - modify]
end
subgraph Expression
ETSV[expression-to-sql-visitor.vala - modify]
end
subgraph Builder
PB[projection-builder.vala]
end
subgraph SQLGen
PSB[projection-sql-builder.vala]
end
subgraph Query
PM[projection-mapper.vala]
PQ[projection-query.vala]
end
subgraph Integration
OS[orm-session.vala - modify]
MB[meson.build - modify]
TEST[projection-test.vala - create]
end
PE --> PD
PD --> ST
PD --> AA
PD --> VT
PD --> FNR
ST --> FNR
PD --> PB
ST --> PB
SD --> SQLD
PD --> SD
PD --> ETSV
VT --> ETSV
AA --> PSB
VT --> PSB
FNR --> PSB
SQLD --> PSB
ETSV --> PSB
PD --> PM
ST --> PM
PD --> PQ
PSB --> PQ
PM --> PQ
PB --> OS
PQ --> OS
PE --> OS
PD --> OS
OS --> MB
PQ --> TEST
PM --> TEST
Decision: Each projection maintains its own variable scope. Variable names are translated to SQL aliases using the format val_N_TypeName.
Rationale: This prevents collisions when nesting projections and makes debugging easier.
Caveat: When nesting projections, the child's variables must be translated based on TYPE, not name, since the parent may use different variable names.
Decision: Automatically detect aggregate functions and split mixed expressions into WHERE (non-aggregate) and HAVING (aggregate) clauses.
Rationale: SQL requires this split, and automating it improves developer experience.
Caveat: When OR combines aggregate and non-aggregate expressions, wrap in subquery. This may have performance implications.
Decision: select_many collections are grouped client-side after fetching flat results.
Rationale: Avoids complex JSON aggregation SQL that varies between databases.
Caveat: Large result sets may have memory implications. Consider pagination.
Decision: The entry point variable type must exactly match the child projection's source type.
Rationale: Type safety prevents runtime errors.
Caveat: This is checked at registration time, not compile time. Consider adding compile-time checks via generics if possible.
Decision: Friendly names must be unique within a projection.
Rationale: Used for WHERE/ORDER BY expressions - duplicates would be ambiguous.
Caveat: Nested projection friendly names are accessed via dot notation (e.g., profile.id).
Decision: Binary blob types should use Invercargill.BinaryData interface with Invercargill.DataStructures.ByteBuffer() - NOT uint8[] on external APIs.
Rationale: Consistency with the Invercargill framework conventions.
ProjectionBuilder Tests
VariableTranslator Tests
FriendlyNameResolver Tests
AggregateAnalyzer Tests
Simple Projection Query
Join Projections
Aggregate Projections
Nested Projections
Error Cases
When switching to Code mode, the implementation should be broken into these tasks:
// Fluent method returns this
public EntityMapperBuilder<T> column<TProp>(string name, ...) {
// Configure internal state
return this;
}
// Terminal method builds result
public EntityMapper<T> build() {
var mapper = new EntityMapper<T>();
// Apply configuration
return mapper;
}
// Store state internally
private Expression? _filter;
private Vector<OrderByClause> _orderings;
// Fluent methods modify state
public Query<T> where(string expression) {
_filter = ExpressionParser.parse(expression);
return this;
}
// Terminal method delegates to session
public Enumerable<T> materialise() throws SqlError {
return _session.execute_query<T>(this);
}
public class ExpressionToSqlVisitor : Object, ExpressionVisitor {
private StringBuilder _sql;
public void visit_binary(BinaryExpression expr) {
_sql.append("(");
expr.left.accept(this);
_sql.append(operator);
expr.right.accept(this);
_sql.append(")");
}
public string get_sql() { return _sql.str; }
}
// Define result type
public class UserSummary : Object {
public int64 id { get; set; }
public string name { get; set; }
}
// Register
session.register_projection<UserSummary>(p => p
.source<User>("u")
.select<int64>("id", "u.id", (x, v) => x.id = v)
.select<string>("name", "u.name", (x, v) => x.name = v)
);
// Query
var users = session.projection_query<UserSummary>()
.where("id > 100")
.order_by("name")
.materialise();
public class OrderStats : Object {
public int64 user_id { get; set; }
public int64 order_count { get; set; }
public double total_spent { get; set; }
}
session.register_projection<OrderStats>(p => p
.source<User>("u")
.join<Order>("o", "u.id == o.user_id")
.group_by("u.id")
.select<int64>("user_id", "u.id", (x, v) => x.user_id = v)
.select<int64>("order_count", "COUNT(o.id)", (x, v) => x.order_count = v)
.select<double>("total_spent", "SUM(o.total)", (x, v) => x.total_spent = v)
);
var stats = session.projection_query<OrderStats>()
.where("order_count >= 5") // Becomes HAVING
.materialise();
public class ProfileSummary : Object {
public int64 id { get; set; }
public string bio { get; set; }
}
public class UserWithProfile : Object {
public int64 id { get; set; }
public ProfileSummary profile { get; set; }
}
session.register_projection<ProfileSummary>(p => p
.source<Profile>("pr")
.select<int64>("id", "pr.id", (x, v) => x.id = v)
.select<string>("bio", "pr.bio", (x, v) => x.bio = v)
);
session.register_projection<UserWithProfile>(p => p
.source<User>("u")
.join<Profile>("pr", "u.profile_id == pr.id")
.select<int64>("id", "u.id", (x, v) => x.id = v)
.select<ProfileSummary>("profile", "pr", (x, v) => x.profile = v)
);
var users = session.projection_query<UserWithProfile>()
.where("profile.id == 5") // Navigate into nested
.materialise();