Add shader preprocessor with #include support and refactor shaders to remove a bunch of duplicates

assets/shaders/camera.glsl

@@ -0,0 +1,9 @@
+#ifndef CAMERA_GLSL
+#define CAMERA_GLSL
+
+layout(std140, binding = 0) uniform Matrices {
+  mat4 projection;
+  mat4 view;
+};
+
+#endif // CAMERA_GLSL

assets/shaders/cube_shadow.glsl

@@ -1,8 +1,4 @@
-// UBOs
-layout(std140, binding = 0) uniform Matrices {
-  mat4 projection;
-  mat4 view;
-};
+#include "camera.glsl"
 
 // Uniforms
 layout(location = 1) uniform mat4 model;

assets/shaders/debug.glsl

@@ -1,8 +1,4 @@
-// UBOs
-layout(std140, binding = 0) uniform Matrices {
-  mat4 projection;
-  mat4 view;
-};
+#include "camera.glsl"
 
 layout(location = 2) uniform vec3 color;
 

assets/shaders/draw_cmds_data.glsl

@@ -0,0 +1,14 @@
+#ifndef DRAW_CMDS_DATA_GLSL
+#define DRAW_CMDS_DATA_GLSL
+
+struct DrawCmdData {
+  mat4 transform;
+  int materialIdx;
+};
+
+layout(std430, binding = 3) readonly buffer DrawCmdDatas {
+  // Access by gl_BaseInstance + gl_InstanceID
+  DrawCmdData draw_data[];
+};
+
+#endif // DRAW_CMDS_DATA_GLSL

assets/shaders/material.glsl

@@ -0,0 +1,44 @@
+#ifndef MATERIAL_GLSL
+#define MATERIAL_GLSL
+
+// You have to enable GL_ARB_bindless_texture extension if you're importing this
+
+struct Material {
+  vec4 albedo;
+  sampler2D albedo_map;
+  vec2 albedo_map_uv_scale;
+  sampler2D normal_map;
+  vec2 normal_map_uv_scale;
+  float metallic;
+  sampler2D metallic_map;
+  vec2 metallic_map_uv_scale;
+  float roughness;
+  sampler2D roughness_map;
+  vec2 roughness_map_uv_scale;
+  vec3 emission;
+  sampler2D emission_map;
+  vec2 emission_map_uv_scale;
+};
+
+layout(std430, binding = 2) readonly buffer Materials {
+  uint materials_count;
+  Material materials[];
+};
+
+vec4 getAlbedo(int materialIdx) {
+  return textureSize(materials[materialIdx].albedo_map, 0) == ivec2(0) ? vec4(pow(materials[materialIdx].albedo.rgb, vec3(2.2)), materials[materialIdx].albedo.a) : texture(materials[materialIdx].albedo_map, VertexOut.uv * materials[materialIdx].albedo_map_uv_scale);
+}
+
+float getRoughness(int materialIdx) {
+  return max(0.01, textureSize(materials[materialIdx].roughness_map, 0) == ivec2(0) ? materials[materialIdx].roughness : texture(materials[materialIdx].roughness_map, VertexOut.uv * materials[materialIdx].roughness_map_uv_scale).g);
+}
+
+float getMetallic(int materialIdx) {
+  return textureSize(materials[materialIdx].metallic_map, 0) == ivec2(0) ? materials[materialIdx].metallic : texture(materials[materialIdx].metallic_map, VertexOut.uv * materials[materialIdx].metallic_map_uv_scale).b;
+}
+
+vec3 getEmission(int materialIdx) {
+  return textureSize(materials[materialIdx].emission_map, 0) == ivec2(0) ? materials[materialIdx].emission : texture(materials[materialIdx].emission_map, VertexOut.uv * materials[materialIdx].emission_map_uv_scale).rgb;
+}
+
+#endif // MATERIAL_GLSL

assets/shaders/mesh.glsl

@@ -1,28 +1,15 @@
-#extension GL_ARB_bindless_texture : enable
+#extension GL_ARB_bindless_texture : require
 #extension GL_KHR_shader_subgroup_ballot : enable
 #extension GL_KHR_shader_subgroup_vote : enable
 
+#include "camera.glsl"
+#include "draw_cmds_data.glsl"
+
 // Keep in sync with cpu
 #define MAX_POINT_LIGHTS 8
 #define PI 3.1415926535897932384626433832795
 #define CSM_SPLITS 4
 
-struct DrawCmdData {
-  mat4 transform;
-  int materialIdx;
-};
-
-// UBOs
-layout(std140, binding = 0) uniform Matrices {
-  mat4 projection;
-  mat4 view;
-};
-
-layout(std430, binding = 3) readonly buffer DrawCmdDatas {
-  // Access by gl_DrawID
-  DrawCmdData draw_data[];
-};
-
 layout(location = 16, bindless_sampler) uniform sampler2DArrayShadow shadow_maps;
 layout(location = 17, bindless_sampler) uniform samplerCubeArrayShadow cube_shadow_maps;
 layout(location = 22, bindless_sampler) uniform sampler2D brdfLut;
@@ -78,6 +65,8 @@ void main() {
 
 #if FRAGMENT_SHADER
 
+#include "material.glsl"
+
 // Types
 struct Light {
   vec4 vPos;
@@ -96,34 +85,11 @@ struct Light {
   vec4 csm_split_points; // TODO: Maybe increase to 8, though it's probably too many
 };
 
-struct Material {
-  vec4 albedo;
-  sampler2D albedo_map;
-  vec2 albedo_map_uv_scale;
-  sampler2D normal_map;
-  vec2 normal_map_uv_scale;
-  float metallic;
-  sampler2D metallic_map;
-  vec2 metallic_map_uv_scale;
-  float roughness;
-  sampler2D roughness_map;
-  vec2 roughness_map_uv_scale;
-  vec3 emission;
-  sampler2D emission_map;
-  vec2 emission_map_uv_scale;
-};
-
-
 layout(std430, binding = 1) readonly buffer Lights {
   uint lights_count;
   Light lights[];
 };
 
-layout(std430, binding = 2) readonly buffer Materials {
-  uint materials_count;
-  Material materials[];
-};
-
 out vec4 FragColor;
 
 int getShadowMapIndex(int lightIdx) {
@@ -146,22 +112,6 @@ int getCSMSplit(int lightIdx, float depth) {
   return CSM_SPLITS - 1;
 }
 
-vec4 getAlbedo(int materialIdx) {
-  return textureSize(materials[materialIdx].albedo_map, 0) == ivec2(0) ? vec4(pow(materials[materialIdx].albedo.rgb, vec3(2.2)), materials[materialIdx].albedo.a) : texture(materials[materialIdx].albedo_map, VertexOut.uv * materials[materialIdx].albedo_map_uv_scale);
-}
-
-float getRoughness(int materialIdx) {
-  return max(0.01, textureSize(materials[materialIdx].roughness_map, 0) == ivec2(0) ? materials[materialIdx].roughness : texture(materials[materialIdx].roughness_map, VertexOut.uv * materials[materialIdx].roughness_map_uv_scale).g);
-}
-
-float getMetallic(int materialIdx) {
-  return textureSize(materials[materialIdx].metallic_map, 0) == ivec2(0) ? materials[materialIdx].metallic : texture(materials[materialIdx].metallic_map, VertexOut.uv * materials[materialIdx].metallic_map_uv_scale).b;
-}
-
-vec3 getEmission(int materialIdx) {
-  return textureSize(materials[materialIdx].emission_map, 0) == ivec2(0) ? materials[materialIdx].emission : texture(materials[materialIdx].emission_map, VertexOut.uv * materials[materialIdx].emission_map_uv_scale).rgb;
-}
-
 vec3 schlickFresnel(int matIdx, float NDotV) {
   vec3 f0 = mix(vec3(0.04), getAlbedo(matIdx).rgb, getMetallic(matIdx));
 
@@ -330,10 +280,11 @@ vec3 ibl(int matIdx, vec3 N, vec3 V) {
   float ambient_diff = dot(N, VertexOut.vUp) * 0.5 + 0.5;
   float ambient_spec = dot(R, VertexOut.vUp) * 0.5 + 0.5;
 
-  vec3 irradiance = vec3(1.0, 0.9764705882352941, 0.9921568627450981) * 79 * ambient_diff;
+  // TODO: don't hardcode this
+  vec3 irradiance = vec3(1.0, 0.9764705882352941, 0.9921568627450981) * 79 * ambient_diff * 0.02;
   vec3 diffuse = irradiance * getAlbedo(matIdx).rgb;
 
-  vec3 reflectedColor = vec3(0.9, 0.9064705882352941, 0.9921568627450981) * 79 * ambient_spec;
+  vec3 reflectedColor = vec3(0.9, 0.9064705882352941, 0.9921568627450981) * 79 * ambient_spec * 0.02;
   vec2 envBRDF = textureLod(brdfLut, vec2(NDotV, getRoughness(matIdx)), 0).rg;
   vec3 specular = reflectedColor * (F * envBRDF.x + envBRDF.y);
   return kD * diffuse + specular;

assets/shaders/post_process.glsl

@@ -69,7 +69,7 @@ vec3 linearToSRGB(vec3 color) {
 
 void main() {
   vec3 hdr_color = texture(screen_sampler, VertexOut.uv).rgb;
-  hdr_color = ACESFitted(hdr_color * 0.02);
+  hdr_color = ACESFitted(hdr_color);
 
   FragColor.rgb = hdr_color;
   FragColor.a = 1;

assets/shaders/shadow.glsl

@@ -1,10 +1,7 @@
-// UBOs
-layout(std140, binding = 0) uniform Matrices {
-  mat4 projection;
-  mat4 view;
-};
+#include "camera.glsl"
 
-layout(std430, binding = 3) readonly buffer DrawCmdDatas {
+// Shadows use a different format, so don't use draw_cmds_data.glsl
+layout(std430, binding = 3) readonly buffer DrawCmdDatasShadow {
   // Access by gl_DrawID
   mat4 transforms[];
 };

assets/shaders/unlit.glsl

@@ -1,10 +1,6 @@
-#extension GL_ARB_bindless_texture : enable
+#extension GL_ARB_bindless_texture : require
 
-// UBOs
-layout(std140, binding = 0) uniform Matrices {
-  mat4 projection;
-  mat4 view;
-};
+#include "camera.glsl"
 
 // Uniforms
 layout(location = 1) uniform mat4 model;

assets/shaders/z_prepass.glsl

@@ -1,20 +1,8 @@
-#extension GL_ARB_bindless_texture : enable
+#extension GL_ARB_bindless_texture : require
 
-struct DrawCmdData {
-  mat4 transform;
-  int materialIdx;
-};
+#include "camera.glsl"
+#include "draw_cmds_data.glsl"
 
-// UBOs
-layout(std140, binding = 0) uniform Matrices {
-  mat4 projection;
-  mat4 view;
-};
-
-layout(std430, binding = 3) readonly buffer DrawCmdDatas {
-  // Access by gl_DrawID
-  DrawCmdData draw_data[];
-};
 
 VERTEX_EXPORT flat uint DrawID;
 
@@ -40,31 +28,7 @@ void main() {
 
 #if FRAGMENT_SHADER
 
-struct Material {
-  vec4 albedo;
-  sampler2D albedo_map;
-  vec2 albedo_map_uv_scale;
-  sampler2D normal_map;
-  vec2 normal_map_uv_scale;
-  float metallic;
-  sampler2D metallic_map;
-  vec2 metallic_map_uv_scale;
-  float roughness;
-  sampler2D roughness_map;
-  vec2 roughness_map_uv_scale;
-  vec3 emission;
-  sampler2D emission_map;
-  vec2 emission_map_uv_scale;
-};
-
-layout(std430, binding = 2) readonly buffer Materials {
-  uint materials_count;
-  Material materials[];
-};
-
-vec4 getAlbedo(int materialIdx) {
-  return textureSize(materials[materialIdx].albedo_map, 0) == ivec2(0) ? vec4(pow(materials[materialIdx].albedo.rgb, vec3(2.2)), materials[materialIdx].albedo.a) : texture(materials[materialIdx].albedo_map, VertexOut.uv * materials[materialIdx].albedo_map_uv_scale);
-}
+#include "material.glsl"
 
 out vec4 FragColor;
 

build.zig

@@ -29,8 +29,8 @@ pub fn build(b: *Build) void {
     });
     const zalgebra_dep = b.dependency("zalgebra", .{});
 
-    const assets_mod = b.addModule("assets", .{ .root_source_file = .{ .src_path = .{ .owner = b, .sub_path = "src/assets/root.zig" } } });
-    const asset_manifest_mod = b.addModule("asset_manifest", .{ .root_source_file = .{ .src_path = .{ .owner = b, .sub_path = "src/gen/asset_manifest.zig" } } });
+    const assets_mod = b.addModule("assets", .{ .root_source_file = b.path("src/assets/root.zig") });
+    const asset_manifest_mod = b.addModule("asset_manifest", .{ .root_source_file = b.path("src/gen/asset_manifest.zig") });
     asset_manifest_mod.addImport("assets", assets_mod);
 
     const assets_step = b.step("assets", "Build and install assets");
@@ -194,16 +194,19 @@ fn buildAssets(b: *std.Build, install_assetc_step: *Step, step: *Step, assetc: *
         if (!is_known_ext) continue;
 
         const run_assetc = b.addRunArtifact(assetc);
+        run_assetc.rename_step_with_output_arg = false;
+
         gen_asset_manifest.addAssetListFile(run_assetc.captureStdOut());
         run_assetc.step.dependOn(install_assetc_step);
 
         run_assetc.addPathDir(b.pathFromRoot("libs/ispc_texcomp/lib"));
 
         // Absolute input file arg, this will add it to step deps, cache and all that good stuff
-        run_assetc.addFileArg(.{ .src_path = .{ .owner = b, .sub_path = b.pathJoin(&.{ path, entry.path }) } });
+        run_assetc.addFileArg(b.path(b.pathJoin(&.{ path, entry.path })));
 
         // Generated output dir. Output asset(s) will be placed there at the same relative path as input
         const result_dir = run_assetc.addOutputFileArg("assets");
+        run_assetc.setName(b.fmt("assetc ({s})", .{entry.basename}));
 
         const install_assets = b.addInstallDirectory(.{
             .source_dir = result_dir,
@@ -230,7 +233,7 @@ fn buildAssetCompiler(b: *Build, optimize: std.builtin.OptimizeMode, assets_mod:
     const assetc = b.addExecutable(.{
         .name = "assetc",
         .target = b.host,
-        .root_source_file = .{ .src_path = .{ .owner = b, .sub_path = "tools/asset_compiler.zig" } },
+        .root_source_file = b.path("tools/asset_compiler.zig"),
         .optimize = optimize,
     });
     assetc.linkLibC();
@@ -239,12 +242,12 @@ fn buildAssetCompiler(b: *Build, optimize: std.builtin.OptimizeMode, assets_mod:
         b.installFile("libs/ispc_texcomp/lib/ispc_texcomp.dll", "ispc_texcomp.dll");
         b.installFile("libs/ispc_texcomp/lib/ispc_texcomp.pdb", "ispc_texcomp.pdb");
     }
-    assetc.addLibraryPath(.{ .src_path = .{ .owner = b, .sub_path = "libs/ispc_texcomp/lib" } });
-    assetc.addIncludePath(.{ .src_path = .{ .owner = b, .sub_path = "libs/ispc_texcomp/include" } });
+    assetc.addLibraryPath(b.path("libs/ispc_texcomp/lib"));
+    assetc.addIncludePath(b.path("libs/ispc_texcomp/include"));
     assetc.linkSystemLibrary("ispc_texcomp");
 
     const zalgebra_mod = zalgebra_dep.module("zalgebra");
-    const formats_mod = b.addModule("formats", .{ .root_source_file = .{ .src_path = .{ .owner = b, .sub_path = "src/formats.zig" } } });
+    const formats_mod = b.addModule("formats", .{ .root_source_file = b.path("src/formats.zig") });
     formats_mod.addImport("zalgebra", zalgebra_mod);
     formats_mod.addImport("assets", assets_mod);
     assetc.root_module.addImport("formats", formats_mod);
@@ -256,7 +259,7 @@ fn buildAssetCompiler(b: *Build, optimize: std.builtin.OptimizeMode, assets_mod:
     assetc.linkLibCpp();
 
     assetc.addCSourceFile(.{ .file = b.path("libs/stb/stb_image.c"), .flags = &.{"-std=c99"} });
-    assetc.addIncludePath(.{ .src_path = .{ .owner = b, .sub_path = "libs/stb" } });
+    assetc.addIncludePath(b.path("libs/stb"));
 
     return assetc;
 }

src/AssetManager.zig

@@ -785,18 +785,621 @@ fn loadShader(self: *AssetManager, id: AssetId) LoadedShader {
     };
 }
 
+const ShaderTokenizer = struct {
+    const Self = @This();
+
+    pub const TokenType = enum {
+        Unknown,
+
+        OpenParen,
+        OpenBrace,
+        OpenBracket,
+
+        ClosedParen,
+        ClosedBrace,
+        ClosedBracket,
+
+        Comma,
+        Colon,
+        Semicolon,
+        Question,
+        Tilde,
+
+        Dot,
+
+        Star,
+        Plus,
+        Dash,
+        Slash,
+        Percent,
+        Caret,
+        Bar,
+        Ampersand,
+
+        StarEquals,
+        PlusEquals,
+        DashEquals,
+        SlashEquals,
+        PercentEquals,
+        CaretEquals,
+        BarEquals,
+        AmpersandEquals,
+
+        DoubleBar,
+        DoubleAmpersand,
+
+        Equals,
+        EqualsEquals,
+
+        Bang,
+        BangEquals,
+
+        Greater,
+        GreaterGreater, // >>
+        Less,
+        LessLess, // <<
+
+        GreaterEquals,
+        LessEquals,
+
+        String,
+        Directive,
+
+        Number,
+        Identifier,
+
+        End,
+    };
+
+    pub const Token = struct {
+        type: TokenType = .Unknown,
+        text: []const u8 = "",
+
+        // Start and end of the token including things like quotes
+        start: usize = 0,
+        end: usize = 0,
+    };
+
+    at: usize = 0,
+    data: []const u8,
+
+    pub fn init(data: []const u8) Self {
+        return Self{ .data = data };
+    }
+
+    fn peek(self: *const Self) ?u8 {
+        if (self.at + 1 < self.data.len) {
+            return self.data[self.at + 1];
+        }
+
+        return null;
+    }
+
+    fn matchDoubleSymbol(self: *Self, token: *Token, next_sym: u8, no_match: TokenType, match: TokenType) bool {
+        if (self.peek()) |next_char| {
+            if (next_char == next_sym) {
+                token.type = match;
+                token.end += 1;
+                token.text = self.data[token.start..token.end];
+                return true;
+            } else {
+                token.type = no_match;
+            }
+        } else {
+            token.type = no_match;
+        }
+        return false;
+    }
+
+    fn isHex(c: u8) bool {
+        return switch (c) {
+            '0'...'9' => true,
+            'a'...'f', 'A'...'F' => true,
+            else => false,
+        };
+    }
+
+    fn matchInteger(self: *Self) ?Token {
+        var token = Token{ .type = .Number, .start = self.at };
+        var has_sign = false;
+        if (self.data[self.at] == '-') {
+            self.at += 1;
+            has_sign = true;
+        }
+
+        switch (self.data[self.at]) {
+            '0' => {
+                if (self.peek()) |next_sym| {
+                    switch (next_sym) {
+                        'x', 'X' => {
+                            // HEX
+                            self.at += 2;
+
+                            while (isHex(self.data[self.at])) {
+                                self.at += 1;
+                            }
+                        },
+                        '0'...'9' => {
+                            self.at += 1;
+                            // Octal, maybe invalid (8, 9 are invalid)
+                            while (isNum(self.data[self.at])) {
+                                self.at += 1;
+                            }
+                        },
+                        else => {
+                            self.at += 1;
+                        },
+                    }
+                } else {
+                    self.at += 1;
+                }
+            },
+            '1'...'9' => {
+                while (self.at < self.data.len and isNum(self.data[self.at])) {
+                    self.at += 1;
+                }
+            },
+            else => {},
+        }
+
+        switch (self.data[self.at]) {
+            'u', 'U' => {
+                self.at += 1;
+            },
+            else => {},
+        }
+        token.end = self.at;
+        token.text = self.data[token.start..token.end];
+
+        const len_without_sign = if (has_sign) token.text.len - 1 else token.text.len;
+        if (len_without_sign == 0) {
+            return null;
+        }
+
+        return token;
+    }
+
+    fn eatDigitSequence(self: *Self) void {
+        while (self.at < self.data.len and isNum(self.data[self.at])) {
+            self.at += 1;
+        }
+    }
+
+    fn matchFloat(self: *Self) ?Token {
+        var token = Token{ .type = .Number, .start = self.at };
+
+        var has_sign = false;
+        switch (self.data[self.at]) {
+            '-', '+' => {
+                self.at += 1;
+                has_sign = true;
+            },
+            else => {},
+        }
+
+        self.eatDigitSequence();
+        if (self.data[self.at] == '.') {
+            self.at += 1;
+        }
+        self.eatDigitSequence();
+        // Exponent
+        if (self.data[self.at] == 'e' or self.data[self.at] == 'E') {
+            self.at += 1;
+        }
+        //Suffix
+        switch (self.data[self.at]) {
+            'f', 'F' => {
+                self.at += 1;
+            },
+            'l' => {
+                if (self.peek() == 'f') {
+                    self.at += 2;
+                }
+            },
+            'L' => {
+                if (self.peek() == 'F') {
+                    self.at += 2;
+                }
+            },
+            else => {},
+        }
+
+        token.end = self.at;
+        token.text = self.data[token.start..token.end];
+
+        const len_without_sign = if (has_sign) token.text.len - 1 else token.text.len;
+        if (len_without_sign == 0) {
+            return null;
+        }
+
+        return token;
+    }
+
+    fn matchNumber(self: *Self, token: *Token) bool {
+        const start = self.at;
+        const maybe_int_token = self.matchInteger();
+        self.at = start;
+        const maybe_float_token = self.matchFloat();
+
+        if (maybe_int_token != null and maybe_float_token != null) {
+            const int_token = maybe_int_token.?;
+            const float_token = maybe_float_token.?;
+            if (int_token.end > float_token.end) {
+                self.at = int_token.end;
+                token.* = int_token;
+            } else {
+                self.at = float_token.end;
+                token.* = float_token;
+            }
+
+            return true;
+        }
+
+        if (maybe_float_token) |result| {
+            self.at = result.end;
+            token.* = result;
+            return true;
+        }
+
+        if (maybe_int_token) |result| {
+            self.at = result.end;
+            token.* = result;
+            return true;
+        }
+
+        return false;
+    }
+
+    pub fn next(self: *Self) Token {
+        self.eatWhitespace();
+
+        if (self.at == self.data.len) {
+            return Token{ .type = .End };
+        }
+
+        var result = Token{ .type = .Unknown, .start = self.at, .end = self.at + 1, .text = self.data[self.at .. self.at + 1] };
+
+        if (self.at < self.data.len) {
+            switch (self.data[self.at]) {
+                '(' => {
+                    result.type = .OpenParen;
+                },
+                '{' => {
+                    result.type = .OpenBrace;
+                },
+                '[' => {
+                    result.type = .OpenBracket;
+                },
+                ')' => {
+                    result.type = .ClosedParen;
+                },
+                '}' => {
+                    result.type = .ClosedBrace;
+                },
+                ']' => {
+                    result.type = .ClosedBracket;
+                },
+                ',' => {
+                    result.type = .Comma;
+                },
+                ':' => {
+                    result.type = .Colon;
+                },
+                ';' => {
+                    result.type = .Semicolon;
+                },
+                '?' => {
+                    result.type = .Question;
+                },
+                '~' => {
+                    result.type = .Tilde;
+                },
+                '.' => {
+                    if (!self.matchNumber(&result)) {
+                        result.type = .Dot;
+                    }
+                },
+                '*' => {
+                    _ = self.matchDoubleSymbol(&result, '=', .Star, .StarEquals);
+                },
+                '+' => {
+                    if (!self.matchNumber(&result)) {
+                        _ = self.matchDoubleSymbol(&result, '=', .Plus, .PlusEquals);
+                    }
+                },
+                '-' => {
+                    if (!self.matchNumber(&result)) {
+                        _ = self.matchDoubleSymbol(&result, '=', .Dash, .DashEquals);
+                    }
+                },
+                '/' => {
+                    _ = self.matchDoubleSymbol(&result, '=', .Slash, .SlashEquals);
+                },
+                '%' => {
+                    _ = self.matchDoubleSymbol(&result, '=', .Percent, .PercentEquals);
+                },
+                '^' => {
+                    _ = self.matchDoubleSymbol(&result, '=', .Caret, .StarEquals);
+                },
+                '|' => {
+                    if (!self.matchDoubleSymbol(&result, '=', .Bar, .BarEquals)) {
+                        _ = self.matchDoubleSymbol(&result, '|', .Bar, .DoubleBar);
+                    }
+                },
+                '&' => {
+                    if (!self.matchDoubleSymbol(&result, '=', .Ampersand, .AmpersandEquals)) {
+                        _ = self.matchDoubleSymbol(&result, '&', .Ampersand, .DoubleAmpersand);
+                    }
+                },
+                '=' => {
+                    _ = self.matchDoubleSymbol(&result, '=', .Equals, .EqualsEquals);
+                },
+                '!' => {
+                    _ = self.matchDoubleSymbol(&result, '=', .Bang, .BangEquals);
+                },
+                '<' => {
+                    if (!self.matchDoubleSymbol(&result, '=', .Less, .LessEquals)) {
+                        _ = self.matchDoubleSymbol(&result, '<', .Less, .LessLess);
+                    }
+                },
+                '>' => {
+                    if (!self.matchDoubleSymbol(&result, '=', .Greater, .GreaterEquals)) {
+                        _ = self.matchDoubleSymbol(&result, '>', .Greater, .GreaterGreater);
+                    }
+                },
+                '"' => {
+                    const start = self.at;
+                    self.at += 1;
+
+                    const text_start = self.at;
+
+                    while (self.at < self.data.len and self.data[self.at] != '"') {
+                        if (self.data[self.at] == '\\') {
+                            self.at += 1;
+                        }
+                        self.at += 1;
+                    }
+                    const text_end = self.at;
+
+                    if (self.data[self.at] == '"') {
+                        self.at += 1;
+                    }
+                    const end = self.at;
+
+                    result.type = .String;
+                    result.start = start;
+                    result.text = self.data[text_start..text_end];
+                    result.start = start;
+                    result.end = end;
+                },
+                '#' => {
+                    const start = self.at;
+
+                    self.at += 1;
+                    const text_start = self.at;
+
+                    while (isAlphaNum(self.data[self.at])) {
+                        self.at += 1;
+                    }
+
+                    const end = self.at;
+
+                    result.type = .Directive;
+                    result.text = self.data[text_start..end];
+                    result.start = start;
+                    result.end = end;
+                },
+                'a'...'z', 'A'...'Z', '_' => {
+                    const start = self.at;
+
+                    while (self.at < self.data.len and (isAlphaNum(self.data[self.at]))) {
+                        self.at += 1;
+                    }
+
+                    const end = self.at;
+
+                    result.type = .Identifier;
+                    result.text = self.data[start..end];
+                    result.start = start;
+                    result.end = end;
+                },
+                '0'...'9' => {
+                    const matched = self.matchNumber(&result);
+                    std.debug.assert(matched);
+                },
+                else => {},
+            }
+        } else {
+            result.type = .End;
+        }
+        self.at = result.end;
+
+        return result;
+    }
+
+    fn isWhitespace(c: u8) bool {
+        return c == ' ' or c == '\t' or c == '\n' or c == '\r';
+    }
+
+    fn isAlpha(c: u8) bool {
+        return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z');
+    }
+
+    fn isNum(c: u8) bool {
+        return (c >= '0' and c <= '9');
+    }
+
+    fn isAlphaNum(c: u8) bool {
+        return isAlpha(c) or isNum(c) or c == '_';
+    }
+
+    fn eatWhitespace(self: *Self) void {
+        var consuming = true;
+        while (consuming) {
+            if (self.at >= self.data.len) {
+                return;
+            }
+
+            if (isWhitespace(self.data[self.at])) {
+                self.at += 1;
+            } else if (self.data[self.at] == '/' and self.peek() == '/') {
+                self.at += 2;
+                while (self.at < self.data.len and self.data[self.at] != '\n') {
+                    if (self.data[self.at] == '\\' and self.peek() == '\n') {
+                        self.at += 1;
+                    }
+                    self.at += 1;
+                }
+            } else if (self.data[self.at] == '/' and self.peek() == '*') {
+                self.at += 2;
+
+                while (self.at < self.data.len and self.at < self.data.len and !(self.data[self.at] == '*' and self.peek() == '/')) {
+                    self.at += 1;
+                }
+                if (self.at < self.data.len and self.data[self.at] == '*' and self.peek() == '/') {
+                    self.at += 2;
+                }
+            } else {
+                consuming = false;
+            }
+        }
+    }
+};
+
+test "ShaderTokenizer" {
+    const testing = std.testing;
+
+    var tokenizer = ShaderTokenizer.init(
+        \\// UBOs asdkfljlka ajksfk\
+        \\
+        \\#include "../my_file\".glsl"
+        \\
+        \\layout(std140, binding = 0) uniform Matrices {
+        \\  mat4 projection;
+        \\  mat4 view;
+        \\};
+        \\
+        \\layout(location = 2) uniform vec3 color;
+        \\
+        \\// Input, output blocks
+        \\
+        \\#if VERTEX_SHADER
+        \\
+        \\layout(location = 0) in vec3 aPos;
+        \\
+        \\void main() {
+        \\  gl_Position = projection * view * vec4(aPos.xyz, 1.0);
+        \\}
+        \\#endif // VERTEX_SHADER
+        \\
+        \\#if FRAGMENT_SHADER
+        \\
+        \\out vec4 FragColor;
+        \\
+        \\void main() {
+        \\  FragColor += vec4(vec3(1.0), 1.0f);
+        \\  bool logic = (true && false || _asdfjkh) | (1u << -123U);
+        \\ 0xAf123FF
+        \\ -0x123
+        \\ -09872
+        \\ .1
+        \\ .12f
+        \\ 0.2LF
+        \\ +0
+        \\ -0f
+        \\}
+        \\
+        \\
+        \\#endif // FRAGMNET_SHADER
+    );
+
+    var token = tokenizer.next();
+    while (token.type != .End) : (token = tokenizer.next()) {
+        try std.io.getStdErr().writer().print("{} \"{s}\"\n", .{ token.type, token.text });
+        try testing.expect(token.type != .Unknown);
+    }
+}
+
 fn loadShaderErr(self: *AssetManager, id: AssetId) !LoadedShader {
     const path = asset_manifest.getPath(id);
+    const dir = std.fs.path.dirname(path) orelse @panic("No dir");
 
-    const data = try self.loadFile(self.allocator, path, SHADER_MAX_BYTES);
+    const data = try self.loadFile(self.frame_arena, path, SHADER_MAX_BYTES);
 
-    const loaded_shader = LoadedShader{ .source = data.bytes };
+    var included_asset_ids = std.ArrayList(AssetId).init(self.frame_arena);
+    var preprocessed_segments = std.SegmentedList([]const u8, 64){};
+    var final_len: usize = 0;
+
+    // Preprocess
+    {
+        var tokenizer = ShaderTokenizer.init(data.bytes);
+
+        var last_offset: usize = 0;
+
+        var token = tokenizer.next();
+        while (token.type != .End) : (token = tokenizer.next()) {
+            switch (token.type) {
+                .Directive => {
+                    if (std.mem.eql(u8, token.text, "include")) {
+
+                        // Append section of text up to this directive
+                        try preprocessed_segments.append(self.frame_arena, data.bytes[last_offset..token.start]);
+                        final_len += token.start - last_offset;
+
+                        const include_path = tokenizer.next();
+                        // Next section will start after this directive, this allows replacing #include with its content
+                        last_offset = include_path.end;
+
+                        if (include_path.type != .String) {
+                            return error.InvalidInclude;
+                        }
+
+                        const included_file_path = try std.fs.path.resolve(self.frame_arena, &.{ dir, include_path.text });
+
+                        const included_asset_id = assets.AssetPath.fromString(included_file_path).hash();
+                        if (included_asset_id != 0) {
+                            try included_asset_ids.append(included_asset_id);
+                            const included_shader = try self.loadShaderErr(included_asset_id);
+                            try preprocessed_segments.append(self.frame_arena, included_shader.source);
+                            final_len += included_shader.source.len;
+                        }
+                    }
+                },
+                else => {},
+            }
+        }
+
+        {
+            const remaining = data.bytes.len - last_offset;
+            if (remaining > 0) {
+                try preprocessed_segments.append(self.frame_arena, data.bytes[last_offset..data.bytes.len]);
+                final_len += remaining;
+            }
+        }
+    }
+
+    var result_source = try self.allocator.alloc(u8, final_len);
+
+    // Join source sections
+    {
+        var cursor: usize = 0;
+        var iter = preprocessed_segments.constIterator(0);
+
+        while (iter.next()) |slice| {
+            @memcpy(result_source[cursor .. cursor + slice.len], slice.*);
+            cursor += slice.len;
+        }
+    }
+
+    const loaded_shader = LoadedShader{ .source = result_source };
     {
         self.rw_lock.lock();
         defer self.rw_lock.unlock();
 
         try self.loaded_assets.put(self.allocator, id, .{ .shader = loaded_shader });
         try self.modified_times.put(self.allocator, id, data.modified);
+
+        try self.addDependencies(id, included_asset_ids.items);
     }
 
     return loaded_shader;

src/assets/root.zig

@@ -1,3 +1,5 @@
+const std = @import("std");
+
 pub const AssetId = u64;
 
 pub const Handle = struct {
@@ -8,3 +10,149 @@ pub const Handle = struct {
     pub const Texture = extern struct { id: AssetId = 0 };
     pub const Material = extern struct { id: AssetId = 0 };
 };
+
+pub const AssetType = enum {
+    Scene,
+    Mesh,
+    Shader,
+    ShaderProgram,
+    Texture,
+    Material,
+
+    pub fn pluralName(self: AssetType) []const u8 {
+        return switch (self) {
+            .Scene => "Scenes",
+            .Mesh => "Meshes",
+            .Shader => "Shaders",
+            .ShaderProgram => "ShaderPrograms",
+            .Texture => "Textures",
+            .Material => "Materials",
+        };
+    }
+
+    pub fn ext(self: AssetType) []const u8 {
+        return switch (self) {
+            .Scene => "scn",
+            .Mesh => "mesh",
+            .Shader => "glsl",
+            .ShaderProgram => "prog",
+            .Texture => "tex",
+            .Material => "mat",
+        };
+    }
+};
+
+pub const AssetPath = union(enum) {
+    invalid: void, // translates to handle with id: 0
+    simple: []const u8,
+    nested: struct {
+        path: []const u8,
+        sub_path: []const u8,
+    },
+
+    pub fn hash(self: AssetPath) u64 {
+        switch (self) {
+            .invalid => return 0,
+            else => {},
+        }
+        var hasher = std.hash.Wyhash.init(0);
+        std.hash.autoHashStrat(&hasher, self.getPath(), .Deep);
+        std.hash.autoHashStrat(&hasher, self.getSubPath(), .Deep);
+        return hasher.final();
+    }
+
+    pub fn subPath(self: AssetPath, sub_path: []const u8) AssetPath {
+        return switch (self) {
+            .invalid => self,
+            .simple => |path| AssetPath{ .nested = .{ .path = path, .sub_path = sub_path } },
+            .nested => |nested| AssetPath{ .nested = .{ .path = nested.path, .sub_path = sub_path } },
+        };
+    }
+
+    pub fn getPath(self: AssetPath) []const u8 {
+        return switch (self) {
+            .invalid => "",
+            .simple => |path| path,
+            .nested => |nested| nested.path,
+        };
+    }
+
+    pub fn getSubPath(self: AssetPath) ?[]const u8 {
+        return switch (self) {
+            .invalid => null,
+            .nested => |nested| nested.sub_path,
+            else => null,
+        };
+    }
+
+    pub inline fn strLen(self: AssetPath) usize {
+        return switch (self) {
+            .invalid => 0,
+            .simple => |path| path.len,
+            .nested => |nested| return nested.path.len + nested.sub_path.len + 1,
+        };
+    }
+
+    pub fn writeString(self: AssetPath, writer: anytype) !void {
+        switch (self) {
+            .invalid => {},
+            .simple => |path| {
+                try writer.writeAll(path);
+            },
+            .nested => |nested| {
+                try writer.writeAll(nested.path);
+                try writer.writeByte('#');
+                try writer.writeAll(nested.sub_path);
+            },
+        }
+    }
+
+    pub fn toString(self: AssetPath, out_buf: []u8) ![]u8 {
+        return self.toStringSep(out_buf, '#');
+    }
+
+    pub fn toStringSep(self: AssetPath, out_buf: []u8, sep: u8) ![]u8 {
+        if (out_buf.len < self.strLen()) {
+            return error.BufferTooSmall;
+        }
+
+        switch (self) {
+            .invalid => {
+                return out_buf[0..0];
+            },
+            .simple => |path| {
+                @memcpy(out_buf[0..path.len], path);
+                return out_buf[0..path.len];
+            },
+            .nested => |nested| {
+                @memcpy(out_buf[0..nested.path.len], nested.path);
+                out_buf[nested.path.len] = sep;
+                @memcpy(out_buf[nested.path.len + 1 .. nested.path.len + 1 + nested.sub_path.len], nested.sub_path);
+
+                return out_buf[0..self.strLen()];
+            },
+        }
+    }
+
+    pub fn toStringAlloc(self: AssetPath, allocator: std.mem.Allocator) ![]u8 {
+        const buf = try allocator.alloc(u8, self.strLen());
+        errdefer allocator.free(buf);
+        return try self.toString(buf);
+    }
+
+    pub fn fromString(str: []const u8) AssetPath {
+        if (str.len == 0) {
+            return .invalid;
+        }
+        if (std.mem.indexOf(u8, str, "#")) |sep_idx| {
+            return .{
+                .nested = .{
+                    .path = str[0..sep_idx],
+                    .sub_path = str[sep_idx + 1 ..],
+                },
+            };
+        }
+
+        return .{ .simple = str };
+    }
+};

src/game.zig

@@ -192,7 +192,7 @@ export fn game_init(global_allocator: *std.mem.Allocator) void {
     _ = globals.g_mem.world.addEntity(.{
         .flags = .{ .dir_light = true, .rotate = true },
         .transform = .{ .rot = Quat.fromEulerAngles(Vec3.new(70, 0, 0)) },
-        .light = .{ .color_intensity = Vec4.new(1.00, 0.805, 0.570, 790) },
+        .light = .{ .color_intensity = Vec4.new(1.00, 0.805, 0.8, 790 * 0.02) },
         .rotate = .{ .axis = Vec3.up(), .rate = -10 },
     });