From 9226b61988f7264914874002ba185640f0c99a71 Mon Sep 17 00:00:00 2001 From: sergeypdev Date: Sun, 1 Sep 2024 11:40:25 +0400 Subject: [PATCH] Add shader preprocessor with #include support and refactor shaders to remove a bunch of duplicates --- assets/shaders/camera.glsl | 9 + assets/shaders/cube_shadow.glsl | 6 +- assets/shaders/debug.glsl | 6 +- assets/shaders/draw_cmds_data.glsl | 14 + assets/shaders/material.glsl | 44 +++ assets/shaders/mesh.glsl | 67 +--- assets/shaders/post_process.glsl | 2 +- assets/shaders/shadow.glsl | 9 +- assets/shaders/unlit.glsl | 8 +- assets/shaders/z_prepass.glsl | 44 +-- build.zig | 19 +- src/AssetManager.zig | 607 ++++++++++++++++++++++++++++- src/assets/root.zig | 148 +++++++ src/game.zig | 2 +- 14 files changed, 853 insertions(+), 132 deletions(-) create mode 100644 assets/shaders/camera.glsl create mode 100644 assets/shaders/draw_cmds_data.glsl create mode 100644 assets/shaders/material.glsl diff --git a/assets/shaders/camera.glsl b/assets/shaders/camera.glsl new file mode 100644 index 0000000..6a34f62 --- /dev/null +++ b/assets/shaders/camera.glsl @@ -0,0 +1,9 @@ +#ifndef CAMERA_GLSL +#define CAMERA_GLSL + +layout(std140, binding = 0) uniform Matrices { + mat4 projection; + mat4 view; +}; + +#endif // CAMERA_GLSL diff --git a/assets/shaders/cube_shadow.glsl b/assets/shaders/cube_shadow.glsl index a182082..26fdfb9 100644 --- a/assets/shaders/cube_shadow.glsl +++ b/assets/shaders/cube_shadow.glsl @@ -1,8 +1,4 @@ -// UBOs -layout(std140, binding = 0) uniform Matrices { - mat4 projection; - mat4 view; -}; +#include "camera.glsl" // Uniforms layout(location = 1) uniform mat4 model; diff --git a/assets/shaders/debug.glsl b/assets/shaders/debug.glsl index 79e334d..2e68bf2 100644 --- a/assets/shaders/debug.glsl +++ b/assets/shaders/debug.glsl @@ -1,8 +1,4 @@ -// UBOs -layout(std140, binding = 0) uniform Matrices { - mat4 projection; - mat4 view; -}; +#include "camera.glsl" layout(location = 2) uniform vec3 color; diff --git a/assets/shaders/draw_cmds_data.glsl b/assets/shaders/draw_cmds_data.glsl new file mode 100644 index 0000000..6cf2f27 --- /dev/null +++ b/assets/shaders/draw_cmds_data.glsl @@ -0,0 +1,14 @@ +#ifndef DRAW_CMDS_DATA_GLSL +#define DRAW_CMDS_DATA_GLSL + +struct DrawCmdData { + mat4 transform; + int materialIdx; +}; + +layout(std430, binding = 3) readonly buffer DrawCmdDatas { + // Access by gl_BaseInstance + gl_InstanceID + DrawCmdData draw_data[]; +}; + +#endif // DRAW_CMDS_DATA_GLSL diff --git a/assets/shaders/material.glsl b/assets/shaders/material.glsl new file mode 100644 index 0000000..bdbd47e --- /dev/null +++ b/assets/shaders/material.glsl @@ -0,0 +1,44 @@ +#ifndef MATERIAL_GLSL +#define MATERIAL_GLSL + +// You have to enable GL_ARB_bindless_texture extension if you're importing this + +struct Material { + vec4 albedo; + sampler2D albedo_map; + vec2 albedo_map_uv_scale; + sampler2D normal_map; + vec2 normal_map_uv_scale; + float metallic; + sampler2D metallic_map; + vec2 metallic_map_uv_scale; + float roughness; + sampler2D roughness_map; + vec2 roughness_map_uv_scale; + vec3 emission; + sampler2D emission_map; + vec2 emission_map_uv_scale; +}; + +layout(std430, binding = 2) readonly buffer Materials { + uint materials_count; + Material materials[]; +}; + +vec4 getAlbedo(int materialIdx) { + return textureSize(materials[materialIdx].albedo_map, 0) == ivec2(0) ? vec4(pow(materials[materialIdx].albedo.rgb, vec3(2.2)), materials[materialIdx].albedo.a) : texture(materials[materialIdx].albedo_map, VertexOut.uv * materials[materialIdx].albedo_map_uv_scale); +} + +float getRoughness(int materialIdx) { + return max(0.01, textureSize(materials[materialIdx].roughness_map, 0) == ivec2(0) ? materials[materialIdx].roughness : texture(materials[materialIdx].roughness_map, VertexOut.uv * materials[materialIdx].roughness_map_uv_scale).g); +} + +float getMetallic(int materialIdx) { + return textureSize(materials[materialIdx].metallic_map, 0) == ivec2(0) ? materials[materialIdx].metallic : texture(materials[materialIdx].metallic_map, VertexOut.uv * materials[materialIdx].metallic_map_uv_scale).b; +} + +vec3 getEmission(int materialIdx) { + return textureSize(materials[materialIdx].emission_map, 0) == ivec2(0) ? materials[materialIdx].emission : texture(materials[materialIdx].emission_map, VertexOut.uv * materials[materialIdx].emission_map_uv_scale).rgb; +} + +#endif // MATERIAL_GLSL diff --git a/assets/shaders/mesh.glsl b/assets/shaders/mesh.glsl index e4b32ff..b7eb232 100644 --- a/assets/shaders/mesh.glsl +++ b/assets/shaders/mesh.glsl @@ -1,28 +1,15 @@ -#extension GL_ARB_bindless_texture : enable +#extension GL_ARB_bindless_texture : require #extension GL_KHR_shader_subgroup_ballot : enable #extension GL_KHR_shader_subgroup_vote : enable +#include "camera.glsl" +#include "draw_cmds_data.glsl" + // Keep in sync with cpu #define MAX_POINT_LIGHTS 8 #define PI 3.1415926535897932384626433832795 #define CSM_SPLITS 4 -struct DrawCmdData { - mat4 transform; - int materialIdx; -}; - -// UBOs -layout(std140, binding = 0) uniform Matrices { - mat4 projection; - mat4 view; -}; - -layout(std430, binding = 3) readonly buffer DrawCmdDatas { - // Access by gl_DrawID - DrawCmdData draw_data[]; -}; - layout(location = 16, bindless_sampler) uniform sampler2DArrayShadow shadow_maps; layout(location = 17, bindless_sampler) uniform samplerCubeArrayShadow cube_shadow_maps; layout(location = 22, bindless_sampler) uniform sampler2D brdfLut; @@ -78,6 +65,8 @@ void main() { #if FRAGMENT_SHADER +#include "material.glsl" + // Types struct Light { vec4 vPos; @@ -96,34 +85,11 @@ struct Light { vec4 csm_split_points; // TODO: Maybe increase to 8, though it's probably too many }; -struct Material { - vec4 albedo; - sampler2D albedo_map; - vec2 albedo_map_uv_scale; - sampler2D normal_map; - vec2 normal_map_uv_scale; - float metallic; - sampler2D metallic_map; - vec2 metallic_map_uv_scale; - float roughness; - sampler2D roughness_map; - vec2 roughness_map_uv_scale; - vec3 emission; - sampler2D emission_map; - vec2 emission_map_uv_scale; -}; - - layout(std430, binding = 1) readonly buffer Lights { uint lights_count; Light lights[]; }; -layout(std430, binding = 2) readonly buffer Materials { - uint materials_count; - Material materials[]; -}; - out vec4 FragColor; int getShadowMapIndex(int lightIdx) { @@ -146,22 +112,6 @@ int getCSMSplit(int lightIdx, float depth) { return CSM_SPLITS - 1; } -vec4 getAlbedo(int materialIdx) { - return textureSize(materials[materialIdx].albedo_map, 0) == ivec2(0) ? vec4(pow(materials[materialIdx].albedo.rgb, vec3(2.2)), materials[materialIdx].albedo.a) : texture(materials[materialIdx].albedo_map, VertexOut.uv * materials[materialIdx].albedo_map_uv_scale); -} - -float getRoughness(int materialIdx) { - return max(0.01, textureSize(materials[materialIdx].roughness_map, 0) == ivec2(0) ? materials[materialIdx].roughness : texture(materials[materialIdx].roughness_map, VertexOut.uv * materials[materialIdx].roughness_map_uv_scale).g); -} - -float getMetallic(int materialIdx) { - return textureSize(materials[materialIdx].metallic_map, 0) == ivec2(0) ? materials[materialIdx].metallic : texture(materials[materialIdx].metallic_map, VertexOut.uv * materials[materialIdx].metallic_map_uv_scale).b; -} - -vec3 getEmission(int materialIdx) { - return textureSize(materials[materialIdx].emission_map, 0) == ivec2(0) ? materials[materialIdx].emission : texture(materials[materialIdx].emission_map, VertexOut.uv * materials[materialIdx].emission_map_uv_scale).rgb; -} - vec3 schlickFresnel(int matIdx, float NDotV) { vec3 f0 = mix(vec3(0.04), getAlbedo(matIdx).rgb, getMetallic(matIdx)); @@ -330,10 +280,11 @@ vec3 ibl(int matIdx, vec3 N, vec3 V) { float ambient_diff = dot(N, VertexOut.vUp) * 0.5 + 0.5; float ambient_spec = dot(R, VertexOut.vUp) * 0.5 + 0.5; - vec3 irradiance = vec3(1.0, 0.9764705882352941, 0.9921568627450981) * 79 * ambient_diff; + // TODO: don't hardcode this + vec3 irradiance = vec3(1.0, 0.9764705882352941, 0.9921568627450981) * 79 * ambient_diff * 0.02; vec3 diffuse = irradiance * getAlbedo(matIdx).rgb; - vec3 reflectedColor = vec3(0.9, 0.9064705882352941, 0.9921568627450981) * 79 * ambient_spec; + vec3 reflectedColor = vec3(0.9, 0.9064705882352941, 0.9921568627450981) * 79 * ambient_spec * 0.02; vec2 envBRDF = textureLod(brdfLut, vec2(NDotV, getRoughness(matIdx)), 0).rg; vec3 specular = reflectedColor * (F * envBRDF.x + envBRDF.y); return kD * diffuse + specular; diff --git a/assets/shaders/post_process.glsl b/assets/shaders/post_process.glsl index cc5b7da..a46b74e 100644 --- a/assets/shaders/post_process.glsl +++ b/assets/shaders/post_process.glsl @@ -69,7 +69,7 @@ vec3 linearToSRGB(vec3 color) { void main() { vec3 hdr_color = texture(screen_sampler, VertexOut.uv).rgb; - hdr_color = ACESFitted(hdr_color * 0.02); + hdr_color = ACESFitted(hdr_color); FragColor.rgb = hdr_color; FragColor.a = 1; diff --git a/assets/shaders/shadow.glsl b/assets/shaders/shadow.glsl index 32be33d..55c0f1b 100644 --- a/assets/shaders/shadow.glsl +++ b/assets/shaders/shadow.glsl @@ -1,10 +1,7 @@ -// UBOs -layout(std140, binding = 0) uniform Matrices { - mat4 projection; - mat4 view; -}; +#include "camera.glsl" -layout(std430, binding = 3) readonly buffer DrawCmdDatas { +// Shadows use a different format, so don't use draw_cmds_data.glsl +layout(std430, binding = 3) readonly buffer DrawCmdDatasShadow { // Access by gl_DrawID mat4 transforms[]; }; diff --git a/assets/shaders/unlit.glsl b/assets/shaders/unlit.glsl index 23e9f6c..e8ea3c4 100644 --- a/assets/shaders/unlit.glsl +++ b/assets/shaders/unlit.glsl @@ -1,10 +1,6 @@ -#extension GL_ARB_bindless_texture : enable +#extension GL_ARB_bindless_texture : require -// UBOs -layout(std140, binding = 0) uniform Matrices { - mat4 projection; - mat4 view; -}; +#include "camera.glsl" // Uniforms layout(location = 1) uniform mat4 model; diff --git a/assets/shaders/z_prepass.glsl b/assets/shaders/z_prepass.glsl index 66e14ce..29f260a 100644 --- a/assets/shaders/z_prepass.glsl +++ b/assets/shaders/z_prepass.glsl @@ -1,20 +1,8 @@ -#extension GL_ARB_bindless_texture : enable +#extension GL_ARB_bindless_texture : require -struct DrawCmdData { - mat4 transform; - int materialIdx; -}; +#include "camera.glsl" +#include "draw_cmds_data.glsl" -// UBOs -layout(std140, binding = 0) uniform Matrices { - mat4 projection; - mat4 view; -}; - -layout(std430, binding = 3) readonly buffer DrawCmdDatas { - // Access by gl_DrawID - DrawCmdData draw_data[]; -}; VERTEX_EXPORT flat uint DrawID; @@ -40,31 +28,7 @@ void main() { #if FRAGMENT_SHADER -struct Material { - vec4 albedo; - sampler2D albedo_map; - vec2 albedo_map_uv_scale; - sampler2D normal_map; - vec2 normal_map_uv_scale; - float metallic; - sampler2D metallic_map; - vec2 metallic_map_uv_scale; - float roughness; - sampler2D roughness_map; - vec2 roughness_map_uv_scale; - vec3 emission; - sampler2D emission_map; - vec2 emission_map_uv_scale; -}; - -layout(std430, binding = 2) readonly buffer Materials { - uint materials_count; - Material materials[]; -}; - -vec4 getAlbedo(int materialIdx) { - return textureSize(materials[materialIdx].albedo_map, 0) == ivec2(0) ? vec4(pow(materials[materialIdx].albedo.rgb, vec3(2.2)), materials[materialIdx].albedo.a) : texture(materials[materialIdx].albedo_map, VertexOut.uv * materials[materialIdx].albedo_map_uv_scale); -} +#include "material.glsl" out vec4 FragColor; diff --git a/build.zig b/build.zig index b79b8da..3516f5f 100644 --- a/build.zig +++ b/build.zig @@ -29,8 +29,8 @@ pub fn build(b: *Build) void { }); const zalgebra_dep = b.dependency("zalgebra", .{}); - const assets_mod = b.addModule("assets", .{ .root_source_file = .{ .src_path = .{ .owner = b, .sub_path = "src/assets/root.zig" } } }); - const asset_manifest_mod = b.addModule("asset_manifest", .{ .root_source_file = .{ .src_path = .{ .owner = b, .sub_path = "src/gen/asset_manifest.zig" } } }); + const assets_mod = b.addModule("assets", .{ .root_source_file = b.path("src/assets/root.zig") }); + const asset_manifest_mod = b.addModule("asset_manifest", .{ .root_source_file = b.path("src/gen/asset_manifest.zig") }); asset_manifest_mod.addImport("assets", assets_mod); const assets_step = b.step("assets", "Build and install assets"); @@ -194,16 +194,19 @@ fn buildAssets(b: *std.Build, install_assetc_step: *Step, step: *Step, assetc: * if (!is_known_ext) continue; const run_assetc = b.addRunArtifact(assetc); + run_assetc.rename_step_with_output_arg = false; + gen_asset_manifest.addAssetListFile(run_assetc.captureStdOut()); run_assetc.step.dependOn(install_assetc_step); run_assetc.addPathDir(b.pathFromRoot("libs/ispc_texcomp/lib")); // Absolute input file arg, this will add it to step deps, cache and all that good stuff - run_assetc.addFileArg(.{ .src_path = .{ .owner = b, .sub_path = b.pathJoin(&.{ path, entry.path }) } }); + run_assetc.addFileArg(b.path(b.pathJoin(&.{ path, entry.path }))); // Generated output dir. Output asset(s) will be placed there at the same relative path as input const result_dir = run_assetc.addOutputFileArg("assets"); + run_assetc.setName(b.fmt("assetc ({s})", .{entry.basename})); const install_assets = b.addInstallDirectory(.{ .source_dir = result_dir, @@ -230,7 +233,7 @@ fn buildAssetCompiler(b: *Build, optimize: std.builtin.OptimizeMode, assets_mod: const assetc = b.addExecutable(.{ .name = "assetc", .target = b.host, - .root_source_file = .{ .src_path = .{ .owner = b, .sub_path = "tools/asset_compiler.zig" } }, + .root_source_file = b.path("tools/asset_compiler.zig"), .optimize = optimize, }); assetc.linkLibC(); @@ -239,12 +242,12 @@ fn buildAssetCompiler(b: *Build, optimize: std.builtin.OptimizeMode, assets_mod: b.installFile("libs/ispc_texcomp/lib/ispc_texcomp.dll", "ispc_texcomp.dll"); b.installFile("libs/ispc_texcomp/lib/ispc_texcomp.pdb", "ispc_texcomp.pdb"); } - assetc.addLibraryPath(.{ .src_path = .{ .owner = b, .sub_path = "libs/ispc_texcomp/lib" } }); - assetc.addIncludePath(.{ .src_path = .{ .owner = b, .sub_path = "libs/ispc_texcomp/include" } }); + assetc.addLibraryPath(b.path("libs/ispc_texcomp/lib")); + assetc.addIncludePath(b.path("libs/ispc_texcomp/include")); assetc.linkSystemLibrary("ispc_texcomp"); const zalgebra_mod = zalgebra_dep.module("zalgebra"); - const formats_mod = b.addModule("formats", .{ .root_source_file = .{ .src_path = .{ .owner = b, .sub_path = "src/formats.zig" } } }); + const formats_mod = b.addModule("formats", .{ .root_source_file = b.path("src/formats.zig") }); formats_mod.addImport("zalgebra", zalgebra_mod); formats_mod.addImport("assets", assets_mod); assetc.root_module.addImport("formats", formats_mod); @@ -256,7 +259,7 @@ fn buildAssetCompiler(b: *Build, optimize: std.builtin.OptimizeMode, assets_mod: assetc.linkLibCpp(); assetc.addCSourceFile(.{ .file = b.path("libs/stb/stb_image.c"), .flags = &.{"-std=c99"} }); - assetc.addIncludePath(.{ .src_path = .{ .owner = b, .sub_path = "libs/stb" } }); + assetc.addIncludePath(b.path("libs/stb")); return assetc; } diff --git a/src/AssetManager.zig b/src/AssetManager.zig index dc5a2b9..28cbf11 100644 --- a/src/AssetManager.zig +++ b/src/AssetManager.zig @@ -785,18 +785,621 @@ fn loadShader(self: *AssetManager, id: AssetId) LoadedShader { }; } +const ShaderTokenizer = struct { + const Self = @This(); + + pub const TokenType = enum { + Unknown, + + OpenParen, + OpenBrace, + OpenBracket, + + ClosedParen, + ClosedBrace, + ClosedBracket, + + Comma, + Colon, + Semicolon, + Question, + Tilde, + + Dot, + + Star, + Plus, + Dash, + Slash, + Percent, + Caret, + Bar, + Ampersand, + + StarEquals, + PlusEquals, + DashEquals, + SlashEquals, + PercentEquals, + CaretEquals, + BarEquals, + AmpersandEquals, + + DoubleBar, + DoubleAmpersand, + + Equals, + EqualsEquals, + + Bang, + BangEquals, + + Greater, + GreaterGreater, // >> + Less, + LessLess, // << + + GreaterEquals, + LessEquals, + + String, + Directive, + + Number, + Identifier, + + End, + }; + + pub const Token = struct { + type: TokenType = .Unknown, + text: []const u8 = "", + + // Start and end of the token including things like quotes + start: usize = 0, + end: usize = 0, + }; + + at: usize = 0, + data: []const u8, + + pub fn init(data: []const u8) Self { + return Self{ .data = data }; + } + + fn peek(self: *const Self) ?u8 { + if (self.at + 1 < self.data.len) { + return self.data[self.at + 1]; + } + + return null; + } + + fn matchDoubleSymbol(self: *Self, token: *Token, next_sym: u8, no_match: TokenType, match: TokenType) bool { + if (self.peek()) |next_char| { + if (next_char == next_sym) { + token.type = match; + token.end += 1; + token.text = self.data[token.start..token.end]; + return true; + } else { + token.type = no_match; + } + } else { + token.type = no_match; + } + return false; + } + + fn isHex(c: u8) bool { + return switch (c) { + '0'...'9' => true, + 'a'...'f', 'A'...'F' => true, + else => false, + }; + } + + fn matchInteger(self: *Self) ?Token { + var token = Token{ .type = .Number, .start = self.at }; + var has_sign = false; + if (self.data[self.at] == '-') { + self.at += 1; + has_sign = true; + } + + switch (self.data[self.at]) { + '0' => { + if (self.peek()) |next_sym| { + switch (next_sym) { + 'x', 'X' => { + // HEX + self.at += 2; + + while (isHex(self.data[self.at])) { + self.at += 1; + } + }, + '0'...'9' => { + self.at += 1; + // Octal, maybe invalid (8, 9 are invalid) + while (isNum(self.data[self.at])) { + self.at += 1; + } + }, + else => { + self.at += 1; + }, + } + } else { + self.at += 1; + } + }, + '1'...'9' => { + while (self.at < self.data.len and isNum(self.data[self.at])) { + self.at += 1; + } + }, + else => {}, + } + + switch (self.data[self.at]) { + 'u', 'U' => { + self.at += 1; + }, + else => {}, + } + token.end = self.at; + token.text = self.data[token.start..token.end]; + + const len_without_sign = if (has_sign) token.text.len - 1 else token.text.len; + if (len_without_sign == 0) { + return null; + } + + return token; + } + + fn eatDigitSequence(self: *Self) void { + while (self.at < self.data.len and isNum(self.data[self.at])) { + self.at += 1; + } + } + + fn matchFloat(self: *Self) ?Token { + var token = Token{ .type = .Number, .start = self.at }; + + var has_sign = false; + switch (self.data[self.at]) { + '-', '+' => { + self.at += 1; + has_sign = true; + }, + else => {}, + } + + self.eatDigitSequence(); + if (self.data[self.at] == '.') { + self.at += 1; + } + self.eatDigitSequence(); + // Exponent + if (self.data[self.at] == 'e' or self.data[self.at] == 'E') { + self.at += 1; + } + //Suffix + switch (self.data[self.at]) { + 'f', 'F' => { + self.at += 1; + }, + 'l' => { + if (self.peek() == 'f') { + self.at += 2; + } + }, + 'L' => { + if (self.peek() == 'F') { + self.at += 2; + } + }, + else => {}, + } + + token.end = self.at; + token.text = self.data[token.start..token.end]; + + const len_without_sign = if (has_sign) token.text.len - 1 else token.text.len; + if (len_without_sign == 0) { + return null; + } + + return token; + } + + fn matchNumber(self: *Self, token: *Token) bool { + const start = self.at; + const maybe_int_token = self.matchInteger(); + self.at = start; + const maybe_float_token = self.matchFloat(); + + if (maybe_int_token != null and maybe_float_token != null) { + const int_token = maybe_int_token.?; + const float_token = maybe_float_token.?; + if (int_token.end > float_token.end) { + self.at = int_token.end; + token.* = int_token; + } else { + self.at = float_token.end; + token.* = float_token; + } + + return true; + } + + if (maybe_float_token) |result| { + self.at = result.end; + token.* = result; + return true; + } + + if (maybe_int_token) |result| { + self.at = result.end; + token.* = result; + return true; + } + + return false; + } + + pub fn next(self: *Self) Token { + self.eatWhitespace(); + + if (self.at == self.data.len) { + return Token{ .type = .End }; + } + + var result = Token{ .type = .Unknown, .start = self.at, .end = self.at + 1, .text = self.data[self.at .. self.at + 1] }; + + if (self.at < self.data.len) { + switch (self.data[self.at]) { + '(' => { + result.type = .OpenParen; + }, + '{' => { + result.type = .OpenBrace; + }, + '[' => { + result.type = .OpenBracket; + }, + ')' => { + result.type = .ClosedParen; + }, + '}' => { + result.type = .ClosedBrace; + }, + ']' => { + result.type = .ClosedBracket; + }, + ',' => { + result.type = .Comma; + }, + ':' => { + result.type = .Colon; + }, + ';' => { + result.type = .Semicolon; + }, + '?' => { + result.type = .Question; + }, + '~' => { + result.type = .Tilde; + }, + '.' => { + if (!self.matchNumber(&result)) { + result.type = .Dot; + } + }, + '*' => { + _ = self.matchDoubleSymbol(&result, '=', .Star, .StarEquals); + }, + '+' => { + if (!self.matchNumber(&result)) { + _ = self.matchDoubleSymbol(&result, '=', .Plus, .PlusEquals); + } + }, + '-' => { + if (!self.matchNumber(&result)) { + _ = self.matchDoubleSymbol(&result, '=', .Dash, .DashEquals); + } + }, + '/' => { + _ = self.matchDoubleSymbol(&result, '=', .Slash, .SlashEquals); + }, + '%' => { + _ = self.matchDoubleSymbol(&result, '=', .Percent, .PercentEquals); + }, + '^' => { + _ = self.matchDoubleSymbol(&result, '=', .Caret, .StarEquals); + }, + '|' => { + if (!self.matchDoubleSymbol(&result, '=', .Bar, .BarEquals)) { + _ = self.matchDoubleSymbol(&result, '|', .Bar, .DoubleBar); + } + }, + '&' => { + if (!self.matchDoubleSymbol(&result, '=', .Ampersand, .AmpersandEquals)) { + _ = self.matchDoubleSymbol(&result, '&', .Ampersand, .DoubleAmpersand); + } + }, + '=' => { + _ = self.matchDoubleSymbol(&result, '=', .Equals, .EqualsEquals); + }, + '!' => { + _ = self.matchDoubleSymbol(&result, '=', .Bang, .BangEquals); + }, + '<' => { + if (!self.matchDoubleSymbol(&result, '=', .Less, .LessEquals)) { + _ = self.matchDoubleSymbol(&result, '<', .Less, .LessLess); + } + }, + '>' => { + if (!self.matchDoubleSymbol(&result, '=', .Greater, .GreaterEquals)) { + _ = self.matchDoubleSymbol(&result, '>', .Greater, .GreaterGreater); + } + }, + '"' => { + const start = self.at; + self.at += 1; + + const text_start = self.at; + + while (self.at < self.data.len and self.data[self.at] != '"') { + if (self.data[self.at] == '\\') { + self.at += 1; + } + self.at += 1; + } + const text_end = self.at; + + if (self.data[self.at] == '"') { + self.at += 1; + } + const end = self.at; + + result.type = .String; + result.start = start; + result.text = self.data[text_start..text_end]; + result.start = start; + result.end = end; + }, + '#' => { + const start = self.at; + + self.at += 1; + const text_start = self.at; + + while (isAlphaNum(self.data[self.at])) { + self.at += 1; + } + + const end = self.at; + + result.type = .Directive; + result.text = self.data[text_start..end]; + result.start = start; + result.end = end; + }, + 'a'...'z', 'A'...'Z', '_' => { + const start = self.at; + + while (self.at < self.data.len and (isAlphaNum(self.data[self.at]))) { + self.at += 1; + } + + const end = self.at; + + result.type = .Identifier; + result.text = self.data[start..end]; + result.start = start; + result.end = end; + }, + '0'...'9' => { + const matched = self.matchNumber(&result); + std.debug.assert(matched); + }, + else => {}, + } + } else { + result.type = .End; + } + self.at = result.end; + + return result; + } + + fn isWhitespace(c: u8) bool { + return c == ' ' or c == '\t' or c == '\n' or c == '\r'; + } + + fn isAlpha(c: u8) bool { + return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z'); + } + + fn isNum(c: u8) bool { + return (c >= '0' and c <= '9'); + } + + fn isAlphaNum(c: u8) bool { + return isAlpha(c) or isNum(c) or c == '_'; + } + + fn eatWhitespace(self: *Self) void { + var consuming = true; + while (consuming) { + if (self.at >= self.data.len) { + return; + } + + if (isWhitespace(self.data[self.at])) { + self.at += 1; + } else if (self.data[self.at] == '/' and self.peek() == '/') { + self.at += 2; + while (self.at < self.data.len and self.data[self.at] != '\n') { + if (self.data[self.at] == '\\' and self.peek() == '\n') { + self.at += 1; + } + self.at += 1; + } + } else if (self.data[self.at] == '/' and self.peek() == '*') { + self.at += 2; + + while (self.at < self.data.len and self.at < self.data.len and !(self.data[self.at] == '*' and self.peek() == '/')) { + self.at += 1; + } + if (self.at < self.data.len and self.data[self.at] == '*' and self.peek() == '/') { + self.at += 2; + } + } else { + consuming = false; + } + } + } +}; + +test "ShaderTokenizer" { + const testing = std.testing; + + var tokenizer = ShaderTokenizer.init( + \\// UBOs asdkfljlka ajksfk\ + \\ + \\#include "../my_file\".glsl" + \\ + \\layout(std140, binding = 0) uniform Matrices { + \\ mat4 projection; + \\ mat4 view; + \\}; + \\ + \\layout(location = 2) uniform vec3 color; + \\ + \\// Input, output blocks + \\ + \\#if VERTEX_SHADER + \\ + \\layout(location = 0) in vec3 aPos; + \\ + \\void main() { + \\ gl_Position = projection * view * vec4(aPos.xyz, 1.0); + \\} + \\#endif // VERTEX_SHADER + \\ + \\#if FRAGMENT_SHADER + \\ + \\out vec4 FragColor; + \\ + \\void main() { + \\ FragColor += vec4(vec3(1.0), 1.0f); + \\ bool logic = (true && false || _asdfjkh) | (1u << -123U); + \\ 0xAf123FF + \\ -0x123 + \\ -09872 + \\ .1 + \\ .12f + \\ 0.2LF + \\ +0 + \\ -0f + \\} + \\ + \\ + \\#endif // FRAGMNET_SHADER + ); + + var token = tokenizer.next(); + while (token.type != .End) : (token = tokenizer.next()) { + try std.io.getStdErr().writer().print("{} \"{s}\"\n", .{ token.type, token.text }); + try testing.expect(token.type != .Unknown); + } +} + fn loadShaderErr(self: *AssetManager, id: AssetId) !LoadedShader { const path = asset_manifest.getPath(id); + const dir = std.fs.path.dirname(path) orelse @panic("No dir"); - const data = try self.loadFile(self.allocator, path, SHADER_MAX_BYTES); + const data = try self.loadFile(self.frame_arena, path, SHADER_MAX_BYTES); - const loaded_shader = LoadedShader{ .source = data.bytes }; + var included_asset_ids = std.ArrayList(AssetId).init(self.frame_arena); + var preprocessed_segments = std.SegmentedList([]const u8, 64){}; + var final_len: usize = 0; + + // Preprocess + { + var tokenizer = ShaderTokenizer.init(data.bytes); + + var last_offset: usize = 0; + + var token = tokenizer.next(); + while (token.type != .End) : (token = tokenizer.next()) { + switch (token.type) { + .Directive => { + if (std.mem.eql(u8, token.text, "include")) { + + // Append section of text up to this directive + try preprocessed_segments.append(self.frame_arena, data.bytes[last_offset..token.start]); + final_len += token.start - last_offset; + + const include_path = tokenizer.next(); + // Next section will start after this directive, this allows replacing #include with its content + last_offset = include_path.end; + + if (include_path.type != .String) { + return error.InvalidInclude; + } + + const included_file_path = try std.fs.path.resolve(self.frame_arena, &.{ dir, include_path.text }); + + const included_asset_id = assets.AssetPath.fromString(included_file_path).hash(); + if (included_asset_id != 0) { + try included_asset_ids.append(included_asset_id); + const included_shader = try self.loadShaderErr(included_asset_id); + try preprocessed_segments.append(self.frame_arena, included_shader.source); + final_len += included_shader.source.len; + } + } + }, + else => {}, + } + } + + { + const remaining = data.bytes.len - last_offset; + if (remaining > 0) { + try preprocessed_segments.append(self.frame_arena, data.bytes[last_offset..data.bytes.len]); + final_len += remaining; + } + } + } + + var result_source = try self.allocator.alloc(u8, final_len); + + // Join source sections + { + var cursor: usize = 0; + var iter = preprocessed_segments.constIterator(0); + + while (iter.next()) |slice| { + @memcpy(result_source[cursor .. cursor + slice.len], slice.*); + cursor += slice.len; + } + } + + const loaded_shader = LoadedShader{ .source = result_source }; { self.rw_lock.lock(); defer self.rw_lock.unlock(); try self.loaded_assets.put(self.allocator, id, .{ .shader = loaded_shader }); try self.modified_times.put(self.allocator, id, data.modified); + + try self.addDependencies(id, included_asset_ids.items); } return loaded_shader; diff --git a/src/assets/root.zig b/src/assets/root.zig index b949b99..8f36992 100644 --- a/src/assets/root.zig +++ b/src/assets/root.zig @@ -1,3 +1,5 @@ +const std = @import("std"); + pub const AssetId = u64; pub const Handle = struct { @@ -8,3 +10,149 @@ pub const Handle = struct { pub const Texture = extern struct { id: AssetId = 0 }; pub const Material = extern struct { id: AssetId = 0 }; }; + +pub const AssetType = enum { + Scene, + Mesh, + Shader, + ShaderProgram, + Texture, + Material, + + pub fn pluralName(self: AssetType) []const u8 { + return switch (self) { + .Scene => "Scenes", + .Mesh => "Meshes", + .Shader => "Shaders", + .ShaderProgram => "ShaderPrograms", + .Texture => "Textures", + .Material => "Materials", + }; + } + + pub fn ext(self: AssetType) []const u8 { + return switch (self) { + .Scene => "scn", + .Mesh => "mesh", + .Shader => "glsl", + .ShaderProgram => "prog", + .Texture => "tex", + .Material => "mat", + }; + } +}; + +pub const AssetPath = union(enum) { + invalid: void, // translates to handle with id: 0 + simple: []const u8, + nested: struct { + path: []const u8, + sub_path: []const u8, + }, + + pub fn hash(self: AssetPath) u64 { + switch (self) { + .invalid => return 0, + else => {}, + } + var hasher = std.hash.Wyhash.init(0); + std.hash.autoHashStrat(&hasher, self.getPath(), .Deep); + std.hash.autoHashStrat(&hasher, self.getSubPath(), .Deep); + return hasher.final(); + } + + pub fn subPath(self: AssetPath, sub_path: []const u8) AssetPath { + return switch (self) { + .invalid => self, + .simple => |path| AssetPath{ .nested = .{ .path = path, .sub_path = sub_path } }, + .nested => |nested| AssetPath{ .nested = .{ .path = nested.path, .sub_path = sub_path } }, + }; + } + + pub fn getPath(self: AssetPath) []const u8 { + return switch (self) { + .invalid => "", + .simple => |path| path, + .nested => |nested| nested.path, + }; + } + + pub fn getSubPath(self: AssetPath) ?[]const u8 { + return switch (self) { + .invalid => null, + .nested => |nested| nested.sub_path, + else => null, + }; + } + + pub inline fn strLen(self: AssetPath) usize { + return switch (self) { + .invalid => 0, + .simple => |path| path.len, + .nested => |nested| return nested.path.len + nested.sub_path.len + 1, + }; + } + + pub fn writeString(self: AssetPath, writer: anytype) !void { + switch (self) { + .invalid => {}, + .simple => |path| { + try writer.writeAll(path); + }, + .nested => |nested| { + try writer.writeAll(nested.path); + try writer.writeByte('#'); + try writer.writeAll(nested.sub_path); + }, + } + } + + pub fn toString(self: AssetPath, out_buf: []u8) ![]u8 { + return self.toStringSep(out_buf, '#'); + } + + pub fn toStringSep(self: AssetPath, out_buf: []u8, sep: u8) ![]u8 { + if (out_buf.len < self.strLen()) { + return error.BufferTooSmall; + } + + switch (self) { + .invalid => { + return out_buf[0..0]; + }, + .simple => |path| { + @memcpy(out_buf[0..path.len], path); + return out_buf[0..path.len]; + }, + .nested => |nested| { + @memcpy(out_buf[0..nested.path.len], nested.path); + out_buf[nested.path.len] = sep; + @memcpy(out_buf[nested.path.len + 1 .. nested.path.len + 1 + nested.sub_path.len], nested.sub_path); + + return out_buf[0..self.strLen()]; + }, + } + } + + pub fn toStringAlloc(self: AssetPath, allocator: std.mem.Allocator) ![]u8 { + const buf = try allocator.alloc(u8, self.strLen()); + errdefer allocator.free(buf); + return try self.toString(buf); + } + + pub fn fromString(str: []const u8) AssetPath { + if (str.len == 0) { + return .invalid; + } + if (std.mem.indexOf(u8, str, "#")) |sep_idx| { + return .{ + .nested = .{ + .path = str[0..sep_idx], + .sub_path = str[sep_idx + 1 ..], + }, + }; + } + + return .{ .simple = str }; + } +}; diff --git a/src/game.zig b/src/game.zig index f866994..834449b 100644 --- a/src/game.zig +++ b/src/game.zig @@ -192,7 +192,7 @@ export fn game_init(global_allocator: *std.mem.Allocator) void { _ = globals.g_mem.world.addEntity(.{ .flags = .{ .dir_light = true, .rotate = true }, .transform = .{ .rot = Quat.fromEulerAngles(Vec3.new(70, 0, 0)) }, - .light = .{ .color_intensity = Vec4.new(1.00, 0.805, 0.570, 790) }, + .light = .{ .color_intensity = Vec4.new(1.00, 0.805, 0.8, 790 * 0.02) }, .rotate = .{ .axis = Vec3.up(), .rate = -10 }, });