Redo how assets are loaded, remove explicit load/unload calls

This will make code simpler because game code will never explicitly
unload assets or retain runtime asset handles!

build.zig

@@ -119,6 +119,37 @@ pub fn build(b: *Build) void {
     test_step.dependOn(&run_exe_unit_tests.step);
 }
 
+const NestedAssetDef = union(enum) {
+    path: std.StringHashMapUnmanaged(NestedAssetDef),
+    asset: usize,
+
+    pub fn put(self: *NestedAssetDef, allocator: std.mem.Allocator, path: []const u8, id: usize) !void {
+        var iter = try std.fs.path.componentIterator(path);
+        const filename = iter.last().?.name;
+        _ = iter.first();
+        // Skip first one because it's always "assets"
+        _ = iter.next();
+
+        var current = &self.path;
+
+        while (iter.next()) |comp| {
+            if (comp.name.ptr == filename.ptr) break;
+            const gop = try current.getOrPut(allocator, comp.name);
+            gop.value_ptr.* = NestedAssetDef{ .path = .{} };
+            current = &gop.value_ptr.path;
+        }
+
+        try current.put(allocator, std.fs.path.stem(filename), NestedAssetDef{ .asset = id });
+    }
+
+    pub fn deinit(self: *NestedAssetDef, allocator: std.mem.Allocator) void {
+        switch (self.*) {
+            .path => |*path| path.deinit(allocator),
+            else => {},
+        }
+    }
+};
+
 // Find all assets and cook them using assetc
 fn buildAssets(b: *std.Build, step: *Step, assetc: *Step.Compile, path: []const u8) !void {
     const assetsPath = b.pathFromRoot(path);
@@ -126,15 +157,23 @@ fn buildAssets(b: *std.Build, step: *Step, assetc: *Step.Compile, path: []const
     var assetsDir = try std.fs.openDirAbsolute(assetsPath, .{ .iterate = true });
     defer assetsDir.close();
 
+    var asset_id: usize = 1; // Start at 1 because asset id 0 = null asset
+    var meshes = NestedAssetDef{ .path = .{} };
+    var asset_paths = std.ArrayList([]const u8).init(b.allocator);
+
     var walker = try assetsDir.walk(b.allocator);
     defer walker.deinit();
 
     while (try walker.next()) |entry| {
-        if (std.mem.eql(u8, ".obj", std.fs.path.extension(entry.basename))) {
+        if (std.mem.endsWith(u8, entry.basename, ".obj")) {
             const run_assetc = b.addRunArtifact(assetc);
             run_assetc.addFileArg(.{ .path = b.pathJoin(&.{ path, entry.path }) });
-            const out_name = try std.mem.concat(b.allocator, u8, &.{ std.fs.path.stem(entry.basename), ".mesh" });
+            const out_name = try std.mem.concat(
-            const out_file = run_assetc.addOutputFileArg(out_name);
+                b.allocator,
+                u8,
+                &.{ std.fs.path.stem(entry.basename), ".mesh" },
+            );
+            const compiled_file = run_assetc.addOutputFileArg(out_name);
 
             const out_path = b.pathJoin(&.{
                 std.fs.path.dirname(entry.path) orelse ".",
@@ -142,13 +181,76 @@ fn buildAssets(b: *std.Build, step: *Step, assetc: *Step.Compile, path: []const
                 out_name,
             });
             const install_asset = b.addInstallFileWithDir(
-                out_file,
+                compiled_file,
                 .prefix,
                 out_path,
             );
             step.dependOn(&install_asset.step);
+
+            {
+                const id = asset_id;
+                asset_id += 1;
+                try meshes.put(b.allocator, out_path, id);
+                try asset_paths.append(out_path);
             }
         }
+
+        if (std.mem.endsWith(u8, entry.basename, ".glsl")) {
+            const out_path = b.pathJoin(&.{
+                path,
+                entry.path,
+            });
+            const install_shader = b.addInstallFileWithDir(.{ .path = out_path }, .prefix, out_path);
+            step.dependOn(&install_shader.step);
+        }
+    }
+
+    const manifest_step = try writeAssetManifest(b, step, asset_paths.items, &meshes);
+    assetc.step.dependOn(&manifest_step.step);
+}
+
+fn writeNestedAssetDef(writer: anytype, handle: []const u8, name: []const u8, asset_def: *NestedAssetDef, indent: usize) !void {
+    switch (asset_def.*) {
+        .path => |*path| {
+            var iter = path.iterator();
+
+            try writer.writeByteNTimes(' ', indent * 4);
+            try std.fmt.format(writer, "pub const {} = struct {{\n", .{std.zig.fmtId(name)});
+            while (iter.next()) |entry| {
+                try writeNestedAssetDef(writer, handle, entry.key_ptr.*, entry.value_ptr, indent + 1);
+            }
+            try writer.writeByteNTimes(' ', indent * 4);
+            try std.fmt.format(writer, "}};\n", .{});
+        },
+        .asset => |id| {
+            try writer.writeByteNTimes(' ', indent * 4);
+            try std.fmt.format(writer, "pub const {} = Handle.{s}{{ .id = {} }};\n", .{ std.zig.fmtId(name), handle, id });
+        },
+    }
+}
+
+fn writeAssetManifest(b: *Build, asset_step: *Step, asset_paths: [][]const u8, meshes: *NestedAssetDef) !*Step.WriteFile {
+    var mesh_asset_manifest = std.ArrayList(u8).init(b.allocator);
+    const writer = mesh_asset_manifest.writer();
+
+    try writer.writeAll("// Generated file, do not edit manually!\n\n");
+    try writer.writeAll("const Handle = @import(\"Assets.zig\").Handle;\n\n");
+
+    try writeNestedAssetDef(writer, "Mesh", "Meshes", meshes, 0);
+
+    try writer.writeAll("pub const asset_paths = [_][]const u8{\n");
+    for (asset_paths) |path| {
+        try std.fmt.format(writer, "    \"{}\",\n", .{std.zig.fmtEscapes(path)});
+    }
+    try writer.writeAll("};\n");
+
+    try writer.writeAll("pub fn getPath(asset_id: u32) []const u8 { return asset_paths[asset_id - 1]; }\n");
+
+    const result = mesh_asset_manifest.toOwnedSlice() catch @panic("OOM");
+    const write_step = b.addWriteFiles();
+    write_step.addBytesToSource(result, "src/asset_manifest.zig");
+    asset_step.dependOn(&write_step.step);
+    return write_step;
 }
 
 fn buildAssetCompiler(b: *Build, optimize: std.builtin.OptimizeMode) *Step.Compile {

src/Assets.zig

@@ -1,7 +1,24 @@
+// TODO:
+// - Don't allocate asset ids dynamically
+// - Store asset memory usage on CPU and GPU
+// - Use LRU to evict unused assets based on available memory
+//   (if we have enough memory never free, lol)
+//
+// NOTE: 1
+//   Renderer/Game code will touch assets each time they are used
+//   so LRU should work pretty well I think and I don't have to retain asset ids
+//   since they'll be pre-generated constants.
+//
+// NOTE: 2
+//   It makes hot reloading easier because it eliminates load*() calls completely
+//   Because each time an asset is used it's touched by using code, hot reload in asset
+//   server only needs to free assets and not actually reload them, cause they'll be reloaded
+//   next time they're used
 const std = @import("std");
 const gl = @import("gl.zig");
 const fs_utils = @import("fs/utils.zig");
 const formats = @import("formats.zig");
+const asset_manifest = @import("asset_manifest.zig");
 
 pub const Assets = @This();
 
@@ -35,9 +52,8 @@ pub const Handle = struct {
         id: AssetId = 0,
 
         // Returns a VAO
-        pub fn resolve(self: Mesh, assets: *Assets) *LoadedMesh {
+        pub fn resolve(self: Mesh, assets: *Assets) *const LoadedMesh {
-            const asset = assets.loaded_assets.getPtr(self.id) orelse unreachable;
+            if (assets.loaded_assets.getPtr(self.id)) |asset| {
-
                 switch (asset.*) {
                     .mesh => |*mesh| {
                         return mesh;
@@ -45,21 +61,31 @@ pub const Handle = struct {
                     else => unreachable,
                 }
             }
+
+            return loadMesh(assets, self.id);
+        }
     };
 };
 
 allocator: std.mem.Allocator,
 frame_arena: std.mem.Allocator,
 
+// All assets are relative to exe dir
+exe_dir: std.fs.Dir,
+
 loaded_assets: std.AutoHashMapUnmanaged(AssetId, LoadedAsset) = .{},
 
-// NOTE: asset id 0 - invalid asset
+next_id: AssetId = 10,
-next_id: AssetId = 1,
 
 pub fn init(allocator: std.mem.Allocator, frame_arena: std.mem.Allocator) Assets {
+    var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
+    const exe_dir_path = std.fs.selfExeDirPath(&buf) catch @panic("can't find self exe dir path");
+    const exe_dir = std.fs.openDirAbsolute(exe_dir_path, .{}) catch @panic("can't open self exe dir path");
+
     return .{
         .allocator = allocator,
         .frame_arena = frame_arena,
+        .exe_dir = exe_dir,
     };
 }
 
@@ -73,12 +99,12 @@ pub fn watchChanges(self: *Assets) void {
     while (iter.next()) |entry| {
         switch (entry.value_ptr.*) {
             .shaderProgram => |*shader| {
-                if (didUpdate(shader.definition.vertex, &shader.vert_modified) or didUpdate(shader.definition.fragment, &shader.frag_modified)) {
+                if (self.didUpdate(shader.definition.vertex, &shader.vert_modified) or self.didUpdate(shader.definition.fragment, &shader.frag_modified)) {
                     self.reloadAsset(entry.key_ptr.*);
                 }
             },
             .mesh => |*mesh| {
-                if (didUpdate(mesh.path, &mesh.modified)) {
+                if (self.didUpdate(asset_manifest.getPath(entry.key_ptr.*), &mesh.modified)) {
                     self.reloadAsset(entry.key_ptr.*);
                 }
             },
@@ -86,8 +112,8 @@ pub fn watchChanges(self: *Assets) void {
     }
 }
 
-fn didUpdate(path: []const u8, last_modified: *i128) bool {
+fn didUpdate(self: *Assets, path: []const u8, last_modified: *i128) bool {
-    const mod = fs_utils.getFileModifiedRelative(path) catch |err| {
+    const mod = fs_utils.getFileModifiedRelative(self.exe_dir, path) catch |err| {
         std.log.err("ERROR: {}\nfailed to check file modtime {s}\n", .{ err, path });
         return false;
     };
@@ -105,8 +131,11 @@ pub fn reloadAsset(self: *Assets, asset_id: AssetId) void {
                 std.log.err("Failed to reload shader program {}\n", .{err});
             };
         },
-        .mesh => |*mesh| {
+        .mesh => {
-            _ = self.loadMeshErr(mesh.path) catch |err| {
+            std.log.debug("reloading mesh {s}\n", .{asset_manifest.getPath(asset_id)});
+            _ = self.loadMeshErr(
+                asset_id,
+            ) catch |err| {
                 std.log.err("Fauled to reload mesh {}\n", .{err});
             };
         },
@@ -126,8 +155,8 @@ pub fn loadShaderProgram(self: *Assets, params: ShaderProgramDefinition) Handle.
 
         return .{ .id = 0 };
     };
-
+    const id = self.nextId();
-    const id = self.putLoadedAsset(.{
+    self.loaded_assets.put(self.allocator, id, .{
         .shaderProgram = .{
             .definition = .{
                 .vertex = self.allocator.dupe(u8, params.vertex) catch @panic("OOM"),
@@ -143,11 +172,11 @@ pub fn loadShaderProgram(self: *Assets, params: ShaderProgramDefinition) Handle.
 }
 
 fn loadShaderProgramErr(self: *Assets, prog: gl.GLuint, params: ShaderProgramDefinition) !struct { vert_modified: i128, frag_modified: i128 } {
-    const vertex_file = try loadFile(self.frame_arena, params.vertex, SHADER_MAX_BYTES);
+    const vertex_file = try self.loadFile(self.frame_arena, params.vertex, SHADER_MAX_BYTES);
     const vertex_shader = try loadShader(self.frame_arena, .vertex, vertex_file.bytes);
     defer gl.deleteShader(vertex_shader);
 
-    const fragment_file = try loadFile(self.frame_arena, params.fragment, SHADER_MAX_BYTES);
+    const fragment_file = try self.loadFile(self.frame_arena, params.fragment, SHADER_MAX_BYTES);
     const fragment_shader = try loadShader(self.frame_arena, .fragment, fragment_file.bytes);
     defer gl.deleteShader(fragment_shader);
 
@@ -178,18 +207,37 @@ fn loadShaderProgramErr(self: *Assets, prog: gl.GLuint, params: ShaderProgramDef
     return .{ .vert_modified = vertex_file.modified, .frag_modified = fragment_file.modified };
 }
 
-pub fn loadMesh(self: *Assets, path: []const u8) Handle.Mesh {
+const NullMesh = LoadedMesh{
-    const id = self.loadMeshErr(path) catch |err| {
+    .modified = 0,
-        std.log.err("Error: {} loading mesh at path: {s}", .{ err, path });
-        return .{ .id = 0 };
-    };
 
-    return .{ .id = id };
+    .positions = BufferSlice{
+        .buffer = 0,
+        .offset = 0,
+        .stride = 0,
+    },
+    .normals = BufferSlice{
+        .buffer = 0,
+        .offset = 0,
+        .stride = 0,
+    },
+    .indices = IndexSlice{
+        .buffer = 0,
+        .offset = 0,
+        .count = 0,
+        .type = gl.UNSIGNED_SHORT,
+    },
+};
+
+pub fn loadMesh(self: *Assets, id: AssetId) *const LoadedMesh {
+    return self.loadMeshErr(id) catch |err| {
+        std.log.err("Error: {} loading mesh at path: {s}", .{ err, asset_manifest.getPath(id) });
+        return &NullMesh;
+    };
 }
 
-fn loadMeshErr(self: *Assets, path: []const u8) !AssetId {
+fn loadMeshErr(self: *Assets, id: AssetId) !*const LoadedMesh {
-    const data = try loadFile(self.frame_arena, path, MESH_MAX_BYTES);
+    const path = asset_manifest.getPath(id);
-
+    const data = try self.loadFile(self.frame_arena, path, MESH_MAX_BYTES);
     const mesh = formats.Mesh.fromBuffer(data.bytes);
 
     var bufs = [_]gl.GLuint{ 0, 0, 0 };
@@ -225,7 +273,6 @@ fn loadMeshErr(self: *Assets, path: []const u8) !AssetId {
     // gl.bindVertexBuffer(_bindingindex: GLuint, _buffer: GLuint, _offset: GLintptr, _stride: GLsizei)
 
     const loaded_mesh = LoadedMesh{
-        .path = try self.allocator.dupe(u8, path),
         .modified = data.modified,
 
         .positions = .{
@@ -246,15 +293,14 @@ fn loadMeshErr(self: *Assets, path: []const u8) !AssetId {
         },
     };
 
-    return try self.putLoadedAsset(.{ .mesh = loaded_mesh });
+    try self.loaded_assets.put(self.allocator, id, .{ .mesh = loaded_mesh });
+    return @ptrCast(&self.loaded_assets.getPtr(id).?.mesh);
 }
 
-fn putLoadedAsset(self: *Assets, asset: LoadedAsset) !AssetId {
+fn nextId(self: *Assets) AssetId {
     const id = self.next_id;
     self.next_id += 1;
 
-    try self.loaded_assets.put(self.allocator, id, asset);
-
     return id;
 }
 
@@ -272,7 +318,6 @@ const LoadedShaderProgram = struct {
 };
 
 const LoadedMesh = struct {
-    path: []const u8,
     modified: i128,
 
     positions: BufferSlice,
@@ -285,7 +330,7 @@ pub const BufferSlice = struct {
     offset: gl.GLintptr,
     stride: gl.GLsizei,
 
-    pub fn bind(self: *BufferSlice, index: gl.GLuint) void {
+    pub fn bind(self: *const BufferSlice, index: gl.GLuint) void {
         gl.bindVertexBuffer(index, self.buffer, self.offset, self.stride);
     }
 };
@@ -318,8 +363,9 @@ const AssetData = struct {
     bytes: []u8,
     modified: i128,
 };
-fn loadFile(allocator: std.mem.Allocator, path: []const u8, max_size: usize) !AssetData {
+
-    const file = try std.fs.cwd().openFile(path, .{});
+fn loadFile(self: *Assets, allocator: std.mem.Allocator, path: []const u8, max_size: usize) !AssetData {
+    const file = try self.exe_dir.openFile(path, .{});
     defer file.close();
     const meta = try file.metadata();
     const bytes = try file.reader().readAllAlloc(allocator, max_size);

src/asset_manifest.zig

@@ -0,0 +1,11 @@
+// Generated file, do not edit manually!
+
+const Handle = @import("Assets.zig").Handle;
+
+pub const Meshes = struct {
+    pub const bunny = Handle.Mesh{ .id = 1 };
+};
+pub const asset_paths = [_][]const u8{
+    ".\\assets\\bunny.mesh",
+};
+pub fn getPath(asset_id: u32) []const u8 { return asset_paths[asset_id - 1]; }

src/fs/utils.zig

@@ -1,7 +1,7 @@
 const std = @import("std");
 
-pub fn getFileModifiedRelative(path: []const u8) !i128 {
+pub fn getFileModifiedRelative(dir: std.fs.Dir, path: []const u8) !i128 {
-    var lib_file = try std.fs.cwd().openFile(path, .{});
+    var lib_file = try dir.openFile(path, .{});
     defer lib_file.close();
     var lib_file_meta = try lib_file.metadata();
 

src/game.zig

@@ -6,6 +6,7 @@ const formats = @import("formats.zig");
 const zlm = @import("zlm");
 const Vec3 = zlm.Vec3;
 const Mat4 = zlm.Mat4;
+const a = @import("asset_manifest.zig");
 
 const FRAME_ARENA_SIZE = 1024 * 1024 * 512;
 
@@ -172,10 +173,10 @@ export fn game_init(global_allocator: *std.mem.Allocator) void {
     gl.vertexAttribPointer(0, 3, gl.FLOAT, gl.FALSE, @sizeOf(f32) * 3, @ptrFromInt(0));
     gl.enableVertexAttribArray(0);
 
-    g_mem.shader_program = g_assets.loadShaderProgram(.{ .vertex = "src/shaders/vert.glsl", .fragment = "src/shaders/frag.glsl" });
+    g_mem.shader_program = g_assets.loadShaderProgram(.{ .vertex = "assets/shaders/vert.glsl", .fragment = "assets/shaders/frag.glsl" });
 
     // MESH PROGRAM
-    g_mem.mesh_program = g_assets.loadShaderProgram(.{ .vertex = "src/shaders/mesh.vert.glsl", .fragment = "src/shaders/mesh.frag.glsl" });
+    g_mem.mesh_program = g_assets.loadShaderProgram(.{ .vertex = "assets/shaders/mesh.vert.glsl", .fragment = "assets/shaders/mesh.frag.glsl" });
     const mesh_program_name = g_mem.mesh_program.resolve(g_assets);
 
     gl.uniformBlockBinding(mesh_program_name, 0, UBO.CameraMatrices.value());
@@ -200,7 +201,7 @@ export fn game_init(global_allocator: *std.mem.Allocator) void {
 
     // MESH ITSELF
     // TODO: asset paths relative to exe
-    g_mem.mesh = g_assets.loadMesh("zig-out/assets/bunny.mesh");
+    g_mem.mesh = a.Meshes.bunny;
 
     var camera_ubo: gl.GLuint = 0;
     gl.createBuffers(1, &camera_ubo);
@@ -275,7 +276,7 @@ export fn game_update() bool {
     // gl.fenceSync(_condition: GLenum, _flags: GLbitfield)
     camera_matrix.* = .{
         .projection = Mat4.createPerspective(
-            std.math.degreesToRadians(f32, 20), //fov
+            std.math.degreesToRadians(f32, 30),
             f_width / f_height,
             0.1,
             100.0,