From 637ad859799946aaf78093bb6a6a9d791b70270f Mon Sep 17 00:00:00 2001 From: sergeypdev Date: Sat, 24 Aug 2024 17:23:55 +0400 Subject: [PATCH] Async asset watch --- src/AssetManager.zig | 290 +++++++++++++++++++++++++++++++------------ src/game.zig | 2 + 2 files changed, 211 insertions(+), 81 deletions(-) diff --git a/src/AssetManager.zig b/src/AssetManager.zig index 4d2b601..cb49631 100644 --- a/src/AssetManager.zig +++ b/src/AssetManager.zig @@ -24,6 +24,7 @@ const checkGLError = @import("Render.zig").checkGLError; const BuddyAllocator = @import("BuddyAllocator.zig"); const Vec2 = @import("zalgebra").Vec2; const Vec3 = @import("zalgebra").Vec3; +const sdl = @import("sdl.zig"); const tracy = @import("tracy"); pub const AssetId = assets.AssetId; @@ -50,12 +51,98 @@ dependencies: std.AutoHashMapUnmanaged(AssetId, std.SegmentedList(AssetId, 4)) = // Mapping from asset to all assets that depend on it dependees: std.AutoHashMapUnmanaged(AssetId, std.SegmentedList(AssetId, 4)) = .{}, loaded_assets: std.AutoHashMapUnmanaged(AssetId, LoadedAsset) = .{}, +rw_lock: std.Thread.RwLock.DefaultRwLock = .{}, +asset_watcher: AssetWatcher = undefined, vertex_heap: VertexBufferHeap, -pub fn init(allocator: std.mem.Allocator, frame_arena: std.mem.Allocator) AssetManager { - // basisu.init_transcoder(); +const AssetWatcher = struct { + assetman: *AssetManager, + fba: std.heap.FixedBufferAllocator, + thread: ?*sdl.SDL_Thread = null, + finished: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + const THREAD_MEMORY = 1024 * 1024 * 32; + + pub fn init(assetman: *AssetManager) !AssetWatcher { + const memory_bytes = try assetman.allocator.alloc(u8, THREAD_MEMORY); + + return AssetWatcher{ + .assetman = assetman, + .fba = std.heap.FixedBufferAllocator.init(memory_bytes), + }; + } + + pub fn deinit(self: *AssetWatcher) void { + self.finished.store(true, .unordered); + if (self.thread) |thread| { + var status: c_int = 0; + sdl.SDL_WaitThread(thread, &status); + } + } + + pub fn startWatching(self: *AssetWatcher) void { + self.thread = sdl.SDL_CreateThread(watcherThread, "AssetManager Watcher", @ptrCast(self)) orelse { + std.log.err("SDL Error: {s}\n", .{sdl.SDL_GetError()}); + @panic("SDL_CreateThread"); + }; + } + + fn watcherThread(userdata: ?*anyopaque) callconv(.C) c_int { + const self: *AssetWatcher = @alignCast(@ptrCast(userdata)); + + while (!self.finished.load(.unordered)) { + self.fba.reset(); + + self.watchChanges() catch |err| { + std.log.err("Watch Changes error: {}\n", .{err}); + }; + } + + return 0; + } + + // TODO: proper watching + fn watchChanges(self: *AssetWatcher) !void { + const zone = tracy.initZone(@src(), .{ .name = "AssetWatcher.watchChanges" }); + defer zone.deinit(); + + var updatedList = std.SegmentedList(AssetId, 128){}; + + var loaded_assets: std.AutoHashMapUnmanaged(AssetId, LoadedAsset) = .{}; + var modified_times: std.AutoHashMapUnmanaged(AssetId, i128) = .{}; + + { + self.assetman.rw_lock.lockShared(); + defer self.assetman.rw_lock.unlockShared(); + + loaded_assets = try self.assetman.loaded_assets.clone(self.fba.allocator()); + modified_times = try self.assetman.modified_times.clone(self.fba.allocator()); + } + + { + var iter = loaded_assets.iterator(); + while (iter.next()) |entry| { + const modified_time = modified_times.get(entry.key_ptr.*) orelse @panic("Modified Time Missing"); + if (self.assetman.didUpdate(asset_manifest.getPath(entry.key_ptr.*), modified_time)) { + try updatedList.append(self.fba.allocator(), entry.key_ptr.*); + } + } + } + + if (updatedList.len > 0) { + self.assetman.rw_lock.lock(); + defer self.assetman.rw_lock.unlock(); + + var iter = updatedList.iterator(0); + while (iter.next()) |asset_id| { + self.assetman.unloadAssetWithDependees(asset_id.*); + } + } + } +}; + +pub fn init(allocator: std.mem.Allocator, frame_arena: std.mem.Allocator) AssetManager { var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; const exe_dir_path = std.fs.selfExeDirPath(&buf) catch @panic("can't find self exe dir path"); const exe_dir = std.fs.openDirAbsolute(exe_dir_path, .{}) catch @panic("can't open self exe dir path"); @@ -68,14 +155,37 @@ pub fn init(allocator: std.mem.Allocator, frame_arena: std.mem.Allocator) AssetM }; } +pub fn initWatch(self: *AssetManager) void { + self.asset_watcher = AssetWatcher.init(self) catch @panic("AssetWatcher.init"); + + self.asset_watcher.startWatching(); +} + pub fn deinit(self: *AssetManager) void { + self.asset_watcher.deinit(); + + var iter = self.loaded_assets.valueIterator(); + while (iter.next()) |asset| { + self.freeAsset(asset); + } + + self.modified_times.deinit(self.allocator); + self.dependees.deinit(self.allocator); + self.dependencies.deinit(self.allocator); self.loaded_assets.deinit(self.allocator); } +fn resolveAsset(self: *AssetManager, handle: AssetId) ?*LoadedAsset { + self.rw_lock.lockShared(); + defer self.rw_lock.unlockShared(); + + return self.loaded_assets.getPtr(handle); +} + pub fn resolveShader(self: *AssetManager, handle: Handle.Shader) LoadedShader { if (handle.id == 0) return NullShader; - if (self.loaded_assets.getPtr(handle.id)) |asset| { + if (self.resolveAsset(handle.id)) |asset| { return asset.shader; } @@ -85,7 +195,7 @@ pub fn resolveShader(self: *AssetManager, handle: Handle.Shader) LoadedShader { pub fn resolveShaderProgram(self: *AssetManager, handle: Handle.ShaderProgram) LoadedShaderProgram { if (handle.id == 0) return NullShaderProgram; - if (self.loaded_assets.getPtr(handle.id)) |asset| { + if (self.resolveAsset(handle.id)) |asset| { switch (asset.*) { .shaderProgram => |shader| { return shader; @@ -100,7 +210,7 @@ pub fn resolveShaderProgram(self: *AssetManager, handle: Handle.ShaderProgram) L pub fn resolveMesh(self: *AssetManager, handle: Handle.Mesh) LoadedMesh { if (handle.id == 0) return NullMesh; - if (self.loaded_assets.getPtr(handle.id)) |asset| { + if (self.resolveAsset(handle.id)) |asset| { switch (asset.*) { .mesh => |mesh| { return mesh; @@ -115,7 +225,7 @@ pub fn resolveMesh(self: *AssetManager, handle: Handle.Mesh) LoadedMesh { pub fn resolveTexture(self: *AssetManager, handle: Handle.Texture) LoadedTexture { if (handle.id == 0) return NullTexture; - if (self.loaded_assets.getPtr(handle.id)) |asset| { + if (self.resolveAsset(handle.id)) |asset| { switch (asset.*) { .texture => |texture| { return texture; @@ -130,7 +240,7 @@ pub fn resolveTexture(self: *AssetManager, handle: Handle.Texture) LoadedTexture pub fn resolveScene(self: *AssetManager, handle: Handle.Scene) formats.Scene { if (handle.id == 0) return NullScene.scene; - if (self.loaded_assets.getPtr(handle.id)) |asset| { + if (self.resolveAsset(handle.id)) |asset| { switch (asset.*) { .scene => |scene| { return scene.scene; @@ -145,7 +255,7 @@ pub fn resolveScene(self: *AssetManager, handle: Handle.Scene) formats.Scene { pub fn resolveMaterial(self: *AssetManager, handle: Handle.Material) formats.Material { if (handle.id == 0) return NullMaterial; - if (self.loaded_assets.getPtr(handle.id)) |asset| { + if (self.resolveAsset(handle.id)) |asset| { switch (asset.*) { .material => |material| { return material; @@ -157,31 +267,12 @@ pub fn resolveMaterial(self: *AssetManager, handle: Handle.Material) formats.Mat return self.loadMaterial(handle.id); } -// TODO: proper watching -pub fn watchChanges(self: *AssetManager) void { - const zone = tracy.initZone(@src(), .{ .name = "AssetManager.watchChanges" }); - defer zone.deinit(); - - var iter = self.loaded_assets.iterator(); - while (iter.next()) |entry| { - const gop = self.modified_times.getOrPut(self.allocator, entry.key_ptr.*) catch return; - if (!gop.found_existing) { - gop.value_ptr.* = 0; - } - if (self.didUpdate(asset_manifest.getPath(entry.key_ptr.*), gop.value_ptr)) { - self.unloadAssetWithDependees(entry.key_ptr.*); - } - } -} - -fn didUpdate(self: *AssetManager, path: []const u8, last_modified: *i128) bool { +fn didUpdate(self: *AssetManager, path: []const u8, last_modified: i128) bool { const mod = fs_utils.getFileModifiedRelative(self.exe_dir, path) catch |err| { std.log.err("ERROR: {}\nfailed to check file modtime {s}\n", .{ err, path }); return false; }; - const updated = mod != last_modified.*; - last_modified.* = mod; - return updated; + return mod != last_modified; } pub const ShaderProgramDefinition = struct { @@ -262,10 +353,16 @@ fn loadShaderProgramErr(self: *AssetManager, id: AssetId) !LoadedShaderProgram { const loaded_shader_program = LoadedShaderProgram{ .program = prog, }; - try self.loaded_assets.put(self.allocator, id, .{ - .shaderProgram = loaded_shader_program, - }); - try self.modified_times.put(self.allocator, id, data.modified); + + { + self.rw_lock.lock(); + defer self.rw_lock.unlock(); + + try self.loaded_assets.put(self.allocator, id, .{ + .shaderProgram = loaded_shader_program, + }); + try self.modified_times.put(self.allocator, id, data.modified); + } return loaded_shader_program; } @@ -389,8 +486,13 @@ fn loadMeshErr(self: *AssetManager, id: AssetId) !LoadedMesh { }, }; - try self.loaded_assets.put(self.allocator, id, .{ .mesh = loaded_mesh }); - try self.modified_times.put(self.allocator, id, data.modified); + { + self.rw_lock.lock(); + defer self.rw_lock.unlock(); + + try self.loaded_assets.put(self.allocator, id, .{ .mesh = loaded_mesh }); + try self.modified_times.put(self.allocator, id, data.modified); + } return loaded_mesh; } @@ -462,12 +564,18 @@ fn loadTextureErr(self: *AssetManager, id: AssetId) !LoadedTexture { .handle = handle, .uv_scale = uv_scale, }; - try self.loaded_assets.put( - self.allocator, - id, - .{ .texture = loaded_texture }, - ); - try self.modified_times.put(self.allocator, id, data.modified); + + { + self.rw_lock.lock(); + defer self.rw_lock.unlock(); + + try self.loaded_assets.put( + self.allocator, + id, + .{ .texture = loaded_texture }, + ); + try self.modified_times.put(self.allocator, id, data.modified); + } return loaded_texture; } @@ -490,14 +598,19 @@ fn loadSceneErr(self: *AssetManager, id: AssetId) !LoadedScene { .scene = scene, }; - try self.loaded_assets.put( - self.allocator, - id, - .{ - .scene = loaded_scene, - }, - ); - try self.modified_times.put(self.allocator, id, data.modified); + { + self.rw_lock.lock(); + defer self.rw_lock.unlock(); + + try self.loaded_assets.put( + self.allocator, + id, + .{ + .scene = loaded_scene, + }, + ); + try self.modified_times.put(self.allocator, id, data.modified); + } return loaded_scene; } @@ -516,14 +629,19 @@ fn loadMaterialErr(self: *AssetManager, id: AssetId) !formats.Material { const material = formats.Material.fromBuffer(data.bytes); - try self.loaded_assets.put( - self.allocator, - id, - .{ - .material = material, - }, - ); - try self.modified_times.put(self.allocator, id, data.modified); + { + self.rw_lock.lock(); + defer self.rw_lock.unlock(); + + try self.loaded_assets.put( + self.allocator, + id, + .{ + .material = material, + }, + ); + try self.modified_times.put(self.allocator, id, data.modified); + } return material; } @@ -643,8 +761,13 @@ fn loadShaderErr(self: *AssetManager, id: AssetId) !LoadedShader { const data = try self.loadFile(self.allocator, path, SHADER_MAX_BYTES); const loaded_shader = LoadedShader{ .source = data.bytes }; - try self.loaded_assets.put(self.allocator, id, .{ .shader = loaded_shader }); - try self.modified_times.put(self.allocator, id, data.modified); + { + self.rw_lock.lock(); + defer self.rw_lock.unlock(); + + try self.loaded_assets.put(self.allocator, id, .{ .shader = loaded_shader }); + try self.modified_times.put(self.allocator, id, data.modified); + } return loaded_shader; } @@ -707,34 +830,39 @@ fn deleteDependees(self: *AssetManager, id: AssetId) void { } } +fn freeAsset(self: *AssetManager, asset: *LoadedAsset) void { + switch (asset.*) { + .mesh => |*mesh| { + self.vertex_heap.free(mesh.heap_handle); + }, + .shader => |*shader| { + self.allocator.free(shader.source); + }, + .shaderProgram => |*program| { + gl.deleteProgram(program.program); + }, + .texture => |*texture| { + gl.GL_ARB_bindless_texture.makeTextureHandleNonResidentARB(texture.handle); + gl.deleteTextures(1, &texture.name); + }, + .scene => |*scene| { + self.allocator.free(scene.buf); + }, + .material => {}, + } +} + +// Don't call without write lock fn unloadAssetWithDependees(self: *AssetManager, id: AssetId) void { std.log.debug("unload asset id {}: {s}\n", .{ id, asset_manifest.getPath(id) }); self.deleteDependees(id); - { - const asset = self.loaded_assets.getPtr(id) orelse return; - - switch (asset.*) { - .mesh => |*mesh| { - self.vertex_heap.free(mesh.heap_handle); - }, - .shader => |*shader| { - self.allocator.free(shader.source); - }, - .shaderProgram => |*program| { - gl.deleteProgram(program.program); - }, - .texture => |*texture| { - gl.GL_ARB_bindless_texture.makeTextureHandleNonResidentARB(texture.handle); - gl.deleteTextures(1, &texture.name); - }, - .scene => |*scene| { - self.allocator.free(scene.buf); - }, - .material => {}, - } + if (self.loaded_assets.getPtr(id)) |asset| { + self.freeAsset(asset); } + _ = self.loaded_assets.remove(id); + _ = self.modified_times.remove(id); _ = self.dependees.remove(id); _ = self.dependencies.remove(id); diff --git a/src/game.zig b/src/game.zig index 348e56e..88be45d 100644 --- a/src/game.zig +++ b/src/game.zig @@ -177,6 +177,7 @@ export fn game_init(global_allocator: *std.mem.Allocator) void { globals.g_mem.render.camera = &globals.g_mem.free_cam.camera; std.log.debug("actual ptr: {}, correct ptr {}", .{ globals.g_mem.assetman.frame_arena.ptr, globals.g_mem.frame_fba.allocator().ptr }); globals.g_assetman = &globals.g_mem.assetman; + globals.g_assetman.initWatch(); globals.g_mem.performance_frequency = c.SDL_GetPerformanceFrequency(); globals.g_mem.last_frame_time = c.SDL_GetPerformanceCounter(); @@ -531,6 +532,7 @@ export fn game_update() bool { export fn game_shutdown() void { const gmem = globals.g_mem; std.log.debug("game_shutdown\n", .{}); + gmem.assetman.deinit(); gmem.global_allocator.free(gmem.frame_fba.buffer); gmem.global_allocator.destroy(gmem); gl.disable(gl.DEBUG_OUTPUT);