sergeypdev/engine
1
0
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Texture loading using a dedicated transfer queue, woo

Browse Source
This commit is contained in:
sergeypdev 2024-12-26 02:18:51 +04:00
parent 2215d082c4
commit 0c7aad9070
3 changed files with 282 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

341
src/AssetManager.zig
View File

@ -54,11 +54,13 @@ modified_times: std.AutoHashMapUnmanaged(AssetId, i128) = .{},
dependencies: std.AutoHashMapUnmanaged(AssetId, std.SegmentedList(AssetId, 4)) = .{},
// Mapping from asset to all assets that depend on it
dependees: std.AutoHashMapUnmanaged(AssetId, std.SegmentedList(AssetId, 4)) = .{},
loading_assets: std.AutoHashMapUnmanaged(AssetId, LoadingAsset) = .{},
loaded_assets: std.AutoHashMapUnmanaged(AssetId, LoadedAsset) = .{},
rw_lock: std.Thread.RwLock.DefaultRwLock = .{},
asset_watcher: AssetWatcher = undefined,
// texture_heap: BuddyAllocator,
texture_heap: BuddyAllocator,
texture_heap_memory: vk.DeviceMemory = .null_handle,
vertex_heap: VertexBufferHeap,
gc: *GraphicsContext,
descriptorman: *DescriptorManager,
@ -170,17 +172,24 @@ pub fn init(allocator: std.mem.Allocator, frame_arena: std.mem.Allocator, gc: *G
break :blk &gc.queues.graphics;
};
return .{
var result = AssetManager{
.allocator = allocator,
.frame_arena = frame_arena,
.exe_dir = exe_dir,
// .texture_heap = BuddyAllocator.init(allocator, 64, 22), // 256MB
.texture_heap = try BuddyAllocator.init(allocator, 64, 22), // 256MB
.vertex_heap = VertexBufferHeap.init(allocator) catch @panic("OOM"),
.gc = gc,
.descriptorman = descriptorman,
.queue = queue,
.command_pool = try queue.createCommandPool(.{ .transient_bit = true }),
};
result.texture_heap_memory = try gc.device.allocateMemory(&.{
.allocation_size = result.texture_heap.getSize(),
.memory_type_index = gc.memory_config.gpu.type_index,
}, null);
return result;
}
pub fn initWatch(self: *AssetManager) void {
@ -196,11 +205,13 @@ pub fn deinit(self: *AssetManager) void {
while (iter.next()) |asset| {
self.freeAsset(asset);
}
// TODO: interrupt loading assets
self.modified_times.deinit(self.allocator);
self.dependees.deinit(self.allocator);
self.dependencies.deinit(self.allocator);
self.loaded_assets.deinit(self.allocator);
self.loading_assets.deinit(self.allocator);
}
fn resolveAsset(self: *AssetManager, handle: AssetId) ?*LoadedAsset {
@ -210,6 +221,13 @@ fn resolveAsset(self: *AssetManager, handle: AssetId) ?*LoadedAsset {
return self.loaded_assets.getPtr(handle);
}
fn resolveLoadingAsset(self: *AssetManager, handle: AssetId) ?*LoadingAsset {
self.rw_lock.lockShared();
defer self.rw_lock.unlockShared();
return self.loading_assets.getPtr(handle);
}
const DefinePair = struct {
key: []const u8,
value: []const u8,
@ -274,6 +292,20 @@ pub fn resolveMesh(self: *AssetManager, handle: Handle.Mesh) LoadedMesh {
pub fn resolveTexture(self: *AssetManager, handle: Handle.Texture) LoadedTexture {
if (handle.id == 0) return NullTexture;
if (self.resolveLoadingAsset(handle.id)) |loading_texture| {
const status = self.gc.device.getFenceStatus(loading_texture.texture.transfer_fence) catch @panic("getFenceStatus");
switch (status) {
.success => {
self.finishLoadingAsset(handle.id);
},
.not_ready => {
return NullTexture;
},
else => unreachable,
}
}
if (self.resolveAsset(handle.id)) |asset| {
switch (asset.*) {
.texture => |texture| {
@ -546,8 +578,9 @@ const NullMesh = LoadedMesh{
};
const NullTexture = LoadedTexture{
.name = 0,
.handle = 0,
.image = .null_handle,
.view = .null_handle,
.descriptor_handle = .{},
};
const NullScene = LoadedScene{
@ -649,6 +682,9 @@ fn loadTextureErr(self: *AssetManager, id: AssetId) !LoadedTexture {
var texture = try formats.Texture.fromBuffer(self.allocator, data.bytes);
defer texture.free(self.allocator);
var mip_offsets: []u64 = try self.frame_arena.alloc(u64, texture.header.mip_count);
defer self.frame_arena.free(mip_offsets);
const format: vk.Format = switch (texture.header.format) {
.bc7 => vk.Format.bc7_unorm_block,
.bc6 => vk.Format.bc6h_ufloat_block,
@ -658,7 +694,7 @@ fn loadTextureErr(self: *AssetManager, id: AssetId) !LoadedTexture {
const image = try self.gc.device.createImage(&.{
.image_type = .@"2d",
.format = format,
.extent = vk.Extent3D{ .width = texture.header.padded_width, .height = texture.header.padded_height, .depth = 1 },
.extent = vk.Extent3D{ .width = texture.header.width, .height = texture.header.height, .depth = 1 },
.mip_levels = texture.header.mip_count,
.array_layers = 1,
.samples = .{ .@"1_bit" = true },
@ -679,6 +715,23 @@ fn loadTextureErr(self: *AssetManager, id: AssetId) !LoadedTexture {
// TODO: for now just pray alignment requirements are not crazy
std.debug.assert(std.mem.isAligned(allocation.offset, mem_reqs.alignment));
try self.gc.device.bindImageMemory(image, self.texture_heap_memory, allocation.offset);
const view = try self.gc.device.createImageView(&.{
.image = image,
.format = format,
.view_type = .@"2d",
.components = .{ .r = .r, .g = .g, .b = .b, .a = .a },
.subresource_range = .{
.aspect_mask = .{ .color_bit = true },
.base_array_layer = 0,
.layer_count = 1,
.base_mip_level = 0,
.level_count = texture.header.mip_count,
},
}, null);
errdefer self.gc.device.destroyImageView(view, null);
var data_size: u64 = 0;
for (texture.data) |bytes| {
// TODO: handle copy offset alignment
@ -692,74 +745,186 @@ fn loadTextureErr(self: *AssetManager, id: AssetId) !LoadedTexture {
}, null);
errdefer self.gc.device.destroyBuffer(staging_buffer, null);
// const staging_mem_reqs = self.gc.device.getBufferMemoryRequirements(staging_buffer);
const staging_mem_reqs = self.gc.device.getBufferMemoryRequirements(staging_buffer);
var name: gl.GLuint = 0;
gl.createTextures(gl.TEXTURE_2D, 1, &name);
if (name == 0) {
return error.GLCreateTexture;
}
errdefer gl.deleteTextures(1, &name);
const staging_mem = try self.gc.device.allocateMemory(&.{
.memory_type_index = self.gc.memory_config.cpu.type_index,
.allocation_size = staging_mem_reqs.size,
}, null);
errdefer self.gc.device.freeMemory(staging_mem, null);
const gl_format: gl.GLenum = switch (texture.header.format) {
.bc7 => gl.COMPRESSED_RGBA_BPTC_UNORM,
.bc5 => gl.COMPRESSED_RG_RGTC2,
.bc6 => gl.COMPRESSED_RGB_BPTC_UNSIGNED_FLOAT,
};
try self.gc.device.bindBufferMemory(staging_buffer, staging_mem, 0);
gl.textureStorage2D(
name,
@intCast(texture.mipLevels()),
gl_format,
@intCast(texture.header.padded_width),
@intCast(texture.header.padded_height),
);
checkGLError();
const staging_bytes: []u8 = @as([*]u8, @ptrCast(try self.gc.device.mapMemory(staging_mem, 0, data_size, .{})))[0..data_size];
for (0..texture.mipLevels()) |mip_level| {
const desc = texture.getMipDesc(mip_level);
gl.compressedTextureSubImage2D(
name,
@intCast(mip_level),
0,
0,
@intCast(desc.width),
@intCast(desc.height),
gl_format,
@intCast(texture.data[mip_level].len),
@ptrCast(texture.data[mip_level].ptr),
);
checkGLError();
var offset: usize = 0;
for (0..texture.header.mip_count) |mip| {
mip_offsets[mip] = offset;
@memcpy(staging_bytes[offset .. offset + texture.data[mip].len], texture.data[mip]);
offset += texture.data[mip].len;
}
const uv_scale = Vec2.new(
@as(f32, @floatFromInt(texture.header.width)) / @as(f32, @floatFromInt(texture.header.padded_width)),
@as(f32, @floatFromInt(texture.header.height)) / @as(f32, @floatFromInt(texture.header.padded_height)),
);
const cmds = try self.command_pool.allocateCommandBuffer();
const handle = gl.GL_ARB_bindless_texture.getTextureHandleARB(name);
gl.GL_ARB_bindless_texture.makeTextureHandleResidentARB(handle);
errdefer gl.GL_ARB_bindless_texture.makeTextureHandleNonResidentARB(handle);
try cmds.beginCommandBuffer(&.{ .flags = .{ .one_time_submit_bit = true } });
const loaded_texture = LoadedTexture{
.name = name,
.handle = handle,
.uv_scale = uv_scale,
cmds.pipelineBarrier2(&.{
.memory_barrier_count = 0,
.image_memory_barrier_count = 1,
.p_image_memory_barriers = &.{vk.ImageMemoryBarrier2{
.src_access_mask = .{},
.src_stage_mask = .{},
.old_layout = .undefined,
.dst_stage_mask = .{ .copy_bit = true },
.dst_access_mask = .{ .transfer_write_bit = true },
.new_layout = .transfer_dst_optimal,
.subresource_range = .{
.aspect_mask = .{ .color_bit = true },
.base_array_layer = 0,
.base_mip_level = 0,
.layer_count = 1,
.level_count = texture.header.mip_count,
},
.image = image,
.src_queue_family_index = 0,
.dst_queue_family_index = 0,
}},
});
const regions = try self.frame_arena.alloc(vk.BufferImageCopy2, texture.header.mip_count);
defer self.frame_arena.free(regions);
for (regions, 0..) |*region, i| {
const desc = texture.getMipDesc(i);
region.* = .{
.buffer_offset = mip_offsets[i],
.buffer_row_length = desc.width,
.buffer_image_height = desc.height,
.image_offset = .{ .x = 0, .y = 0, .z = 0 },
.image_extent = .{ .width = desc.width, .height = desc.height, .depth = 1 },
.image_subresource = .{
.aspect_mask = .{ .color_bit = true },
.base_array_layer = 0,
.layer_count = 1,
.mip_level = @intCast(i),
},
};
}
cmds.copyBufferToImage2(&.{
.src_buffer = staging_buffer,
.dst_image = image,
.dst_image_layout = .transfer_dst_optimal,
.region_count = @intCast(regions.len),
.p_regions = regions.ptr,
});
cmds.pipelineBarrier2(&.{
.memory_barrier_count = 0,
.image_memory_barrier_count = 1,
.p_image_memory_barriers = &.{
vk.ImageMemoryBarrier2{
.src_stage_mask = .{ .copy_bit = true },
.src_access_mask = .{ .transfer_write_bit = true },
.old_layout = .transfer_dst_optimal,
.dst_stage_mask = .{ .vertex_shader_bit = true, .fragment_shader_bit = true, .compute_shader_bit = true },
.dst_access_mask = .{ .shader_sampled_read_bit = true },
.new_layout = .shader_read_only_optimal,
.subresource_range = .{
.aspect_mask = .{ .color_bit = true },
.base_array_layer = 0,
.base_mip_level = 0,
.layer_count = 1,
.level_count = texture.header.mip_count,
},
.image = image,
.src_queue_family_index = self.queue.family,
.dst_queue_family_index = self.gc.queues.graphics.family,
},
},
});
try cmds.endCommandBuffer();
const image_copy_fence = try self.gc.device.createFence(&.{}, null);
try self.queue.submit(&.{ .command_buffers = &.{cmds.handle} }, image_copy_fence);
const descriptor_handle = self.descriptorman.image_descriptor_array_2d.alloc(.{ .view = view, .layout = .shader_read_only_optimal }).handle;
const loading_texture = LoadingTexture{
.image = image,
.view = view,
.transfer_fence = image_copy_fence,
.staging_buf = staging_buffer,
.staging_memory = staging_mem,
.descriptor_handle = descriptor_handle,
.command_buffer = cmds.handle,
};
{
self.rw_lock.lock();
defer self.rw_lock.unlock();
try self.loaded_assets.put(
try self.loading_assets.put(
self.allocator,
id,
.{ .texture = loaded_texture },
.{ .texture = loading_texture },
);
try self.modified_times.put(self.allocator, id, data.modified);
}
return loaded_texture;
return NullTexture;
// return loaded_texture;
// var name: gl.GLuint = 0;
// gl.createTextures(gl.TEXTURE_2D, 1, &name);
// if (name == 0) {
// return error.GLCreateTexture;
// }
// errdefer gl.deleteTextures(1, &name);
// const gl_format: gl.GLenum = switch (texture.header.format) {
// .bc7 => gl.COMPRESSED_RGBA_BPTC_UNORM,
// .bc5 => gl.COMPRESSED_RG_RGTC2,
// .bc6 => gl.COMPRESSED_RGB_BPTC_UNSIGNED_FLOAT,
// };
// gl.textureStorage2D(
// name,
// @intCast(texture.mipLevels()),
// gl_format,
// @intCast(texture.header.padded_width),
// @intCast(texture.header.padded_height),
// );
// checkGLError();
// for (0..texture.mipLevels()) |mip_level| {
// const desc = texture.getMipDesc(mip_level);
// gl.compressedTextureSubImage2D(
// name,
// @intCast(mip_level),
// 0,
// 0,
// @intCast(desc.width),
// @intCast(desc.height),
// gl_format,
// @intCast(texture.data[mip_level].len),
// @ptrCast(texture.data[mip_level].ptr),
// );
// checkGLError();
// }
// const uv_scale = Vec2.new(
// @as(f32, @floatFromInt(texture.header.width)) / @as(f32, @floatFromInt(texture.header.padded_width)),
// @as(f32, @floatFromInt(texture.header.height)) / @as(f32, @floatFromInt(texture.header.padded_height)),
// );
// const handle = gl.GL_ARB_bindless_texture.getTextureHandleARB(name);
// gl.GL_ARB_bindless_texture.makeTextureHandleResidentARB(handle);
// errdefer gl.GL_ARB_bindless_texture.makeTextureHandleNonResidentARB(handle);
}
fn loadScene(self: *AssetManager, id: AssetId) LoadedScene {
@ -828,7 +993,51 @@ fn loadMaterialErr(self: *AssetManager, id: AssetId) !formats.Material {
return material;
}
const LoadedAsset = union(enum) {
fn finishLoadingAsset(self: *AssetManager, id: AssetId) void {
if (self.resolveLoadingAsset(id)) |loading_asset| {
switch (loading_asset.*) {
.texture => |loading_texture| {
{
self.rw_lock.lock();
defer self.rw_lock.unlock();
_ = self.loading_assets.remove(id);
self.loaded_assets.put(self.allocator, id, LoadedAsset{ .texture = .{
.image = loading_texture.image,
.view = loading_texture.view,
.descriptor_handle = loading_texture.descriptor_handle,
} }) catch @panic("OOM");
}
self.gc.device.freeCommandBuffers(self.command_pool.handle, 1, &.{loading_texture.command_buffer});
self.gc.device.destroyBuffer(loading_texture.staging_buf, null);
self.gc.device.freeMemory(loading_texture.staging_memory, null);
self.gc.device.destroyFence(loading_texture.transfer_fence, null);
},
else => unreachable,
}
}
}
const AssetType = enum {
shader,
shaderProgram,
mesh,
texture,
scene,
material,
};
const LoadingAsset = union(AssetType) {
shader: void,
shaderProgram: void,
mesh: void,
texture: LoadingTexture,
scene: void,
material: void,
};
const LoadedAsset = union(AssetType) {
shader: LoadedShader,
shaderProgram: LoadedShaderProgram,
mesh: LoadedMesh,
@ -857,10 +1066,21 @@ pub const LoadedMesh = struct {
material: formats.Material,
};
const LoadingTexture = struct {
image: vk.Image,
view: vk.ImageView,
transfer_fence: vk.Fence,
staging_buf: vk.Buffer,
staging_memory: vk.DeviceMemory,
descriptor_handle: DescriptorManager.DescriptorHandle,
// Upload texture command buffer
command_buffer: vk.CommandBuffer,
};
const LoadedTexture = struct {
name: gl.GLuint,
handle: gl.GLuint64,
uv_scale: Vec2 = Vec2.one(),
image: vk.Image,
view: vk.ImageView,
descriptor_handle: DescriptorManager.DescriptorHandle = .{},
};
const LoadedScene = struct {
@ -1670,8 +1890,9 @@ fn freeAsset(self: *AssetManager, asset: *LoadedAsset) void {
self.gc.device.destroyPipeline(program.pipeline, null);
},
.texture => |*texture| {
gl.GL_ARB_bindless_texture.makeTextureHandleNonResidentARB(texture.handle);
gl.deleteTextures(1, &texture.name);
_ = texture; // autofix
// gl.GL_ARB_bindless_texture.makeTextureHandleNonResidentARB(texture.handle);
// gl.deleteTextures(1, &texture.name);
},
.scene => |*scene| {
self.allocator.free(scene.buf);

2
src/Render2.zig
View File

@ -629,7 +629,7 @@ pub fn draw(self: *Render2) !void {
cmds.bindDescriptorSets(.graphics, self.descriptorman.pipeline_layout, 0, 1, &.{global_descriptor_set}, 0, null);
self.pushConstants(cmds, .{ .vertex_bit = true, .fragment_bit = true }, PostProcessPushConstants{
.scene_color_texture = main_render_target.color_descriptor.index(),
.scene_color_texture = self.assetman.resolveTexture(a.Textures.bunny_tex1).descriptor_handle.index,
.scene_color_sampler = self.screen_color_sampler_descriptor_handle.index,
});

0
src/ScratchAllocator.zig Normal file
View File