Cleanup, start replacing basisu with ispc texture compressor

ISPC texcomp is so, so much faster and supports bc6 (hdr) as well as bc5 and others down to bc1.

.gitattributes

@@ -9,6 +9,7 @@
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.tiff filter=lfs diff=lfs merge=lfs -text
+*.exr filter=lfs diff=lfs merge=lfs -text
 *.bmp filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
 *.blend filter=lfs diff=lfs merge=lfs -text

build.zig

@@ -38,12 +38,15 @@ pub fn build(b: *Build) void {
     const assets_step = b.step("assets", "Build and install assets");
     b.getInstallStep().dependOn(assets_step);
 
-    const assetc = buildAssetCompiler(b, basisu_optimize, assets_step, buildOptimize);
+    const assetc = buildAssetCompiler(b, buildOptimize);
+
+    const install_assetc_step = b.addInstallArtifact(assetc, .{ .dest_dir = .{ .override = .prefix } });
+    assets_step.dependOn(&install_assetc_step.step);
 
     assetc.root_module.addImport("assets", assets_mod);
     assetc.root_module.addImport("asset_manifest", asset_manifest_mod);
 
-    const gen_asset_manifest = buildAssets(b, assets_step, assetc, "assets") catch |err| {
+    const gen_asset_manifest = buildAssets(b, &install_assetc_step.step, assets_step, assetc, "assets") catch |err| {
         std.log.err("Failed to build assets {}\n", .{err});
         @panic("buildAssets");
     };
@@ -185,7 +188,7 @@ const NestedAssetDef = union(enum) {
 };
 
 // Find all assets and cook them using assetc
-fn buildAssets(b: *std.Build, step: *Step, assetc: *Step.Compile, path: []const u8) !Build.LazyPath {
+fn buildAssets(b: *std.Build, install_assetc_step: *Step, step: *Step, assetc: *Step.Compile, path: []const u8) !Build.LazyPath {
     const assetsPath = b.pathFromRoot(path);
     defer b.allocator.free(assetsPath);
     var assetsDir = try std.fs.openDirAbsolute(assetsPath, .{ .iterate = true });
@@ -204,6 +207,9 @@ fn buildAssets(b: *std.Build, step: *Step, assetc: *Step.Compile, path: []const
     while (try walker.next()) |entry| {
         if (std.mem.endsWith(u8, entry.basename, ".obj")) {
             const run_assetc = b.addRunArtifact(assetc);
+            run_assetc.step.dependOn(install_assetc_step);
+            run_assetc.addPathDir(b.pathFromRoot("libs/ispc_texcomp/lib"));
+
             run_assetc.addFileArg(.{ .path = b.pathJoin(&.{ path, entry.path }) });
             const out_name = try std.mem.concat(
                 b.allocator,
@@ -251,6 +257,9 @@ fn buildAssets(b: *std.Build, step: *Step, assetc: *Step.Compile, path: []const
         // Shader program
         if (std.mem.endsWith(u8, entry.basename, ".prog")) {
             const run_assetc = b.addRunArtifact(assetc);
+            run_assetc.step.dependOn(install_assetc_step);
+            run_assetc.addPathDir(b.pathFromRoot("libs/ispc_texcomp/lib"));
+
             run_assetc.addFileArg(.{ .path = b.pathJoin(&.{ path, entry.path }) });
             const compiled_file = run_assetc.addOutputFileArg(b.dupe(entry.basename));
 
@@ -273,13 +282,16 @@ fn buildAssets(b: *std.Build, step: *Step, assetc: *Step.Compile, path: []const
             }
         }
 
-        if (std.mem.endsWith(u8, entry.basename, ".png") or std.mem.endsWith(u8, entry.basename, ".jpg")) {
+        if (std.mem.endsWith(u8, entry.basename, ".png") or std.mem.endsWith(u8, entry.basename, ".jpg") or std.mem.endsWith(u8, entry.basename, ".exr")) {
             const run_assetc = b.addRunArtifact(assetc);
+            run_assetc.step.dependOn(install_assetc_step);
+            run_assetc.addPathDir(b.pathFromRoot("libs/ispc_texcomp/lib"));
+
             run_assetc.addFileArg(.{ .path = b.pathJoin(&.{ path, entry.path }) });
             const out_name = try std.mem.concat(
                 b.allocator,
                 u8,
-                &.{ std.fs.path.stem(entry.basename), ".bu" }, // basisu
+                &.{ std.fs.path.stem(entry.basename), ".tex" }, // basisu
             );
             const compiled_file = run_assetc.addOutputFileArg(out_name);
 
@@ -371,7 +383,7 @@ fn writeAssetManifest(
     return manifest_path;
 }
 
-fn buildAssetCompiler(b: *Build, basisu_optimize: std.builtin.OptimizeMode, assets_step: *Step, optimize: std.builtin.OptimizeMode) *Step.Compile {
+fn buildAssetCompiler(b: *Build, optimize: std.builtin.OptimizeMode) *Step.Compile {
     const assimp_dep = b.dependency("zig-assimp", .{
         .target = b.host,
         .optimize = optimize,
@@ -379,13 +391,7 @@ fn buildAssetCompiler(b: *Build, basisu_optimize: std.builtin.OptimizeMode, asse
         .formats = @as([]const u8, "Obj"),
     });
 
-    const basisu_dep = b.dependency("mach-basisu", .{
-        .target = b.host,
-        .optimize = basisu_optimize,
-    });
-
     const assimp_lib = assimp_dep.artifact("assimp");
-    const basisu_lib = basisu_dep.artifact("mach-basisu");
 
     const assetc = b.addExecutable(.{
         .name = "assetc",
@@ -393,19 +399,22 @@ fn buildAssetCompiler(b: *Build, basisu_optimize: std.builtin.OptimizeMode, asse
         .root_source_file = .{ .path = "tools/asset_compiler.zig" },
         .optimize = optimize,
     });
+    assetc.linkLibC();
+
+    b.installFile("libs/ispc_texcomp/lib/ispc_texcomp.dll", "ispc_texcomp.dll");
+    b.installFile("libs/ispc_texcomp/lib/ispc_texcomp.pdb", "ispc_texcomp.pdb");
+    assetc.addLibraryPath(.{ .path = "libs/ispc_texcomp/lib" });
+    assetc.addIncludePath(.{ .path = "libs/ispc_texcomp/include" });
+    assetc.linkSystemLibrary("ispc_texcomp");
 
     assetc.root_module.addAnonymousImport("formats", .{ .root_source_file = .{ .path = "src/formats.zig" } });
-    assetc.root_module.addImport("mach-basisu", basisu_dep.module("mach-basisu"));
 
     assetc.linkLibrary(assimp_lib);
-    assetc.linkLibrary(basisu_lib);
     assetc.linkLibC();
     assetc.linkLibCpp();
 
-    assetc.addCSourceFile(.{ .file = .{ .path = "tools/stb/stb_image.c" }, .flags = &.{"-std=c99"} });
-    assetc.addIncludePath(.{ .path = "tools/stb" });
+    assetc.addCSourceFile(.{ .file = .{ .path = "libs/stb/stb_image.c" }, .flags = &.{"-std=c99"} });
+    assetc.addIncludePath(.{ .path = "libs/stb" });
 
-    const install_step = b.addInstallArtifact(assetc, .{ .dest_dir = .{ .override = .prefix } });
-    assets_step.dependOn(&install_step.step);
     return assetc;
 }

libs/ispc_texcomp/include/ispc_texcomp.h

@@ -0,0 +1,126 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-2019, Intel Corporation
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+////////////////////////////////////////////////////////////////////////////////
+
+#include <stdint.h>
+#include <stdbool.h>
+
+struct rgba_surface
+{
+    uint8_t* ptr;
+    int32_t width;
+    int32_t height;
+    int32_t stride; // in bytes
+};
+
+struct bc7_enc_settings
+{
+    bool mode_selection[4];
+    int refineIterations[8];
+
+    bool skip_mode2;
+    int fastSkipTreshold_mode1;
+    int fastSkipTreshold_mode3;
+    int fastSkipTreshold_mode7;
+
+    int mode45_channel0;
+    int refineIterations_channel;
+
+    int channels;
+};
+
+struct bc6h_enc_settings
+{
+    bool slow_mode;
+    bool fast_mode;
+    int refineIterations_1p;
+    int refineIterations_2p;
+    int fastSkipTreshold;
+};
+
+struct etc_enc_settings
+{
+    int fastSkipTreshold;
+};
+
+struct astc_enc_settings
+{
+    int block_width;
+    int block_height;
+    int channels;
+
+    int fastSkipTreshold;
+    int refineIterations;
+};
+
+// profiles for RGB data (alpha channel will be ignored)
+void GetProfile_ultrafast(struct bc7_enc_settings* settings);
+void GetProfile_veryfast(struct bc7_enc_settings* settings);
+void GetProfile_fast(struct bc7_enc_settings* settings);
+void GetProfile_basic(struct bc7_enc_settings* settings);
+void GetProfile_slow(struct bc7_enc_settings* settings);
+
+// profiles for RGBA inputs
+void GetProfile_alpha_ultrafast(struct bc7_enc_settings* settings);
+void GetProfile_alpha_veryfast(struct bc7_enc_settings* settings);
+void GetProfile_alpha_fast(struct bc7_enc_settings* settings);
+void GetProfile_alpha_basic(struct bc7_enc_settings* settings);
+void GetProfile_alpha_slow(struct bc7_enc_settings* settings);
+
+// profiles for BC6H (RGB HDR)
+void GetProfile_bc6h_veryfast(struct bc6h_enc_settings* settings);
+void GetProfile_bc6h_fast(struct bc6h_enc_settings* settings);
+void GetProfile_bc6h_basic(struct bc6h_enc_settings* settings);
+void GetProfile_bc6h_slow(struct bc6h_enc_settings* settings);
+void GetProfile_bc6h_veryslow(struct bc6h_enc_settings* settings);
+
+// profiles for ETC
+void GetProfile_etc_slow(struct etc_enc_settings* settings);
+
+// profiles for ASTC
+void GetProfile_astc_fast(struct astc_enc_settings* settings, int block_width, int block_height);
+void GetProfile_astc_alpha_fast(struct astc_enc_settings* settings, int block_width, int block_height);
+void GetProfile_astc_alpha_slow(struct astc_enc_settings* settings, int block_width, int block_height);
+
+// helper function to replicate border pixels for the desired block sizes (bpp = 32 or 64)
+void ReplicateBorders(struct rgba_surface* dst_slice, const struct rgba_surface* src_tex, int x, int y, int bpp);
+
+/*
+Notes:
+    - input width and height need to be a multiple of block size
+    - LDR input is 32 bit/pixel (sRGB), HDR is 64 bit/pixel (half float)
+        - for BC4 input is 8bit/pixel (R8), for BC5 input is 16bit/pixel (RG8)
+    - dst buffer must be allocated with enough space for the compressed texture:
+        - 8 bytes/block for BC1/BC4/ETC1,
+        - 16 bytes/block for BC3/BC5/BC6H/BC7/ASTC
+    - the blocks are stored in raster scan order (natural CPU texture layout)
+    - use the GetProfile_* functions to select various speed/quality tradeoffs
+    - the RGB profiles are slightly faster as they ignore the alpha channel
+*/
+
+void CompressBlocksBC1(const struct rgba_surface* src, uint8_t* dst);
+void CompressBlocksBC3(const struct rgba_surface* src, uint8_t* dst);
+void CompressBlocksBC4(const struct rgba_surface* src, uint8_t* dst);
+void CompressBlocksBC5(const struct rgba_surface* src, uint8_t* dst);
+void CompressBlocksBC6H(const struct rgba_surface* src, uint8_t* dst, struct bc6h_enc_settings* settings);
+void CompressBlocksBC7(const struct rgba_surface* src, uint8_t* dst, struct bc7_enc_settings* settings);
+void CompressBlocksETC1(const struct rgba_surface* src, uint8_t* dst, struct etc_enc_settings* settings);
+void CompressBlocksASTC(const struct rgba_surface* src, uint8_t* dst, struct astc_enc_settings* settings);

src/AssetManager.zig

@@ -247,7 +247,6 @@ const NullMesh = LoadedMesh{
     },
 };
 
-// TODO: create empty texture instead, this will crash
 const NullTexture = LoadedTexture{
     .name = 0,
     .handle = 0,

src/formats.zig

@@ -158,3 +158,64 @@ fn writeFloat(writer: anytype, value: f32, endian: std.builtin.Endian) !void {
     const val: u32 = @bitCast(value);
     try writer.writeInt(u32, val, endian);
 }
+
+pub const Texture = struct {
+    pub const Format = enum(u32) {
+        bc5, // uncorrelated 2 channel, used for normal maps
+        bc6, // f16 for hdr textures
+        bc7, // normal rgba textures, assumed linear colors
+        bc7_srgb, // normal rgba textures, assumed srgb
+    };
+
+    pub const MAGIC = [_]u8{ 'T', 'X', 'F', 'M' };
+
+    pub const Header = extern struct {
+        magic: [4]u8 = MAGIC,
+        format: Format,
+        mip_levels: u32,
+        width: u32,
+        height: u32,
+    };
+
+    header: Header,
+    data: []u8,
+
+    pub fn fromBuffer(buf: []u8) !Texture {
+        const header: *align(1) Header = @ptrCast(buf.ptr);
+
+        if (!std.mem.eql(u8, &header.magic, &MAGIC)) {
+            return error.MagicMatch;
+        }
+
+        return Texture{
+            .header = header.*,
+            .data = buf[@sizeOf(Header)..],
+        };
+    }
+};
+
+// TODO: this doesn't respect endiannes at all
+pub fn writeTexture(writer: anytype, value: Texture) !void {
+    try writer.writeStruct(value.header);
+    try writer.writeAll(value.data);
+}
+
+test "texture write/parse" {
+    var data = [_]u8{ 'h', 'e', 'l', 'l', 'o' };
+    const source = Texture{
+        .header = .{
+            .format = .bc7_srgb,
+            .width = 123,
+            .height = 234,
+            .mip_levels = 1,
+        },
+        .data = &data,
+    };
+
+    var buf: [@sizeOf(Texture.Header) + data.len]u8 = undefined;
+    var stream = std.io.fixedBufferStream(&buf);
+    try writeTexture(stream.writer(), source);
+
+    const decoded = try Texture.fromBuffer(&buf);
+    try std.testing.expectEqualDeep(source, decoded);
+}

src/game.zig

@@ -305,9 +305,6 @@ export fn game_update() bool {
     // TODO: make this an entity
     gmem.free_cam.update(gmem.delta_time, move, look.scale(0.008));
 
-    // RENDER
-    // gl.fenceSync(_condition: GLenum, _flags: GLbitfield)
-
     const f_width: f32 = @floatFromInt(ginit.width);
     const f_height: f32 = @floatFromInt(ginit.height);
 
@@ -453,7 +450,4 @@ fn toggleFullScreen() !void {
     }
 
     ginit.fullscreen = !ginit.fullscreen;
-
-    // c.SDL_GL_GetDrawableSize(ginit.window, &ginit.width, &ginit.height);
-    // gl.viewport(0, 0, ginit.width, ginit.height);
 }

tools/asset_compiler.zig

@@ -10,8 +10,9 @@ const c = @cImport({
     @cInclude("assimp/postprocess.h");
 
     @cInclude("stb_image.h");
+
+    @cInclude("ispc_texcomp.h");
 });
-const basisu = @import("mach-basisu");
 
 const ASSET_MAX_BYTES = 1024 * 1024 * 1024;
 
@@ -20,6 +21,7 @@ const AssetType = enum {
     Shader,
     ShaderProgram,
     Texture,
+    HDRTexture,
 };
 
 pub fn resolveAssetTypeByExtension(path: []const u8) ?AssetType {
@@ -35,6 +37,9 @@ pub fn resolveAssetTypeByExtension(path: []const u8) ?AssetType {
     if (std.mem.endsWith(u8, path, ".png") or std.mem.endsWith(u8, path, ".jpg")) {
         return .Texture;
     }
+    if (std.mem.endsWith(u8, path, ".exr")) {
+        return .HDRTexture;
+    }
     return null;
 }
 
@@ -54,7 +59,8 @@ pub fn main() !void {
     switch (asset_type) {
         .Mesh => try processMesh(allocator, input, output),
         .ShaderProgram => try processShaderProgram(allocator, std.mem.span(input), output),
-        .Texture => try processTexture(allocator, input, output),
+        .Texture => try processTexture(allocator, input, output, false),
+        .HDRTexture => try processTexture(allocator, input, output, true),
         else => return error.CantProcessAssetType,
     }
 }
@@ -187,42 +193,67 @@ fn processShaderProgram(allocator: std.mem.Allocator, absolute_input: []const u8
     try buf_writer.flush();
 }
 
-fn processTexture(allocator: std.mem.Allocator, input: [*:0]const u8, output: []const u8) !void {
-    _ = allocator; // autofix
-    var x: c_int = undefined;
-    var y: c_int = undefined;
+fn processTexture(allocator: std.mem.Allocator, input: [*:0]const u8, output: []const u8, hdr: bool) !void {
+    _ = hdr; // autofix
+    var width_int: c_int = undefined;
+    var height_int: c_int = undefined;
     var comps: c_int = undefined;
 
     c.stbi_set_flip_vertically_on_load(1);
-    const FORCED_COMPONENTS = 3; // force rgb
-    const data_c = @as(?[*]u8, @ptrCast(c.stbi_load(input, &x, &y, &comps, FORCED_COMPONENTS))) orelse return error.ImageLoadError;
+    // const FORCED_COMPONENTS = 3; // force rgb
+    const data_c = c.stbi_load(input, &width_int, &height_int, &comps, 0);
+    if (data_c == null) {
+        return error.ImageLoadError;
+    }
     defer c.stbi_image_free(data_c);
 
-    const data = data_c[0 .. @as(usize, @intCast(x)) * @as(usize, @intCast(y)) * FORCED_COMPONENTS];
+    const width: usize = @intCast(width_int);
+    const height: usize = @intCast(height_int);
 
-    basisu.init_encoder();
+    // TODO: support textures not divisible by 4
+    if (width % 4 != 0 or height % 4 != 0) {
+        std.log.debug("Image size: {}X{}\n", .{ width, height });
+        return error.ImageSizeShouldBeDivisibleBy4;
+    }
 
-    var params = basisu.CompressorParams.init(@intCast(try std.Thread.getCpuCount()));
-    defer params.deinit();
+    const blocks_x: usize = width / 4;
+    const blocks_y: usize = height / 4;
 
-    const img = params.getImageSource(0);
-    img.fill(data, @intCast(x), @intCast(y), @intCast(FORCED_COMPONENTS));
+    const rgba_surf = c.rgba_surface{
+        .ptr = data_c,
+        .width = @intCast(blocks_x),
+        .height = @intCast(blocks_y),
+        .stride = width_int * comps,
+    };
+    var settings: c.bc7_enc_settings = undefined;
 
-    // TODO: configure per-texture somehow
-    params.setQualityLevel(64);
-    params.setBasisFormat(basisu.BasisTextureFormat.uastc4x4);
-    params.setColorSpace(basisu.ColorSpace.srgb);
-    params.setGenerateMipMaps(true);
+    if (comps == 3) {
+        c.GetProfile_fast(&settings);
+    } else if (comps == 4) {
+        c.GetProfile_alpha_fast(&settings);
+    } else {
+        std.log.debug("Channel count: {}\n", .{comps});
+        return error.UnsupportedChannelCount;
+    }
 
-    var compressor = try basisu.Compressor.init(params);
-    defer compressor.deinit();
+    const out_data = try allocator.alignedAlloc(u8, 16, blocks_x * blocks_y * 16);
 
-    try compressor.process();
+    c.CompressBlocksBC7(&rgba_surf, out_data.ptr, &settings);
+
+    const texture = formats.Texture{
+        .header = .{
+            .format = .bc7_srgb,
+            .width = @intCast(width),
+            .height = @intCast(height),
+            .mip_levels = 1,
+        },
+        .data = out_data,
+    };
 
     const out_file = try std.fs.createFileAbsolute(output, .{});
     defer out_file.close();
     var buf_writer = std.io.bufferedWriter(out_file.writer());
 
-    try buf_writer.writer().writeAll(compressor.output());
+    try formats.writeTexture(buf_writer.writer(), texture);
     try buf_writer.flush();
 }