<commit_before>const Blake3 = @import("crypto.zig").Blake3;
const fs = @import("fs.zig");
const base64 = @import("base64.zig");
const ArrayList = @import("array_list.zig").ArrayList;
const debug = @import("debug.zig");
const testing = @import("testing.zig");
const mem = @import("mem.zig");
const fmt = @import("fmt.zig");
const Allocator = mem.Allocator;
const Buffer = @import("buffer.zig").Buffer;
const os = @import("os.zig");

const base64_encoder = fs.base64_encoder;
const base64_decoder = fs.base64_decoder;
const BIN_DIGEST_LEN = 48;
const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);

pub const File = struct {
    path: ?[]const u8,
    stat: fs.File.Stat,
    file_handle: os.fd_t,
    bin_digest: [BIN_DIGEST_LEN]u8,
    contents: ?[]const u8,

    pub fn deinit(self: *@This(), alloc: *Allocator) void {
        if (self.path) |owned_slice| {
            alloc.free(owned_slice);
            self.path = null;
        }
        if (self.contents) |owned_slice| {
            alloc.free(owned_slice);
            self.contents = null;
        }
    }
};

pub const CacheHash = struct {
    alloc: *Allocator,
    blake3: Blake3,
    manifest_dir: fs.Dir,
    manifest_file: ?fs.File,
    manifest_dirty: bool,
    force_check_manifest: bool,
    files: ArrayList(File),
    b64_digest: [BASE64_DIGEST_LEN]u8,

    pub fn init(alloc: *Allocator, manifest_dir_path: []const u8) !@This() {
        try fs.cwd().makePath(manifest_dir_path);
        const manifest_dir = try fs.cwd().openDirTraverse(manifest_dir_path);

        return CacheHash{
            .alloc = alloc,
            .blake3 = Blake3.init(),
            .manifest_dir = manifest_dir,
            .manifest_file = null,
            .manifest_dirty = false,
            .force_check_manifest = false,
            .files = ArrayList(File).init(alloc),
            .b64_digest = undefined,
        };
    }

    pub fn addSlice(self: *@This(), val: []const u8) void {
        debug.assert(self.manifest_file == null);

        self.blake3.update(val);
        self.blake3.update(&[_]u8{0});
    }

    pub fn addBool(self: *@This(), val: bool) void {
        debug.assert(self.manifest_file == null);
        self.blake3.update(&[_]u8{@boolToInt(val)});
    }

    pub fn addInt(self: *@This(), val: var) void {
        debug.assert(self.manifest_file == null);

        switch (@typeInfo(@TypeOf(val))) {
            .Int => |int_info| {
                if (int_info.bits == 0 or int_info.bits % 8 != 0) {
                    @compileError("Unsupported integer size. Please use a multiple of 8, manually convert to a u8 slice.");
                }

                const buf_len = @divExact(int_info.bits, 8);
                var buf: [buf_len]u8 = undefined;
                mem.writeIntNative(@TypeOf(val), &buf, val);
                self.addSlice(&buf);

                self.blake3.update(&[_]u8{0});
            },
            else => @compileError("Type must be an integer."),
        }
    }

    pub fn add(self: *@This(), val: var) void {
        debug.assert(self.manifest_file == null);

        const val_type = @TypeOf(val);
        switch (@typeInfo(val_type)) {
            .Int => self.addInt(val),
            .Bool => self.addBool(val),
            .Array => |array_info| if (array_info.child == u8) {
                self.addSlice(val[0..]);
            } else {
                @compileError("Unsupported array type");
            },
            .Pointer => |ptr_info| switch (ptr_info.size) {
                .Slice => if (ptr_info.child == u8) {
                    self.addSlice(val);
                },
                .One => self.add(val.*),
                else => {
                    @compileLog("Pointer type: ", ptr_info.size, ptr_info.child);
                    @compileError("Unsupported pointer type");
                },
            },
            else => @compileError("Unsupported type"),
        }
    }

    pub fn addFile(self: *@This(), file_path: []const u8) !void {
        debug.assert(self.manifest_file == null);

        var cache_hash_file = try self.files.addOne();
        cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});

        self.addSlice(cache_hash_file.path.?);
    }

    pub fn hit(self: *@This()) !?[BASE64_DIGEST_LEN]u8 {
        debug.assert(self.manifest_file == null);

        var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
        self.blake3.final(&bin_digest);

        base64_encoder.encode(self.b64_digest[0..], &bin_digest);

        if (self.files.toSlice().len == 0 and !self.force_check_manifest) {
            return self.b64_digest;
        }

        self.blake3 = Blake3.init();
        self.blake3.update(&bin_digest);

        {
            const manifest_file_path = try fmt.allocPrint(self.alloc, "{}.txt", .{self.b64_digest});
            defer self.alloc.free(manifest_file_path);

            self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{ .read = true, .truncate = false });
        }

        // create a buffer instead of using readAllAlloc
        // See: https://github.com/ziglang/zig/issues/4656
        var file_buffer = try Buffer.initCapacity(self.alloc, 16 * 1024);
        defer file_buffer.deinit();

        // TODO: Figure out a good max value?
        try self.manifest_file.?.inStream().stream.readAllBuffer(&file_buffer, 16 * 1024);
        const file_contents = file_buffer.toSliceConst();

        const input_file_count = self.files.len;
        var any_file_changed = false;
        var line_iter = mem.tokenize(file_contents, "\n");
        var idx: usize = 0;
        while (line_iter.next()) |line| {
            defer idx += 1;

            var cache_hash_file: *File = undefined;
            if (idx < input_file_count) {
                cache_hash_file = self.files.ptrAt(idx);
            } else {
                cache_hash_file = try self.files.addOne();
                cache_hash_file.path = null;
            }

            var iter = mem.tokenize(line, " ");
            const file_handle_str = iter.next() orelse return error.InvalidFormat;
            const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
            const digest_str = iter.next() orelse return error.InvalidFormat;<commit_msg>Use `readAllAlloc`<commit_after>const Blake3 = @import("crypto.zig").Blake3;
const fs = @import("fs.zig");
const base64 = @import("base64.zig");
const ArrayList = @import("array_list.zig").ArrayList;
const debug = @import("debug.zig");
const testing = @import("testing.zig");
const mem = @import("mem.zig");
const fmt = @import("fmt.zig");
const Allocator = mem.Allocator;
const Buffer = @import("buffer.zig").Buffer;
const os = @import("os.zig");

const base64_encoder = fs.base64_encoder;
const base64_decoder = fs.base64_decoder;
const BIN_DIGEST_LEN = 48;
const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);

pub const File = struct {
    path: ?[]const u8,
    stat: fs.File.Stat,
    file_handle: os.fd_t,
    bin_digest: [BIN_DIGEST_LEN]u8,
    contents: ?[]const u8,

    pub fn deinit(self: *@This(), alloc: *Allocator) void {
        if (self.path) |owned_slice| {
            alloc.free(owned_slice);
            self.path = null;
        }
        if (self.contents) |owned_slice| {
            alloc.free(owned_slice);
            self.contents = null;
        }
    }
};

pub const CacheHash = struct {
    alloc: *Allocator,
    blake3: Blake3,
    manifest_dir: fs.Dir,
    manifest_file: ?fs.File,
    manifest_dirty: bool,
    force_check_manifest: bool,
    files: ArrayList(File),
    b64_digest: [BASE64_DIGEST_LEN]u8,

    pub fn init(alloc: *Allocator, manifest_dir_path: []const u8) !@This() {
        try fs.cwd().makePath(manifest_dir_path);
        const manifest_dir = try fs.cwd().openDirTraverse(manifest_dir_path);

        return CacheHash{
            .alloc = alloc,
            .blake3 = Blake3.init(),
            .manifest_dir = manifest_dir,
            .manifest_file = null,
            .manifest_dirty = false,
            .force_check_manifest = false,
            .files = ArrayList(File).init(alloc),
            .b64_digest = undefined,
        };
    }

    pub fn addSlice(self: *@This(), val: []const u8) void {
        debug.assert(self.manifest_file == null);

        self.blake3.update(val);
        self.blake3.update(&[_]u8{0});
    }

    pub fn addBool(self: *@This(), val: bool) void {
        debug.assert(self.manifest_file == null);
        self.blake3.update(&[_]u8{@boolToInt(val)});
    }

    pub fn addInt(self: *@This(), val: var) void {
        debug.assert(self.manifest_file == null);

        switch (@typeInfo(@TypeOf(val))) {
            .Int => |int_info| {
                if (int_info.bits == 0 or int_info.bits % 8 != 0) {
                    @compileError("Unsupported integer size. Please use a multiple of 8, manually convert to a u8 slice.");
                }

                const buf_len = @divExact(int_info.bits, 8);
                var buf: [buf_len]u8 = undefined;
                mem.writeIntNative(@TypeOf(val), &buf, val);
                self.addSlice(&buf);

                self.blake3.update(&[_]u8{0});
            },
            else => @compileError("Type must be an integer."),
        }
    }

    pub fn add(self: *@This(), val: var) void {
        debug.assert(self.manifest_file == null);

        const val_type = @TypeOf(val);
        switch (@typeInfo(val_type)) {
            .Int => self.addInt(val),
            .Bool => self.addBool(val),
            .Array => |array_info| if (array_info.child == u8) {
                self.addSlice(val[0..]);
            } else {
                @compileError("Unsupported array type");
            },
            .Pointer => |ptr_info| switch (ptr_info.size) {
                .Slice => if (ptr_info.child == u8) {
                    self.addSlice(val);
                },
                .One => self.add(val.*),
                else => {
                    @compileLog("Pointer type: ", ptr_info.size, ptr_info.child);
                    @compileError("Unsupported pointer type");
                },
            },
            else => @compileError("Unsupported type"),
        }
    }

    pub fn addFile(self: *@This(), file_path: []const u8) !void {
        debug.assert(self.manifest_file == null);

        var cache_hash_file = try self.files.addOne();
        cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});

        self.addSlice(cache_hash_file.path.?);
    }

    pub fn hit(self: *@This()) !?[BASE64_DIGEST_LEN]u8 {
        debug.assert(self.manifest_file == null);

        var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
        self.blake3.final(&bin_digest);

        base64_encoder.encode(self.b64_digest[0..], &bin_digest);

        if (self.files.toSlice().len == 0 and !self.force_check_manifest) {
            return self.b64_digest;
        }

        self.blake3 = Blake3.init();
        self.blake3.update(&bin_digest);

        {
            const manifest_file_path = try fmt.allocPrint(self.alloc, "{}.txt", .{self.b64_digest});
            defer self.alloc.free(manifest_file_path);

            self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{ .read = true, .truncate = false });
        }

        // TODO: Figure out a good max value?
        const file_contents = try self.manifest_file.?.inStream().stream.readAllAlloc(self.alloc, 16 * 1024);
        defer self.alloc.free(file_contents);

        const input_file_count = self.files.len;
        var any_file_changed = false;
        var line_iter = mem.tokenize(file_contents, "\n");
        var idx: usize = 0;
        while (line_iter.next()) |line| {
            defer idx += 1;

            var cache_hash_file: *File = undefined;
            if (idx < input_file_count) {
                cache_hash_file = self.files.ptrAt(idx);
            } else {
                cache_hash_file = try self.files.addOne();
                cache_hash_file.path = null;
            }

            var iter = mem.tokenize(line, " ");
            const file_handle_str = iter.next() orelse return error.InvalidFormat;
            const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
            const digest_str = iter.next() orelse return error.InvalidFormat;
{'fraction_non_alphanumeric': 0.13566986998304126, 'fraction_numerical': 0.014536057498182993, 'mean_word_length': 2.143945163747144, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2188801', 'n_tokens_mistral': 4073, 'n_tokens_neox': 3826, 'n_words': 792}