<commit_before>const std = @import("std");
const builtin = @import("builtin");
const assert = std.debug.assert;
const mem = std.mem;
const meta = std.meta;

/// Describes how pointer types should be hashed.
pub const HashStrategy = enum {
    /// Do not follow pointers, only hash their value.
    Shallow,

    /// Follow pointers, hash the pointee content.
    /// Only dereferences one level, ie. it is changed into .Shallow when a
    /// pointer type is encountered.
    Deep,

    /// Follow pointers, hash the pointee content.
    /// Dereferences all pointers encountered.
    /// Assumes no cycle.
    DeepRecursive,
};

/// Helper function to hash a pointer and mutate the strategy if needed.
pub fn hashPointer(hasher: var, key: var, comptime strat: HashStrategy) void {
    const info = @typeInfo(@TypeOf(key));

    switch (info.Pointer.size) {
        .One => switch (strat) {
            .Shallow => hash(hasher, @ptrToInt(key), .Shallow),
            .Deep => hash(hasher, key.*, .Shallow),
            .DeepRecursive => hash(hasher, key.*, .DeepRecursive),
        },

        .Slice => switch (strat) {
            .Shallow => {
                hashPointer(hasher, key.ptr, .Shallow);
                hash(hasher, key.len, .Shallow);
            },
            .Deep => hashArray(hasher, key, .Shallow),
            .DeepRecursive => hashArray(hasher, key, .DeepRecursive),
        },

        .Many,
        .C,
        => switch (strat) {
            .Shallow => hash(hasher, @ptrToInt(key), .Shallow),
            else => @compileError(
                \\ unknown-length pointers and C pointers cannot be hashed deeply.
                \\ Consider providing your own hash function.
            ),
        },
    }
}

/// Helper function to hash a set of contiguous objects, from an array or slice.
pub fn hashArray(hasher: var, key: var, comptime strat: HashStrategy) void {
    switch (strat) {
        .Shallow => {
            // TODO detect via a trait when Key has no padding bits to
            // hash it as an array of bytes.
            // Otherwise, hash every element.
            for (key) |element| {
                hash(hasher, element, .Shallow);
            }
        },
        else => {
            for (key) |element| {
                hash(hasher, element, strat);
            }
        },
    }
}

/// Provides generic hashing for any eligible type.
/// Strategy is provided to determine if pointers should be followed or not.
pub fn hash(hasher: var, key: var, comptime strat: HashStrategy) void {
    const Key = @TypeOf(key);
    switch (@typeInfo(Key)) {
        .NoReturn,
        .Opaque,
        .Undefined,
        .Void,
        .Null,
        .BoundFn,
        .ComptimeFloat,
        .ComptimeInt,
        .Type,
        .EnumLiteral,
        .Frame,
        => @compileError("cannot hash this type"),

        // Help the optimizer see that hashing an int is easy by inlining!
        // TODO Check if the situation is better after #561 is resolved.
        .Int => @call(.{ .modifier = .always_inline }, hasher.update, .{std.mem.asBytes(&key)}),

        .Float => |info| hash(hasher, @bitCast(std.meta.Int(false, info.bits), key), strat),

        .Bool => hash(hasher, @boolToInt(key), strat),
        .Enum => hash(hasher, @enumToInt(key), strat),
        .ErrorSet => hash(hasher, @errorToInt(key), strat),
        .AnyFrame, .Fn => hash(hasher, @ptrToInt(key), strat),

        .Pointer => @call(.{ .modifier = .always_inline }, hashPointer, .{ hasher, key, strat }),

        .Optional => if (key) |k| hash(hasher, k, strat),

        .Array => hashArray(hasher, key, strat),

        .Vector => |info| {
            if (info.child.bit_count % 8 == 0) {
                // If there's no unused bits in the child type, we can just hash
                // this as an array of bytes.
                hasher.update(mem.asBytes(&key));
            } else {
                // Otherwise, hash every element.
                // TODO remove the copy to an array once field access is done.
                const array: [info.len]info.child = key;
                comptime var i = 0;
                inline while (i < info.len) : (i += 1) {
                    hash(hasher, array[i], strat);
                }
            }
        },

        .Struct => |info| {<commit_msg>Optimization of vectors hashing<commit_after>const std = @import("std");
const builtin = @import("builtin");
const assert = std.debug.assert;
const mem = std.mem;
const meta = std.meta;

/// Describes how pointer types should be hashed.
pub const HashStrategy = enum {
    /// Do not follow pointers, only hash their value.
    Shallow,

    /// Follow pointers, hash the pointee content.
    /// Only dereferences one level, ie. it is changed into .Shallow when a
    /// pointer type is encountered.
    Deep,

    /// Follow pointers, hash the pointee content.
    /// Dereferences all pointers encountered.
    /// Assumes no cycle.
    DeepRecursive,
};

/// Helper function to hash a pointer and mutate the strategy if needed.
pub fn hashPointer(hasher: var, key: var, comptime strat: HashStrategy) void {
    const info = @typeInfo(@TypeOf(key));

    switch (info.Pointer.size) {
        .One => switch (strat) {
            .Shallow => hash(hasher, @ptrToInt(key), .Shallow),
            .Deep => hash(hasher, key.*, .Shallow),
            .DeepRecursive => hash(hasher, key.*, .DeepRecursive),
        },

        .Slice => switch (strat) {
            .Shallow => {
                hashPointer(hasher, key.ptr, .Shallow);
                hash(hasher, key.len, .Shallow);
            },
            .Deep => hashArray(hasher, key, .Shallow),
            .DeepRecursive => hashArray(hasher, key, .DeepRecursive),
        },

        .Many,
        .C,
        => switch (strat) {
            .Shallow => hash(hasher, @ptrToInt(key), .Shallow),
            else => @compileError(
                \\ unknown-length pointers and C pointers cannot be hashed deeply.
                \\ Consider providing your own hash function.
            ),
        },
    }
}

/// Helper function to hash a set of contiguous objects, from an array or slice.
pub fn hashArray(hasher: var, key: var, comptime strat: HashStrategy) void {
    switch (strat) {
        .Shallow => {
            // TODO detect via a trait when Key has no padding bits to
            // hash it as an array of bytes.
            // Otherwise, hash every element.
            for (key) |element| {
                hash(hasher, element, .Shallow);
            }
        },
        else => {
            for (key) |element| {
                hash(hasher, element, strat);
            }
        },
    }
}

/// Provides generic hashing for any eligible type.
/// Strategy is provided to determine if pointers should be followed or not.
pub fn hash(hasher: var, key: var, comptime strat: HashStrategy) void {
    const Key = @TypeOf(key);
    switch (@typeInfo(Key)) {
        .NoReturn,
        .Opaque,
        .Undefined,
        .Void,
        .Null,
        .BoundFn,
        .ComptimeFloat,
        .ComptimeInt,
        .Type,
        .EnumLiteral,
        .Frame,
        => @compileError("cannot hash this type"),

        // Help the optimizer see that hashing an int is easy by inlining!
        // TODO Check if the situation is better after #561 is resolved.
        .Int => @call(.{ .modifier = .always_inline }, hasher.update, .{std.mem.asBytes(&key)}),

        .Float => |info| hash(hasher, @bitCast(std.meta.Int(false, info.bits), key), strat),

        .Bool => hash(hasher, @boolToInt(key), strat),
        .Enum => hash(hasher, @enumToInt(key), strat),
        .ErrorSet => hash(hasher, @errorToInt(key), strat),
        .AnyFrame, .Fn => hash(hasher, @ptrToInt(key), strat),

        .Pointer => @call(.{ .modifier = .always_inline }, hashPointer, .{ hasher, key, strat }),

        .Optional => if (key) |k| hash(hasher, k, strat),

        .Array => hashArray(hasher, key, strat),

        .Vector => |info| {
            if (info.child.bit_count % 8 == 0) {
                // If there's no unused bits in the child type, we can just hash
                // this as an array of bytes.
                hasher.update(mem.asBytes(&key));
            } else {
                // Otherwise, hash every element.
                comptime var i = 0;
                inline while (i < info.len) : (i += 1) {
                    hash(hasher, key[i], strat);
                }
            }
        },

        .Struct => |info| {
{'fraction_non_alphanumeric': 0.1279587532224045, 'fraction_numerical': 0.0016404968361846731, 'mean_word_length': 1.7937806873977087, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4409505', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2447, 'n_words': 795}