<commit_before>
import ArrayList

/**
 * Container for key/value entries in the hash table
 */
HashEntry: cover {

    key, value: Pointer
    next: HashEntry*

    init: func@ ~keyVal (=key, =value) {
        next = null
    }

}

nullHashEntry: HashEntry <None, None>
memset(nullHashEntry&, 0, HashEntry size)

stringEquals: func <K> (k1, k2: K) -> Bool {
    assert(K == String)
    k1 as String equals?(k2 as String)
}

cstringEquals: func <K> (k1, k2: K) -> Bool {
    k1 as CString == k2 as CString
}


pointerEquals: func <K> (k1, k2: K) -> Bool {
    k1 as Pointer == k2 as Pointer
}

intEquals: func <K> (k1, k2: K) -> Bool {
    k1 as Int == k2 as Int
}

charEquals: func <K> (k1, k2: K) -> Bool {
    k1 as Char == k2 as Char
}

/** used when we don't have a custom comparing function for the key type */
genericEquals: func <K> (k1, k2: K) -> Bool {
    // FIXME rock should turn == between generic vars into a memcmp itself
    memcmp(k1, k2, K size) == 0
}

intHash: func <K> (key: K) -> SizeT {
    result: SizeT = key as Int
    return result
}

pointerHash: func <K> (key: K) -> SizeT {
    return (key as Pointer) as SizeT
}

charHash: func <K> (key: K) -> SizeT {
    // both casts are necessary
    // Casting 'key' directly to UInt would deref a pointer to UInt
    // which would read random memory just after the char, which is not a good idea..
    return (key as Char) as SizeT
}

/**
   Port of Austin Appleby's Murmur Hash implementation
   http://code.google.com/p/smhasher/

   :param: key The key to hash
   :param: seed The seed value
 */
murmurHash: func <K> (keyTagazok: K) -> SizeT {

    seed: SizeT = 1 // TODO: figure out what makes a good seed value?

    len := K size
    m = 0x5bd1e995 : const SizeT
    r = 24 : const SSizeT
    l := len

    h : SizeT = seed ^ len
    data := (keyTagazok&) as Octet*

    while (true) {
        k := (data as SizeT*)@

        k *= m
        k ^= k >> r
        k *= m

        h *= m
        h ^= k

        data += 4
        if(len < 4) break
        len -= 4
    }

    t := 0

    if(len == 3) h ^= data[2] << 16
    if(len == 2) h ^= data[1] << 8
    if(len == 1) h ^= data[0]

    t *= m; t ^= t >> r; t *= m; h *= m; h ^= t;
    l *= m; l ^= l >> r; l *= m; h *= m; h ^= l;

    h ^= h >> 13
    h *= m
    h ^= h >> 15

    return h
}

/**
 * khash's ac_X31_hash_string
 * http://attractivechaos.awardspace.com/khash.h.html
 * @access private
 * @param s The string to hash
 * @return UInt
 */
ac_X31_hash: func <K> (key: K) -> SizeT {
    assert(key != null)
    s : Char* = (K == String) ? (key as String) toCString() as Char* : key as Char*
    h = s@ : SizeT
    if (h) {
        s += 1
        while (s@) {
            h = (h << 5) - h + s@
            s += 1
        }
    }
    return h
}

/* this function seem is called by List */
getStandardEquals: func <T> (T: Class) -> Func <T> (T, T) -> Bool {
    // choose comparing function for key type
    if(T == String) {
        stringEquals
    } else if(T == CString) {
        cstringEquals
    } else if(T size == Pointer size) {
        pointerEquals
    } else if(T size == UInt size) {
        intEquals
    } else if(T size == Char size) {
        charEquals
    } else {
        genericEquals
    }
}

getStandardHashFunc: func <T> (T: Class) -> Func <T> (T) -> SizeT {
    if(T == String || T == CString) {
        ac_X31_hash
    } else if(T size == Pointer size) {
        pointerHash
    } else if(T size == UInt size) {
        intHash
    } else if(T size == Char size) {
        charHash
    } else {
        murmurHash
    }
}

/**
 * Simple hash table implementation
 */

HashMap: class <K, V> extends BackIterable<V> {

    _size, capacity: SizeT
    keyEquals: Func <K> (K, K) -> Bool
    hashKey: Func <K> (K) -> SizeT

    buckets: HashEntry[]
    keys: ArrayList<K>

    size: SizeT {
    	get {
            _size
 	}
    }

    /**
     * Returns a new hash map
     */

    init: func {
        init(10)
    }

    /**
     * Returns a hash table of a specified bucket capacity.
     * @param UInt capacity The number of buckets to use
     */
    init: func ~withCapacity (=capacity) {
        _size = 0

        buckets = HashEntry[capacity] new()

        keys = ArrayList<K> new(capacity)

        keyEquals = getStandardEquals(K)
        hashKey = getStandardHashFunc(K)

        T = V // workarounds ftw
    }
<commit_msg>Still #746, default size of HashMap to 100, otherwise it won't work..<commit_after>
import ArrayList

/**
 * Container for key/value entries in the hash table
 */
HashEntry: cover {

    key, value: Pointer
    next: HashEntry*

    init: func@ ~keyVal (=key, =value) {
        next = null
    }

}

nullHashEntry: HashEntry <None, None>
memset(nullHashEntry&, 0, HashEntry size)

stringEquals: func <K> (k1, k2: K) -> Bool {
    assert(K == String)
    k1 as String equals?(k2 as String)
}

cstringEquals: func <K> (k1, k2: K) -> Bool {
    k1 as CString == k2 as CString
}


pointerEquals: func <K> (k1, k2: K) -> Bool {
    k1 as Pointer == k2 as Pointer
}

intEquals: func <K> (k1, k2: K) -> Bool {
    k1 as Int == k2 as Int
}

charEquals: func <K> (k1, k2: K) -> Bool {
    k1 as Char == k2 as Char
}

/** used when we don't have a custom comparing function for the key type */
genericEquals: func <K> (k1, k2: K) -> Bool {
    // FIXME rock should turn == between generic vars into a memcmp itself
    memcmp(k1, k2, K size) == 0
}

intHash: func <K> (key: K) -> SizeT {
    result: SizeT = key as Int
    return result
}

pointerHash: func <K> (key: K) -> SizeT {
    return (key as Pointer) as SizeT
}

charHash: func <K> (key: K) -> SizeT {
    // both casts are necessary
    // Casting 'key' directly to UInt would deref a pointer to UInt
    // which would read random memory just after the char, which is not a good idea..
    return (key as Char) as SizeT
}

/**
   Port of <NAME>'s Murmur Hash implementation
   http://code.google.com/p/smhasher/

   :param: key The key to hash
   :param: seed The seed value
 */
murmurHash: func <K> (keyTagazok: K) -> SizeT {

    seed: SizeT = 1 // TODO: figure out what makes a good seed value?

    len := K size
    m = 0x5bd1e995 : const SizeT
    r = 24 : const SSizeT
    l := len

    h : SizeT = seed ^ len
    data := (keyTagazok&) as Octet*

    while (true) {
        k := (data as SizeT*)@

        k *= m
        k ^= k >> r
        k *= m

        h *= m
        h ^= k

        data += 4
        if(len < 4) break
        len -= 4
    }

    t := 0

    if(len == 3) h ^= data[2] << 16
    if(len == 2) h ^= data[1] << 8
    if(len == 1) h ^= data[0]

    t *= m; t ^= t >> r; t *= m; h *= m; h ^= t;
    l *= m; l ^= l >> r; l *= m; h *= m; h ^= l;

    h ^= h >> 13
    h *= m
    h ^= h >> 15

    return h
}

/**
 * khash's ac_X31_hash_string
 * http://attractivechaos.awardspace.com/khash.h.html
 * @access private
 * @param s The string to hash
 * @return UInt
 */
ac_X31_hash: func <K> (key: K) -> SizeT {
    assert(key != null)
    s : Char* = (K == String) ? (key as String) toCString() as Char* : key as Char*
    h = s@ : SizeT
    if (h) {
        s += 1
        while (s@) {
            h = (h << 5) - h + s@
            s += 1
        }
    }
    return h
}

/* this function seem is called by List */
getStandardEquals: func <T> (T: Class) -> Func <T> (T, T) -> Bool {
    // choose comparing function for key type
    if(T == String) {
        stringEquals
    } else if(T == CString) {
        cstringEquals
    } else if(T size == Pointer size) {
        pointerEquals
    } else if(T size == UInt size) {
        intEquals
    } else if(T size == Char size) {
        charEquals
    } else {
        genericEquals
    }
}

getStandardHashFunc: func <T> (T: Class) -> Func <T> (T) -> SizeT {
    if(T == String || T == CString) {
        ac_X31_hash
    } else if(T size == Pointer size) {
        pointerHash
    } else if(T size == UInt size) {
        intHash
    } else if(T size == Char size) {
        charHash
    } else {
        murmurHash
    }
}

/**
 * Simple hash table implementation
 */

HashMap: class <K, V> extends BackIterable<V> {

    _size, capacity: SizeT
    keyEquals: Func <K> (K, K) -> Bool
    hashKey: Func <K> (K) -> SizeT

    buckets: HashEntry[]
    keys: ArrayList<K>

    size: SizeT {
    	get {
            _size
 	}
    }

    /**
     * Returns a new hash map
     */

    init: func {
        init(100)
    }

    /**
     * Returns a hash table of a specified bucket capacity.
     * @param UInt capacity The number of buckets to use
     */
    init: func ~withCapacity (=capacity) {
        _size = 0

        buckets = HashEntry[capacity] new()

        keys = ArrayList<K> new(capacity)

        keyEquals = getStandardEquals(K)
        hashKey = getStandardHashFunc(K)

        T = V // workarounds ftw
    }

{'fraction_non_alphanumeric': 0.1365025466893039, 'fraction_numerical': 0.015280135823429542, 'mean_word_length': 1.9621186724773718, 'pattern_counts': {'":': 0, '<': 62, '<?xml version=': 0, '>': 102, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7425675', 'n_tokens_mistral': 3268, 'n_tokens_neox': 3119, 'n_words': 1104}