<commit_before>'use strict';

module.exports = Bloom;

function Bloom () {

};

Bloom.prototype.add = function () {

};

Bloom.prototype.exist = function () {

};









<commit_msg>finish bloom filter.<commit_after>'use strict';

var crypto = require('crypto');

module.exports = Bloom;

function Bloom () {
    this.buckets = [];
};

Bloom.prototype.add = function (data) {
    var md5Value, sha1Value;
    md5Value = md5(data);
    fillBucket(md5Value, this.buckets);
    sha1Value = sha1(data);
    fillBucket(sha1Value, this.buckets);
};

Bloom.prototype.exist = function (data) {
    var md5Value, sha1Value;
    md5Value = md5(data);
    if(!visited(md5Value, this.buckets))
        return false;
    sha1Value = sha1(data);
    if(!visited(sha1Value, this.buckets))
        return false;
    return true;
};

function visited (hashedValue, buckets) {
    var bucketIndex, positionIndex;
    bucketIndex = getBucketIndex(hashedValue);
    positionIndex = getPositionIndex(hashedValue);
    return ( buckets[bucketIndex] & ( 1 << positionIndex ) );
};

function fillBucket (hashedValue, buckets) {
    var bucketIndex, positionIndex;
    bucketIndex = getBucketIndex(hashedValue);
    positionIndex = getPositionIndex(hashedValue);
    buckets[bucketIndex] = buckets[bucketIndex] || 0;
    buckets[bucketIndex] = buckets[bucketIndex] | ( 1 << positionIndex );
};

function getBucketIndex (hashedValue) {
    return hashedValue >>> 5; 
};

function getPositionIndex (hashedValue) {
    return hashedValue & 31;
};

function md5 (data) {
    return crypto.createHash('md5').update(data).digest().readUInt16LE(0);
};

function sha1 (data) {
    return crypto.createHash('sha1').update(data).digest().readUInt32LE(0);
};




{'fraction_non_alphanumeric': 0.13657407407407407, 'fraction_numerical': 0.018518518518518517, 'mean_word_length': 4.040816326530612, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5578361', 'n_tokens_mistral': 575, 'n_tokens_neox': 550, 'n_words': 111}