<commit_before>/**
    @license
    Copyright (C) 2015 <NAME>
    License: MIT
    See license.txt for full license text.
*/
"use strict";

let toString = Object.prototype.toString;
let log = console.log.bind(console); /* eslint no-unused-vars: 0 */

/**
    Class for containing a max reference counter
    as well as two stacks of references to objects.
    To be used with deepCopy and equals.

    @class Contains two reference stacks as well as a defined max stack depth.
*/
class RecurseCounter {
    constructor(maxStackDepth) {
        this.xStack = [];
        this.yStack = [];
        this.count = 0;
        this.maxStackDepth = maxStackDepth;
    }

    push(x, y) {
        this.xStack.push(x);
        this.yStack.push(y);
        this.count++;
    }

    pop() {
        this.xStack.pop();
        this.yStack.pop();
        this.count--;
    }
}

const toStringTagTemp = Symbol('@@toStringTagTemp'); // make sure this is a symbol, so we don't step on anyone else's toes.
const generatorFnProto = (function*(){}).prototype;

// NOTE: to fix a bug with how babel-runtime works without polyfilling Object.prototype.toString,
// we need to make sure that the following polyfilled items will return the expected string
// with the getType() method.
//
// if we don't do this, then toString.call() on these items will most likely return [object Object],
// which will in turn cause methods like equals() and clone() to handle them incorrectly.
Object.defineProperty(Promise.prototype, toStringTagTemp, {
    get() { return '[object GeneratorFunction]'; },
    enumerable: false
});
Object.defineProperty(Map.prototype, toStringTagTemp, {
    get() { return '[object Map]'; },
    enumerable: false
});
Object.defineProperty(Set.prototype, toStringTagTemp, {
    get() { return '[object Set]'; },
    enumerable: false
});
Object.defineProperty(WeakMap.prototype, toStringTagTemp, {
    get() { return '[object WeakMap]'; },
    enumerable: false
});
Object.defineProperty(WeakSet.prototype, toStringTagTemp, {
    get() { return '[object WeakSet]'; },
    enumerable: false
});
Object.defineProperty(generatorFnProto, toStringTagTemp, { // make sure this doesn't collide with anything else.
    get() { return '[object GeneratorFunction]'; },
    enumerable: false
});

export function getType(val) {
    if (val && val[toStringTagTemp])
        return val[toStringTagTemp];
    return toString.call(val);
}

export var types = {
      'arguments'         : getType(arguments) // will this work? babel may be accidentally saving us here. swap to iife if necessary
    , 'array'             : getType([])
    , 'boolean'           : getType(true)
    // buffer doesn't work, toString.call(Buffer) returns [object Object]
    , 'date'              : getType(new Date())
    , 'error'             : getType(new Error())
    // , 'generator'         : getType(generatorProto)
    , 'generatorFunction' : getType(generatorFnProto)
    , 'function'          : getType(() => {})
    , 'map'               : getType(new Map())
    , 'null'              : getType(null)
    , 'number'            : getType(0)
    , 'object'            : getType({})
    , 'promise'           : getType(new Promise(() => {}))
    , 'regexp'            : getType(new RegExp())
    , 'string'            : getType('')
    , 'set'               : getType(new Set())
    , 'symbol'            : getType(Symbol())
    , 'undefined'         : getType(undefined)
    , 'weakmap'           : getType(new WeakMap())
    , 'weakset'           : getType(new WeakSet())
};

export function setType(key, value) {
    types[key] = getType(value);
}

function _clone(source, rc) {
    if (rc.count > rc.maxStackDepth) throw new Error("Stack depth exceeded: " + rc.stackMaxDepth + "!");
    switch (getType(source)) {
        // case types.buffer:
            // return _bufferCopy(source, new Buffer(source.length));
        // case types.object:
        case types.array:
            return _singleCopy(source, [], rc);
        case types.regexp:
            return _singleCopy(source, new RegExp(source), rc);
        case types.date:
            return _singleCopy(source, new Date(source), rc);
        case types.set:
            return _setCopy(source, new Set(), rc);
        case types.map:
            return _mapCopy(source, new Map(), rc);
        case types.boolean:
        case types.null:
        case types.number:
        case types.string:
        case types.undefined:
            return source;
        case types.function:
            return source;
        case types.object:
        default:
            if (Buffer.isBuffer(source)) // boo, extra checks on each object because of bad buffer toStringTag
                return _bufferCopy(source, new Buffer(source.length));
            else
                return _objectCopy(source, Object.create(Reflect.getPrototypeOf(source)), rc);
    }
}

function _instanceCopy(sourceRef, copyRef, rc, copier) {
    let origIndex = rc.xStack.indexOf(sourceRef);
    if (origIndex === -1) {
        rc.push(sourceRef, copyRef);
        forEach(sourceRef, function(value, key) {
            copier(copyRef, value, key);
        });
        rc.pop();
        return copyRef;
    }
    else
        return rc.yStack[origIndex];
}

function _objectCopy(sourceRef, copyRef, rc) {
    let origIndex = rc.xStack.indexOf(sourceRef);
    if (origIndex === -1) {
        rc.push(sourceRef, copyRef);
        for (let [key, val] of Object.entries(sourceRef))
            copyRef[key] = _clone(val, rc);
        let symbols = Object.getOwnPropertySymbols(sourceRef);
        for (let symbol of symbols)
            copyRef[symbol] = _clone(sourceRef[symbol], rc);
        rc.pop();
        return copyRef;
    }
    else
        return rc.yStack[origIndex];
}

function _setCopy(sourceRef, copyRef, rc) {
    return _instanceCopy(sourceRef, copyRef, rc, (set, val) => {
        set.add(_clone(val, rc));
    });
}

function _mapCopy(sourceRef, copyRef, rc) {
    return _instanceCopy(sourceRef, copyRef, rc, (map, val, key) => {
        map.set(key, _clone(val, rc));
    });
}

function _singleCopy(sourceRef, copyRef, rc) {
    return _instanceCopy(sourceRef, copyRef, rc, (item, val, key) => {
        copyRef[key] = _clone(val, rc);
    });
}

function _bufferCopy(sourceRef, copyRef, rc) {
    sourceRef.copy(copyRef);
    return copyRef;
}

export function clone(origSource) {
    let origIndex = -1;
    let rc = new RecursiveCounter(1000);
    return _clone.call(null, origSource, rc);
}

/**
    Internal method for comparing the equivalence of items.

    @param x The first item to compare.
    @param y The second item to compare.
    @param rc The running counter for comparing circular references.
    @returns {boolean} An indication as to whether or not x and y were equal.
*/
function _equals(x, y, rc) {
    if (rc.count > rc.maxStackDepth) throw new Error("Stack depth exceeded: " + rc.maxStackDepth + "!");
    // check for reference and primitive equality
    if (x === y)
        return true;
    // check for type equality
    let xType = getType(x);<commit_msg>docs: additional method documentation in util.js<commit_after>/**
    @license
    Copyright (C) 2015 <NAME>
    License: MIT
    See license.txt for full license text.
*/
"use strict";

let toString = Object.prototype.toString;
let log = console.log.bind(console); /* eslint no-unused-vars: 0 */

/**
    Class for containing a max reference counter
    as well as two stacks of references to objects.
    To be used with deepCopy and equals.

    @class Contains two reference stacks as well as a defined max stack depth.
*/
class RecurseCounter {
    constructor(maxStackDepth) {
        this.xStack = [];
        this.yStack = [];
        this.count = 0;
        this.maxStackDepth = maxStackDepth;
    }

    push(x, y) {
        this.xStack.push(x);
        this.yStack.push(y);
        this.count++;
    }

    pop() {
        this.xStack.pop();
        this.yStack.pop();
        this.count--;
    }
}

const toStringTagTemp = Symbol('@@toStringTagTemp'); // make sure this is a symbol, so we don't step on anyone else's toes.
const generatorFnProto = (function*(){}).prototype;

// NOTE: to fix a bug with how babel-runtime works without polyfilling Object.prototype.toString,
// we need to make sure that the following polyfilled items will return the expected string
// with the getType() method.
//
// if we don't do this, then toString.call() on these items will most likely return [object Object],
// which will in turn cause methods like equals() and clone() to handle them incorrectly.
Object.defineProperty(Promise.prototype, toStringTagTemp, {
    get() { return '[object GeneratorFunction]'; },
    enumerable: false
});
Object.defineProperty(Map.prototype, toStringTagTemp, {
    get() { return '[object Map]'; },
    enumerable: false
});
Object.defineProperty(Set.prototype, toStringTagTemp, {
    get() { return '[object Set]'; },
    enumerable: false
});
Object.defineProperty(WeakMap.prototype, toStringTagTemp, {
    get() { return '[object WeakMap]'; },
    enumerable: false
});
Object.defineProperty(WeakSet.prototype, toStringTagTemp, {
    get() { return '[object WeakSet]'; },
    enumerable: false
});
Object.defineProperty(generatorFnProto, toStringTagTemp, { // make sure this doesn't collide with anything else.
    get() { return '[object GeneratorFunction]'; },
    enumerable: false
});

export function getType(val) {
    if (val && val[toStringTagTemp])
        return val[toStringTagTemp];
    return toString.call(val);
}

export var types = {
      'arguments'         : getType(arguments) // will this work? babel may be accidentally saving us here. swap to iife if necessary
    , 'array'             : getType([])
    , 'boolean'           : getType(true)
    // buffer doesn't work, toString.call(Buffer) returns [object Object]
    , 'date'              : getType(new Date())
    , 'error'             : getType(new Error())
    // , 'generator'         : getType(generatorProto)
    , 'generatorFunction' : getType(generatorFnProto)
    , 'function'          : getType(() => {})
    , 'map'               : getType(new Map())
    , 'null'              : getType(null)
    , 'number'            : getType(0)
    , 'object'            : getType({})
    , 'promise'           : getType(new Promise(() => {}))
    , 'regexp'            : getType(new RegExp())
    , 'string'            : getType('')
    , 'set'               : getType(new Set())
    , 'symbol'            : getType(Symbol())
    , 'undefined'         : getType(undefined)
    , 'weakmap'           : getType(new WeakMap())
    , 'weakset'           : getType(new WeakSet())
};

export function setType(key, value) {
    types[key] = getType(value);
}

function _clone(source, rc) {
    if (rc.count > rc.maxStackDepth) throw new Error("Stack depth exceeded: " + rc.stackMaxDepth + "!");
    switch (getType(source)) {
        // case types.buffer:
            // return _bufferCopy(source, new Buffer(source.length));
        // case types.object:
        case types.array:
            return _singleCopy(source, [], rc);
        case types.regexp:
            return _singleCopy(source, new RegExp(source), rc);
        case types.date:
            return _singleCopy(source, new Date(source), rc);
        case types.set:
            return _setCopy(source, new Set(), rc);
        case types.map:
            return _mapCopy(source, new Map(), rc);
        case types.boolean:
        case types.null:
        case types.number:
        case types.string:
        case types.undefined:
            return source;
        case types.function:
            return source;
        case types.object:
        default:
            if (Buffer.isBuffer(source)) // boo, extra checks on each object because of bad buffer toStringTag
                return _bufferCopy(source, new Buffer(source.length));
            else
                return _objectCopy(source, Object.create(Reflect.getPrototypeOf(source)), rc);
    }
}

/**
    The shared instance copier based on provided callback.

    @param sourceRef A reference to the source.
    @param copyRef A reference to the copy.
    @param rc The current recurse and reference counter.
    @param copier The callback used to make a copy of the property list from source to copy reference.
    @returns {any} The reference to the copy.
*/
function _instanceCopy(sourceRef, copyRef, rc, copier) {
    let origIndex = rc.xStack.indexOf(sourceRef);
    if (origIndex === -1) {
        rc.push(sourceRef, copyRef);
        forEach(sourceRef, function(value, key) {
            copier(copyRef, value, key);
        });
        rc.pop();
        return copyRef;
    }
    else
        return rc.yStack[origIndex];
}

/**
    Copies data from one object to another.

    @param sourceRef A reference to the object source.
    @param copyRef A reference to the object copy.
    @param rc The current recurse and reference counter.
    @returns {Object} The reference to the object copy.
*/
function _objectCopy(sourceRef, copyRef, rc) {
    let origIndex = rc.xStack.indexOf(sourceRef);
    if (origIndex === -1) {
        rc.push(sourceRef, copyRef);
        for (let [key, val] of Object.entries(sourceRef))
            copyRef[key] = _clone(val, rc);
        let symbols = Object.getOwnPropertySymbols(sourceRef);
        for (let symbol of symbols)
            copyRef[symbol] = _clone(sourceRef[symbol], rc);
        rc.pop();
        return copyRef;
    }
    else
        return rc.yStack[origIndex];
}

/**
    Copies data from one set to another.

    @param sourceRef A reference to the set source.
    @param copyRef A reference to the set copy.
    @param rc The current recurse and reference counter.
    @returns {Set} The reference to the set copy.
*/
function _setCopy(sourceRef, copyRef, rc) {
    return _instanceCopy(sourceRef, copyRef, rc, (set, val) => {
        set.add(_clone(val, rc));
    });
}

/**
    Copies data from one map to another.

    @param sourceRef A reference to the map source.
    @param copyRef A reference to the map copy.
    @param rc The current recurse and reference counter.
    @returns {Map} The reference to the map copy.
*/
function _mapCopy(sourceRef, copyRef, rc) {
    return _instanceCopy(sourceRef, copyRef, rc, (map, val, key) => {
        map.set(key, _clone(val, rc));
    });
}

/**
    Copies data from one item to another.
    Designed to a be a generic copy interface, where possible.

    @param sourceRef A reference to the source.
    @param copyRef A reference to the copy.
    @param rc The current recurse and reference counter.
    @returns {any} The reference to the copy.
*/
function _singleCopy(sourceRef, copyRef, rc) {
    return _instanceCopy(sourceRef, copyRef, rc, (item, val, key) => {
        copyRef[key] = _clone(val, rc);
    });
}

/**
    Copies data from one buffer to another.

    @param sourceRef A reference to the buffer source.
    @param copyRef A reference to the buffer copy.
    @returns {Buffer} The reference to the buffer copy.
*/
function _bufferCopy(sourceRef, copyRef) {
    sourceRef.copy(copyRef);
    return copyRef;
}

/**
    Creates a deep clone of the provided source.

    @param origSource The item to clone.
    @returns {any} The cloned item.
*/
export function clone(origSource) {
    let origIndex = -1;
    let rc = new RecursiveCounter(1000);
    return _clone.call(null, origSource, rc);
}

/**
    Internal method for comparing the equivalence of items.

    @param x The first item to compare.
    @param y The second item to compare.
    @param rc The running counter for comparing circular references.
    @returns {boolean} An indication as to whether or not x and y were equal.
*/
function _equals(x, y, rc) {
    if (rc.count > rc.maxStackDepth) throw new Error("Stack depth exceeded: " + rc.maxStackDepth + "!");
    // check for reference and primitive equality
    if (x === y)
        return true;
    // check for type equality
    let xType = getType(x);
{'fraction_non_alphanumeric': 0.11836938228510877, 'fraction_numerical': 0.0017453094807704296, 'mean_word_length': 2.575663026521061, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6455818', 'n_tokens_mistral': 4660, 'n_tokens_neox': 4612, 'n_words': 1473}