<commit_before>
var iconv = module.exports;

// All codecs and aliases are kept here, keyed by encoding name/alias.
// They are lazy loaded in `iconv.getCodec` from `encodings/index.js`.
iconv.encodings = null;

// Characters emitted in case of error.
iconv.defaultCharUnicode = '�';
iconv.defaultCharSingleByte = '?';

// Public API.
iconv.encode = function encode(str, encoding, options) {
    str = "" + (str || ""); // Ensure string.

    var encoder = iconv.getCodec(encoding).encoder(options);

    var res = encoder.write(str);
    var trail = encoder.end();
    
    return (trail && trail.length > 0) ? Buffer.concat([res, trail]) : res;
}

iconv.decode = function decode(buf, encoding, options) {
    buf = (buf instanceof Buffer) ? buf : new Buffer("" + (buf || ""), "binary"); // Ensure buffer.

    var decoder = iconv.getCodec(encoding).decoder(options);

    var res = decoder.write(buf);
    var trail = decoder.end();

    return (trail && trail.length > 0) ? (res + trail) : res;
}

iconv.encodingExists = function encodingExists(enc) {
    try {
        iconv.getCodec(enc);
        return true;
    } catch (e) {
        return false;
    }
}

// Legacy aliases to convert functions
iconv.toEncoding = iconv.encode;
iconv.fromEncoding = iconv.decode;

// Search for a codec in iconv.encodings. Cache codec data.
iconv._codecDataCache = {};
iconv.getCodec = function getCodec(encoding) {
    if (!iconv.encodings)
        iconv.encodings = require("../encodings"); // Lazy load all encoding definitions.
    
    // Canonicalize encoding name: strip all non-alphanumeric chars and appended year.
    var enc = (''+encoding).toLowerCase().replace(/[^0-9a-z]|:\d{4}$/g, "");

    var codecOptions, saveEnc;
    while (true) {
        var codecData = iconv._codecDataCache[enc];
        if (codecData)
            return codecData;

        var codec = iconv.encodings[enc];

        switch (typeof codec) {
            case "string": // Direct alias to other encoding.
                enc = codec;
                break;

            case "object": // Alias with additional options. Can be layered.
                if (!codecOptions) {
                    codecOptions = codec;
                    saveEnc = enc;
                }
                else {
                    for (var key in codec)
                        codecOptions[key] = codec[key];
                }

                enc = codec.type;
                break;

            case "function": // Codec itself.
                codecOptions.iconv = iconv;
                codecOptions.encodingName = saveEnc;
                codecData = codec(codecOptions);
                iconv._codecDataCache[saveEnc || enc] = codecData; // Save it to be reused later.
                return codecData;

            default:
                throw new Error("Encoding not recognized: '" + encoding + "' (searched as: '"+enc+"')");
        }
    }
}

iconv.encodeStream = iconv.decodeStream = function() { throw new Error("Iconv-lite streams supported only since Node v0.10."); }
iconv.supportsStreams = false;

// Load extensions in Node.
var nodeVer = typeof process !== 'undefined' && process.versions && 
                process.versions.node && process.versions.node.split(".").map(Number);

if (nodeVer) {
    // Load streaming support in Node v0.10+
    if (nodeVer[0] > 0 || nodeVer[1] >= 10) {
        require("./str"+"eams")(iconv); // Hide this module from Browserify by using expression in require().
    }

    // Load Node primitive extensions.
    require("./ext"+"end-node")(iconv);
}
<commit_msg>Add comments & minor code cleanup of index.js.<commit_after>
var iconv = module.exports;

// All codecs and aliases are kept here, keyed by encoding name/alias.
// They are lazy loaded in `iconv.getCodec` from `encodings/index.js`.
iconv.encodings = null;

// Characters emitted in case of error.
iconv.defaultCharUnicode = '�';
iconv.defaultCharSingleByte = '?';

// Public API.
iconv.encode = function encode(str, encoding, options) {
    str = "" + (str || ""); // Ensure string.

    var encoder = iconv.getCodec(encoding).encoder(options);

    var res = encoder.write(str);
    var trail = encoder.end();
    
    return (trail && trail.length > 0) ? Buffer.concat([res, trail]) : res;
}

iconv.decode = function decode(buf, encoding, options) {
    buf = (buf instanceof Buffer) ? buf : new Buffer("" + (buf || ""), "binary"); // Ensure buffer.

    var decoder = iconv.getCodec(encoding).decoder(options);

    var res = decoder.write(buf);
    var trail = decoder.end();

    return (trail && trail.length > 0) ? (res + trail) : res;
}

iconv.encodingExists = function encodingExists(enc) {
    try {
        iconv.getCodec(enc);
        return true;
    } catch (e) {
        return false;
    }
}

// Legacy aliases to convert functions
iconv.toEncoding = iconv.encode;
iconv.fromEncoding = iconv.decode;

// Search for a codec in iconv.encodings. Cache codec data in iconv._codecDataCache.
iconv._codecDataCache = {};
iconv.getCodec = function getCodec(encoding) {
    if (!iconv.encodings)
        iconv.encodings = require("../encodings"); // Lazy load all encoding definitions.
    
    // Canonicalize encoding name: strip all non-alphanumeric chars and appended year.
    var enc = (''+encoding).toLowerCase().replace(/[^0-9a-z]|:\d{4}$/g, "");

    // Traverse iconv.encodings to find actual codec.
    var codecData, codecOptions;
    while (true) {
        codecData = iconv._codecDataCache[enc];
        if (codecData)
            return codecData;

        var codec = iconv.encodings[enc];

        switch (typeof codec) {
            case "string": // Direct alias to other encoding.
                enc = codec;
                break;

            case "object": // Alias with options. Can be layered.
                if (!codecOptions) {
                    codecOptions = codec;
                    codecOptions.encodingName = enc;
                }
                else {
                    for (var key in codec)
                        codecOptions[key] = codec[key];
                }

                enc = codec.type;
                break;

            case "function": // Codec itself.
                if (!codecOptions)
                    codecOptions = { encodingName: enc };
                codecOptions.iconv = iconv;

                // The codec function must load all tables and return object with .encoder and .decoder methods.
                // It'll be called only once (for each different options object).
                codecData = codec.call(iconv.encodings, codecOptions);

                iconv._codecDataCache[codecOptions.encodingName] = codecData; // Save it to be reused later.
                return codecData;

            default:
                throw new Error("Encoding not recognized: '" + encoding + "' (searched as: '"+enc+"')");
        }
    }
}

iconv.encodeStream = iconv.decodeStream = function() { throw new Error("Iconv-lite streams supported only since Node v0.10."); }
iconv.supportsStreams = false;

// Load extensions in Node.
var nodeVer = typeof process !== 'undefined' && process.versions && 
                process.versions.node && process.versions.node.split(".").map(Number);

if (nodeVer) {
    // Load streaming support in Node v0.10+
    if (nodeVer[0] > 0 || nodeVer[1] >= 10) {
        require("./str"+"eams")(iconv); // Hide this module from Browserify by using expression in require().
    }

    // Load Node primitive extensions.
    require("./ext"+"end-node")(iconv);
}

{'fraction_non_alphanumeric': 0.12736911862160372, 'fraction_numerical': 0.004241219350563287, 'mean_word_length': 2.200169635284139, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '447436', 'n_tokens_mistral': 2274, 'n_tokens_neox': 2135, 'n_words': 616}