<commit_before>// ==UserScript==
// @name         Magic™ Tag Review 2
// @namespace    http://github.com/Tiny-Giant
// @version      1.0.1.2
// @description  Custom review queue for tag oriented reviewing with the ability to filter by close votes and delete votes
// @author       @TinyGiant
// @contributor  @Makyen
// @include      /^https?:\/\/\w*.?stackoverflow\.com\/review*/
// @grant        GM_getValue
// @grant        GM_setValue
// @grant        GM_addValueChangeListener
// @run-at       document-start
// ==/UserScript==
/* jshint -W097 */
/* jshint esnext: true */
/* globals $, GM_setValue, GM_getValue, StackExchange, GM_info */
'use strict';

/** Start Reusable Utilities **/
        
const executeInPage = function(functionToRunInPage, leaveInPage, id) {
    //Execute a function in the page context.
    // Any additional arguments for this function are passed into the page to the functionToRunInPage.
    // Such arguments must be JSON-ifiable (also Date, Function, and RegExp) (prototypes are not copied).
    // Using () => doesn't set arguments, so can't use it to define this function.
    // This has to be done without jQuery, as jQuery creates the script
    // within this context, not the page context, which results in
    // permission denied to run the function.
    function convertToText(args) {
        //This uses the fact that the arguments are converted to text which is interpreted
        //  within a <script>. That means we can create other types of objects by recreating their
        //  normal JavaScript representation.
        //  It's actually easier to do this without JSON.strigify() for the whole Object/Array.
        var asText = '';
        var level = 0;
        function lineSeparator(adj, isntLast) {
            level += adj - ((typeof isntLast === 'undefined' || isntLast) ? 0 : 1);
            asText += (isntLast ? ',' : '') + '\n' + (new Array(level * 2 + 1)).join('');
        }
        function recurseObject(obj) {
            if (Array.isArray(obj)) {
                asText += '[';
                lineSeparator(1);
                obj.forEach(function(value, index, array) {
                    recurseObject(value);
                    lineSeparator(0, index !== array.length - 1);
                });
                asText += ']';
            } else if (obj === null) {
                asText += 'null';
            //undefined
            } else if (obj === void(0)) {
                asText += 'void(0)';
            //Special cases for Number
            } else if (Number.isNaN(obj)) {
                asText += 'Number.NaN';
            } else if (obj === 1/0) {
                asText += '1/0';
            } else if (obj === 1/-0) {
                asText += '1/-0';
            //function
            } else if (obj instanceof RegExp || typeof obj === 'function') {
                asText +=  obj.toString();
            } else if (obj instanceof Date) {
                asText += 'new Date("' + obj.toJSON() + '")';
            } else if (typeof obj === 'object') {
                asText += '{';
                lineSeparator(1);
                Object.keys(obj).forEach(function(prop, index, array) {
                    asText += JSON.stringify(prop) + ': ';
                    recurseObject(obj[prop]);
                    lineSeparator(0, index !== array.length - 1);
                });
                asText += '}';
            } else if (['boolean', 'number', 'string'].indexOf(typeof obj) > -1) {
                asText += JSON.stringify(obj);
            } else {
                console.log('Didn\'t handle: typeof obj:', typeof obj, '::  obj:', obj);
            }
        }
        recurseObject(args);
        return asText;
    }
    var newScript = document.createElement('script');
    if(typeof id === 'string' && id) {
        newScript.id = id;
    }
    var args = [];
    //using .slice(), or other Array methods, on arguments prevents optimization
    for(var index=3;index<arguments.length;index++){
        args.push(arguments[index]);
    }
    newScript.textContent = '(' + functionToRunInPage.toString() + ').apply(null,' + convertToText(args) + ");";
    (document.head || document.documentElement).appendChild(newScript);
    if(!leaveInPage) {
        //Synchronous scripts are executed immediately and can be immediately removed.
        //Scripts with asynchronous functionality *may* need to remain in the page until complete.
        document.head.removeChild(newScript);
    }
    return newScript;
};


/**
 * @typedef XHRListener
 * @type {function}
 * @param {object} data - Object containing the request object, the property being accessed or method being called, and the value that would be returned
 * @param {XMLHttpRequest} data.xhr - The request object
 * @param {string} [data.property] - The property being accessed (get/set)
 * @param {string} [data.method] - the method being called (call)
 * @param {*} data.value - The value being returned or set (get/set/call)
 * @returns {undefined}
 */
/**
 * @typedef XHRListenerObject
 * @type {object}
 * @param {RegExp} regex - To be matched against XMLHttpRequest.responseURL in order to execute the listener
 * @param {XHRListener} [get] - Called when get operations occur on the request object, 
 * @param {XHRListener} [set] - Called when set operations occur on the request object
 * @param {XHRListener} [call] - Called when methods of the request object are called
 */
/**
 * @function addXHRListener - Intercepts XMLHttpRequests 
 * @param {XHRListenerObject} listener
 * @return {bool} false
 * @github https://github.com/Tiny-Giant/JS-Examples/blob/master/addXHRListener.js
 */
const inPageAddXHRListener = listen => {
    window.addXHRListener = (_ => {
        const XHR = XMLHttpRequest;

        const listeners = [];
        
        XMLHttpRequest = new Proxy(XHR, {
            construct: (target, args) => {
                const callall = (type, data) => (listeners.forEach(e => type in e && e.regex.test(xhr.responseURL) && e[type](data)), data);

                const xhr = new XHR();

                return new Proxy(xhr, {
                    get: (t, k) => {
                        if(typeof xhr[k] === 'function') {
                            return new Proxy(xhr[k], {
                                apply: async (target, self, args) => {
                                    const result = await xhr[k](...args);
                                    const data = callall('call', { xhr, 'method': k, 'args': args.join(', '), 'value': result });
                                    return data.value;
                                }
                            });
                        }
                        const data = callall('get', { xhr, 'property': k, 'value': xhr[k] });
                        return data.value;
                    },
                    set: (t, k, v) => {
                        const data = callall('set', { xhr, 'property': k, 'value': v });
                        xhr[k] = data.value;
                        return true;
                    }
                });
            }
        });

        return listener => (listeners.push(listener), !0);
    })();
};
/* For de bug porpoises */
/*addXHRListener({
    regex: /./,
    get: data => console.log(data.xhr.responseURL, 'get', data),
    set: data => console.log(data.xhr.responseURL, 'set', data),
    call: data => console.log(data.xhr.responseURL, 'call', data)
});*/
executeInPage(inPageAddXHRListener, true, 'magicTagReview-addXHRListener');

/** @Proxy store - Wraps GM_(set/get)value with a prefix to prevent interference with other scripts */
//Use an in-script cache for what's being stored to GM storage. A side effect is that updates in other tabs
//  are NOT seen cross-tab once the value is first read from storage. This provides semi-independance across
//  tabs.
var storageCache = {};
const store = new Proxy({}, {
    get: (t, k) => (typeof storageCache[k] === 'undefined' ? storageCache[k] = GM_getValue(`MagicTagReview-${ k }`) : storageCache[k]),
    //Immediately update the cache and GM storage.
    //Saving the data to GM storage asynchronously doesn't solve the problem.
    //Storing large amounts of data to GM storage in Chrome/Tampermonkey is expensive. It will freeze the tab and
    //  all Tampermonkey UI tabs (i.e. ones in the background context) for several seconds (it may affect all Tampermonkey
    //  operations (at least in the background context, which will probably affect loading scripts in new pages).
    //  This is not the case in Firefox/Tampermonkey or Firefox/Greasemonkey.
    set: (t, k, v) => (storageCache[k] = v, GM_setValue(`MagicTagReview-${ k }`, storageCache[k]), true)
});
<commit_msg>Storage performance improvements   Add compression (LZ-String) for question list and queue.   Store filter settings/options as a single JSON value in localStorage, not GM storage.   Clean-up/transfer old data storage to current.<commit_after>// ==UserScript==
// @name         Magic™ Tag Review 2
// @namespace    http://github.com/Tiny-Giant
// @version      1.0.1.2
// @description  Custom review queue for tag oriented reviewing with the ability to filter by close votes and delete votes
// @author       @TinyGiant
// @contributor  @Makyen
// @include      /^https?:\/\/\w*.?stackoverflow\.com\/review*/
// @grant        GM_getValue
// @grant        GM_setValue
// @grant        GM_listValues
// @grant        GM_deleteValue
// @grant        GM_addValueChangeListener
// @run-at       document-start
// ==/UserScript==
/* jshint -W097 */
/* jshint esnext: true */
/* globals $, GM_setValue, GM_getValue, StackExchange, GM_info */
'use strict';

/** Start Reusable Utilities **/
        
const executeInPage = function(functionToRunInPage, leaveInPage, id) {
    //Execute a function in the page context.
    // Any additional arguments for this function are passed into the page to the functionToRunInPage.
    // Such arguments must be JSON-ifiable (also Date, Function, and RegExp) (prototypes are not copied).
    // Using () => doesn't set arguments, so can't use it to define this function.
    // This has to be done without jQuery, as jQuery creates the script
    // within this context, not the page context, which results in
    // permission denied to run the function.
    function convertToText(args) {
        //This uses the fact that the arguments are converted to text which is interpreted
        //  within a <script>. That means we can create other types of objects by recreating their
        //  normal JavaScript representation.
        //  It's actually easier to do this without JSON.strigify() for the whole Object/Array.
        var asText = '';
        var level = 0;
        function lineSeparator(adj, isntLast) {
            level += adj - ((typeof isntLast === 'undefined' || isntLast) ? 0 : 1);
            asText += (isntLast ? ',' : '') + '\n' + (new Array(level * 2 + 1)).join('');
        }
        function recurseObject(obj) {
            if (Array.isArray(obj)) {
                asText += '[';
                lineSeparator(1);
                obj.forEach(function(value, index, array) {
                    recurseObject(value);
                    lineSeparator(0, index !== array.length - 1);
                });
                asText += ']';
            } else if (obj === null) {
                asText += 'null';
            //undefined
            } else if (obj === void(0)) {
                asText += 'void(0)';
            //Special cases for Number
            } else if (Number.isNaN(obj)) {
                asText += 'Number.NaN';
            } else if (obj === 1/0) {
                asText += '1/0';
            } else if (obj === 1/-0) {
                asText += '1/-0';
            //function
            } else if (obj instanceof RegExp || typeof obj === 'function') {
                asText +=  obj.toString();
            } else if (obj instanceof Date) {
                asText += 'new Date("' + obj.toJSON() + '")';
            } else if (typeof obj === 'object') {
                asText += '{';
                lineSeparator(1);
                Object.keys(obj).forEach(function(prop, index, array) {
                    asText += JSON.stringify(prop) + ': ';
                    recurseObject(obj[prop]);
                    lineSeparator(0, index !== array.length - 1);
                });
                asText += '}';
            } else if (['boolean', 'number', 'string'].indexOf(typeof obj) > -1) {
                asText += JSON.stringify(obj);
            } else {
                console.log('Didn\'t handle: typeof obj:', typeof obj, '::  obj:', obj);
            }
        }
        recurseObject(args);
        return asText;
    }
    var newScript = document.createElement('script');
    if(typeof id === 'string' && id) {
        newScript.id = id;
    }
    var args = [];
    //using .slice(), or other Array methods, on arguments prevents optimization
    for(var index=3;index<arguments.length;index++){
        args.push(arguments[index]);
    }
    newScript.textContent = '(' + functionToRunInPage.toString() + ').apply(null,' + convertToText(args) + ");";
    (document.head || document.documentElement).appendChild(newScript);
    if(!leaveInPage) {
        //Synchronous scripts are executed immediately and can be immediately removed.
        //Scripts with asynchronous functionality *may* need to remain in the page until complete.
        document.head.removeChild(newScript);
    }
    return newScript;
};


/**
 * @typedef XHRListener
 * @type {function}
 * @param {object} data - Object containing the request object, the property being accessed or method being called, and the value that would be returned
 * @param {XMLHttpRequest} data.xhr - The request object
 * @param {string} [data.property] - The property being accessed (get/set)
 * @param {string} [data.method] - the method being called (call)
 * @param {*} data.value - The value being returned or set (get/set/call)
 * @returns {undefined}
 */
/**
 * @typedef XHRListenerObject
 * @type {object}
 * @param {RegExp} regex - To be matched against XMLHttpRequest.responseURL in order to execute the listener
 * @param {XHRListener} [get] - Called when get operations occur on the request object, 
 * @param {XHRListener} [set] - Called when set operations occur on the request object
 * @param {XHRListener} [call] - Called when methods of the request object are called
 */
/**
 * @function addXHRListener - Intercepts XMLHttpRequests 
 * @param {XHRListenerObject} listener
 * @return {bool} false
 * @github https://github.com/Tiny-Giant/JS-Examples/blob/master/addXHRListener.js
 */
const inPageAddXHRListener = listen => {
    window.addXHRListener = (_ => {
        const XHR = XMLHttpRequest;

        const listeners = [];
        
        XMLHttpRequest = new Proxy(XHR, {
            construct: (target, args) => {
                const callall = (type, data) => (listeners.forEach(e => type in e && e.regex.test(xhr.responseURL) && e[type](data)), data);

                const xhr = new XHR();

                return new Proxy(xhr, {
                    get: (t, k) => {
                        if(typeof xhr[k] === 'function') {
                            return new Proxy(xhr[k], {
                                apply: async (target, self, args) => {
                                    const result = await xhr[k](...args);
                                    const data = callall('call', { xhr, 'method': k, 'args': args.join(', '), 'value': result });
                                    return data.value;
                                }
                            });
                        }
                        const data = callall('get', { xhr, 'property': k, 'value': xhr[k] });
                        return data.value;
                    },
                    set: (t, k, v) => {
                        const data = callall('set', { xhr, 'property': k, 'value': v });
                        xhr[k] = data.value;
                        return true;
                    }
                });
            }
        });

        return listener => (listeners.push(listener), !0);
    })();
};
/* For de bug porpoises */
/*addXHRListener({
    regex: /./,
    get: data => console.log(data.xhr.responseURL, 'get', data),
    set: data => console.log(data.xhr.responseURL, 'set', data),
    call: data => console.log(data.xhr.responseURL, 'call', data)
});*/
executeInPage(inPageAddXHRListener, true, 'magicTagReview-addXHRListener');

//LZ-String 1.4.4
// Copyright (c) 2013 Pieroxy <<EMAIL>>
// This work is free. You can redistribute it and/or modify it
// under the terms of the WTFPL, Version 2
// For more information see LICENSE.txt or http://www.wtfpl.net/
//
// For more information, the home page:
// http://pieroxy.net/blog/pages/lz-string/testing.html
//
// LZ-based compression algorithm, version 1.4.4
// From: https://github.com/pieroxy/lz-string/blob/master/libs/lz-string.min.js
var LZString=function(){function o(o,r){if(!t[o]){t[o]={};for(var n=0;n<o.length;n++)t[o][o.charAt(n)]=n}return t[o][r]}var r=String.fromCharCode,n="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",e="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-$",t={},i={compressToBase64:function(o){if(null==o)return"";var r=i._compress(o,6,function(o){return n.charAt(o)});switch(r.length%4){default:case 0:return r;case 1:return r+"===";case 2:return r+"==";case 3:return r+"="}},decompressFromBase64:function(r){return null==r?"":""==r?null:i._decompress(r.length,32,function(e){return o(n,r.charAt(e))})},compressToUTF16:function(o){return null==o?"":i._compress(o,15,function(o){return r(o+32)})+" "},decompressFromUTF16:function(o){return null==o?"":""==o?null:i._decompress(o.length,16384,function(r){return o.charCodeAt(r)-32})},compressToUint8Array:function(o){for(var r=i.compress(o),n=new Uint8Array(2*r.length),e=0,t=r.length;t>e;e++){var s=r.charCodeAt(e);n[2*e]=s>>>8,n[2*e+1]=s%256}return n},decompressFromUint8Array:function(o){if(null===o||void 0===o)return i.decompress(o);for(var n=new Array(o.length/2),e=0,t=n.length;t>e;e++)n[e]=256*o[2*e]+o[2*e+1];var s=[];return n.forEach(function(o){s.push(r(o))}),i.decompress(s.join(""))},compressToEncodedURIComponent:function(o){return null==o?"":i._compress(o,6,function(o){return e.charAt(o)})},decompressFromEncodedURIComponent:function(r){return null==r?"":""==r?null:(r=r.replace(/ /g,"+"),i._decompress(r.length,32,function(n){return o(e,r.charAt(n))}))},compress:function(o){return i._compress(o,16,function(o){return r(o)})},_compress:function(o,r,n){if(null==o)return"";var e,t,i,s={},p={},u="",c="",a="",l=2,f=3,h=2,d=[],m=0,v=0;for(i=0;i<o.length;i+=1)if(u=o.charAt(i),Object.prototype.hasOwnProperty.call(s,u)||(s[u]=f++,p[u]=!0),c=a+u,Object.prototype.hasOwnProperty.call(s,c))a=c;else{if(Object.prototype.hasOwnProperty.call(p,a)){if(a.charCodeAt(0)<256){for(e=0;h>e;e++)m<<=1,v==r-1?(v=0,d.push(n(m)),m=0):v++;for(t=a.charCodeAt(0),e=0;8>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1}else{for(t=1,e=0;h>e;e++)m=m<<1|t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t=0;for(t=a.charCodeAt(0),e=0;16>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1}l--,0==l&&(l=Math.pow(2,h),h++),delete p[a]}else for(t=s[a],e=0;h>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1;l--,0==l&&(l=Math.pow(2,h),h++),s[c]=f++,a=String(u)}if(""!==a){if(Object.prototype.hasOwnProperty.call(p,a)){if(a.charCodeAt(0)<256){for(e=0;h>e;e++)m<<=1,v==r-1?(v=0,d.push(n(m)),m=0):v++;for(t=a.charCodeAt(0),e=0;8>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1}else{for(t=1,e=0;h>e;e++)m=m<<1|t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t=0;for(t=a.charCodeAt(0),e=0;16>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1}l--,0==l&&(l=Math.pow(2,h),h++),delete p[a]}else for(t=s[a],e=0;h>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1;l--,0==l&&(l=Math.pow(2,h),h++)}for(t=2,e=0;h>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1;for(;;){if(m<<=1,v==r-1){d.push(n(m));break}v++}return d.join("")},decompress:function(o){return null==o?"":""==o?null:i._decompress(o.length,32768,function(r){return o.charCodeAt(r)})},_decompress:function(o,n,e){var t,i,s,p,u,c,a,l,f=[],h=4,d=4,m=3,v="",w=[],A={val:e(0),position:n,index:1};for(i=0;3>i;i+=1)f[i]=i;for(p=0,c=Math.pow(2,2),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;switch(t=p){case 0:for(p=0,c=Math.pow(2,8),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;l=r(p);break;case 1:for(p=0,c=Math.pow(2,16),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;l=r(p);break;case 2:return""}for(f[3]=l,s=l,w.push(l);;){if(A.index>o)return"";for(p=0,c=Math.pow(2,m),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;switch(l=p){case 0:for(p=0,c=Math.pow(2,8),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;f[d++]=r(p),l=d-1,h--;break;case 1:for(p=0,c=Math.pow(2,16),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;f[d++]=r(p),l=d-1,h--;break;case 2:return w.join("")}if(0==h&&(h=Math.pow(2,m),m++),f[l])v=f[l];else{if(l!==d)return null;v=s+s.charAt(0)}w.push(v),f[d++]=s+v.charAt(0),h--,s=v,0==h&&(h=Math.pow(2,m),m++)}}};return i}();"function"==typeof define&&define.amd?define(function(){return LZString}):"undefined"!=typeof module&&null!=module&&(module.exports=LZString);


/** @Proxy store - Wraps GM_(set/get)value */
//Use an in-script cache for what's being stored to GM storage. A side effect is that updates in other tabs
//  are NOT seen cross-tab once the value is first read from storage. This provides semi-independance across
//  tabs.
var storageCache = {};
try {
    storageCache = JSON.parse(localStorage['MagicTagReview-storeageCache']);
} catch(e) {
}
var storageCacheSeparate = {};
var storageCacheTimeout=0;
const storageSeparateList = ['queue', 'question_list', 'current'];
const store = new Proxy({}, {
    get: (target, key) => {
        if (storageSeparateList.indexOf(key) > -1) {
            if(key === 'current') {
                var current = +localStorage['MagicTagReview-' + key];
                return current ? current : 0;
            } else {
                if(typeof storageCacheSeparate[key] === 'undefined') {
                    const lzString = GM_getValue(`${key}-LZString`);
                    const uncompressed = LZString.decompressFromBase64(lzString);
                    storageCacheSeparate[key] = uncompressed;
                }
                return storageCacheSeparate[key];
            }
        } else {
            return storageCache[key];
        }
    },
    //Immediately update the cache and GM storage.
    //Saving the data to GM storage asynchronously doesn't solve the problem of long times to store data.
    //Storing large amounts of data or multiple values to GM storage in Chrome/Tampermonkey is expensive. It will freeze the tab and
    //  all Tampermonkey UI tabs (i.e. ones in the background context) for several seconds (it may affect all Tampermonkey
    //  operations (at least in the background context, which will probably affect loading scripts in new pages).
    //  Thus, only the queue and question_list are stored in GM storage. Everything else is stored in localStorage.
    set: (target, key, value) => {
        if (storageSeparateList.indexOf(key) > -1) {
            if(key === 'current') {
                localStorage['MagicTagReview-' + key] = value;
            } else {
                storageCacheSeparate[key] = value;
                const compressed = LZString.compressToBase64(value);
                const uncompressed = LZString.decompressFromBase64(compressed);
                GM_setValue(`${key}-LZString`, compressed);
                const lzString = GM_getValue(`${key}-LZString`);
                const uncompressed2 = LZString.decompressFromBase64(lzString);
            }
        } else {
            storageCache[key] = value;
            //Don't stall execution for storing the chache (this doesn't actually help Chrome/Tampermonkey).
            clearTimeout(storageCacheTimeout);
            storageCacheTimeout = setTimeout(_ => {
                localStorage['MagicTagReview-storeageCache'] = JSON.stringify(storageCache);
            }, 0);
        }
        return true;
    }
});

//Clean-up from earlier released version of the script/Transfer data from original GM_setValue keys to storageCache.
//We need to clear localStorage *and* GM_storage
var gmValuesList = GM_listValues();
gmValuesList.forEach(key => {
    if(key.indexOf('MagicTagReview-') === 0) {
        var mainKey = key.replace(/^MagicTagReview-/,'');
        var value = GM_getValue(key);
        store[mainKey] = value;
        GM_deleteValue(key);
    }
});

{'fraction_non_alphanumeric': 0.17132741954731692, 'fraction_numerical': 0.0144397151835793, 'mean_word_length': 2.640312771503041, 'pattern_counts': {'":': 6, '<': 49, '<?xml version=': 0, '>': 93, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '885047', 'n_tokens_mistral': 8361, 'n_tokens_neox': 8072, 'n_words': 1808}