<commit_before>(function(ns)
{

  var JSONC = {},
      _nCode = -1,
      toString = {}.toString;

  if ( !Array.prototype.contains )
  {
    Array.prototype.contains = function ( v )
    {
      var i;
      for ( i = 0; i < this.length; i++ )
      {
        if ( this[i][1] === v )
        {
          return true;
        }
      }
      return false;
    };
  }
  if ( !Array.prototype.unique )
  {
    Array.prototype.unique = function () {
      var i,
          arr = [];
      for ( i = 0; i < this.length; i++)
      {
        if ( !arr.contains( this[i][1] ) ) {
          arr.push( this[i] );
        }
      }
      return arr;
    };
  }
  if(!RegExp.escape)
  {
    RegExp.escape = function( text )
    {
      return text.replace( /[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&" );
    };
  }
  function _isObject ( obj )
  {
    return toString.call( obj ) === "[object Object]";
  }
  function _isArray ( obj )
  {
    return toString.call( obj ) === "[object Array]";
  }
  function biDimensionalArrayToObject( aArr )
  {
    var obj = {},
        nIndex,
        nLen = aArr.length,
        oItem;
    for( nIndex = 0; nIndex < nLen; nIndex++ )
    {
      oItem = aArr[nIndex];
      obj[oItem[0]] = oItem[1];
    }
    return obj;
  }
  function _numberToKey ( index, totalChar, offset ){
    totalChar = totalChar || 26;
    offset = offset || 65;
    var aArr = [];
    var currentChar = index;
    while( currentChar >= totalChar )
    {
      aArr.push( (currentChar % totalChar) + offset );
      currentChar = Math.floor( currentChar / totalChar - 1 );
    }
    aArr.push(currentChar + offset);
    return aArr.reverse();
  }
  function _getSpecialKey ( aKeys )
  {
    return String.fromCharCode.apply(String, aKeys);
  }
  function _getKeys ( json, aKeys )
  {
    var aKey,
      sKey,
      oItem;

    for ( sKey in json )
    {

      if ( json.hasOwnProperty( sKey ) )
      {
        oItem = json[sKey];
        if ( _isObject( oItem ) || _isArray( oItem ) )
        {
          aKeys = aKeys.concat( _getKeys( oItem, aKeys).unique() );
        }
        if ( isNaN( Number( sKey ) ) )
        {
          if ( !aKeys.contains( sKey ) )
          {
            _nCode += 1;
            aKey = [];
            aKey.push( _getSpecialKey( _numberToKey( _nCode ) ), sKey );
            aKeys.push( aKey );
          }
        }
      }
    }
    return aKeys;
  }
  JSONC.compress = function ( json, optKeys )
  {
    if(!optKeys)
    {
      _nCode = -1;
    }
    var aKeys = optKeys || [],
      oKeys,
      aKey,
      obj,
      str,
      nIndex,
      nLenKeys;
    if ( _isArray( json ) )
    {
      for ( nIndex = 0, nLenKeys = json.length; nIndex < nLenKeys; nIndex++ )
      {
        json[nIndex] = this.compress( json[nIndex], aKeys );
      }
    }
    else
    {
      aKeys = _getKeys( json, aKeys );
      aKeys = aKeys.unique();
      oKeys = biDimensionalArrayToObject( aKeys );

      str = JSON.stringify( json );
      nLenKeys = aKeys.length;

      for ( nIndex = 0; nIndex < nLenKeys; nIndex++ )
      {
        aKey = aKeys[nIndex];
        str = str.replace( new RegExp( RegExp.escape( '"' + aKey[1] + '"' ), "g" ), '"' + aKey[0] + '"' );
      }

      obj = JSON.parse( str );
      obj._ = oKeys;
    }
    return obj || json;
  };<commit_msg>Reduce complexity in compress method<commit_after>(function(ns)
{

  var JSONC = {},
      _nCode = -1,
      toString = {}.toString;

  if ( !Array.prototype.contains )
  {
    Array.prototype.contains = function ( v )
    {
      var i;
      for ( i = 0; i < this.length; i++ )
      {
        if ( this[i][1] === v )
        {
          return true;
        }
      }
      return false;
    };
  }
  if ( !Array.prototype.unique )
  {
    Array.prototype.unique = function () {
      var i,
          arr = [];
      for ( i = 0; i < this.length; i++)
      {
        if ( !arr.contains( this[i][1] ) ) {
          arr.push( this[i] );
        }
      }
      return arr;
    };
  }
  if(!RegExp.escape)
  {
    RegExp.escape = function( text )
    {
      return text.replace( /[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&" );
    };
  }
  function _isObject ( obj )
  {
    return toString.call( obj ) === "[object Object]";
  }
  function _isArray ( obj )
  {
    return toString.call( obj ) === "[object Array]";
  }
  function biDimensionalArrayToObject( aArr )
  {
    var obj = {},
        nIndex,
        nLen = aArr.length,
        oItem;
    for( nIndex = 0; nIndex < nLen; nIndex++ )
    {
      oItem = aArr[nIndex];
      obj[oItem[0]] = oItem[1];
    }
    return obj;
  }
  function _numberToKey ( index, totalChar, offset ){
    totalChar = totalChar || 26;
    offset = offset || 65;
    var aArr = [];
    var currentChar = index;
    while( currentChar >= totalChar )
    {
      aArr.push( (currentChar % totalChar) + offset );
      currentChar = Math.floor( currentChar / totalChar - 1 );
    }
    aArr.push(currentChar + offset);
    return aArr.reverse();
  }
  function _getSpecialKey ( aKeys )
  {
    return String.fromCharCode.apply(String, aKeys);
  }
  function _getKeys ( json, aKeys )
  {
    var aKey,
      sKey,
      oItem;

    for ( sKey in json )
    {

      if ( json.hasOwnProperty( sKey ) )
      {
        oItem = json[sKey];
        if ( _isObject( oItem ) || _isArray( oItem ) )
        {
          aKeys = aKeys.concat( _getKeys( oItem, aKeys).unique() );
        }
        if ( isNaN( Number( sKey ) ) )
        {
          if ( !aKeys.contains( sKey ) )
          {
            _nCode += 1;
            aKey = [];
            aKey.push( _getSpecialKey( _numberToKey( _nCode ) ), sKey );
            aKeys.push( aKey );
          }
        }
      }
    }
    return aKeys;
  }
  function compressArray ( json, aKeys )
  {
    var nIndex,
        nLenKeys;

    for ( nIndex = 0, nLenKeys = json.length; nIndex < nLenKeys; nIndex++ )
    {
      json[nIndex] = JSONC.compress( json[nIndex], aKeys );
    }
  }
  function compressOther ( json, aKeys )
  {
    var oKeys,
        aKey,
        str,
        nLenKeys,
        nIndex,
        obj;
    aKeys = _getKeys( json, aKeys );
    aKeys = aKeys.unique();
    oKeys = biDimensionalArrayToObject( aKeys );

    str = JSON.stringify( json );
    nLenKeys = aKeys.length;

    for ( nIndex = 0; nIndex < nLenKeys; nIndex++ )
    {
      aKey = aKeys[nIndex];
      str = str.replace( new RegExp( RegExp.escape( '"' + aKey[1] + '"' ), "g" ), '"' + aKey[0] + '"' );
    }

    obj = JSON.parse( str );
    obj._ = oKeys;
    return obj;
  }
  JSONC.compress = function ( json, optKeys )
  {
    if(!optKeys)
    {
      _nCode = -1;
    }
    var aKeys = optKeys || [],
      obj;
    if ( _isArray( json ) )
    {
      compressArray( json, aKeys );
    }
    else
    {
      obj = compressOther ( json, aKeys );
    }
    return obj || json;
  };
{'fraction_non_alphanumeric': 0.15084125822970007, 'fraction_numerical': 0.005559619604974396, 'mean_word_length': 1.6724003127443314, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2520495', 'n_tokens_mistral': 2436, 'n_tokens_neox': 2406, 'n_words': 516}