<commit_before><?php
/*
 * @author Anakeen
 * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License
 * @package FDL
*/

namespace Dcp;

class ExtensionFilterIteratorDecorator extends \FilterIterator
{
    private $_ext;
    /**
     * Check whether the current element of the iterator is acceptable
     *
     * @see FilterIterator::accept()
     *
     * @return bool true if the current element is acceptable, otherwise false.
     *
     */
    public function accept()
    {
        if (substr($this->current() , -1 * strlen($this->_ext)) === $this->_ext) {
            return is_readable($this->current());
        }
        return false;
    }
    /**
     * define which extension is allowed
     *
     * @param string $pExt extension
     *
     * @return void
     */
    public function setExtension($pExt)
    {
        $this->_ext = $pExt;
    }
}
/**
 * Interface for classes in charge of detecting classes declarations in files
 *
 * @author <EMAIL> <<EMAIL>>
 *
 */
interface IClassHunter {
    /**
     * Find all classes in given file
     *
     * @param string $pFileName file name
     *
     * @return array of strings
     *
     */
    public function find($pFileName);
}
/**
 * Class in charge of detecting classes declarations in files (only for PHP 5.3)
 *
 * @author <EMAIL> <<EMAIL>>
 *
 */
class ClassHunterForPHP5_3 implements IClassHunter
{
    /**
     * Find all classes in given file
     *
     * @param string $pFileName file name
     *
     * @return array of strings
     *
     */
    public function find($pFileName)
    {
        $toReturn = array();
        $tokens = token_get_all(file_get_contents($pFileName, false));
        $currentNamespace = '';
        $namespaceHunt = false;
        $validatedNamespaceHunt = false;
        $classHunt = false;
        $whitespaceCount = 0;
        foreach ($tokens as $token) {
            if (is_array($token)) {
                if ($token[0] === T_INTERFACE || $token[0] === T_CLASS) {
                    $classHunt = true;
                    continue;
                } elseif ($token[0] === T_NAMESPACE) {
                    $namespaceHunt = true;
                    continue;
                }
                if ($classHunt && $token[0] === T_STRING) {
                    $toReturn[(strlen($currentNamespace) > 0 ? $currentNamespace . '\\' : '') . $token[1]] = $pFileName;
                    $classHunt = false;
                } elseif ($namespaceHunt && $validatedNamespaceHunt && ($token[0] === T_STRING || $token[0] === T_NS_SEPARATOR)) {
                    $currentNamespace.= $token[1];
                } elseif ($namespaceHunt && !$validatedNamespaceHunt && $token[0] === T_WHITESPACE) {
                    $currentNamespace = '';
                    $validatedNamespaceHunt = true;
                } elseif ($namespaceHunt && !$validatedNamespaceHunt && $token[0] !== T_WHITESPACE) {
                    $namespaceHunt = false;
                }
            } else {
                if ($token === ';' || $token === '{') {
                    //can validate namespace only when using namespace{} syntax
                    if ($namespaceHunt && !$validatedNamespaceHunt && $token === '{') {
                        $currentNamespace = '';
                    }
                    $classHunt = false;
                    $namespaceHunt = false;
                    $validatedNamespaceHunt = false;
                }
            }
        }
        return $toReturn;
    }
}

class DirectoriesAutoloaderException extends \Exception
{
}
class DirectoriesAutoloader
{
    /**
     * @var \dcp\ClassHunterForPHP5_3
     */
    private $_classHunterStrategy;
    //--- Singleton
    
    /**
     * make constructor private
     *
     * @return \dcp\DirectoriesAutoloader
     */
    private function __construct()
    {
    }
    /**
     *
     * @var DirectoriesAutoloader
     */
    private static $_instance = false;
    /**
     * get singleton for DirectoriesAutoloader
     *
     * @param string $pTmpPath     path to tmp dir
     * @param string $pTmpFileName tmp file name
     *
     * @return \dcp\DirectoriesAutoloader
     */
    public static function instance($pTmpPath, $pTmpFileName = 'directoriesautoloader.cache.php')
    {
        if (self::$_instance === false) {
            self::$_instance = new DirectoriesAutoloader();
            self::$_instance->setCachePath($pTmpPath);
            self::$_instance->setCacheFileName($pTmpFileName);
            self::$_instance->_classHunterStrategy = ClassHunterFactory::create(PHP_VERSION);
        }
        return self::$_instance;
    }
    //--- /Singleton
    
    /**
     * register autoloader against SPL
     *
     * @return boolean
     */
    public function register()
    {
        return spl_autoload_register(array(
            $this,
            'autoload'
        ));
    }
    /**
     * force autoloader to regenerate cache now!
     *
     * @return \dcp\DirectoriesAutoloader
     */
    public function forceRegenerate()
    {
        $this->_canRegenerate = true;
        $this->_classes = null;
        $this->_saveIncache();
        return self::$_instance;
    }
    //--- Cache
    private $_cachePath;
    private $_cacheFileName = 'directoriesautoloader.cache.php';
    /**
     * define the path of tmp directory
     *
     * @param string $pTmp cache directory path
     *
     * @throws \dcp\DirectoriesAutoloaderException
     *
     * @return void
     */
    public function setCachePath($pTmp)
    {
        if (!is_writable($pTmp)) {
            throw new DirectoriesAutoloaderException('Cannot write in given CachePath [' . $pTmp . ']');
        }
        $this->_cachePath = $pTmp;
    }
    /**
     * define the name of tmp file
     *
     * @param string $ptmpFileName tmp file name
     *
     * @return void
     */
    public function setCacheFileName($ptmpFileName)
    {
        $this->_cacheFileName = $ptmpFileName;
    }
    /**
     * get the fully qualified tmp file path
     *
     * @return string
     */
    public function getCacheFilePath()
    {
        return $this->_cachePath . '/' . $this->_cacheFileName;
    }
    //--- /Cache
    //--- custom filters
    private $_customFilterClasses = null;
    /**
     * add an instance of FilterIterator as custom filter
     *
     * @param string $pCustomFilterClass className of customFilter to add
     *
     * @throws \dcp\DirectoriesAutoloaderException
     *
     * @return \dcp\DirectoriesAutoloader
     */
    public function addCustomFilter($pCustomFilterClass)
    {
        $filterClassParents = class_parents($pCustomFilterClass);
        if (isset($filterClassParents['FilterIterator'])) {
            if (!is_array($this->_customFilterClasses)) {
                $this->_customFilterClasses = array(
                    $pCustomFilterClass
                );
                //error_log("adding [$pCustomFilterClass] as custom filter");
                
            } else {
                $this->_customFilterClasses[] = $pCustomFilterClass;
            }
        } else {
            throw new DirectoriesAutoloaderException('Custom filter class [' . $pCustomFilterClass . '] does not inherits from FilterIterator');
        }
        return self::$_instance;
    }
    //--- /custom filters
    //--- Autoload
    
    /**
     * autoloader
     *
     * @param string $pClassName name of the class to load
     *
     * @return boolean
     */
    public function autoload($pClassName)
    {
        //do we already know this class?
        if ($this->_loadClass($pClassName)) {
            return true;
        }
        //If we are allowed to regenerate autoload file, we try again
        if ($this->_canRegenerate) {
            $this->_canRegenerate = false; //avoid loops
            $this->_includesAll();
            $this->addFamilies('FDLGEN');
            $this->_saveInCache();
            return $this->autoload($pClassName);
        }
        //we really found nothing
        return false;
    }
    private $_canRegenerate = true;
    //--- /Autoload
    
    /**
     * look for all classes into registered directories
     *
     * @return void
     */
    private function _includesAll()
    {
        $cwd = getcwd();
        //include known classes
        foreach ($this->_directories as $directory => $recursive) {
            /*
             * Relative directories are handled relative to _cachePath
            */
            $changedCwd = false;
            if (strpos($directory, '/') !== 0) {
                $ret = chdir($this->_cachePath . DIRECTORY_SEPARATOR . $directory);
                if ($ret === false) {
                    continue;
                }
                $changedCwd = true;
            }
            
            $directories = new \AppendIterator();
            //add all paths that we want to browse
            if ($recursive) {
                $directories->append(new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($directory)));
            } else {
                $directories->append(new \DirectoryIterator($directory));
            }
            if (is_array($this->_customFilterClasses)) {
                foreach ($this->_customFilterClasses as $customFilterClass) {
                    //error_log("trying to use [$customFilterClass] as filter");
                    //error_log("directories was a " . get_class($directories));
                    $directories = new $customFilterClass($directories);
                    //error_log("directories is now a " . get_class($directories));
                    
                }
            }
            //restrict files to php ones
            $files = new ExtensionFilterIteratorDecorator($directories);
            $files->setExtension('.php');
            foreach ($files as $fileName) {
                $classes = $this->_classHunterStrategy->find((string)$fileName);
                foreach ($classes as $className => $fileName) {
                    $this->_classes[strtolower($className) ] = $fileName;
                }
            }
            
            if ($changedCwd) {
                chdir($cwd);
            }
        }
        //error_log('included all classes as ' . var_export($this->_classes, true));
        
    }
    /**
     * array of files containing classes indexed by class name
     *
     * @var array $_classes
     */
    private $_classes = array();
    /**
     * write cache file
     *
     * @see DirectoriesAutoloader::_classes
     * @see DirectoriesAutoloader::getCacheFilePath
     *
     * @throws \dcp\DirectoriesAutoloaderException
     *
     * @return void
     */
    private function _saveIncache()
    {
        foreach ($this->_classes as $className => & $fileName) {
            if (substr($fileName, 0, 2) == './') {
                $fileName = substr($fileName, 2);
            }
        }
        unset($fileName);
        $toSave = '<?php' . PHP_EOL;
        $toSave.= '// Cache generated at: ' . date(DATE_W3C) . PHP_EOL;
        $toSave.= '$classes = ' . var_export($this->_classes, true);
        $toSave.= '; ?>';
        //error_log('will save classes');
        if (file_put_contents($this->getCacheFilePath() , $toSave) === false) {
            throw new DirectoriesAutoloaderException('Cannot write cache file ' . $this->getCacheFilePath());
        }
    }
    /**
     * try to load a class
     *
     * @param string $pClassName class name
     *
     * @return boolean
     */
    private function _loadClass($pClassName)
    {
        $className = strtolower($pClassName);
        if (count($this->_classes) === 0) {
            if (is_readable($this->getCacheFilePath())) {
                //error_log('Loading classes from [' . $this->getCacheFilePath() . ']');
                $classes = array();
                require $this->getCacheFilePath(); // load $classes here
                $this->_classes = $classes;
            }
        }
        if (isset($this->_classes[$className])) {
            require_once $this->_classes[$className];
            //error_log('loaded class [' . $className . ']');
            return true;
        }
        return false;
    }
    /**
     * add a directory to autoloaded directories
     *
     * @param string  $pDirectory directory path
     * @param boolean $pRecursive should we recursively scan this directory<commit_msg>Re-generate autoloader cache when cached file does not exists (closes #2508)<commit_after><?php
/*
 * @author Anakeen
 * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License
 * @package FDL
*/

namespace Dcp;

class ExtensionFilterIteratorDecorator extends \FilterIterator
{
    private $_ext;
    /**
     * Check whether the current element of the iterator is acceptable
     *
     * @see FilterIterator::accept()
     *
     * @return bool true if the current element is acceptable, otherwise false.
     *
     */
    public function accept()
    {
        if (substr($this->current() , -1 * strlen($this->_ext)) === $this->_ext) {
            return is_readable($this->current());
        }
        return false;
    }
    /**
     * define which extension is allowed
     *
     * @param string $pExt extension
     *
     * @return void
     */
    public function setExtension($pExt)
    {
        $this->_ext = $pExt;
    }
}
/**
 * Interface for classes in charge of detecting classes declarations in files
 *
 * @author <EMAIL> <<EMAIL>>
 *
 */
interface IClassHunter {
    /**
     * Find all classes in given file
     *
     * @param string $pFileName file name
     *
     * @return array of strings
     *
     */
    public function find($pFileName);
}
/**
 * Class in charge of detecting classes declarations in files (only for PHP 5.3)
 *
 * @author <EMAIL> <<EMAIL>>
 *
 */
class ClassHunterForPHP5_3 implements IClassHunter
{
    /**
     * Find all classes in given file
     *
     * @param string $pFileName file name
     *
     * @return array of strings
     *
     */
    public function find($pFileName)
    {
        $toReturn = array();
        $tokens = token_get_all(file_get_contents($pFileName, false));
        $currentNamespace = '';
        $namespaceHunt = false;
        $validatedNamespaceHunt = false;
        $classHunt = false;
        $whitespaceCount = 0;
        foreach ($tokens as $token) {
            if (is_array($token)) {
                if ($token[0] === T_INTERFACE || $token[0] === T_CLASS) {
                    $classHunt = true;
                    continue;
                } elseif ($token[0] === T_NAMESPACE) {
                    $namespaceHunt = true;
                    continue;
                }
                if ($classHunt && $token[0] === T_STRING) {
                    $toReturn[(strlen($currentNamespace) > 0 ? $currentNamespace . '\\' : '') . $token[1]] = $pFileName;
                    $classHunt = false;
                } elseif ($namespaceHunt && $validatedNamespaceHunt && ($token[0] === T_STRING || $token[0] === T_NS_SEPARATOR)) {
                    $currentNamespace.= $token[1];
                } elseif ($namespaceHunt && !$validatedNamespaceHunt && $token[0] === T_WHITESPACE) {
                    $currentNamespace = '';
                    $validatedNamespaceHunt = true;
                } elseif ($namespaceHunt && !$validatedNamespaceHunt && $token[0] !== T_WHITESPACE) {
                    $namespaceHunt = false;
                }
            } else {
                if ($token === ';' || $token === '{') {
                    //can validate namespace only when using namespace{} syntax
                    if ($namespaceHunt && !$validatedNamespaceHunt && $token === '{') {
                        $currentNamespace = '';
                    }
                    $classHunt = false;
                    $namespaceHunt = false;
                    $validatedNamespaceHunt = false;
                }
            }
        }
        return $toReturn;
    }
}

class DirectoriesAutoloaderException extends \Exception
{
}
class DirectoriesAutoloader
{
    /**
     * @var \dcp\ClassHunterForPHP5_3
     */
    private $_classHunterStrategy;
    private $_lockfd = false;
    //--- Singleton
    
    /**
     * make constructor private
     *
     * @return \dcp\DirectoriesAutoloader
     */
    private function __construct()
    {
    }
    /**
     *
     * @var DirectoriesAutoloader
     */
    private static $_instance = false;
    /**
     * get singleton for DirectoriesAutoloader
     *
     * @param string $pTmpPath     path to tmp dir
     * @param string $pTmpFileName tmp file name
     *
     * @return \dcp\DirectoriesAutoloader
     */
    public static function instance($pTmpPath, $pTmpFileName = 'directoriesautoloader.cache.php')
    {
        if (self::$_instance === false) {
            self::$_instance = new DirectoriesAutoloader();
            self::$_instance->setCachePath($pTmpPath);
            self::$_instance->setCacheFileName($pTmpFileName);
            self::$_instance->_classHunterStrategy = ClassHunterFactory::create(PHP_VERSION);
        }
        return self::$_instance;
    }
    //--- /Singleton
    
    /**
     * register autoloader against SPL
     *
     * @return boolean
     */
    public function register()
    {
        return spl_autoload_register(array(
            $this,
            'autoload'
        ));
    }
    /**
     * force autoloader to regenerate cache now!
     *
     * @return \dcp\DirectoriesAutoloader
     */
    public function forceRegenerate()
    {
        $this->_canRegenerate = true;
        $this->_classes = array();
        $this->_saveIncache();
        return self::$_instance;
    }
    //--- Cache
    private $_cachePath;
    private $_cacheFileName = 'directoriesautoloader.cache.php';
    /**
     * define the path of tmp directory
     *
     * @param string $pTmp cache directory path
     *
     * @throws \dcp\DirectoriesAutoloaderException
     *
     * @return void
     */
    public function setCachePath($pTmp)
    {
        if (!is_writable($pTmp)) {
            throw new DirectoriesAutoloaderException('Cannot write in given CachePath [' . $pTmp . ']');
        }
        $this->_cachePath = $pTmp;
    }
    /**
     * define the name of tmp file
     *
     * @param string $ptmpFileName tmp file name
     *
     * @return void
     */
    public function setCacheFileName($ptmpFileName)
    {
        $this->_cacheFileName = $ptmpFileName;
    }
    /**
     * get the fully qualified tmp file path
     *
     * @return string
     */
    public function getCacheFilePath()
    {
        return $this->_cachePath . '/' . $this->_cacheFileName;
    }
    /**
     * get the cache lock file
     *
     * @return string
     */
    private function _getCacheLockFilePath()
    {
        return $this->getCacheFilePath() . '.lock';
    }
    //--- /Cache
    //--- custom filters
    private $_customFilterClasses = null;
    /**
     * add an instance of FilterIterator as custom filter
     *
     * @param string $pCustomFilterClass className of customFilter to add
     *
     * @throws \dcp\DirectoriesAutoloaderException
     *
     * @return \dcp\DirectoriesAutoloader
     */
    public function addCustomFilter($pCustomFilterClass)
    {
        $filterClassParents = class_parents($pCustomFilterClass);
        if (isset($filterClassParents['FilterIterator'])) {
            if (!is_array($this->_customFilterClasses)) {
                $this->_customFilterClasses = array(
                    $pCustomFilterClass
                );
                //error_log("adding [$pCustomFilterClass] as custom filter");
                
            } else {
                $this->_customFilterClasses[] = $pCustomFilterClass;
            }
        } else {
            throw new DirectoriesAutoloaderException('Custom filter class [' . $pCustomFilterClass . '] does not inherits from FilterIterator');
        }
        return self::$_instance;
    }
    //--- /custom filters
    //--- Autoload
    
    /**
     * @throws DirectoriesAutoloaderException
     */
    private function _lock()
    {
        if ($this->_lockfd !== false) {
            throw new DirectoriesAutoloaderException(sprintf("Cache lock is already opened."));
        }
        $lockfile = $this->_getCacheLockFilePath();
        $this->_lockfd = fopen($lockfile, 'a');
        if ($this->_lockfd === false) {
            throw new DirectoriesAutoloaderException(sprintf("Error opening cache lock file '%s'.", $lockfile));
        }
        if (flock($this->_lockfd, LOCK_EX) === false) {
            throw new DirectoriesAutoloaderException(sprintf("Error locking cache lock file '%s'.", $lockfile));
        }
    }
    /**
     * @throws DirectoriesAutoloaderException
     */
    private function _unlock()
    {
        if ($this->_lockfd === false) {
            throw new DirectoriesAutoloaderException(sprintf("Cache lock not opened."));
        }
        if (flock($this->_lockfd, LOCK_UN) === false) {
            fclose($this->_lockfd);
            $this->_lockfd = false;
            throw new DirectoriesAutoloaderException(sprintf("Error unlocking cache lock."));
        }
        fclose($this->_lockfd);
        $this->_lockfd = false;
    }
    /**
     * @param $pClassName
     * @return bool
     * @throws \Exception
     */
    private function _regenerate($pClassName)
    {
        $this->_lock();
        /*
         * Re-check the cache in case of a concurrent process
         * which might have already re-built the cache.
        */
        $this->_classes = array();
        if ($this->_loadClass($pClassName)) {
            $this->_unlock();
            return true;
        }
        /*
         * Re-build the cache
        */
        try {
            $this->_includesAll();
            $this->addFamilies('FDLGEN');
            $this->_saveInCache();
        }
        catch(\Exception $e) {
            $this->_unlock();
            throw $e;
        }
        $this->_unlock();
    }
    /**
     * autoloader
     *
     * @param string $pClassName name of the class to load
     *
     * @return boolean
     */
    public function autoload($pClassName)
    {
        //do we already know this class?
        if ($this->_loadClass($pClassName)) {
            return true;
        }
        //If we are allowed to regenerate autoload file, we try again
        if ($this->_canRegenerate) {
            $this->_regenerate($pClassName);
            $this->_canRegenerate = false; //avoid loops
            return $this->autoload($pClassName);
        }
        //we really found nothing
        return false;
    }
    private $_canRegenerate = true;
    //--- /Autoload
    
    /**
     * look for all classes into registered directories
     *
     * @return void
     */
    private function _includesAll()
    {
        $cwd = getcwd();
        //include known classes
        foreach ($this->_directories as $directory => $recursive) {
            /*
             * Relative directories are handled relative to _cachePath
            */
            $changedCwd = false;
            if (strpos($directory, '/') !== 0) {
                $ret = chdir($this->_cachePath . DIRECTORY_SEPARATOR . $directory);
                if ($ret === false) {
                    continue;
                }
                $changedCwd = true;
            }
            
            $directories = new \AppendIterator();
            //add all paths that we want to browse
            if ($recursive) {
                $directories->append(new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($directory)));
            } else {
                $directories->append(new \DirectoryIterator($directory));
            }
            if (is_array($this->_customFilterClasses)) {
                foreach ($this->_customFilterClasses as $customFilterClass) {
                    //error_log("trying to use [$customFilterClass] as filter");
                    //error_log("directories was a " . get_class($directories));
                    $directories = new $customFilterClass($directories);
                    //error_log("directories is now a " . get_class($directories));
                    
                }
            }
            //restrict files to php ones
            $files = new ExtensionFilterIteratorDecorator($directories);
            $files->setExtension('.php');
            foreach ($files as $fileName) {
                $classes = $this->_classHunterStrategy->find((string)$fileName);
                foreach ($classes as $className => $fileName) {
                    $this->_classes[strtolower($className) ] = $fileName;
                }
            }
            
            if ($changedCwd) {
                chdir($cwd);
            }
        }
        //error_log('included all classes as ' . var_export($this->_classes, true));
        
    }
    /**
     * array of files containing classes indexed by class name
     *
     * @var array $_classes
     */
    private $_classes = array();
    /**
     * write cache file
     *
     * @see DirectoriesAutoloader::_classes
     * @see DirectoriesAutoloader::getCacheFilePath
     *
     * @throws \dcp\DirectoriesAutoloaderException
     *
     * @return void
     */
    private function _saveIncache()
    {
        foreach ($this->_classes as $className => & $fileName) {
            if (substr($fileName, 0, 2) == './') {
                $fileName = substr($fileName, 2);
            }
        }
        unset($fileName);
        
        $toSave = '<?php' . PHP_EOL;
        $toSave.= '// Cache generated at: ' . date(DATE_W3C) . PHP_EOL;
        $toSave.= '$classes = ' . var_export($this->_classes, true);
        $toSave.= '; ?>';
        /*
         * Atomically write .autoloader.cache file
        */
        $cacheFileName = $this->getCacheFilePath();
        $cacheDirName = dirname($cacheFileName);
        $tmpFile = tempnam($cacheDirName, '.autoloader.cache.tmp');
        if ($tmpFile === false) {
            throw new DirectoriesAutoloaderException("Error creating temporary autoloader cache file.");
        }
        if (file_put_contents($tmpFile, $toSave) === false) {
            unlink($tmpFile);
            throw new DirectoriesAutoloaderException(sprintf("Error writing cache content to temporary file '%s'", $tmpFile));
        }
        if (rename($tmpFile, $cacheFileName) === false) {
            unlink($tmpFile);
            throw new DirectoriesAutoloaderException(sprintf("Error renaming temporary cache file '%s' to '%s'.", $tmpFile, $cacheFileName));
        }
        chmod($cacheFileName, (0666 ^ umask()));
    }
    /**
     * try to load a class
     *
     * @param string $pClassName class name
     *
     * @return boolean
     */
    private function _loadClass($pClassName)
    {
        $className = strtolower($pClassName);
        if (count($this->_classes) === 0) {
            if (is_readable($this->getCacheFilePath())) {
                //error_log('Loading classes from [' . $this->getCacheFilePath() . ']');
                $classes = array();
                require $this->getCacheFilePath(); // load $classes here
                $this->_classes = $classes;
            }
        }
        if (isset($this->_classes[$className])) {
            include_once('WHAT/Lib.Prefix.php');
            if (!file_exists(DEFAULT_PUBDIR . DIRECTORY_SEPARATOR . $this->_classes[$className])) {
                return false;
            }
            require_once $this->_classes[$className];
            if (!class_exists($className, false)) {
                return false;
            }
            return true;
        }
        return false;
    }
    /**
     * add a directory to autoloaded directories
     *
     * @param string  $pDirectory directory path
     * @param boolean $pRecursive should we recursively scan this directory
{'fraction_non_alphanumeric': 0.12169368982426, 'fraction_numerical': 0.002219120226207094, 'mean_word_length': 1.781483325037332, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 138, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2194624', 'n_tokens_mistral': 7735, 'n_tokens_neox': 7580, 'n_words': 1749}