<commit_before><?php
    /*********************************************************************************
     * Zurmo is a customer relationship management program developed by
     * Zurmo, Inc. Copyright (C) 2013 Zurmo Inc.
     *
     * Zurmo is free software; you can redistribute it and/or modify it under
     * the terms of the GNU General Public License version 3 as published by the
     * Free Software Foundation with the addition of the following permission added
     * to Section 15 as permitted in Section 7(a): FOR ANY PART OF THE COVERED WORK
     * IN WHICH THE COPYRIGHT IS OWNED BY ZURMO, ZURMO DISCLAIMS THE WARRANTY
     * OF NON INFRINGEMENT OF THIRD PARTY RIGHTS.
     *
     * Zurmo is distributed in the hope that it will be useful, but WITHOUT
     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
     * details.
     *
     * You should have received a copy of the GNU General Public License along with
     * this program; if not, see http://www.gnu.org/licenses or write to the Free
     * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
     * 02110-1301 USA.
     *
     * You can contact Zurmo, Inc. with a mailing address at 27 North Wacker Drive
     * Suite 370 Chicago, IL 60606. or at email address <EMAIL>.
     *
     * The interactive user interfaces in original and modified versions
     * of this program must display Appropriate Legal Notices, as required under
     * Section 5 of the GNU General Public License version 3.
     *
     * In accordance with Section 7(b) of the GNU General Public License version 3,
     * these Appropriate Legal Notices must retain the display of the Zurmo
     * logo and Zurmo copyright notice. If the display of the logo is not reasonably
     * feasible for technical reasons, the Appropriate Legal Notices must display the words
     * "Copyright Zurmo Inc. 2013. All rights reserved".
     ********************************************************************************/

    /**
     * Helper functionality for aggregating the messages
     * from the framework and all installed modules.
     */
    class MessageUtil
    {
        /**
         * Returns the messages made by aggregating in order
         * the messages found int the framework, then in each
         * of the dependent modules up to then including
         * the module for the currently controller.
         */
        public static function getMessages($language, $moduleName = null, $category = null)
        {
            assert('is_string($language)');
            assert('strlen($language) == 2 || strlen($language) == 5');
            if ($moduleName === null)
            {
                $directories = self::getMessageDirectoriesForAllModules();
            }
            elseif ($moduleName != 'framework')
            {
                $directories = self::getMessageDirectoriesForNamedModuleAndDependencies($moduleName);
            }
            else
            {
                $directories = array();
            }
            array_unshift($directories, 'core');
            $messages = array();
            foreach ($directories as $directory)
            {
                $messagesDirectoryName = COMMON_ROOT . "/protected/$directory/messages/$language";
                if (is_dir($messagesDirectoryName))
                {
                    $fileNames = scandir($messagesDirectoryName);
                    foreach ($fileNames as $fileName)
                    {
                        if (pathinfo($fileName, PATHINFO_EXTENSION) == 'php')
                        {
                            if ($category === null || pathinfo($fileName, PATHINFO_FILENAME) == $category)
                            {
                                $fileName = "$messagesDirectoryName/$fileName";
                                if (file_exists($fileName))
                                {
                                    $messages = array_merge($messages, require($fileName));
                                }
                            }
                        }
                    }
                }
            }
            return $messages;
        }

        private static function getMessageDirectoriesForCurrentModuleAndDependencies()
        {
            if (Yii::app()->controller instanceof CController)
            {
                $currentModule = Yii::app()->controller->getModule();
                return self::getMessageDirectoriesForModuleAndDependencies($currentModule);
            }
            return array();
        }

        private static function getMessageDirectoriesForNamedModuleAndDependencies($moduleName)
        {
            $module = Yii::app()->findModule($moduleName);
            assert('$module instanceof Module');
            return self::getMessageDirectoriesForModuleAndDependencies($module);
        }

        private static function getMessageDirectoriesForModuleAndDependencies(Module $module)
        {
            $dependencies = Module::getDependenciesForModule($module);
            $directories = array();
            foreach ($dependencies as $dependency)
            {
                $directories[] = "modules/$dependency";
            }
            return $directories;
        }

        /**
         * Ignores nested modules. Assumes nested modules have their messages in their parent module.
         * @return all available module message directories
         */
        private static function getMessageDirectoriesForAllModules()
        {
             $modules = Module::getModuleObjects();
             foreach ($modules as $module)
             {
                if (get_class($module->getParentModule()) == 'MessageUtil')
                {
                    $directories[] = 'modules/' . $module->getDirectoryName();
                }
             }
             return $directories;
        }
    }

    function getLanguagesMessages($basePath, $moduleName, $language)
    {
        $messagesDirectoryName = "$basePath/protected/modules/$moduleName/messages/$language";
        $messageFileName = "$messagesDirectoryName/Default.php";
        if (file_exists($messageFileName))
        {
            return require($messageFileName);
        }
        else
        {
            return array();
        }
    }

    function checkLanguagesMessageFilesContainAllTheSameValidMessages($messagesDirectoryName, $moduleName)
    {
        assert('is_string($messagesDirectoryName)');
        assert('is_dir   ($messagesDirectoryName)');
        $problems = array();
        $languages = getLanguages($messagesDirectoryName);
        if (count($languages) > 0)
        {
            $languagesToMessageFiles = array();
            foreach ($languages as $language)
            {
                $languagesToMessageFiles[$language] = array();
                $languageMessagesDirectoryName = "$messagesDirectoryName/$language";
                $languagesToMessageFiles[$language]['messageDirectory'] = $languageMessagesDirectoryName;
                $entries = scandir($languageMessagesDirectoryName);
                foreach ($entries as $entry)
                {
                    $fullEntryName = "$languageMessagesDirectoryName/$entry";
                    if (is_file($fullEntryName) &&
                        pathinfo($entry, PATHINFO_EXTENSION) == 'php')
                    {
                        $languagesToMessageFiles[$language][] = $entry;
                    }
                }
            }
            for ($i = 0; $i < count($languages); $i++)
            {
                $problems = array_merge($problems,
                                        checkLanguagesMessageFilesFormat($languagesToMessageFiles[$languages[$i]],
                                                                         $messagesDirectoryName));
            }
            // Compares the first language to the others.
            for ($i = 1; $i < count($languages); $i++)
            {
                $problems = array_merge($problems,
                                        diffLanguagesMessageFiles($languagesToMessageFiles[$languages[0]],
                                                                  $languagesToMessageFiles[$languages[$i]],
                                                                  $messagesDirectoryName,
                                                                  $moduleName));
            }
        }
        else
        {
            $problems[] = 'No languages were found in the following directory:' . $messagesDirectoryName . "\n";
        }
        return $problems;
    }

    function checkFirstLanguagesMessageFilesMessagesReallyExistInSource($messagesDirectoryName, $moduleName)
    {
        assert('is_string($messagesDirectoryName)');
        assert('is_dir   ($messagesDirectoryName)');
        $problems = array();
        $fileNamesToCategoriesToMessages = findFileNameToCategoryToMessage($messagesDirectoryName . '/..');
        $categoriesToMessagesToFileNames = convertFileNameToCategoryToMessageToCategoryToMessageToFileName($fileNamesToCategoriesToMessages);
        unset($categoriesToMessagesToFileNames['yii']);
        $languages = getLanguages($messagesDirectoryName);
        if (count($languages) > 0)
        {
            $firstLanguage = $languages[0];
            foreach ($categoriesToMessagesToFileNames as $category => $messagesToFileNames)
            {
                $messageFileName = "$messagesDirectoryName/$firstLanguage/$category.php";
                if (is_file($messageFileName))
                {
                    $messagesInMessageFile = require($messageFileName);
                    if (!is_array($messagesInMessageFile))
                    {
                        $problems[] = "$messageFileName is not a valid message file.\n";
                        continue;
                    }
                    $messagesInMessageFile = array_keys($messagesInMessageFile);
                }
                else
                {
                    continue;
                }
                $messagesInSourceFiles = array_keys($messagesToFileNames);
                $messagesNotInSourceFiles = array_diff($messagesInMessageFile, $messagesInSourceFiles);
                foreach ($messagesNotInSourceFiles as $message)
                {
                    if (strpos($message, 'ModulePluralLabel') === false &&
                       strpos($message, 'ModuleSingularLabel') === false &&
                       strpos($message, 'ModulePluralLowerCaseLabel') === false &&
                       strpos($message, 'ModuleSingularLowerCaseLabel') === false )
                    {
                        //There is no current way to resolve this type of issue. It is possible a label has been pushed
                        //back into framework for example, even though there is no mention of the label in framework.  This
                        //is so it is not duplicated in various modules. This is turned off for now from the message checker.
                        //$problems[] = "(This might be ok) '$message' in $firstLanguage/$category.php not in any source file in $moduleName.";
                    }
                }
            }
        }
        else
        {
            $problems[] = 'No languages were found in the following directory:' . $messagesDirectoryName . "\n";
        }
        return $problems;
    }

    function checkFirstLanguagesMessageFilesContainAllExistingMessages($messagesDirectoryName, $moduleName, $basePath)
    {
        assert('is_string($messagesDirectoryName)');
        assert('is_dir   ($messagesDirectoryName)');
        $problems = array();
        $fileNamesToCategoriesToMessages = findFileNameToCategoryToMessage($messagesDirectoryName . '/..');
        $categoriesToMessagesToFileNames = convertFileNameToCategoryToMessageToCategoryToMessageToFileName($fileNamesToCategoriesToMessages);
        unset($categoriesToMessagesToFileNames['yii']);
        $languages = getLanguages($messagesDirectoryName);
        if (count($languages) > 0)
        {
            $firstLanguage = $languages[0];
            foreach ($categoriesToMessagesToFileNames as $category => $messagesToFileNames)
            {
                $existingMessages   = array_keys(MessageUtil::getMessages($firstLanguage, $moduleName, $category));
                $duplicateCount = count($existingMessages) != count(array_unique($existingMessages));
                if ($duplicateCount != 0)
                {
                    $problems[] = "$moduleName and its dependencies contain $duplicateCount duplicate entries.";
                }
                $yiiMessageFileName = "$messagesDirectoryName/$firstLanguage/$category.php";
                $yiiMessages        = require("$basePath/../../yii/framework/messages/$firstLanguage/yii.php");
                foreach ($messagesToFileNames as $message => $fileNames)
                {
                    $fileNames = join(', ', $fileNames);
                    if (!in_array($message, $existingMessages) &&
                        !in_array($message, $yiiMessages))
                    {
                        $problems[] = "'$message' in $fileNames not in $firstLanguage/$category.php in $moduleName or its dependencies.";
                    }
                    elseif (in_array($message, $existingMessages) &&
                        in_array($message, $yiiMessages))
                    {
                        $problems[] = "'$message' in $fileNames in $firstLanguage/$category.php in $moduleName or its dependencies is a duplicate of definition already in Yii.";
                    }
                }
            }
        }
        else
        {
            $problems[] = 'No languages were found in the following directory:' . $messagesDirectoryName . "\n";
        }
        return $problems;
    }

    function checkForYiiTCallsThatAreNotHowThisScriptExpectsThemToBe($messagesDirectoryName, $moduleName)
    {
        assert('is_string($messagesDirectoryName)');
        assert('is_dir   ($messagesDirectoryName)');
        $problems = array();
        $fileNamesToUnexpectedlyFormattedYiiTs = findFileNameToUnexpectedlyFormattedYiiT($messagesDirectoryName . '/..');
        foreach ($fileNamesToUnexpectedlyFormattedYiiTs as $fileName => $unexpectedlyFormattedYiiTs)
        {
            foreach ($unexpectedlyFormattedYiiTs as $code)
            {
                $problems[] = "$code in $fileName in $moduleName.";
            }
        }
        return $problems;
    }

    function findFileNameToCategoryToMessage($path, $forcedCategory = 'Default')
    {
        assert('is_string($path)');
        assert('is_dir   ($path)');
        $fileNamesToCategoriesToMessages = array();
        $f = opendir($path);
        assert('$f !== false');
        while ($entry = readdir($f))<commit_msg>Activated the right Contexts during message export<commit_after><?php
    /*********************************************************************************
     * Zurmo is a customer relationship management program developed by
     * Zurmo, Inc. Copyright (C) 2013 Zurmo Inc.
     *
     * Zurmo is free software; you can redistribute it and/or modify it under
     * the terms of the GNU General Public License version 3 as published by the
     * Free Software Foundation with the addition of the following permission added
     * to Section 15 as permitted in Section 7(a): FOR ANY PART OF THE COVERED WORK
     * IN WHICH THE COPYRIGHT IS OWNED BY ZURMO, ZURMO DISCLAIMS THE WARRANTY
     * OF NON INFRINGEMENT OF THIRD PARTY RIGHTS.
     *
     * Zurmo is distributed in the hope that it will be useful, but WITHOUT
     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
     * details.
     *
     * You should have received a copy of the GNU General Public License along with
     * this program; if not, see http://www.gnu.org/licenses or write to the Free
     * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
     * 02110-1301 USA.
     *
     * You can contact Zurmo, Inc. with a mailing address at 27 North Wacker Drive
     * Suite 370 Chicago, IL 60606. or at email address <EMAIL>.
     *
     * The interactive user interfaces in original and modified versions
     * of this program must display Appropriate Legal Notices, as required under
     * Section 5 of the GNU General Public License version 3.
     *
     * In accordance with Section 7(b) of the GNU General Public License version 3,
     * these Appropriate Legal Notices must retain the display of the Zurmo
     * logo and Zurmo copyright notice. If the display of the logo is not reasonably
     * feasible for technical reasons, the Appropriate Legal Notices must display the words
     * "Copyright Zurmo Inc. 2013. All rights reserved".
     ********************************************************************************/

    /**
     * Helper functionality for aggregating the messages
     * from the framework and all installed modules.
     */
    class MessageUtil
    {
        /**
         * Returns the messages made by aggregating in order
         * the messages found int the framework, then in each
         * of the dependent modules up to then including
         * the module for the currently controller.
         */
        public static function getMessages($language, $moduleName = null, $category = null)
        {
            assert('is_string($language)');
            assert('strlen($language) == 2 || strlen($language) == 5');
            if ($moduleName === null)
            {
                $directories = self::getMessageDirectoriesForAllModules();
            }
            elseif ($moduleName != 'framework')
            {
                $directories = self::getMessageDirectoriesForNamedModuleAndDependencies($moduleName);
            }
            else
            {
                $directories = array();
            }
            array_unshift($directories, 'core');
            $messages = array();
            foreach ($directories as $directory)
            {
                $messagesDirectoryName = COMMON_ROOT . "/protected/$directory/messages/$language";
                if (is_dir($messagesDirectoryName))
                {
                    $fileNames = scandir($messagesDirectoryName);
                    foreach ($fileNames as $fileName)
                    {
                        if (pathinfo($fileName, PATHINFO_EXTENSION) == 'php')
                        {
                            if ($category === null || pathinfo($fileName, PATHINFO_FILENAME) == $category)
                            {
                                $fileName = "$messagesDirectoryName/$fileName";
                                if (file_exists($fileName))
                                {
                                    $messages = array_merge($messages, require($fileName));
                                }
                            }
                        }
                    }
                }
            }
            return $messages;
        }

        private static function getMessageDirectoriesForCurrentModuleAndDependencies()
        {
            if (Yii::app()->controller instanceof CController)
            {
                $currentModule = Yii::app()->controller->getModule();
                return self::getMessageDirectoriesForModuleAndDependencies($currentModule);
            }
            return array();
        }

        private static function getMessageDirectoriesForNamedModuleAndDependencies($moduleName)
        {
            $module = Yii::app()->findModule($moduleName);
            assert('$module instanceof Module');
            return self::getMessageDirectoriesForModuleAndDependencies($module);
        }

        private static function getMessageDirectoriesForModuleAndDependencies(Module $module)
        {
            $dependencies = Module::getDependenciesForModule($module);
            $directories = array();
            foreach ($dependencies as $dependency)
            {
                $directories[] = "modules/$dependency";
            }
            return $directories;
        }

        /**
         * Ignores nested modules. Assumes nested modules have their messages in their parent module.
         * @return all available module message directories
         */
        private static function getMessageDirectoriesForAllModules()
        {
             $modules = Module::getModuleObjects();
             foreach ($modules as $module)
             {
                if (get_class($module->getParentModule()) == 'MessageUtil')
                {
                    $directories[] = 'modules/' . $module->getDirectoryName();
                }
             }
             return $directories;
        }
    }

    function getLanguagesMessages($basePath, $moduleName, $language)
    {
        $messagesDirectoryName = "$basePath/protected/modules/$moduleName/messages/$language";
        $messageFileName = "$messagesDirectoryName/Default.php";
        if (file_exists($messageFileName))
        {
            return require($messageFileName);
        }
        else
        {
            return array();
        }
    }

    function checkLanguagesMessageFilesContainAllTheSameValidMessages($messagesDirectoryName, $moduleName)
    {
        assert('is_string($messagesDirectoryName)');
        assert('is_dir   ($messagesDirectoryName)');
        $problems = array();
        $languages = getLanguages($messagesDirectoryName);
        if (count($languages) > 0)
        {
            $languagesToMessageFiles = array();
            foreach ($languages as $language)
            {
                $languagesToMessageFiles[$language] = array();
                $languageMessagesDirectoryName = "$messagesDirectoryName/$language";
                $languagesToMessageFiles[$language]['messageDirectory'] = $languageMessagesDirectoryName;
                $entries = scandir($languageMessagesDirectoryName);
                foreach ($entries as $entry)
                {
                    $fullEntryName = "$languageMessagesDirectoryName/$entry";
                    if (is_file($fullEntryName) &&
                        pathinfo($entry, PATHINFO_EXTENSION) == 'php')
                    {
                        $languagesToMessageFiles[$language][] = $entry;
                    }
                }
            }
            for ($i = 0; $i < count($languages); $i++)
            {
                $problems = array_merge($problems,
                                        checkLanguagesMessageFilesFormat($languagesToMessageFiles[$languages[$i]],
                                                                         $messagesDirectoryName));
            }
            // Compares the first language to the others.
            for ($i = 1; $i < count($languages); $i++)
            {
                $problems = array_merge($problems,
                                        diffLanguagesMessageFiles($languagesToMessageFiles[$languages[0]],
                                                                  $languagesToMessageFiles[$languages[$i]],
                                                                  $messagesDirectoryName,
                                                                  $moduleName));
            }
        }
        else
        {
            $problems[] = 'No languages were found in the following directory:' . $messagesDirectoryName . "\n";
        }
        return $problems;
    }

    function checkFirstLanguagesMessageFilesMessagesReallyExistInSource($messagesDirectoryName, $moduleName)
    {
        assert('is_string($messagesDirectoryName)');
        assert('is_dir   ($messagesDirectoryName)');
        $problems = array();
        $fileNamesToCategoriesToMessages = findFileNameToCategoryToMessage($messagesDirectoryName . '/..');
        $categoriesToMessagesToFileNames = convertFileNameToCategoryToMessageToCategoryToMessageToFileName($fileNamesToCategoriesToMessages);
        unset($categoriesToMessagesToFileNames['yii']);
        $languages = getLanguages($messagesDirectoryName);
        if (count($languages) > 0)
        {
            $firstLanguage = $languages[0];
            foreach ($categoriesToMessagesToFileNames as $category => $messagesToFileNames)
            {
                $messageFileName = "$messagesDirectoryName/$firstLanguage/$category.php";
                if (is_file($messageFileName))
                {
                    $messagesInMessageFile = require($messageFileName);
                    if (!is_array($messagesInMessageFile))
                    {
                        $problems[] = "$messageFileName is not a valid message file.\n";
                        continue;
                    }
                    $messagesInMessageFile = array_keys($messagesInMessageFile);
                }
                else
                {
                    continue;
                }
                $messagesInSourceFiles = array_keys($messagesToFileNames);
                $messagesNotInSourceFiles = array_diff($messagesInMessageFile, $messagesInSourceFiles);
                foreach ($messagesNotInSourceFiles as $message)
                {
                    if (strpos($message, 'ModulePluralLabel') === false &&
                       strpos($message, 'ModuleSingularLabel') === false &&
                       strpos($message, 'ModulePluralLowerCaseLabel') === false &&
                       strpos($message, 'ModuleSingularLowerCaseLabel') === false )
                    {
                        //There is no current way to resolve this type of issue. It is possible a label has been pushed
                        //back into framework for example, even though there is no mention of the label in framework.  This
                        //is so it is not duplicated in various modules. This is turned off for now from the message checker.
                        //$problems[] = "(This might be ok) '$message' in $firstLanguage/$category.php not in any source file in $moduleName.";
                    }
                }
            }
        }
        else
        {
            $problems[] = 'No languages were found in the following directory:' . $messagesDirectoryName . "\n";
        }
        return $problems;
    }

    function checkFirstLanguagesMessageFilesContainAllExistingMessages($messagesDirectoryName, $moduleName, $basePath)
    {
        assert('is_string($messagesDirectoryName)');
        assert('is_dir   ($messagesDirectoryName)');
        $problems = array();
        $fileNamesToCategoriesToMessages = findFileNameToCategoryToMessage($messagesDirectoryName . '/..');
        $categoriesToMessagesToFileNames = convertFileNameToCategoryToMessageToCategoryToMessageToFileName($fileNamesToCategoriesToMessages);
        unset($categoriesToMessagesToFileNames['yii']);
        $languages = getLanguages($messagesDirectoryName);
        if (count($languages) > 0)
        {
            $firstLanguage = $languages[0];
            foreach ($categoriesToMessagesToFileNames as $category => $messagesToFileNames)
            {
                $existingMessages   = array_keys(MessageUtil::getMessages($firstLanguage, $moduleName, $category));
                $duplicateCount = count($existingMessages) != count(array_unique($existingMessages));
                if ($duplicateCount != 0)
                {
                    $problems[] = "$moduleName and its dependencies contain $duplicateCount duplicate entries.";
                }
                $yiiMessageFileName = "$messagesDirectoryName/$firstLanguage/$category.php";
                $yiiMessages        = require("$basePath/../../yii/framework/messages/$firstLanguage/yii.php");
                foreach ($messagesToFileNames as $message => $fileNames)
                {
                    $fileNames = join(', ', $fileNames);
                    if (!in_array($message, $existingMessages) &&
                        !in_array($message, $yiiMessages))
                    {
                        $problems[] = "'$message' in $fileNames not in $firstLanguage/$category.php in $moduleName or its dependencies.";
                    }
                    elseif (in_array($message, $existingMessages) &&
                        in_array($message, $yiiMessages))
                    {
                        $problems[] = "'$message' in $fileNames in $firstLanguage/$category.php in $moduleName or its dependencies is a duplicate of definition already in Yii.";
                    }
                }
            }
        }
        else
        {
            $problems[] = 'No languages were found in the following directory:' . $messagesDirectoryName . "\n";
        }
        return $problems;
    }

    function checkForYiiTCallsThatAreNotHowThisScriptExpectsThemToBe($messagesDirectoryName, $moduleName)
    {
        assert('is_string($messagesDirectoryName)');
        assert('is_dir   ($messagesDirectoryName)');
        $problems = array();
        $fileNamesToUnexpectedlyFormattedYiiTs = findFileNameToUnexpectedlyFormattedYiiT($messagesDirectoryName . '/..');
        foreach ($fileNamesToUnexpectedlyFormattedYiiTs as $fileName => $unexpectedlyFormattedYiiTs)
        {
            foreach ($unexpectedlyFormattedYiiTs as $code)
            {
                $problems[] = "$code in $fileName in $moduleName.";
            }
        }
        return $problems;
    }

    function findFileNameToCategoryToMessage($path, $forcedCategory = '')
    {
        assert('is_string($path)');
        assert('is_dir   ($path)');
        $fileNamesToCategoriesToMessages = array();
        $f = opendir($path);
        assert('$f !== false');
        while ($entry = readdir($f))
{'fraction_non_alphanumeric': 0.10113408939292862, 'fraction_numerical': 0.0032021347565043363, 'mean_word_length': 1.8426092727789893, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5817905', 'n_tokens_mistral': 7011, 'n_tokens_neox': 6624, 'n_words': 1738}