<commit_before><?php

namespace Concrete\Core\Multilingual\Service;
use Concrete\Attribute\Select\Option as SelectAttributeOption;
use Concrete\Core\Attribute\Key\Key as AttributeKey;
use Concrete\Core\Attribute\Set as AttributeSet;
use Concrete\Core\Attribute\Type as AttributeType;
use Concrete\Core\Job\Set as JobSet;
use Concrete\Core\Multilingual\Page\Section\Section;
use Concrete\Core\Permission\Access\Entity\Type as PermissionAccessEntityType;
use Concrete\Core\Permission\Key\Key as PermissionKey;
use Concrete\Core\User\Group\Group;
use Concrete\Core\User\Group\GroupSet;
use Gettext\Extractors\Mo;
use Gettext\Extractors\Po;
use Gettext\Translations;
use Gettext\Extractors\PhpCode;
use Gettext\Generators\Po as PoGenerator;
use Gettext\Generators\Mo as MoGenerator;

defined('C5_EXECUTE') or die("Access Denied.");

class Extractor
{
    public function __construct()
    {
        // register concrete5 translator functions
        PhpCode::$functions['t'] = '__';
        PhpCode::$functions['t2'] = 'n__';
        PhpCode::$functions['tc'] = 'p__';
    }

    /**
     * return \GetText\Translations $translations;
     */
    public function extractTranslatableSiteStrings()
    {
        // first, we figure out which files we're going to operate on.
        $directories = array(
            DIR_APPLICATION . '/' . DIRNAME_BLOCKS,
            DIR_APPLICATION . '/' . DIRNAME_ELEMENTS,
            DIR_APPLICATION . '/' . DIRNAME_CONTROLLERS,
            DIR_APPLICATION . '/' . DIRNAME_MAIL_TEMPLATES,
            DIR_APPLICATION . '/' . DIRNAME_PAGE_TYPES,
            DIR_APPLICATION . '/' . DIRNAME_PAGES,
            DIR_APPLICATION . '/' . DIRNAME_THEMES,
            DIR_APPLICATION . '/' . DIRNAME_VIEWS
        );

        $files = array();
        foreach($directories as $directory) {
            $directoryIterator = new \RecursiveDirectoryIterator($directory);
            $iterator = new \RecursiveIteratorIterator($directoryIterator);
            $results = new \RegexIterator($iterator, '/^.+\.php$/i', \RecursiveRegexIterator::GET_MATCH);
            foreach($results as $result) {
                $files[] = $result[0];
            }
        }

        $translations = new Translations();
        foreach($files as $file) {
            $fileTranslations = Translations::fromPhpCodeFile($file);
            $translations->mergeWith($fileTranslations);
        }

        // Now, we grab dynamic content that's part of our site that we translate dynamically
        $dynamicTranslations = $this->getDynamicTranslations();
        $translations->mergeWith($dynamicTranslations);

        return $translations;
    }

    public function getDynamicTranslations()
    {
        $translations = new Translations();
        $translations->mergeWith(AttributeSet::exportTranslations());
        $translations->mergeWith(AttributeKey::exportTranslations());
        $translations->mergeWith(AttributeType::exportTranslations());<commit_msg>Fix #1695<commit_after><?php

namespace Concrete\Core\Multilingual\Service;
use Concrete\Attribute\Select\Option as SelectAttributeOption;
use Concrete\Core\Attribute\Key\Key as AttributeKey;
use Concrete\Core\Attribute\Set as AttributeSet;
use Concrete\Core\Attribute\Type as AttributeType;
use Concrete\Core\Job\Set as JobSet;
use Concrete\Core\Multilingual\Page\Section\Section;
use Concrete\Core\Permission\Access\Entity\Type as PermissionAccessEntityType;
use Concrete\Core\Permission\Key\Key as PermissionKey;
use Concrete\Core\User\Group\Group;
use Concrete\Core\User\Group\GroupSet;
use Gettext\Extractors\Mo;
use Gettext\Extractors\Po;
use Gettext\Translations;
use Gettext\Extractors\PhpCode;
use Gettext\Generators\Po as PoGenerator;
use Gettext\Generators\Mo as MoGenerator;

defined('C5_EXECUTE') or die("Access Denied.");

class Extractor
{
    public function __construct()
    {
        // register concrete5 translator functions
        PhpCode::$functions['t'] = '__';
        PhpCode::$functions['t2'] = 'n__';
        PhpCode::$functions['tc'] = 'p__';
    }

    /**
     * return \GetText\Translations $translations;
     */
    public function extractTranslatableSiteStrings()
    {
        // first, we figure out which files we're going to operate on.
        $directories = array(
            DIR_APPLICATION . '/' . DIRNAME_BLOCKS,
            DIR_APPLICATION . '/' . DIRNAME_ELEMENTS,
            DIR_APPLICATION . '/' . DIRNAME_CONTROLLERS,
            DIR_APPLICATION . '/' . DIRNAME_MAIL_TEMPLATES,
            DIR_APPLICATION . '/' . DIRNAME_PAGE_TYPES,
            DIR_APPLICATION . '/' . DIRNAME_PAGES,
            DIR_APPLICATION . '/' . DIRNAME_THEMES,
            DIR_APPLICATION . '/' . DIRNAME_VIEWS
        );

        $files = array();
        foreach($directories as $directory) {
            if (!is_dir($directory)) {
                continue;
            }
            $directoryIterator = new \RecursiveDirectoryIterator($directory);
            $iterator = new \RecursiveIteratorIterator($directoryIterator);
            $results = new \RegexIterator($iterator, '/^.+\.php$/i', \RecursiveRegexIterator::GET_MATCH);
            foreach($results as $result) {
                $files[] = $result[0];
            }
        }

        $translations = new Translations();
        foreach($files as $file) {
            $fileTranslations = Translations::fromPhpCodeFile($file);
            $translations->mergeWith($fileTranslations);
        }

        // Now, we grab dynamic content that's part of our site that we translate dynamically
        $dynamicTranslations = $this->getDynamicTranslations();
        $translations->mergeWith($dynamicTranslations);

        return $translations;
    }

    public function getDynamicTranslations()
    {
        $translations = new Translations();
        $translations->mergeWith(AttributeSet::exportTranslations());
        $translations->mergeWith(AttributeKey::exportTranslations());
        $translations->mergeWith(AttributeType::exportTranslations());
{'fraction_non_alphanumeric': 0.1151222758276493, 'fraction_numerical': 0.001996340043254034, 'mean_word_length': 2.9268451992161983, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4503374', 'n_tokens_mistral': 1779, 'n_tokens_neox': 1711, 'n_words': 331}