<commit_before><?php
namespace org\dokuwiki\translatorBundle\Services\Language;

class LanguageFileParser {

    protected $content;
    protected $header;

    /**
     * @var AuthorList
     */
    protected $author;
    protected $lang;
    protected $totalLineNumbers;
    protected $file = '';
    protected $trimmedEnding;

    public static $MODE_PHP = 'php';
    public static $MODE_COMMENT_SINGLE_LINE = 'comment single line';
    public static $MODE_COMMENT_MULTI_LINE = 'comment multi line';
    public static $MODE_LANG = 'lang';
    public static $MODE_PHP_END = 'php end';
    public static $MODE_PHP_UNKNOWN = 'php unknown';

    public function loadFile($file) {
        if (!is_file($file)) {
            throw new LanguageFileDoesNotExistException();
        }
        $this->file = $file;
        $content = file_get_contents($file);
        $lines = explode("\n", $content);
        $this->totalLineNumbers = count($lines) - 1;

        $content = rtrim($content);
        $position = strlen($content);
        $this->trimmedEnding = substr($this->content, $position);
        $this->content = ltrim($content);
    }

    public static function parseLangPHP($file) {
        $parser = new LanguageFileParser();
        $parser->loadFile($file);
        return $parser->parse();
    }

    public function parse() {
        $this->author = new AuthorList();
        $this->lang = array();
        $this->lineNumber = 0;
        $this->header = '';

        $this->goToStart();

        $mode = LanguageFileParser::$MODE_PHP;
        while (strlen($this->content) !== 0) {
            if ($mode === LanguageFileParser::$MODE_PHP) {
                $mode = $this->determineNextMode();
            } elseif ($mode === LanguageFileParser::$MODE_COMMENT_MULTI_LINE) {
                $mode = $this->processMultiLineComment();<commit_msg>remove unused var lineNumber from LanguageFileParser<commit_after><?php
namespace org\dokuwiki\translatorBundle\Services\Language;

class LanguageFileParser {

    protected $content;
    protected $header;

    /**
     * @var AuthorList
     */
    protected $author;
    protected $lang;
    protected $totalLineNumbers;
    protected $file = '';
    protected $trimmedEnding;

    public static $MODE_PHP = 'php';
    public static $MODE_COMMENT_SINGLE_LINE = 'comment single line';
    public static $MODE_COMMENT_MULTI_LINE = 'comment multi line';
    public static $MODE_LANG = 'lang';
    public static $MODE_PHP_END = 'php end';
    public static $MODE_PHP_UNKNOWN = 'php unknown';

    public function loadFile($file) {
        if (!is_file($file)) {
            throw new LanguageFileDoesNotExistException();
        }
        $this->file = $file;
        $content = file_get_contents($file);
        $lines = explode("\n", $content);
        $this->totalLineNumbers = count($lines) - 1;

        $content = rtrim($content);
        $position = strlen($content);
        $this->trimmedEnding = substr($this->content, $position);
        $this->content = ltrim($content);
    }

    public static function parseLangPHP($file) {
        $parser = new LanguageFileParser();
        $parser->loadFile($file);
        return $parser->parse();
    }

    public function parse() {
        $this->author = new AuthorList();
        $this->lang = array();
        $this->header = '';

        $this->goToStart();

        $mode = LanguageFileParser::$MODE_PHP;
        while (strlen($this->content) !== 0) {
            if ($mode === LanguageFileParser::$MODE_PHP) {
                $mode = $this->determineNextMode();
            } elseif ($mode === LanguageFileParser::$MODE_COMMENT_MULTI_LINE) {
                $mode = $this->processMultiLineComment();
{'fraction_non_alphanumeric': 0.1366576819407008, 'fraction_numerical': 0.0013477088948787063, 'mean_word_length': 2.696215139442231, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5398222', 'n_tokens_mistral': 1058, 'n_tokens_neox': 1040, 'n_words': 232}