<commit_before><?php

class Twig_Extensions_Extension_Gettext_Extractor {
    
    const VARIABLE     = 0;
    const MSGID        = 'msgid';
    const MSGID_PLURAL = 'msgid_plural';
    const DOMAIN       = 'domain';
    const CATEGORY     = 'category';
    const CONTEXT      = 'context';
    
    protected $strings = array();
    
    protected $comments = array();
    
    protected $file;
    
    protected $lexer;
    
    protected $parser;
    
    protected $POStringFactory;
    
    public function __construct(Twig_Extensions_Extension_Gettext_POString_Factory_Interface $POStringFactory) {
        $this->POStringFactory = $POStringFactory;
        
        $twig         = new Twig_Environment(new Twig_Loader_String);
        $this->lexer  = new Twig_Extensions_Extension_Gettext_Lexer($twig);
        $this->parser = new Twig_Parser($twig);
    }
    
    public function extractFile($file) {
        $this->strings = array();
        $this->file    = $file;
        
        $source = file_get_contents($file);
        $tokens = $this->lexer->tokenize($source);
        $tokens = $this->extractComments($tokens);
        $node   = $this->parser->parse($tokens);
        $this->processNode($node);
        
        return $this->strings;
    }
    
    protected function extractComments(Twig_TokenStream $stream) {
        $tokens = array();
        
        while (!$stream->isEOF()) {
            $token = $stream->next();
            
            switch ($token->getType()) {
                case Twig_Extensions_Extension_Gettext_Token::COMMENT :
                    $this->comments[] = $token;
                    break;
                default :
                    $tokens[] = $token;
            }
        }
        
        $tokens[] = $stream->getCurrent();
        <commit_msg>fixing comment extraction across multiple files<commit_after><?php

class Twig_Extensions_Extension_Gettext_Extractor {
    
    const VARIABLE     = 0;
    const MSGID        = 'msgid';
    const MSGID_PLURAL = 'msgid_plural';
    const DOMAIN       = 'domain';
    const CATEGORY     = 'category';
    const CONTEXT      = 'context';
    
    protected $strings = array();
    
    protected $comments = array();
    
    protected $file;
    
    protected $lexer;
    
    protected $parser;
    
    protected $POStringFactory;
    
    public function __construct(Twig_Extensions_Extension_Gettext_POString_Factory_Interface $POStringFactory) {
        $this->POStringFactory = $POStringFactory;
        
        $twig         = new Twig_Environment(new Twig_Loader_String);
        $this->lexer  = new Twig_Extensions_Extension_Gettext_Lexer($twig);
        $this->parser = new Twig_Parser($twig);
    }
    
    public function extractFile($file) {
        $this->strings = array();
        $this->file    = $file;
        
        $source = file_get_contents($file);
        $tokens = $this->lexer->tokenize($source);
        $tokens = $this->extractComments($tokens);
        $node   = $this->parser->parse($tokens);
        $this->processNode($node);
        
        return $this->strings;
    }
    
    protected function extractComments(Twig_TokenStream $stream) {
        $tokens = array();
        $this->comments = array();

        while (!$stream->isEOF()) {
            $token = $stream->next();
            
            switch ($token->getType()) {
                case Twig_Extensions_Extension_Gettext_Token::COMMENT :
                    $this->comments[] = $token;
                    break;
                default :
                    $tokens[] = $token;
            }
        }
        
        $tokens[] = $stream->getCurrent();
        
{'fraction_non_alphanumeric': 0.11564625850340136, 'fraction_numerical': 0.0005442176870748299, 'mean_word_length': 1.9220985691573926, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3308860', 'n_tokens_mistral': 1046, 'n_tokens_neox': 1027, 'n_words': 189}