<commit_before><?php
namespace Peast\Syntax\ES6;

use Peast\Syntax\Token;

class Parser extends \Peast\Syntax\Parser
{
    protected $moduleMode = false;
    
    protected $config;
    
    public function __construct($module = false)
    {
        $this->moduleMode = $module;
    }
    
    public function setSource($source, $encoding = null)
    {
        $this->scanner = new Scanner($source, $encoding);
        return $this;
    }
    
    protected function expressionToPattern($node)
    {
        $retNode = $node;
        if ($node instanceof Node\ArrayExpression) {
            
            $loc = $node->getLocation();
            $elems = array();
            foreach ($node->getElements() as $elem) {
                $elems[] = $this->expressionToPattern($elem);
            }
                
            $retNode = $this->createNode("ArrayPattern", $loc->getStart());
            $retNode->setElements($elems);
            $this->completeNode($retNode, $loc->getEnd());
            
        } elseif ($node instanceof Node\ObjectExpression) {
            
            $loc = $node->getLocation();
            $props = array();
            foreach ($node->getProperties() as $prop) {
                $props[] = $this->expressionToPattern($prop);
            }
                
            $retNode = $this->createNode("ObjectPattern", $loc->getStart());
            $retNode->setProperties($props);
            $this->completeNode($retNode, $loc->getEnd());
            
        } elseif ($node instanceof Node\Property) {
            
            $loc = $node->getLocation();
            $retNode = $this->createNode("AssignmentProperty", $loc->getStart());
            $retNode->setValue($node->getValue());
            $retNode->setKey($node->getKey());
            $retNode->setMethod($node->getMethod());
            $retNode->setShorthand($node->getShorthand());
            $retNode->setComputed($node->getComputed());
            $this->completeNode($retNode, $loc->getEnd());
            
        } elseif ($node instanceof Node\SpreadElement) {
            
            $loc = $node->getLocation();
            $retNode = $this->createNode("RestElement", $loc->getStart());
            $retNode->setArgument($this->expressionToPattern($node->getArgument()));
            $this->completeNode($retNode, $loc->getEnd());
            
        } elseif ($node instanceof Node\AssignmentExpression) {
            
            $loc = $node->getLocation();
            $retNode = $this->createNode("AssignmentPattern", $loc->getStart());
            $retNode->setLeft($this->expressionToPattern($node->getLeft()));
            $retNode->setRight($node->getRight());
            $this->completeNode($retNode, $loc->getEnd());
            
        }
        return $retNode;
    }
    
    public function parse()
    {
        if ($this->moduleMode) {
            return $this->parseModule();
        } else {
            return $this->parseScript();
        }
    }
    
    protected function parseScript()
    {
        $body = $this->parseStatementList();
        $node = $this->createNode(
            "Program", $body ? $body : $this->scanner->getPosition()
        );
        $node->setSourceType($node::SOURCE_TYPE_SCRIPT);
        if ($body) {
            $node->setBody($body);
        }
        $program = $this->completeNode($node);
        if ($this->scanner->getToken()) {
            return $this->error();
        }
        return $program;
    }
    
    protected function parseModule()
    {
        $body = $this->parseModuleItemList();
        $node = $this->createNode(
            "Program", $body ? $body : $this->scanner->getPosition()
        );
        $node->setSourceType($node::SOURCE_TYPE_MODULE);
        if ($body) {
            $node->setBody($body);
        }
        $program = $this->completeNode($node);
        if ($this->scanner->getToken()) {
            return $this->error();
        }
        return $program;
    }
    
    protected function parseStatementList($yield = false, $return = false)
    {
        $items = array();
        while ($item = $this->parseStatementListItem($yield, $return)) {
            $items[] = $item;
        }
        return count($items) ? $items : null;
    }<commit_msg>Compressed parseModule and parseScript in a single method<commit_after><?php
namespace Peast\Syntax\ES6;

use Peast\Syntax\Token;

class Parser extends \Peast\Syntax\Parser
{
    protected $moduleMode = false;
    
    protected $config;
    
    public function __construct($module = false)
    {
        $this->moduleMode = $module;
    }
    
    public function setSource($source, $encoding = null)
    {
        $this->scanner = new Scanner($source, $encoding);
        return $this;
    }
    
    protected function expressionToPattern($node)
    {
        $retNode = $node;
        if ($node instanceof Node\ArrayExpression) {
            
            $loc = $node->getLocation();
            $elems = array();
            foreach ($node->getElements() as $elem) {
                $elems[] = $this->expressionToPattern($elem);
            }
                
            $retNode = $this->createNode("ArrayPattern", $loc->getStart());
            $retNode->setElements($elems);
            $this->completeNode($retNode, $loc->getEnd());
            
        } elseif ($node instanceof Node\ObjectExpression) {
            
            $loc = $node->getLocation();
            $props = array();
            foreach ($node->getProperties() as $prop) {
                $props[] = $this->expressionToPattern($prop);
            }
                
            $retNode = $this->createNode("ObjectPattern", $loc->getStart());
            $retNode->setProperties($props);
            $this->completeNode($retNode, $loc->getEnd());
            
        } elseif ($node instanceof Node\Property) {
            
            $loc = $node->getLocation();
            $retNode = $this->createNode("AssignmentProperty", $loc->getStart());
            $retNode->setValue($node->getValue());
            $retNode->setKey($node->getKey());
            $retNode->setMethod($node->getMethod());
            $retNode->setShorthand($node->getShorthand());
            $retNode->setComputed($node->getComputed());
            $this->completeNode($retNode, $loc->getEnd());
            
        } elseif ($node instanceof Node\SpreadElement) {
            
            $loc = $node->getLocation();
            $retNode = $this->createNode("RestElement", $loc->getStart());
            $retNode->setArgument($this->expressionToPattern($node->getArgument()));
            $this->completeNode($retNode, $loc->getEnd());
            
        } elseif ($node instanceof Node\AssignmentExpression) {
            
            $loc = $node->getLocation();
            $retNode = $this->createNode("AssignmentPattern", $loc->getStart());
            $retNode->setLeft($this->expressionToPattern($node->getLeft()));
            $retNode->setRight($node->getRight());
            $this->completeNode($retNode, $loc->getEnd());
            
        }
        return $retNode;
    }
    
    public function parse()
    {
        $body = $this->moduleMode ?
                $this->parseModuleItemList() :
                $this->parseStatementList();
        $node = $this->createNode(
            "Program", $body ? $body : $this->scanner->getPosition()
        );
        $node->setSourceType($this->moduleMode ?
                             $node::SOURCE_TYPE_MODULE :
                             $node::SOURCE_TYPE_SCRIPT);
        if ($body) {
            $node->setBody($body);
        }
        $program = $this->completeNode($node);
        if ($this->scanner->getToken()) {
            return $this->error();
        }
        return $program;
    }
    
    protected function parseStatementList($yield = false, $return = false)
    {
        $items = array();
        while ($item = $this->parseStatementListItem($yield, $return)) {
            $items[] = $item;
        }
        return count($items) ? $items : null;
    }
{'fraction_non_alphanumeric': 0.1558473736372646, 'fraction_numerical': 0.0002477700693756194, 'mean_word_length': 1.8863067572399, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5605915', 'n_tokens_mistral': 2168, 'n_tokens_neox': 2174, 'n_words': 364}