<commit_before><?php 
namespace Bolt\Storage\Query;

use Bolt\Storage\EntityManager;


/**
*  Handler class to convert the DSL for content queries into an
*  object representation.
* 
*  @author <NAME> <<EMAIL>>
*/
class ContentQueryParser
{
    
    protected $em;
    
    protected $query;
    
    protected $params;
    

    protected $contentTypes = [];
    
    protected $operation;
    
    protected $identifier;
    
    protected $selectType = 'all';
    
    protected $limit;
        
    protected $sqlParams = [];
    
    protected $operations = ['search', 'latest', 'first'];
    
    protected $getquery;
    
    protected $printquery;
    
    public function __construct(EntityManager $em, $query, array $params = [])
    {
        $this->em = $em;
        $this->query = $query;
        $this->params = $params;
    }
    
    public function parse()
    {
        $this->parseContent();
        $this->operation = $this->parseOperation();
        $this->parseDirectives();
    }
    
    /**
     * Parses the content area of the querystring.
     * 
     * @return void
     */
    protected function parseContent()
    {
        $contentString = strtok($this->query, '/');
        
        $content = array();
        $delim = "(),";
        $tok = strtok($contentString, $delim);
        while ($tok !== false) {
            $content[] = $tok;
            $tok = strtok($delim);
        }       
        
        $this->contentTypes = $content;
    }
    
    protected function parseOperation()
    {
        $operation = 'select';
        
        $queryParts = explode('/', $this->query);
        array_shift($queryParts);
        
        if (!count($queryParts)) {
            return $operation;
        }
        
        if (in_array($queryParts[0], $this->operations)) {
            $operation = array_shift($queryParts);
            $this->limit = array_shift($queryParts);
            $this->identifier = implode(',', $queryParts);
        } else {
            $this->identifier = implode(',', $queryParts);
        }
        
        return $operation;
        
    }
    
    protected function parseDirectives()
    {
        foreach ($this->params as $key => $value) {
            if ($key == 'printquery') {
                $this->printquery = true;
            }
            
            if ($key == 'getquery') {
                $this->getquery = $value;
            }
            
            if ($key == 'returnsingle') {
                $this->limit = 1;
            }
        }
    }
    
    public function getContentTypes()
    {
        return $this->contentTypes;
    }
    
    public function getOperation()
    {
        return $this->operation;
    }
    
    public function getIdentifier()
    {
        return $this->identifier;
    }
    
    public function getLimit()
    {
        return $this->limit;
    }
    

}<commit_msg>add support to add and remove operations<commit_after><?php 
namespace Bolt\Storage\Query;

use Bolt\Storage\EntityManager;


/**
*  Handler class to convert the DSL for content queries into an
*  object representation.
* 
*  @author <NAME> <<EMAIL>>
*/
class ContentQueryParser
{
    
    protected $em;
    
    protected $query;
    
    protected $params;
    

    protected $contentTypes = [];
    
    protected $operation;
    
    protected $identifier;
    
    protected $selectType = 'all';
    
    protected $limit;
        
    protected $sqlParams = [];
    
    protected $operations = ['search', 'latest', 'first', 'random'];
    
    protected $getquery;
    
    protected $printquery;
    
    public function __construct(EntityManager $em, $query, array $params = [])
    {
        $this->em = $em;
        $this->query = $query;
        $this->params = $params;
    }
    
    public function parse()
    {
        $this->parseContent();
        $this->operation = $this->parseOperation();
        $this->parseDirectives();
    }
    
    /**
     * Parses the content area of the querystring.
     * 
     * @return void
     */
    protected function parseContent()
    {
        $contentString = strtok($this->query, '/');
        
        $content = array();
        $delim = "(),";
        $tok = strtok($contentString, $delim);
        while ($tok !== false) {
            $content[] = $tok;
            $tok = strtok($delim);
        }       
        
        $this->contentTypes = $content;
    }
    
    protected function parseOperation()
    {
        $operation = 'select';
        
        $queryParts = explode('/', $this->query);
        array_shift($queryParts);
        
        if (!count($queryParts)) {
            return $operation;
        }
        
        if (in_array($queryParts[0], $this->operations)) {
            $operation = array_shift($queryParts);
            $this->limit = array_shift($queryParts);
            $this->identifier = implode(',', $queryParts);
        } else {
            $this->identifier = implode(',', $queryParts);
        }
        
        return $operation;
        
    }
    
    protected function parseDirectives()
    {
        foreach ($this->params as $key => $value) {
            if ($key == 'printquery') {
                $this->printquery = true;
            }
            
            if ($key == 'getquery') {
                $this->getquery = $value;
            }
            
            if ($key == 'returnsingle') {
                $this->limit = 1;
            }
        }
    }
    
    public function getContentTypes()
    {
        return $this->contentTypes;
    }
    
    public function getOperation()
    {
        return $this->operation;
    }
    
    public function getIdentifier()
    {
        return $this->identifier;
    }
    
    public function getLimit()
    {
        return $this->limit;
    }
    
    /**
     * Adds a new operation to the list supported
     * @param string $operation name of operation to parse for
     */
    public function addOperation($operation)
    {
        if (!in_array($operation, $this->operations)) {
            $this->operations[] = $operation;
        }
    }
    
    /**
     * Removes an operation from the list supported
     * @param string $operation name of operation to remove
     */
    public function removeOperation($operation)
    {
        if (in_array($operation, $this->operations)) {
            $key = array_search($operation, $this->operations);
            unset($this->operations[$key]);
        }
    }
    

}
{'fraction_non_alphanumeric': 0.13050847457627118, 'fraction_numerical': 0.0006163328197226503, 'mean_word_length': 1.641839641839642, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '963437', 'n_tokens_mistral': 1817, 'n_tokens_neox': 1790, 'n_words': 401}