<commit_before><?php
/**
 * This file is part of the MathExecutor package
 *
 * (c) <NAME>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code
 */

namespace avadim\MathExecutor\Classes;

use avadim\MathExecutor\Classes\Token\AbstractOperator;
use avadim\MathExecutor\Classes\Token\InterfaceOperator;
use avadim\MathExecutor\Classes\Token\TokenComma;
use avadim\MathExecutor\Classes\Token\TokenFunction;
use avadim\MathExecutor\Classes\Token\TokenLeftBracket;
use avadim\MathExecutor\Classes\Token\TokenNumber;
use avadim\MathExecutor\Classes\Token\TokenRightBracket;
use avadim\MathExecutor\Classes\Token\TokenVariable;
use avadim\MathExecutor\Exception\IncorrectBracketsException;
use avadim\MathExecutor\Exception\IncorrectExpressionException;
use avadim\MathExecutor\Exception\UnknownFunctionException;
use avadim\MathExecutor\Exception\UnknownTokenException;

/**
 * @author <NAME> <<EMAIL>>
 */
class Lexer
{
    /**
     * @var TokenFactory
     */
    private $tokenFactory;

    public function __construct($tokenFactory)
    {
        $this->tokenFactory = $tokenFactory;
    }

    /**
     * @param  string                                      $input Source string of equation
     *
     * @return array                                       Tokens stream
     *
     * @throws IncorrectExpressionException
     * @throws UnknownFunctionException
     * @throws UnknownTokenException
     */
    public function stringToTokensStream($input)
    {
        $matches = array();
        preg_match_all($this->tokenFactory->getTokenParserRegex(), $input, $matches);
        $tokenFactory = $this->tokenFactory;
        $tokensStream = array_map(
            function ($token) use ($tokenFactory) {
                return $tokenFactory->createToken($token);
            },
            $matches[0]
        );

        return $tokensStream;
    }
<commit_msg>fix error with minus operator<commit_after><?php
/**
 * This file is part of the MathExecutor package
 *
 * (c) <NAME>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code
 */

namespace avadim\MathExecutor\Classes;

use avadim\MathExecutor\Classes\Token\AbstractOperator;
use avadim\MathExecutor\Classes\Token\InterfaceOperator;
use avadim\MathExecutor\Classes\Token\TokenComma;
use avadim\MathExecutor\Classes\Token\TokenFunction;
use avadim\MathExecutor\Classes\Token\TokenLeftBracket;
use avadim\MathExecutor\Classes\Token\TokenNumber;
use avadim\MathExecutor\Classes\Token\TokenRightBracket;
use avadim\MathExecutor\Classes\Token\TokenVariable;
use avadim\MathExecutor\Exception\IncorrectBracketsException;
use avadim\MathExecutor\Exception\IncorrectExpressionException;
use avadim\MathExecutor\Exception\UnknownFunctionException;
use avadim\MathExecutor\Exception\UnknownTokenException;

/**
 * @author <NAME> <<EMAIL>>
 */
class Lexer
{
    /**
     * @var TokenFactory
     */
    private $tokenFactory;

    public function __construct($tokenFactory)
    {
        $this->tokenFactory = $tokenFactory;
    }

    /**
     * @param  string                                      $input Source string of equation
     *
     * @return array                                       Tokens stream
     *
     * @throws IncorrectExpressionException
     * @throws UnknownFunctionException
     * @throws UnknownTokenException
     */
    public function stringToTokensStream($input)
    {
        $matches = array();
        $input = preg_replace_callback('/([\)\w])\s*\-(\d)/', function ($matches){
            return $matches[1] . ' - ' . $matches[2];
        }, $input);
        preg_match_all($this->tokenFactory->getTokenParserRegex(), $input, $matches);
        $tokenFactory = $this->tokenFactory;
        $tokensStream = array_map(
            function ($token) use ($tokenFactory) {
                return $tokenFactory->createToken($token);
            },
            $matches[0]
        );

        return $tokensStream;
    }

{'fraction_non_alphanumeric': 0.1033292231812577, 'fraction_numerical': 0.0009864364981504315, 'mean_word_length': 2.926427879961278, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5964663', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1098, 'n_words': 228}