<commit_before><?php

namespace Bavix\Flow;

use Bavix\Helpers\Arr;
use Bavix\Lexer\Lexer;
use Bavix\SDK\FileLoader;

/**
 * Class Lexem
 *
 * @package Bavix\Flow
 */
class Lexem
{

    /**
     * @var array
     */
    protected $types = [
        'callable' => '[.\w\s(,\'")]+',
        'variable' => '[\w\'":.\[\]()\s]+',
        'array'    => '(array\(|\[)[\s\S]*(\]|\))',
        'ternary'  => '\X+\?\X*:\X+',
        'any'      => '\X+',
    ];

    /**
     * @var array
     */
    protected $default = [
        'types' => [
            'variable'
        ]
    ];

    /**
     * @var array
     */
    protected $props = [];

    /**
     * @var array
     */
    protected $data = [];

    /**
     * @var array
     */
    protected $closed = [];

    /**
     * @var Lexer
     */
    protected $lexer;

    /**
     * @var string
     */
    protected $root;

    /**
     * @var Flow
     */
    protected $flow;

    /**
     * Lexem constructor.
     *
     * @param Flow $flow
     */
    public function __construct(Flow $flow)
    {
        $this->root = \dirname(__DIR__, 2) . '/lexemes';
        $this->flow = $flow;
    }

    /**
     * @return Lexer
     */
    protected function lexer(): Lexer
    {
        if (!$this->lexer)
        {
            $this->lexer = new Lexer();
        }

        return $this->lexer;
    }

    /**
     * @param $file
     *
     * @return FileLoader\DataInterface|null
     */
    protected function loader($file)
    {
        foreach (FileLoader::extensions() as $ext)
        {
            try<commit_msg>update lexem<commit_after><?php

namespace Bavix\Flow;

use Bavix\Helpers\Arr;
use Bavix\Lexer\Lexer;
use Bavix\SDK\FileLoader;

/**
 * Class Lexem
 *
 * @package Bavix\Flow
 */
class Lexem
{

    /**
     * @var array
     */
    protected $types = [
        'callable' => '[.\w\s(,\'")]+',
        'variable' => '[\w\'":.\[\]()\s]+',
        'array'    => '(array\(|\[)[\s\S]*(\]|\))',
        'ternary'  => '\X+\?\X*:\X+',
        'any'      => '\X+',
    ];

    /**
     * @var array
     */
    protected $default = [
        'types' => [
            'variable'
        ]
    ];

    /**
     * @var array
     */
    protected $props = [];

    /**
     * @var array
     */
    protected $data = [];

    /**
     * @var array
     */
    protected $closed = [];

    /**
     * @var Lexer
     */
    protected $lexer;

    /**
     * @var string
     */
    protected $root;

    /**
     * @var Flow
     */
    protected $flow;

    /**
     * Lexem constructor.
     *
     * @param Flow $flow
     * @param string $root
     */
    public function __construct(Flow $flow, string $root = null)
    {
        if (!$root)
        {
            $root = \dirname(__DIR__, 2);
        }

        $this->root = $root . '/lexemes';
        $this->flow = $flow;
    }

    /**
     * @return Lexer
     */
    protected function lexer(): Lexer
    {
        if (!$this->lexer)
        {
            $this->lexer = new Lexer();
        }

        return $this->lexer;
    }

    /**
     * @param $file
     *
     * @return FileLoader\DataInterface|null
     */
    protected function loader($file)
    {
        foreach (FileLoader::extensions() as $ext)
        {
            try
{'fraction_non_alphanumeric': 0.19281547436291066, 'fraction_numerical': 0.0006140620202640467, 'mean_word_length': 1.3421998562185478, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '612765', 'n_tokens_mistral': 1144, 'n_tokens_neox': 1102, 'n_words': 208}