<commit_before><?php

/*
 * This file is part of the Yodler package.
 *
 * (c) aes3xs <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Aes3xs\Yodler\Heap;

use Aes3xs\Yodler\Exception\RuntimeException;
use Aes3xs\Yodler\Exception\VariableCircularReferenceException;
use Aes3xs\Yodler\Exception\VariableNotFoundException;
use Aes3xs\Yodler\Common\CallableHelper;
use Aes3xs\Yodler\Variable\VariableListInterface;
use Symfony\Component\ExpressionLanguage\ExpressionLanguage;

/**
 * Heap implementation.
 */
class Heap implements HeapInterface
{
    /**
     * @var VariableListInterface[]
     */
    protected $collection = [];

    /**
     * @var \Twig_Environment
     */
    protected $twig;

    /**
     * @var ExpressionLanguage
     */
    protected $expressionLanguage;

    /**
     * Constructor.
     *
     * @param \Twig_Environment $twig
     * @param ExpressionLanguage $expressionLanguage
     */
    public function __construct(\Twig_Environment $twig, ExpressionLanguage $expressionLanguage)
    {
        $this->twig = $twig;
        $this->expressionLanguage = $expressionLanguage;
    }

    /**
     * Add variable list to heap with first priority.
     *
     * @param VariableListInterface $variables
     */
    public function addVariables(VariableListInterface $variables)
    {
        array_unshift($this->collection, $variables);
    }

    /**
     * {@inheritdoc}
     */
    public function has($name)
    {
        foreach ($this->collection as $variables) {
            if ($variables->has($name)) {
                return true;
            }
        }

        return false;
    }

    /**
     * {@inheritdoc}
     */
    public function get($name)
    {
        foreach ($this->collection as $variables) {
            if ($variables->has($name)) {
                return $variables->get($name);
            }
        }

        throw new VariableNotFoundException($name);
    }

    /**
     * {@inheritdoc}
     */
    public function resolve($name)
    {
        $value = $this->get($name);

        return is_callable($value) ? $this->resolveCallbackWithCallstack($value) : $value;
    }

    /**
     * {@inheritdoc}
     */
    public function resolveString($string)
    {
        $data = [];

        foreach ($this->all() as $name => $value) {
            if (!is_callable($value)) {
                $data[$name] = $value;
            }
        }

        $template = $this->twig->createTemplate($string);

        $result = $template->render($data);

        return $result;
    }

    /**
     * {@inheritdoc}
     */
    public function resolveExpression($expression)
    {
        $data = [];

        foreach ($this->all() as $name => $value) {
            if (!is_callable($value)) {
                $data[$name] = $value;
            }
        }

        return $this->expressionLanguage->evaluate($expression, $data);
    }

    /**
     * {@inheritdoc}
     */
    public function getDependencies($name)
    {
        $value = $this->get($name);

        return is_callable($value) ? $this->getDependenciesWithCallstack($value) : [];
    }

    /**
     * Return all variables flatten to single list by name and priority.
     *
     * @return array
     */
    protected function all()
    {
        $result = [];

        foreach ($this->collection as $variables) {
            $result += $variables->all();
        }

        return $result;
    }

    /**
     * Resolve callback using arguments from the heap.
     *
     * Has circular reference detection.
     *
     * @param callable $callback
     * @param array $callstack
     *
     * @return mixed
     *
     * @throws VariableCircularReferenceException
     */
    protected function resolveCallbackWithCallstack(callable $callback, &$callstack = [])
    {
        $arguments = CallableHelper::extractArguments($callback);

        $callArguments = [];

        foreach ($arguments as $name) {

            $value = $this->get($name);

            if (is_callable($value)) {

                if (in_array($name, $callstack)) {
                    throw new VariableCircularReferenceException($name, $callstack);
                }

                $callstack[] = $name;

                $value = $this->resolveCallbackWithCallstack($value, $callstack);

                $extractedName = array_pop($callstack);

                if ($extractedName !== $name) {
                    throw new RuntimeException(sprintf('Extracted name `%s` doesn\'t match called `%s`', $extractedName, $name));
                }
            }

            $callArguments[$name] = $value;
        }

        return CallableHelper::call($callback, $callArguments);
    }

    /**
     * Return callback dependencies using arguments from the heap.
     *
     * Has circular reference detection.
     *
     * @param callable $callback
     * @param array $callstack
     *
     * @return array
     *
     * @throws VariableCircularReferenceException
     */
    protected function getDependenciesWithCallstack(callable $callback, &$callstack = [])
    {
        $arguments = CallableHelper::extractArguments($callback);

        $dependencies = $arguments;

        foreach ($arguments as $name) {

            if (!$this->has($name)) {
                continue;
            }

            $value = $this->get($name);

            if (is_callable($value)) {

                if (in_array($name, $callstack)) {
                    throw new VariableCircularReferenceException($name, $callstack);
                }

                $callstack[] = $name;

                $dependencies = array_merge($dependencies, $this->getDependenciesWithCallstack($value, $callstack));

                $extractedName = array_pop($callstack);

                if ($extractedName !== $name) {
                    throw new RuntimeException(sprintf('Extracted name `%s` doesn\'t match called `%s`', $extractedName, $name));
                }
            }

            $callArguments[$name] = $value;
        }

        $dependencies = array_unique($dependencies);

        return $dependencies;
    }
}
<commit_msg>Add twig and expression variables extraction in order to resolve only used variables<commit_after><?php

/*
 * This file is part of the Yodler package.
 *
 * (c) aes3xs <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Aes3xs\Yodler\Heap;

use Aes3xs\Yodler\Exception\RuntimeException;
use Aes3xs\Yodler\Exception\VariableCircularReferenceException;
use Aes3xs\Yodler\Exception\VariableNotFoundException;
use Aes3xs\Yodler\Common\CallableHelper;
use Aes3xs\Yodler\Variable\VariableListInterface;
use Symfony\Component\ExpressionLanguage\ExpressionLanguage;
use Symfony\Component\ExpressionLanguage\Node\NameNode;
use Symfony\Component\ExpressionLanguage\Node\Node;

/**
 * Heap implementation.
 */
class Heap implements HeapInterface
{
    /**
     * @var VariableListInterface[]
     */
    protected $collection = [];

    /**
     * @var \Twig_Environment
     */
    protected $twig;

    /**
     * @var ExpressionLanguage
     */
    protected $expressionLanguage;

    /**
     * Constructor.
     *
     * @param \Twig_Environment $twig
     * @param ExpressionLanguage $expressionLanguage
     */
    public function __construct(\Twig_Environment $twig, ExpressionLanguage $expressionLanguage)
    {
        $this->twig = $twig;
        $this->expressionLanguage = $expressionLanguage;
    }

    /**
     * Add variable list to heap with first priority.
     *
     * @param VariableListInterface $variables
     */
    public function addVariables(VariableListInterface $variables)
    {
        array_unshift($this->collection, $variables);
    }

    /**
     * {@inheritdoc}
     */
    public function has($name)
    {
        foreach ($this->collection as $variables) {
            if ($variables->has($name)) {
                return true;
            }
        }

        return false;
    }

    /**
     * {@inheritdoc}
     */
    public function get($name)
    {
        foreach ($this->collection as $variables) {
            if ($variables->has($name)) {
                return $variables->get($name);
            }
        }

        throw new VariableNotFoundException($name);
    }

    /**
     * {@inheritdoc}
     */
    public function resolve($name)
    {
        $value = $this->get($name);

        return is_callable($value) ? $this->resolveCallbackWithCallstack($value) : $value;
    }

    /**
     * {@inheritdoc}
     */
    public function resolveString($string)
    {
        $dependencies = $this->getTwigVariables($string);
        $data = [];
        foreach ($dependencies as $name) {
            $data[$name] = $this->resolve($name);
        }
        $template = $this->twig->createTemplate($string);
        return $template->render($data);
    }

    /**
     * Return twig variables by source.
     *
     * http://stackoverflow.com/a/40105067
     *
     * @param $source
     *
     * @return array
     */
    protected function getTwigVariables($source)
    {
        $tokens = $this->twig->tokenize($source);
        $parsed = $this->twig->getParser()->parse($tokens);
        $collected = [];
        $this->collectTwigNodes($parsed, $collected);
        return array_keys($collected);
    }

    /**
     * @param \Twig_Node[] $nodes
     * @param array $collected
     */
    protected function collectTwigNodes($nodes, array &$collected)
    {
        foreach ($nodes as $node) {
            $childNodes = $node->getIterator()->getArrayCopy();
            if (!empty($childNodes)) {
                $this->collectTwigNodes($childNodes, $collected); // recursion
            } elseif ($node instanceof \Twig_Node_Expression_Name) {
                $name = $node->getAttribute('name');
                $collected[$name] = $node; // ensure unique values
            }
        }
    }

    /**
     * {@inheritdoc}
     */
    public function resolveExpression($expression)
    {
        $dependencies = $this->getExpressionVariables($expression);
        $data = [];
        foreach ($dependencies as $name) {
            $data[$name] = $this->resolve($name);
        }
        return $this->expressionLanguage->evaluate($expression, $data);
    }

    /**
     * @param $expression
     *
     * @return array
     */
    protected function getExpressionVariables($expression)
    {
        $nodes = $this->expressionLanguage->parse($expression, array_keys($this->all()))->getNodes()->nodes;
        $collected = [];
        $this->collectExpressionNameNodes($nodes, $collected);
        return array_keys($collected);
    }

    /**
     * @param array $nodes
     * @param array $collected
     */
    protected function collectExpressionNameNodes(array $nodes, array &$collected)
    {
        /** @var Node $node */
        foreach ($nodes as $node) {
            if ($node instanceof NameNode) {
                $name = $node->attributes['name'];
                $collected[$name] = $name; // ensure unique values
            }
            $this->collectExpressionNameNodes($node->nodes, $collected); // recursion
        }
    }

    /**
     * {@inheritdoc}
     */
    public function getDependencies($name)
    {
        $value = $this->get($name);

        return is_callable($value) ? $this->getDependenciesWithCallstack($value) : [];
    }

    /**
     * Return all variables flatten to single list by name and priority.
     *
     * @return array
     */
    protected function all()
    {
        $result = [];

        foreach ($this->collection as $variables) {
            $result += $variables->all();
        }

        return $result;
    }

    /**
     * Resolve callback using arguments from the heap.
     *
     * Has circular reference detection.
     *
     * @param callable $callback
     * @param array $callstack
     *
     * @return mixed
     *
     * @throws VariableCircularReferenceException
     */
    protected function resolveCallbackWithCallstack(callable $callback, &$callstack = [])
    {
        $arguments = CallableHelper::extractArguments($callback);

        $callArguments = [];

        foreach ($arguments as $name) {

            $value = $this->get($name);

            if (is_callable($value)) {

                if (in_array($name, $callstack)) {
                    throw new VariableCircularReferenceException($name, $callstack);
                }

                $callstack[] = $name;

                $value = $this->resolveCallbackWithCallstack($value, $callstack);

                $extractedName = array_pop($callstack);

                if ($extractedName !== $name) {
                    throw new RuntimeException(sprintf('Extracted name `%s` doesn\'t match called `%s`', $extractedName, $name));
                }
            }

            $callArguments[$name] = $value;
        }

        return CallableHelper::call($callback, $callArguments);
    }

    /**
     * Return callback dependencies using arguments from the heap.
     *
     * Has circular reference detection.
     *
     * @param callable $callback
     * @param array $callstack
     *
     * @return array
     *
     * @throws VariableCircularReferenceException
     */
    protected function getDependenciesWithCallstack(callable $callback, &$callstack = [])
    {
        $arguments = CallableHelper::extractArguments($callback);

        $dependencies = $arguments;

        foreach ($arguments as $name) {

            if (!$this->has($name)) {
                continue;
            }

            $value = $this->get($name);

            if (is_callable($value)) {

                if (in_array($name, $callstack)) {
                    throw new VariableCircularReferenceException($name, $callstack);
                }

                $callstack[] = $name;

                $dependencies = array_merge($dependencies, $this->getDependenciesWithCallstack($value, $callstack));

                $extractedName = array_pop($callstack);

                if ($extractedName !== $name) {
                    throw new RuntimeException(sprintf('Extracted name `%s` doesn\'t match called `%s`', $extractedName, $name));
                }
            }

            $callArguments[$name] = $value;
        }

        $dependencies = array_unique($dependencies);

        return $dependencies;
    }
}

{'fraction_non_alphanumeric': 0.1276287664621113, 'fraction_numerical': 0.00151692753223471, 'mean_word_length': 2.0521885521885523, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6391628', 'n_tokens_mistral': 4031, 'n_tokens_neox': 3846, 'n_words': 823}