<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using SassSharp.Exceptions;
using SassSharp.IO;
using SassSharp.Model;
using SassSharp.Model.Expressions;
using SassSharp.Model.Nodes;

namespace SassSharp
{
    internal class ScssReader : StreamReader
    {
        private int _lineNumber;


        public ScssReader(PathFile file) : base(file.GetStream())
        {
            File = file;
        }

        public PathFile File { get; set; }


        public override int Peek()
        {
            var c = base.Peek();

            //Ignore CR
            while ((char) c == '\r')
            {
                base.Read();
                c = base.Peek();
            }

            return c;
        }

        public override int Read()
        {
            var c = base.Read();

            //Ignore CR
            while ((char) c == '\r')
            {
                c = base.Read();
            }

            if ((char) c == '\n')
                ++_lineNumber;

            return c;
        }

        /// <summary>
        ///     Skips whitespace
        /// </summary>
        /// <returns>true if whitespace was found</returns>
        private bool SkipWhitespace()
        {
            var foundWhitespace = false;

            while (!EndOfStream)
            {
                var c = (char) Peek();

                if (char.IsWhiteSpace(c))
                {
                    foundWhitespace = true;
                    Read();
                }
                else
                {
                    return foundWhitespace;
                }
            }

            return false;
        }

        /// <summary>
        ///     Reads one character, throwing an exception if not as expected.
        /// </summary>
        /// <param name="c"></param>
        private void Expect(char expected)
        {
            SkipWhitespace();

            if (EndOfStream)
                throw new ScssReaderException($"Expected {expected}, end of stream", File.Path, _lineNumber);

            var c = (char) Read();

            if (c != expected)
                throw new ScssReaderException($"Expected '{expected}', found '{c}'", File.Path, _lineNumber);
        }

        private void Optional(char expected, bool skipWhitespace = true)
        {
            if (skipWhitespace)
                SkipWhitespace();

            if (EndOfStream)
                throw new ScssReaderException($"Expected {expected}, end of stream", File.Path, _lineNumber);

            var c = (char) Peek();

            if (c == expected)
                Read();
        }

        private string ReadUntil(char endChar)
        {
            var buffer = new StringBuilder();
            while (!EndOfStream)
            {
                var c = (char) Read();

                if (c == endChar)
                    return buffer.ToString();

                buffer.Append(c);
            }

            throw new Exception($"Expected {endChar}");
        }

        internal ScssPackage ReadTree()
        {
            var package = new ScssPackage(File);
            package.LoadBuiltInFunctions();

            ScopeNode currentScope = package;
            ReadScopeContent(currentScope);

            return package;
        }

        private void ReadScopeContent(ScopeNode currentScope)
        {
            var buffer = new StringBuilder();
            var interpolations = new List<ValueList>();

            //States
            var inCommentStart = false;
            var paranthesesLevel = 0;
            var inDoubleQuotes = false;

            while (!EndOfStream)
            {
                var c = (char) Read();

                if (inDoubleQuotes)
                {
                    buffer.Append(c);

                    if (c == '"')
                    {
                        inDoubleQuotes = false;
                    }
                }
                else
                {
                    switch (c)
                    {
                        case ' ':
                            //Ignore starting white space
                            if (buffer.Length == 0)
                                break;
                            goto default;
                        case '@':
                            if (buffer.Length != 0)
                                throw new Exception("Must be first character");

                            ReadAt(currentScope);

                            break;
                        case '$':
                            var name = ReadUntil(':');
                            var vn = new VariableNode(name);
                            vn.Expression = new Expression(ReadValueList(';'));
                            Expect(';');
                            currentScope.Add(vn);
                            break;
                        case '/':
                            if (inCommentStart)
                            {
                                ReadInlineComment();

                                inCommentStart = false;

                                //Remove the first slash from the buffer
                                buffer.Length--;
                            }
                            else
                            {
                                buffer.Append(c);
                                inCommentStart = true;
                            }
                            break;
                        case '*':
                            if (inCommentStart)
                            {
                                var node = ReadComment();
                                currentScope.Add(node);
                                inCommentStart = false;

                                //Remove the first slash from the buffer
                                buffer.Length--;
                                break;
                            }
                            goto default;
                        case ';':
                        {
                            throw new ScssReaderException("Unexpected ;", File.Path, _lineNumber);
                        }
                        case ':':
                            var pc = (char) Peek();
                            if (char.IsLetter(pc)) //hover etc.
                            {
                                buffer.Append(c);
                            }
                            else
                            {
                                var pn = new PropertyNode();
                                pn.Name = new ScssString(buffer.ToString().Trim(), interpolations);
                                pn.Expression = new Expression(ReadValueList(';', '{'));
                                buffer.Clear();

                                var pc2 = (char) Read();
                                if (pc2 == ';')
                                {
                                    currentScope.Add(pn);
                                }
                                else if (pc2 == '{')
                                {
                                    var result = new NamespaceNode(pn);
                                    ReadScopeContent(result);
                                    currentScope.Add(result);
                                }
                                else
                                {
                                    throw new ScssReaderException("Expected { or ;", File.Path, _lineNumber);
                                }
                            }
                            break;
                        case '"':
                            inDoubleQuotes = true;
                            goto default;
                        case '(':
                            paranthesesLevel++;
                            goto default;
                        case ')':
                            paranthesesLevel--;
                            goto default;
                        case '{':
                        {
                            var node = new SelectorNode();
                            node.Selector = new ScssString(buffer.ToString().Trim(), interpolations);
                            ReadScopeContent(node);
                            currentScope.Add(node);
                            buffer.Clear();
                        }
                            break;
                        case '}':
                            return;
                        case '#':
                            if (Peek() == '{')
                            {
                                inCommentStart = false;
                                var value = ReadInterpolation();
                                buffer.Append("{" + interpolations.Count + "}");
                                interpolations.Add(value);
                                break;
                            }
                            else
                            {
                                goto default;
                            }
                        case '\n':
                            inCommentStart = false;
                            break;
                        default:
                            inCommentStart = false;
                            buffer.Append(c);
                            break;
                    }
                }
            }
        }

        private ValueList ReadInterpolation()
        {
            Expect('{');
            ValueList vl = ReadValueList('}');
            Expect('}');
            return vl;
        }

        private void ReadAt(ScopeNode currentScope)
        {
            string type = ReadName();

            SkipWhitespace();

            switch (type)
            {
                case "media":
                    ReadMedia(currentScope);
                    break;
                case "mixin":
                    ReadMixin(currentScope);
                    break;
                case "content":
                    ReadContent(currentScope);
                    break;
                case "include":
                    ReadInclude(currentScope);
                    break;
                case "import":
                    ReadImport(currentScope);
                    break;
                case "function":
                    ReadFunction(currentScope);
                    break;
                case "return":
                    ReadReturn(currentScope);
                    break;
                case "if":
                    ReadIf(currentScope);
                    break;
                case "else":
                    ReadElse(currentScope);
                    break;
                default:
                    throw new ScssReaderException($"Could not recognize @{type}", File.Path, _lineNumber);
            }
        }


        private void ReadIf(ScopeNode currentScope)
        {
            var val = ReadValueList('{');
            Expect('{');

            var ifNode = new IfNode(val);
            currentScope.Add(ifNode);

            ReadScopeContent(ifNode);
        }

        private void ReadElse(ScopeNode currentScope)
        {
            ElseNode elseNode;

            if (Peek() == 'i') //Assume if
            {
                Read();
                Expect('f');
                SkipWhitespace();

                var val = ReadValueList('{');
                Expect('{');

                elseNode = new ElseNode(val);
            }
            else
            {
                elseNode = new ElseNode();
            }

            var parentIf = currentScope.Nodes.Last() as IfNode;

            if (parentIf == null)
                throw new Exception("Else without if");

            parentIf.Elses.Add(elseNode);

            ReadScopeContent(elseNode);
        }


        private void ReadImport(ScopeNode currentScope)
        {
            var path = ReadUntil(';');

            currentScope.Add(new ImportNode(path));
        }


        private void ReadMedia(ScopeNode currentScope)
        {
            var def = ReadUntil('{');

            MediaNode node = new MediaNode()
            {
                Definition = def.TrimEnd()
            };


            ReadScopeContent(node);
            currentScope.Add(node);
        }


        private void ReadMixin(ScopeNode currentScope)
        {
            var name = ReadName();

            VariableNode[] args = new VariableNode[0];

            SkipWhitespace();

            if (Peek() == '(')
                args = ReadArgumentDefinition().ToArray();

            var node = new MixinNode(name, args);
            Expect('{');
            ReadScopeContent(node);
            currentScope.Add(node);
        }

        private void ReadContent(ScopeNode currentScope)
        {
            currentScope.Add(new ContentNode());
            Optional(';');
        }

        private void ReadInclude(ScopeNode currentScope)
        {
            var name = ReadName();
            ValueList args = new ValueList();

            SkipWhitespace();

            if (Peek() == '(')
                args = ReadArgumentCall();

            SkipWhitespace();

            var result = new IncludeNode(name, args);
            currentScope.Add(result);

            if (Peek() == '{')
            {
                Expect('{');
                ReadScopeContent(result);
            }
            else
                Expect(';');
        }

        private void ReadFunction(ScopeNode currentScope)
        {
            var name = ReadName();

            var args = ReadArgumentDefinition();

            var node = new FunctionNode(name, args.ToArray());
            Expect('{');
            ReadScopeContent(node);
            currentScope.Add(node);
        }

        private void ReadReturn(ScopeNode currentScope)
        {
            ExpressionNode value = ReadValueList(';');
            Expect(';');
            var result = new ReturnNode(new Expression(value));
            currentScope.Add(result);
        }

        private ValueList ReadArgumentCall()
        {
            Expect('(');
            var result = ReadValueList(')');
            Expect(')');
            return result;
        }

        private IEnumerable<VariableNode> ReadArgumentDefinition()
        {
            var result = new List<VariableNode>();

            Expect('(');

            while (!EndOfStream)
            {
                var c = (char) Read();

                switch (c)
                {
                    case '$':
                        var name = ReadName();

                        var node = new VariableNode(name);
                        result.Add(node);

                        SkipWhitespace();
                        c = (char) Peek();
                        if (c == ':')
                        {
                            Read();
                            node.Expression = new Expression(ReadValueList(',', ')'));
                        }

                        break;
                    case ',':
                    case ' ':
                        break;
                    case ')':
                        return result;
                }

                //var expr = ReadValue();
            }

            throw new Exception("Could not find argument ending");
        }


        /// <summary>
        ///     Reads a name, stopping at (, {, ; and whitespace.  Avoids consuming the last character
        /// </summary>
        /// <returns></returns>
        private string ReadName()
        {
            var firstChar = (char) Peek();

            if (!char.IsLetter(firstChar))
                throw new Exception("First character in name must be a letter");

            var buffer = new StringBuilder();
            buffer.Append((char) Read());

            while (!EndOfStream)
            {
                var c = (char) Peek();

                if (!char.IsLetter(c)
                    && !char.IsDigit(c)
                    && c != '-'
                    && c != '_')
                    break;

                buffer.Append((char) Read());
            }

            return buffer.ToString();
        }

        private ValueList ReadValueList(params char[] expectedEndCharaters)
        {
            ValueList commaList = null;
            ValueList spaceList = null;
            string key = null;

            var tempNodes = new List<ExpressionNode>();
            var op = '+';

            while (!EndOfStream)
            {
                var afterSpace = SkipWhitespace();
                var c = (char) Peek();

                if (expectedEndCharaters.Contains(c))
                {
                    var node = Expression.CalculateTree(tempNodes.ToArray());

                    if (commaList != null)
                    {
                        if (spaceList != null)
                        {
                            spaceList.Add(node);
                            commaList.Add(spaceList);
                            spaceList = null;
                        }
                        else
                        {
                            commaList.Add(node);
                        }

                        return commaList;
                    }
                    if (spaceList == null)
                    {
                        spaceList = new ValueList();
                    }
                    spaceList.Add(node);
                    return spaceList;
                }

                switch (c)
                {
                    case ')':
                    case '{':
                    case ';':
                    {
                        throw new Exception("Unexpected end character");
                    }
                    case '!':
                        Read();
                        c = (char) Peek();
                        if (c == '=')
                        {
                            op = '!';
                            Read();
                            break;
                        }
                        var name = ReadName();

                        if (spaceList == null)
                        {
                            spaceList = new ValueList();
                        }

                        spaceList.Add(Expression.CalculateTree(tempNodes.ToArray()));
                        tempNodes.Clear();

                        //Make a special modifierNode?
                        tempNodes.Add(new ValueNode($"!{name}")
                        {
                            Operator = '+'
                        });

                        continue;
                    case '=':
                        Read();
                        if (Peek() != '=')
                        {
                            op = 'e';
                        }
                        else
                        {
                            op = c;
                            Read();
                            Expect('=');
                        }
                        break;
                    case '-':
                    case '+':
                    case '*':
                    case '/':
                    case '<':
                    case '>':
                        op = c;
                        Read();
                        break;
                    case ',':
                    {
                        Read();

                        if (commaList == null)
                        {
                            commaList = new ValueList();
                            commaList.PreferComma = true;
                        }

                        var node = Expression.CalculateTree(tempNodes.ToArray());
                        tempNodes.Clear();
<commit_msg>Fix bug<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using SassSharp.Exceptions;
using SassSharp.IO;
using SassSharp.Model;
using SassSharp.Model.Expressions;
using SassSharp.Model.Nodes;

namespace SassSharp
{
    internal class ScssReader : StreamReader
    {
        private int _lineNumber;


        public ScssReader(PathFile file) : base(file.GetStream())
        {
            File = file;
        }

        public PathFile File { get; set; }


        public override int Peek()
        {
            var c = base.Peek();

            //Ignore CR
            while ((char) c == '\r')
            {
                base.Read();
                c = base.Peek();
            }

            return c;
        }

        public override int Read()
        {
            var c = base.Read();

            //Ignore CR
            while ((char) c == '\r')
            {
                c = base.Read();
            }

            if ((char) c == '\n')
                ++_lineNumber;

            return c;
        }

        /// <summary>
        ///     Skips whitespace
        /// </summary>
        /// <returns>true if whitespace was found</returns>
        private bool SkipWhitespace()
        {
            var foundWhitespace = false;

            while (!EndOfStream)
            {
                var c = (char) Peek();

                if (char.IsWhiteSpace(c))
                {
                    foundWhitespace = true;
                    Read();
                }
                else
                {
                    return foundWhitespace;
                }
            }

            return false;
        }

        /// <summary>
        ///     Reads one character, throwing an exception if not as expected.
        /// </summary>
        /// <param name="c"></param>
        private void Expect(char expected)
        {
            SkipWhitespace();

            if (EndOfStream)
                throw new ScssReaderException($"Expected {expected}, end of stream", File.Path, _lineNumber);

            var c = (char) Read();

            if (c != expected)
                throw new ScssReaderException($"Expected '{expected}', found '{c}'", File.Path, _lineNumber);
        }

        private void Optional(char expected, bool skipWhitespace = true)
        {
            if (skipWhitespace)
                SkipWhitespace();

            if (EndOfStream)
                throw new ScssReaderException($"Expected {expected}, end of stream", File.Path, _lineNumber);

            var c = (char) Peek();

            if (c == expected)
                Read();
        }

        private string ReadUntil(char endChar)
        {
            var buffer = new StringBuilder();
            while (!EndOfStream)
            {
                var c = (char) Read();

                if (c == endChar)
                    return buffer.ToString();

                buffer.Append(c);
            }

            throw new Exception($"Expected {endChar}");
        }

        internal ScssPackage ReadTree()
        {
            var package = new ScssPackage(File);
            package.LoadBuiltInFunctions();

            ScopeNode currentScope = package;
            ReadScopeContent(currentScope);

            return package;
        }

        private void ReadScopeContent(ScopeNode currentScope)
        {
            var buffer = new StringBuilder();
            var interpolations = new List<ValueList>();

            //States
            var inCommentStart = false;
            var paranthesesLevel = 0;
            var inDoubleQuotes = false;

            while (!EndOfStream)
            {
                var c = (char) Read();

                if (inDoubleQuotes)
                {
                    buffer.Append(c);

                    if (c == '"')
                    {
                        inDoubleQuotes = false;
                    }
                }
                else
                {
                    switch (c)
                    {
                        case ' ':
                            //Ignore starting white space
                            if (buffer.Length == 0)
                                break;
                            goto default;
                        case '@':
                            if (buffer.Length != 0)
                                throw new Exception("Must be first character");

                            ReadAt(currentScope);

                            break;
                        case '$':
                            var name = ReadUntil(':');
                            var vn = new VariableNode(name);
                            vn.Expression = new Expression(ReadValueList(';'));
                            Expect(';');
                            currentScope.Add(vn);
                            break;
                        case '/':
                            if (inCommentStart)
                            {
                                ReadInlineComment();

                                inCommentStart = false;

                                //Remove the first slash from the buffer
                                buffer.Length--;
                            }
                            else
                            {
                                buffer.Append(c);
                                inCommentStart = true;
                            }
                            break;
                        case '*':
                            if (inCommentStart)
                            {
                                var node = ReadComment();
                                currentScope.Add(node);
                                inCommentStart = false;

                                //Remove the first slash from the buffer
                                buffer.Length--;
                                break;
                            }
                            goto default;
                        case ';':
                        {
                            throw new ScssReaderException("Unexpected ;", File.Path, _lineNumber);
                        }
                        case ':':
                            var pc = (char) Peek();
                            if (char.IsLetter(pc)) //hover etc.
                            {
                                buffer.Append(c);
                            }
                            else
                            {
                                var pn = new PropertyNode();
                                pn.Name = new ScssString(buffer.ToString().Trim(), interpolations);
                                pn.Expression = new Expression(ReadValueList(';', '{'));
                                buffer.Clear();

                                var pc2 = (char) Read();
                                if (pc2 == ';')
                                {
                                    currentScope.Add(pn);
                                }
                                else if (pc2 == '{')
                                {
                                    var result = new NamespaceNode(pn);
                                    ReadScopeContent(result);
                                    currentScope.Add(result);
                                }
                                else
                                {
                                    throw new ScssReaderException("Expected { or ;", File.Path, _lineNumber);
                                }
                            }
                            break;
                        case '"':
                            inDoubleQuotes = true;
                            goto default;
                        case '(':
                            paranthesesLevel++;
                            goto default;
                        case ')':
                            paranthesesLevel--;
                            goto default;
                        case '{':
                        {
                            var node = new SelectorNode();
                            node.Selector = new ScssString(buffer.ToString().Trim(), interpolations);
                            ReadScopeContent(node);
                            currentScope.Add(node);
                            buffer.Clear();
                        }
                            break;
                        case '}':
                            return;
                        case '#':
                            if (Peek() == '{')
                            {
                                inCommentStart = false;
                                var value = ReadInterpolation();
                                buffer.Append("{" + interpolations.Count + "}");
                                interpolations.Add(value);
                                break;
                            }
                            else
                            {
                                goto default;
                            }
                        case '\n':
                            inCommentStart = false;
                            break;
                        default:
                            inCommentStart = false;
                            buffer.Append(c);
                            break;
                    }
                }
            }
        }

        private ValueList ReadInterpolation()
        {
            Expect('{');
            ValueList vl = ReadValueList('}');
            Expect('}');
            return vl;
        }

        private void ReadAt(ScopeNode currentScope)
        {
            string type = ReadName();

            SkipWhitespace();

            switch (type)
            {
                case "media":
                    ReadMedia(currentScope);
                    break;
                case "mixin":
                    ReadMixin(currentScope);
                    break;
                case "content":
                    ReadContent(currentScope);
                    break;
                case "include":
                    ReadInclude(currentScope);
                    break;
                case "import":
                    ReadImport(currentScope);
                    break;
                case "function":
                    ReadFunction(currentScope);
                    break;
                case "return":
                    ReadReturn(currentScope);
                    break;
                case "if":
                    ReadIf(currentScope);
                    break;
                case "else":
                    ReadElse(currentScope);
                    break;
                default:
                    throw new ScssReaderException($"Could not recognize @{type}", File.Path, _lineNumber);
            }
        }


        private void ReadIf(ScopeNode currentScope)
        {
            var val = ReadValueList('{');
            Expect('{');

            var ifNode = new IfNode(val);
            currentScope.Add(ifNode);

            ReadScopeContent(ifNode);
        }

        private void ReadElse(ScopeNode currentScope)
        {
            ElseNode elseNode;

            if (Peek() == 'i') //Assume if
            {
                Read();
                Expect('f');
                SkipWhitespace();

                var val = ReadValueList('{');
                Expect('{');

                elseNode = new ElseNode(val);
            }
            else
            {
                elseNode = new ElseNode();
            }

            var parentIf = currentScope.Nodes.Last() as IfNode;

            if (parentIf == null)
                throw new Exception("Else without if");

            parentIf.Elses.Add(elseNode);

            ReadScopeContent(elseNode);
        }


        private void ReadImport(ScopeNode currentScope)
        {
            var path = ReadUntil(';');

            currentScope.Add(new ImportNode(path));
        }


        private void ReadMedia(ScopeNode currentScope)
        {
            var def = ReadUntil('{');

            MediaNode node = new MediaNode()
            {
                Definition = def.TrimEnd()
            };


            ReadScopeContent(node);
            currentScope.Add(node);
        }


        private void ReadMixin(ScopeNode currentScope)
        {
            var name = ReadName();

            VariableNode[] args = new VariableNode[0];

            SkipWhitespace();

            if (Peek() == '(')
                args = ReadArgumentDefinition().ToArray();

            var node = new MixinNode(name, args);
            Expect('{');
            ReadScopeContent(node);
            currentScope.Add(node);
        }

        private void ReadContent(ScopeNode currentScope)
        {
            currentScope.Add(new ContentNode());
            Optional(';');
        }

        private void ReadInclude(ScopeNode currentScope)
        {
            var name = ReadName();
            ValueList args = new ValueList();

            SkipWhitespace();

            if (Peek() == '(')
                args = ReadArgumentCall();

            SkipWhitespace();

            var result = new IncludeNode(name, args);
            currentScope.Add(result);

            if (Peek() == '{')
            {
                Expect('{');
                ReadScopeContent(result);
            }
            else
                Expect(';');
        }

        private void ReadFunction(ScopeNode currentScope)
        {
            var name = ReadName();

            var args = ReadArgumentDefinition();

            var node = new FunctionNode(name, args.ToArray());
            Expect('{');
            ReadScopeContent(node);
            currentScope.Add(node);
        }

        private void ReadReturn(ScopeNode currentScope)
        {
            ExpressionNode value = ReadValueList(';');
            Expect(';');
            var result = new ReturnNode(new Expression(value));
            currentScope.Add(result);
        }

        private ValueList ReadArgumentCall()
        {
            Expect('(');
            var result = ReadValueList(')');
            Expect(')');
            return result;
        }

        private IEnumerable<VariableNode> ReadArgumentDefinition()
        {
            var result = new List<VariableNode>();

            Expect('(');

            while (!EndOfStream)
            {
                var c = (char) Read();

                switch (c)
                {
                    case '$':
                        var name = ReadName();

                        var node = new VariableNode(name);
                        result.Add(node);

                        SkipWhitespace();
                        c = (char) Peek();
                        if (c == ':')
                        {
                            Read();
                            node.Expression = new Expression(ReadValueList(',', ')'));
                        }

                        break;
                    case ',':
                    case ' ':
                        break;
                    case ')':
                        return result;
                }

                //var expr = ReadValue();
            }

            throw new Exception("Could not find argument ending");
        }


        /// <summary>
        ///     Reads a name, stopping at (, {, ; and whitespace.  Avoids consuming the last character
        /// </summary>
        /// <returns></returns>
        private string ReadName()
        {
            var firstChar = (char) Peek();

            if (!char.IsLetter(firstChar))
                throw new Exception("First character in name must be a letter");

            var buffer = new StringBuilder();
            buffer.Append((char) Read());

            while (!EndOfStream)
            {
                var c = (char) Peek();

                if (!char.IsLetter(c)
                    && !char.IsDigit(c)
                    && c != '-'
                    && c != '_')
                    break;

                buffer.Append((char) Read());
            }

            return buffer.ToString();
        }

        private ValueList ReadValueList(params char[] expectedEndCharaters)
        {
            ValueList commaList = null;
            ValueList spaceList = null;
            string key = null;

            var tempNodes = new List<ExpressionNode>();
            var op = '+';

            while (!EndOfStream)
            {
                var afterSpace = SkipWhitespace();
                var c = (char) Peek();

                if (expectedEndCharaters.Contains(c))
                {
                    var node = Expression.CalculateTree(tempNodes.ToArray());

                    if (commaList != null)
                    {
                        if (spaceList != null)
                        {
                            spaceList.Add(node);
                            commaList.Add(spaceList);
                            spaceList = null;
                        }
                        else
                        {
                            commaList.Add(node);
                        }

                        return commaList;
                    }
                    if (spaceList == null)
                    {
                        spaceList = new ValueList();
                    }
                    spaceList.Add(node);
                    return spaceList;
                }

                switch (c)
                {
                    case ')':
                    case '{':
                    case ';':
                    {
                        throw new Exception("Unexpected end character");
                    }
                    case '!':
                        Read();
                        c = (char) Peek();
                        if (c == '=')
                        {
                            op = '!';
                            Read();
                            break;
                        }
                        var name = ReadName();

                        if (spaceList == null)
                        {
                            spaceList = new ValueList();
                        }

                        spaceList.Add(Expression.CalculateTree(tempNodes.ToArray()));
                        tempNodes.Clear();

                        //Make a special modifierNode?
                        tempNodes.Add(new ValueNode($"!{name}")
                        {
                            Operator = '+'
                        });

                        continue;
                    case '=':
                        Read();
                        if (Peek() != '=')
                        {
                            op = 'e';
                        }
                        else
                        {
                            op = c;
                            Expect('=');
                        }
                        break;
                    case '-':
                    case '+':
                    case '*':
                    case '/':
                    case '<':
                    case '>':
                        op = c;
                        Read();
                        break;
                    case ',':
                    {
                        Read();

                        if (commaList == null)
                        {
                            commaList = new ValueList();
                            commaList.PreferComma = true;
                        }

                        var node = Expression.CalculateTree(tempNodes.ToArray());
                        tempNodes.Clear();

{'fraction_non_alphanumeric': 0.08892532795156408, 'fraction_numerical': 0.00035317860746720485, 'mean_word_length': 0.7608048682983165, 'pattern_counts': {'":': 18, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6413088', 'n_tokens_mistral': 8789, 'n_tokens_neox': 7986, 'n_words': 1856}