<commit_before>﻿namespace TAUtil.Tdf
{
    using System.Globalization;
    using System.IO;
    using System.Linq;
    using System.Text;
    using System.Text.RegularExpressions;

    public class TdfParser
    {
        private const int EofSignal = -1;

        private readonly TextReader reader;

        private readonly ITdfNodeAdapter adapter;

        private int currentLine = 1;

        private int currentColumn = 1;

        public TdfParser(Stream s, ITdfNodeAdapter adapter)
            : this(new StreamReader(s), adapter)
        {
        }

        public TdfParser(TextReader reader, ITdfNodeAdapter adapter)
        {
            this.reader = new TdfLexingReader(reader);
            this.adapter = adapter;
        }

        public void Load()
        {
            this.OptionalWhitespace();

            while (!this.Accept(EofSignal))
            {
                this.Block();
                this.OptionalWhitespace();
            }
        }

        private void Block()
        {
            var title = this.BlockTitle();

            this.adapter.BeginBlock(title);

            this.OptionalWhitespace();
            this.BlockBody();

            this.adapter.EndBlock();
        }

        private string BlockTitle()
        {
            this.Expect('[');
            var s = this.BlockName();
            this.Expect(']');

            return s;
        }

        private string BlockName()
        {
            StringBuilder s = new StringBuilder();

            char next = (char)this.Next();
            while (this.AcceptNotAny(']', EofSignal))
            {
                s.Append(next);
                next = (char)this.Next();
            }

            return s.ToString();
        }

        private void BlockBody()
        {
            this.Expect('{');
            this.OptionalWhitespace();

            while (!this.Accept('}'))
            {
                this.BlockBodyEntry();
                this.OptionalWhitespace();
            }
        }

        private void BlockBodyEntry()
        {
            if (this.Next() == '[')
            {
                this.Block();
            }
            else
            {
                this.Property();
            }
        }

        private void Property()
        {
            var name = this.PropertyName();
            this.Expect('=');
            var value = this.PropertyValue();
            this.Terminator();

            this.adapter.AddProperty(name, value);
        }

        private void Terminator()
        {
            if (!this.AcceptAny(';', '\n'))
            {
                this.Error();
            }
        }

        private string PropertyName()
        {
            var sb = new StringBuilder();
            char next = (char)this.Next();

            if (!this.AcceptNotAny('=', '\n', ';', EofSignal))
            {
                this.Error();
            }

            sb.Append(next);
            next = (char)this.Next();

            while (this.AcceptNotAny('=', '\n', ';', EofSignal))
            {
                sb.Append(next);
                next = (char)this.Next();
            }

            return sb.ToString();
        }

        private string PropertyValue()
        {
            var sb = new StringBuilder();

            char next = (char)this.Next();

            while (this.AcceptNotAny('\n', ';', EofSignal))
            {
                sb.Append(next);
                next = (char)this.Next();
            }

            return sb.ToString();
        }

        private void OptionalWhitespace()
        {
            while (this.AcceptWhitespace())
            {
                // keep accepting
            }
        }

        private bool AcceptWhitespace()
        {
            return this.AcceptAny(' ', '\n', '\t');
        }

        private int Next()
        {
            return this.reader.Peek();
        }

        private void Consume()
        {
            this.currentColumn++;

            if (this.Next() == '\n')<commit_msg>Permit newline in tdf property values<commit_after>﻿namespace TAUtil.Tdf
{
    using System.Globalization;
    using System.IO;
    using System.Linq;
    using System.Text;
    using System.Text.RegularExpressions;

    public class TdfParser
    {
        private const int EofSignal = -1;

        private readonly TextReader reader;

        private readonly ITdfNodeAdapter adapter;

        private int currentLine = 1;

        private int currentColumn = 1;

        public TdfParser(Stream s, ITdfNodeAdapter adapter)
            : this(new StreamReader(s), adapter)
        {
        }

        public TdfParser(TextReader reader, ITdfNodeAdapter adapter)
        {
            this.reader = new TdfLexingReader(reader);
            this.adapter = adapter;
        }

        public void Load()
        {
            this.OptionalWhitespace();

            while (!this.Accept(EofSignal))
            {
                this.Block();
                this.OptionalWhitespace();
            }
        }

        private void Block()
        {
            var title = this.BlockTitle();

            this.adapter.BeginBlock(title);

            this.OptionalWhitespace();
            this.BlockBody();

            this.adapter.EndBlock();
        }

        private string BlockTitle()
        {
            this.Expect('[');
            var s = this.BlockName();
            this.Expect(']');

            return s;
        }

        private string BlockName()
        {
            StringBuilder s = new StringBuilder();

            char next = (char)this.Next();
            while (this.AcceptNotAny(']', EofSignal))
            {
                s.Append(next);
                next = (char)this.Next();
            }

            return s.ToString();
        }

        private void BlockBody()
        {
            this.Expect('{');
            this.OptionalWhitespace();

            while (!this.Accept('}'))
            {
                this.BlockBodyEntry();
                this.OptionalWhitespace();
            }
        }

        private void BlockBodyEntry()
        {
            if (this.Next() == '[')
            {
                this.Block();
            }
            else
            {
                this.Property();
            }
        }

        private void Property()
        {
            var name = this.PropertyName();
            this.Expect('=');
            var value = this.PropertyValue();
            this.Terminator();

            this.adapter.AddProperty(name, value);
        }

        private void Terminator()
        {
            if (!this.Accept(';'))
            {
                this.Error();
            }
        }

        private string PropertyName()
        {
            var sb = new StringBuilder();
            char next = (char)this.Next();

            if (!this.AcceptNotAny('=', '\n', ';', EofSignal))
            {
                this.Error();
            }

            sb.Append(next);
            next = (char)this.Next();

            while (this.AcceptNotAny('=', '\n', ';', EofSignal))
            {
                sb.Append(next);
                next = (char)this.Next();
            }

            return sb.ToString();
        }

        private string PropertyValue()
        {
            var sb = new StringBuilder();

            char next = (char)this.Next();

            while (this.AcceptNotAny(';', EofSignal))
            {
                sb.Append(next);
                next = (char)this.Next();
            }

            return sb.ToString();
        }

        private void OptionalWhitespace()
        {
            while (this.AcceptWhitespace())
            {
                // keep accepting
            }
        }

        private bool AcceptWhitespace()
        {
            return this.AcceptAny(' ', '\n', '\t');
        }

        private int Next()
        {
            return this.reader.Peek();
        }

        private void Consume()
        {
            this.currentColumn++;

            if (this.Next() == '\n')
{'fraction_non_alphanumeric': 0.11470333209463644, 'fraction_numerical': 0.0007432181345224824, 'mean_word_length': 1.1646112600536194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '319346', 'n_tokens_mistral': 2098, 'n_tokens_neox': 1984, 'n_words': 419}