<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace BibTeXLibrary
{
    public class BibParser : IDisposable
    {
        #region Const Field

        #endregion

        #region Private Field
        /// <summary>
        /// Input text stream.
        /// </summary>
        private readonly TextReader _inputText;

        /// <summary>
        /// Line No. counter
        /// </summary>
        private int _lineCount = 1;
        #endregion

        #region Constructor
        public BibParser(TextReader inputText)
        {
            _inputText = inputText;
        }
        #endregion

        #region Public Method
        public void GetResult()
        {
            Parser();
        }
        #endregion

        #region Private Method
        private void Parser()
        {
            int curState = 0;
            BibEntry bib = null;

            foreach (var token in Lexer())
            {
                

                switch(curState)
                {
                    case 0:
                        if(token.Type == TokenType.Start)
                        {
                            curState++;
                            bib = new BibEntry();
                        }
                        break;

                    case 1:
                        if(token.Type == TokenType.Name)
                        {
                            curState++;
                            bib.Type = token.Value;
                        }
                        break;
                }
            }
        }
<commit_msg>Update state machine<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace BibTeXLibrary
{
    public class BibParser : IDisposable
    {
        #region Const Field

        #endregion

        #region Private Field
        /// <summary>
        /// Input text stream.
        /// </summary>
        private readonly TextReader _inputText;

        /// <summary>
        /// Line No. counter
        /// </summary>
        private int _lineCount = 1;
        #endregion

        #region Constructor
        public BibParser(TextReader inputText)
        {
            _inputText = inputText;
        }
        #endregion

        #region Public Method
        public void GetResult()
        {
            Parser();
        }
        #endregion

        #region Private Method
        private void Parser()
        {
            int curState = 0;
            BibEntry bib = null;

            string tagName = "";

            foreach (var token in Lexer())
            {
                switch(curState)
                {
                    case 0:
                        if(token.Type == TokenType.Start)
                        {
                            curState = 1;
                            bib = new BibEntry();
                        }
                        break;

                    case 1:
                        if(token.Type == TokenType.Name)
                        {
                            curState = 2;
                            bib.Type = token.Value;
                        }
                        break;

                    case 2:
                        if (token.Type == TokenType.LeftBrace)
                        {
                            curState = 3;
                        }
                        break;

                    case 3:
                        if (token.Type == TokenType.Name)
                        {
                            curState++;
                            bib.Key = token.Value;
                        }
                        else if(token.Type == TokenType.Comma)
                        {
                            curState = 5;
                        }
                        break;

                    case 4:
                        if (token.Type == TokenType.Comma)
                        {
                            curState = 5;
                        }
                        break;

                    case 5:
                        if (token.Type == TokenType.Name)
                        {
                            curState = 6;
                            tagName = token.Value;
                            bib[tagName] = "";
                        }
                        break;

                    case 6:
                        if (token.Type == TokenType.Equal)
                        {
                            curState = 7;
                        }
                        break;

                    case 7:
                        if (token.Type == TokenType.String)
                        {
                            curState = 8;
                            bib[tagName] += token.Value;
                        }
                        break;

                    case 8:
                        if (token.Type == TokenType.Concatenation)
                        {
                            curState = 7;
                        }
                        else if(token.Type == TokenType.Comma)
                        {
                            curState = 9;
                        }
                        else if(token.Type == TokenType.RightBrace)
                        {
                            curState = 10;
                            // Add to result list
                        }
                        break;

                    case 9:
                        if (token.Type == TokenType.String)
                        {
                            curState = 8;
                            bib[tagName] += token.Value;
                        }
                        else if(token.Type == TokenType.RightBrace)
                        {
                            curState = 10;
                        }
                        break;

                    case 10:
                        if (token.Type == TokenType.Start)
                        {
                            curState = 1;
                        }
                        break;
                }
            }
        }

{'fraction_non_alphanumeric': 0.06920858699134892, 'fraction_numerical': 0.005446972124319128, 'mean_word_length': 0.6283255086071987, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5654012', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1260, 'n_words': 300}