<commit_before>﻿using sly.lexer.fsm;
using System;
using System.Collections.Generic;
using System.Text;

namespace sly.lexer
{

    public enum GenericToken
    {
        Identifier,
        Int,
        Double,
        KeyWord,
        String,
        SugarToken
    }

    public enum EOLType
    {
        Windows,
        Nix,
        Environment
    }

    public class GenericLexer<IN> : ILexer<IN> where IN : struct
    {
        private const string in_string = "in_string";
        private const string string_end = "string_end";
        private const string start = "start";
        private const string in_int = "in_int";
        private const string start_double = "start_double";
        private const string in_double = "in_double";
        private const string in_identifier = "in_identifier";
        private const string token_property = "token";
        public const string DerivedToken = "derivedToken";
        private const string defaultIdentifierKey = "identifier";
        private FSMLexer<GenericToken, GenericToken> LexerFsm;

        private Dictionary<GenericToken, Dictionary<string, IN>> derivedTokens;
        private IN identifierDerivedToken;
        private FSMLexerBuilder<GenericToken, GenericToken> FSMBuilder;


        public GenericLexer(EOLType eolType)
        {
            InitializeStaticLexer(eolType);
            derivedTokens = new Dictionary<GenericToken, Dictionary<string, IN>>();
        }

        private void InitializeStaticLexer(EOLType eolType)
        {
            FSMBuilder = new FSMLexerBuilder<GenericToken, GenericToken>();


            // conf
            FSMBuilder.IgnoreWS()
                .WhiteSpace(' ')
                .WhiteSpace('\t')
                .IgnoreEOL();
            switch (eolType)
            {
                case EOLType.Windows:
                    {
                        FSMBuilder.UseWindowsEOL();
                        break;
                    }
                case EOLType.Nix:
                    {
                        FSMBuilder.UseNixEOL();
                        break;
                    }
                case EOLType.Environment:
                    {
                        FSMBuilder.UseEnvironmentEOL();
                        break;
                    }
            }


            // start machine definition
            FSMBuilder.Mark(start);



            // string literal
            FSMBuilder.Transition('\"', GenericToken.String)
                .Mark(in_string)
                .ExceptTransitionTo(new char[] { '\"', '\\' }, "in_string", GenericToken.String)
                .Transition('\\', GenericToken.String)
                .Mark("escape")
                .AnyTransitionTo(' ', "in_string", GenericToken.String)
                .Transition('\"', GenericToken.String)
                .End(GenericToken.String)
                .Mark(string_end);

            // identifier
            FSMBuilder.GoTo(start).
            RangeTransition('a', 'z', GenericToken.Identifier).
            Mark(in_identifier)
            .End(GenericToken.Identifier);

            FSMBuilder.GoTo(start).
            RangeTransitionTo('A', 'Z', in_identifier, GenericToken.Identifier, GenericToken.Identifier).
            RangeTransitionTo('a', 'z', in_identifier,GenericToken.Identifier).
            RangeTransitionTo('A', 'Z', in_identifier, GenericToken.Identifier).
            End(GenericToken.Identifier);


            //numeric
            FSMBuilder.GoTo(start)
            .RangeTransition('0', '9', GenericToken.Int, GenericToken.Double)
            .Mark(in_int)
            .RangeTransitionTo('0', '9', in_int, GenericToken.Int, GenericToken.Double)
            .End(GenericToken.Int)
            .Transition('.', GenericToken.Double)
            .Mark(start_double)
            .RangeTransition('0', '9', GenericToken.Int, GenericToken.Int, GenericToken.Double)
            .Mark(in_double)
            .RangeTransitionTo('0', '9', in_double, GenericToken.Int, GenericToken.Double)
            .End(GenericToken.Double);


            string code = "{\"d\" : 42.42 , \"i\" : 42 , \"s\" : \"quarante-deux\" }";
            LexerFsm = FSMBuilder.Fsm;
        }


        public void AddLexeme(GenericToken generic, IN token)
        {
            if (generic == GenericToken.Identifier)
            {
                identifierDerivedToken = token;
                var derived = new Dictionary<string, IN>();
                if (derivedTokens.ContainsKey(generic))
                {
                    derived = derivedTokens[generic];
                }
                derived[defaultIdentifierKey] = token;
                return;
            }

            NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
            {
                if (match.Result.TokenID == GenericToken.Identifier)<commit_msg>cleaning<commit_after>﻿using sly.lexer.fsm;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace sly.lexer
{

    public enum GenericToken
    {
        Identifier,
        Int,
        Double,
        KeyWord,
        String,
        SugarToken
    }

    public enum EOLType
    {
        Windows,
        Nix,
        Environment
    }

    public class GenericLexer<IN> : ILexer<IN> where IN : struct
    {
        private const string in_string = "in_string";
        private const string string_end = "string_end";
        private const string start = "start";
        private const string in_int = "in_int";
        private const string start_double = "start_double";
        private const string in_double = "in_double";
        private const string in_identifier = "in_identifier";
        private const string token_property = "token";
        public const string DerivedToken = "derivedToken";
        private const string defaultIdentifierKey = "identifier";
        private const string escape_string = "escape";
        private FSMLexer<GenericToken, GenericToken> LexerFsm;

        private Dictionary<GenericToken, Dictionary<string, IN>> derivedTokens;
        private IN identifierDerivedToken;
        private FSMLexerBuilder<GenericToken, GenericToken> FSMBuilder;


        public GenericLexer(EOLType eolType, params GenericToken[] staticTokens)
        {
            InitializeStaticLexer(eolType,staticTokens);
            derivedTokens = new Dictionary<GenericToken, Dictionary<string, IN>>();
        }

        private void InitializeStaticLexer(EOLType eolType, params GenericToken[] staticTokens)
        {
            FSMBuilder = new FSMLexerBuilder<GenericToken, GenericToken>();


            // conf
            FSMBuilder.IgnoreWS()
                .WhiteSpace(' ')
                .WhiteSpace('\t')
                .IgnoreEOL();
            switch (eolType)
            {
                case EOLType.Windows:
                    {
                        FSMBuilder.UseWindowsEOL();
                        break;
                    }
                case EOLType.Nix:
                    {
                        FSMBuilder.UseNixEOL();
                        break;
                    }
                case EOLType.Environment:
                    {
                        FSMBuilder.UseEnvironmentEOL();
                        break;
                    }
            }


            // start machine definition
            FSMBuilder.Mark(start);



            if (staticTokens.ToList().Contains(GenericToken.String))
            {

                // string literal
                FSMBuilder.Transition('\"', GenericToken.String)
                    .Mark(in_string)
                    .ExceptTransitionTo(new char[] { '\"', '\\' }, in_string, GenericToken.String)
                    .Transition('\\', GenericToken.String)
                    .Mark(escape_string)
                    .AnyTransitionTo(' ', in_string, GenericToken.String)
                    .Transition('\"', GenericToken.String)
                    .End(GenericToken.String)
                    .Mark(string_end);
            }

            if (staticTokens.ToList().Contains(GenericToken.Identifier))
            {
                // identifier
                FSMBuilder.GoTo(start).
            RangeTransition('a', 'z', GenericToken.Identifier).
            Mark(in_identifier)
            .End(GenericToken.Identifier);

                FSMBuilder.GoTo(start).
                RangeTransitionTo('A', 'Z', in_identifier, GenericToken.Identifier, GenericToken.Identifier).
                RangeTransitionTo('a', 'z', in_identifier, GenericToken.Identifier).
                RangeTransitionTo('A', 'Z', in_identifier, GenericToken.Identifier).
                End(GenericToken.Identifier);
            }

            //numeric
            if (staticTokens.ToList().Contains(GenericToken.Int) || staticTokens.ToList().Contains(GenericToken.Double))
            {
                FSMBuilder = FSMBuilder.GoTo(start)
            .RangeTransition('0', '9', GenericToken.Int, GenericToken.Double)
            .Mark(in_int)
            .RangeTransitionTo('0', '9', in_int, GenericToken.Int, GenericToken.Double)
            .End(GenericToken.Int);
                if (staticTokens.ToList().Contains(GenericToken.Double))
                {
                    FSMBuilder.Transition('.', GenericToken.Double)
                    .Mark(start_double)
                    .RangeTransition('0', '9', GenericToken.Int, GenericToken.Int, GenericToken.Double)
                    .Mark(in_double)
                    .RangeTransitionTo('0', '9', in_double, GenericToken.Int, GenericToken.Double)
                    .End(GenericToken.Double);
                }
            }

            LexerFsm = FSMBuilder.Fsm;
        }


        public void AddLexeme(GenericToken generic, IN token)
        {
            if (generic == GenericToken.Identifier)
            {
                identifierDerivedToken = token;
                var derived = new Dictionary<string, IN>();
                if (derivedTokens.ContainsKey(generic))
                {
                    derived = derivedTokens[generic];
                }
                derived[defaultIdentifierKey] = token;
                return;
            }

            NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
            {
                if (match.Result.TokenID == GenericToken.Identifier)
{'fraction_non_alphanumeric': 0.09630556090051953, 'fraction_numerical': 0.0021166057340773524, 'mean_word_length': 1.6039579158316633, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5970473', 'n_tokens_mistral': 2661, 'n_tokens_neox': 2516, 'n_words': 507}