<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace CAPNet
{
    /// <summary>
    /// 
    /// </summary>
    public static class XmlSpaceDelimitedElementsParser
    {

        private const int decisionState = 0;
        private const int addCharInSpaceContainingAddress = 1;
        private const int addCharInAddressWithNoSpace = 2;

        private static int state;
        private static string partialElement;
        private static int currentPosition;
        private static char[] representationChars;
        private static List<string> addresses;

        /// <summary>
        /// 
        /// </summary>
        /// <param name="representation"></param>
        /// <returns></returns>
        public static IEnumerable<string> GetElements(this string representation)
        {
            addresses = new List<string>();
            representationChars = representation.ToCharArray();
            state = decisionState;
            partialElement = "";

            for (currentPosition = 0; currentPosition < representationChars.Count(); currentPosition++)
            {
                char currentChar = representationChars[currentPosition];
                switch (state)
                {
                    case decisionState:
                        DecisionState(currentChar);
                        break;

                    case addCharInAddressWithNoSpace:
                        AddCharInAddressWithNoSpaceState(currentChar);
                        break;

                    case addCharInSpaceContainingAddress:

                        AddCharInSpaceContainingAddressState(currentChar);
                        break;
                }
            }

            addresses.RemoveAll(content=>content.Equals(""));
            return addresses;
        }

        private static void AddCharInSpaceContainingAddressState(char currentChar)
        {
            if (currentChar.IsElementCaracter() || currentChar.IsSpace())
            {
                state = addCharInSpaceContainingAddress;
                partialElement += currentChar;
                if (currentPosition == representationChars.Count() - 1)
                {
                    addresses.Add(partialElement);
                }
            }
            else if (currentChar.IsQuote())
            {
                addresses.Add(partialElement);
                partialElement = "";
                state = decisionState;
            }
        }

        private static void AddCharInAddressWithNoSpaceState(char currentChar)
        {
            if (currentChar.IsElementCaracter())
            {
                state = addCharInAddressWithNoSpace;
                partialElement += currentChar;
                if (currentPosition == representationChars.Count() - 1)
                {
                    addresses.Add(partialElement);
                }
            }
            else if (currentChar.IsSpace())
            {
                addresses.Add(partialElement);
                partialElement = "";
                state = decisionState;
            }
            else if (currentChar.IsQuote())
            {
                state = addCharInSpaceContainingAddress;
            }
        }

        private static void DecisionState(char currentChar)
        {
            if (currentChar.IsSpace())
            {
                state = addCharInAddressWithNoSpace;
            }
            else if (currentChar.IsQuote())
            {
                state = addCharInSpaceContainingAddress;
            }
            else if (currentChar.IsElementCaracter())
            {
                state = addCharInAddressWithNoSpace;
                partialElement += currentChar;
            }
        }

        private static bool IsElementCaracter(this char tested)
        {
            return tested.CompareTo('"') != 0 && tested.CompareTo(' ') != 0;
        }

        private static bool IsSpace(this char tested)
        {
            return tested.CompareTo(' ') == 0;
        }

        private static bool IsQuote(this char tested)
        {
            return tested.CompareTo('"') == 0;
        }

    }

}
<commit_msg>Formated document<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace CAPNet
{
    /// <summary>
    /// 
    /// </summary>
    public static class XmlSpaceDelimitedElementsParser
    {

        private const int decisionState = 0;
        private const int addCharInSpaceContainingAddress = 1;
        private const int addCharInAddressWithNoSpace = 2;

        private static int state;
        private static string partialElement;
        private static int currentPosition;
        private static char[] representationChars;
        private static List<string> addresses;

        /// <summary>
        /// 
        /// </summary>
        /// <param name="representation"></param>
        /// <returns></returns>
        public static IEnumerable<string> GetElements(this string representation)
        {
            addresses = new List<string>();
            representationChars = representation.ToCharArray();
            state = decisionState;
            partialElement = "";

            for (currentPosition = 0; currentPosition < representationChars.Count(); currentPosition++)
            {
                char currentChar = representationChars[currentPosition];
                switch (state)
                {
                    case decisionState:
                        DecisionState(currentChar);
                        break;

                    case addCharInAddressWithNoSpace:
                        AddCharInAddressWithNoSpaceState(currentChar);
                        break;

                    case addCharInSpaceContainingAddress:

                        AddCharInSpaceContainingAddressState(currentChar);
                        break;
                }
            }

            addresses.RemoveAll(content => content.Equals(""));
            return addresses;
        }

        private static void AddCharInSpaceContainingAddressState(char currentChar)
        {
            if (currentChar.IsElementCaracter() || currentChar.IsSpace())
            {
                state = addCharInSpaceContainingAddress;
                partialElement += currentChar;
                if (currentPosition == representationChars.Count() - 1)
                {
                    addresses.Add(partialElement);
                }
            }
            else if (currentChar.IsQuote())
            {
                addresses.Add(partialElement);
                partialElement = "";
                state = decisionState;
            }
        }

        private static void AddCharInAddressWithNoSpaceState(char currentChar)
        {
            if (currentChar.IsElementCaracter())
            {
                state = addCharInAddressWithNoSpace;
                partialElement += currentChar;
                if (currentPosition == representationChars.Count() - 1)
                {
                    addresses.Add(partialElement);
                }
            }
            else if (currentChar.IsSpace())
            {
                addresses.Add(partialElement);
                partialElement = "";
                state = decisionState;
            }
            else if (currentChar.IsQuote())
            {
                state = addCharInSpaceContainingAddress;
            }
        }

        private static void DecisionState(char currentChar)
        {
            if (currentChar.IsSpace())
            {
                state = addCharInAddressWithNoSpace;
            }
            else if (currentChar.IsQuote())
            {
                state = addCharInSpaceContainingAddress;
            }
            else if (currentChar.IsElementCaracter())
            {
                state = addCharInAddressWithNoSpace;
                partialElement += currentChar;
            }
        }

        private static bool IsElementCaracter(this char tested)
        {
            return tested.CompareTo('"') != 0 && tested.CompareTo(' ') != 0;
        }

        private static bool IsSpace(this char tested)
        {
            return tested.CompareTo(' ') == 0;
        }

        private static bool IsQuote(this char tested)
        {
            return tested.CompareTo('"') == 0;
        }

    }

}

{'fraction_non_alphanumeric': 0.08010335917312661, 'fraction_numerical': 0.002349072116513977, 'mean_word_length': 1.4468390804597702, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6098222', 'n_tokens_mistral': 1995, 'n_tokens_neox': 1852, 'n_words': 402}