<commit_before>using System;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;

namespace AvsAnLib.Internals {
    public static class NodeSerializer {
        public static Node Deserialize(string rawDict) {
            var mutableRoot = new Node();
            foreach (
                Match m in Regex.Matches(rawDict, @"([^(\n]*)\(([0-9a-f]*):([0-9a-f]*)\)", RegexOptions.CultureInvariant)
                )
                mutableRoot.LoadPrefixRatio(
                    m.Groups[1].Value,
                    0,
                    new Ratio {
                        aCount = parseHex(m.Groups[2].Value),
                        anCount = parseHex(m.Groups[3].Value)
                    });
            return mutableRoot;
        }

        static int parseHex(string str) {
            return str == "" ? 0 : Int32.Parse(str, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture);
        }

        public static string SerializeReadably(Node node) {
            return SerializeImpl(node, true);
        }

        public static string Serialize(Node node) {
            return SerializeImpl(node, false);
        }

        static string SerializeImpl(Node node, bool withNewlines) {
            var sb = new StringBuilder();
            SerializeImpl(node, sb, "", withNewlines);
            return sb.ToString();
        }<commit_msg>Replace regex based parser with handrolled (faster) implementation.<commit_after>using System.Text;

namespace AvsAnLib.Internals {
    public static class NodeSerializer {
        public static Node Deserialize(string rawDict) {
            var mutableRoot = new Node();
            int pos = 0;
            int len = rawDict.Length;
            while (true) {
                while (true) {
                    if (pos >= len)
                        return mutableRoot;
                    if (rawDict[pos] != ' ' && rawDict[pos] != '\n' && rawDict[pos] != '\r')
                        break;
                    pos++;
                }
                int start = pos;
                while (true) {
                    if (pos >= len)
                        return mutableRoot;
                    if (rawDict[pos] == '(')
                        break;
                    pos++;
                }
                int paren0 = pos;
                pos++;
                while (true) {
                    if (pos >= len)
                        return mutableRoot;
                    if (rawDict[pos] == ':')
                        break;
                    pos++;
                }
                int sep = pos;
                while (true) {
                    if (pos >= len)
                        return mutableRoot;
                    if (rawDict[pos] == ')')
                        break;
                    pos++;
                }
                int paren1 = pos;
                mutableRoot.LoadPrefixRatio(
                   rawDict.Substring(start, paren0 - start),
                   0,
                   new Ratio {
                       aCount = parseHex(rawDict, paren0 + 1, sep),
                       anCount = parseHex(rawDict, sep + 1, paren1)
                   });
                pos++;
            }
        }

        static int parseHex(string str, int start, int end) {
            int retval = 0;
            while (start < end) {
                retval = retval << 4;
                char c = str[start];
                int charVal;
                if (c <= '9')
                    charVal = c - '0';
                else if (c <= 'F')
                    charVal = c - 'A' + 10;
                else
                    charVal = c - 'a' + 10;
                retval += charVal;
                start++;
            }
            return retval;
        }

        public static string SerializeReadably(Node node) {
            return SerializeImpl(node, true);
        }

        public static string Serialize(Node node) {
            return SerializeImpl(node, false);
        }

        static string SerializeImpl(Node node, bool withNewlines) {
            var sb = new StringBuilder();
            SerializeImpl(node, sb, "", withNewlines);
            return sb.ToString();
        }
{'fraction_non_alphanumeric': 0.09693396226415095, 'fraction_numerical': 0.006603773584905661, 'mean_word_length': 1.1026276648487854, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7243105', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1010, 'n_words': 270}