<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Globalization;

namespace ValveResourceFormat.KeyValues
{
    //Different type of value blocks for KeyValues3
    public enum KVType
    {
        OBJECT,
        ARRAY,
        BOOLEAN,
        INTEGER,
        DOUBLE,
        STRING,
        STRING_MULTI,
        FLAGGED_STRING
    }

    //Datastructure for a KV Object
    public class KVObject
    {
        public string Key;
        public Dictionary<String, KVValue> Properties;
        private bool IsArray;
        public int Count;

        public KVObject(string name)
        {
            Key = name;
            Properties = new Dictionary<String, KVValue>();
            IsArray = false;
            Count = 0;
        }

        public KVObject(string name, bool isArray)
            : this(name)
        {
            IsArray = isArray;
        }

        //Add a property to the structure
        public virtual void AddProperty(String name, KVValue value)
        {
            if (IsArray)
            {
                //Make up a key for the dictionary
                Properties.Add(Count.ToString(), value);
            }
            else
            {
                Properties.Add(name, value);
            }

            Count++;
        }

        public void Serialize(StringBuilder stringBuilder, int indent)
        {
            if (IsArray)
            {
                SerializeArray(stringBuilder, indent);
            }
            else
            {
                SerializeObject(stringBuilder, indent);
            }
        }

        //Serialize the contents of the KV object
        private void SerializeObject(StringBuilder stringBuilder, int indent)
        {
            //Don't enter the top-most object
            if (indent > 1)
            {
                stringBuilder.Append("\n");
            }

            for (var i = 0; i < indent - 1; i++)
            {
                stringBuilder.Append("\t");
            }
            stringBuilder.Append("{\n");

            foreach (var pair in Properties)
            {
                for (var i = 0; i < indent; i++)
                {
                    stringBuilder.Append("\t");
                }

                stringBuilder.Append(pair.Key);
                stringBuilder.Append(" = ");

                PrintValue(stringBuilder, pair.Value, indent);

                stringBuilder.Append("\n");
            }

            for (var i = 0; i < indent - 1; i++)
            {
                stringBuilder.Append("\t");
            }
            stringBuilder.Append("}");
        }

        private void SerializeArray(StringBuilder stringBuilder, int indent)
        {
            stringBuilder.Append("\n");
            for (var i = 0; i < indent - 1; i++)
            {
                stringBuilder.Append("\t");
            }
            stringBuilder.Append("[\n");

            //Need to preserve the order
            for (var i = 0; i < Count; i++)
            {
                for (var j = 0; j < indent; j++)
                {
                    stringBuilder.Append("\t");
                }

                PrintValue(stringBuilder, Properties[i.ToString()], indent);

                stringBuilder.Append(",\n");
            }

            for (var i = 0; i < indent - 1; i++)
            {
                stringBuilder.Append("\t");
            }
            stringBuilder.AppendLine("]");
        }

        private string EscapeUnescaped(string input, char toEscape)
        {
            int index = 1;
            while (true)
            {
                index = input.IndexOf(toEscape, index);

                //Break out of the loop if no more occurrences were found
                if (index == -1)
                {
                    break;
                }

                if (input.ElementAt(index - 1) != '\\')
                {
                    input = input.Insert(index, "\\");
                }

                //Don't read this one again
                index++;
            }
            return input;
        }

        //Print a value in the correct representation
        private void PrintValue(StringBuilder stringBuilder, KVValue kvValue, int indent)
        {
            KVType type = kvValue.Type;
            object value = kvValue.Value;

            switch (type)
            {
                case KVType.OBJECT:
                    ((KVObject)value).Serialize(stringBuilder, indent + 1);
                    break;
                case KVType.ARRAY:
                    ((KVObject)value).Serialize(stringBuilder, indent + 1);
                    break;
                case KVType.FLAGGED_STRING:
                    stringBuilder.Append((string)value);
                    break;
                case KVType.STRING:
                    stringBuilder.Append("\"");
                    stringBuilder.Append(EscapeUnescaped((string)value, '"'));
                    stringBuilder.Append("\"");
                    break;
                case KVType.STRING_MULTI:
                    stringBuilder.Append("\"\"\"\n");
                    stringBuilder.Append(EscapeUnescaped((string)value, '"'));
                    stringBuilder.Append("\n\"\"\"");
                    break;
                case KVType.BOOLEAN:
                    stringBuilder.Append((bool)value ? "true" : "false");
                    break;
                case KVType.DOUBLE:
                    stringBuilder.Append(((double)value).ToString("#0.000000", CultureInfo.InvariantCulture));
                    break;
                case KVType.INTEGER:
                    stringBuilder.Append((int)value);
                    break;
                default:
                    //Unknown type encountered
                    throw new Exception("Unknown type encountered.");
            }
        }
    }

    //Class to hold type + value
    public class KVValue
    {
        public KVType Type;
        public object Value;

        public KVValue(KVType type, object value)
        {
            Type = type;
            Value = value;
        }
    }
}<commit_msg>Removed escaping from multi-line strings<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Globalization;

namespace ValveResourceFormat.KeyValues
{
    //Different type of value blocks for KeyValues3
    public enum KVType
    {
        OBJECT,
        ARRAY,
        BOOLEAN,
        INTEGER,
        DOUBLE,
        STRING,
        STRING_MULTI,
        FLAGGED_STRING
    }

    //Datastructure for a KV Object
    public class KVObject
    {
        public string Key;
        public Dictionary<String, KVValue> Properties;
        private bool IsArray;
        public int Count;

        public KVObject(string name)
        {
            Key = name;
            Properties = new Dictionary<String, KVValue>();
            IsArray = false;
            Count = 0;
        }

        public KVObject(string name, bool isArray)
            : this(name)
        {
            IsArray = isArray;
        }

        //Add a property to the structure
        public virtual void AddProperty(String name, KVValue value)
        {
            if (IsArray)
            {
                //Make up a key for the dictionary
                Properties.Add(Count.ToString(), value);
            }
            else
            {
                Properties.Add(name, value);
            }

            Count++;
        }

        public void Serialize(StringBuilder stringBuilder, int indent)
        {
            if (IsArray)
            {
                SerializeArray(stringBuilder, indent);
            }
            else
            {
                SerializeObject(stringBuilder, indent);
            }
        }

        //Serialize the contents of the KV object
        private void SerializeObject(StringBuilder stringBuilder, int indent)
        {
            //Don't enter the top-most object
            if (indent > 1)
            {
                stringBuilder.Append("\n");
            }

            for (var i = 0; i < indent - 1; i++)
            {
                stringBuilder.Append("\t");
            }
            stringBuilder.Append("{\n");

            foreach (var pair in Properties)
            {
                for (var i = 0; i < indent; i++)
                {
                    stringBuilder.Append("\t");
                }

                stringBuilder.Append(pair.Key);
                stringBuilder.Append(" = ");

                PrintValue(stringBuilder, pair.Value, indent);

                stringBuilder.Append("\n");
            }

            for (var i = 0; i < indent - 1; i++)
            {
                stringBuilder.Append("\t");
            }
            stringBuilder.Append("}");
        }

        private void SerializeArray(StringBuilder stringBuilder, int indent)
        {
            stringBuilder.Append("\n");
            for (var i = 0; i < indent - 1; i++)
            {
                stringBuilder.Append("\t");
            }
            stringBuilder.Append("[\n");

            //Need to preserve the order
            for (var i = 0; i < Count; i++)
            {
                for (var j = 0; j < indent; j++)
                {
                    stringBuilder.Append("\t");
                }

                PrintValue(stringBuilder, Properties[i.ToString()], indent);

                stringBuilder.Append(",\n");
            }

            for (var i = 0; i < indent - 1; i++)
            {
                stringBuilder.Append("\t");
            }
            stringBuilder.AppendLine("]");
        }

        private string EscapeUnescaped(string input, char toEscape)
        {
            int index = 1;
            while (true)
            {
                index = input.IndexOf(toEscape, index);

                //Break out of the loop if no more occurrences were found
                if (index == -1)
                {
                    break;
                }

                if (input.ElementAt(index - 1) != '\\')
                {
                    input = input.Insert(index, "\\");
                }

                //Don't read this one again
                index++;
            }
            return input;
        }

        //Print a value in the correct representation
        private void PrintValue(StringBuilder stringBuilder, KVValue kvValue, int indent)
        {
            KVType type = kvValue.Type;
            object value = kvValue.Value;

            switch (type)
            {
                case KVType.OBJECT:
                    ((KVObject)value).Serialize(stringBuilder, indent + 1);
                    break;
                case KVType.ARRAY:
                    ((KVObject)value).Serialize(stringBuilder, indent + 1);
                    break;
                case KVType.FLAGGED_STRING:
                    stringBuilder.Append((string)value);
                    break;
                case KVType.STRING:
                    stringBuilder.Append("\"");
                    stringBuilder.Append(EscapeUnescaped((string)value, '"'));
                    stringBuilder.Append("\"");
                    break;
                case KVType.STRING_MULTI:
                    stringBuilder.Append("\"\"\"\n");
                    stringBuilder.Append((string)value);
                    stringBuilder.Append("\n\"\"\"");
                    break;
                case KVType.BOOLEAN:
                    stringBuilder.Append((bool)value ? "true" : "false");
                    break;
                case KVType.DOUBLE:
                    stringBuilder.Append(((double)value).ToString("#0.000000", CultureInfo.InvariantCulture));
                    break;
                case KVType.INTEGER:
                    stringBuilder.Append((int)value);
                    break;
                default:
                    //Unknown type encountered
                    throw new Exception("Unknown type encountered.");
            }
        }
    }

    //Class to hold type + value
    public class KVValue
    {
        public KVType Type;
        public object Value;

        public KVValue(KVType type, object value)
        {
            Type = type;
            Value = value;
        }
    }
}
{'fraction_non_alphanumeric': 0.09625798108785258, 'fraction_numerical': 0.004202699426169886, 'mean_word_length': 1.1352890422778257, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6707119', 'n_tokens_mistral': 3162, 'n_tokens_neox': 2916, 'n_words': 734}