<commit_before>﻿using UnityEngine;
using System.Collections;
using System.Collections.Generic;

// Helper class to read an ini file into a nested dictionary
public static class IniRead{
    // Function takes path to ini file and returns data object
    // Returns null on error

    public static IniData ReadFromIni(string path)
    {
        string[] lines;
        
        // Read the whole file
        try
        {
            lines = System.IO.File.ReadAllLines(path);
        }
        catch(System.Exception e)
        {
            return null;
        }
        // Create a dictionary for the first section
        Dictionary<string, string> entryData = new Dictionary<string, string>();
        // Name for the current section
        string entryName = "";
        // Create object to hold output
        IniData output = new IniData();
        // Section headers have these chars removed
        char[] charsToTrim = { '[', ']' };

        // Parse all lines
        foreach (string l in lines)
        {
            // Start of new section
            if (l.Length > 0 && l.Trim()[0] == '[')
            {
                // If not first section, add the last section of data
                if (entryName != "")
                {
                    if (!output.Add(entryName, entryData))
                    {
                        Debug.Log("Warning: duplicate section \"" + entryName + "\" in " + path + " will be ignored.");
                    }
                }
                // create new data for new section
                entryData = new Dictionary<string, string>();
                // Get name of new section
                entryName = l.Trim().Trim(charsToTrim);
                // Blank section names not allowed, but not fatal
                if(entryName.Equals(""))
                {
                    Debug.Log("Warning: empty section in " + path + " will be ignored.");
                }
            }
            // If the line is not a comment (starts with ;)
            else if (l.Length > 0 && l.Trim()[0] != ';')
            {
                int equalsLocation = l.IndexOf('=');
                // Add data as entry with no value
                if (equalsLocation == -1)
                    entryData.Add(l.Trim(), "");
                // If there is an = add data as key and value
                else
                {
                    string key = l.Substring(0, equalsLocation).Trim();
                    if(entryData.ContainsKey(key))
                    {
                        Debug.Log("Warning: duplicate \"" + key + "\" data in section \"" + entryName + "\" in " + path + " will be ignored.");
                    }
                    else
                    {
                        string value = FFGLookup(l.Substring(equalsLocation + 1).Trim().Trim('\"'));
                        entryData.Add(key, SymbolReplace(value));
                    }
                }
                // This won't go anywhere if we don't have a section
                if (entryName.Equals(""))
                {
                    Debug.Log("Warning: data without section in " + path + " will be ignored.");
                }
            }
        }

        // Add the last section
        if (entryName != "")
        {
            if (!output.Add(entryName, entryData))
            {
                Debug.Log("Warning: duplicate section \"" + entryName + "\" in " + path + " will be ignored.");
            }
        }

        return output;
    }

    public static string SymbolReplace(string input)
    {
        string output = input;
        output = output.Replace("{heart}", "≥");
        output = output.Replace("{fatigue}", "∏");
        output = output.Replace("{might}", "∂");
        output = output.Replace("{will}", "π");
        output = output.Replace("{knowledge}", "∑");
        output = output.Replace("{awareness}", "μ");
        output = output.Replace("{action}", "∞");
        output = output.Replace("{shield}", "±");
        output = output.Replace("{surge}", "≥");
        return output;
    }

    public static string FFGLookup(string input)
    {
        string output = input;
        while (output.IndexOf("{ffg:") != -1)
        {
            int lookupStart = output.IndexOf("{ffg:") + "{ffg:".Length;
            int lookupEnd = output.Substring(lookupStart).IndexOf("}");
            string lookup = output.Substring(lookupStart, lookupEnd);
            string result = FFGKeyLookup(lookup);
            output = output.Replace("{ffg:" + lookup + "}", result);
        }
        return output;
    }

    public static string FFGKeyLookup(string key)
    {
        try
        {
            Game game = Game.Get();

            if (game.ffgText == null || game.ffgText.Length == 0)
            {
                game.ffgText = System.IO.File.ReadAllLines(ContentData.ContentPath() + "D2E/ffg/text/Localization.txt");
            }
            for (int i = 0; i < game.ffgText.Length; i++)
            {
                string[] values = game.ffgText[i].Split(',');
                if (values.Length > 1 && values[0].Equals(key))
                {
                    string returnValue = values[1];
                    int nextQuote = 0;

                    while (true)
                    {
                        nextQuote = returnValue.IndexOf("\"", nextQuote + 1);
                        if (nextQuote == returnValue.Length - 1)
                        {
                            return returnValue.Replace("\"\"", "\"").Trim('\"');
                        }<commit_msg>Fixing ',' handling in text import.<commit_after>﻿using UnityEngine;
using System.Collections;
using System.Collections.Generic;

// Helper class to read an ini file into a nested dictionary
public static class IniRead{
    // Function takes path to ini file and returns data object
    // Returns null on error

    public static IniData ReadFromIni(string path)
    {
        string[] lines;
        
        // Read the whole file
        try
        {
            lines = System.IO.File.ReadAllLines(path);
        }
        catch(System.Exception e)
        {
            return null;
        }
        // Create a dictionary for the first section
        Dictionary<string, string> entryData = new Dictionary<string, string>();
        // Name for the current section
        string entryName = "";
        // Create object to hold output
        IniData output = new IniData();
        // Section headers have these chars removed
        char[] charsToTrim = { '[', ']' };

        // Parse all lines
        foreach (string l in lines)
        {
            // Start of new section
            if (l.Length > 0 && l.Trim()[0] == '[')
            {
                // If not first section, add the last section of data
                if (entryName != "")
                {
                    if (!output.Add(entryName, entryData))
                    {
                        Debug.Log("Warning: duplicate section \"" + entryName + "\" in " + path + " will be ignored.");
                    }
                }
                // create new data for new section
                entryData = new Dictionary<string, string>();
                // Get name of new section
                entryName = l.Trim().Trim(charsToTrim);
                // Blank section names not allowed, but not fatal
                if(entryName.Equals(""))
                {
                    Debug.Log("Warning: empty section in " + path + " will be ignored.");
                }
            }
            // If the line is not a comment (starts with ;)
            else if (l.Length > 0 && l.Trim()[0] != ';')
            {
                int equalsLocation = l.IndexOf('=');
                // Add data as entry with no value
                if (equalsLocation == -1)
                    entryData.Add(l.Trim(), "");
                // If there is an = add data as key and value
                else
                {
                    string key = l.Substring(0, equalsLocation).Trim();
                    if(entryData.ContainsKey(key))
                    {
                        Debug.Log("Warning: duplicate \"" + key + "\" data in section \"" + entryName + "\" in " + path + " will be ignored.");
                    }
                    else
                    {
                        string value = FFGLookup(l.Substring(equalsLocation + 1).Trim().Trim('\"'));
                        entryData.Add(key, SymbolReplace(value));
                    }
                }
                // This won't go anywhere if we don't have a section
                if (entryName.Equals(""))
                {
                    Debug.Log("Warning: data without section in " + path + " will be ignored.");
                }
            }
        }

        // Add the last section
        if (entryName != "")
        {
            if (!output.Add(entryName, entryData))
            {
                Debug.Log("Warning: duplicate section \"" + entryName + "\" in " + path + " will be ignored.");
            }
        }

        return output;
    }

    public static string SymbolReplace(string input)
    {
        string output = input;
        output = output.Replace("{heart}", "≥");
        output = output.Replace("{fatigue}", "∏");
        output = output.Replace("{might}", "∂");
        output = output.Replace("{will}", "π");
        output = output.Replace("{knowledge}", "∑");
        output = output.Replace("{awareness}", "μ");
        output = output.Replace("{action}", "∞");
        output = output.Replace("{shield}", "±");
        output = output.Replace("{surge}", "≥");
        return output;
    }

    public static string FFGLookup(string input)
    {
        string output = input;
        while (output.IndexOf("{ffg:") != -1)
        {
            int lookupStart = output.IndexOf("{ffg:") + "{ffg:".Length;
            int lookupEnd = output.Substring(lookupStart).IndexOf("}");
            string lookup = output.Substring(lookupStart, lookupEnd);
            string result = FFGKeyLookup(lookup);
            output = output.Replace("{ffg:" + lookup + "}", result);
        }
        return output;
    }

    public static string FFGKeyLookup(string key)
    {
        try
        {
            Game game = Game.Get();

            if (game.ffgText == null || game.ffgText.Length == 0)
            {
                game.ffgText = System.IO.File.ReadAllLines(ContentData.ContentPath() + "D2E/ffg/text/Localization.txt");
            }
            for (int i = 0; i < game.ffgText.Length; i++)
            {
                string[] values = game.ffgText[i].Split(",".ToCharArray(), 2);
                if (values.Length > 1 && values[0].Equals(key))
                {
                    string returnValue = values[1];
                    int nextQuote = 0;

                    while (true)
                    {
                        nextQuote = returnValue.IndexOf("\"", nextQuote + 1);
                        if (nextQuote == returnValue.Length - 1)
                        {
                            return returnValue.Replace("\"\"", "\"").Trim('\"');
                        }
{'fraction_non_alphanumeric': 0.12541313086199196, 'fraction_numerical': 0.0031263957123715946, 'mean_word_length': 1.3310430980637102, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2865649', 'n_tokens_mistral': 2952, 'n_tokens_neox': 2746, 'n_words': 777}