<commit_before>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;

namespace Polyglot
{
    public static class LocalizationImporter
    {
        /// <summary>
        /// A dictionary with the key of the text item you want and a list of all the languages.
        /// </summary>
        private static Dictionary<string, List<string>> languageStrings = new Dictionary<string, List<string>>();

        private static List<string> EmptyList = new List<string>();

        private static List<LocalizationAsset> InputFiles = new List<LocalizationAsset>();

        static LocalizationImporter()
        {
            Initialize();
        }

        private static void Initialize()
        {
            var settings = Localization.Instance;
            if (settings == null)
            {
                Debug.LogError("Could not find a Localization Settings file in Resources.");
                return;
            }

            languageStrings.Clear();
            ImportFromFiles(settings);
            ImportFromGoogle(settings);
        }

        private static void ImportFromGoogle(Localization settings)
        {
            if (settings.CustomDocument.DownloadOnStart)
            {
                Download(settings.PolyglotDocument, s => Import(s, settings.PolyglotDocument.Format));
            }

            if (settings.CustomDocument.DownloadOnStart)
            {
                Download(settings.CustomDocument, s => Import(s, settings.CustomDocument.Format));
            }
        }

        private static void Import(string text, GoogleDriveDownloadFormat format)
        {
            ImportTextFile(text, format);
        }

        private static IEnumerator Download(LocalizationDocument document, Action<string> done, Func<float, bool> progressbar = null)
        {
            return GoogleDownload.DownloadSheet(document.DocsId, document.SheetId, done, document.Format, progressbar);
        }
        
        public static IEnumerator DownloadPolyglotSheet( Func<float, bool> progressbar = null)
        {
            var settings = Localization.Instance;
            if (settings == null)
            {
                Debug.LogError("Could not find a Localization Settings file in Resources.");
                return null;
            } 
            return Download(settings.PolyglotDocument, s => Import(s, settings.PolyglotDocument.Format), progressbar);
        }

        public static IEnumerator DownloadCustomSheet(Func<float, bool> progressbar = null)
        {
            var settings = Localization.Instance;
            if (settings == null)
            {
                Debug.LogError("Could not find a Localization Settings file in Resources.");
                return null;
            } 
            return Download(settings.CustomDocument, s => Import(s, settings.CustomDocument.Format), progressbar);
        }

        private static void ImportFromFiles(Localization settings)
        {
            InputFiles.Clear();
            InputFiles.AddRange(settings.InputFiles);
            ImportInputFiles();
        }

        private static void ImportInputFiles()
        {
            for (var index = 0; index < InputFiles.Count; index++)
            {
                var inputAsset = InputFiles[index];

                if (inputAsset == null)
                {
                    Debug.LogError("Input File at index [" + index + "] is null");
                    continue;
                }
                
                if (inputAsset.TextAsset == null)
                {
                    Debug.LogError("Input File Text Asset at index [" + index + "] is null");
                    continue;
                }

                ImportTextFile(inputAsset.TextAsset.text, inputAsset.Format);
            }
        }

        private static void ImportTextFile(string text, GoogleDriveDownloadFormat format)
        {
            List<List<string>> rows;
            text = text.Replace("\r\n", "\n");
            if (format == GoogleDriveDownloadFormat.CSV)
            {
                rows = CsvReader.Parse(text);
            }
            else
            {
                rows = TsvReader.Parse(text);
            }
            var canBegin = false;
            
            for (int rowIndex = 0; rowIndex < rows.Count; rowIndex++)
            {
                var row = rows[rowIndex];
                var key = row[0];
                
                if (string.IsNullOrEmpty(key) || IsLineBreak(key) || row.Count <= 1)
                {
                    //Ignore empty lines in the sheet
                    continue;
                }
                
                if (!canBegin)
                {
                    if (key == "Polyglot" || key == "PolyMaster" || key == "BEGIN")
                    {
                        canBegin = true;
                        continue;
                    }
                }

                if (!canBegin)
                {
                    continue;
                }

                if (key == "END")
                {
                    break;
                }

                //Remove key
                row.RemoveAt(0);
                //Remove description
                row.RemoveAt(0);

                if (languageStrings.ContainsKey(key))
                {
                    Debug.Log("The key '" + key + "' already exist, but is now overwritten");
                    languageStrings[key] = row;
                    continue;
                }
                languageStrings.Add(key, row);
            }
        }

        /// <summary><commit_msg>Cleanup union double if check to simpler version<commit_after>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;

namespace Polyglot
{
    public static class LocalizationImporter
    {
        /// <summary>
        /// A dictionary with the key of the text item you want and a list of all the languages.
        /// </summary>
        private static Dictionary<string, List<string>> languageStrings = new Dictionary<string, List<string>>();

        private static List<string> EmptyList = new List<string>();

        private static List<LocalizationAsset> InputFiles = new List<LocalizationAsset>();

        static LocalizationImporter()
        {
            Initialize();
        }

        private static void Initialize()
        {
            var settings = Localization.Instance;
            if (settings == null)
            {
                Debug.LogError("Could not find a Localization Settings file in Resources.");
                return;
            }

            languageStrings.Clear();
            ImportFromFiles(settings);
            ImportFromGoogle(settings);
        }

        private static void ImportFromGoogle(Localization settings)
        {
            if (settings.CustomDocument.DownloadOnStart)
            {
                Download(settings.PolyglotDocument, s => Import(s, settings.PolyglotDocument.Format));
            }

            if (settings.CustomDocument.DownloadOnStart)
            {
                Download(settings.CustomDocument, s => Import(s, settings.CustomDocument.Format));
            }
        }

        private static void Import(string text, GoogleDriveDownloadFormat format)
        {
            ImportTextFile(text, format);
        }

        private static IEnumerator Download(LocalizationDocument document, Action<string> done, Func<float, bool> progressbar = null)
        {
            return GoogleDownload.DownloadSheet(document.DocsId, document.SheetId, done, document.Format, progressbar);
        }
        
        public static IEnumerator DownloadPolyglotSheet( Func<float, bool> progressbar = null)
        {
            var settings = Localization.Instance;
            if (settings == null)
            {
                Debug.LogError("Could not find a Localization Settings file in Resources.");
                return null;
            } 
            return Download(settings.PolyglotDocument, s => Import(s, settings.PolyglotDocument.Format), progressbar);
        }

        public static IEnumerator DownloadCustomSheet(Func<float, bool> progressbar = null)
        {
            var settings = Localization.Instance;
            if (settings == null)
            {
                Debug.LogError("Could not find a Localization Settings file in Resources.");
                return null;
            } 
            return Download(settings.CustomDocument, s => Import(s, settings.CustomDocument.Format), progressbar);
        }

        private static void ImportFromFiles(Localization settings)
        {
            InputFiles.Clear();
            InputFiles.AddRange(settings.InputFiles);
            ImportInputFiles();
        }

        private static void ImportInputFiles()
        {
            for (var index = 0; index < InputFiles.Count; index++)
            {
                var inputAsset = InputFiles[index];

                if (inputAsset == null)
                {
                    Debug.LogError("Input File at index [" + index + "] is null");
                    continue;
                }
                
                if (inputAsset.TextAsset == null)
                {
                    Debug.LogError("Input File Text Asset at index [" + index + "] is null");
                    continue;
                }

                ImportTextFile(inputAsset.TextAsset.text, inputAsset.Format);
            }
        }

        private static void ImportTextFile(string text, GoogleDriveDownloadFormat format)
        {
            List<List<string>> rows;
            text = text.Replace("\r\n", "\n");
            if (format == GoogleDriveDownloadFormat.CSV)
            {
                rows = CsvReader.Parse(text);
            }
            else
            {
                rows = TsvReader.Parse(text);
            }
            var canBegin = false;
            
            for (int rowIndex = 0; rowIndex < rows.Count; rowIndex++)
            {
                var row = rows[rowIndex];
                var key = row[0];
                
                if (string.IsNullOrEmpty(key) || IsLineBreak(key) || row.Count <= 1)
                {
                    //Ignore empty lines in the sheet
                    continue;
                }
                
                if (!canBegin)
                {
                    if (key == "Polyglot" || key == "PolyMaster" || key == "BEGIN")
                    {
                        canBegin = true;
                    }
                    continue;
                }

                if (key == "END")
                {
                    break;
                }

                //Remove key
                row.RemoveAt(0);
                //Remove description
                row.RemoveAt(0);

                if (languageStrings.ContainsKey(key))
                {
                    Debug.Log("The key '" + key + "' already exist, but is now overwritten");
                    languageStrings[key] = row;
                    continue;
                }
                languageStrings.Add(key, row);
            }
        }

        /// <summary>
{'fraction_non_alphanumeric': 0.08798226164079823, 'fraction_numerical': 0.0010643015521064302, 'mean_word_length': 1.3728956228956228, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2550911', 'n_tokens_mistral': 2680, 'n_tokens_neox': 2524, 'n_words': 651}