<commit_before>﻿using System;
using System.Collections.Generic;
using System.Data.OleDb;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ResourcesFirstTranslations.Data;

namespace Importer.Corsavy
{
    class Program
    {
        private const string ConnectionString = 
                        @"Provider=Microsoft.Jet.OLEDB.4.0;" + 
                        @"Data Source=LocalizeDb_DL_Corsavy.mdb;" + 
                        @"User Id=;Password=;";

        // TODO: which languages do we import? (atm > 2k missing are removed)
        private static readonly List<string> IgnoreOnImport = new List<string>()
        {
            "en", "goisern", // en isn't really in the database, goisern was a fake translation
            "urdu", "af", "id", "fa", "th", "vi", "be", "hr", "he", "ar"<commit_msg>Make official that 2k missing translations doesn't get you into the new system<commit_after>﻿using System;
using System.Collections.Generic;
using System.Data.OleDb;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ResourcesFirstTranslations.Data;

namespace Importer.Corsavy
{
    class Program
    {
        private const string ConnectionString = 
                        @"Provider=Microsoft.Jet.OLEDB.4.0;" + 
                        @"Data Source=LocalizeDb_DL_Corsavy.mdb;" + 
                        @"User Id=;Password=;";

        // NOTE: Languages > 2k missing are removed, statistics as of 6/6/2014
        //   Urdu 	urdu 	2538
        //   Afrikaans 	af 	2537
        //   Indonesian 	id 	2534
        //   Persian 	fa 	2492
        //   Thai 	th 	2491
        //   Vietnamese 	vi 	2406
        //   Goiserisch 	goisern 	2393
        //   Belarusian 	be 	2366
        //   Croatian 	hr 	2351
        //   Hebrew 	he 	2343
        //   Arabic 	ar 	2254
        private static readonly List<string> IgnoreOnImport = new List<string>()
        {
            "en", "goisern", // en isn't really in the database, goisern was a fake translation
            "urdu", "af", "id", "fa", "th", "vi", "be", "hr", "he", "ar"
{'fraction_non_alphanumeric': 0.10923581809657759, 'fraction_numerical': 0.026722925457102673, 'mean_word_length': 2.0838150289017343, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7225817', 'n_tokens_mistral': 709, 'n_tokens_neox': 658, 'n_words': 183}