<commit_before>using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.Collections.Generic;
using PokemonGoRaidBot.Tests.MockedObjects;
using PokemonGoRaidBot.Services.Parsing;
using System.IO;
using System;

namespace PokemonGoRaidBot.Tests
{
    [TestClass]
    public class MessageParserTests
    {
        private List<string> GoodStringsEnglish = new List<string>(new string[] 
        {
            "Raikou at 17th/stout, looks like I am only one here",
            "Raikou hatches at Potager (11th/Ogden) in 58 min. Anyone up for 3:30 raid there?",
            "Tyranitar at Broadway and 18th",
            "ttar at 3rd and Broadwy",
            "Tyranitar 5th and Bannock. By Denver Health. 46 min. I know it's super early",
            "Raikou at Dayton and Girard.  Moon Rock.  36 minutes left.  Need one more.",

        });
        private List<string> BadStringsEnglish = new List<string>(new string[]
        {
            "Raikou is cool, google.com/maps",
            "BLAHBLAHBLAH",
        });


        private List<string> GoodStringsDutch = new List<string>(new string[]
        {
            "Tyranitar in Julianapark 1 uur en 20 minuten resterend",
            "Tyranitar at Centraal Station nog 30 minuten",
        });

        private List<string> BadStringsDutch = new List<string>(new string[]
        {
            "Raikou is cool, google.com/maps",
            "BLAHBLAHBLAH",
        });

        #region en-US
        [TestMethod]
        public void TestGoodStringsEnglish()
        {
            var parser = GetParser();
            foreach (var str in GoodStringsEnglish)
            {
                var post = parser.ParsePost(new MockedChatMessage(str, Objects.ChatTypes.Discord));

                if(!post.IsValid)
                    Assert.IsTrue(post.IsValid);
            }
        }

        [TestMethod]
        public void TestBadStringsEnglish()
        {
            var parser = GetParser();
            foreach (var str in BadStringsEnglish)
            {
                var post = parser.ParsePost(new MockedChatMessage(str, Objects.ChatTypes.Discord));

                if (post.IsValid)
                    Assert.IsFalse(post.IsValid);
            }
        }
        #endregion

        #region nl-NL
        [TestMethod]
        public void TestGoodStringsDutch()
        {
            var parser = GetParser("nl-NL");
            foreach (var str in GoodStringsDutch)
            {
                var post = parser.ParsePost(new MockedChatMessage(str, Objects.ChatTypes.Discord));

                if (!post.IsValid)
                    Assert.IsTrue(post.IsValid);
            }
        }

        [TestMethod]
        public void TestBadStringsDutch()
        {
            var parser = GetParser("nl-NL");
            foreach (var str in BadStringsDutch)
            {
                var post = parser.ParsePost(new MockedChatMessage(str, Objects.ChatTypes.Discord));

                if (post.IsValid)
                    Assert.IsFalse(post.IsValid);
            }
        }
        #endregion

        #region util
        private MessageParser GetParser(string language)
        {
            return GetParser(new MockedBotServerConfiguration()
            {
                Language = language
            });
        }

        private MessageParser GetParser(MockedBotServerConfiguration serverConfig = null)
        {
            var parser = new MessageParser(serverConfig ?? new MockedBotServerConfiguration(), serverConfig?.Language ?? "en-us", 0);

            return parser;
        }
        #endregion
    }
}
<commit_msg>Adding tests<commit_after>using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.Collections.Generic;
using PokemonGoRaidBot.Tests.MockedObjects;
using PokemonGoRaidBot.Services.Parsing;
using System.IO;
using System;

namespace PokemonGoRaidBot.Tests
{
    [TestClass]
    public class MessageParserTests
    {
        private List<string> GoodStringsEnglish = new List<string>(new string[] 
        {
            "Raikou at 17th/stout, looks like I am only one here",
            "Raikou hatches at Potager (11th/Ogden) in 58 min. Anyone up for 3:30 raid there?",
            "Tyranitar at Broadway and 18th",
            "ttar at 3rd and Broadwy",
            "Tyranitar 5th and Bannock. By Denver Health. 46 min. I know it's super early",
            "Raikou at Dayton and Girard.  Moon Rock.  36 minutes left.  Need one more.",

        });
        private List<string> BadStringsEnglish = new List<string>(new string[]
        {
            "Raikou is cool, google.com/maps",
            "BLAHBLAHBLAH",
        });


        private List<string> GoodStringsDutch = new List<string>(new string[]
        {
            "Ttar in Jlianapark nog 56 minuten",
            "Snorlax naast Centraal Station 20 min resterend",
            "Lapras bij Longboat nog 76 mins",
            "Magikarp bij de Dom 2 en een half uur resterend",
            "Tyranitar in Julianapark 1 uur en 20 minuten resterend",
            "Tyranitar at Centraal Station nog 30 minuten",
        });

        private List<string> BadStringsDutch = new List<string>(new string[]
        {
            "Raikou is cool, google.com/maps",
            "BLAHBLAHBLAH",
        });

        #region en-US
        [TestMethod]
        public void TestGoodStringsEnglish()
        {
            var parser = GetParser();
            foreach (var str in GoodStringsEnglish)
            {
                var post = parser.ParsePost(new MockedChatMessage(str, Objects.ChatTypes.Discord));

                if(!post.IsValid)
                    Assert.IsTrue(post.IsValid);
            }
        }

        [TestMethod]
        public void TestBadStringsEnglish()
        {
            var parser = GetParser();
            foreach (var str in BadStringsEnglish)
            {
                var post = parser.ParsePost(new MockedChatMessage(str, Objects.ChatTypes.Discord));

                if (post.IsValid)
                    Assert.IsFalse(post.IsValid);
            }
        }
        #endregion

        #region nl-NL
        [TestMethod]
        public void TestGoodStringsDutch()
        {
            var parser = GetParser("nl-NL");
            foreach (var str in GoodStringsDutch)
            {
                var post = parser.ParsePost(new MockedChatMessage(str, Objects.ChatTypes.Discord));

                if (!post.IsValid)
                    Assert.IsTrue(post.IsValid);
            }
        }

        [TestMethod]
        public void TestBadStringsDutch()
        {
            var parser = GetParser("nl-NL");
            foreach (var str in BadStringsDutch)
            {
                var post = parser.ParsePost(new MockedChatMessage(str, Objects.ChatTypes.Discord));

                if (post.IsValid)
                    Assert.IsFalse(post.IsValid);
            }
        }
        #endregion

        #region util
        private MessageParser GetParser(string language)
        {
            return GetParser(new MockedBotServerConfiguration()
            {
                Language = language
            });
        }

        private MessageParser GetParser(MockedBotServerConfiguration serverConfig = null)
        {
            var parser = new MessageParser(serverConfig ?? new MockedBotServerConfiguration(), serverConfig?.Language ?? "en-us", 0);

            return parser;
        }
        #endregion
    }
}

{'fraction_non_alphanumeric': 0.08515782404298186, 'fraction_numerical': 0.007118871725990598, 'mean_word_length': 1.7304730473047305, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6980733', 'n_tokens_mistral': 2084, 'n_tokens_neox': 1938, 'n_words': 514}