<commit_before>﻿using FluentScheduler;
using System;
using System.Text;
using System.Text.RegularExpressions;
using Unamit.Enums;

namespace Unamit.Services
{
  public class Babybytes : Registry
  {
    public Babybytes()
    {
      Schedule(Process).ToRunEvery(1).Days().At(3, 0);
    }

    public void Process()
    {
      const string start = "<ul class='names_list'>";
      const string end = "</ul>";

      var regex = new Regex(@"<span .*?>(.*?)<\/span><\/a> (.*?) (jongensnaam|meisjesnaam|gemengdenaam).", RegexOptions.Compiled);
      var sb = new StringBuilder();

      try
      {
        sb.AppendLine("DECLARE @Table TABLE ([Name] nvarchar(150), [Group] nvarchar(150), [Gender] int)");

        for (var i = 1; i < 2500; i++)
        {
          var found = false;
          var s = Utility.Get($"http://www.babybytes.nl/namen/?page={i}");

          s = s.Substring(s.IndexOf(start) + start.Length);
          s = s.Substring(0, s.IndexOf(end));

          foreach (Match m in regex.Matches(s))
          {
            var name = m.Groups[1].Value.Replace("'", "''");
            var group = m.Groups[2].Value.Replace("'", "''");
            var gender = m.Groups[3].Value.ToLower() == "meisjesnaam" ? Gender.Female : m.Groups[3].Value.ToLower() == "jongensnaam" ? Gender.Male : Gender.Unisex;

            if (group.ToLower() == "nieuwe" || group.ToLower() == "lezerbijdrage") group = null;

            found = true;
            sb.AppendLine($"INSERT INTO @Table ([Name], [Group], [Gender]) VALUES ('{name}', '{group}', {(int)gender})");
          }

          if (i % 10 == 0) Finish(sb); // debug
          if (!found) break;
        }

        Finish(sb);
      }
      catch (Exception ex)<commit_msg>sanitize babybytes data<commit_after>﻿using FluentScheduler;
using System;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Unamit.Enums;

namespace Unamit.Services
{
  public class Babybytes : Registry
  {
    public Babybytes()
    {
      Schedule(Process).ToRunEvery(1).Days().At(3, 0);
    }

    public void Process()
    {
      const string start = "<ul class='names_list'>";
      const string end = "</ul>";

      var bogus = new[] { "nieuwe", "lezerbijdrage", "lezerbijdage", "onbekend", "zelf bedacht", "the bold and the beautyfull", "suzanne", "indianennaam" };

      var regex = new Regex(@"<span .*?>(.*?)<\/span><\/a> (.*?) (jongensnaam|meisjesnaam|gemengdenaam).", RegexOptions.Compiled);
      var sb = new StringBuilder();

      try
      {
        sb.AppendLine("DECLARE @Table TABLE ([Name] nvarchar(150), [Group] nvarchar(150), [Gender] int)");

        for (var i = 1; i < 2500; i++)
        {
          var found = false;
          var s = Utility.Get($"http://www.babybytes.nl/namen/?page={i}");

          s = s.Substring(s.IndexOf(start) + start.Length);
          s = s.Substring(0, s.IndexOf(end));

          foreach (Match m in regex.Matches(s))
          {
            var name = m.Groups[1].Value.Replace("'", "''");
            var group = m.Groups[2].Value.Replace("'", "''");
            var gender = m.Groups[3].Value.ToLower() == "meisjesnaam" ? Gender.Female : m.Groups[3].Value.ToLower() == "jongensnaam" ? Gender.Male : Gender.Unisex;

            if (bogus.Contains(group.ToLower())) group = null;
            if (group.ToLower() == "nederlands") group = "Nederlandse";
            if (group.ToLower() == "zweden") group = "Zweedse";
            if (group.ToLower() == "spaans") group = "Spaanse";
            if (group.ToLower() == "scandinavisch ") group = "Scandinavische";
            if (group.ToLower() == "keltisch") group = "Keltische";
            if (group.ToLower() == "japans") group = "Japanse";
            if (group.ToLower() == "itali") group = "Italiaanse";
            if (group.ToLower() == "italiaans") group = "Italiaanse";
            if (group.ToLower() == "iran") group = "Iraanse";
            if (group.ToLower() == "hebreeuws") group = "Hebreeuwse";
            if (group.ToLower() == "georgië") group = "Georgische";
            if (group.ToLower() == "grieks/frans") group = "Griekse";
            if (group.ToLower() == "fries") group = "Friese";
            if (group.ToLower() == "frans ") group = "Franse";
            if (group.ToLower() == "engelse-arabische") group = "Arabische";
            if (group.ToLower() == "arabisch,fries") group = "Arabische";
            if (group.ToLower() == "arabisch") group = "Arabische";
            if (group.ToLower() == "engels") group = "Engelse";
            if (group.ToLower() == "aramese") group = "Armeense";

            found = true;
            sb.AppendLine($"INSERT INTO @Table ([Name], [Group], [Gender]) VALUES ('{name}', '{group}', {(int)gender})");
          }

          if (i % 10 == 0) Finish(sb); // debug
          if (!found) break;
        }

        Finish(sb);
      }
      catch (Exception ex)
{'fraction_non_alphanumeric': 0.18354430379746836, 'fraction_numerical': 0.00898325847284606, 'mean_word_length': 2.430672268907563, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6617648', 'n_tokens_mistral': 1693, 'n_tokens_neox': 1609, 'n_words': 359}