<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml;

namespace Corporal
{
    public class SpeechAct
    {
        private string act;
        private string tax;
        private List<string> sentences=new List<string>();

        public string Act { get { return this.act; } set { this.act = value; } }
        public string Tax{ get { return this.tax; } set { this.tax= value; } }
        public List<string> Sentences{ get { return this.sentences; } set { this.sentences = value; } }

        public SpeechAct(string Act)
        {
            this.act = Act;
        }
        public static StringBuilder Regexify(string Directory)
        {
            StringBuilder sb = new StringBuilder();
            List<SpeechAct> acts = new List<SpeechAct>();
            foreach (FileInfo file in new DirectoryInfo(Directory).GetFiles("*.dat"))
            {
                XmlDocument doc = new XmlDocument();
                doc.Load(file.FullName);
                XmlNode node = doc.DocumentElement.SelectSingleNode("/list/header/speechact_type");
                SpeechAct act = new SpeechAct(node.InnerText);
                act.Tax = doc.DocumentElement.SelectSingleNode("/list/header/taxonomie").InnerText;
                foreach (string s in doc.DocumentElement.SelectSingleNode("/list/body").InnerText.Split(Environment.NewLine.ToCharArray()))
                {
                    act.Sentences.Add(s);
                }
                acts.Add(act);
            }

            return sb;
        }
    }
}<commit_msg>read speechacts and convert them to regular expressions<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml;

namespace Corporal
{
    public class SpeechAct
    {
        private string act;
        private string tax;
        private List<string> sentences=new List<string>();

        public string Act { get { return this.act; } set { this.act = value; } }
        public string Tax{ get { return this.tax; } set { this.tax= value; } }
        public List<string> Sentences{ get { return this.sentences; } set { this.sentences = value; } }

        public SpeechAct(string Act)
        {
            this.act = Act;
        }
        public static List<SpeechAct> Regexify(string Directory)
        {
            List<SpeechAct> acts = new List<SpeechAct>();
            foreach (FileInfo file in new DirectoryInfo(Directory).GetFiles("*.dat"))
            {
                XmlDocument doc = new XmlDocument();
                doc.Load(file.FullName);
                XmlNode node = doc.DocumentElement.SelectSingleNode("/list/header/speechact_type");
                SpeechAct act = new SpeechAct(node.InnerText);
                act.Tax = doc.DocumentElement.SelectSingleNode("/list/header/taxonomie").InnerText;
                foreach (string s in doc.DocumentElement.SelectSingleNode("/list/body").InnerText.Split(Environment.NewLine.ToCharArray()))
                {
                    if (!string.IsNullOrEmpty(s))
                        act.Sentences.Add(Prepare(s));
                }
                acts.Add(act);
            }
            return acts;
        }
        private static string Prepare(string sentence)
        {
            // convert matching syntax to regex
            sentence = sentence.Replace(" . ", @" .* ");          // Find commas with or without spaces
            sentence = sentence.Replace("...", @" .* ");          // Match everything here
            sentence = sentence.Replace(" ? ", @"[ ]?\?[ ]?");    // Mask Questions
            sentence = sentence.Replace(" ! ", @"[ ]?\![ ]?");    // Mask !

            // make common mistakes more fuzzy
            sentence = Regex.Replace(sentence, @"\,[ ]?dass[ ]?", @"\,[ ]?da[s]?[ß]?[ss]?[ ]?");
            //sentence = sentence.Replace(",dass ", );    // match dass, das and daß when comma before
            //sentence = sentence.Replace(", dass ", @"\,\w?da[s]?[ß]?[ss]?\w?");    // match dass, das and daß when comma before
            sentence = sentence.Replace(" , ", @"[ ]?\,[ ]?");    // Find commas with or without spaces

            return sentence;
        }
    }
}
{'fraction_non_alphanumeric': 0.12511436413540714, 'fraction_numerical': 0.0, 'mean_word_length': 1.9931553730321698, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1509406', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1142, 'n_words': 284}