<commit_before>﻿// Copyright (c) IxMilia.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;

namespace IxMilia.Classics.Tasks
{
    public class ProcessFile : Task
    {
        [Required]
        public string File { get; set; }

        [Required]
        public string Dictionary { get; set; }

        [Required]
        public string SuffixFile { get; set; }

        [Required]
        public string[] SuffixWords { get; set; }

        [Required]
        public string Output { get; set; }

        public int MaxLines { get; set; }

        private readonly Dictionary<string, HashSet<string>> _suffixes = new Dictionary<string, HashSet<string>>();
        private readonly Dictionary<string, DictionaryEntry> _entries = new Dictionary<string, DictionaryEntry>();
        private readonly Regex _wordMatcher = new Regex("[a-zA-Z]+", RegexOptions.Compiled);
        private readonly Regex _definitionMatcher = new Regex(@"^#(.*)  ([A-Z].*)\[(.....)\] :: (.*)$", RegexOptions.Compiled);
        //                                                        ^^^^ word
        //                                                              ^^^^^^^^^ part of speech
        //                                                                       ^^^^^^^^^^^ flags
        //                                                                                      ^^^^ definition

        private int _bookNumber;
        private IEnumerable<XElement> _elements;
        private int _currentLine;

        public override bool Execute()
        {
            LoadFile();
            LoadSuffixes();
            LoadDictionary();

            _currentLine = 1;
            foreach (var element in _elements)
            {
                if (MaxLines > 0 && _currentLine > MaxLines)
                {
                    break;
                }

                AssertCurrentLineNumber(element);
                switch (element.Name.LocalName)
                {
                    case "milestone":
                        break;
                    case "l":
                        var words = BreakLineIntoWords(element.Value);
                        foreach (var word in words)
                        {
                            var definitions = GetDefinitions(word);
                            if (definitions.Length == 0)
                            {
                                Log.LogError($"Unable to define {word} on line {_currentLine}.");
                            }
                            else
                            {
                                foreach (var definition in definitions)
                                {
                                    Console.WriteLine($"Found definition on line {_currentLine}: {word} = {definition.Entry} - {definition.Definition}");
                                }
                            }
                        }

                        _currentLine++;
                        break;
                }
            }

            if (MaxLines > 0)
            {
                Log.LogError($"Failing the build because {nameof(MaxLines)} was specified.");
            }

            return true;
        }

        private void LoadFile()
        {
            var file = XDocument.Load(File).Root;
            var content = file.Element("text").Element("body").Element("div1");
            _bookNumber = int.Parse(content.Attribute("n").Value);
            _elements = content.Elements();
        }

        private void LoadSuffixes()
        {
            var suffixLines = System.IO.File.ReadAllLines(SuffixFile)
                .Where(line => !string.IsNullOrEmpty(line) && !line.StartsWith("#"));
            foreach (var line in suffixLines)
            {
                var parts = line.Split("|".ToCharArray(), 2);<commit_msg>add simple defining statistics<commit_after>﻿// Copyright (c) IxMilia.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;

namespace IxMilia.Classics.Tasks
{
    public class ProcessFile : Task
    {
        [Required]
        public string File { get; set; }

        [Required]
        public string Dictionary { get; set; }

        [Required]
        public string SuffixFile { get; set; }

        [Required]
        public string[] SuffixWords { get; set; }

        [Required]
        public string Output { get; set; }

        public int MaxLines { get; set; }

        private readonly Dictionary<string, HashSet<string>> _suffixes = new Dictionary<string, HashSet<string>>();
        private readonly Dictionary<string, DictionaryEntry> _entries = new Dictionary<string, DictionaryEntry>();
        private readonly Regex _wordMatcher = new Regex("[a-zA-Z]+", RegexOptions.Compiled);
        private readonly Regex _definitionMatcher = new Regex(@"^#(.*)  ([A-Z].*)\[(.....)\] :: (.*)$", RegexOptions.Compiled);
        //                                                        ^^^^ word
        //                                                              ^^^^^^^^^ part of speech
        //                                                                       ^^^^^^^^^^^ flags
        //                                                                                      ^^^^ definition

        private int _bookNumber;
        private IEnumerable<XElement> _elements;
        private int _currentLine;
        private int _definedWords;
        private int _undefinedWords;

        public override bool Execute()
        {
            LoadFile();
            LoadSuffixes();
            LoadDictionary();

            _currentLine = 1;
            _definedWords = 0;
            _undefinedWords = 0;
            foreach (var element in _elements)
            {
                if (MaxLines > 0 && _currentLine > MaxLines)
                {
                    break;
                }

                AssertCurrentLineNumber(element);
                switch (element.Name.LocalName)
                {
                    case "milestone":
                        break;
                    case "l":
                        var words = BreakLineIntoWords(element.Value);
                        foreach (var word in words)
                        {
                            var definitions = GetDefinitions(word);
                            if (definitions.Length == 0)
                            {
                                _undefinedWords++;
                                Log.LogError($"Unable to define {word} on line {_currentLine}.");
                            }
                            else
                            {
                                _definedWords += definitions.Length;
                                foreach (var definition in definitions)
                                {
                                    Console.WriteLine($"Found definition on line {_currentLine}: {word} = {definition.Entry} - {definition.Definition}");
                                }
                            }
                        }

                        _currentLine++;
                        break;
                }
            }

            if (MaxLines > 0)
            {
                Log.LogError($"Failing the build because {nameof(MaxLines)} was specified.");
            }

            Log.LogMessage($"Defined {_definedWords} words with {_undefinedWords} undefined.  {_definedWords * 100.0 / (_definedWords + _undefinedWords)}% success rate.");

            return true;
        }

        private void LoadFile()
        {
            var file = XDocument.Load(File).Root;
            var content = file.Element("text").Element("body").Element("div1");
            _bookNumber = int.Parse(content.Attribute("n").Value);
            _elements = content.Elements();
        }

        private void LoadSuffixes()
        {
            var suffixLines = System.IO.File.ReadAllLines(SuffixFile)
                .Where(line => !string.IsNullOrEmpty(line) && !line.StartsWith("#"));
            foreach (var line in suffixLines)
            {
                var parts = line.Split("|".ToCharArray(), 2);
{'fraction_non_alphanumeric': 0.09694938756644327, 'fraction_numerical': 0.002542177027963947, 'mean_word_length': 1.211857909532328, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '920761', 'n_tokens_mistral': 2001, 'n_tokens_neox': 1885, 'n_words': 491}