<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Speech.Recognition;

namespace lex4all
{
    public static class Evaluation
    {

        /// <summary>
        /// Performs recognition and computes accuracy on the given testing files.
        /// </summary>
        /// <param name="TestDict">Keys are words, values are lists of audio file names</param>
        public static void Test(Dictionary<String, String[]> testDict, string evalGramFilename)<commit_msg>Parse lexicon file to get list of words<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Linq;
using Microsoft.Speech.Recognition;
using System.Diagnostics;

namespace lex4all
{
    public static class Evaluation
    {
        /// <summary>
        /// Get the set of words (graphemes) in the given lexicon file.
        /// Lexicon file must use the namespace below.
        /// </summary>
        /// <param name="lexFileName">Path to lexicon file</param>
        /// <returns>Array of words</returns>
        public static string[] ReadLexicon(string lexFileName)
        {
       

            Debug.WriteLine("Starting to read lexicon");
            XDocument lexDoc = XDocument.Load(lexFileName);
            Debug.WriteLine("lexdoc created");

            XNamespace ns = "http://www.w3.org/2005/01/pronunciation-lexicon";

            IEnumerable<XElement> childList =
                from el in lexDoc.Root.Elements(ns + "lexeme")
                select el.Element(ns + "grapheme");
            foreach (XElement e in childList)
                Debug.WriteLine(e.Value);
            
            string[] words =
                (from lexeme in lexDoc.Root.Elements(ns+"lexeme")
                select lexeme.Element(ns+"grapheme").Value).ToArray();

            Debug.WriteLine(String.Join(",", words));

            return words;

            
        }

        /// <summary>
        /// Performs recognition and computes accuracy on the given testing files.
        /// </summary>
        /// <param name="TestDict">Keys are words, values are lists of audio file names</param>
        public static void Evaluate(Dictionary<String, String[]> testDict, string evalGramFilename)
{'fraction_non_alphanumeric': 0.10646387832699619, 'fraction_numerical': 0.0038022813688212928, 'mean_word_length': 2.3588652482269503, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5493137', 'n_tokens_mistral': 643, 'n_tokens_neox': 610, 'n_words': 183}