<commit_before>using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Linq;
using System.Xml.Linq;
using MARC4J.Net.Properties;
using System.Net;

namespace MARC4J.Net.Converter
{
    /// <summary>
    ///  <c>ReverseCodeTableHash</c> defines a data structure to facilitate
    ///  UnicodeToAnsel character conversion.
    /// </summary>
    public class ReverseCodeTableHash : ReverseCodeTable
    {
        #region Default Instance

        private static ReverseCodeTableHash _defaultInstance;
        public static ReverseCodeTableHash DefaultInstance
        {
            get 
            {
                if (_defaultInstance == null)
                {
                    using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(Resources.codetables)))
                    {
                        _defaultInstance = new ReverseCodeTableHash(stream);
                    }
                }
                return _defaultInstance; 
            }
        }

        #endregion

        #region Fields
        
        protected IDictionary<char, IDictionary<int, char[]>> charsets = null;

        protected HashSet<char> combining = null; 

        #endregion

        public override bool IsCombining(char c)
        {
            return combining.Contains(c);
        }

        public override IDictionary<int, char[]> GetCharTable(char c)
        {
            return charsets[c];
        }

        public ReverseCodeTableHash(Stream byteStream)
        {
            try
            {
                var reader = new ReverseCodeTableXmlReader();
                reader.Read(byteStream);
                charsets = reader.CharSets;
                combining = reader.CombiningChars;
            }
            catch (Exception e)
            {
                throw new MarcException(e.Message, e);
            }

        }

        public ReverseCodeTableHash(String filename)
        {
            try
            {
                using (var fs = new FileStream(filename, FileMode.Open))
                {
                    var reader = new ReverseCodeTableXmlReader();
                    reader.Read(fs);
                    charsets = reader.CharSets;
                    combining = reader.CombiningChars;
                }
            }
            catch (Exception e)
            {
                throw new MarcException(e.Message, e);
            }
        }

        public ReverseCodeTableHash(Uri uri)
        {
            try
            {
                using (var stream = WebRequest.Create(uri).GetResponse().GetResponseStream())
                {
                    var reader = new ReverseCodeTableXmlReader();
                    reader.Read(stream);
                    charsets = reader.CharSets;
                    combining = reader.CombiningChars;
                }
            }
            catch (Exception e)
            {
                throw new MarcException(e.Message, e);
            }
        }
    }
}<commit_msg>return null when UnicodeToAnsel conversion does not find unicode character in code table<commit_after>using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Linq;
using System.Xml.Linq;
using MARC4J.Net.Properties;
using System.Net;

namespace MARC4J.Net.Converter
{
    /// <summary>
    ///  <c>ReverseCodeTableHash</c> defines a data structure to facilitate
    ///  UnicodeToAnsel character conversion.
    /// </summary>
    public class ReverseCodeTableHash : ReverseCodeTable
    {
        #region Default Instance

        private static ReverseCodeTableHash _defaultInstance;
        public static ReverseCodeTableHash DefaultInstance
        {
            get 
            {
                if (_defaultInstance == null)
                {
                    using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(Resources.codetables)))
                    {
                        _defaultInstance = new ReverseCodeTableHash(stream);
                    }
                }
                return _defaultInstance; 
            }
        }

        #endregion

        #region Fields
        
        protected IDictionary<char, IDictionary<int, char[]>> charsets = null;

        protected HashSet<char> combining = null; 

        #endregion

        public override bool IsCombining(char c)
        {
            return combining.Contains(c);
        }

        public override IDictionary<int, char[]> GetCharTable(char c)
        {
            IDictionary<int, char[]> value = null;
            if (charsets.TryGetValue(c, out value))
            {
                return value;
            }

            return null;
        }

        public ReverseCodeTableHash(Stream byteStream)
        {
            try
            {
                var reader = new ReverseCodeTableXmlReader();
                reader.Read(byteStream);
                charsets = reader.CharSets;
                combining = reader.CombiningChars;
            }
            catch (Exception e)
            {
                throw new MarcException(e.Message, e);
            }

        }

        public ReverseCodeTableHash(String filename)
        {
            try
            {
                using (var fs = new FileStream(filename, FileMode.Open))
                {
                    var reader = new ReverseCodeTableXmlReader();
                    reader.Read(fs);
                    charsets = reader.CharSets;
                    combining = reader.CombiningChars;
                }
            }
            catch (Exception e)
            {
                throw new MarcException(e.Message, e);
            }
        }

        public ReverseCodeTableHash(Uri uri)
        {
            try
            {
                using (var stream = WebRequest.Create(uri).GetResponse().GetResponseStream())
                {
                    var reader = new ReverseCodeTableXmlReader();
                    reader.Read(stream);
                    charsets = reader.CharSets;
                    combining = reader.CombiningChars;
                }
            }
            catch (Exception e)
            {
                throw new MarcException(e.Message, e);
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.07470718581829693, 'fraction_numerical': 0.000949667616334283, 'mean_word_length': 1.326583210603829, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2640352', 'n_tokens_mistral': 1566, 'n_tokens_neox': 1454, 'n_words': 342}