<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Xml;

namespace IpRanges
{
    public static class IPRangesParser
    {
        public static IEnumerable<IPRangesGroup> ParseFromResources(string resourcePrefix = null)
        {
            foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
            {
                if (IsDynamicAssembly(assembly))
                    continue;

                string[] names;
                try { names = assembly.GetManifestResourceNames(); }
                catch { continue; }

                foreach (var resName in names)
                {
                    if (!String.IsNullOrEmpty(resourcePrefix))
                    {
                        if (!resName.StartsWith(resourcePrefix)) continue;
                    }

                    if (!resName.EndsWith(".xml")) continue;

                    IPRangesGroup group;
                    var stream = assembly.GetManifestResourceStream(resName);
                    if (stream == null) continue;

                    using (var reader = new XmlTextReader(stream))
                    {
                        Exception exception;
                        if (!TryParseFromXml(reader, out group, out exception, true))
                        {
                            throw exception;
                        }
                    }

                    if (group.Regions.Any())
                        yield return group;
                }
            }
        }

        private static bool IsDynamicAssembly(Assembly assembly)
        {
            return (assembly.ManifestModule is System.Reflection.Emit.ModuleBuilder);
        }

        public static IPRangesGroup ParseFromXml(string xml)
        {
            if (xml == null) throw new ArgumentNullException("xml");

            using (var mem = new MemoryStream(Encoding.Default.GetBytes(xml)))
            using (var reader = new StreamReader(mem))
                return ParseFromXml(reader.BaseStream);
        }

        public static IPRangesGroup ParseFromXml(Stream stream)
        {
            if (stream == null) throw new ArgumentNullException("stream");

            using (var reader = new XmlTextReader(stream))
                return ParseFromXml(reader);
        }<commit_msg>better detection of dynamic assemblies<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Xml;

namespace IpRanges
{
    public static class IPRangesParser
    {
        public static IEnumerable<IPRangesGroup> ParseFromResources(string resourcePrefix = null)
        {
            foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
            {
                if (IsDynamicAssembly(assembly))
                    continue;

                string[] names;
                try { names = assembly.GetManifestResourceNames(); }
                catch { continue; }

                foreach (var resName in names)
                {
                    if (!String.IsNullOrEmpty(resourcePrefix))
                    {
                        if (!resName.StartsWith(resourcePrefix)) continue;
                    }

                    if (!resName.EndsWith(".xml")) continue;

                    IPRangesGroup group;
                    var stream = assembly.GetManifestResourceStream(resName);
                    if (stream == null) continue;

                    using (var reader = new XmlTextReader(stream))
                    {
                        Exception exception;
                        if (!TryParseFromXml(reader, out group, out exception, true))
                        {
                            throw exception;
                        }
                    }

                    if (group.Regions.Any())
                        yield return group;
                }
            }
        }

        private static bool IsDynamicAssembly(Assembly assembly)
        {
            return (assembly.IsDynamic || assembly.ManifestModule is System.Reflection.Emit.ModuleBuilder);
        }

        public static IPRangesGroup ParseFromXml(string xml)
        {
            if (xml == null) throw new ArgumentNullException("xml");

            using (var mem = new MemoryStream(Encoding.Default.GetBytes(xml)))
            using (var reader = new StreamReader(mem))
                return ParseFromXml(reader.BaseStream);
        }

        public static IPRangesGroup ParseFromXml(Stream stream)
        {
            if (stream == null) throw new ArgumentNullException("stream");

            using (var reader = new XmlTextReader(stream))
                return ParseFromXml(reader);
        }
{'fraction_non_alphanumeric': 0.07635315908623173, 'fraction_numerical': 0.0, 'mean_word_length': 1.4846625766871167, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5771277', 'n_tokens_mistral': 1129, 'n_tokens_neox': 1060, 'n_words': 277}