<commit_before>using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Reflection;
using System.Xml;
using System.Xml.Linq;


namespace Behemoth.Alg
{
  /// <summary>
  /// Utils related to the innards of the runtime.
  /// </summary>
  public static class MemUtil
  {
    /// <summary>
    /// Look for a static method from a class and then from its parents. Call
    /// the method if found and return the result.
    /// </summary>
    /// <params name="type">
    /// The type of the class where the static method lookup starts.
    /// </params>
    /// <type name="methodName">
    /// The name of the method to look for.
    /// </type>
    /// <type name="args">
    /// Arguments passed to the static method call.
    /// </type>
    /// <returns>
    /// The result of the static method call, if the method was found. Null
    /// otherwise.
    /// </returns>
    public static Object CallInheritedStaticMethod(
      Type type, string methodName, Object[] args)
    {
      try
      {
        return type.InvokeMember(
          methodName, BindingFlags.Static | BindingFlags.InvokeMethod,
          null, null, args);
      }
      catch (MissingMethodException)
      {
        if (type.BaseType != null)
        {
          return CallInheritedStaticMethod(type.BaseType, methodName, args);
        }
        else
        {
          return null;
        }
      }
    }


    /// <summary>
    /// Read an XDocument object from a byte array representing an XML file.
    /// </summary>
    public static XDocument ReadXml(byte[] data)
    {
      var stream = new MemoryStream(data);

      // Make settings that don't try to read DTD.
      var settings = new XmlReaderSettings();
      settings.ProhibitDtd = false;
      var reader = XmlReader.Create(stream, settings);

      return XDocument.Load(reader);
    }


    public static byte[] ReadBytes(Stream stream)
    {
      var result = new MemoryStream();
      var chunkSize = 8192;
      byte[] buffer = new byte[chunkSize];

      while (true)
      {
        var bytesRead = stream.Read(buffer, 0, chunkSize);

        if (bytesRead == 0)
        {
          // EOF.
          break;
        }
        result.Write(buffer, 0, bytesRead);
      }

      return result.ToArray();
    }
<commit_msg>DTD resolver bugfix in ReadXml.<commit_after>using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Reflection;
using System.Xml;
using System.Xml.Linq;


namespace Behemoth.Alg
{
  /// <summary>
  /// Utils related to the innards of the runtime.
  /// </summary>
  public static class MemUtil
  {
    /// <summary>
    /// Look for a static method from a class and then from its parents. Call
    /// the method if found and return the result.
    /// </summary>
    /// <params name="type">
    /// The type of the class where the static method lookup starts.
    /// </params>
    /// <type name="methodName">
    /// The name of the method to look for.
    /// </type>
    /// <type name="args">
    /// Arguments passed to the static method call.
    /// </type>
    /// <returns>
    /// The result of the static method call, if the method was found. Null
    /// otherwise.
    /// </returns>
    public static Object CallInheritedStaticMethod(
      Type type, string methodName, Object[] args)
    {
      try
      {
        return type.InvokeMember(
          methodName, BindingFlags.Static | BindingFlags.InvokeMethod,
          null, null, args);
      }
      catch (MissingMethodException)
      {
        if (type.BaseType != null)
        {
          return CallInheritedStaticMethod(type.BaseType, methodName, args);
        }
        else
        {
          return null;
        }
      }
    }


    /// <summary>
    /// Read an XDocument object from a byte array representing an XML file.
    /// </summary>
    public static XDocument ReadXml(byte[] data)
    {
      var stream = new MemoryStream(data);

      // Make settings that don't try to read the DTD...
      var settings = new XmlReaderSettings();
      settings.XmlResolver = null;
      // ...nor complain about the presence of the DTD.
      settings.ProhibitDtd = false;

      var reader = XmlReader.Create(stream, settings);

      return XDocument.Load(reader);
    }


    public static byte[] ReadBytes(Stream stream)
    {
      var result = new MemoryStream();
      var chunkSize = 8192;
      byte[] buffer = new byte[chunkSize];

      while (true)
      {
        var bytesRead = stream.Read(buffer, 0, chunkSize);

        if (bytesRead == 0)
        {
          // EOF.
          break;
        }
        result.Write(buffer, 0, bytesRead);
      }

      return result.ToArray();
    }

{'fraction_non_alphanumeric': 0.11409395973154363, 'fraction_numerical': 0.002936241610738255, 'mean_word_length': 1.9311616472034419, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3674345', 'n_tokens_mistral': 1362, 'n_tokens_neox': 1317, 'n_words': 439}