<commit_before>﻿using System;
using System.IO;
using System.Linq;

namespace Vba.Language.Preprocessor
{
    public class SourceTextReader : ReadLineTextReader
    {
        private TextReader reader;
        private PreprocessorStateManager manager;

        public SourceTextReader(string source)
        {
            manager = new PreprocessorStateManager();
            reader = new StringReader(source);
        }

        public override string ReadLine()
        {
            string line = reader.ReadLine();
            while (line != null) // null indicates we have reached the end of the source.
            {
                if (IsPreprocessorStatement(line))
                {
                    manager.ExecutePreprocessorStatement(line);
                }
                else if (manager.IsActiveRegion)
                {
                    return line;
                }

                line = reader.ReadLine();
            }
        }

        /// <summary>
        /// Checks if the given statement is a preprocessor statement.
        /// </summary>
        /// <param name="line">A string representing a line of source code.</param>
        /// <returns>True if the given statement is a preprocessor statement, false otherwise.</returns>
        /// <exception cref="ArgumentException">Thrown if the input contains line break characters.</exception>
        private bool IsPreprocessorStatement(string line)
        {
            if (line.Contains("\r") || line.Contains("\n"))
            {
                throw new ArgumentException("Parameter 'line' cannot contain line break characters.");
            }

            var firstChar = (from c in line
                             where c != '\t' && c != ' '
                             select c).FirstOrDefault();

            return firstChar == '#';
        }
    }
}<commit_msg>SourceTextReader: Compiler fix. Add dispose override for text reader.<commit_after>﻿using System;
using System.IO;
using System.Linq;

namespace Vba.Language.Preprocessor
{
    public class SourceTextReader : ReadLineTextReader
    {
        private TextReader reader;
        private PreprocessorStateManager manager;

        public SourceTextReader(string source)
        {
            manager = new PreprocessorStateManager();
            reader = new StringReader(source);
        }

        public override string ReadLine()
        {
            string line = reader.ReadLine();
            while (line != null) // null indicates we have reached the end of the source.
            {
                if (IsPreprocessorStatement(line))
                {
                    manager.ExecutePreprocessorStatement(line);
                }
                else if (manager.IsActiveRegion)
                {
                    return line;
                }

                line = reader.ReadLine();
            }
            return null;
        }

        /// <summary>
        /// Checks if the given statement is a preprocessor statement.
        /// </summary>
        /// <param name="line">A string representing a line of source code.</param>
        /// <returns>True if the given statement is a preprocessor statement, false otherwise.</returns>
        /// <exception cref="ArgumentException">Thrown if the input contains line break characters.</exception>
        private bool IsPreprocessorStatement(string line)
        {
            if (line.Contains("\r") || line.Contains("\n"))
            {
                throw new ArgumentException("Parameter 'line' cannot contain line break characters.");
            }

            var firstChar = (from c in line
                             where c != '\t' && c != ' '
                             select c).FirstOrDefault();

            return firstChar == '#';
        }

        protected override void Dispose(bool disposing)
        {
            if (disposing)
            {
                reader.Dispose();
            }
            base.Dispose(disposing);
        }
    }
}
{'fraction_non_alphanumeric': 0.090863704443335, 'fraction_numerical': 0.0, 'mean_word_length': 1.5441269841269842, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6644584', 'n_tokens_mistral': 920, 'n_tokens_neox': 883, 'n_words': 275}