<commit_before>﻿using System.IO;

namespace Vba.Language.Preprocessor
{
    public class SourceTextReader : ReadLineTextReader
    {
        private TextReader reader;
        private PreprocessorStateManager manager;
        private VbaHeader header;

        public SourceTextReader(string source)
        {
            manager = new PreprocessorStateManager();
            reader = new StringReader(source);
            header = new VbaHeader();
        }

        public override string ReadLine()
        {
            // when ReadLine returns null we are at the end of the file.
            var line = reader.ReadLine();

            // first parse the header.
            while (line != null && header.ProcessHeaderStatement(line))
            {
                line = reader.ReadLine();
            }

            // after we exit the top loop, line will be the first non-header line.
            while (line != null) 
            {
                if (PreprocessorStateManager.IsPreprocessorStatement(line))
                {
                    manager.ExecutePreprocessorStatement(line);
                }
                else if (manager.IsActiveRegion)
                {
                    return line;
                }

                line = reader.ReadLine();
            }
            return null;
        }

        protected override void Dispose(bool disposing)
        {
            if (disposing)
            {
                reader.Dispose();
            }
            base.Dispose(disposing);
        }
    }
}<commit_msg>Added additional constructor to SourceTextReader implementation to accept a generic TextReader instance.<commit_after>﻿using System.IO;

namespace Vba.Language.Preprocessor
{
    public class SourceTextReader : ReadLineTextReader
    {
        private TextReader reader;
        private PreprocessorStateManager manager;
        private VbaHeader header;

        public SourceTextReader(string source) : this(new StringReader(source)) { }

        public SourceTextReader(TextReader reader)
        {
            this.reader = reader;
            manager = new PreprocessorStateManager();
            header = new VbaHeader();
        }

        public override string ReadLine()
        {
            // when ReadLine returns null we are at the end of the file.
            var line = reader.ReadLine();

            // first parse the header.
            while (line != null && header.ProcessHeaderStatement(line))
            {
                line = reader.ReadLine();
            }

            // after we exit the top loop, line will be the first non-header line.
            while (line != null) 
            {
                if (PreprocessorStateManager.IsPreprocessorStatement(line))
                {
                    manager.ExecutePreprocessorStatement(line);
                }
                else if (manager.IsActiveRegion)
                {
                    return line;
                }

                line = reader.ReadLine();
            }
            return null;
        }

        protected override void Dispose(bool disposing)
        {
            if (disposing)
            {
                if (reader != null)
                {
                    reader.Dispose();
                    reader = null;
                }
            }
            base.Dispose(disposing);
        }
    }
}
{'fraction_non_alphanumeric': 0.07197867298578199, 'fraction_numerical': 0.0, 'mean_word_length': 1.3435114503816794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5554104', 'n_tokens_mistral': 788, 'n_tokens_neox': 745, 'n_words': 210}