<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;

namespace BrightData.Input
{
    internal class CsvParser
    {
        readonly StreamReader _stream;
        readonly char _delimiter;
        readonly char[] _buffer;
        readonly char _quote;
        readonly List<StringBuilder> _columns = new List<StringBuilder>();
        int _currIndex;
        bool _inQuote;

        public CsvParser(StreamReader stream, char delimiter, char quote = '"')
        {
            _stream = stream;
            _delimiter = delimiter;
            _quote = quote;
            _buffer = new char[32768];
        }

        public Action<int>? OnProgress { get; set; }
        public Action? OnComplete { get; set; }

        public IEnumerable<string[]> Parse()
        {
            while(!_stream.EndOfStream) {
                foreach(var line in ReadPage())
                    yield return line;
                OnProgress?.Invoke((int) (_stream.BaseStream.Position * 100 / _stream.BaseStream.Length));
            }
            var lastLine = GetLine();
            if(lastLine.Any(s => s.Length > 0))
                yield return lastLine;
            OnComplete?.Invoke();
        }

        StringBuilder GetBuilder(bool increment)
        {
            if(increment)
                _currIndex++;
            if(_columns.Count == _currIndex)
                _columns.Add(new StringBuilder());
            return _columns[_currIndex];
        }

        IEnumerable<string[]> ReadPage()
        {
            var len = _stream.Read(_buffer);
            var curr = GetBuilder(false);

            for (var i = 0; i < len; i++)
            {
                var ch = _buffer[i];
                if (ch == '\r') {
                    // ReSharper disable once RedundantJumpStatement
                    continue;
                }
                if(ch == '\n' && !_inQuote) {
                    var line = GetLine();
                    if(line.Any(s => s.Length > 0))
                        yield return line;
                    curr = GetBuilder(false);
                } else if(ch == _delimiter && !_inQuote)
                    curr = GetBuilder(true);
                else if(ch == _quote) {
                    // TODO: look for escaped quotes?
                    _inQuote = !_inQuote;
                }
                else {
                    curr.Append(ch);
                }
            }
        }

        string[] GetLine()
        {
            var ret = new string[_columns.Count];
            for (int i = 0, len = _columns.Count; i < len; i++)
            {
                var sb = _columns[i];
                ret[i] = sb.ToString();
                sb.Clear();
            }
            _currIndex = 0;
            return ret;
        }
    }
}<commit_msg>csv parser now reuses arrays<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;

namespace BrightData.Input
{
    internal class CsvParser
    {
        readonly StreamReader _stream;
        readonly char _delimiter;
        readonly char[] _buffer;
        readonly char _quote;
        readonly List<StringBuilder> _columns = new List<StringBuilder>();
        readonly Dictionary<int, string[]> _stringBuffers = new Dictionary<int, string[]>();
        int _currIndex;
        bool _inQuote;

        public CsvParser(StreamReader stream, char delimiter, char quote = '"')
        {
            _stream = stream;
            _delimiter = delimiter;
            _quote = quote;
            _buffer = new char[32768];
        }

        public Action<int>? OnProgress { get; set; }
        public Action? OnComplete { get; set; }

        public IEnumerable<string[]> Parse()
        {
            while(!_stream.EndOfStream) {
                foreach(var line in ReadPage())
                    yield return line;
                OnProgress?.Invoke((int) (_stream.BaseStream.Position * 100 / _stream.BaseStream.Length));
            }
            var lastLine = GetLine();
            if(lastLine.Any(s => s.Length > 0))
                yield return lastLine;
            OnComplete?.Invoke();
        }

        StringBuilder GetBuilder(bool increment)
        {
            if(increment)
                _currIndex++;
            if(_columns.Count == _currIndex)
                _columns.Add(new StringBuilder());
            return _columns[_currIndex];
        }

        IEnumerable<string[]> ReadPage()
        {
            var len = _stream.Read(_buffer);
            var curr = GetBuilder(false);

            for (var i = 0; i < len; i++)
            {
                var ch = _buffer[i];
                if (ch == '\r') {
                    // ReSharper disable once RedundantJumpStatement
                    continue;
                }
                if(ch == '\n' && !_inQuote) {
                    var line = GetLine();
                    if(line.Any(s => s.Length > 0))
                        yield return line;
                    curr = GetBuilder(false);
                } else if(ch == _delimiter && !_inQuote)
                    curr = GetBuilder(true);
                else if(ch == _quote) {
                    // TODO: look for escaped quotes?
                    _inQuote = !_inQuote;
                }
                else {
                    curr.Append(ch);
                }
            }
        }

        string[] GetLine()
        {
            var ret = GetBuffer(_columns.Count);
            for (int i = 0, len = _columns.Count; i < len; i++) {
                var sb = _columns[i];
                ret[i] = sb.ToString();
                sb.Clear();
            }
            _currIndex = 0;
            return ret;
        }

        string[] GetBuffer(int length)
        {
            if (!_stringBuffers.TryGetValue(length, out var ret))
                _stringBuffers.Add(length, ret = new string[length]);
            return ret;
        }
    }
}
{'fraction_non_alphanumeric': 0.10161049310974597, 'fraction_numerical': 0.004316785654989208, 'mean_word_length': 1.293983244478294, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5422615', 'n_tokens_mistral': 1613, 'n_tokens_neox': 1515, 'n_words': 386}