<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using EDDiscovery2.DB;

namespace EDDiscovery
{
    public class LogReaderBase
    {
        // File buffer
        protected byte[] buffer;
        protected long fileptr;
        protected int bufferpos;
        protected int bufferlen;

        protected Stream stream;

        // File Information
        public string FileName { get { return Path.Combine(TravelLogUnit.Path, TravelLogUnit.Name); } }
        public long filePos { get { return TravelLogUnit.Size; } }
        public TravelLogUnit TravelLogUnit { get; protected set; }

        public LogReaderBase(string filename)
        {
            FileInfo fi = new FileInfo(filename);

            this.TravelLogUnit = new TravelLogUnit
            {
                Name = fi.Name,
                Path = fi.DirectoryName,
                Size = 0
            };

            this.stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
        }

        public LogReaderBase(TravelLogUnit tlu)
        {
            this.TravelLogUnit = tlu;
            this.stream = File.Open(Path.Combine(tlu.Path, tlu.Name), FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
        }

        public bool ReadLine(out string line)
        {
            // Initialize buffer if not yet allocated
            if (buffer == null)
            {
                buffer = new byte[16384];
                bufferpos = 0;
                bufferlen = 0;
                fileptr = TravelLogUnit.Size;
            }

            // Loop while no end-of-line is found
            while (true)
            {
                int endlinepos = -1;

                // Only look for an end-of-line if not already at end of buffer
                if (bufferpos < bufferlen)
                {
                    // Find the next end-of-line
                    endlinepos = Array.IndexOf(buffer, (byte)'\n', bufferpos, bufferlen - bufferpos) - bufferpos;

                    // End-of-line found
                    if (endlinepos >= 0)
                    {
                        // Include the end-of-line in the line length
                        int linelen = endlinepos + 1;

                        // Trim any trailing carriage-return
                        if (endlinepos > 0 && buffer[bufferpos + endlinepos - 1] == '\r')
                        {
                            endlinepos--;
                        }

                        // Return the trimmed string
                        byte[] buf = new byte[endlinepos];
                        Buffer.BlockCopy(buffer, bufferpos, buf, 0, endlinepos);
                        bufferpos += linelen;
                        TravelLogUnit.Size += linelen;
                        line = new String(buf.Select(c => (char)c).ToArray());

                        return true;
                    }
                }

                // No end-of-line found
                // Move remaining data to start of buffer
                if (bufferpos != 0)<commit_msg>Use Encoding.UTF8.GetString()<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using EDDiscovery2.DB;

namespace EDDiscovery
{
    public class LogReaderBase
    {
        // File buffer
        protected byte[] buffer;
        protected long fileptr;
        protected int bufferpos;
        protected int bufferlen;

        protected Stream stream;

        // File Information
        public string FileName { get { return Path.Combine(TravelLogUnit.Path, TravelLogUnit.Name); } }
        public long filePos { get { return TravelLogUnit.Size; } }
        public TravelLogUnit TravelLogUnit { get; protected set; }

        public LogReaderBase(string filename)
        {
            FileInfo fi = new FileInfo(filename);

            this.TravelLogUnit = new TravelLogUnit
            {
                Name = fi.Name,
                Path = fi.DirectoryName,
                Size = 0
            };

            this.stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
        }

        public LogReaderBase(TravelLogUnit tlu)
        {
            this.TravelLogUnit = tlu;
            this.stream = File.Open(Path.Combine(tlu.Path, tlu.Name), FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
        }

        public bool ReadLine(out string line)
        {
            // Initialize buffer if not yet allocated
            if (buffer == null)
            {
                buffer = new byte[16384];
                bufferpos = 0;
                bufferlen = 0;
                fileptr = TravelLogUnit.Size;
            }

            // Loop while no end-of-line is found
            while (true)
            {
                int endlinepos = -1;

                // Only look for an end-of-line if not already at end of buffer
                if (bufferpos < bufferlen)
                {
                    // Find the next end-of-line
                    endlinepos = Array.IndexOf(buffer, (byte)'\n', bufferpos, bufferlen - bufferpos) - bufferpos;

                    // End-of-line found
                    if (endlinepos >= 0)
                    {
                        // Include the end-of-line in the line length
                        int linelen = endlinepos + 1;

                        // Trim any trailing carriage-return
                        if (endlinepos > 0 && buffer[bufferpos + endlinepos - 1] == '\r')
                        {
                            endlinepos--;
                        }

                        // Return the trimmed string
                        byte[] buf = new byte[endlinepos];
                        Buffer.BlockCopy(buffer, bufferpos, buf, 0, endlinepos);
                        bufferpos += linelen;
                        TravelLogUnit.Size += linelen;
                        line = System.Text.Encoding.UTF8.GetString(buf);

                        return true;
                    }
                }

                // No end-of-line found
                // Move remaining data to start of buffer
                if (bufferpos != 0)
{'fraction_non_alphanumeric': 0.08114175753688262, 'fraction_numerical': 0.005452212957023733, 'mean_word_length': 1.2330827067669172, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2223854', 'n_tokens_mistral': 1607, 'n_tokens_neox': 1475, 'n_words': 420}