<commit_before>using System;
using System.IO;

public class MemoryFileStream : Stream {
    public override bool CanRead {
        get { return true; }
    }
    public override bool CanSeek {
        get { return true; }
    }
    public override bool CanTimeout {
        get { return false; }
    }
    public override bool CanWrite {
        get { return true; }
    }
    public override long Length {
        get { return _length; }
    }
    public override long Position {
        get { return _position; }
        set { _position = value; }
    }

    public string Name {
        get { return filename; }
    }

    long _length;
    long _position;
    byte[] data;

    string filename;

    public MemoryFileStream(string filename) {
        this.filename = filename;
        
        FileStream input = new FileStream(filename, FileMode.Open);
        _length = input.Length;

        data = new byte[Length];
        _position = 0;
        input.Read(data, 0, (int)Length);
        input.Close();
    }

    public override void Flush() {
        FileStream output = new FileStream(filename, FileMode.Open);
        output.Write(data, 0, (int)Length);
        output.Close();
    }

    public override void SetLength(long len) {
        byte[] newData = new byte[len];
        Array.Copy(data, newData, Math.Min(len, _length));
        data = newData;

        _length = len;
    }

    public override long Seek(long dest, SeekOrigin origin) {
        switch (origin) {
            case SeekOrigin.End:
                Position = Length - dest;
                break;
            case SeekOrigin.Begin:
                Position = dest;
                break;
            case SeekOrigin.Current:
                Position += dest;
                break;
        }
        return Position;
    }

    public override int Read(byte[] buffer, int offset, int count) {
        int size = count;
        if (Position + count > Length)
            size = (int)(Length-Position);
        Array.Copy(data, Position, buffer, offset, size);
        Position = Position + size;
        return size;
    }
    public override void Write(byte[] buffer, int offset, int count) {
        if (Position + count > Length)
            SetLength(Position + count);
        Array.Copy(buffer, offset, data, Position, count);
        Position = Position + count;
        if (Position > Length)
            Position = Length;
    }

    public override int ReadByte() {
        int ret = data[Position];
        Position++;
        return ret;
    }
    public override void WriteByte(byte value) {
        data[Position] = value;
        Position++;
    }
}<commit_msg>MemoryFileStream keeps track of if it's modified<commit_after>using System;
using System.IO;

public class MemoryFileStream : Stream {
    public override bool CanRead {
        get { return true; }
    }
    public override bool CanSeek {
        get { return true; }
    }
    public override bool CanTimeout {
        get { return false; }
    }
    public override bool CanWrite {
        get { return true; }
    }
    public override long Length {
        get { return _length; }
    }
    public override long Position {
        get { return _position; }
        set { _position = value; }
    }

    public string Name {
        get { return filename; }
    }

    long _length;
    long _position;
    byte[] data;
    bool modified = false;

    string filename;

    public MemoryFileStream(string filename) {
        this.filename = filename;
        
        FileStream input = new FileStream(filename, FileMode.Open);
        _length = input.Length;

        data = new byte[Length];
        _position = 0;
        modified = false;
        input.Read(data, 0, (int)Length);
        input.Close();
    }

    public override void Flush() {
        if (modified) {
            FileStream output = new FileStream(filename, FileMode.Open);
            output.Write(data, 0, (int)Length);
            output.Close();
            modified = false;
        }
    }

    public override void SetLength(long len) {
        if (Length != len) {
            modified = true;
            byte[] newData = new byte[len];
            Array.Copy(data, newData, Math.Min(len, _length));
            data = newData;

            _length = len;
        }
    }

    public override long Seek(long dest, SeekOrigin origin) {
        switch (origin) {
            case SeekOrigin.End:
                Position = Length - dest;
                break;
            case SeekOrigin.Begin:
                Position = dest;
                break;
            case SeekOrigin.Current:
                Position += dest;
                break;
        }
        return Position;
    }

    public override int Read(byte[] buffer, int offset, int count) {
        int size = count;
        if (Position + count > Length)
            size = (int)(Length-Position);
        Array.Copy(data, Position, buffer, offset, size);
        Position = Position + size;
        return size;
    }
    public override void Write(byte[] buffer, int offset, int count) {
        if (Position + count > Length)
            SetLength(Position + count);
        Array.Copy(buffer, offset, data, Position, count);
        Position = Position + count;
        if (Position > Length)
            Position = Length;
        modified = true;
    }

    public override int ReadByte() {
        int ret = data[Position];
        Position++;
        return ret;
    }
    public override void WriteByte(byte value) {
        data[Position] = value;
        Position++;
        modified = true;
    }
}
{'fraction_non_alphanumeric': 0.09263869132290185, 'fraction_numerical': 0.0010668563300142249, 'mean_word_length': 1.7385589094449854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3607482', 'n_tokens_mistral': 1510, 'n_tokens_neox': 1467, 'n_words': 462}