<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace Lokad.Cqrs.Feature.TapeStorage
{
    public class SingleThreadFileTapeWriter : ISingleThreadTapeWriter
    {
        readonly string _dataFileName;
        readonly string _indexFileName;

        public SingleThreadFileTapeWriter(string name)
        {
            _dataFileName = Path.ChangeExtension(name, ".tmd");
            _indexFileName = Path.ChangeExtension(name, ".tmi");
        }

        /// <summary>
        /// For now it opens files for every call.
        /// </summary>
        /// <param name="records"></param>
        public void SaveRecords(IEnumerable<byte[]> records)
        {
            if (records == null)
                throw new ArgumentNullException("records");

            if (!records.Any())
                return;

            var writers = CreateWriters();

            try
            {
                var dataWriter = writers.DataWriter;
                var indexWriter = writers.IndexWriter;
                var dataStream = dataWriter.BaseStream;
                var indexStream = indexWriter.BaseStream;

                foreach (var record in records)
                {
                    if (record.Length == 0)
                        throw new ArgumentException("Record must contain at least one byte.");

                    indexWriter.Write(dataStream.Position);

                    dataWriter.Write(record.Length);
                    dataWriter.Write(record);

                    dataStream.Flush();
                    indexStream.Flush();
                }
            }
            finally
            {
                DisposeWriters(writers);
            }
        }

        Writers CreateWriters()
        {
            var dataExists = File.Exists(_dataFileName);
            var indexExists = File.Exists(_indexFileName);

            if ((dataExists || indexExists) && (!dataExists || !indexExists))
            {
                if (!dataExists)
                    throw new InvalidOperationException("Index file found but no data file.");

                throw new InvalidOperationException("Data file found but no index file.");
            }

            Writers writers;

            var dataStream = new FileStream(_dataFileName, FileMode.Append, FileAccess.Write, FileShare.Read);
            writers.DataWriter = new BinaryWriter(dataStream);

            var indexStream = new FileStream(_indexFileName, FileMode.Append, FileAccess.Write, FileShare.Read);
            writers.IndexWriter = new BinaryWriter(indexStream);

            return writers;
        }

        static void DisposeWriters(Writers writers)
        {
            writers.DataWriter.Dispose(); // will dispose BaseStream too
            writers.IndexWriter.Dispose(); // will dispose BaseStream too
        }

        struct Writers
        {
            internal BinaryWriter DataWriter;
            internal BinaryWriter IndexWriter;
        }
    }
}<commit_msg>Checked that index will never be more than long.MaxValue.<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace Lokad.Cqrs.Feature.TapeStorage
{
    public class SingleThreadFileTapeWriter : ISingleThreadTapeWriter
    {
        readonly string _dataFileName;
        readonly string _indexFileName;

        public SingleThreadFileTapeWriter(string name)
        {
            _dataFileName = Path.ChangeExtension(name, ".tmd");
            _indexFileName = Path.ChangeExtension(name, ".tmi");
        }

        /// <summary>
        /// For now it opens files for every call.
        /// </summary>
        /// <param name="records"></param>
        public void SaveRecords(IEnumerable<byte[]> records)
        {
            if (records == null)
                throw new ArgumentNullException("records");

            if (!records.Any())
                return;

            var writers = CreateWriters();

            try
            {
                var dataWriter = writers.DataWriter;
                var indexWriter = writers.IndexWriter;
                var dataStream = dataWriter.BaseStream;
                var indexStream = indexWriter.BaseStream;

                // Used only to enforce the rule that index must not be more than long.MaxValue
                var index = indexStream.Position / sizeof(long);

                foreach (var record in records)
                {
                    if (record.Length == 0)
                        throw new ArgumentException("Record must contain at least one byte.");

                    if (index > long.MaxValue - 1)
                        throw new IndexOutOfRangeException("Index is more than long.MaxValue.");
                    index++;

                    indexWriter.Write(dataStream.Position);

                    dataWriter.Write(record.Length);
                    dataWriter.Write(record);

                    dataStream.Flush();
                    indexStream.Flush();
                }
            }
            finally
            {
                DisposeWriters(writers);
            }
        }

        Writers CreateWriters()
        {
            var dataExists = File.Exists(_dataFileName);
            var indexExists = File.Exists(_indexFileName);

            if ((dataExists || indexExists) && (!dataExists || !indexExists))
            {
                if (!dataExists)
                    throw new InvalidOperationException("Index file found but no data file.");

                throw new InvalidOperationException("Data file found but no index file.");
            }

            Writers writers;

            var dataStream = new FileStream(_dataFileName, FileMode.Append, FileAccess.Write, FileShare.Read);
            writers.DataWriter = new BinaryWriter(dataStream);

            var indexStream = new FileStream(_indexFileName, FileMode.Append, FileAccess.Write, FileShare.Read);
            writers.IndexWriter = new BinaryWriter(indexStream);

            return writers;
        }

        static void DisposeWriters(Writers writers)
        {
            writers.DataWriter.Dispose(); // will dispose BaseStream too
            writers.IndexWriter.Dispose(); // will dispose BaseStream too
        }

        struct Writers
        {
            internal BinaryWriter DataWriter;
            internal BinaryWriter IndexWriter;
        }
    }
}
{'fraction_non_alphanumeric': 0.08018085908063301, 'fraction_numerical': 0.000452147701582517, 'mean_word_length': 1.5117335352006056, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2564646', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1444, 'n_words': 384}