<commit_before>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace Sir.Postings
{
    public class StreamRepository : IDisposable, ILogger
    {
        private readonly IConfigurationProvider _config;
        private const string DataFileNameFormat = "{0}.pos";
        private readonly IDictionary<(ulong, long), IList<long>> _cache;

        public StreamRepository(IConfigurationProvider config)
        {
            _config = config;
            _cache = new ConcurrentDictionary<(ulong, long), IList<long>>();
        }

        public async Task Concat(ulong collectionId, IDictionary<long, IList<long>> offsets)
        {
            foreach (var list in offsets)
            {
                var canonical = list.Key;

                foreach (var offset in list.Value)
                {
                    await Concat(collectionId, canonical, offset);
                }
            }
        }

        public async Task Concat(ulong collectionId, long offset1, long offset2)
        {
            if (offset1 == offset2)
            {
                throw new InvalidOperationException();
            }

            var offset2Buf = BitConverter.GetBytes(offset2);

            using (var data = CreateReadableWritableDataStream(collectionId))
            {
                data.Seek(offset1 + (sizeof(long)*2), SeekOrigin.Begin);

                var lastPageBuf = new byte[sizeof(long)];
                await data.ReadAsync(lastPageBuf);
                var lastPage = BitConverter.ToInt64(lastPageBuf, 0);

                if (lastPage == offset1)
                {
                    data.Seek(offset1 + sizeof(ulong), SeekOrigin.Begin);

                    await data.WriteAsync(offset2Buf);
                    await data.WriteAsync(offset2Buf);
                }
                else
                {
                    data.Seek(lastPage + sizeof(ulong), SeekOrigin.Begin);

                    await data.WriteAsync(offset2Buf);
                    await data.WriteAsync(offset2Buf);
                }
            }
        }

        public async Task<IList<long>> Read(ulong collectionId, IList<long> offsets)
        {
            var result = new List<long>();

            foreach (var offset in offsets)
            {
                foreach (var x in await Read(collectionId, offset))
                {
                    result.Add(x);
                }
            }

            return result;<commit_msg>introduce a bug to hide the other bug (list should already be distinct)<commit_after>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace Sir.Postings
{
    public class StreamRepository : IDisposable, ILogger
    {
        private readonly IConfigurationProvider _config;
        private const string DataFileNameFormat = "{0}.pos";
        private readonly IDictionary<(ulong, long), IList<long>> _cache;

        public StreamRepository(IConfigurationProvider config)
        {
            _config = config;
            _cache = new ConcurrentDictionary<(ulong, long), IList<long>>();
        }

        public async Task Concat(ulong collectionId, IDictionary<long, IList<long>> offsets)
        {
            foreach (var list in offsets)
            {
                var canonical = list.Key;

                foreach (var offset in list.Value)
                {
                    await Concat(collectionId, canonical, offset);
                }
            }
        }

        public async Task Concat(ulong collectionId, long offset1, long offset2)
        {
            if (offset1 == offset2)
            {
                throw new InvalidOperationException();
            }

            var offset2Buf = BitConverter.GetBytes(offset2);

            using (var data = CreateReadableWritableDataStream(collectionId))
            {
                data.Seek(offset1 + (sizeof(long)*2), SeekOrigin.Begin);

                var lastPageBuf = new byte[sizeof(long)];
                await data.ReadAsync(lastPageBuf);
                var lastPage = BitConverter.ToInt64(lastPageBuf, 0);

                if (lastPage == offset1)
                {
                    data.Seek(offset1 + sizeof(ulong), SeekOrigin.Begin);

                    await data.WriteAsync(offset2Buf);
                    await data.WriteAsync(offset2Buf);
                }
                else
                {
                    data.Seek(lastPage + sizeof(ulong), SeekOrigin.Begin);

                    await data.WriteAsync(offset2Buf);
                    await data.WriteAsync(offset2Buf);
                }
            }
        }

        public async Task<IEnumerable<long>> Read(ulong collectionId, IList<long> offsets)
        {
            var result = new HashSet<long>();

            foreach (var offset in offsets)
            {
                foreach (var x in await Read(collectionId, offset))
                {
                    result.Add(x);
                }
            }

            return result;
{'fraction_non_alphanumeric': 0.08293521880374546, 'fraction_numerical': 0.006879419071278425, 'mean_word_length': 1.5949429846306395, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3612873', 'n_tokens_mistral': 1346, 'n_tokens_neox': 1268, 'n_words': 311}