<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;

namespace YaR.MailRuCloud.Api.Base.Streams
{
    internal class DownloadStream : Stream
    {
        private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(DownloadStream));

        private const int InnerBufferSize = 65536 * 2;

        private readonly Func<long, long, File, HttpWebRequest> _requestGenerator;
        private readonly IList<File> _files;
        private readonly long? _start;
        private readonly long? _end;

        private RingBufferedStream _innerStream;
        private bool _initialized;

        public DownloadStream(Func<long, long, File, HttpWebRequest> requestGenerator, File file, long? start = null, long? end = null)
            : this(requestGenerator, file.Parts, start, end)
        {
        }

        private DownloadStream(Func<long, long, File, HttpWebRequest> requestGenerator, IList<File> files, long? start = null, long? end = null)
        {
            var globalLength = files.Sum(f => f.OriginalSize);

            _requestGenerator = requestGenerator ?? throw new ArgumentNullException(nameof(requestGenerator));
            _files = files;
            _start = start;
            _end = end >= globalLength ? globalLength - 1 : end;

            Length = _start != null && _end != null
                ? _end.Value - _start.Value + 1
                : globalLength;
        }

        public void Open()
        {
            _innerStream = new RingBufferedStream(InnerBufferSize) {ReadTimeout = 15 * 1000, WriteTimeout = 15 * 1000};
            _copyTask = GetFileStream();

            _initialized = true;
        }

        private Task _copyTask;

        private async Task<object> GetFileStream()
        {
            var totalLength = Length;
            long glostart = _start ?? 0;
            long gloend = _end == null || 
                _start == _end && _end == 0 ? totalLength : _end.Value + 1;

            long fileStart = 0;
            long fileEnd = 0;

            foreach (var file in _files)
            {
                var clofile = file;

                fileEnd += clofile.OriginalSize;

                if (glostart >= fileEnd || gloend <= fileStart)
                {
                    fileStart += clofile.OriginalSize;
                    continue;
                }
                
                long clostart = Math.Max(0, glostart - fileStart);
                long cloend = gloend - fileStart - 1;

                await GetWebResponse(clostart, cloend, clofile).ConfigureAwait(false);

                fileStart += file.OriginalSize;
            }

            _innerStream.Flush();

            return _innerStream;
        }

        private async Task<WebResponse> GetWebResponse(long clostart, long cloend, File clofile)
        {
            var request = _requestGenerator(clostart, cloend, clofile);
            Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri}");

            try
            {
                var response = await request.GetResponseAsync().ConfigureAwait(false);
                using (var responseStream = response.GetResponseStream())
                {
                    responseStream.ReadTimeout = 15 * 1000;
                    await responseStream.CopyToAsync(_innerStream).ConfigureAwait(false);
                }
                return response;
            }
            catch (Exception e)
            {
                throw;
            }
        }

        private bool _disposed;
        protected override void Dispose(bool disposing)
        {<commit_msg>response to using<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;

namespace YaR.MailRuCloud.Api.Base.Streams
{
    internal class DownloadStream : Stream
    {
        private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(DownloadStream));

        private const int InnerBufferSize = 65536 * 2;

        private readonly Func<long, long, File, HttpWebRequest> _requestGenerator;
        private readonly IList<File> _files;
        private readonly long? _start;
        private readonly long? _end;

        private RingBufferedStream _innerStream;
        private bool _initialized;

        public DownloadStream(Func<long, long, File, HttpWebRequest> requestGenerator, File file, long? start = null, long? end = null)
            : this(requestGenerator, file.Parts, start, end)
        {
        }

        private DownloadStream(Func<long, long, File, HttpWebRequest> requestGenerator, IList<File> files, long? start = null, long? end = null)
        {
            var globalLength = files.Sum(f => f.OriginalSize);

            _requestGenerator = requestGenerator ?? throw new ArgumentNullException(nameof(requestGenerator));
            _files = files;
            _start = start;
            _end = end >= globalLength ? globalLength - 1 : end;

            Length = _start != null && _end != null
                ? _end.Value - _start.Value + 1
                : globalLength;
        }

        public void Open()
        {
            _innerStream = new RingBufferedStream(InnerBufferSize) {ReadTimeout = 15 * 1000, WriteTimeout = 15 * 1000};
            _copyTask = GetFileStream();

            _initialized = true;
        }

        private Task _copyTask;

        private async Task<object> GetFileStream()
        {
            var totalLength = Length;
            long glostart = _start ?? 0;
            long gloend = _end == null || 
                _start == _end && _end == 0 ? totalLength : _end.Value + 1;

            long fileStart = 0;
            long fileEnd = 0;

            foreach (var file in _files)
            {
                var clofile = file;

                fileEnd += clofile.OriginalSize;

                if (glostart >= fileEnd || gloend <= fileStart)
                {
                    fileStart += clofile.OriginalSize;
                    continue;
                }
                
                long clostart = Math.Max(0, glostart - fileStart);
                long cloend = gloend - fileStart - 1;

                await Download(clostart, cloend, clofile).ConfigureAwait(false);

                fileStart += file.OriginalSize;
            }

            _innerStream.Flush();

            return _innerStream;
        }

        private async Task Download(long clostart, long cloend, File clofile)
        {
            var request = _requestGenerator(clostart, cloend, clofile);
            Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri}");

            try
            {
                using (var response = await request.GetResponseAsync().ConfigureAwait(false))
                using (var responseStream = response.GetResponseStream())
                {
                    await responseStream.CopyToAsync(_innerStream).ConfigureAwait(false);
                }
            }
            catch (Exception e)
            {
                throw;
            }
        }

        private bool _disposed;
        protected override void Dispose(bool disposing)
        {
{'fraction_non_alphanumeric': 0.08396213472355604, 'fraction_numerical': 0.008780353958018933, 'mean_word_length': 1.7613636363636365, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1788517', 'n_tokens_mistral': 1952, 'n_tokens_neox': 1815, 'n_words': 482}