<commit_before>﻿using NLog;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;

namespace PaletteInsightAgent.Output
{
    class FileUploader
    {
        private static readonly Logger Log = LogManager.GetCurrentClassLogger();
        /// <summary>
        /// The directory we store the succesfully uploaded files
        /// </summary>
        private const string PROCESSED_PREFIX = @"processed/";
        /// <summary>
        /// The directory where the files that have errors (invalid names, etc.)
        /// </summary>
        private const string ERROR_PREFIX = @"errors/";

        public static string DataFilePattern
        {
            get
            {
                return "*" + OutputSerializer.Extension;
            }
        }

        public static string ProcessedPath
        {
            get
            {
                return Path.Combine(OutputSerializer.DATA_FOLDER, PROCESSED_PREFIX);
            }
        }

        public static string ErrorPath
        {
            get
            {
                return Path.Combine(OutputSerializer.DATA_FOLDER, ERROR_PREFIX);
            }
        }

        public static readonly object FileUploadLock = new object();
        public static readonly int fileUploadLockTimeout = 1000;

        /// <summary>
        /// Start a single write loop
        /// </summary>
        /// <param name="output"></param>
        public static void Start(IOutput output, int processedFilesTTL, long storageLimit)
        {
            DoUpload(output, OutputSerializer.DATA_FOLDER);
            DeleteOldFiles(ProcessedPath, processedFilesTTL);
            // Storage limit is given in megabytes in config, but we will work with
            // bytes while applying the storage limit
            ApplyStorageLimit(storageLimit * 1024 * 1024);
        }

        private static void DeleteOldFiles(string from, int ttl)
        {
            if (!Directory.Exists(from))
            {
                // No directory, no old files in there.
                return;
            }

            Directory.EnumerateFiles(from)
                .Select(f => new FileInfo(f))
                .Where(f => f.CreationTime < DateTime.Now.AddSeconds(-ttl))
                .ToList()
                .ForEach(f => f.Delete());
        }

        /// <summary>
        /// Deletes oldest file so that storage size can fit in the configured
        /// value. For example if the storage limit is set to 1Gb and the
        /// size of the stored files are larger than 1Gb, then oldest files
        /// will be started to be deleted, until the size of the stored files
        /// are less than half of the configured limit (which is 500 Mb in this
        /// example).
        /// </summary>
        /// <param name="storageLimitInBytes">Given in bytes</param>
        private static void ApplyStorageLimit(long storageLimitInBytes)
        {
            IList<FileInfo> storedFiles = CollectStoredFiles();

            long cumulatedSize = 0;
            foreach (var file in storedFiles)
            {
                cumulatedSize += file.Length;
            }
<commit_msg>Handle file not found exceptions while determining the used storage<commit_after>﻿using NLog;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;

namespace PaletteInsightAgent.Output
{
    class FileUploader
    {
        private static readonly Logger Log = LogManager.GetCurrentClassLogger();
        /// <summary>
        /// The directory we store the succesfully uploaded files
        /// </summary>
        private const string PROCESSED_PREFIX = @"processed/";
        /// <summary>
        /// The directory where the files that have errors (invalid names, etc.)
        /// </summary>
        private const string ERROR_PREFIX = @"errors/";

        public static string DataFilePattern
        {
            get
            {
                return "*" + OutputSerializer.Extension;
            }
        }

        public static string ProcessedPath
        {
            get
            {
                return Path.Combine(OutputSerializer.DATA_FOLDER, PROCESSED_PREFIX);
            }
        }

        public static string ErrorPath
        {
            get
            {
                return Path.Combine(OutputSerializer.DATA_FOLDER, ERROR_PREFIX);
            }
        }

        public static readonly object FileUploadLock = new object();
        public static readonly int fileUploadLockTimeout = 1000;

        /// <summary>
        /// Start a single write loop
        /// </summary>
        /// <param name="output"></param>
        public static void Start(IOutput output, int processedFilesTTL, long storageLimit)
        {
            DoUpload(output, OutputSerializer.DATA_FOLDER);
            DeleteOldFiles(ProcessedPath, processedFilesTTL);
            // Storage limit is given in megabytes in config, but we will work with
            // bytes while applying the storage limit
            ApplyStorageLimit(storageLimit * 1024 * 1024);
        }

        private static void DeleteOldFiles(string from, int ttl)
        {
            if (!Directory.Exists(from))
            {
                // No directory, no old files in there.
                return;
            }

            Directory.EnumerateFiles(from)
                .Select(f => new FileInfo(f))
                .Where(f => f.CreationTime < DateTime.Now.AddSeconds(-ttl))
                .ToList()
                .ForEach(f => f.Delete());
        }

        /// <summary>
        /// Deletes oldest file so that storage size can fit in the configured
        /// value. For example if the storage limit is set to 1Gb and the
        /// size of the stored files are larger than 1Gb, then oldest files
        /// will be started to be deleted, until the size of the stored files
        /// are less than half of the configured limit (which is 500 Mb in this
        /// example).
        /// </summary>
        /// <param name="storageLimitInBytes">Given in bytes</param>
        private static void ApplyStorageLimit(long storageLimitInBytes)
        {
            IList<FileInfo> storedFiles = CollectStoredFiles();

            long cumulatedSize = 0;
            foreach (var file in storedFiles)
            {
                try
                {
                    cumulatedSize += file.Length;
                }
                catch (Exception ex)
                {
                    if (ex is IOException || ex is FileNotFoundException)
                    {
                        // This is not an error, because file may be gone after we iterated them.
                        // Either it has been uploaded in the meantime, or it has been renamed
                        // because the CSV writing finished. (Removed the .writing postfix.)
                    }
                    else
                    {
                        Log.Error("Unexpected error while determining size of file: {0}. Error message: {1}", file.Name, ex.Message);
                    }
                }
            }

{'fraction_non_alphanumeric': 0.08192403659550873, 'fraction_numerical': 0.005267535347934572, 'mean_word_length': 1.5685297258810964, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2261149', 'n_tokens_mistral': 1777, 'n_tokens_neox': 1668, 'n_words': 576}